Nothing backs up the k3s datastore or Gitea's repositories, packages and app.ini. W1 covers platform-postgres only. One SSD failure loses the source, CI artefacts, published libraries and every cluster Secret. Add two root systemd timers that write to a new AIStor bucket on the HDD (platform-host-backups, same AIStor as W1's postgres backups): - k3s datastore, daily: sqlite3 .backup of the live state.db plus the server token (k3s backup/restore docs: the token is needed to read it back), integrity_check, tar.gz, sha256, upload. Off-host copy is prepared behind two variables (gpg recipient file, rclone target) and stays off until the owner chooses a place. - Gitea, weekly: gitea dump --skip-db streamed from the live container, listing checked for app.ini and repositories, sha256, upload. The DB is in the CNPG backup. Each run writes textfile metrics for the alerts in the next commit. create-host-backup-credentials.sh makes the bucket (50GiB quota, lifecycle 56 days for gitea/, 30 days for k3s-datastore/), a bucket-scoped policy and user, and a root-only env file; no key is printed and mc gets keys only through a 0600 env file. Rejected: in-cluster CronJobs. The datastore is a root-owned host file and the dump needs the Gitea DB connection; a hostPath root pod or a second Gitea-labelled pod is a larger privilege than a root timer. Risk: not installed here (owner sudo steps). The k3s archive holds the token; together with the datastore it decrypts every Secret. It lives in a bucket only the backup user can read. Verification (desktop): a dump streamed with --file - is a valid tar.gz with app.ini; repositories appear as repos/<owner>/<name>.git (29, matching the live count). The scripts parse with bash -n.
87 lines
4.2 KiB
Bash
87 lines
4.2 KiB
Bash
#!/usr/bin/env bash
|
|
# Back up the k3s datastore (sqlite) and the server token to AIStor (HDD sdb3).
|
|
# Root only. Run by platform-k3s-datastore-backup.timer (infrastructure/host/backup).
|
|
#
|
|
# k3s docs (datastore/backup-restore, SQLite): copy server/db and server/token;
|
|
# the token is required to read the datastore back. `sqlite3 .backup` takes a
|
|
# consistent copy of the live database.
|
|
#
|
|
# Off-host copy (prepared, off by default): set in /etc/platform-backup/aistor.env
|
|
# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc
|
|
# OFFHOST_RCLONE_TARGET=<rclone remote:path>
|
|
# The archive is then encrypted to that public key and copied with rclone.
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
# shellcheck source=../lib/host-backup.sh
|
|
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
|
|
|
|
readonly SERVER=/var/lib/rancher/k3s/server
|
|
readonly NAME=k3s_datastore_backup
|
|
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root"
|
|
command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required"
|
|
hb_load_aistor_env
|
|
|
|
started="$(date +%s)"
|
|
on_error() {
|
|
hb_write_metrics "$NAME" <<PROM
|
|
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
|
|
# TYPE platform_backup_last_run_success gauge
|
|
platform_backup_last_run_success{backup="k3s-datastore"} 0
|
|
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} ${previous_success:-0}
|
|
PROM
|
|
}
|
|
previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)"
|
|
trap on_error ERR
|
|
|
|
work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")"
|
|
trap 'rm -rf -- "$work"' EXIT
|
|
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
stage="$work/k3s-datastore-$ts"
|
|
mkdir -m 0700 "$stage"
|
|
|
|
sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'"
|
|
check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')"
|
|
[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check"
|
|
install -m 0600 "$SERVER/token" "$stage/token"
|
|
{
|
|
echo "created=$ts"
|
|
echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)"
|
|
echo "host=$(hostname)"
|
|
echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')"
|
|
} >"$stage/META"
|
|
tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts"
|
|
sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256"
|
|
size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")"
|
|
|
|
hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null
|
|
hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null
|
|
|
|
offhost=0
|
|
if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then
|
|
command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed"
|
|
gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \
|
|
--output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz"
|
|
rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg"
|
|
offhost=1
|
|
fi
|
|
|
|
trap - ERR
|
|
hb_write_metrics "$NAME" <<PROM
|
|
# HELP platform_backup_last_success_timestamp_seconds Unix time of the last successful backup.
|
|
# TYPE platform_backup_last_success_timestamp_seconds gauge
|
|
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
|
|
# TYPE platform_backup_last_run_success gauge
|
|
platform_backup_last_run_success{backup="k3s-datastore"} 1
|
|
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
# HELP platform_backup_last_size_bytes Size of the last successful backup archive.
|
|
# TYPE platform_backup_last_size_bytes gauge
|
|
platform_backup_last_size_bytes{backup="k3s-datastore"} $size
|
|
platform_backup_last_duration_seconds{backup="k3s-datastore"} $(( $(date +%s) - started ))
|
|
# HELP platform_backup_offhost_copy 1 when the last run also wrote the encrypted off-host copy.
|
|
# TYPE platform_backup_offhost_copy gauge
|
|
platform_backup_offhost_copy{backup="k3s-datastore"} $offhost
|
|
PROM
|
|
hb_log "k3s datastore backup k3s-datastore-$ts.tar.gz ($size bytes) uploaded; off-host=$offhost"
|