AIStor runs inside the k3s cluster that the datastore archive restores. After losing the whole cluster the AIStor copy cannot be read until k3s is running again, which is what the archive is for. Also keep the seven newest archives in /srv/k3s/aistor/host-backups/k3s-datastore (root 0700) on the same HDD filesystem (sdb3). The directory is next to AIStor's drive directory data/, so AIStor does not list it as a bucket, and it is readable by mounting sdb3 alone. A metric reports how many local copies exist. Rejected: the NTFS archive partition (sdb2). It has no Unix permissions and Windows on the same machine can read it; the archive holds the server token. Risk: the files live inside the AIStor PersistentVolume root, which is mounted into the MinIO pod (uid 1000); mode 0700 root keeps the pod from reading them. Verification: bash -n; /srv/k3s/aistor contains only data/ (2026-11-12).
100 lines
5.1 KiB
Bash
100 lines
5.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Back up the k3s datastore (sqlite) and the server token to AIStor (HDD sdb3).
|
|
# Root only. Run by platform-k3s-datastore-backup.timer (infrastructure/host/backup).
|
|
#
|
|
# k3s docs (datastore/backup-restore, SQLite): copy server/db and server/token;
|
|
# the token is required to read the datastore back. `sqlite3 .backup` takes a
|
|
# consistent copy of the live database.
|
|
#
|
|
# Off-host copy (prepared, off by default): set in /etc/platform-backup/aistor.env
|
|
# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc
|
|
# OFFHOST_RCLONE_TARGET=<rclone remote:path>
|
|
# The archive is then encrypted to that public key and copied with rclone.
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
# shellcheck source=../lib/host-backup.sh
|
|
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
|
|
|
|
readonly SERVER=/var/lib/rancher/k3s/server
|
|
readonly NAME=k3s_datastore_backup
|
|
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root"
|
|
command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required"
|
|
hb_load_aistor_env
|
|
|
|
started="$(date +%s)"
|
|
on_error() {
|
|
hb_write_metrics "$NAME" <<PROM
|
|
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
|
|
# TYPE platform_backup_last_run_success gauge
|
|
platform_backup_last_run_success{backup="k3s-datastore"} 0
|
|
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} ${previous_success:-0}
|
|
PROM
|
|
}
|
|
previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)"
|
|
trap on_error ERR
|
|
|
|
work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")"
|
|
trap 'rm -rf -- "$work"' EXIT
|
|
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
stage="$work/k3s-datastore-$ts"
|
|
mkdir -m 0700 "$stage"
|
|
|
|
sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'"
|
|
check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')"
|
|
[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check"
|
|
install -m 0600 "$SERVER/token" "$stage/token"
|
|
{
|
|
echo "created=$ts"
|
|
echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)"
|
|
echo "host=$(hostname)"
|
|
echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')"
|
|
} >"$stage/META"
|
|
tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts"
|
|
sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256"
|
|
size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")"
|
|
|
|
hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null
|
|
hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null
|
|
|
|
# Local copy outside AIStor on the same HDD (sdb3). AIStor runs inside the
|
|
# cluster this archive restores, so after losing the cluster the AIStor copy
|
|
# cannot be read until k3s is back; this copy can. The directory sits next to
|
|
# AIStor's drive directory (data/), so AIStor does not see it as a bucket.
|
|
readonly LOCAL_DIR=/srv/k3s/aistor/host-backups/k3s-datastore
|
|
install -d -m 0700 -o root -g root "$(dirname "$LOCAL_DIR")" "$LOCAL_DIR"
|
|
install -m 0600 "$work/k3s-datastore-$ts.tar.gz" "$work/k3s-datastore-$ts.tar.gz.sha256" "$LOCAL_DIR/"
|
|
ls -1t "$LOCAL_DIR"/k3s-datastore-*.tar.gz | tail -n +8 | while read -r old; do rm -f -- "$old" "$old.sha256"; done
|
|
local_copies="$(ls -1 "$LOCAL_DIR"/k3s-datastore-*.tar.gz | wc -l)"
|
|
|
|
offhost=0
|
|
if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then
|
|
command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed"
|
|
gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \
|
|
--output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz"
|
|
rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg"
|
|
offhost=1
|
|
fi
|
|
|
|
trap - ERR
|
|
hb_write_metrics "$NAME" <<PROM
|
|
# HELP platform_backup_last_success_timestamp_seconds Unix time of the last successful backup.
|
|
# TYPE platform_backup_last_success_timestamp_seconds gauge
|
|
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
|
|
# TYPE platform_backup_last_run_success gauge
|
|
platform_backup_last_run_success{backup="k3s-datastore"} 1
|
|
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
# HELP platform_backup_last_size_bytes Size of the last successful backup archive.
|
|
# TYPE platform_backup_last_size_bytes gauge
|
|
platform_backup_last_size_bytes{backup="k3s-datastore"} $size
|
|
platform_backup_last_duration_seconds{backup="k3s-datastore"} $(( $(date +%s) - started ))
|
|
# HELP platform_backup_local_copies Archives kept outside AIStor on the HDD.
|
|
# TYPE platform_backup_local_copies gauge
|
|
platform_backup_local_copies{backup="k3s-datastore"} $local_copies
|
|
# HELP platform_backup_offhost_copy 1 when the last run also wrote the encrypted off-host copy.
|
|
# TYPE platform_backup_offhost_copy gauge
|
|
platform_backup_offhost_copy{backup="k3s-datastore"} $offhost
|
|
PROM
|
|
hb_log "k3s datastore backup k3s-datastore-$ts.tar.gz ($size bytes) uploaded; off-host=$offhost"
|