feat(backup): keep k3s datastore archives outside AIStor on the HDD

AIStor runs inside the k3s cluster that the datastore archive restores.
After losing the whole cluster the AIStor copy cannot be read until k3s
is running again, which is what the archive is for.

Also keep the seven newest archives in
/srv/k3s/aistor/host-backups/k3s-datastore (root 0700) on the same HDD
filesystem (sdb3). The directory is next to AIStor's drive directory
data/, so AIStor does not list it as a bucket, and it is readable by
mounting sdb3 alone. A metric reports how many local copies exist.

Rejected: the NTFS archive partition (sdb2). It has no Unix
permissions and Windows on the same machine can read it; the archive
holds the server token.

Risk: the files live inside the AIStor PersistentVolume root, which is
mounted into the MinIO pod (uid 1000); mode 0700 root keeps the pod
from reading them.

Verification: bash -n; /srv/k3s/aistor contains only data/ (2026-11-12).
This commit is contained in:
DongHyeonka
2026-11-12 01:14:45 +09:00
parent ba3dddc139
commit 9e9ea94461
2 changed files with 17 additions and 1 deletions
+13
View File
@@ -57,6 +57,16 @@ size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")"
hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null
hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null
# Local copy outside AIStor on the same HDD (sdb3). AIStor runs inside the
# cluster this archive restores, so after losing the cluster the AIStor copy
# cannot be read until k3s is back; this copy can. The directory sits next to
# AIStor's drive directory (data/), so AIStor does not see it as a bucket.
readonly LOCAL_DIR=/srv/k3s/aistor/host-backups/k3s-datastore
install -d -m 0700 -o root -g root "$(dirname "$LOCAL_DIR")" "$LOCAL_DIR"
install -m 0600 "$work/k3s-datastore-$ts.tar.gz" "$work/k3s-datastore-$ts.tar.gz.sha256" "$LOCAL_DIR/"
ls -1t "$LOCAL_DIR"/k3s-datastore-*.tar.gz | tail -n +8 | while read -r old; do rm -f -- "$old" "$old.sha256"; done
local_copies="$(ls -1 "$LOCAL_DIR"/k3s-datastore-*.tar.gz | wc -l)"
offhost=0
if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then
command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed"
@@ -79,6 +89,9 @@ platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
# TYPE platform_backup_last_size_bytes gauge
platform_backup_last_size_bytes{backup="k3s-datastore"} $size
platform_backup_last_duration_seconds{backup="k3s-datastore"} $(( $(date +%s) - started ))
# HELP platform_backup_local_copies Archives kept outside AIStor on the HDD.
# TYPE platform_backup_local_copies gauge
platform_backup_local_copies{backup="k3s-datastore"} $local_copies
# HELP platform_backup_offhost_copy 1 when the last run also wrote the encrypted off-host copy.
# TYPE platform_backup_offhost_copy gauge
platform_backup_offhost_copy{backup="k3s-datastore"} $offhost