Merge pull request 'feat(backup): back up k3s and Gitea to AIStor with alerts and drills' (#6) from donghyeon.kang/w6-host-backups into main

This commit was merged in pull request #6.
This commit is contained in:
2026-11-11 16:48:31 +00:00
25 changed files with 1083 additions and 0 deletions
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
# Create the AIStor bucket, lifecycle rules, policy and user that the host backup
# timers use, and the root-only /etc/platform-backup/aistor.env.
#
# sudo bash scripts/bootstrap/create-host-backup-credentials.sh
#
# Run once, by the owner, on the desktop. Refuses to run when the env file exists;
# never rotates or deletes a bucket, policy, user or object. No key is printed.
set -Eeuo pipefail
set +x
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# shellcheck source=../lib/host-backup.sh
source "$ROOT/scripts/lib/host-backup.sh"
readonly BUCKET=platform-host-backups
readonly POLICY=platform-host-backups
readonly QUOTA=50GiB
readonly POLICY_FILE="$ROOT/services/platform-host-checks/bucket-policy.json"
readonly PASS_LINE=PLATFORM_HOST_BACKUP_BOOTSTRAP_PASS
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; }
[[ "$(id -u)" -eq 0 ]] || hb_fail "run with sudo"
for c in docker jq openssl; do command -v "$c" >/dev/null || hb_fail "missing command: $c"; done
jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON"
[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it"
ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')"
[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found"
endpoint="http://${ip}:80"
work="$(mktemp -d)"
trap 'rm -rf -- "$work"' EXIT
kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env"
grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format"
printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY"
openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY"
cp "$POLICY_FILE" "$work/policy.json"
printf '%s\n' \
"Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \
"Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \
"User: a new generated access key attached to $POLICY" \
"File: $HOST_BACKUP_ENV (root, 0600)" \
'No key value will be printed.'
printf 'Type APPLY to create them: '
IFS= read -r confirmation
[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY"
# Every mc call is silenced so no key reaches the terminal.
docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \
--entrypoint /bin/sh "$MC_IMAGE" -ec '
. /run/b/root.env
ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)"
mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null
mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null
mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null
mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null
mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null
mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null
mc admin user add root "$ak" "$sk" >/dev/null
mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null
mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null
probe=".bootstrap-probe-$(date +%s)"
printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null
[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ]
mc rm "backup/'"$BUCKET"'/$probe" >/dev/null
if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi
echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written"
install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")"
{
echo "AISTOR_BUCKET=$BUCKET"
echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")"
echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")"
echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):"
echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc"
echo "# OFFHOST_RCLONE_TARGET=remote:path"
} >"$HOST_BACKUP_ENV"
chmod 0600 "$HOST_BACKUP_ENV"
printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV."
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env bash
# Install the host backup and check timers (infrastructure/host/backup).
#
# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing
# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs
#
# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root
# timers never execute files the desktop user can edit.
set -Eeuo pipefail
umask 022
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly LIB=/usr/local/lib/platform-backup
readonly UNIT_DIR=/etc/systemd/system
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
readonly WORK_ROOT=/var/lib/platform-backup/work
readonly USER_NAME=donghyeon
readonly -a SCRIPTS=(
scripts/lib/host-backup.sh:lib/host-backup.sh
scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh
scripts/backup/gitea-backup.sh:backup/gitea-backup.sh
scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh
scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh
services/platform-host-checks/README.md:README.md
)
readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry)
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
mode="${1:-}"
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; }
echo "== check"
for pair in "${SCRIPTS[@]}"; do
src="${pair%%:*}"
[[ -f "$ROOT/$src" ]] || fail "missing $src"
[[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src"
done
for t in "${TIMERS[@]}"; do
for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done
done
if command -v systemd-analyze >/dev/null; then
verify_dir="$(mktemp -d)"
trap 'rm -rf -- "$verify_dir"' EXIT
cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/"
# ExecStart paths do not exist before installation; only syntax and keys are checked.
out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)"
[[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; }
echo " units: systemd-analyze verify clean"
fi
echo "== plan"
for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done
for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done
echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)"
echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check"
[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; }
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first"
echo "== install"
for pair in "${SCRIPTS[@]}"; do
install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}"
done
install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR"
install -d -o root -g root -m 0700 "$WORK_ROOT"
for t in "${TIMERS[@]}"; do
install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/"
done
systemctl daemon-reload
for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done
echo "== first runs"
for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do
if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi
done
echo "== metrics"
grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet"
systemctl list-timers 'platform-*' --no-pager
echo "HOST CHECKS INSTALLED"