From ba3dddc13966198a2b6b157fd6330f642de3ae90 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:13:02 +0900 Subject: [PATCH] feat(validate): add restore drills for the k3s and Gitea backups A backup nobody has restored is not known to work. The 2026-08 k3s recovery bundle never got its isolated restore drill. k3s-datastore-restore-drill.sh downloads the latest archive, checks its sha256 and integrity, starts the same k3s version in a network-less throwaway container on the copy, and checks the API comes up, the Secret count is close to live and every Secret decrypts with the token. gitea-backup-restore-drill.sh extracts the latest dump and checks app.ini (INTERNAL_TOKEN, SECRET_KEY), git fsck on every repository and the repository count against live Gitea. Both accept a local copy and write a drill timestamp metric. Risk: read-only for the cluster. The k3s drill runs the restored control plane with --network none, so its controllers reach nothing. Verification (desktop): - test-k3s-datastore-restore-drill.sh: a fresh k3s with three test Secrets (stored encrypted, k8s:enc:) restores with all Secrets decrypted in 7s; before folding the WAL the test saw 2 of 3 rows, which is why the real backup uses sqlite3 .backup. - gitea-backup-restore-drill.sh on a live repository dump: 29 repositories, 0 fsck failures, 29 live; the dump was deleted after. --- scripts/validate/README.md | 10 ++ .../validate/gitea-backup-restore-drill.sh | 75 ++++++++++++++ .../validate/k3s-datastore-restore-drill.sh | 99 +++++++++++++++++++ .../test-k3s-datastore-restore-drill.sh | 41 ++++++++ 4 files changed, 225 insertions(+) create mode 100644 scripts/validate/gitea-backup-restore-drill.sh create mode 100644 scripts/validate/k3s-datastore-restore-drill.sh create mode 100644 scripts/validate/test-k3s-datastore-restore-drill.sh diff --git a/scripts/validate/README.md b/scripts/validate/README.md index b30e14a..9e5ca12 100644 --- a/scripts/validate/README.md +++ b/scripts/validate/README.md @@ -596,3 +596,13 @@ PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ - `--textfile FILE`: `platform_live_drift_objects`, `platform_live_drift_check_success`, `platform_live_drift_last_run_timestamp_seconds`를 Prometheus textfile 형식으로 쓴다. - Secret 값의 차이는 보지 않는다. + +## 호스트 백업·점검 시험 + +- `test-platform-host-checks-alerts.sh`: `services/platform-host-checks/prometheus-rules.yaml`의 promtool + 단위 시험(백업 신선도, 실패, 지표 없음, drift, k3s 인증서). docker 필요, 클러스터 접근 없음. +- `k3s-cert-expiry.sh`: 6443·10250 serving 인증서 만료를 textfile 지표로 쓴다(타이머가 매일 돌림). +- `live-drift-check-main.sh`: checkout의 작업 트리가 아니라 `origin/main`으로 `live-drift-check.sh`를 돌린다. +- `k3s-datastore-restore-drill.sh`, `gitea-backup-restore-drill.sh`: 복원 리허설. 인자 없이 root로 돌리면 AIStor의 + 최신 백업을 쓰고, `--from-dir`·`--from-file`로 로컬 사본을 줄 수 있다. +- `test-k3s-datastore-restore-drill.sh`: 운영 데이터 없이 k3s 복원 리허설 경로를 시험한다(일회용 k3s, 시험 Secret 셋). diff --git a/scripts/validate/gitea-backup-restore-drill.sh b/scripts/validate/gitea-backup-restore-drill.sh new file mode 100644 index 0000000..0dfc9c4 --- /dev/null +++ b/scripts/validate/gitea-backup-restore-drill.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Restore drill for the Gitea backup (file level). +# +# sudo bash scripts/validate/gitea-backup-restore-drill.sh # latest dump in AIStor +# bash scripts/validate/gitea-backup-restore-drill.sh --from-file DUMP.tar.gz +# +# Extracts the dump into a throwaway directory and checks what a restore needs: +# app.ini is present, every repository passes `git fsck`, and the repository +# count matches the live Gitea. The database is restored from the CloudNativePG +# backup instead (services/platform-postgres-backup); the full restore order is +# in docs/runbooks/gitea-restore.md. Nothing is written to the cluster. +set -Eeuo pipefail +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +# shellcheck source=../lib/host-backup.sh +source "$ROOT/scripts/lib/host-backup.sh" +readonly NAME=gitea_restore_drill + +from_file="" +case "${1:-}" in + --from-file) from_file="${2:?--from-file needs a path}" ;; + "") ;; + *) sed -n '2,11p' "$0" >&2; exit 2 ;; +esac +if [[ "$(id -u)" -eq 0 ]]; then + kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; } +fi + +base="$HB_WORK_ROOT"; [[ -w "$base" ]] || base="$(mktemp -d)" +work="$(mktemp -d "$base/gitea-drill.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT + +if [[ -z "$from_file" ]]; then + [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" + hb_load_aistor_env + latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/gitea/" | awk '{print $NF}' | grep -E '^gitea-dump-.*\.tar\.gz$' | sort | tail -1)" + [[ -n "$latest" ]] || hb_fail "no Gitea dump found" + hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/gitea/$latest" "backup/$AISTOR_BUCKET/gitea/$latest.sha256" /work/ >/dev/null + [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" + from_file="$work/$latest" + hb_log "drill on $latest" +fi + +mkdir "$work/restore" +tar -C "$work/restore" -xzf "$from_file" +[[ -s "$work/restore/app.ini" ]] || hb_fail "app.ini is missing from the dump" +for key in INTERNAL_TOKEN SECRET_KEY; do + grep -q "^${key}[[:space:]]*=" "$work/restore/app.ini" || hb_fail "app.ini has no ${key} (needed to restore)" +done + +total=0 broken=0 +while IFS= read -r -d '' repo; do + total=$((total + 1)) + if ! git -c safe.directory='*' --git-dir="$repo" fsck --no-dangling --no-progress >/dev/null 2>"$work/fsck.err"; then + broken=$((broken + 1)) + hb_log "fsck failed: ${repo#"$work/restore/"}: $(head -1 "$work/fsck.err")" + fi +done < <(find "$work/restore/repos" -mindepth 2 -maxdepth 2 -type d -name '*.git' -print0) +hb_log "repositories in dump: $total, fsck failures: $broken" +(( total > 0 )) || hb_fail "the dump has no repositories" +(( broken == 0 )) || hb_fail "$broken repositories failed git fsck" + +live="$(kubectl -n gitea exec deploy/gitea -c gitea -- sh -c 'find /data/git/gitea-repositories -mindepth 2 -maxdepth 2 -name "*.git" -type d | wc -l' 2>/dev/null || echo "")" +if [[ -n "$live" ]]; then + hb_log "live Gitea repositories: $live" + (( total >= live - 2 )) || hb_fail "the dump has $total repositories, live Gitea has $live" +fi + +hb_write_metrics "$NAME" < 0 )); do + case "$1" in + --from-dir) from_dir="${2:?}"; shift 2 ;; + --expect-secrets) expect_secrets="${2:?}"; shift 2 ;; + *) sed -n '2,13p' "$0" >&2; exit 2 ;; + esac +done + +base="${HB_WORK_ROOT}" +[[ -w "$base" ]] || base="$(mktemp -d)" +work="$(mktemp -d "$base/k3s-drill.XXXXXX")" +container="k3s-restore-drill-$$" +cleanup() { + docker rm -f "$container" >/dev/null 2>&1 || true + # The drill container writes as root; remove its files with the same image. + docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT + +if [[ -z "$from_dir" ]]; then + [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" + hb_load_aistor_env + latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)" + [[ -n "$latest" ]] || hb_fail "no k3s datastore backup found" + hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null + [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" + tar -C "$work" -xzf "$work/$latest" + from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)" + hb_log "drill on $latest" + if [[ -z "$expect_secrets" ]]; then + expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)" + hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)" + fi +fi +[[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token" + +install -d -m 0700 "$work/server/db" +cp "$from_dir/state.db" "$work/server/db/state.db" +cp "$from_dir/token" "$work/server/token" +[[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed" + +started="$(date +%s)" +docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \ + -v "$work/server:/var/lib/rancher/k3s/server" \ + "$K3S_IMAGE" server --disable-agent --secrets-encryption \ + --token-file /var/lib/rancher/k3s/server/token \ + --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ + --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null + +ready=0 +for _ in $(seq 1 60); do + if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi + sleep 5 +done +if (( ready == 0 )); then + docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2 + hb_fail "restored k3s did not become ready in 5 minutes" +fi +namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)" +secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)" +# -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero. +decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')" +hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s" +if [[ -n "$expect_secrets" ]]; then + diff=$(( secrets - expect_secrets )) + (( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets" +fi +(( decrypted > 0 )) || hb_fail "no Secret carried data after restore" + +hb_write_metrics "$NAME" <&2; exit 1; } +work="$(mktemp -d)" +seed="w6-k3s-drill-seed-$$" +cleanup() { + docker rm -f "$seed" >/dev/null 2>&1 || true + docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT +mkdir -p "$work/server" "$work/copy" + +docker run -d --name "$seed" --network none --tmpfs /run --tmpfs /var/run \ + -v "$work/server:/var/lib/rancher/k3s/server" "$K3S_IMAGE" server --disable-agent --secrets-encryption \ + --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ + --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null +for _ in $(seq 1 60); do docker exec "$seed" kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 5; done +for i in 1 2 3; do docker exec "$seed" kubectl create secret generic "drill-test-$i" --from-literal=value="test-$i" >/dev/null; done +expected="$(docker exec "$seed" kubectl get secrets -A --no-headers | wc -l)" +docker stop -t 30 "$seed" >/dev/null +docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c \ + "cp /w/server/db/state.db* /w/server/token /w/copy/ && chown $(id -u):$(id -g) /w/copy/*" +# Fold the WAL into state.db, as `sqlite3 .backup` does in the real backup. +sqlite3 "$work/copy/state.db" 'PRAGMA wal_checkpoint(TRUNCATE);' >/dev/null +rm -f "$work/copy/state.db-wal" "$work/copy/state.db-shm" +# The test Secrets must be stored encrypted (k8s:enc:...), so the drill really +# proves that the token unlocks the encryption key. +encrypted="$(sqlite3 "$work/copy/state.db" "SELECT COUNT(*) FROM kine WHERE name LIKE '/registry/secrets/default/drill-test-%' AND CAST(value AS TEXT) LIKE 'k8s:enc:%';")" +echo "seed: $expected Secrets, encrypted test Secret rows=$encrypted" +(( encrypted >= 3 )) || { echo "test Secrets are not encrypted at rest in the seed datastore" >&2; exit 1; } + +bash "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh" --from-dir "$work/copy" --expect-secrets "$expected" +echo "K3S DATASTORE RESTORE DRILL TEST PASS"