A backup nobody has restored is not known to work. The 2026-08 k3s recovery bundle never got its isolated restore drill. k3s-datastore-restore-drill.sh downloads the latest archive, checks its sha256 and integrity, starts the same k3s version in a network-less throwaway container on the copy, and checks the API comes up, the Secret count is close to live and every Secret decrypts with the token. gitea-backup-restore-drill.sh extracts the latest dump and checks app.ini (INTERNAL_TOKEN, SECRET_KEY), git fsck on every repository and the repository count against live Gitea. Both accept a local copy and write a drill timestamp metric. Risk: read-only for the cluster. The k3s drill runs the restored control plane with --network none, so its controllers reach nothing. Verification (desktop): - test-k3s-datastore-restore-drill.sh: a fresh k3s with three test Secrets (stored encrypted, k8s:enc:) restores with all Secrets decrypted in 7s; before folding the WAL the test saw 2 of 3 rows, which is why the real backup uses sqlite3 .backup. - gitea-backup-restore-drill.sh on a live repository dump: 29 repositories, 0 fsck failures, 29 live; the dump was deleted after.
76 lines
3.4 KiB
Bash
76 lines
3.4 KiB
Bash
#!/usr/bin/env bash
|
|
# Restore drill for the Gitea backup (file level).
|
|
#
|
|
# sudo bash scripts/validate/gitea-backup-restore-drill.sh # latest dump in AIStor
|
|
# bash scripts/validate/gitea-backup-restore-drill.sh --from-file DUMP.tar.gz
|
|
#
|
|
# Extracts the dump into a throwaway directory and checks what a restore needs:
|
|
# app.ini is present, every repository passes `git fsck`, and the repository
|
|
# count matches the live Gitea. The database is restored from the CloudNativePG
|
|
# backup instead (services/platform-postgres-backup); the full restore order is
|
|
# in docs/runbooks/gitea-restore.md. Nothing is written to the cluster.
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
# shellcheck source=../lib/host-backup.sh
|
|
source "$ROOT/scripts/lib/host-backup.sh"
|
|
readonly NAME=gitea_restore_drill
|
|
|
|
from_file=""
|
|
case "${1:-}" in
|
|
--from-file) from_file="${2:?--from-file needs a path}" ;;
|
|
"") ;;
|
|
*) sed -n '2,11p' "$0" >&2; exit 2 ;;
|
|
esac
|
|
if [[ "$(id -u)" -eq 0 ]]; then
|
|
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; }
|
|
fi
|
|
|
|
base="$HB_WORK_ROOT"; [[ -w "$base" ]] || base="$(mktemp -d)"
|
|
work="$(mktemp -d "$base/gitea-drill.XXXXXX")"
|
|
trap 'rm -rf -- "$work"' EXIT
|
|
|
|
if [[ -z "$from_file" ]]; then
|
|
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials"
|
|
hb_load_aistor_env
|
|
latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/gitea/" | awk '{print $NF}' | grep -E '^gitea-dump-.*\.tar\.gz$' | sort | tail -1)"
|
|
[[ -n "$latest" ]] || hb_fail "no Gitea dump found"
|
|
hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/gitea/$latest" "backup/$AISTOR_BUCKET/gitea/$latest.sha256" /work/ >/dev/null
|
|
[[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest"
|
|
from_file="$work/$latest"
|
|
hb_log "drill on $latest"
|
|
fi
|
|
|
|
mkdir "$work/restore"
|
|
tar -C "$work/restore" -xzf "$from_file"
|
|
[[ -s "$work/restore/app.ini" ]] || hb_fail "app.ini is missing from the dump"
|
|
for key in INTERNAL_TOKEN SECRET_KEY; do
|
|
grep -q "^${key}[[:space:]]*=" "$work/restore/app.ini" || hb_fail "app.ini has no ${key} (needed to restore)"
|
|
done
|
|
|
|
total=0 broken=0
|
|
while IFS= read -r -d '' repo; do
|
|
total=$((total + 1))
|
|
if ! git -c safe.directory='*' --git-dir="$repo" fsck --no-dangling --no-progress >/dev/null 2>"$work/fsck.err"; then
|
|
broken=$((broken + 1))
|
|
hb_log "fsck failed: ${repo#"$work/restore/"}: $(head -1 "$work/fsck.err")"
|
|
fi
|
|
done < <(find "$work/restore/repos" -mindepth 2 -maxdepth 2 -type d -name '*.git' -print0)
|
|
hb_log "repositories in dump: $total, fsck failures: $broken"
|
|
(( total > 0 )) || hb_fail "the dump has no repositories"
|
|
(( broken == 0 )) || hb_fail "$broken repositories failed git fsck"
|
|
|
|
live="$(kubectl -n gitea exec deploy/gitea -c gitea -- sh -c 'find /data/git/gitea-repositories -mindepth 2 -maxdepth 2 -name "*.git" -type d | wc -l' 2>/dev/null || echo "")"
|
|
if [[ -n "$live" ]]; then
|
|
hb_log "live Gitea repositories: $live"
|
|
(( total >= live - 2 )) || hb_fail "the dump has $total repositories, live Gitea has $live"
|
|
fi
|
|
|
|
hb_write_metrics "$NAME" <<PROM
|
|
# HELP platform_backup_restore_drill_last_success_timestamp_seconds Unix time of the last successful restore drill.
|
|
# TYPE platform_backup_restore_drill_last_success_timestamp_seconds gauge
|
|
platform_backup_restore_drill_last_success_timestamp_seconds{backup="gitea"} $(date +%s)
|
|
PROM
|
|
echo "GITEA RESTORE DRILL PASS"
|