A backup nobody has restored is not known to work. The 2026-08 k3s recovery bundle never got its isolated restore drill. k3s-datastore-restore-drill.sh downloads the latest archive, checks its sha256 and integrity, starts the same k3s version in a network-less throwaway container on the copy, and checks the API comes up, the Secret count is close to live and every Secret decrypts with the token. gitea-backup-restore-drill.sh extracts the latest dump and checks app.ini (INTERNAL_TOKEN, SECRET_KEY), git fsck on every repository and the repository count against live Gitea. Both accept a local copy and write a drill timestamp metric. Risk: read-only for the cluster. The k3s drill runs the restored control plane with --network none, so its controllers reach nothing. Verification (desktop): - test-k3s-datastore-restore-drill.sh: a fresh k3s with three test Secrets (stored encrypted, k8s:enc:) restores with all Secrets decrypted in 7s; before folding the WAL the test saw 2 of 3 rows, which is why the real backup uses sqlite3 .backup. - gitea-backup-restore-drill.sh on a live repository dump: 29 repositories, 0 fsck failures, 29 live; the dump was deleted after.
100 lines
5.0 KiB
Bash
100 lines
5.0 KiB
Bash
#!/usr/bin/env bash
|
|
# Restore drill for the k3s datastore backup.
|
|
#
|
|
# sudo bash scripts/validate/k3s-datastore-restore-drill.sh # latest backup in AIStor
|
|
# bash scripts/validate/k3s-datastore-restore-drill.sh --from-dir DIR [--expect-secrets N]
|
|
# DIR holds state.db and token (an extracted archive, or a test datastore)
|
|
#
|
|
# Starts a throwaway k3s server (same version as production, --network none,
|
|
# no agent) on a copy of the backup, and checks that the API answers, that the
|
|
# Secret count matches and that every Secret decrypts (the token unlocks the
|
|
# encryption config stored in the datastore). Prints no Secret value. The
|
|
# container and the copy are removed at the end.
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
# shellcheck source=../lib/host-backup.sh
|
|
source "$ROOT/scripts/lib/host-backup.sh"
|
|
# Same version as the desktop server (k3s --version: v1.36.2+k3s1).
|
|
readonly K3S_IMAGE="docker.io/rancher/k3s:v1.36.2-k3s1@sha256:6a47cea22c4b834d4ba72c89d291696b79ebe406251f90b446e4dff03513dd87"
|
|
readonly NAME=k3s_datastore_restore_drill
|
|
|
|
from_dir="" expect_secrets=""
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--from-dir) from_dir="${2:?}"; shift 2 ;;
|
|
--expect-secrets) expect_secrets="${2:?}"; shift 2 ;;
|
|
*) sed -n '2,13p' "$0" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
base="${HB_WORK_ROOT}"
|
|
[[ -w "$base" ]] || base="$(mktemp -d)"
|
|
work="$(mktemp -d "$base/k3s-drill.XXXXXX")"
|
|
container="k3s-restore-drill-$$"
|
|
cleanup() {
|
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
|
# The drill container writes as root; remove its files with the same image.
|
|
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true
|
|
rm -rf -- "$work"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [[ -z "$from_dir" ]]; then
|
|
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials"
|
|
hb_load_aistor_env
|
|
latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)"
|
|
[[ -n "$latest" ]] || hb_fail "no k3s datastore backup found"
|
|
hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null
|
|
[[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest"
|
|
tar -C "$work" -xzf "$work/$latest"
|
|
from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)"
|
|
hb_log "drill on $latest"
|
|
if [[ -z "$expect_secrets" ]]; then
|
|
expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)"
|
|
hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)"
|
|
fi
|
|
fi
|
|
[[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token"
|
|
|
|
install -d -m 0700 "$work/server/db"
|
|
cp "$from_dir/state.db" "$work/server/db/state.db"
|
|
cp "$from_dir/token" "$work/server/token"
|
|
[[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed"
|
|
|
|
started="$(date +%s)"
|
|
docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \
|
|
-v "$work/server:/var/lib/rancher/k3s/server" \
|
|
"$K3S_IMAGE" server --disable-agent --secrets-encryption \
|
|
--token-file /var/lib/rancher/k3s/server/token \
|
|
--bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \
|
|
--disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null
|
|
|
|
ready=0
|
|
for _ in $(seq 1 60); do
|
|
if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi
|
|
sleep 5
|
|
done
|
|
if (( ready == 0 )); then
|
|
docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2
|
|
hb_fail "restored k3s did not become ready in 5 minutes"
|
|
fi
|
|
namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)"
|
|
secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)"
|
|
# -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero.
|
|
decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')"
|
|
hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s"
|
|
if [[ -n "$expect_secrets" ]]; then
|
|
diff=$(( secrets - expect_secrets ))
|
|
(( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets"
|
|
fi
|
|
(( decrypted > 0 )) || hb_fail "no Secret carried data after restore"
|
|
|
|
hb_write_metrics "$NAME" <<PROM
|
|
# HELP platform_backup_restore_drill_last_success_timestamp_seconds Unix time of the last successful restore drill.
|
|
# TYPE platform_backup_restore_drill_last_success_timestamp_seconds gauge
|
|
platform_backup_restore_drill_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
|
PROM
|
|
echo "K3S DATASTORE RESTORE DRILL PASS"
|