#!/usr/bin/env bash # Restore drill for the k3s datastore backup. # # sudo bash scripts/validate/k3s-datastore-restore-drill.sh # latest backup in AIStor # bash scripts/validate/k3s-datastore-restore-drill.sh --from-dir DIR [--expect-secrets N] # DIR holds state.db and token (an extracted archive, or a test datastore) # # Starts a throwaway k3s server (same version as production, --network none, # no agent) on a copy of the backup, and checks that the API answers, that the # Secret count matches and that every Secret decrypts (the token unlocks the # encryption config stored in the datastore). Prints no Secret value. The # container and the copy are removed at the end. set -Eeuo pipefail umask 077 readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" # shellcheck source=../lib/host-backup.sh source "$ROOT/scripts/lib/host-backup.sh" # Same version as the desktop server (k3s --version: v1.36.2+k3s1). readonly K3S_IMAGE="docker.io/rancher/k3s:v1.36.2-k3s1@sha256:6a47cea22c4b834d4ba72c89d291696b79ebe406251f90b446e4dff03513dd87" readonly NAME=k3s_datastore_restore_drill from_dir="" expect_secrets="" while (( $# > 0 )); do case "$1" in --from-dir) from_dir="${2:?}"; shift 2 ;; --expect-secrets) expect_secrets="${2:?}"; shift 2 ;; *) sed -n '2,13p' "$0" >&2; exit 2 ;; esac done base="${HB_WORK_ROOT}" [[ -w "$base" ]] || base="$(mktemp -d)" work="$(mktemp -d "$base/k3s-drill.XXXXXX")" container="k3s-restore-drill-$$" cleanup() { docker rm -f "$container" >/dev/null 2>&1 || true # The drill container writes as root; remove its files with the same image. docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true rm -rf -- "$work" } trap cleanup EXIT if [[ -z "$from_dir" ]]; then [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" hb_load_aistor_env latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)" [[ -n "$latest" ]] || hb_fail "no k3s datastore backup found" hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" tar -C "$work" -xzf "$work/$latest" from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)" hb_log "drill on $latest" if [[ -z "$expect_secrets" ]]; then expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)" hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)" fi fi [[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token" install -d -m 0700 "$work/server/db" cp "$from_dir/state.db" "$work/server/db/state.db" cp "$from_dir/token" "$work/server/token" [[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed" started="$(date +%s)" docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \ -v "$work/server:/var/lib/rancher/k3s/server" \ "$K3S_IMAGE" server --disable-agent --secrets-encryption \ --token-file /var/lib/rancher/k3s/server/token \ --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null ready=0 for _ in $(seq 1 60); do if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi sleep 5 done if (( ready == 0 )); then docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2 hb_fail "restored k3s did not become ready in 5 minutes" fi namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)" secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)" # -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero. decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')" hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s" if [[ -n "$expect_secrets" ]]; then diff=$(( secrets - expect_secrets )) (( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets" fi (( decrypted > 0 )) || hb_fail "no Secret carried data after restore" hb_write_metrics "$NAME" <