From 77daa39a41b07707f8b315b02ee9559be4f9bc57 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:13:02 +0900 Subject: [PATCH] feat(backup): back up the k3s datastore and Gitea to AIStor Nothing backs up the k3s datastore or Gitea's repositories, packages and app.ini. W1 covers platform-postgres only. One SSD failure loses the source, CI artefacts, published libraries and every cluster Secret. Add two root systemd timers that write to a new AIStor bucket on the HDD (platform-host-backups, same AIStor as W1's postgres backups): - k3s datastore, daily: sqlite3 .backup of the live state.db plus the server token (k3s backup/restore docs: the token is needed to read it back), integrity_check, tar.gz, sha256, upload. Off-host copy is prepared behind two variables (gpg recipient file, rclone target) and stays off until the owner chooses a place. - Gitea, weekly: gitea dump --skip-db streamed from the live container, listing checked for app.ini and repositories, sha256, upload. The DB is in the CNPG backup. Each run writes textfile metrics for the alerts in the next commit. create-host-backup-credentials.sh makes the bucket (50GiB quota, lifecycle 56 days for gitea/, 30 days for k3s-datastore/), a bucket-scoped policy and user, and a root-only env file; no key is printed and mc gets keys only through a 0600 env file. Rejected: in-cluster CronJobs. The datastore is a root-owned host file and the dump needs the Gitea DB connection; a hostPath root pod or a second Gitea-labelled pod is a larger privilege than a root timer. Risk: not installed here (owner sudo steps). The k3s archive holds the token; together with the datastore it decrypts every Secret. It lives in a bucket only the backup user can read. Verification (desktop): a dump streamed with --file - is a valid tar.gz with app.ini; repositories appear as repos//.git (29, matching the live count). The scripts parse with bash -n. --- .../host/backup/platform-gitea-backup.service | 12 +++ .../host/backup/platform-gitea-backup.timer | 10 +++ .../platform-k3s-datastore-backup.service | 11 +++ .../platform-k3s-datastore-backup.timer | 10 +++ scripts/backup/gitea-backup.sh | 61 +++++++++++++ scripts/backup/k3s-datastore-backup.sh | 86 +++++++++++++++++++ .../create-host-backup-credentials.sh | 81 +++++++++++++++++ scripts/lib/host-backup.sh | 55 ++++++++++++ .../platform-host-checks/bucket-policy.json | 15 ++++ 9 files changed, 341 insertions(+) create mode 100644 infrastructure/host/backup/platform-gitea-backup.service create mode 100644 infrastructure/host/backup/platform-gitea-backup.timer create mode 100644 infrastructure/host/backup/platform-k3s-datastore-backup.service create mode 100644 infrastructure/host/backup/platform-k3s-datastore-backup.timer create mode 100644 scripts/backup/gitea-backup.sh create mode 100644 scripts/backup/k3s-datastore-backup.sh create mode 100644 scripts/bootstrap/create-host-backup-credentials.sh create mode 100644 scripts/lib/host-backup.sh create mode 100644 services/platform-host-checks/bucket-policy.json diff --git a/infrastructure/host/backup/platform-gitea-backup.service b/infrastructure/host/backup/platform-gitea-backup.service new file mode 100644 index 0000000..3cbc4e5 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.service @@ -0,0 +1,12 @@ +[Unit] +Description=Back up Gitea (gitea dump --skip-db) to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/gitea-backup.sh +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=1h diff --git a/infrastructure/host/backup/platform-gitea-backup.timer b/infrastructure/host/backup/platform-gitea-backup.timer new file mode 100644 index 0000000..add1a49 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly Gitea backup + +[Timer] +OnCalendar=Sun *-*-* 04:00:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.service b/infrastructure/host/backup/platform-k3s-datastore-backup.service new file mode 100644 index 0000000..cdd58bd --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=Back up the k3s datastore and token to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/k3s-datastore-backup.sh +Nice=10 +IOSchedulingClass=idle diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.timer b/infrastructure/host/backup/platform-k3s-datastore-backup.timer new file mode 100644 index 0000000..4fd6337 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily k3s datastore backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/backup/gitea-backup.sh b/scripts/backup/gitea-backup.sh new file mode 100644 index 0000000..c0c96f5 --- /dev/null +++ b/scripts/backup/gitea-backup.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Back up Gitea (repositories, packages, attachments, Actions data, app.ini) to +# AIStor (HDD sdb3). Root only. Run by platform-gitea-backup.timer. +# +# `gitea dump --skip-db` runs inside the live Gitea container and streams a +# tar.gz to stdout (logs go to stderr). The database is backed up separately by +# CloudNativePG (services/platform-postgres-backup). The dump contains app.ini, +# whose secrets are needed to restore, so the archive is sensitive. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly NAME=gitea_backup +readonly KCFG=/etc/rancher/k3s/k3s.yaml +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +hb_load_aistor_env +kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig "$KCFG" "$@"; } + +started="$(date +%s)" +previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)" +on_error() { + hb_write_metrics "$NAME" <"$work/$archive" 2>"$work/dump.log" +tar -tzf "$work/$archive" >"$work/list.txt" +grep -qx 'app.ini' "$work/list.txt" || hb_fail "the dump has no app.ini" +repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)" +(( repos > 0 )) || hb_fail "the dump has no repositories" +sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256" +size="$(stat -c %s "$work/$archive")" + +hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "backup/$AISTOR_BUCKET/gitea/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null + +trap - ERR +hb_write_metrics "$NAME" < +# The archive is then encrypted to that public key and copied with rclone. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly SERVER=/var/lib/rancher/k3s/server +readonly NAME=k3s_datastore_backup +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required" +hb_load_aistor_env + +started="$(date +%s)" +on_error() { + hb_write_metrics "$NAME" </dev/null || echo 0)" +trap on_error ERR + +work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT +ts="$(date -u +%Y%m%dT%H%M%SZ)" +stage="$work/k3s-datastore-$ts" +mkdir -m 0700 "$stage" + +sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'" +check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')" +[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check" +install -m 0600 "$SERVER/token" "$stage/token" +{ + echo "created=$ts" + echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)" + echo "host=$(hostname)" + echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')" +} >"$stage/META" +tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts" +sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256" +size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")" + +hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null + +offhost=0 +if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then + command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed" + gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \ + --output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz" + rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg" + offhost=1 +fi + +trap - ERR +hb_write_metrics "$NAME" </dev/null || hb_fail "missing command: $c"; done +jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON" +[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it" + +ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" +[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found" +endpoint="http://${ip}:80" + +work="$(mktemp -d)" +trap 'rm -rf -- "$work"' EXIT +kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env" +grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format" +printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY" +openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY" +cp "$POLICY_FILE" "$work/policy.json" + +printf '%s\n' \ + "Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \ + "Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \ + "User: a new generated access key attached to $POLICY" \ + "File: $HOST_BACKUP_ENV (root, 0600)" \ + 'No key value will be printed.' +printf 'Type APPLY to create them: ' +IFS= read -r confirmation +[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY" + +# Every mc call is silenced so no key reaches the terminal. +docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \ + --entrypoint /bin/sh "$MC_IMAGE" -ec ' +. /run/b/root.env +ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)" +mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null +mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null +mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null +mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null +mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null +mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null +mc admin user add root "$ak" "$sk" >/dev/null +mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null +mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null +probe=".bootstrap-probe-$(date +%s)" +printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null +[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ] +mc rm "backup/'"$BUCKET"'/$probe" >/dev/null +if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi +echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written" + +install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")" +{ + echo "AISTOR_BUCKET=$BUCKET" + echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")" + echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")" + echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):" + echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc" + echo "# OFFHOST_RCLONE_TARGET=remote:path" +} >"$HOST_BACKUP_ENV" +chmod 0600 "$HOST_BACKUP_ENV" +printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV." diff --git a/scripts/lib/host-backup.sh b/scripts/lib/host-backup.sh new file mode 100644 index 0000000..33954fb --- /dev/null +++ b/scripts/lib/host-backup.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Shared helpers for the host backup and host check scripts. Source only. +# shellcheck shell=bash + +readonly HOST_BACKUP_ENV=/etc/platform-backup/aistor.env +readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector +# Backup staging (root 0700, same SSD). Not /var/tmp: dockerd must see the path. +readonly HB_WORK_ROOT=/var/lib/platform-backup/work +# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh. +readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b" + +hb_log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; } +hb_fail() { hb_log "FAIL: $*" >&2; exit 1; } + +# Write Prometheus textfile metrics atomically: hb_write_metrics NAME <<'PROM' ... PROM +hb_write_metrics() { + local name="$1" tmp + [[ -d "$TEXTFILE_DIR" ]] || { hb_log "textfile dir $TEXTFILE_DIR missing; metrics not written"; cat >/dev/null; return 0; } + tmp="$(mktemp "$TEXTFILE_DIR/.${name}.XXXXXX")" + cat >"$tmp" + chmod 0644 "$tmp" + mv -f -- "$tmp" "$TEXTFILE_DIR/${name}.prom" +} + +# Load the AIStor endpoint and the backup user's keys (root-only file). +hb_load_aistor_env() { + [[ -f "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV is missing; run scripts/bootstrap/create-host-backup-credentials.sh" + [[ "$(stat -c '%u %a' "$HOST_BACKUP_ENV")" == "0 600" ]] || hb_fail "$HOST_BACKUP_ENV must be owned by root with mode 0600" + # shellcheck disable=SC1090 + . "$HOST_BACKUP_ENV" + : "${AISTOR_BUCKET:?}" "${AISTOR_ACCESS_KEY:?}" "${AISTOR_SECRET_KEY:?}" + # The host cannot resolve cluster DNS, so use the Service ClusterIP (port 80 -> 9000). + if [[ -z "${AISTOR_ENDPOINT:-}" ]]; then + local ip + ip="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml -n object-storage \ + get service minio -o jsonpath='{.spec.clusterIP}')" + [[ -n "$ip" ]] || hb_fail "cannot find the object-storage/minio Service" + AISTOR_ENDPOINT="http://${ip}:80" + fi +} + +# Run the pinned mc client on the host network with the backup user's keys. +# The keys go through a 0600 env file, never the command line (ps would show it). +# Usage: hb_mc +hb_mc() { + local work="$1"; shift + local envfile="$work/.mc.env" + if [[ ! -f "$envfile" ]]; then + ( umask 077; printf 'MC_HOST_backup=%s\nMC_CONFIG_DIR=/tmp/mc\n' \ + "${AISTOR_ENDPOINT/:\/\//://${AISTOR_ACCESS_KEY}:${AISTOR_SECRET_KEY}@}" >"$envfile" ) + fi + docker run --rm --network host --user "$(id -u):$(id -g)" \ + --env-file "$envfile" -v "$work:/work" --tmpfs /tmp \ + "$MC_IMAGE" "$@" +} diff --git a/services/platform-host-checks/bucket-policy.json b/services/platform-host-checks/bucket-policy.json new file mode 100644 index 0000000..cf6e7a6 --- /dev/null +++ b/services/platform-host-checks/bucket-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::platform-host-backups"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::platform-host-backups/*"] + } + ] +}