diff --git a/infrastructure/host/backup/platform-gitea-backup.service b/infrastructure/host/backup/platform-gitea-backup.service new file mode 100644 index 0000000..3cbc4e5 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.service @@ -0,0 +1,12 @@ +[Unit] +Description=Back up Gitea (gitea dump --skip-db) to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/gitea-backup.sh +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=1h diff --git a/infrastructure/host/backup/platform-gitea-backup.timer b/infrastructure/host/backup/platform-gitea-backup.timer new file mode 100644 index 0000000..add1a49 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly Gitea backup + +[Timer] +OnCalendar=Sun *-*-* 04:00:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.service b/infrastructure/host/backup/platform-k3s-datastore-backup.service new file mode 100644 index 0000000..cdd58bd --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=Back up the k3s datastore and token to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/k3s-datastore-backup.sh +Nice=10 +IOSchedulingClass=idle diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.timer b/infrastructure/host/backup/platform-k3s-datastore-backup.timer new file mode 100644 index 0000000..4fd6337 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily k3s datastore backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/backup/gitea-backup.sh b/scripts/backup/gitea-backup.sh new file mode 100644 index 0000000..c0c96f5 --- /dev/null +++ b/scripts/backup/gitea-backup.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Back up Gitea (repositories, packages, attachments, Actions data, app.ini) to +# AIStor (HDD sdb3). Root only. Run by platform-gitea-backup.timer. +# +# `gitea dump --skip-db` runs inside the live Gitea container and streams a +# tar.gz to stdout (logs go to stderr). The database is backed up separately by +# CloudNativePG (services/platform-postgres-backup). The dump contains app.ini, +# whose secrets are needed to restore, so the archive is sensitive. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly NAME=gitea_backup +readonly KCFG=/etc/rancher/k3s/k3s.yaml +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +hb_load_aistor_env +kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig "$KCFG" "$@"; } + +started="$(date +%s)" +previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)" +on_error() { + hb_write_metrics "$NAME" <"$work/$archive" 2>"$work/dump.log" +tar -tzf "$work/$archive" >"$work/list.txt" +grep -qx 'app.ini' "$work/list.txt" || hb_fail "the dump has no app.ini" +repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)" +(( repos > 0 )) || hb_fail "the dump has no repositories" +sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256" +size="$(stat -c %s "$work/$archive")" + +hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "backup/$AISTOR_BUCKET/gitea/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null + +trap - ERR +hb_write_metrics "$NAME" < +# The archive is then encrypted to that public key and copied with rclone. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly SERVER=/var/lib/rancher/k3s/server +readonly NAME=k3s_datastore_backup +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required" +hb_load_aistor_env + +started="$(date +%s)" +on_error() { + hb_write_metrics "$NAME" </dev/null || echo 0)" +trap on_error ERR + +work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT +ts="$(date -u +%Y%m%dT%H%M%SZ)" +stage="$work/k3s-datastore-$ts" +mkdir -m 0700 "$stage" + +sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'" +check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')" +[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check" +install -m 0600 "$SERVER/token" "$stage/token" +{ + echo "created=$ts" + echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)" + echo "host=$(hostname)" + echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')" +} >"$stage/META" +tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts" +sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256" +size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")" + +hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null + +offhost=0 +if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then + command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed" + gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \ + --output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz" + rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg" + offhost=1 +fi + +trap - ERR +hb_write_metrics "$NAME" </dev/null || hb_fail "missing command: $c"; done +jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON" +[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it" + +ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" +[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found" +endpoint="http://${ip}:80" + +work="$(mktemp -d)" +trap 'rm -rf -- "$work"' EXIT +kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env" +grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format" +printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY" +openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY" +cp "$POLICY_FILE" "$work/policy.json" + +printf '%s\n' \ + "Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \ + "Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \ + "User: a new generated access key attached to $POLICY" \ + "File: $HOST_BACKUP_ENV (root, 0600)" \ + 'No key value will be printed.' +printf 'Type APPLY to create them: ' +IFS= read -r confirmation +[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY" + +# Every mc call is silenced so no key reaches the terminal. +docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \ + --entrypoint /bin/sh "$MC_IMAGE" -ec ' +. /run/b/root.env +ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)" +mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null +mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null +mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null +mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null +mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null +mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null +mc admin user add root "$ak" "$sk" >/dev/null +mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null +mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null +probe=".bootstrap-probe-$(date +%s)" +printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null +[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ] +mc rm "backup/'"$BUCKET"'/$probe" >/dev/null +if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi +echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written" + +install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")" +{ + echo "AISTOR_BUCKET=$BUCKET" + echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")" + echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")" + echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):" + echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc" + echo "# OFFHOST_RCLONE_TARGET=remote:path" +} >"$HOST_BACKUP_ENV" +chmod 0600 "$HOST_BACKUP_ENV" +printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV." diff --git a/scripts/lib/host-backup.sh b/scripts/lib/host-backup.sh new file mode 100644 index 0000000..33954fb --- /dev/null +++ b/scripts/lib/host-backup.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Shared helpers for the host backup and host check scripts. Source only. +# shellcheck shell=bash + +readonly HOST_BACKUP_ENV=/etc/platform-backup/aistor.env +readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector +# Backup staging (root 0700, same SSD). Not /var/tmp: dockerd must see the path. +readonly HB_WORK_ROOT=/var/lib/platform-backup/work +# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh. +readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b" + +hb_log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; } +hb_fail() { hb_log "FAIL: $*" >&2; exit 1; } + +# Write Prometheus textfile metrics atomically: hb_write_metrics NAME <<'PROM' ... PROM +hb_write_metrics() { + local name="$1" tmp + [[ -d "$TEXTFILE_DIR" ]] || { hb_log "textfile dir $TEXTFILE_DIR missing; metrics not written"; cat >/dev/null; return 0; } + tmp="$(mktemp "$TEXTFILE_DIR/.${name}.XXXXXX")" + cat >"$tmp" + chmod 0644 "$tmp" + mv -f -- "$tmp" "$TEXTFILE_DIR/${name}.prom" +} + +# Load the AIStor endpoint and the backup user's keys (root-only file). +hb_load_aistor_env() { + [[ -f "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV is missing; run scripts/bootstrap/create-host-backup-credentials.sh" + [[ "$(stat -c '%u %a' "$HOST_BACKUP_ENV")" == "0 600" ]] || hb_fail "$HOST_BACKUP_ENV must be owned by root with mode 0600" + # shellcheck disable=SC1090 + . "$HOST_BACKUP_ENV" + : "${AISTOR_BUCKET:?}" "${AISTOR_ACCESS_KEY:?}" "${AISTOR_SECRET_KEY:?}" + # The host cannot resolve cluster DNS, so use the Service ClusterIP (port 80 -> 9000). + if [[ -z "${AISTOR_ENDPOINT:-}" ]]; then + local ip + ip="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml -n object-storage \ + get service minio -o jsonpath='{.spec.clusterIP}')" + [[ -n "$ip" ]] || hb_fail "cannot find the object-storage/minio Service" + AISTOR_ENDPOINT="http://${ip}:80" + fi +} + +# Run the pinned mc client on the host network with the backup user's keys. +# The keys go through a 0600 env file, never the command line (ps would show it). +# Usage: hb_mc +hb_mc() { + local work="$1"; shift + local envfile="$work/.mc.env" + if [[ ! -f "$envfile" ]]; then + ( umask 077; printf 'MC_HOST_backup=%s\nMC_CONFIG_DIR=/tmp/mc\n' \ + "${AISTOR_ENDPOINT/:\/\//://${AISTOR_ACCESS_KEY}:${AISTOR_SECRET_KEY}@}" >"$envfile" ) + fi + docker run --rm --network host --user "$(id -u):$(id -g)" \ + --env-file "$envfile" -v "$work:/work" --tmpfs /tmp \ + "$MC_IMAGE" "$@" +} diff --git a/services/platform-host-checks/bucket-policy.json b/services/platform-host-checks/bucket-policy.json new file mode 100644 index 0000000..cf6e7a6 --- /dev/null +++ b/services/platform-host-checks/bucket-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::platform-host-backups"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::platform-host-backups/*"] + } + ] +}