From 77daa39a41b07707f8b315b02ee9559be4f9bc57 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:13:02 +0900 Subject: [PATCH 1/5] feat(backup): back up the k3s datastore and Gitea to AIStor Nothing backs up the k3s datastore or Gitea's repositories, packages and app.ini. W1 covers platform-postgres only. One SSD failure loses the source, CI artefacts, published libraries and every cluster Secret. Add two root systemd timers that write to a new AIStor bucket on the HDD (platform-host-backups, same AIStor as W1's postgres backups): - k3s datastore, daily: sqlite3 .backup of the live state.db plus the server token (k3s backup/restore docs: the token is needed to read it back), integrity_check, tar.gz, sha256, upload. Off-host copy is prepared behind two variables (gpg recipient file, rclone target) and stays off until the owner chooses a place. - Gitea, weekly: gitea dump --skip-db streamed from the live container, listing checked for app.ini and repositories, sha256, upload. The DB is in the CNPG backup. Each run writes textfile metrics for the alerts in the next commit. create-host-backup-credentials.sh makes the bucket (50GiB quota, lifecycle 56 days for gitea/, 30 days for k3s-datastore/), a bucket-scoped policy and user, and a root-only env file; no key is printed and mc gets keys only through a 0600 env file. Rejected: in-cluster CronJobs. The datastore is a root-owned host file and the dump needs the Gitea DB connection; a hostPath root pod or a second Gitea-labelled pod is a larger privilege than a root timer. Risk: not installed here (owner sudo steps). The k3s archive holds the token; together with the datastore it decrypts every Secret. It lives in a bucket only the backup user can read. Verification (desktop): a dump streamed with --file - is a valid tar.gz with app.ini; repositories appear as repos//.git (29, matching the live count). The scripts parse with bash -n. --- .../host/backup/platform-gitea-backup.service | 12 +++ .../host/backup/platform-gitea-backup.timer | 10 +++ .../platform-k3s-datastore-backup.service | 11 +++ .../platform-k3s-datastore-backup.timer | 10 +++ scripts/backup/gitea-backup.sh | 61 +++++++++++++ scripts/backup/k3s-datastore-backup.sh | 86 +++++++++++++++++++ .../create-host-backup-credentials.sh | 81 +++++++++++++++++ scripts/lib/host-backup.sh | 55 ++++++++++++ .../platform-host-checks/bucket-policy.json | 15 ++++ 9 files changed, 341 insertions(+) create mode 100644 infrastructure/host/backup/platform-gitea-backup.service create mode 100644 infrastructure/host/backup/platform-gitea-backup.timer create mode 100644 infrastructure/host/backup/platform-k3s-datastore-backup.service create mode 100644 infrastructure/host/backup/platform-k3s-datastore-backup.timer create mode 100644 scripts/backup/gitea-backup.sh create mode 100644 scripts/backup/k3s-datastore-backup.sh create mode 100644 scripts/bootstrap/create-host-backup-credentials.sh create mode 100644 scripts/lib/host-backup.sh create mode 100644 services/platform-host-checks/bucket-policy.json diff --git a/infrastructure/host/backup/platform-gitea-backup.service b/infrastructure/host/backup/platform-gitea-backup.service new file mode 100644 index 0000000..3cbc4e5 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.service @@ -0,0 +1,12 @@ +[Unit] +Description=Back up Gitea (gitea dump --skip-db) to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/gitea-backup.sh +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=1h diff --git a/infrastructure/host/backup/platform-gitea-backup.timer b/infrastructure/host/backup/platform-gitea-backup.timer new file mode 100644 index 0000000..add1a49 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly Gitea backup + +[Timer] +OnCalendar=Sun *-*-* 04:00:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.service b/infrastructure/host/backup/platform-k3s-datastore-backup.service new file mode 100644 index 0000000..cdd58bd --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=Back up the k3s datastore and token to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/k3s-datastore-backup.sh +Nice=10 +IOSchedulingClass=idle diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.timer b/infrastructure/host/backup/platform-k3s-datastore-backup.timer new file mode 100644 index 0000000..4fd6337 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily k3s datastore backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/backup/gitea-backup.sh b/scripts/backup/gitea-backup.sh new file mode 100644 index 0000000..c0c96f5 --- /dev/null +++ b/scripts/backup/gitea-backup.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Back up Gitea (repositories, packages, attachments, Actions data, app.ini) to +# AIStor (HDD sdb3). Root only. Run by platform-gitea-backup.timer. +# +# `gitea dump --skip-db` runs inside the live Gitea container and streams a +# tar.gz to stdout (logs go to stderr). The database is backed up separately by +# CloudNativePG (services/platform-postgres-backup). The dump contains app.ini, +# whose secrets are needed to restore, so the archive is sensitive. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly NAME=gitea_backup +readonly KCFG=/etc/rancher/k3s/k3s.yaml +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +hb_load_aistor_env +kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig "$KCFG" "$@"; } + +started="$(date +%s)" +previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)" +on_error() { + hb_write_metrics "$NAME" <"$work/$archive" 2>"$work/dump.log" +tar -tzf "$work/$archive" >"$work/list.txt" +grep -qx 'app.ini' "$work/list.txt" || hb_fail "the dump has no app.ini" +repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)" +(( repos > 0 )) || hb_fail "the dump has no repositories" +sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256" +size="$(stat -c %s "$work/$archive")" + +hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "backup/$AISTOR_BUCKET/gitea/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null + +trap - ERR +hb_write_metrics "$NAME" < +# The archive is then encrypted to that public key and copied with rclone. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly SERVER=/var/lib/rancher/k3s/server +readonly NAME=k3s_datastore_backup +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required" +hb_load_aistor_env + +started="$(date +%s)" +on_error() { + hb_write_metrics "$NAME" </dev/null || echo 0)" +trap on_error ERR + +work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT +ts="$(date -u +%Y%m%dT%H%M%SZ)" +stage="$work/k3s-datastore-$ts" +mkdir -m 0700 "$stage" + +sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'" +check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')" +[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check" +install -m 0600 "$SERVER/token" "$stage/token" +{ + echo "created=$ts" + echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)" + echo "host=$(hostname)" + echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')" +} >"$stage/META" +tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts" +sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256" +size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")" + +hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null + +offhost=0 +if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then + command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed" + gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \ + --output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz" + rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg" + offhost=1 +fi + +trap - ERR +hb_write_metrics "$NAME" </dev/null || hb_fail "missing command: $c"; done +jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON" +[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it" + +ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" +[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found" +endpoint="http://${ip}:80" + +work="$(mktemp -d)" +trap 'rm -rf -- "$work"' EXIT +kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env" +grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format" +printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY" +openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY" +cp "$POLICY_FILE" "$work/policy.json" + +printf '%s\n' \ + "Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \ + "Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \ + "User: a new generated access key attached to $POLICY" \ + "File: $HOST_BACKUP_ENV (root, 0600)" \ + 'No key value will be printed.' +printf 'Type APPLY to create them: ' +IFS= read -r confirmation +[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY" + +# Every mc call is silenced so no key reaches the terminal. +docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \ + --entrypoint /bin/sh "$MC_IMAGE" -ec ' +. /run/b/root.env +ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)" +mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null +mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null +mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null +mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null +mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null +mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null +mc admin user add root "$ak" "$sk" >/dev/null +mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null +mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null +probe=".bootstrap-probe-$(date +%s)" +printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null +[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ] +mc rm "backup/'"$BUCKET"'/$probe" >/dev/null +if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi +echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written" + +install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")" +{ + echo "AISTOR_BUCKET=$BUCKET" + echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")" + echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")" + echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):" + echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc" + echo "# OFFHOST_RCLONE_TARGET=remote:path" +} >"$HOST_BACKUP_ENV" +chmod 0600 "$HOST_BACKUP_ENV" +printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV." diff --git a/scripts/lib/host-backup.sh b/scripts/lib/host-backup.sh new file mode 100644 index 0000000..33954fb --- /dev/null +++ b/scripts/lib/host-backup.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Shared helpers for the host backup and host check scripts. Source only. +# shellcheck shell=bash + +readonly HOST_BACKUP_ENV=/etc/platform-backup/aistor.env +readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector +# Backup staging (root 0700, same SSD). Not /var/tmp: dockerd must see the path. +readonly HB_WORK_ROOT=/var/lib/platform-backup/work +# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh. +readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b" + +hb_log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; } +hb_fail() { hb_log "FAIL: $*" >&2; exit 1; } + +# Write Prometheus textfile metrics atomically: hb_write_metrics NAME <<'PROM' ... PROM +hb_write_metrics() { + local name="$1" tmp + [[ -d "$TEXTFILE_DIR" ]] || { hb_log "textfile dir $TEXTFILE_DIR missing; metrics not written"; cat >/dev/null; return 0; } + tmp="$(mktemp "$TEXTFILE_DIR/.${name}.XXXXXX")" + cat >"$tmp" + chmod 0644 "$tmp" + mv -f -- "$tmp" "$TEXTFILE_DIR/${name}.prom" +} + +# Load the AIStor endpoint and the backup user's keys (root-only file). +hb_load_aistor_env() { + [[ -f "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV is missing; run scripts/bootstrap/create-host-backup-credentials.sh" + [[ "$(stat -c '%u %a' "$HOST_BACKUP_ENV")" == "0 600" ]] || hb_fail "$HOST_BACKUP_ENV must be owned by root with mode 0600" + # shellcheck disable=SC1090 + . "$HOST_BACKUP_ENV" + : "${AISTOR_BUCKET:?}" "${AISTOR_ACCESS_KEY:?}" "${AISTOR_SECRET_KEY:?}" + # The host cannot resolve cluster DNS, so use the Service ClusterIP (port 80 -> 9000). + if [[ -z "${AISTOR_ENDPOINT:-}" ]]; then + local ip + ip="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml -n object-storage \ + get service minio -o jsonpath='{.spec.clusterIP}')" + [[ -n "$ip" ]] || hb_fail "cannot find the object-storage/minio Service" + AISTOR_ENDPOINT="http://${ip}:80" + fi +} + +# Run the pinned mc client on the host network with the backup user's keys. +# The keys go through a 0600 env file, never the command line (ps would show it). +# Usage: hb_mc +hb_mc() { + local work="$1"; shift + local envfile="$work/.mc.env" + if [[ ! -f "$envfile" ]]; then + ( umask 077; printf 'MC_HOST_backup=%s\nMC_CONFIG_DIR=/tmp/mc\n' \ + "${AISTOR_ENDPOINT/:\/\//://${AISTOR_ACCESS_KEY}:${AISTOR_SECRET_KEY}@}" >"$envfile" ) + fi + docker run --rm --network host --user "$(id -u):$(id -g)" \ + --env-file "$envfile" -v "$work:/work" --tmpfs /tmp \ + "$MC_IMAGE" "$@" +} diff --git a/services/platform-host-checks/bucket-policy.json b/services/platform-host-checks/bucket-policy.json new file mode 100644 index 0000000..cf6e7a6 --- /dev/null +++ b/services/platform-host-checks/bucket-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::platform-host-backups"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::platform-host-backups/*"] + } + ] +} From 012452cf7897f2396f91fd2e1caaaa4b6073eeb7 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:13:02 +0900 Subject: [PATCH 2/5] feat(observability): alert on host backups, drift and k3s serving certs Backups, the daily drift check and the k3s serving certificates have no alert. k3s renews certificates only when it starts within 120 days of expiry, and the two kube-prometheus-stack kubelet certificate alerts can never fire because the k3s kubelet exports no kubelet_certificate_manager_* metric. Add a PrometheusRule (platform-host-checks) for backup age, failed runs, missing metrics, drift left for 6h, a stale drift check, and serving certificate expiry (warning under 100 days, critical under 14). Add the two remaining host timers (certificate expiry via openssl on 6443 and 10250; the drift check against origin/main) and an installer that copies the scripts to a root-owned path, creates the textfile and work directories and enables all four timers. Disable the two dead kubelet certificate rules in kube-prometheus-stack. Risk: the metrics need the node-exporter textfile collector, a one-line change in W1's node-exporter values that was asked of W1. Until then PlatformHostBackupMetricsAbsent fires after a day, which is the point. Verification (desktop): - test-platform-host-checks-alerts.sh: promtool check and unit tests pass; with the k3s threshold changed from 2 to 5 days they fail. - render-observability-core.sh passes; the render has no Kubelet* certificate alerts and still has both KubeClientCertificateExpiration. - k3s-cert-expiry.sh as the desktop user reads both certificates. - install-host-checks.sh --check: systemd-analyze verify is clean. --- .../backup/platform-k3s-cert-expiry.service | 8 + .../backup/platform-k3s-cert-expiry.timer | 9 ++ .../backup/platform-live-drift-check.service | 16 ++ .../backup/platform-live-drift-check.timer | 10 ++ scripts/bootstrap/install-host-checks.sh | 78 +++++++++ scripts/validate/k3s-cert-expiry.sh | 25 +++ scripts/validate/live-drift-check-main.sh | 16 ++ .../test-platform-host-checks-alerts.sh | 151 ++++++++++++++++++ .../kube-prometheus-stack/values/home.yaml | 5 + services/platform-host-checks/README.md | 83 ++++++++++ .../platform-host-checks/kustomization.yaml | 5 + .../prometheus-rules.yaml | 88 ++++++++++ 12 files changed, 494 insertions(+) create mode 100644 infrastructure/host/backup/platform-k3s-cert-expiry.service create mode 100644 infrastructure/host/backup/platform-k3s-cert-expiry.timer create mode 100644 infrastructure/host/backup/platform-live-drift-check.service create mode 100644 infrastructure/host/backup/platform-live-drift-check.timer create mode 100644 scripts/bootstrap/install-host-checks.sh create mode 100644 scripts/validate/k3s-cert-expiry.sh create mode 100644 scripts/validate/live-drift-check-main.sh create mode 100644 scripts/validate/test-platform-host-checks-alerts.sh create mode 100644 services/platform-host-checks/README.md create mode 100644 services/platform-host-checks/kustomization.yaml create mode 100644 services/platform-host-checks/prometheus-rules.yaml diff --git a/infrastructure/host/backup/platform-k3s-cert-expiry.service b/infrastructure/host/backup/platform-k3s-cert-expiry.service new file mode 100644 index 0000000..4215754 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-cert-expiry.service @@ -0,0 +1,8 @@ +[Unit] +Description=Record the k3s API server and kubelet serving certificate expiry +After=k3s.service + +[Service] +Type=oneshot +User=donghyeon +ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/k3s-cert-expiry.sh diff --git a/infrastructure/host/backup/platform-k3s-cert-expiry.timer b/infrastructure/host/backup/platform-k3s-cert-expiry.timer new file mode 100644 index 0000000..c885829 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-cert-expiry.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily k3s certificate expiry check + +[Timer] +OnCalendar=*-*-* 05:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-live-drift-check.service b/infrastructure/host/backup/platform-live-drift-check.service new file mode 100644 index 0000000..1838a38 --- /dev/null +++ b/infrastructure/host/backup/platform-live-drift-check.service @@ -0,0 +1,16 @@ +[Unit] +Description=Compare the cluster with platform-core origin/main (kubectl diff, server dry-run) +After=k3s.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=donghyeon +Environment=PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm +Environment=PLATFORM_CHECKOUT=/home/donghyeon/workspace/platform +ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/live-drift-check-main.sh +# Exit 1 means "drift found"; the metric and its alert report that. Only 2 (the +# check itself failed) marks the unit failed. +SuccessExitStatus=1 +Nice=10 +TimeoutStartSec=30min diff --git a/infrastructure/host/backup/platform-live-drift-check.timer b/infrastructure/host/backup/platform-live-drift-check.timer new file mode 100644 index 0000000..1267b5a --- /dev/null +++ b/infrastructure/host/backup/platform-live-drift-check.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily live drift check + +[Timer] +OnCalendar=*-*-* 05:15:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/bootstrap/install-host-checks.sh b/scripts/bootstrap/install-host-checks.sh new file mode 100644 index 0000000..21d13ee --- /dev/null +++ b/scripts/bootstrap/install-host-checks.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Install the host backup and check timers (infrastructure/host/backup). +# +# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing +# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs +# +# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root +# timers never execute files the desktop user can edit. +set -Eeuo pipefail +umask 022 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly LIB=/usr/local/lib/platform-backup +readonly UNIT_DIR=/etc/systemd/system +readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector +readonly WORK_ROOT=/var/lib/platform-backup/work +readonly USER_NAME=donghyeon +readonly -a SCRIPTS=( + scripts/lib/host-backup.sh:lib/host-backup.sh + scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh + scripts/backup/gitea-backup.sh:backup/gitea-backup.sh + scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh + scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh + services/platform-host-checks/README.md:README.md +) +readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry) + +fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +mode="${1:-}" +[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; } + +echo "== check" +for pair in "${SCRIPTS[@]}"; do + src="${pair%%:*}" + [[ -f "$ROOT/$src" ]] || fail "missing $src" + [[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src" +done +for t in "${TIMERS[@]}"; do + for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done +done +if command -v systemd-analyze >/dev/null; then + verify_dir="$(mktemp -d)" + trap 'rm -rf -- "$verify_dir"' EXIT + cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/" + # ExecStart paths do not exist before installation; only syntax and keys are checked. + out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)" + [[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; } + echo " units: systemd-analyze verify clean" +fi +echo "== plan" +for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done +for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done +echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)" +echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check" +[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; } + +[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)" +[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first" +echo "== install" +for pair in "${SCRIPTS[@]}"; do + install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}" +done +install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR" +install -d -o root -g root -m 0700 "$WORK_ROOT" +for t in "${TIMERS[@]}"; do + install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/" +done +systemctl daemon-reload +for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done + +echo "== first runs" +for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do + if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi +done +echo "== metrics" +grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet" +systemctl list-timers 'platform-*' --no-pager +echo "HOST CHECKS INSTALLED" diff --git a/scripts/validate/k3s-cert-expiry.sh b/scripts/validate/k3s-cert-expiry.sh new file mode 100644 index 0000000..55f5032 --- /dev/null +++ b/scripts/validate/k3s-cert-expiry.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Write the notAfter time of the k3s API server (6443) and kubelet (10250) +# serving certificates as Prometheus textfile metrics. No root needed. +# k3s renews these only when it starts and they are within 120 days of expiry +# (docs.k3s.io cli/certificate), and the kubelet exposes no certificate metric, +# so nothing else in the cluster watches them. +set -Eeuo pipefail +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +lines="" +for target in apiserver:6443 kubelet:10250; do + name="${target%%:*}" port="${target##*:}" + end="$(echo | timeout 10 openssl s_client -connect "127.0.0.1:$port" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//')" || true + [[ -n "$end" ]] || hb_fail "could not read the $name certificate on 127.0.0.1:$port" + lines+="platform_k3s_certificate_not_after_seconds{certificate=\"$name\"} $(date -d "$end" +%s)"$'\n' + hb_log "$name certificate notAfter $end" +done +hb_write_metrics platform_k3s_certificates <"$work/tests.yaml" <<'EOF' +rule_files: + - rules.yaml +evaluation_interval: 1m +tests: + # k3s datastore: last success at t=0; stale after 2 days, alert after 1h more. + - interval: 1m + input_series: + - series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}' + values: '0x5500' + - series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}' + values: '0x5500' + - series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}' + values: '0x5500' + - series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}' + values: '0x5500' + alert_rule_test: + - eval_time: 24h + alertname: PlatformK3sDatastoreBackupTooOld + exp_alerts: [] + - eval_time: 50h + alertname: PlatformK3sDatastoreBackupTooOld + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: k3s datastore has not been backed up for more than 2 days + description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST. + - eval_time: 90h + alertname: PlatformGiteaBackupTooOld + exp_alerts: [] + - eval_time: 90h + alertname: PlatformHostBackupMetricsAbsent + exp_alerts: [] + - interval: 1m + input_series: + - series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}' + values: '0x11700' + alert_rule_test: + - eval_time: 194h + alertname: PlatformGiteaBackupTooOld + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: Gitea has not been backed up for more than 8 days + description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST. + # A failed run is reported with its backup label. + - interval: 1m + input_series: + - series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}' + values: '0x30' + - series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}' + values: '1x30' + alert_rule_test: + - eval_time: 15m + alertname: PlatformHostBackupFailed + exp_alerts: + - exp_labels: {severity: warning, backup: gitea} + exp_annotations: + summary: "The last gitea backup run failed" + description: "Check `journalctl -u platform-gitea-backup` on desktop." + # Metrics never written (timers or textfile collector missing). + - interval: 1h + input_series: + - series: 'up{job="prometheus-node-exporter"}' + values: '1x30' + alert_rule_test: + - eval_time: 25h + alertname: PlatformHostBackupMetricsAbsent + exp_alerts: + - exp_labels: {severity: warning, backup: k3s-datastore} + exp_annotations: + summary: Host backup metrics are missing + description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. + - exp_labels: {severity: warning, backup: gitea} + exp_annotations: + summary: Host backup metrics are missing + description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. + # Drift for 7h fires; drift 0 does not. + - interval: 1m + input_series: + - series: 'platform_live_drift_objects' + values: '3x500' + - series: 'platform_live_drift_last_run_timestamp_seconds' + values: '0x500' + - series: 'platform_live_drift_check_success' + values: '1x500' + alert_rule_test: + - eval_time: 5h + alertname: PlatformLiveDriftDetected + exp_alerts: [] + - eval_time: 7h + alertname: PlatformLiveDriftDetected + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: "3 live objects differ from platform-core origin/main" + description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them. + - eval_time: 7h + alertname: PlatformLiveDriftCheckStale + exp_alerts: [] + # Certificates: 50 days left -> warning only; 10 days left -> warning and critical. + - interval: 1m + input_series: + - series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}' + values: '4320000x200' + - series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}' + values: '864000x200' + - series: 'platform_k3s_certificate_check_timestamp_seconds' + values: '0x200' + alert_rule_test: + - eval_time: 2h + alertname: PlatformK3sServingCertificateExpiry + exp_alerts: + - exp_labels: {severity: warning, certificate: apiserver} + exp_annotations: + summary: "k3s apiserver serving certificate expires in less than 100 days" + description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). + - exp_labels: {severity: warning, certificate: kubelet} + exp_annotations: + summary: "k3s kubelet serving certificate expires in less than 100 days" + description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). + - exp_labels: {severity: critical, certificate: kubelet} + exp_annotations: + summary: "k3s kubelet serving certificate expires in less than 14 days" + description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`. + - eval_time: 2h + alertname: PlatformK3sCertificateCheckStale + exp_alerts: [] +EOF + +docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml +docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml +echo "PLATFORM HOST CHECK ALERTS PASS" diff --git a/services/observability/kube-prometheus-stack/values/home.yaml b/services/observability/kube-prometheus-stack/values/home.yaml index 2c0f3a0..e55abf3 100644 --- a/services/observability/kube-prometheus-stack/values/home.yaml +++ b/services/observability/kube-prometheus-stack/values/home.yaml @@ -34,6 +34,11 @@ kubeProxy: defaultRules: disabled: Watchdog: true + # k3s kubelet exposes no kubelet_certificate_manager_* metrics, so these two + # can never fire. Serving certificates are watched by platform-host-checks + # (PlatformK3sServingCertificateExpiry) instead. Review 2026-11-11 A11. + KubeletClientCertificateExpiration: true + KubeletServerCertificateExpiration: true rules: etcd: false kubeControllerManager: false diff --git a/services/platform-host-checks/README.md b/services/platform-host-checks/README.md new file mode 100644 index 0000000..861b943 --- /dev/null +++ b/services/platform-host-checks/README.md @@ -0,0 +1,83 @@ +# 호스트 백업과 점검 + +desktop 호스트에서 systemd 타이머로 도는 백업과 점검이다. 결과는 node-exporter textfile 수집기 +(`/var/lib/node_exporter/textfile_collector/*.prom`)로 Prometheus에 올라가고, 이 폴더의 PrometheusRule이 +경보를 낸다. 근거와 결정은 `docs/reviews/2026-11-11-k3s-maturity-review.md` A1·A11·A13·A14, 8절 결정 1·6이다. + +## 무엇을 어디에 얼마나 + +| 대상 | 방법 | 주기 | 보관 | 위치 | +| --- | --- | --- | --- | --- | +| k3s datastore + `server/token` | `sqlite3 .backup` → tar.gz + sha256 | 매일 03:30 KST | 30일 | AIStor `platform-host-backups/k3s-datastore/` (HDD sdb3) | +| Gitea (저장소·패키지·첨부·Actions·`app.ini`) | 라이브 파드에서 `gitea dump --skip-db` | 일요일 04:00 KST | 56일(8주) | AIStor `platform-host-backups/gitea/` | +| platform-postgres (Gitea·Keycloak·tech-log DB) | CloudNativePG base backup + WAL | 매일 03:00 KST | 30일 | AIStor `platform-postgres-backups/` (W1, `services/platform-postgres-backup`) | +| AIStor 데이터(Loki·Tempo) | **백업하지 않는다** | — | — | 8절 결정 6: 잃어도 되는 관측 데이터. 디스크를 잃으면 로그·추적 기록만 사라진다 | +| Prometheus·Grafana·Alertmanager PV | 백업하지 않는다 | — | — | 설정은 이 저장소가 정본이고, 지표는 다시 쌓인다 | + +- Gitea DB는 dump에 없다(`--skip-db`). Gitea를 복원하려면 CNPG 복원이 먼저다(`docs/runbooks/gitea-restore.md`). +- k3s 백업에는 `token`이 들어 있다. token과 datastore가 같이 있으면 클러스터의 모든 Secret을 풀 수 있다. + Gitea dump의 `app.ini`에도 비밀 값이 있다. 그래서 둘 다 AIStor의 전용 사용자만 읽고 쓴다. +- AIStor는 같은 desktop의 HDD다. SSD가 죽으면 남지만 desktop 자체를 잃으면 같이 잃는다. + +## off-host 사본 (준비만, 기본 꺼짐) + +위치는 소유자가 정한다(8절 결정 1). `/etc/platform-backup/aistor.env`에 두 줄을 넣으면 k3s datastore +백업이 그 공개 키로 암호화한 사본을 `rclone`으로 보낸다. + +```bash +OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc # 공개 키. 개인 키는 desktop에 두지 않는다 +OFFHOST_RCLONE_TARGET=: # rclone 설치·설정 필요 +``` + +지표 `platform_backup_offhost_copy{backup="k3s-datastore"}`가 1이 된다. Gitea dump의 off-host 사본은 아직 없다. + +## 점검 + +| 타이머 | 사용자 | 하는 일 | 지표 | +| --- | --- | --- | --- | +| `platform-k3s-cert-expiry` | donghyeon | 6443·10250 serving 인증서 notAfter | `platform_k3s_certificate_not_after_seconds` | +| `platform-live-drift-check` | donghyeon | origin/main 렌더 → `kubectl diff`(서버 dry-run) | `platform_live_drift_objects` | + +## 경보 (`prometheus-rules.yaml`) + +| 경보 | 조건 | +| --- | --- | +| `PlatformK3sDatastoreBackupTooOld` | 마지막 성공이 2일보다 오래됨 | +| `PlatformGiteaBackupTooOld` | 8일보다 오래됨 | +| `PlatformHostBackupFailed` | 마지막 실행 실패 | +| `PlatformHostBackupMetricsAbsent` | 지표가 하루 동안 없음(타이머나 textfile 수집기가 안 돎) | +| `PlatformLiveDriftDetected` | drift가 6시간 넘게 남음 | +| `PlatformLiveDriftCheckStale` | drift 검사가 2일 넘게 완료되지 않음 | +| `PlatformK3sServingCertificateExpiry` | 100일 미만 warning, 14일 미만 critical | +| `PlatformK3sCertificateCheckStale` | 인증서 점검이 2일 넘게 안 돎 | + +`scripts/validate/test-platform-host-checks-alerts.sh`가 promtool 단위 시험으로 각 경보가 울려야 할 때 울리고 +아닐 때 안 울리는지 본다. 같은 PR에서 kube-prometheus-stack의 `KubeletClientCertificateExpiration`· +`KubeletServerCertificateExpiration`을 껐다. k3s kubelet은 그 지표를 내지 않아 울릴 수 없었다. + +## 적용 (소유자, 순서대로) + +전제: node-exporter textfile 수집기(W1, `services/observability/node-exporter/values/home.yaml`의 +`--collector.textfile.directory=/host/root/var/lib/node_exporter/textfile_collector`)가 들어간 main checkout. + +```bash +cd /home/donghyeon/workspace/platform && git pull --ff-only +sudo bash scripts/bootstrap/create-host-backup-credentials.sh # bucket·정책·사용자·/etc/platform-backup/aistor.env (APPLY 입력) +bash scripts/bootstrap/install-host-checks.sh --check +sudo bash scripts/bootstrap/install-host-checks.sh --execute # 타이머 설치·켜기·첫 실행 +kubectl apply -k services/platform-host-checks # 경보 규칙 +sudo bash scripts/validate/k3s-datastore-restore-drill.sh # 첫 백업으로 복원 리허설 +sudo bash scripts/validate/gitea-backup-restore-drill.sh +``` + +`install-host-checks.sh --execute` 마지막에 `platform_backup_last_run_success{backup="k3s-datastore"} 1`과 +`{backup="gitea"} 1`이 보여야 한다. 리허설 둘은 각각 `K3S DATASTORE RESTORE DRILL PASS`, `GITEA RESTORE DRILL PASS`로 끝나야 한다. + +## 복원 리허설 + +- k3s: 백업을 내려받아 sha256을 맞춘 뒤, 운영과 같은 버전의 k3s를 **네트워크 없는** 일회용 컨테이너로 + 띄워 API가 뜨는지, Secret 수가 라이브와 비슷한지(±5), 모든 Secret이 복호화되는지 본다. + `scripts/validate/test-k3s-datastore-restore-drill.sh`는 운영 데이터 없이 같은 경로를 시험한다(새 k3s에 + 시험 Secret 셋 → 암호화돼 저장된 것을 확인 → 복원). +- Gitea: dump를 풀어 `app.ini`(INTERNAL_TOKEN·SECRET_KEY), 모든 저장소의 `git fsck`, 라이브와 저장소 수를 본다. +- 분기마다 한 번 돌린다. 성공하면 `platform_backup_restore_drill_last_success_timestamp_seconds`가 갱신된다. diff --git a/services/platform-host-checks/kustomization.yaml b/services/platform-host-checks/kustomization.yaml new file mode 100644 index 0000000..a4bdf33 --- /dev/null +++ b/services/platform-host-checks/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - prometheus-rules.yaml diff --git a/services/platform-host-checks/prometheus-rules.yaml b/services/platform-host-checks/prometheus-rules.yaml new file mode 100644 index 0000000..ab7e556 --- /dev/null +++ b/services/platform-host-checks/prometheus-rules.yaml @@ -0,0 +1,88 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-host-checks + namespace: observability + labels: + observability.hyeonworks.com/instance: home + app.kubernetes.io/part-of: platform +spec: + groups: + - name: platform-host-backups + rules: + - alert: PlatformK3sDatastoreBackupTooOld + expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}) > 2 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: k3s datastore has not been backed up for more than 2 days + description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST. + - alert: PlatformGiteaBackupTooOld + expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="gitea"}) > 8 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: Gitea has not been backed up for more than 8 days + description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST. + - alert: PlatformHostBackupFailed + expr: max by (backup) (platform_backup_last_run_success) == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "The last {{ $labels.backup }} backup run failed" + description: "Check `journalctl -u platform-{{ $labels.backup }}-backup` on desktop." + - alert: PlatformHostBackupMetricsAbsent + expr: absent(platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}) or absent(platform_backup_last_run_timestamp_seconds{backup="gitea"}) + for: 1d + labels: + severity: warning + annotations: + summary: Host backup metrics are missing + description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. + - name: platform-host-drift + rules: + - alert: PlatformLiveDriftDetected + expr: max(platform_live_drift_objects) > 0 + for: 6h + labels: + severity: warning + annotations: + summary: "{{ $value }} live objects differ from platform-core origin/main" + description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them. + - alert: PlatformLiveDriftCheckStale + expr: time() - max(platform_live_drift_last_run_timestamp_seconds) > 2 * 86400 or max(platform_live_drift_check_success) == 0 + for: 1h + labels: + severity: warning + annotations: + summary: The daily live drift check has not completed + description: Check `journalctl -u platform-live-drift-check` on desktop. + - name: platform-k3s-certificates + rules: + - alert: PlatformK3sServingCertificateExpiry + expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 100 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 100 days" + description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). + - alert: PlatformK3sServingCertificateExpiry + expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 14 * 86400 + for: 10m + labels: + severity: critical + annotations: + summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 14 days" + description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`. + - alert: PlatformK3sCertificateCheckStale + expr: time() - max(platform_k3s_certificate_check_timestamp_seconds) > 2 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: The daily k3s certificate expiry check has not run + description: Check `journalctl -u platform-k3s-cert-expiry` on desktop. From ba3dddc13966198a2b6b157fd6330f642de3ae90 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:13:02 +0900 Subject: [PATCH 3/5] feat(validate): add restore drills for the k3s and Gitea backups A backup nobody has restored is not known to work. The 2026-08 k3s recovery bundle never got its isolated restore drill. k3s-datastore-restore-drill.sh downloads the latest archive, checks its sha256 and integrity, starts the same k3s version in a network-less throwaway container on the copy, and checks the API comes up, the Secret count is close to live and every Secret decrypts with the token. gitea-backup-restore-drill.sh extracts the latest dump and checks app.ini (INTERNAL_TOKEN, SECRET_KEY), git fsck on every repository and the repository count against live Gitea. Both accept a local copy and write a drill timestamp metric. Risk: read-only for the cluster. The k3s drill runs the restored control plane with --network none, so its controllers reach nothing. Verification (desktop): - test-k3s-datastore-restore-drill.sh: a fresh k3s with three test Secrets (stored encrypted, k8s:enc:) restores with all Secrets decrypted in 7s; before folding the WAL the test saw 2 of 3 rows, which is why the real backup uses sqlite3 .backup. - gitea-backup-restore-drill.sh on a live repository dump: 29 repositories, 0 fsck failures, 29 live; the dump was deleted after. --- scripts/validate/README.md | 10 ++ .../validate/gitea-backup-restore-drill.sh | 75 ++++++++++++++ .../validate/k3s-datastore-restore-drill.sh | 99 +++++++++++++++++++ .../test-k3s-datastore-restore-drill.sh | 41 ++++++++ 4 files changed, 225 insertions(+) create mode 100644 scripts/validate/gitea-backup-restore-drill.sh create mode 100644 scripts/validate/k3s-datastore-restore-drill.sh create mode 100644 scripts/validate/test-k3s-datastore-restore-drill.sh diff --git a/scripts/validate/README.md b/scripts/validate/README.md index b30e14a..9e5ca12 100644 --- a/scripts/validate/README.md +++ b/scripts/validate/README.md @@ -596,3 +596,13 @@ PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ - `--textfile FILE`: `platform_live_drift_objects`, `platform_live_drift_check_success`, `platform_live_drift_last_run_timestamp_seconds`를 Prometheus textfile 형식으로 쓴다. - Secret 값의 차이는 보지 않는다. + +## 호스트 백업·점검 시험 + +- `test-platform-host-checks-alerts.sh`: `services/platform-host-checks/prometheus-rules.yaml`의 promtool + 단위 시험(백업 신선도, 실패, 지표 없음, drift, k3s 인증서). docker 필요, 클러스터 접근 없음. +- `k3s-cert-expiry.sh`: 6443·10250 serving 인증서 만료를 textfile 지표로 쓴다(타이머가 매일 돌림). +- `live-drift-check-main.sh`: checkout의 작업 트리가 아니라 `origin/main`으로 `live-drift-check.sh`를 돌린다. +- `k3s-datastore-restore-drill.sh`, `gitea-backup-restore-drill.sh`: 복원 리허설. 인자 없이 root로 돌리면 AIStor의 + 최신 백업을 쓰고, `--from-dir`·`--from-file`로 로컬 사본을 줄 수 있다. +- `test-k3s-datastore-restore-drill.sh`: 운영 데이터 없이 k3s 복원 리허설 경로를 시험한다(일회용 k3s, 시험 Secret 셋). diff --git a/scripts/validate/gitea-backup-restore-drill.sh b/scripts/validate/gitea-backup-restore-drill.sh new file mode 100644 index 0000000..0dfc9c4 --- /dev/null +++ b/scripts/validate/gitea-backup-restore-drill.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Restore drill for the Gitea backup (file level). +# +# sudo bash scripts/validate/gitea-backup-restore-drill.sh # latest dump in AIStor +# bash scripts/validate/gitea-backup-restore-drill.sh --from-file DUMP.tar.gz +# +# Extracts the dump into a throwaway directory and checks what a restore needs: +# app.ini is present, every repository passes `git fsck`, and the repository +# count matches the live Gitea. The database is restored from the CloudNativePG +# backup instead (services/platform-postgres-backup); the full restore order is +# in docs/runbooks/gitea-restore.md. Nothing is written to the cluster. +set -Eeuo pipefail +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +# shellcheck source=../lib/host-backup.sh +source "$ROOT/scripts/lib/host-backup.sh" +readonly NAME=gitea_restore_drill + +from_file="" +case "${1:-}" in + --from-file) from_file="${2:?--from-file needs a path}" ;; + "") ;; + *) sed -n '2,11p' "$0" >&2; exit 2 ;; +esac +if [[ "$(id -u)" -eq 0 ]]; then + kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; } +fi + +base="$HB_WORK_ROOT"; [[ -w "$base" ]] || base="$(mktemp -d)" +work="$(mktemp -d "$base/gitea-drill.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT + +if [[ -z "$from_file" ]]; then + [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" + hb_load_aistor_env + latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/gitea/" | awk '{print $NF}' | grep -E '^gitea-dump-.*\.tar\.gz$' | sort | tail -1)" + [[ -n "$latest" ]] || hb_fail "no Gitea dump found" + hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/gitea/$latest" "backup/$AISTOR_BUCKET/gitea/$latest.sha256" /work/ >/dev/null + [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" + from_file="$work/$latest" + hb_log "drill on $latest" +fi + +mkdir "$work/restore" +tar -C "$work/restore" -xzf "$from_file" +[[ -s "$work/restore/app.ini" ]] || hb_fail "app.ini is missing from the dump" +for key in INTERNAL_TOKEN SECRET_KEY; do + grep -q "^${key}[[:space:]]*=" "$work/restore/app.ini" || hb_fail "app.ini has no ${key} (needed to restore)" +done + +total=0 broken=0 +while IFS= read -r -d '' repo; do + total=$((total + 1)) + if ! git -c safe.directory='*' --git-dir="$repo" fsck --no-dangling --no-progress >/dev/null 2>"$work/fsck.err"; then + broken=$((broken + 1)) + hb_log "fsck failed: ${repo#"$work/restore/"}: $(head -1 "$work/fsck.err")" + fi +done < <(find "$work/restore/repos" -mindepth 2 -maxdepth 2 -type d -name '*.git' -print0) +hb_log "repositories in dump: $total, fsck failures: $broken" +(( total > 0 )) || hb_fail "the dump has no repositories" +(( broken == 0 )) || hb_fail "$broken repositories failed git fsck" + +live="$(kubectl -n gitea exec deploy/gitea -c gitea -- sh -c 'find /data/git/gitea-repositories -mindepth 2 -maxdepth 2 -name "*.git" -type d | wc -l' 2>/dev/null || echo "")" +if [[ -n "$live" ]]; then + hb_log "live Gitea repositories: $live" + (( total >= live - 2 )) || hb_fail "the dump has $total repositories, live Gitea has $live" +fi + +hb_write_metrics "$NAME" < 0 )); do + case "$1" in + --from-dir) from_dir="${2:?}"; shift 2 ;; + --expect-secrets) expect_secrets="${2:?}"; shift 2 ;; + *) sed -n '2,13p' "$0" >&2; exit 2 ;; + esac +done + +base="${HB_WORK_ROOT}" +[[ -w "$base" ]] || base="$(mktemp -d)" +work="$(mktemp -d "$base/k3s-drill.XXXXXX")" +container="k3s-restore-drill-$$" +cleanup() { + docker rm -f "$container" >/dev/null 2>&1 || true + # The drill container writes as root; remove its files with the same image. + docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT + +if [[ -z "$from_dir" ]]; then + [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" + hb_load_aistor_env + latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)" + [[ -n "$latest" ]] || hb_fail "no k3s datastore backup found" + hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null + [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" + tar -C "$work" -xzf "$work/$latest" + from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)" + hb_log "drill on $latest" + if [[ -z "$expect_secrets" ]]; then + expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)" + hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)" + fi +fi +[[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token" + +install -d -m 0700 "$work/server/db" +cp "$from_dir/state.db" "$work/server/db/state.db" +cp "$from_dir/token" "$work/server/token" +[[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed" + +started="$(date +%s)" +docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \ + -v "$work/server:/var/lib/rancher/k3s/server" \ + "$K3S_IMAGE" server --disable-agent --secrets-encryption \ + --token-file /var/lib/rancher/k3s/server/token \ + --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ + --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null + +ready=0 +for _ in $(seq 1 60); do + if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi + sleep 5 +done +if (( ready == 0 )); then + docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2 + hb_fail "restored k3s did not become ready in 5 minutes" +fi +namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)" +secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)" +# -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero. +decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')" +hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s" +if [[ -n "$expect_secrets" ]]; then + diff=$(( secrets - expect_secrets )) + (( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets" +fi +(( decrypted > 0 )) || hb_fail "no Secret carried data after restore" + +hb_write_metrics "$NAME" <&2; exit 1; } +work="$(mktemp -d)" +seed="w6-k3s-drill-seed-$$" +cleanup() { + docker rm -f "$seed" >/dev/null 2>&1 || true + docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT +mkdir -p "$work/server" "$work/copy" + +docker run -d --name "$seed" --network none --tmpfs /run --tmpfs /var/run \ + -v "$work/server:/var/lib/rancher/k3s/server" "$K3S_IMAGE" server --disable-agent --secrets-encryption \ + --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ + --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null +for _ in $(seq 1 60); do docker exec "$seed" kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 5; done +for i in 1 2 3; do docker exec "$seed" kubectl create secret generic "drill-test-$i" --from-literal=value="test-$i" >/dev/null; done +expected="$(docker exec "$seed" kubectl get secrets -A --no-headers | wc -l)" +docker stop -t 30 "$seed" >/dev/null +docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c \ + "cp /w/server/db/state.db* /w/server/token /w/copy/ && chown $(id -u):$(id -g) /w/copy/*" +# Fold the WAL into state.db, as `sqlite3 .backup` does in the real backup. +sqlite3 "$work/copy/state.db" 'PRAGMA wal_checkpoint(TRUNCATE);' >/dev/null +rm -f "$work/copy/state.db-wal" "$work/copy/state.db-shm" +# The test Secrets must be stored encrypted (k8s:enc:...), so the drill really +# proves that the token unlocks the encryption key. +encrypted="$(sqlite3 "$work/copy/state.db" "SELECT COUNT(*) FROM kine WHERE name LIKE '/registry/secrets/default/drill-test-%' AND CAST(value AS TEXT) LIKE 'k8s:enc:%';")" +echo "seed: $expected Secrets, encrypted test Secret rows=$encrypted" +(( encrypted >= 3 )) || { echo "test Secrets are not encrypted at rest in the seed datastore" >&2; exit 1; } + +bash "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh" --from-dir "$work/copy" --expect-secrets "$expected" +echo "K3S DATASTORE RESTORE DRILL TEST PASS" From 9e9ea94461bc7a2560c5f755e27fe69d679feb93 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:14:45 +0900 Subject: [PATCH 4/5] feat(backup): keep k3s datastore archives outside AIStor on the HDD AIStor runs inside the k3s cluster that the datastore archive restores. After losing the whole cluster the AIStor copy cannot be read until k3s is running again, which is what the archive is for. Also keep the seven newest archives in /srv/k3s/aistor/host-backups/k3s-datastore (root 0700) on the same HDD filesystem (sdb3). The directory is next to AIStor's drive directory data/, so AIStor does not list it as a bucket, and it is readable by mounting sdb3 alone. A metric reports how many local copies exist. Rejected: the NTFS archive partition (sdb2). It has no Unix permissions and Windows on the same machine can read it; the archive holds the server token. Risk: the files live inside the AIStor PersistentVolume root, which is mounted into the MinIO pod (uid 1000); mode 0700 root keeps the pod from reading them. Verification: bash -n; /srv/k3s/aistor contains only data/ (2026-11-12). --- scripts/backup/k3s-datastore-backup.sh | 13 +++++++++++++ services/platform-host-checks/README.md | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/backup/k3s-datastore-backup.sh b/scripts/backup/k3s-datastore-backup.sh index 3f5433a..f5f5b3e 100644 --- a/scripts/backup/k3s-datastore-backup.sh +++ b/scripts/backup/k3s-datastore-backup.sh @@ -57,6 +57,16 @@ size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")" hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null +# Local copy outside AIStor on the same HDD (sdb3). AIStor runs inside the +# cluster this archive restores, so after losing the cluster the AIStor copy +# cannot be read until k3s is back; this copy can. The directory sits next to +# AIStor's drive directory (data/), so AIStor does not see it as a bucket. +readonly LOCAL_DIR=/srv/k3s/aistor/host-backups/k3s-datastore +install -d -m 0700 -o root -g root "$(dirname "$LOCAL_DIR")" "$LOCAL_DIR" +install -m 0600 "$work/k3s-datastore-$ts.tar.gz" "$work/k3s-datastore-$ts.tar.gz.sha256" "$LOCAL_DIR/" +ls -1t "$LOCAL_DIR"/k3s-datastore-*.tar.gz | tail -n +8 | while read -r old; do rm -f -- "$old" "$old.sha256"; done +local_copies="$(ls -1 "$LOCAL_DIR"/k3s-datastore-*.tar.gz | wc -l)" + offhost=0 if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed" @@ -79,6 +89,9 @@ platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s) # TYPE platform_backup_last_size_bytes gauge platform_backup_last_size_bytes{backup="k3s-datastore"} $size platform_backup_last_duration_seconds{backup="k3s-datastore"} $(( $(date +%s) - started )) +# HELP platform_backup_local_copies Archives kept outside AIStor on the HDD. +# TYPE platform_backup_local_copies gauge +platform_backup_local_copies{backup="k3s-datastore"} $local_copies # HELP platform_backup_offhost_copy 1 when the last run also wrote the encrypted off-host copy. # TYPE platform_backup_offhost_copy gauge platform_backup_offhost_copy{backup="k3s-datastore"} $offhost diff --git a/services/platform-host-checks/README.md b/services/platform-host-checks/README.md index 861b943..524a986 100644 --- a/services/platform-host-checks/README.md +++ b/services/platform-host-checks/README.md @@ -8,7 +8,7 @@ desktop 호스트에서 systemd 타이머로 도는 백업과 점검이다. 결 | 대상 | 방법 | 주기 | 보관 | 위치 | | --- | --- | --- | --- | --- | -| k3s datastore + `server/token` | `sqlite3 .backup` → tar.gz + sha256 | 매일 03:30 KST | 30일 | AIStor `platform-host-backups/k3s-datastore/` (HDD sdb3) | +| k3s datastore + `server/token` | `sqlite3 .backup` → tar.gz + sha256 | 매일 03:30 KST | AIStor 30일 + 로컬 최근 7개 | AIStor `platform-host-backups/k3s-datastore/`와 `/srv/k3s/aistor/host-backups/k3s-datastore/`(root 0700). 둘 다 HDD sdb3 | | Gitea (저장소·패키지·첨부·Actions·`app.ini`) | 라이브 파드에서 `gitea dump --skip-db` | 일요일 04:00 KST | 56일(8주) | AIStor `platform-host-backups/gitea/` | | platform-postgres (Gitea·Keycloak·tech-log DB) | CloudNativePG base backup + WAL | 매일 03:00 KST | 30일 | AIStor `platform-postgres-backups/` (W1, `services/platform-postgres-backup`) | | AIStor 데이터(Loki·Tempo) | **백업하지 않는다** | — | — | 8절 결정 6: 잃어도 되는 관측 데이터. 디스크를 잃으면 로그·추적 기록만 사라진다 | @@ -18,6 +18,9 @@ desktop 호스트에서 systemd 타이머로 도는 백업과 점검이다. 결 - k3s 백업에는 `token`이 들어 있다. token과 datastore가 같이 있으면 클러스터의 모든 Secret을 풀 수 있다. Gitea dump의 `app.ini`에도 비밀 값이 있다. 그래서 둘 다 AIStor의 전용 사용자만 읽고 쓴다. - AIStor는 같은 desktop의 HDD다. SSD가 죽으면 남지만 desktop 자체를 잃으면 같이 잃는다. +- k3s 백업을 AIStor 밖에도 두는 이유: AIStor는 바로 그 k3s 위에서 돈다. 클러스터를 통째로 잃으면 AIStor를 + 다시 띄우기 전에는 AIStor 안의 사본을 꺼낼 수 없다. 로컬 사본은 sdb3를 mount하기만 하면 읽힌다 + (`docs/runbooks/k3s-datastore-restore.md`). 폴더는 AIStor 드라이브 폴더(`data/`) 옆이라 bucket으로 보이지 않는다. ## off-host 사본 (준비만, 기본 꺼짐) From cb78ccdb0bfae27fdb6727e127d5e64dff963512 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 01:38:24 +0900 Subject: [PATCH 5/5] feat(backup): store the Gitea version next to each dump A Gitea dump must be restored with the Gitea version that wrote it, and a restore checks the repository count against the dump. Neither was recorded outside the archive, so the restore runbook had nothing to compare with. Upload a small .meta file with each dump: creation time, the Deployment image, the gitea --version output and the repository count. docs/runbooks/gitea-restore.md reads it. Risk: none for the backup itself; the file holds no secret. Verification: bash -n; kubectl -n gitea exec deploy/gitea -c gitea -- gitea --version prints "gitea version 1.27.0 built with ..." on the desktop, so the third field is the version. --- scripts/backup/gitea-backup.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/backup/gitea-backup.sh b/scripts/backup/gitea-backup.sh index c0c96f5..32de28b 100644 --- a/scripts/backup/gitea-backup.sh +++ b/scripts/backup/gitea-backup.sh @@ -41,8 +41,15 @@ repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)" (( repos > 0 )) || hb_fail "the dump has no repositories" sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256" size="$(stat -c %s "$work/$archive")" +# What a restore must match: restore with the same Gitea version (docs/runbooks/gitea-restore.md). +{ + echo "created=$ts" + echo "gitea_image=$(kubectl -n gitea get deploy gitea -o jsonpath='{.spec.template.spec.containers[0].image}')" + echo "gitea_version=$(kubectl -n gitea exec deploy/gitea -c gitea -- gitea --version 2>/dev/null | awk '{print $3}')" + echo "repositories=$repos" +} >"$work/$archive.meta" -hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "backup/$AISTOR_BUCKET/gitea/" >/dev/null +hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "/work/$archive.meta" "backup/$AISTOR_BUCKET/gitea/" >/dev/null hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null trap - ERR