diff --git a/infrastructure/host/backup/platform-gitea-backup.service b/infrastructure/host/backup/platform-gitea-backup.service new file mode 100644 index 0000000..3cbc4e5 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.service @@ -0,0 +1,12 @@ +[Unit] +Description=Back up Gitea (gitea dump --skip-db) to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/gitea-backup.sh +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=1h diff --git a/infrastructure/host/backup/platform-gitea-backup.timer b/infrastructure/host/backup/platform-gitea-backup.timer new file mode 100644 index 0000000..add1a49 --- /dev/null +++ b/infrastructure/host/backup/platform-gitea-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly Gitea backup + +[Timer] +OnCalendar=Sun *-*-* 04:00:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-k3s-cert-expiry.service b/infrastructure/host/backup/platform-k3s-cert-expiry.service new file mode 100644 index 0000000..4215754 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-cert-expiry.service @@ -0,0 +1,8 @@ +[Unit] +Description=Record the k3s API server and kubelet serving certificate expiry +After=k3s.service + +[Service] +Type=oneshot +User=donghyeon +ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/k3s-cert-expiry.sh diff --git a/infrastructure/host/backup/platform-k3s-cert-expiry.timer b/infrastructure/host/backup/platform-k3s-cert-expiry.timer new file mode 100644 index 0000000..c885829 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-cert-expiry.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily k3s certificate expiry check + +[Timer] +OnCalendar=*-*-* 05:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.service b/infrastructure/host/backup/platform-k3s-datastore-backup.service new file mode 100644 index 0000000..cdd58bd --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=Back up the k3s datastore and token to AIStor (platform-host-backups) +Documentation=file:///usr/local/lib/platform-backup/README.md +After=k3s.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/k3s-datastore-backup.sh +Nice=10 +IOSchedulingClass=idle diff --git a/infrastructure/host/backup/platform-k3s-datastore-backup.timer b/infrastructure/host/backup/platform-k3s-datastore-backup.timer new file mode 100644 index 0000000..4fd6337 --- /dev/null +++ b/infrastructure/host/backup/platform-k3s-datastore-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily k3s datastore backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/host/backup/platform-live-drift-check.service b/infrastructure/host/backup/platform-live-drift-check.service new file mode 100644 index 0000000..1838a38 --- /dev/null +++ b/infrastructure/host/backup/platform-live-drift-check.service @@ -0,0 +1,16 @@ +[Unit] +Description=Compare the cluster with platform-core origin/main (kubectl diff, server dry-run) +After=k3s.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=donghyeon +Environment=PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm +Environment=PLATFORM_CHECKOUT=/home/donghyeon/workspace/platform +ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/live-drift-check-main.sh +# Exit 1 means "drift found"; the metric and its alert report that. Only 2 (the +# check itself failed) marks the unit failed. +SuccessExitStatus=1 +Nice=10 +TimeoutStartSec=30min diff --git a/infrastructure/host/backup/platform-live-drift-check.timer b/infrastructure/host/backup/platform-live-drift-check.timer new file mode 100644 index 0000000..1267b5a --- /dev/null +++ b/infrastructure/host/backup/platform-live-drift-check.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily live drift check + +[Timer] +OnCalendar=*-*-* 05:15:00 +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/backup/gitea-backup.sh b/scripts/backup/gitea-backup.sh new file mode 100644 index 0000000..32de28b --- /dev/null +++ b/scripts/backup/gitea-backup.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Back up Gitea (repositories, packages, attachments, Actions data, app.ini) to +# AIStor (HDD sdb3). Root only. Run by platform-gitea-backup.timer. +# +# `gitea dump --skip-db` runs inside the live Gitea container and streams a +# tar.gz to stdout (logs go to stderr). The database is backed up separately by +# CloudNativePG (services/platform-postgres-backup). The dump contains app.ini, +# whose secrets are needed to restore, so the archive is sensitive. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly NAME=gitea_backup +readonly KCFG=/etc/rancher/k3s/k3s.yaml +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +hb_load_aistor_env +kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig "$KCFG" "$@"; } + +started="$(date +%s)" +previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)" +on_error() { + hb_write_metrics "$NAME" <"$work/$archive" 2>"$work/dump.log" +tar -tzf "$work/$archive" >"$work/list.txt" +grep -qx 'app.ini' "$work/list.txt" || hb_fail "the dump has no app.ini" +repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)" +(( repos > 0 )) || hb_fail "the dump has no repositories" +sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256" +size="$(stat -c %s "$work/$archive")" +# What a restore must match: restore with the same Gitea version (docs/runbooks/gitea-restore.md). +{ + echo "created=$ts" + echo "gitea_image=$(kubectl -n gitea get deploy gitea -o jsonpath='{.spec.template.spec.containers[0].image}')" + echo "gitea_version=$(kubectl -n gitea exec deploy/gitea -c gitea -- gitea --version 2>/dev/null | awk '{print $3}')" + echo "repositories=$repos" +} >"$work/$archive.meta" + +hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "/work/$archive.meta" "backup/$AISTOR_BUCKET/gitea/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null + +trap - ERR +hb_write_metrics "$NAME" < +# The archive is then encrypted to that public key and copied with rclone. +set -Eeuo pipefail +umask 077 +# shellcheck source=../lib/host-backup.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh" + +readonly SERVER=/var/lib/rancher/k3s/server +readonly NAME=k3s_datastore_backup +[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root" +command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required" +hb_load_aistor_env + +started="$(date +%s)" +on_error() { + hb_write_metrics "$NAME" </dev/null || echo 0)" +trap on_error ERR + +work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT +ts="$(date -u +%Y%m%dT%H%M%SZ)" +stage="$work/k3s-datastore-$ts" +mkdir -m 0700 "$stage" + +sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'" +check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')" +[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check" +install -m 0600 "$SERVER/token" "$stage/token" +{ + echo "created=$ts" + echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)" + echo "host=$(hostname)" + echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')" +} >"$stage/META" +tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts" +sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256" +size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")" + +hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null +hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null + +# Local copy outside AIStor on the same HDD (sdb3). AIStor runs inside the +# cluster this archive restores, so after losing the cluster the AIStor copy +# cannot be read until k3s is back; this copy can. The directory sits next to +# AIStor's drive directory (data/), so AIStor does not see it as a bucket. +readonly LOCAL_DIR=/srv/k3s/aistor/host-backups/k3s-datastore +install -d -m 0700 -o root -g root "$(dirname "$LOCAL_DIR")" "$LOCAL_DIR" +install -m 0600 "$work/k3s-datastore-$ts.tar.gz" "$work/k3s-datastore-$ts.tar.gz.sha256" "$LOCAL_DIR/" +ls -1t "$LOCAL_DIR"/k3s-datastore-*.tar.gz | tail -n +8 | while read -r old; do rm -f -- "$old" "$old.sha256"; done +local_copies="$(ls -1 "$LOCAL_DIR"/k3s-datastore-*.tar.gz | wc -l)" + +offhost=0 +if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then + command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed" + gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \ + --output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz" + rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg" + offhost=1 +fi + +trap - ERR +hb_write_metrics "$NAME" </dev/null || hb_fail "missing command: $c"; done +jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON" +[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it" + +ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" +[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found" +endpoint="http://${ip}:80" + +work="$(mktemp -d)" +trap 'rm -rf -- "$work"' EXIT +kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env" +grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format" +printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY" +openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY" +cp "$POLICY_FILE" "$work/policy.json" + +printf '%s\n' \ + "Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \ + "Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \ + "User: a new generated access key attached to $POLICY" \ + "File: $HOST_BACKUP_ENV (root, 0600)" \ + 'No key value will be printed.' +printf 'Type APPLY to create them: ' +IFS= read -r confirmation +[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY" + +# Every mc call is silenced so no key reaches the terminal. +docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \ + --entrypoint /bin/sh "$MC_IMAGE" -ec ' +. /run/b/root.env +ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)" +mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null +mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null +mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null +mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null +mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null +mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null +mc admin user add root "$ak" "$sk" >/dev/null +mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null +mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null +probe=".bootstrap-probe-$(date +%s)" +printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null +[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ] +mc rm "backup/'"$BUCKET"'/$probe" >/dev/null +if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi +echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written" + +install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")" +{ + echo "AISTOR_BUCKET=$BUCKET" + echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")" + echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")" + echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):" + echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc" + echo "# OFFHOST_RCLONE_TARGET=remote:path" +} >"$HOST_BACKUP_ENV" +chmod 0600 "$HOST_BACKUP_ENV" +printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV." diff --git a/scripts/bootstrap/install-host-checks.sh b/scripts/bootstrap/install-host-checks.sh new file mode 100644 index 0000000..21d13ee --- /dev/null +++ b/scripts/bootstrap/install-host-checks.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Install the host backup and check timers (infrastructure/host/backup). +# +# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing +# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs +# +# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root +# timers never execute files the desktop user can edit. +set -Eeuo pipefail +umask 022 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly LIB=/usr/local/lib/platform-backup +readonly UNIT_DIR=/etc/systemd/system +readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector +readonly WORK_ROOT=/var/lib/platform-backup/work +readonly USER_NAME=donghyeon +readonly -a SCRIPTS=( + scripts/lib/host-backup.sh:lib/host-backup.sh + scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh + scripts/backup/gitea-backup.sh:backup/gitea-backup.sh + scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh + scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh + services/platform-host-checks/README.md:README.md +) +readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry) + +fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +mode="${1:-}" +[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; } + +echo "== check" +for pair in "${SCRIPTS[@]}"; do + src="${pair%%:*}" + [[ -f "$ROOT/$src" ]] || fail "missing $src" + [[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src" +done +for t in "${TIMERS[@]}"; do + for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done +done +if command -v systemd-analyze >/dev/null; then + verify_dir="$(mktemp -d)" + trap 'rm -rf -- "$verify_dir"' EXIT + cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/" + # ExecStart paths do not exist before installation; only syntax and keys are checked. + out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)" + [[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; } + echo " units: systemd-analyze verify clean" +fi +echo "== plan" +for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done +for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done +echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)" +echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check" +[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; } + +[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)" +[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first" +echo "== install" +for pair in "${SCRIPTS[@]}"; do + install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}" +done +install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR" +install -d -o root -g root -m 0700 "$WORK_ROOT" +for t in "${TIMERS[@]}"; do + install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/" +done +systemctl daemon-reload +for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done + +echo "== first runs" +for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do + if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi +done +echo "== metrics" +grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet" +systemctl list-timers 'platform-*' --no-pager +echo "HOST CHECKS INSTALLED" diff --git a/scripts/lib/host-backup.sh b/scripts/lib/host-backup.sh new file mode 100644 index 0000000..33954fb --- /dev/null +++ b/scripts/lib/host-backup.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Shared helpers for the host backup and host check scripts. Source only. +# shellcheck shell=bash + +readonly HOST_BACKUP_ENV=/etc/platform-backup/aistor.env +readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector +# Backup staging (root 0700, same SSD). Not /var/tmp: dockerd must see the path. +readonly HB_WORK_ROOT=/var/lib/platform-backup/work +# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh. +readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b" + +hb_log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; } +hb_fail() { hb_log "FAIL: $*" >&2; exit 1; } + +# Write Prometheus textfile metrics atomically: hb_write_metrics NAME <<'PROM' ... PROM +hb_write_metrics() { + local name="$1" tmp + [[ -d "$TEXTFILE_DIR" ]] || { hb_log "textfile dir $TEXTFILE_DIR missing; metrics not written"; cat >/dev/null; return 0; } + tmp="$(mktemp "$TEXTFILE_DIR/.${name}.XXXXXX")" + cat >"$tmp" + chmod 0644 "$tmp" + mv -f -- "$tmp" "$TEXTFILE_DIR/${name}.prom" +} + +# Load the AIStor endpoint and the backup user's keys (root-only file). +hb_load_aistor_env() { + [[ -f "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV is missing; run scripts/bootstrap/create-host-backup-credentials.sh" + [[ "$(stat -c '%u %a' "$HOST_BACKUP_ENV")" == "0 600" ]] || hb_fail "$HOST_BACKUP_ENV must be owned by root with mode 0600" + # shellcheck disable=SC1090 + . "$HOST_BACKUP_ENV" + : "${AISTOR_BUCKET:?}" "${AISTOR_ACCESS_KEY:?}" "${AISTOR_SECRET_KEY:?}" + # The host cannot resolve cluster DNS, so use the Service ClusterIP (port 80 -> 9000). + if [[ -z "${AISTOR_ENDPOINT:-}" ]]; then + local ip + ip="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml -n object-storage \ + get service minio -o jsonpath='{.spec.clusterIP}')" + [[ -n "$ip" ]] || hb_fail "cannot find the object-storage/minio Service" + AISTOR_ENDPOINT="http://${ip}:80" + fi +} + +# Run the pinned mc client on the host network with the backup user's keys. +# The keys go through a 0600 env file, never the command line (ps would show it). +# Usage: hb_mc +hb_mc() { + local work="$1"; shift + local envfile="$work/.mc.env" + if [[ ! -f "$envfile" ]]; then + ( umask 077; printf 'MC_HOST_backup=%s\nMC_CONFIG_DIR=/tmp/mc\n' \ + "${AISTOR_ENDPOINT/:\/\//://${AISTOR_ACCESS_KEY}:${AISTOR_SECRET_KEY}@}" >"$envfile" ) + fi + docker run --rm --network host --user "$(id -u):$(id -g)" \ + --env-file "$envfile" -v "$work:/work" --tmpfs /tmp \ + "$MC_IMAGE" "$@" +} diff --git a/scripts/validate/README.md b/scripts/validate/README.md index b30e14a..9e5ca12 100644 --- a/scripts/validate/README.md +++ b/scripts/validate/README.md @@ -596,3 +596,13 @@ PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ - `--textfile FILE`: `platform_live_drift_objects`, `platform_live_drift_check_success`, `platform_live_drift_last_run_timestamp_seconds`를 Prometheus textfile 형식으로 쓴다. - Secret 값의 차이는 보지 않는다. + +## 호스트 백업·점검 시험 + +- `test-platform-host-checks-alerts.sh`: `services/platform-host-checks/prometheus-rules.yaml`의 promtool + 단위 시험(백업 신선도, 실패, 지표 없음, drift, k3s 인증서). docker 필요, 클러스터 접근 없음. +- `k3s-cert-expiry.sh`: 6443·10250 serving 인증서 만료를 textfile 지표로 쓴다(타이머가 매일 돌림). +- `live-drift-check-main.sh`: checkout의 작업 트리가 아니라 `origin/main`으로 `live-drift-check.sh`를 돌린다. +- `k3s-datastore-restore-drill.sh`, `gitea-backup-restore-drill.sh`: 복원 리허설. 인자 없이 root로 돌리면 AIStor의 + 최신 백업을 쓰고, `--from-dir`·`--from-file`로 로컬 사본을 줄 수 있다. +- `test-k3s-datastore-restore-drill.sh`: 운영 데이터 없이 k3s 복원 리허설 경로를 시험한다(일회용 k3s, 시험 Secret 셋). diff --git a/scripts/validate/gitea-backup-restore-drill.sh b/scripts/validate/gitea-backup-restore-drill.sh new file mode 100644 index 0000000..0dfc9c4 --- /dev/null +++ b/scripts/validate/gitea-backup-restore-drill.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Restore drill for the Gitea backup (file level). +# +# sudo bash scripts/validate/gitea-backup-restore-drill.sh # latest dump in AIStor +# bash scripts/validate/gitea-backup-restore-drill.sh --from-file DUMP.tar.gz +# +# Extracts the dump into a throwaway directory and checks what a restore needs: +# app.ini is present, every repository passes `git fsck`, and the repository +# count matches the live Gitea. The database is restored from the CloudNativePG +# backup instead (services/platform-postgres-backup); the full restore order is +# in docs/runbooks/gitea-restore.md. Nothing is written to the cluster. +set -Eeuo pipefail +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +# shellcheck source=../lib/host-backup.sh +source "$ROOT/scripts/lib/host-backup.sh" +readonly NAME=gitea_restore_drill + +from_file="" +case "${1:-}" in + --from-file) from_file="${2:?--from-file needs a path}" ;; + "") ;; + *) sed -n '2,11p' "$0" >&2; exit 2 ;; +esac +if [[ "$(id -u)" -eq 0 ]]; then + kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; } +fi + +base="$HB_WORK_ROOT"; [[ -w "$base" ]] || base="$(mktemp -d)" +work="$(mktemp -d "$base/gitea-drill.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT + +if [[ -z "$from_file" ]]; then + [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" + hb_load_aistor_env + latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/gitea/" | awk '{print $NF}' | grep -E '^gitea-dump-.*\.tar\.gz$' | sort | tail -1)" + [[ -n "$latest" ]] || hb_fail "no Gitea dump found" + hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/gitea/$latest" "backup/$AISTOR_BUCKET/gitea/$latest.sha256" /work/ >/dev/null + [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" + from_file="$work/$latest" + hb_log "drill on $latest" +fi + +mkdir "$work/restore" +tar -C "$work/restore" -xzf "$from_file" +[[ -s "$work/restore/app.ini" ]] || hb_fail "app.ini is missing from the dump" +for key in INTERNAL_TOKEN SECRET_KEY; do + grep -q "^${key}[[:space:]]*=" "$work/restore/app.ini" || hb_fail "app.ini has no ${key} (needed to restore)" +done + +total=0 broken=0 +while IFS= read -r -d '' repo; do + total=$((total + 1)) + if ! git -c safe.directory='*' --git-dir="$repo" fsck --no-dangling --no-progress >/dev/null 2>"$work/fsck.err"; then + broken=$((broken + 1)) + hb_log "fsck failed: ${repo#"$work/restore/"}: $(head -1 "$work/fsck.err")" + fi +done < <(find "$work/restore/repos" -mindepth 2 -maxdepth 2 -type d -name '*.git' -print0) +hb_log "repositories in dump: $total, fsck failures: $broken" +(( total > 0 )) || hb_fail "the dump has no repositories" +(( broken == 0 )) || hb_fail "$broken repositories failed git fsck" + +live="$(kubectl -n gitea exec deploy/gitea -c gitea -- sh -c 'find /data/git/gitea-repositories -mindepth 2 -maxdepth 2 -name "*.git" -type d | wc -l' 2>/dev/null || echo "")" +if [[ -n "$live" ]]; then + hb_log "live Gitea repositories: $live" + (( total >= live - 2 )) || hb_fail "the dump has $total repositories, live Gitea has $live" +fi + +hb_write_metrics "$NAME" </dev/null | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//')" || true + [[ -n "$end" ]] || hb_fail "could not read the $name certificate on 127.0.0.1:$port" + lines+="platform_k3s_certificate_not_after_seconds{certificate=\"$name\"} $(date -d "$end" +%s)"$'\n' + hb_log "$name certificate notAfter $end" +done +hb_write_metrics platform_k3s_certificates < 0 )); do + case "$1" in + --from-dir) from_dir="${2:?}"; shift 2 ;; + --expect-secrets) expect_secrets="${2:?}"; shift 2 ;; + *) sed -n '2,13p' "$0" >&2; exit 2 ;; + esac +done + +base="${HB_WORK_ROOT}" +[[ -w "$base" ]] || base="$(mktemp -d)" +work="$(mktemp -d "$base/k3s-drill.XXXXXX")" +container="k3s-restore-drill-$$" +cleanup() { + docker rm -f "$container" >/dev/null 2>&1 || true + # The drill container writes as root; remove its files with the same image. + docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT + +if [[ -z "$from_dir" ]]; then + [[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials" + hb_load_aistor_env + latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)" + [[ -n "$latest" ]] || hb_fail "no k3s datastore backup found" + hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null + [[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest" + tar -C "$work" -xzf "$work/$latest" + from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)" + hb_log "drill on $latest" + if [[ -z "$expect_secrets" ]]; then + expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)" + hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)" + fi +fi +[[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token" + +install -d -m 0700 "$work/server/db" +cp "$from_dir/state.db" "$work/server/db/state.db" +cp "$from_dir/token" "$work/server/token" +[[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed" + +started="$(date +%s)" +docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \ + -v "$work/server:/var/lib/rancher/k3s/server" \ + "$K3S_IMAGE" server --disable-agent --secrets-encryption \ + --token-file /var/lib/rancher/k3s/server/token \ + --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ + --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null + +ready=0 +for _ in $(seq 1 60); do + if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi + sleep 5 +done +if (( ready == 0 )); then + docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2 + hb_fail "restored k3s did not become ready in 5 minutes" +fi +namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)" +secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)" +# -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero. +decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')" +hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s" +if [[ -n "$expect_secrets" ]]; then + diff=$(( secrets - expect_secrets )) + (( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets" +fi +(( decrypted > 0 )) || hb_fail "no Secret carried data after restore" + +hb_write_metrics "$NAME" <&2; exit 1; } +work="$(mktemp -d)" +seed="w6-k3s-drill-seed-$$" +cleanup() { + docker rm -f "$seed" >/dev/null 2>&1 || true + docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT +mkdir -p "$work/server" "$work/copy" + +docker run -d --name "$seed" --network none --tmpfs /run --tmpfs /var/run \ + -v "$work/server:/var/lib/rancher/k3s/server" "$K3S_IMAGE" server --disable-agent --secrets-encryption \ + --bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \ + --disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null +for _ in $(seq 1 60); do docker exec "$seed" kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 5; done +for i in 1 2 3; do docker exec "$seed" kubectl create secret generic "drill-test-$i" --from-literal=value="test-$i" >/dev/null; done +expected="$(docker exec "$seed" kubectl get secrets -A --no-headers | wc -l)" +docker stop -t 30 "$seed" >/dev/null +docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c \ + "cp /w/server/db/state.db* /w/server/token /w/copy/ && chown $(id -u):$(id -g) /w/copy/*" +# Fold the WAL into state.db, as `sqlite3 .backup` does in the real backup. +sqlite3 "$work/copy/state.db" 'PRAGMA wal_checkpoint(TRUNCATE);' >/dev/null +rm -f "$work/copy/state.db-wal" "$work/copy/state.db-shm" +# The test Secrets must be stored encrypted (k8s:enc:...), so the drill really +# proves that the token unlocks the encryption key. +encrypted="$(sqlite3 "$work/copy/state.db" "SELECT COUNT(*) FROM kine WHERE name LIKE '/registry/secrets/default/drill-test-%' AND CAST(value AS TEXT) LIKE 'k8s:enc:%';")" +echo "seed: $expected Secrets, encrypted test Secret rows=$encrypted" +(( encrypted >= 3 )) || { echo "test Secrets are not encrypted at rest in the seed datastore" >&2; exit 1; } + +bash "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh" --from-dir "$work/copy" --expect-secrets "$expected" +echo "K3S DATASTORE RESTORE DRILL TEST PASS" diff --git a/scripts/validate/test-platform-host-checks-alerts.sh b/scripts/validate/test-platform-host-checks-alerts.sh new file mode 100644 index 0000000..cecc215 --- /dev/null +++ b/scripts/validate/test-platform-host-checks-alerts.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml. +# Needs docker and python3 (PyYAML). No cluster access. +set -Eeuo pipefail + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +# Same pinned image as test-platform-observability-rules.sh. +readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b" + +work="$(mktemp -d)" +trap 'rm -rf -- "$work"' EXIT +python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY' +import sys, yaml +rule = yaml.safe_load(open(sys.argv[1])) +yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False) +PY + +# promtool evaluates from Unix time 0, so "now" in each test is the eval_time. +cat >"$work/tests.yaml" <<'EOF' +rule_files: + - rules.yaml +evaluation_interval: 1m +tests: + # k3s datastore: last success at t=0; stale after 2 days, alert after 1h more. + - interval: 1m + input_series: + - series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}' + values: '0x5500' + - series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}' + values: '0x5500' + - series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}' + values: '0x5500' + - series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}' + values: '0x5500' + alert_rule_test: + - eval_time: 24h + alertname: PlatformK3sDatastoreBackupTooOld + exp_alerts: [] + - eval_time: 50h + alertname: PlatformK3sDatastoreBackupTooOld + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: k3s datastore has not been backed up for more than 2 days + description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST. + - eval_time: 90h + alertname: PlatformGiteaBackupTooOld + exp_alerts: [] + - eval_time: 90h + alertname: PlatformHostBackupMetricsAbsent + exp_alerts: [] + - interval: 1m + input_series: + - series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}' + values: '0x11700' + alert_rule_test: + - eval_time: 194h + alertname: PlatformGiteaBackupTooOld + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: Gitea has not been backed up for more than 8 days + description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST. + # A failed run is reported with its backup label. + - interval: 1m + input_series: + - series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}' + values: '0x30' + - series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}' + values: '1x30' + alert_rule_test: + - eval_time: 15m + alertname: PlatformHostBackupFailed + exp_alerts: + - exp_labels: {severity: warning, backup: gitea} + exp_annotations: + summary: "The last gitea backup run failed" + description: "Check `journalctl -u platform-gitea-backup` on desktop." + # Metrics never written (timers or textfile collector missing). + - interval: 1h + input_series: + - series: 'up{job="prometheus-node-exporter"}' + values: '1x30' + alert_rule_test: + - eval_time: 25h + alertname: PlatformHostBackupMetricsAbsent + exp_alerts: + - exp_labels: {severity: warning, backup: k3s-datastore} + exp_annotations: + summary: Host backup metrics are missing + description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. + - exp_labels: {severity: warning, backup: gitea} + exp_annotations: + summary: Host backup metrics are missing + description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. + # Drift for 7h fires; drift 0 does not. + - interval: 1m + input_series: + - series: 'platform_live_drift_objects' + values: '3x500' + - series: 'platform_live_drift_last_run_timestamp_seconds' + values: '0x500' + - series: 'platform_live_drift_check_success' + values: '1x500' + alert_rule_test: + - eval_time: 5h + alertname: PlatformLiveDriftDetected + exp_alerts: [] + - eval_time: 7h + alertname: PlatformLiveDriftDetected + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: "3 live objects differ from platform-core origin/main" + description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them. + - eval_time: 7h + alertname: PlatformLiveDriftCheckStale + exp_alerts: [] + # Certificates: 50 days left -> warning only; 10 days left -> warning and critical. + - interval: 1m + input_series: + - series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}' + values: '4320000x200' + - series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}' + values: '864000x200' + - series: 'platform_k3s_certificate_check_timestamp_seconds' + values: '0x200' + alert_rule_test: + - eval_time: 2h + alertname: PlatformK3sServingCertificateExpiry + exp_alerts: + - exp_labels: {severity: warning, certificate: apiserver} + exp_annotations: + summary: "k3s apiserver serving certificate expires in less than 100 days" + description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). + - exp_labels: {severity: warning, certificate: kubelet} + exp_annotations: + summary: "k3s kubelet serving certificate expires in less than 100 days" + description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). + - exp_labels: {severity: critical, certificate: kubelet} + exp_annotations: + summary: "k3s kubelet serving certificate expires in less than 14 days" + description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`. + - eval_time: 2h + alertname: PlatformK3sCertificateCheckStale + exp_alerts: [] +EOF + +docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml +docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml +echo "PLATFORM HOST CHECK ALERTS PASS" diff --git a/services/observability/kube-prometheus-stack/values/home.yaml b/services/observability/kube-prometheus-stack/values/home.yaml index 2c0f3a0..e55abf3 100644 --- a/services/observability/kube-prometheus-stack/values/home.yaml +++ b/services/observability/kube-prometheus-stack/values/home.yaml @@ -34,6 +34,11 @@ kubeProxy: defaultRules: disabled: Watchdog: true + # k3s kubelet exposes no kubelet_certificate_manager_* metrics, so these two + # can never fire. Serving certificates are watched by platform-host-checks + # (PlatformK3sServingCertificateExpiry) instead. Review 2026-11-11 A11. + KubeletClientCertificateExpiration: true + KubeletServerCertificateExpiration: true rules: etcd: false kubeControllerManager: false diff --git a/services/platform-host-checks/README.md b/services/platform-host-checks/README.md new file mode 100644 index 0000000..524a986 --- /dev/null +++ b/services/platform-host-checks/README.md @@ -0,0 +1,86 @@ +# 호스트 백업과 점검 + +desktop 호스트에서 systemd 타이머로 도는 백업과 점검이다. 결과는 node-exporter textfile 수집기 +(`/var/lib/node_exporter/textfile_collector/*.prom`)로 Prometheus에 올라가고, 이 폴더의 PrometheusRule이 +경보를 낸다. 근거와 결정은 `docs/reviews/2026-11-11-k3s-maturity-review.md` A1·A11·A13·A14, 8절 결정 1·6이다. + +## 무엇을 어디에 얼마나 + +| 대상 | 방법 | 주기 | 보관 | 위치 | +| --- | --- | --- | --- | --- | +| k3s datastore + `server/token` | `sqlite3 .backup` → tar.gz + sha256 | 매일 03:30 KST | AIStor 30일 + 로컬 최근 7개 | AIStor `platform-host-backups/k3s-datastore/`와 `/srv/k3s/aistor/host-backups/k3s-datastore/`(root 0700). 둘 다 HDD sdb3 | +| Gitea (저장소·패키지·첨부·Actions·`app.ini`) | 라이브 파드에서 `gitea dump --skip-db` | 일요일 04:00 KST | 56일(8주) | AIStor `platform-host-backups/gitea/` | +| platform-postgres (Gitea·Keycloak·tech-log DB) | CloudNativePG base backup + WAL | 매일 03:00 KST | 30일 | AIStor `platform-postgres-backups/` (W1, `services/platform-postgres-backup`) | +| AIStor 데이터(Loki·Tempo) | **백업하지 않는다** | — | — | 8절 결정 6: 잃어도 되는 관측 데이터. 디스크를 잃으면 로그·추적 기록만 사라진다 | +| Prometheus·Grafana·Alertmanager PV | 백업하지 않는다 | — | — | 설정은 이 저장소가 정본이고, 지표는 다시 쌓인다 | + +- Gitea DB는 dump에 없다(`--skip-db`). Gitea를 복원하려면 CNPG 복원이 먼저다(`docs/runbooks/gitea-restore.md`). +- k3s 백업에는 `token`이 들어 있다. token과 datastore가 같이 있으면 클러스터의 모든 Secret을 풀 수 있다. + Gitea dump의 `app.ini`에도 비밀 값이 있다. 그래서 둘 다 AIStor의 전용 사용자만 읽고 쓴다. +- AIStor는 같은 desktop의 HDD다. SSD가 죽으면 남지만 desktop 자체를 잃으면 같이 잃는다. +- k3s 백업을 AIStor 밖에도 두는 이유: AIStor는 바로 그 k3s 위에서 돈다. 클러스터를 통째로 잃으면 AIStor를 + 다시 띄우기 전에는 AIStor 안의 사본을 꺼낼 수 없다. 로컬 사본은 sdb3를 mount하기만 하면 읽힌다 + (`docs/runbooks/k3s-datastore-restore.md`). 폴더는 AIStor 드라이브 폴더(`data/`) 옆이라 bucket으로 보이지 않는다. + +## off-host 사본 (준비만, 기본 꺼짐) + +위치는 소유자가 정한다(8절 결정 1). `/etc/platform-backup/aistor.env`에 두 줄을 넣으면 k3s datastore +백업이 그 공개 키로 암호화한 사본을 `rclone`으로 보낸다. + +```bash +OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc # 공개 키. 개인 키는 desktop에 두지 않는다 +OFFHOST_RCLONE_TARGET=: # rclone 설치·설정 필요 +``` + +지표 `platform_backup_offhost_copy{backup="k3s-datastore"}`가 1이 된다. Gitea dump의 off-host 사본은 아직 없다. + +## 점검 + +| 타이머 | 사용자 | 하는 일 | 지표 | +| --- | --- | --- | --- | +| `platform-k3s-cert-expiry` | donghyeon | 6443·10250 serving 인증서 notAfter | `platform_k3s_certificate_not_after_seconds` | +| `platform-live-drift-check` | donghyeon | origin/main 렌더 → `kubectl diff`(서버 dry-run) | `platform_live_drift_objects` | + +## 경보 (`prometheus-rules.yaml`) + +| 경보 | 조건 | +| --- | --- | +| `PlatformK3sDatastoreBackupTooOld` | 마지막 성공이 2일보다 오래됨 | +| `PlatformGiteaBackupTooOld` | 8일보다 오래됨 | +| `PlatformHostBackupFailed` | 마지막 실행 실패 | +| `PlatformHostBackupMetricsAbsent` | 지표가 하루 동안 없음(타이머나 textfile 수집기가 안 돎) | +| `PlatformLiveDriftDetected` | drift가 6시간 넘게 남음 | +| `PlatformLiveDriftCheckStale` | drift 검사가 2일 넘게 완료되지 않음 | +| `PlatformK3sServingCertificateExpiry` | 100일 미만 warning, 14일 미만 critical | +| `PlatformK3sCertificateCheckStale` | 인증서 점검이 2일 넘게 안 돎 | + +`scripts/validate/test-platform-host-checks-alerts.sh`가 promtool 단위 시험으로 각 경보가 울려야 할 때 울리고 +아닐 때 안 울리는지 본다. 같은 PR에서 kube-prometheus-stack의 `KubeletClientCertificateExpiration`· +`KubeletServerCertificateExpiration`을 껐다. k3s kubelet은 그 지표를 내지 않아 울릴 수 없었다. + +## 적용 (소유자, 순서대로) + +전제: node-exporter textfile 수집기(W1, `services/observability/node-exporter/values/home.yaml`의 +`--collector.textfile.directory=/host/root/var/lib/node_exporter/textfile_collector`)가 들어간 main checkout. + +```bash +cd /home/donghyeon/workspace/platform && git pull --ff-only +sudo bash scripts/bootstrap/create-host-backup-credentials.sh # bucket·정책·사용자·/etc/platform-backup/aistor.env (APPLY 입력) +bash scripts/bootstrap/install-host-checks.sh --check +sudo bash scripts/bootstrap/install-host-checks.sh --execute # 타이머 설치·켜기·첫 실행 +kubectl apply -k services/platform-host-checks # 경보 규칙 +sudo bash scripts/validate/k3s-datastore-restore-drill.sh # 첫 백업으로 복원 리허설 +sudo bash scripts/validate/gitea-backup-restore-drill.sh +``` + +`install-host-checks.sh --execute` 마지막에 `platform_backup_last_run_success{backup="k3s-datastore"} 1`과 +`{backup="gitea"} 1`이 보여야 한다. 리허설 둘은 각각 `K3S DATASTORE RESTORE DRILL PASS`, `GITEA RESTORE DRILL PASS`로 끝나야 한다. + +## 복원 리허설 + +- k3s: 백업을 내려받아 sha256을 맞춘 뒤, 운영과 같은 버전의 k3s를 **네트워크 없는** 일회용 컨테이너로 + 띄워 API가 뜨는지, Secret 수가 라이브와 비슷한지(±5), 모든 Secret이 복호화되는지 본다. + `scripts/validate/test-k3s-datastore-restore-drill.sh`는 운영 데이터 없이 같은 경로를 시험한다(새 k3s에 + 시험 Secret 셋 → 암호화돼 저장된 것을 확인 → 복원). +- Gitea: dump를 풀어 `app.ini`(INTERNAL_TOKEN·SECRET_KEY), 모든 저장소의 `git fsck`, 라이브와 저장소 수를 본다. +- 분기마다 한 번 돌린다. 성공하면 `platform_backup_restore_drill_last_success_timestamp_seconds`가 갱신된다. diff --git a/services/platform-host-checks/bucket-policy.json b/services/platform-host-checks/bucket-policy.json new file mode 100644 index 0000000..cf6e7a6 --- /dev/null +++ b/services/platform-host-checks/bucket-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::platform-host-backups"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::platform-host-backups/*"] + } + ] +} diff --git a/services/platform-host-checks/kustomization.yaml b/services/platform-host-checks/kustomization.yaml new file mode 100644 index 0000000..a4bdf33 --- /dev/null +++ b/services/platform-host-checks/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - prometheus-rules.yaml diff --git a/services/platform-host-checks/prometheus-rules.yaml b/services/platform-host-checks/prometheus-rules.yaml new file mode 100644 index 0000000..ab7e556 --- /dev/null +++ b/services/platform-host-checks/prometheus-rules.yaml @@ -0,0 +1,88 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-host-checks + namespace: observability + labels: + observability.hyeonworks.com/instance: home + app.kubernetes.io/part-of: platform +spec: + groups: + - name: platform-host-backups + rules: + - alert: PlatformK3sDatastoreBackupTooOld + expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}) > 2 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: k3s datastore has not been backed up for more than 2 days + description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST. + - alert: PlatformGiteaBackupTooOld + expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="gitea"}) > 8 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: Gitea has not been backed up for more than 8 days + description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST. + - alert: PlatformHostBackupFailed + expr: max by (backup) (platform_backup_last_run_success) == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "The last {{ $labels.backup }} backup run failed" + description: "Check `journalctl -u platform-{{ $labels.backup }}-backup` on desktop." + - alert: PlatformHostBackupMetricsAbsent + expr: absent(platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}) or absent(platform_backup_last_run_timestamp_seconds{backup="gitea"}) + for: 1d + labels: + severity: warning + annotations: + summary: Host backup metrics are missing + description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. + - name: platform-host-drift + rules: + - alert: PlatformLiveDriftDetected + expr: max(platform_live_drift_objects) > 0 + for: 6h + labels: + severity: warning + annotations: + summary: "{{ $value }} live objects differ from platform-core origin/main" + description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them. + - alert: PlatformLiveDriftCheckStale + expr: time() - max(platform_live_drift_last_run_timestamp_seconds) > 2 * 86400 or max(platform_live_drift_check_success) == 0 + for: 1h + labels: + severity: warning + annotations: + summary: The daily live drift check has not completed + description: Check `journalctl -u platform-live-drift-check` on desktop. + - name: platform-k3s-certificates + rules: + - alert: PlatformK3sServingCertificateExpiry + expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 100 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 100 days" + description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). + - alert: PlatformK3sServingCertificateExpiry + expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 14 * 86400 + for: 10m + labels: + severity: critical + annotations: + summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 14 days" + description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`. + - alert: PlatformK3sCertificateCheckStale + expr: time() - max(platform_k3s_certificate_check_timestamp_seconds) > 2 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: The daily k3s certificate expiry check has not run + description: Check `journalctl -u platform-k3s-cert-expiry` on desktop.