Merge pull request 'feat(backup): back up k3s and Gitea to AIStor with alerts and drills' (#6) from donghyeon.kang/w6-host-backups into main

This commit was merged in pull request #6.
This commit is contained in:
2026-11-11 16:48:31 +00:00
25 changed files with 1083 additions and 0 deletions
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# Back up Gitea (repositories, packages, attachments, Actions data, app.ini) to
# AIStor (HDD sdb3). Root only. Run by platform-gitea-backup.timer.
#
# `gitea dump --skip-db` runs inside the live Gitea container and streams a
# tar.gz to stdout (logs go to stderr). The database is backed up separately by
# CloudNativePG (services/platform-postgres-backup). The dump contains app.ini,
# whose secrets are needed to restore, so the archive is sensitive.
set -Eeuo pipefail
umask 077
# shellcheck source=../lib/host-backup.sh
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
readonly NAME=gitea_backup
readonly KCFG=/etc/rancher/k3s/k3s.yaml
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root"
hb_load_aistor_env
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig "$KCFG" "$@"; }
started="$(date +%s)"
previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)"
on_error() {
hb_write_metrics "$NAME" <<PROM
platform_backup_last_run_success{backup="gitea"} 0
platform_backup_last_run_timestamp_seconds{backup="gitea"} $(date +%s)
platform_backup_last_success_timestamp_seconds{backup="gitea"} ${previous_success:-0}
PROM
}
trap on_error ERR
work="$(mktemp -d "$HB_WORK_ROOT/gitea.XXXXXX")"
trap 'rm -rf -- "$work"' EXIT
ts="$(date -u +%Y%m%dT%H%M%SZ)"
archive="gitea-dump-$ts.tar.gz"
kubectl -n gitea exec deploy/gitea -c gitea -- \
gitea dump --skip-db --type tar.gz --file - >"$work/$archive" 2>"$work/dump.log"
tar -tzf "$work/$archive" >"$work/list.txt"
grep -qx 'app.ini' "$work/list.txt" || hb_fail "the dump has no app.ini"
repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)"
(( repos > 0 )) || hb_fail "the dump has no repositories"
sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256"
size="$(stat -c %s "$work/$archive")"
# What a restore must match: restore with the same Gitea version (docs/runbooks/gitea-restore.md).
{
echo "created=$ts"
echo "gitea_image=$(kubectl -n gitea get deploy gitea -o jsonpath='{.spec.template.spec.containers[0].image}')"
echo "gitea_version=$(kubectl -n gitea exec deploy/gitea -c gitea -- gitea --version 2>/dev/null | awk '{print $3}')"
echo "repositories=$repos"
} >"$work/$archive.meta"
hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "/work/$archive.meta" "backup/$AISTOR_BUCKET/gitea/" >/dev/null
hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null
trap - ERR
hb_write_metrics "$NAME" <<PROM
# HELP platform_backup_last_success_timestamp_seconds Unix time of the last successful backup.
# TYPE platform_backup_last_success_timestamp_seconds gauge
platform_backup_last_success_timestamp_seconds{backup="gitea"} $(date +%s)
platform_backup_last_run_success{backup="gitea"} 1
platform_backup_last_run_timestamp_seconds{backup="gitea"} $(date +%s)
platform_backup_last_size_bytes{backup="gitea"} $size
platform_backup_last_duration_seconds{backup="gitea"} $(( $(date +%s) - started ))
# HELP platform_backup_gitea_repositories Repositories in the last successful Gitea dump.
# TYPE platform_backup_gitea_repositories gauge
platform_backup_gitea_repositories $repos
PROM
hb_log "gitea backup $archive ($size bytes, $repos repositories) uploaded"
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env bash
# Back up the k3s datastore (sqlite) and the server token to AIStor (HDD sdb3).
# Root only. Run by platform-k3s-datastore-backup.timer (infrastructure/host/backup).
#
# k3s docs (datastore/backup-restore, SQLite): copy server/db and server/token;
# the token is required to read the datastore back. `sqlite3 .backup` takes a
# consistent copy of the live database.
#
# Off-host copy (prepared, off by default): set in /etc/platform-backup/aistor.env
# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc
# OFFHOST_RCLONE_TARGET=<rclone remote:path>
# The archive is then encrypted to that public key and copied with rclone.
set -Eeuo pipefail
umask 077
# shellcheck source=../lib/host-backup.sh
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
readonly SERVER=/var/lib/rancher/k3s/server
readonly NAME=k3s_datastore_backup
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root"
command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required"
hb_load_aistor_env
started="$(date +%s)"
on_error() {
hb_write_metrics "$NAME" <<PROM
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
# TYPE platform_backup_last_run_success gauge
platform_backup_last_run_success{backup="k3s-datastore"} 0
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} ${previous_success:-0}
PROM
}
previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)"
trap on_error ERR
work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")"
trap 'rm -rf -- "$work"' EXIT
ts="$(date -u +%Y%m%dT%H%M%SZ)"
stage="$work/k3s-datastore-$ts"
mkdir -m 0700 "$stage"
sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'"
check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')"
[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check"
install -m 0600 "$SERVER/token" "$stage/token"
{
echo "created=$ts"
echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)"
echo "host=$(hostname)"
echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')"
} >"$stage/META"
tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts"
sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256"
size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")"
hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null
hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null
# Local copy outside AIStor on the same HDD (sdb3). AIStor runs inside the
# cluster this archive restores, so after losing the cluster the AIStor copy
# cannot be read until k3s is back; this copy can. The directory sits next to
# AIStor's drive directory (data/), so AIStor does not see it as a bucket.
readonly LOCAL_DIR=/srv/k3s/aistor/host-backups/k3s-datastore
install -d -m 0700 -o root -g root "$(dirname "$LOCAL_DIR")" "$LOCAL_DIR"
install -m 0600 "$work/k3s-datastore-$ts.tar.gz" "$work/k3s-datastore-$ts.tar.gz.sha256" "$LOCAL_DIR/"
ls -1t "$LOCAL_DIR"/k3s-datastore-*.tar.gz | tail -n +8 | while read -r old; do rm -f -- "$old" "$old.sha256"; done
local_copies="$(ls -1 "$LOCAL_DIR"/k3s-datastore-*.tar.gz | wc -l)"
offhost=0
if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then
command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed"
gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \
--output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz"
rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg"
offhost=1
fi
trap - ERR
hb_write_metrics "$NAME" <<PROM
# HELP platform_backup_last_success_timestamp_seconds Unix time of the last successful backup.
# TYPE platform_backup_last_success_timestamp_seconds gauge
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
# TYPE platform_backup_last_run_success gauge
platform_backup_last_run_success{backup="k3s-datastore"} 1
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
# HELP platform_backup_last_size_bytes Size of the last successful backup archive.
# TYPE platform_backup_last_size_bytes gauge
platform_backup_last_size_bytes{backup="k3s-datastore"} $size
platform_backup_last_duration_seconds{backup="k3s-datastore"} $(( $(date +%s) - started ))
# HELP platform_backup_local_copies Archives kept outside AIStor on the HDD.
# TYPE platform_backup_local_copies gauge
platform_backup_local_copies{backup="k3s-datastore"} $local_copies
# HELP platform_backup_offhost_copy 1 when the last run also wrote the encrypted off-host copy.
# TYPE platform_backup_offhost_copy gauge
platform_backup_offhost_copy{backup="k3s-datastore"} $offhost
PROM
hb_log "k3s datastore backup k3s-datastore-$ts.tar.gz ($size bytes) uploaded; off-host=$offhost"
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
# Create the AIStor bucket, lifecycle rules, policy and user that the host backup
# timers use, and the root-only /etc/platform-backup/aistor.env.
#
# sudo bash scripts/bootstrap/create-host-backup-credentials.sh
#
# Run once, by the owner, on the desktop. Refuses to run when the env file exists;
# never rotates or deletes a bucket, policy, user or object. No key is printed.
set -Eeuo pipefail
set +x
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# shellcheck source=../lib/host-backup.sh
source "$ROOT/scripts/lib/host-backup.sh"
readonly BUCKET=platform-host-backups
readonly POLICY=platform-host-backups
readonly QUOTA=50GiB
readonly POLICY_FILE="$ROOT/services/platform-host-checks/bucket-policy.json"
readonly PASS_LINE=PLATFORM_HOST_BACKUP_BOOTSTRAP_PASS
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; }
[[ "$(id -u)" -eq 0 ]] || hb_fail "run with sudo"
for c in docker jq openssl; do command -v "$c" >/dev/null || hb_fail "missing command: $c"; done
jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON"
[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it"
ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')"
[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found"
endpoint="http://${ip}:80"
work="$(mktemp -d)"
trap 'rm -rf -- "$work"' EXIT
kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env"
grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format"
printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY"
openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY"
cp "$POLICY_FILE" "$work/policy.json"
printf '%s\n' \
"Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \
"Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \
"User: a new generated access key attached to $POLICY" \
"File: $HOST_BACKUP_ENV (root, 0600)" \
'No key value will be printed.'
printf 'Type APPLY to create them: '
IFS= read -r confirmation
[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY"
# Every mc call is silenced so no key reaches the terminal.
docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \
--entrypoint /bin/sh "$MC_IMAGE" -ec '
. /run/b/root.env
ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)"
mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null
mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null
mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null
mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null
mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null
mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null
mc admin user add root "$ak" "$sk" >/dev/null
mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null
mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null
probe=".bootstrap-probe-$(date +%s)"
printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null
[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ]
mc rm "backup/'"$BUCKET"'/$probe" >/dev/null
if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi
echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written"
install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")"
{
echo "AISTOR_BUCKET=$BUCKET"
echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")"
echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")"
echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):"
echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc"
echo "# OFFHOST_RCLONE_TARGET=remote:path"
} >"$HOST_BACKUP_ENV"
chmod 0600 "$HOST_BACKUP_ENV"
printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV."
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env bash
# Install the host backup and check timers (infrastructure/host/backup).
#
# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing
# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs
#
# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root
# timers never execute files the desktop user can edit.
set -Eeuo pipefail
umask 022
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly LIB=/usr/local/lib/platform-backup
readonly UNIT_DIR=/etc/systemd/system
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
readonly WORK_ROOT=/var/lib/platform-backup/work
readonly USER_NAME=donghyeon
readonly -a SCRIPTS=(
scripts/lib/host-backup.sh:lib/host-backup.sh
scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh
scripts/backup/gitea-backup.sh:backup/gitea-backup.sh
scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh
scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh
services/platform-host-checks/README.md:README.md
)
readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry)
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
mode="${1:-}"
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; }
echo "== check"
for pair in "${SCRIPTS[@]}"; do
src="${pair%%:*}"
[[ -f "$ROOT/$src" ]] || fail "missing $src"
[[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src"
done
for t in "${TIMERS[@]}"; do
for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done
done
if command -v systemd-analyze >/dev/null; then
verify_dir="$(mktemp -d)"
trap 'rm -rf -- "$verify_dir"' EXIT
cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/"
# ExecStart paths do not exist before installation; only syntax and keys are checked.
out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)"
[[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; }
echo " units: systemd-analyze verify clean"
fi
echo "== plan"
for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done
for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done
echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)"
echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check"
[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; }
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first"
echo "== install"
for pair in "${SCRIPTS[@]}"; do
install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}"
done
install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR"
install -d -o root -g root -m 0700 "$WORK_ROOT"
for t in "${TIMERS[@]}"; do
install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/"
done
systemctl daemon-reload
for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done
echo "== first runs"
for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do
if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi
done
echo "== metrics"
grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet"
systemctl list-timers 'platform-*' --no-pager
echo "HOST CHECKS INSTALLED"
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env bash
# Shared helpers for the host backup and host check scripts. Source only.
# shellcheck shell=bash
readonly HOST_BACKUP_ENV=/etc/platform-backup/aistor.env
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
# Backup staging (root 0700, same SSD). Not /var/tmp: dockerd must see the path.
readonly HB_WORK_ROOT=/var/lib/platform-backup/work
# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh.
readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b"
hb_log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; }
hb_fail() { hb_log "FAIL: $*" >&2; exit 1; }
# Write Prometheus textfile metrics atomically: hb_write_metrics NAME <<'PROM' ... PROM
hb_write_metrics() {
local name="$1" tmp
[[ -d "$TEXTFILE_DIR" ]] || { hb_log "textfile dir $TEXTFILE_DIR missing; metrics not written"; cat >/dev/null; return 0; }
tmp="$(mktemp "$TEXTFILE_DIR/.${name}.XXXXXX")"
cat >"$tmp"
chmod 0644 "$tmp"
mv -f -- "$tmp" "$TEXTFILE_DIR/${name}.prom"
}
# Load the AIStor endpoint and the backup user's keys (root-only file).
hb_load_aistor_env() {
[[ -f "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV is missing; run scripts/bootstrap/create-host-backup-credentials.sh"
[[ "$(stat -c '%u %a' "$HOST_BACKUP_ENV")" == "0 600" ]] || hb_fail "$HOST_BACKUP_ENV must be owned by root with mode 0600"
# shellcheck disable=SC1090
. "$HOST_BACKUP_ENV"
: "${AISTOR_BUCKET:?}" "${AISTOR_ACCESS_KEY:?}" "${AISTOR_SECRET_KEY:?}"
# The host cannot resolve cluster DNS, so use the Service ClusterIP (port 80 -> 9000).
if [[ -z "${AISTOR_ENDPOINT:-}" ]]; then
local ip
ip="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml -n object-storage \
get service minio -o jsonpath='{.spec.clusterIP}')"
[[ -n "$ip" ]] || hb_fail "cannot find the object-storage/minio Service"
AISTOR_ENDPOINT="http://${ip}:80"
fi
}
# Run the pinned mc client on the host network with the backup user's keys.
# The keys go through a 0600 env file, never the command line (ps would show it).
# Usage: hb_mc <dir mounted at /work> <mc args...>
hb_mc() {
local work="$1"; shift
local envfile="$work/.mc.env"
if [[ ! -f "$envfile" ]]; then
( umask 077; printf 'MC_HOST_backup=%s\nMC_CONFIG_DIR=/tmp/mc\n' \
"${AISTOR_ENDPOINT/:\/\//://${AISTOR_ACCESS_KEY}:${AISTOR_SECRET_KEY}@}" >"$envfile" )
fi
docker run --rm --network host --user "$(id -u):$(id -g)" \
--env-file "$envfile" -v "$work:/work" --tmpfs /tmp \
"$MC_IMAGE" "$@"
}
+10
View File
@@ -596,3 +596,13 @@ PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
- `--textfile FILE`: `platform_live_drift_objects`, `platform_live_drift_check_success`,
`platform_live_drift_last_run_timestamp_seconds`를 Prometheus textfile 형식으로 쓴다.
- Secret 값의 차이는 보지 않는다.
## 호스트 백업·점검 시험
- `test-platform-host-checks-alerts.sh`: `services/platform-host-checks/prometheus-rules.yaml`의 promtool
단위 시험(백업 신선도, 실패, 지표 없음, drift, k3s 인증서). docker 필요, 클러스터 접근 없음.
- `k3s-cert-expiry.sh`: 6443·10250 serving 인증서 만료를 textfile 지표로 쓴다(타이머가 매일 돌림).
- `live-drift-check-main.sh`: checkout의 작업 트리가 아니라 `origin/main`으로 `live-drift-check.sh`를 돌린다.
- `k3s-datastore-restore-drill.sh`, `gitea-backup-restore-drill.sh`: 복원 리허설. 인자 없이 root로 돌리면 AIStor의
최신 백업을 쓰고, `--from-dir`·`--from-file`로 로컬 사본을 줄 수 있다.
- `test-k3s-datastore-restore-drill.sh`: 운영 데이터 없이 k3s 복원 리허설 경로를 시험한다(일회용 k3s, 시험 Secret 셋).
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# Restore drill for the Gitea backup (file level).
#
# sudo bash scripts/validate/gitea-backup-restore-drill.sh # latest dump in AIStor
# bash scripts/validate/gitea-backup-restore-drill.sh --from-file DUMP.tar.gz
#
# Extracts the dump into a throwaway directory and checks what a restore needs:
# app.ini is present, every repository passes `git fsck`, and the repository
# count matches the live Gitea. The database is restored from the CloudNativePG
# backup instead (services/platform-postgres-backup); the full restore order is
# in docs/runbooks/gitea-restore.md. Nothing is written to the cluster.
set -Eeuo pipefail
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# shellcheck source=../lib/host-backup.sh
source "$ROOT/scripts/lib/host-backup.sh"
readonly NAME=gitea_restore_drill
from_file=""
case "${1:-}" in
--from-file) from_file="${2:?--from-file needs a path}" ;;
"") ;;
*) sed -n '2,11p' "$0" >&2; exit 2 ;;
esac
if [[ "$(id -u)" -eq 0 ]]; then
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; }
fi
base="$HB_WORK_ROOT"; [[ -w "$base" ]] || base="$(mktemp -d)"
work="$(mktemp -d "$base/gitea-drill.XXXXXX")"
trap 'rm -rf -- "$work"' EXIT
if [[ -z "$from_file" ]]; then
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials"
hb_load_aistor_env
latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/gitea/" | awk '{print $NF}' | grep -E '^gitea-dump-.*\.tar\.gz$' | sort | tail -1)"
[[ -n "$latest" ]] || hb_fail "no Gitea dump found"
hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/gitea/$latest" "backup/$AISTOR_BUCKET/gitea/$latest.sha256" /work/ >/dev/null
[[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest"
from_file="$work/$latest"
hb_log "drill on $latest"
fi
mkdir "$work/restore"
tar -C "$work/restore" -xzf "$from_file"
[[ -s "$work/restore/app.ini" ]] || hb_fail "app.ini is missing from the dump"
for key in INTERNAL_TOKEN SECRET_KEY; do
grep -q "^${key}[[:space:]]*=" "$work/restore/app.ini" || hb_fail "app.ini has no ${key} (needed to restore)"
done
total=0 broken=0
while IFS= read -r -d '' repo; do
total=$((total + 1))
if ! git -c safe.directory='*' --git-dir="$repo" fsck --no-dangling --no-progress >/dev/null 2>"$work/fsck.err"; then
broken=$((broken + 1))
hb_log "fsck failed: ${repo#"$work/restore/"}: $(head -1 "$work/fsck.err")"
fi
done < <(find "$work/restore/repos" -mindepth 2 -maxdepth 2 -type d -name '*.git' -print0)
hb_log "repositories in dump: $total, fsck failures: $broken"
(( total > 0 )) || hb_fail "the dump has no repositories"
(( broken == 0 )) || hb_fail "$broken repositories failed git fsck"
live="$(kubectl -n gitea exec deploy/gitea -c gitea -- sh -c 'find /data/git/gitea-repositories -mindepth 2 -maxdepth 2 -name "*.git" -type d | wc -l' 2>/dev/null || echo "")"
if [[ -n "$live" ]]; then
hb_log "live Gitea repositories: $live"
(( total >= live - 2 )) || hb_fail "the dump has $total repositories, live Gitea has $live"
fi
hb_write_metrics "$NAME" <<PROM
# HELP platform_backup_restore_drill_last_success_timestamp_seconds Unix time of the last successful restore drill.
# TYPE platform_backup_restore_drill_last_success_timestamp_seconds gauge
platform_backup_restore_drill_last_success_timestamp_seconds{backup="gitea"} $(date +%s)
PROM
echo "GITEA RESTORE DRILL PASS"
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Write the notAfter time of the k3s API server (6443) and kubelet (10250)
# serving certificates as Prometheus textfile metrics. No root needed.
# k3s renews these only when it starts and they are within 120 days of expiry
# (docs.k3s.io cli/certificate), and the kubelet exposes no certificate metric,
# so nothing else in the cluster watches them.
set -Eeuo pipefail
# shellcheck source=../lib/host-backup.sh
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
lines=""
for target in apiserver:6443 kubelet:10250; do
name="${target%%:*}" port="${target##*:}"
end="$(echo | timeout 10 openssl s_client -connect "127.0.0.1:$port" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//')" || true
[[ -n "$end" ]] || hb_fail "could not read the $name certificate on 127.0.0.1:$port"
lines+="platform_k3s_certificate_not_after_seconds{certificate=\"$name\"} $(date -d "$end" +%s)"$'\n'
hb_log "$name certificate notAfter $end"
done
hb_write_metrics platform_k3s_certificates <<PROM
# HELP platform_k3s_certificate_not_after_seconds notAfter of a k3s serving certificate (Unix time).
# TYPE platform_k3s_certificate_not_after_seconds gauge
${lines}# HELP platform_k3s_certificate_check_timestamp_seconds Unix time of the last certificate check.
# TYPE platform_k3s_certificate_check_timestamp_seconds gauge
platform_k3s_certificate_check_timestamp_seconds $(date +%s)
PROM
@@ -0,0 +1,99 @@
#!/usr/bin/env bash
# Restore drill for the k3s datastore backup.
#
# sudo bash scripts/validate/k3s-datastore-restore-drill.sh # latest backup in AIStor
# bash scripts/validate/k3s-datastore-restore-drill.sh --from-dir DIR [--expect-secrets N]
# DIR holds state.db and token (an extracted archive, or a test datastore)
#
# Starts a throwaway k3s server (same version as production, --network none,
# no agent) on a copy of the backup, and checks that the API answers, that the
# Secret count matches and that every Secret decrypts (the token unlocks the
# encryption config stored in the datastore). Prints no Secret value. The
# container and the copy are removed at the end.
set -Eeuo pipefail
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# shellcheck source=../lib/host-backup.sh
source "$ROOT/scripts/lib/host-backup.sh"
# Same version as the desktop server (k3s --version: v1.36.2+k3s1).
readonly K3S_IMAGE="docker.io/rancher/k3s:v1.36.2-k3s1@sha256:6a47cea22c4b834d4ba72c89d291696b79ebe406251f90b446e4dff03513dd87"
readonly NAME=k3s_datastore_restore_drill
from_dir="" expect_secrets=""
while (( $# > 0 )); do
case "$1" in
--from-dir) from_dir="${2:?}"; shift 2 ;;
--expect-secrets) expect_secrets="${2:?}"; shift 2 ;;
*) sed -n '2,13p' "$0" >&2; exit 2 ;;
esac
done
base="${HB_WORK_ROOT}"
[[ -w "$base" ]] || base="$(mktemp -d)"
work="$(mktemp -d "$base/k3s-drill.XXXXXX")"
container="k3s-restore-drill-$$"
cleanup() {
docker rm -f "$container" >/dev/null 2>&1 || true
# The drill container writes as root; remove its files with the same image.
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true
rm -rf -- "$work"
}
trap cleanup EXIT
if [[ -z "$from_dir" ]]; then
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials"
hb_load_aistor_env
latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)"
[[ -n "$latest" ]] || hb_fail "no k3s datastore backup found"
hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null
[[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest"
tar -C "$work" -xzf "$work/$latest"
from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)"
hb_log "drill on $latest"
if [[ -z "$expect_secrets" ]]; then
expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)"
hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)"
fi
fi
[[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token"
install -d -m 0700 "$work/server/db"
cp "$from_dir/state.db" "$work/server/db/state.db"
cp "$from_dir/token" "$work/server/token"
[[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed"
started="$(date +%s)"
docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \
-v "$work/server:/var/lib/rancher/k3s/server" \
"$K3S_IMAGE" server --disable-agent --secrets-encryption \
--token-file /var/lib/rancher/k3s/server/token \
--bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \
--disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null
ready=0
for _ in $(seq 1 60); do
if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi
sleep 5
done
if (( ready == 0 )); then
docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2
hb_fail "restored k3s did not become ready in 5 minutes"
fi
namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)"
secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)"
# -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero.
decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')"
hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s"
if [[ -n "$expect_secrets" ]]; then
diff=$(( secrets - expect_secrets ))
(( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets"
fi
(( decrypted > 0 )) || hb_fail "no Secret carried data after restore"
hb_write_metrics "$NAME" <<PROM
# HELP platform_backup_restore_drill_last_success_timestamp_seconds Unix time of the last successful restore drill.
# TYPE platform_backup_restore_drill_last_success_timestamp_seconds gauge
platform_backup_restore_drill_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
PROM
echo "K3S DATASTORE RESTORE DRILL PASS"
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Daily drift check against origin/main (not the working tree of the checkout),
# so local edits or a stale checkout do not hide or invent drift.
# Run by platform-live-drift-check.timer as the desktop user.
set -Eeuo pipefail
readonly CHECKOUT="${PLATFORM_CHECKOUT:-/home/donghyeon/workspace/platform}"
readonly TEXTFILE=/var/lib/node_exporter/textfile_collector/platform_live_drift.prom
git -C "$CHECKOUT" fetch --quiet origin main
work="$(mktemp -d /tmp/platform-drift-main.XXXXXX)"
trap 'rm -rf -- "$work"' EXIT
git -C "$CHECKOUT" archive origin/main | tar -x -C "$work"
printf 'origin/main %s\n' "$(git -C "$CHECKOUT" rev-parse --short origin/main)"
args=(--accepted "$work/scripts/validate/live-drift-accepted.txt")
[[ -d "$(dirname "$TEXTFILE")" ]] && args+=(--textfile "$TEXTFILE")
bash "$work/scripts/validate/live-drift-check.sh" "${args[@]}"
@@ -0,0 +1,41 @@
#!/usr/bin/env bash
# Test k3s-datastore-restore-drill.sh without production data: start a fresh,
# network-less k3s with secrets encryption, add three test Secrets, stop it,
# and run the drill on its state.db and token. Needs docker; no root.
set -Eeuo pipefail
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly K3S_IMAGE="$(sed -n 's/^readonly K3S_IMAGE="\(.*\)"$/\1/p' "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh")"
[[ -n "$K3S_IMAGE" ]] || { echo "cannot read K3S_IMAGE from the drill script" >&2; exit 1; }
work="$(mktemp -d)"
seed="w6-k3s-drill-seed-$$"
cleanup() {
docker rm -f "$seed" >/dev/null 2>&1 || true
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true
rm -rf -- "$work"
}
trap cleanup EXIT
mkdir -p "$work/server" "$work/copy"
docker run -d --name "$seed" --network none --tmpfs /run --tmpfs /var/run \
-v "$work/server:/var/lib/rancher/k3s/server" "$K3S_IMAGE" server --disable-agent --secrets-encryption \
--bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \
--disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null
for _ in $(seq 1 60); do docker exec "$seed" kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 5; done
for i in 1 2 3; do docker exec "$seed" kubectl create secret generic "drill-test-$i" --from-literal=value="test-$i" >/dev/null; done
expected="$(docker exec "$seed" kubectl get secrets -A --no-headers | wc -l)"
docker stop -t 30 "$seed" >/dev/null
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c \
"cp /w/server/db/state.db* /w/server/token /w/copy/ && chown $(id -u):$(id -g) /w/copy/*"
# Fold the WAL into state.db, as `sqlite3 .backup` does in the real backup.
sqlite3 "$work/copy/state.db" 'PRAGMA wal_checkpoint(TRUNCATE);' >/dev/null
rm -f "$work/copy/state.db-wal" "$work/copy/state.db-shm"
# The test Secrets must be stored encrypted (k8s:enc:...), so the drill really
# proves that the token unlocks the encryption key.
encrypted="$(sqlite3 "$work/copy/state.db" "SELECT COUNT(*) FROM kine WHERE name LIKE '/registry/secrets/default/drill-test-%' AND CAST(value AS TEXT) LIKE 'k8s:enc:%';")"
echo "seed: $expected Secrets, encrypted test Secret rows=$encrypted"
(( encrypted >= 3 )) || { echo "test Secrets are not encrypted at rest in the seed datastore" >&2; exit 1; }
bash "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh" --from-dir "$work/copy" --expect-secrets "$expected"
echo "K3S DATASTORE RESTORE DRILL TEST PASS"
@@ -0,0 +1,151 @@
#!/usr/bin/env bash
# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml.
# Needs docker and python3 (PyYAML). No cluster access.
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# Same pinned image as test-platform-observability-rules.sh.
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
work="$(mktemp -d)"
trap 'rm -rf -- "$work"' EXIT
python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY'
import sys, yaml
rule = yaml.safe_load(open(sys.argv[1]))
yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False)
PY
# promtool evaluates from Unix time 0, so "now" in each test is the eval_time.
cat >"$work/tests.yaml" <<'EOF'
rule_files:
- rules.yaml
evaluation_interval: 1m
tests:
# k3s datastore: last success at t=0; stale after 2 days, alert after 1h more.
- interval: 1m
input_series:
- series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}'
values: '0x5500'
- series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}'
values: '0x5500'
- series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}'
values: '0x5500'
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
values: '0x5500'
alert_rule_test:
- eval_time: 24h
alertname: PlatformK3sDatastoreBackupTooOld
exp_alerts: []
- eval_time: 50h
alertname: PlatformK3sDatastoreBackupTooOld
exp_alerts:
- exp_labels: {severity: warning}
exp_annotations:
summary: k3s datastore has not been backed up for more than 2 days
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
- eval_time: 90h
alertname: PlatformGiteaBackupTooOld
exp_alerts: []
- eval_time: 90h
alertname: PlatformHostBackupMetricsAbsent
exp_alerts: []
- interval: 1m
input_series:
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
values: '0x11700'
alert_rule_test:
- eval_time: 194h
alertname: PlatformGiteaBackupTooOld
exp_alerts:
- exp_labels: {severity: warning}
exp_annotations:
summary: Gitea has not been backed up for more than 8 days
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
# A failed run is reported with its backup label.
- interval: 1m
input_series:
- series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}'
values: '0x30'
- series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}'
values: '1x30'
alert_rule_test:
- eval_time: 15m
alertname: PlatformHostBackupFailed
exp_alerts:
- exp_labels: {severity: warning, backup: gitea}
exp_annotations:
summary: "The last gitea backup run failed"
description: "Check `journalctl -u platform-gitea-backup` on desktop."
# Metrics never written (timers or textfile collector missing).
- interval: 1h
input_series:
- series: 'up{job="prometheus-node-exporter"}'
values: '1x30'
alert_rule_test:
- eval_time: 25h
alertname: PlatformHostBackupMetricsAbsent
exp_alerts:
- exp_labels: {severity: warning, backup: k3s-datastore}
exp_annotations:
summary: Host backup metrics are missing
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
- exp_labels: {severity: warning, backup: gitea}
exp_annotations:
summary: Host backup metrics are missing
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
# Drift for 7h fires; drift 0 does not.
- interval: 1m
input_series:
- series: 'platform_live_drift_objects'
values: '3x500'
- series: 'platform_live_drift_last_run_timestamp_seconds'
values: '0x500'
- series: 'platform_live_drift_check_success'
values: '1x500'
alert_rule_test:
- eval_time: 5h
alertname: PlatformLiveDriftDetected
exp_alerts: []
- eval_time: 7h
alertname: PlatformLiveDriftDetected
exp_alerts:
- exp_labels: {severity: warning}
exp_annotations:
summary: "3 live objects differ from platform-core origin/main"
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
- eval_time: 7h
alertname: PlatformLiveDriftCheckStale
exp_alerts: []
# Certificates: 50 days left -> warning only; 10 days left -> warning and critical.
- interval: 1m
input_series:
- series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}'
values: '4320000x200'
- series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}'
values: '864000x200'
- series: 'platform_k3s_certificate_check_timestamp_seconds'
values: '0x200'
alert_rule_test:
- eval_time: 2h
alertname: PlatformK3sServingCertificateExpiry
exp_alerts:
- exp_labels: {severity: warning, certificate: apiserver}
exp_annotations:
summary: "k3s apiserver serving certificate expires in less than 100 days"
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
- exp_labels: {severity: warning, certificate: kubelet}
exp_annotations:
summary: "k3s kubelet serving certificate expires in less than 100 days"
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
- exp_labels: {severity: critical, certificate: kubelet}
exp_annotations:
summary: "k3s kubelet serving certificate expires in less than 14 days"
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
- eval_time: 2h
alertname: PlatformK3sCertificateCheckStale
exp_alerts: []
EOF
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml
echo "PLATFORM HOST CHECK ALERTS PASS"