feat(backup): back up k3s and Gitea to AIStor with alerts and drills #6

Merged
donghyeon.kang merged 5 commits from donghyeon.kang/w6-host-backups AGit into main 2026-11-11 16:48:32 +00:00
12 changed files with 494 additions and 0 deletions
Showing only changes of commit 012452cf78 - Show all commits
@@ -0,0 +1,8 @@
[Unit]
Description=Record the k3s API server and kubelet serving certificate expiry
After=k3s.service
[Service]
Type=oneshot
User=donghyeon
ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/k3s-cert-expiry.sh
@@ -0,0 +1,9 @@
[Unit]
Description=Daily k3s certificate expiry check
[Timer]
OnCalendar=*-*-* 05:00:00
Persistent=true
[Install]
WantedBy=timers.target
@@ -0,0 +1,16 @@
[Unit]
Description=Compare the cluster with platform-core origin/main (kubectl diff, server dry-run)
After=k3s.service network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User=donghyeon
Environment=PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm
Environment=PLATFORM_CHECKOUT=/home/donghyeon/workspace/platform
ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/live-drift-check-main.sh
# Exit 1 means "drift found"; the metric and its alert report that. Only 2 (the
# check itself failed) marks the unit failed.
SuccessExitStatus=1
Nice=10
TimeoutStartSec=30min
@@ -0,0 +1,10 @@
[Unit]
Description=Daily live drift check
[Timer]
OnCalendar=*-*-* 05:15:00
RandomizedDelaySec=10min
Persistent=true
[Install]
WantedBy=timers.target
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env bash
# Install the host backup and check timers (infrastructure/host/backup).
#
# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing
# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs
#
# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root
# timers never execute files the desktop user can edit.
set -Eeuo pipefail
umask 022
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly LIB=/usr/local/lib/platform-backup
readonly UNIT_DIR=/etc/systemd/system
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
readonly WORK_ROOT=/var/lib/platform-backup/work
readonly USER_NAME=donghyeon
readonly -a SCRIPTS=(
scripts/lib/host-backup.sh:lib/host-backup.sh
scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh
scripts/backup/gitea-backup.sh:backup/gitea-backup.sh
scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh
scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh
services/platform-host-checks/README.md:README.md
)
readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry)
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
mode="${1:-}"
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; }
echo "== check"
for pair in "${SCRIPTS[@]}"; do
src="${pair%%:*}"
[[ -f "$ROOT/$src" ]] || fail "missing $src"
[[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src"
done
for t in "${TIMERS[@]}"; do
for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done
done
if command -v systemd-analyze >/dev/null; then
verify_dir="$(mktemp -d)"
trap 'rm -rf -- "$verify_dir"' EXIT
cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/"
# ExecStart paths do not exist before installation; only syntax and keys are checked.
out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)"
[[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; }
echo " units: systemd-analyze verify clean"
fi
echo "== plan"
for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done
for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done
echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)"
echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check"
[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; }
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first"
echo "== install"
for pair in "${SCRIPTS[@]}"; do
install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}"
done
install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR"
install -d -o root -g root -m 0700 "$WORK_ROOT"
for t in "${TIMERS[@]}"; do
install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/"
done
systemctl daemon-reload
for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done
echo "== first runs"
for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do
if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi
done
echo "== metrics"
grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet"
systemctl list-timers 'platform-*' --no-pager
echo "HOST CHECKS INSTALLED"
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Write the notAfter time of the k3s API server (6443) and kubelet (10250)
# serving certificates as Prometheus textfile metrics. No root needed.
# k3s renews these only when it starts and they are within 120 days of expiry
# (docs.k3s.io cli/certificate), and the kubelet exposes no certificate metric,
# so nothing else in the cluster watches them.
set -Eeuo pipefail
# shellcheck source=../lib/host-backup.sh
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
lines=""
for target in apiserver:6443 kubelet:10250; do
name="${target%%:*}" port="${target##*:}"
end="$(echo | timeout 10 openssl s_client -connect "127.0.0.1:$port" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//')" || true
[[ -n "$end" ]] || hb_fail "could not read the $name certificate on 127.0.0.1:$port"
lines+="platform_k3s_certificate_not_after_seconds{certificate=\"$name\"} $(date -d "$end" +%s)"$'\n'
hb_log "$name certificate notAfter $end"
done
hb_write_metrics platform_k3s_certificates <<PROM
# HELP platform_k3s_certificate_not_after_seconds notAfter of a k3s serving certificate (Unix time).
# TYPE platform_k3s_certificate_not_after_seconds gauge
${lines}# HELP platform_k3s_certificate_check_timestamp_seconds Unix time of the last certificate check.
# TYPE platform_k3s_certificate_check_timestamp_seconds gauge
platform_k3s_certificate_check_timestamp_seconds $(date +%s)
PROM
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Daily drift check against origin/main (not the working tree of the checkout),
# so local edits or a stale checkout do not hide or invent drift.
# Run by platform-live-drift-check.timer as the desktop user.
set -Eeuo pipefail
readonly CHECKOUT="${PLATFORM_CHECKOUT:-/home/donghyeon/workspace/platform}"
readonly TEXTFILE=/var/lib/node_exporter/textfile_collector/platform_live_drift.prom
git -C "$CHECKOUT" fetch --quiet origin main
work="$(mktemp -d /tmp/platform-drift-main.XXXXXX)"
trap 'rm -rf -- "$work"' EXIT
git -C "$CHECKOUT" archive origin/main | tar -x -C "$work"
printf 'origin/main %s\n' "$(git -C "$CHECKOUT" rev-parse --short origin/main)"
args=(--accepted "$work/scripts/validate/live-drift-accepted.txt")
[[ -d "$(dirname "$TEXTFILE")" ]] && args+=(--textfile "$TEXTFILE")
bash "$work/scripts/validate/live-drift-check.sh" "${args[@]}"
@@ -0,0 +1,151 @@
#!/usr/bin/env bash
# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml.
# Needs docker and python3 (PyYAML). No cluster access.
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# Same pinned image as test-platform-observability-rules.sh.
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
work="$(mktemp -d)"
trap 'rm -rf -- "$work"' EXIT
python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY'
import sys, yaml
rule = yaml.safe_load(open(sys.argv[1]))
yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False)
PY
# promtool evaluates from Unix time 0, so "now" in each test is the eval_time.
cat >"$work/tests.yaml" <<'EOF'
rule_files:
- rules.yaml
evaluation_interval: 1m
tests:
# k3s datastore: last success at t=0; stale after 2 days, alert after 1h more.
- interval: 1m
input_series:
- series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}'
values: '0x5500'
- series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}'
values: '0x5500'
- series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}'
values: '0x5500'
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
values: '0x5500'
alert_rule_test:
- eval_time: 24h
alertname: PlatformK3sDatastoreBackupTooOld
exp_alerts: []
- eval_time: 50h
alertname: PlatformK3sDatastoreBackupTooOld
exp_alerts:
- exp_labels: {severity: warning}
exp_annotations:
summary: k3s datastore has not been backed up for more than 2 days
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
- eval_time: 90h
alertname: PlatformGiteaBackupTooOld
exp_alerts: []
- eval_time: 90h
alertname: PlatformHostBackupMetricsAbsent
exp_alerts: []
- interval: 1m
input_series:
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
values: '0x11700'
alert_rule_test:
- eval_time: 194h
alertname: PlatformGiteaBackupTooOld
exp_alerts:
- exp_labels: {severity: warning}
exp_annotations:
summary: Gitea has not been backed up for more than 8 days
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
# A failed run is reported with its backup label.
- interval: 1m
input_series:
- series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}'
values: '0x30'
- series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}'
values: '1x30'
alert_rule_test:
- eval_time: 15m
alertname: PlatformHostBackupFailed
exp_alerts:
- exp_labels: {severity: warning, backup: gitea}
exp_annotations:
summary: "The last gitea backup run failed"
description: "Check `journalctl -u platform-gitea-backup` on desktop."
# Metrics never written (timers or textfile collector missing).
- interval: 1h
input_series:
- series: 'up{job="prometheus-node-exporter"}'
values: '1x30'
alert_rule_test:
- eval_time: 25h
alertname: PlatformHostBackupMetricsAbsent
exp_alerts:
- exp_labels: {severity: warning, backup: k3s-datastore}
exp_annotations:
summary: Host backup metrics are missing
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
- exp_labels: {severity: warning, backup: gitea}
exp_annotations:
summary: Host backup metrics are missing
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
# Drift for 7h fires; drift 0 does not.
- interval: 1m
input_series:
- series: 'platform_live_drift_objects'
values: '3x500'
- series: 'platform_live_drift_last_run_timestamp_seconds'
values: '0x500'
- series: 'platform_live_drift_check_success'
values: '1x500'
alert_rule_test:
- eval_time: 5h
alertname: PlatformLiveDriftDetected
exp_alerts: []
- eval_time: 7h
alertname: PlatformLiveDriftDetected
exp_alerts:
- exp_labels: {severity: warning}
exp_annotations:
summary: "3 live objects differ from platform-core origin/main"
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
- eval_time: 7h
alertname: PlatformLiveDriftCheckStale
exp_alerts: []
# Certificates: 50 days left -> warning only; 10 days left -> warning and critical.
- interval: 1m
input_series:
- series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}'
values: '4320000x200'
- series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}'
values: '864000x200'
- series: 'platform_k3s_certificate_check_timestamp_seconds'
values: '0x200'
alert_rule_test:
- eval_time: 2h
alertname: PlatformK3sServingCertificateExpiry
exp_alerts:
- exp_labels: {severity: warning, certificate: apiserver}
exp_annotations:
summary: "k3s apiserver serving certificate expires in less than 100 days"
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
- exp_labels: {severity: warning, certificate: kubelet}
exp_annotations:
summary: "k3s kubelet serving certificate expires in less than 100 days"
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
- exp_labels: {severity: critical, certificate: kubelet}
exp_annotations:
summary: "k3s kubelet serving certificate expires in less than 14 days"
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
- eval_time: 2h
alertname: PlatformK3sCertificateCheckStale
exp_alerts: []
EOF
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml
echo "PLATFORM HOST CHECK ALERTS PASS"
@@ -34,6 +34,11 @@ kubeProxy:
defaultRules:
disabled:
Watchdog: true
# k3s kubelet exposes no kubelet_certificate_manager_* metrics, so these two
# can never fire. Serving certificates are watched by platform-host-checks
# (PlatformK3sServingCertificateExpiry) instead. Review 2026-11-11 A11.
KubeletClientCertificateExpiration: true
KubeletServerCertificateExpiration: true
rules:
etcd: false
kubeControllerManager: false
+83
View File
@@ -0,0 +1,83 @@
# 호스트 백업과 점검
desktop 호스트에서 systemd 타이머로 도는 백업과 점검이다. 결과는 node-exporter textfile 수집기
(`/var/lib/node_exporter/textfile_collector/*.prom`)로 Prometheus에 올라가고, 이 폴더의 PrometheusRule이
경보를 낸다. 근거와 결정은 `docs/reviews/2026-11-11-k3s-maturity-review.md` A1·A11·A13·A14, 8절 결정 1·6이다.
## 무엇을 어디에 얼마나
| 대상 | 방법 | 주기 | 보관 | 위치 |
| --- | --- | --- | --- | --- |
| k3s datastore + `server/token` | `sqlite3 .backup` → tar.gz + sha256 | 매일 03:30 KST | 30일 | AIStor `platform-host-backups/k3s-datastore/` (HDD sdb3) |
| Gitea (저장소·패키지·첨부·Actions·`app.ini`) | 라이브 파드에서 `gitea dump --skip-db` | 일요일 04:00 KST | 56일(8주) | AIStor `platform-host-backups/gitea/` |
| platform-postgres (Gitea·Keycloak·tech-log DB) | CloudNativePG base backup + WAL | 매일 03:00 KST | 30일 | AIStor `platform-postgres-backups/` (W1, `services/platform-postgres-backup`) |
| AIStor 데이터(Loki·Tempo) | **백업하지 않는다** | — | — | 8절 결정 6: 잃어도 되는 관측 데이터. 디스크를 잃으면 로그·추적 기록만 사라진다 |
| Prometheus·Grafana·Alertmanager PV | 백업하지 않는다 | — | — | 설정은 이 저장소가 정본이고, 지표는 다시 쌓인다 |
- Gitea DB는 dump에 없다(`--skip-db`). Gitea를 복원하려면 CNPG 복원이 먼저다(`docs/runbooks/gitea-restore.md`).
- k3s 백업에는 `token`이 들어 있다. token과 datastore가 같이 있으면 클러스터의 모든 Secret을 풀 수 있다.
Gitea dump의 `app.ini`에도 비밀 값이 있다. 그래서 둘 다 AIStor의 전용 사용자만 읽고 쓴다.
- AIStor는 같은 desktop의 HDD다. SSD가 죽으면 남지만 desktop 자체를 잃으면 같이 잃는다.
## off-host 사본 (준비만, 기본 꺼짐)
위치는 소유자가 정한다(8절 결정 1). `/etc/platform-backup/aistor.env`에 두 줄을 넣으면 k3s datastore
백업이 그 공개 키로 암호화한 사본을 `rclone`으로 보낸다.
```bash
OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc # 공개 키. 개인 키는 desktop에 두지 않는다
OFFHOST_RCLONE_TARGET=<rclone remote>:<path> # rclone 설치·설정 필요
```
지표 `platform_backup_offhost_copy{backup="k3s-datastore"}`가 1이 된다. Gitea dump의 off-host 사본은 아직 없다.
## 점검
| 타이머 | 사용자 | 하는 일 | 지표 |
| --- | --- | --- | --- |
| `platform-k3s-cert-expiry` | donghyeon | 6443·10250 serving 인증서 notAfter | `platform_k3s_certificate_not_after_seconds` |
| `platform-live-drift-check` | donghyeon | origin/main 렌더 → `kubectl diff`(서버 dry-run) | `platform_live_drift_objects` |
## 경보 (`prometheus-rules.yaml`)
| 경보 | 조건 |
| --- | --- |
| `PlatformK3sDatastoreBackupTooOld` | 마지막 성공이 2일보다 오래됨 |
| `PlatformGiteaBackupTooOld` | 8일보다 오래됨 |
| `PlatformHostBackupFailed` | 마지막 실행 실패 |
| `PlatformHostBackupMetricsAbsent` | 지표가 하루 동안 없음(타이머나 textfile 수집기가 안 돎) |
| `PlatformLiveDriftDetected` | drift가 6시간 넘게 남음 |
| `PlatformLiveDriftCheckStale` | drift 검사가 2일 넘게 완료되지 않음 |
| `PlatformK3sServingCertificateExpiry` | 100일 미만 warning, 14일 미만 critical |
| `PlatformK3sCertificateCheckStale` | 인증서 점검이 2일 넘게 안 돎 |
`scripts/validate/test-platform-host-checks-alerts.sh`가 promtool 단위 시험으로 각 경보가 울려야 할 때 울리고
아닐 때 안 울리는지 본다. 같은 PR에서 kube-prometheus-stack의 `KubeletClientCertificateExpiration`·
`KubeletServerCertificateExpiration`을 껐다. k3s kubelet은 그 지표를 내지 않아 울릴 수 없었다.
## 적용 (소유자, 순서대로)
전제: node-exporter textfile 수집기(W1, `services/observability/node-exporter/values/home.yaml`의
`--collector.textfile.directory=/host/root/var/lib/node_exporter/textfile_collector`)가 들어간 main checkout.
```bash
cd /home/donghyeon/workspace/platform && git pull --ff-only
sudo bash scripts/bootstrap/create-host-backup-credentials.sh # bucket·정책·사용자·/etc/platform-backup/aistor.env (APPLY 입력)
bash scripts/bootstrap/install-host-checks.sh --check
sudo bash scripts/bootstrap/install-host-checks.sh --execute # 타이머 설치·켜기·첫 실행
kubectl apply -k services/platform-host-checks # 경보 규칙
sudo bash scripts/validate/k3s-datastore-restore-drill.sh # 첫 백업으로 복원 리허설
sudo bash scripts/validate/gitea-backup-restore-drill.sh
```
`install-host-checks.sh --execute` 마지막에 `platform_backup_last_run_success{backup="k3s-datastore"} 1`과
`{backup="gitea"} 1`이 보여야 한다. 리허설 둘은 각각 `K3S DATASTORE RESTORE DRILL PASS`, `GITEA RESTORE DRILL PASS`로 끝나야 한다.
## 복원 리허설
- k3s: 백업을 내려받아 sha256을 맞춘 뒤, 운영과 같은 버전의 k3s를 **네트워크 없는** 일회용 컨테이너로
띄워 API가 뜨는지, Secret 수가 라이브와 비슷한지(±5), 모든 Secret이 복호화되는지 본다.
`scripts/validate/test-k3s-datastore-restore-drill.sh`는 운영 데이터 없이 같은 경로를 시험한다(새 k3s에
시험 Secret 셋 → 암호화돼 저장된 것을 확인 → 복원).
- Gitea: dump를 풀어 `app.ini`(INTERNAL_TOKEN·SECRET_KEY), 모든 저장소의 `git fsck`, 라이브와 저장소 수를 본다.
- 분기마다 한 번 돌린다. 성공하면 `platform_backup_restore_drill_last_success_timestamp_seconds`가 갱신된다.
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- prometheus-rules.yaml
@@ -0,0 +1,88 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: platform-host-checks
namespace: observability
labels:
observability.hyeonworks.com/instance: home
app.kubernetes.io/part-of: platform
spec:
groups:
- name: platform-host-backups
rules:
- alert: PlatformK3sDatastoreBackupTooOld
expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}) > 2 * 86400
for: 1h
labels:
severity: warning
annotations:
summary: k3s datastore has not been backed up for more than 2 days
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
- alert: PlatformGiteaBackupTooOld
expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="gitea"}) > 8 * 86400
for: 1h
labels:
severity: warning
annotations:
summary: Gitea has not been backed up for more than 8 days
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
- alert: PlatformHostBackupFailed
expr: max by (backup) (platform_backup_last_run_success) == 0
for: 10m
labels:
severity: warning
annotations:
summary: "The last {{ $labels.backup }} backup run failed"
description: "Check `journalctl -u platform-{{ $labels.backup }}-backup` on desktop."
- alert: PlatformHostBackupMetricsAbsent
expr: absent(platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}) or absent(platform_backup_last_run_timestamp_seconds{backup="gitea"})
for: 1d
labels:
severity: warning
annotations:
summary: Host backup metrics are missing
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
- name: platform-host-drift
rules:
- alert: PlatformLiveDriftDetected
expr: max(platform_live_drift_objects) > 0
for: 6h
labels:
severity: warning
annotations:
summary: "{{ $value }} live objects differ from platform-core origin/main"
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
- alert: PlatformLiveDriftCheckStale
expr: time() - max(platform_live_drift_last_run_timestamp_seconds) > 2 * 86400 or max(platform_live_drift_check_success) == 0
for: 1h
labels:
severity: warning
annotations:
summary: The daily live drift check has not completed
description: Check `journalctl -u platform-live-drift-check` on desktop.
- name: platform-k3s-certificates
rules:
- alert: PlatformK3sServingCertificateExpiry
expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 100 * 86400
for: 1h
labels:
severity: warning
annotations:
summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 100 days"
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
- alert: PlatformK3sServingCertificateExpiry
expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 14 * 86400
for: 10m
labels:
severity: critical
annotations:
summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 14 days"
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
- alert: PlatformK3sCertificateCheckStale
expr: time() - max(platform_k3s_certificate_check_timestamp_seconds) > 2 * 86400
for: 1h
labels:
severity: warning
annotations:
summary: The daily k3s certificate expiry check has not run
description: Check `journalctl -u platform-k3s-cert-expiry` on desktop.