feat(observability): alert on host backups, drift and k3s serving certs
Backups, the daily drift check and the k3s serving certificates have no alert. k3s renews certificates only when it starts within 120 days of expiry, and the two kube-prometheus-stack kubelet certificate alerts can never fire because the k3s kubelet exports no kubelet_certificate_manager_* metric. Add a PrometheusRule (platform-host-checks) for backup age, failed runs, missing metrics, drift left for 6h, a stale drift check, and serving certificate expiry (warning under 100 days, critical under 14). Add the two remaining host timers (certificate expiry via openssl on 6443 and 10250; the drift check against origin/main) and an installer that copies the scripts to a root-owned path, creates the textfile and work directories and enables all four timers. Disable the two dead kubelet certificate rules in kube-prometheus-stack. Risk: the metrics need the node-exporter textfile collector, a one-line change in W1's node-exporter values that was asked of W1. Until then PlatformHostBackupMetricsAbsent fires after a day, which is the point. Verification (desktop): - test-platform-host-checks-alerts.sh: promtool check and unit tests pass; with the k3s threshold changed from 2 to 5 days they fail. - render-observability-core.sh passes; the render has no Kubelet* certificate alerts and still has both KubeClientCertificateExpiration. - k3s-cert-expiry.sh as the desktop user reads both certificates. - install-host-checks.sh --check: systemd-analyze verify is clean.
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the host backup and check timers (infrastructure/host/backup).
|
||||
#
|
||||
# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing
|
||||
# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs
|
||||
#
|
||||
# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root
|
||||
# timers never execute files the desktop user can edit.
|
||||
set -Eeuo pipefail
|
||||
umask 022
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
readonly LIB=/usr/local/lib/platform-backup
|
||||
readonly UNIT_DIR=/etc/systemd/system
|
||||
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
|
||||
readonly WORK_ROOT=/var/lib/platform-backup/work
|
||||
readonly USER_NAME=donghyeon
|
||||
readonly -a SCRIPTS=(
|
||||
scripts/lib/host-backup.sh:lib/host-backup.sh
|
||||
scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh
|
||||
scripts/backup/gitea-backup.sh:backup/gitea-backup.sh
|
||||
scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh
|
||||
scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh
|
||||
services/platform-host-checks/README.md:README.md
|
||||
)
|
||||
readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry)
|
||||
|
||||
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
mode="${1:-}"
|
||||
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; }
|
||||
|
||||
echo "== check"
|
||||
for pair in "${SCRIPTS[@]}"; do
|
||||
src="${pair%%:*}"
|
||||
[[ -f "$ROOT/$src" ]] || fail "missing $src"
|
||||
[[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src"
|
||||
done
|
||||
for t in "${TIMERS[@]}"; do
|
||||
for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done
|
||||
done
|
||||
if command -v systemd-analyze >/dev/null; then
|
||||
verify_dir="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$verify_dir"' EXIT
|
||||
cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/"
|
||||
# ExecStart paths do not exist before installation; only syntax and keys are checked.
|
||||
out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)"
|
||||
[[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; }
|
||||
echo " units: systemd-analyze verify clean"
|
||||
fi
|
||||
echo "== plan"
|
||||
for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done
|
||||
for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done
|
||||
echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)"
|
||||
echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check"
|
||||
[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; }
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
|
||||
[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first"
|
||||
echo "== install"
|
||||
for pair in "${SCRIPTS[@]}"; do
|
||||
install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}"
|
||||
done
|
||||
install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR"
|
||||
install -d -o root -g root -m 0700 "$WORK_ROOT"
|
||||
for t in "${TIMERS[@]}"; do
|
||||
install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/"
|
||||
done
|
||||
systemctl daemon-reload
|
||||
for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done
|
||||
|
||||
echo "== first runs"
|
||||
for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do
|
||||
if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi
|
||||
done
|
||||
echo "== metrics"
|
||||
grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet"
|
||||
systemctl list-timers 'platform-*' --no-pager
|
||||
echo "HOST CHECKS INSTALLED"
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# Write the notAfter time of the k3s API server (6443) and kubelet (10250)
|
||||
# serving certificates as Prometheus textfile metrics. No root needed.
|
||||
# k3s renews these only when it starts and they are within 120 days of expiry
|
||||
# (docs.k3s.io cli/certificate), and the kubelet exposes no certificate metric,
|
||||
# so nothing else in the cluster watches them.
|
||||
set -Eeuo pipefail
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
|
||||
|
||||
lines=""
|
||||
for target in apiserver:6443 kubelet:10250; do
|
||||
name="${target%%:*}" port="${target##*:}"
|
||||
end="$(echo | timeout 10 openssl s_client -connect "127.0.0.1:$port" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//')" || true
|
||||
[[ -n "$end" ]] || hb_fail "could not read the $name certificate on 127.0.0.1:$port"
|
||||
lines+="platform_k3s_certificate_not_after_seconds{certificate=\"$name\"} $(date -d "$end" +%s)"$'\n'
|
||||
hb_log "$name certificate notAfter $end"
|
||||
done
|
||||
hb_write_metrics platform_k3s_certificates <<PROM
|
||||
# HELP platform_k3s_certificate_not_after_seconds notAfter of a k3s serving certificate (Unix time).
|
||||
# TYPE platform_k3s_certificate_not_after_seconds gauge
|
||||
${lines}# HELP platform_k3s_certificate_check_timestamp_seconds Unix time of the last certificate check.
|
||||
# TYPE platform_k3s_certificate_check_timestamp_seconds gauge
|
||||
platform_k3s_certificate_check_timestamp_seconds $(date +%s)
|
||||
PROM
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Daily drift check against origin/main (not the working tree of the checkout),
|
||||
# so local edits or a stale checkout do not hide or invent drift.
|
||||
# Run by platform-live-drift-check.timer as the desktop user.
|
||||
set -Eeuo pipefail
|
||||
readonly CHECKOUT="${PLATFORM_CHECKOUT:-/home/donghyeon/workspace/platform}"
|
||||
readonly TEXTFILE=/var/lib/node_exporter/textfile_collector/platform_live_drift.prom
|
||||
|
||||
git -C "$CHECKOUT" fetch --quiet origin main
|
||||
work="$(mktemp -d /tmp/platform-drift-main.XXXXXX)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
git -C "$CHECKOUT" archive origin/main | tar -x -C "$work"
|
||||
printf 'origin/main %s\n' "$(git -C "$CHECKOUT" rev-parse --short origin/main)"
|
||||
args=(--accepted "$work/scripts/validate/live-drift-accepted.txt")
|
||||
[[ -d "$(dirname "$TEXTFILE")" ]] && args+=(--textfile "$TEXTFILE")
|
||||
bash "$work/scripts/validate/live-drift-check.sh" "${args[@]}"
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml.
|
||||
# Needs docker and python3 (PyYAML). No cluster access.
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
# Same pinned image as test-platform-observability-rules.sh.
|
||||
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
||||
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY'
|
||||
import sys, yaml
|
||||
rule = yaml.safe_load(open(sys.argv[1]))
|
||||
yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False)
|
||||
PY
|
||||
|
||||
# promtool evaluates from Unix time 0, so "now" in each test is the eval_time.
|
||||
cat >"$work/tests.yaml" <<'EOF'
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
# k3s datastore: last success at t=0; stale after 2 days, alert after 1h more.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x5500'
|
||||
alert_rule_test:
|
||||
- eval_time: 24h
|
||||
alertname: PlatformK3sDatastoreBackupTooOld
|
||||
exp_alerts: []
|
||||
- eval_time: 50h
|
||||
alertname: PlatformK3sDatastoreBackupTooOld
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: k3s datastore has not been backed up for more than 2 days
|
||||
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
|
||||
- eval_time: 90h
|
||||
alertname: PlatformGiteaBackupTooOld
|
||||
exp_alerts: []
|
||||
- eval_time: 90h
|
||||
alertname: PlatformHostBackupMetricsAbsent
|
||||
exp_alerts: []
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x11700'
|
||||
alert_rule_test:
|
||||
- eval_time: 194h
|
||||
alertname: PlatformGiteaBackupTooOld
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: Gitea has not been backed up for more than 8 days
|
||||
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
|
||||
# A failed run is reported with its backup label.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}'
|
||||
values: '0x30'
|
||||
- series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 15m
|
||||
alertname: PlatformHostBackupFailed
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, backup: gitea}
|
||||
exp_annotations:
|
||||
summary: "The last gitea backup run failed"
|
||||
description: "Check `journalctl -u platform-gitea-backup` on desktop."
|
||||
# Metrics never written (timers or textfile collector missing).
|
||||
- interval: 1h
|
||||
input_series:
|
||||
- series: 'up{job="prometheus-node-exporter"}'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 25h
|
||||
alertname: PlatformHostBackupMetricsAbsent
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, backup: k3s-datastore}
|
||||
exp_annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
- exp_labels: {severity: warning, backup: gitea}
|
||||
exp_annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
# Drift for 7h fires; drift 0 does not.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_live_drift_objects'
|
||||
values: '3x500'
|
||||
- series: 'platform_live_drift_last_run_timestamp_seconds'
|
||||
values: '0x500'
|
||||
- series: 'platform_live_drift_check_success'
|
||||
values: '1x500'
|
||||
alert_rule_test:
|
||||
- eval_time: 5h
|
||||
alertname: PlatformLiveDriftDetected
|
||||
exp_alerts: []
|
||||
- eval_time: 7h
|
||||
alertname: PlatformLiveDriftDetected
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: "3 live objects differ from platform-core origin/main"
|
||||
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
|
||||
- eval_time: 7h
|
||||
alertname: PlatformLiveDriftCheckStale
|
||||
exp_alerts: []
|
||||
# Certificates: 50 days left -> warning only; 10 days left -> warning and critical.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}'
|
||||
values: '4320000x200'
|
||||
- series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}'
|
||||
values: '864000x200'
|
||||
- series: 'platform_k3s_certificate_check_timestamp_seconds'
|
||||
values: '0x200'
|
||||
alert_rule_test:
|
||||
- eval_time: 2h
|
||||
alertname: PlatformK3sServingCertificateExpiry
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, certificate: apiserver}
|
||||
exp_annotations:
|
||||
summary: "k3s apiserver serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- exp_labels: {severity: warning, certificate: kubelet}
|
||||
exp_annotations:
|
||||
summary: "k3s kubelet serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- exp_labels: {severity: critical, certificate: kubelet}
|
||||
exp_annotations:
|
||||
summary: "k3s kubelet serving certificate expires in less than 14 days"
|
||||
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
|
||||
- eval_time: 2h
|
||||
alertname: PlatformK3sCertificateCheckStale
|
||||
exp_alerts: []
|
||||
EOF
|
||||
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml
|
||||
echo "PLATFORM HOST CHECK ALERTS PASS"
|
||||
Reference in New Issue
Block a user