feat(observability): alert on host backups, drift and k3s serving certs
Backups, the daily drift check and the k3s serving certificates have no alert. k3s renews certificates only when it starts within 120 days of expiry, and the two kube-prometheus-stack kubelet certificate alerts can never fire because the k3s kubelet exports no kubelet_certificate_manager_* metric. Add a PrometheusRule (platform-host-checks) for backup age, failed runs, missing metrics, drift left for 6h, a stale drift check, and serving certificate expiry (warning under 100 days, critical under 14). Add the two remaining host timers (certificate expiry via openssl on 6443 and 10250; the drift check against origin/main) and an installer that copies the scripts to a root-owned path, creates the textfile and work directories and enables all four timers. Disable the two dead kubelet certificate rules in kube-prometheus-stack. Risk: the metrics need the node-exporter textfile collector, a one-line change in W1's node-exporter values that was asked of W1. Until then PlatformHostBackupMetricsAbsent fires after a day, which is the point. Verification (desktop): - test-platform-host-checks-alerts.sh: promtool check and unit tests pass; with the k3s threshold changed from 2 to 5 days they fail. - render-observability-core.sh passes; the render has no Kubelet* certificate alerts and still has both KubeClientCertificateExpiration. - k3s-cert-expiry.sh as the desktop user reads both certificates. - install-host-checks.sh --check: systemd-analyze verify is clean.
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml.
|
||||
# Needs docker and python3 (PyYAML). No cluster access.
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
# Same pinned image as test-platform-observability-rules.sh.
|
||||
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
||||
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY'
|
||||
import sys, yaml
|
||||
rule = yaml.safe_load(open(sys.argv[1]))
|
||||
yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False)
|
||||
PY
|
||||
|
||||
# promtool evaluates from Unix time 0, so "now" in each test is the eval_time.
|
||||
cat >"$work/tests.yaml" <<'EOF'
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
# k3s datastore: last success at t=0; stale after 2 days, alert after 1h more.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x5500'
|
||||
alert_rule_test:
|
||||
- eval_time: 24h
|
||||
alertname: PlatformK3sDatastoreBackupTooOld
|
||||
exp_alerts: []
|
||||
- eval_time: 50h
|
||||
alertname: PlatformK3sDatastoreBackupTooOld
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: k3s datastore has not been backed up for more than 2 days
|
||||
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
|
||||
- eval_time: 90h
|
||||
alertname: PlatformGiteaBackupTooOld
|
||||
exp_alerts: []
|
||||
- eval_time: 90h
|
||||
alertname: PlatformHostBackupMetricsAbsent
|
||||
exp_alerts: []
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x11700'
|
||||
alert_rule_test:
|
||||
- eval_time: 194h
|
||||
alertname: PlatformGiteaBackupTooOld
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: Gitea has not been backed up for more than 8 days
|
||||
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
|
||||
# A failed run is reported with its backup label.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}'
|
||||
values: '0x30'
|
||||
- series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 15m
|
||||
alertname: PlatformHostBackupFailed
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, backup: gitea}
|
||||
exp_annotations:
|
||||
summary: "The last gitea backup run failed"
|
||||
description: "Check `journalctl -u platform-gitea-backup` on desktop."
|
||||
# Metrics never written (timers or textfile collector missing).
|
||||
- interval: 1h
|
||||
input_series:
|
||||
- series: 'up{job="prometheus-node-exporter"}'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 25h
|
||||
alertname: PlatformHostBackupMetricsAbsent
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, backup: k3s-datastore}
|
||||
exp_annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
- exp_labels: {severity: warning, backup: gitea}
|
||||
exp_annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
# Drift for 7h fires; drift 0 does not.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_live_drift_objects'
|
||||
values: '3x500'
|
||||
- series: 'platform_live_drift_last_run_timestamp_seconds'
|
||||
values: '0x500'
|
||||
- series: 'platform_live_drift_check_success'
|
||||
values: '1x500'
|
||||
alert_rule_test:
|
||||
- eval_time: 5h
|
||||
alertname: PlatformLiveDriftDetected
|
||||
exp_alerts: []
|
||||
- eval_time: 7h
|
||||
alertname: PlatformLiveDriftDetected
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: "3 live objects differ from platform-core origin/main"
|
||||
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
|
||||
- eval_time: 7h
|
||||
alertname: PlatformLiveDriftCheckStale
|
||||
exp_alerts: []
|
||||
# Certificates: 50 days left -> warning only; 10 days left -> warning and critical.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}'
|
||||
values: '4320000x200'
|
||||
- series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}'
|
||||
values: '864000x200'
|
||||
- series: 'platform_k3s_certificate_check_timestamp_seconds'
|
||||
values: '0x200'
|
||||
alert_rule_test:
|
||||
- eval_time: 2h
|
||||
alertname: PlatformK3sServingCertificateExpiry
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, certificate: apiserver}
|
||||
exp_annotations:
|
||||
summary: "k3s apiserver serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- exp_labels: {severity: warning, certificate: kubelet}
|
||||
exp_annotations:
|
||||
summary: "k3s kubelet serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- exp_labels: {severity: critical, certificate: kubelet}
|
||||
exp_annotations:
|
||||
summary: "k3s kubelet serving certificate expires in less than 14 days"
|
||||
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
|
||||
- eval_time: 2h
|
||||
alertname: PlatformK3sCertificateCheckStale
|
||||
exp_alerts: []
|
||||
EOF
|
||||
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml
|
||||
echo "PLATFORM HOST CHECK ALERTS PASS"
|
||||
Reference in New Issue
Block a user