#!/usr/bin/env bash # promtool unit tests for services/platform-host-checks/prometheus-rules.yaml. # Needs docker and python3 (PyYAML). No cluster access. set -Eeuo pipefail readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" # Same pinned image as test-platform-observability-rules.sh. readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b" work="$(mktemp -d)" trap 'rm -rf -- "$work"' EXIT python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY' import sys, yaml rule = yaml.safe_load(open(sys.argv[1])) yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False) PY # promtool evaluates from Unix time 0, so "now" in each test is the eval_time. cat >"$work/tests.yaml" <<'EOF' rule_files: - rules.yaml evaluation_interval: 1m tests: # k3s datastore: last success at t=0; stale after 2 days, alert after 1h more. - interval: 1m input_series: - series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}' values: '0x5500' - series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}' values: '0x5500' - series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}' values: '0x5500' - series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}' values: '0x5500' alert_rule_test: - eval_time: 24h alertname: PlatformK3sDatastoreBackupTooOld exp_alerts: [] - eval_time: 50h alertname: PlatformK3sDatastoreBackupTooOld exp_alerts: - exp_labels: {severity: warning} exp_annotations: summary: k3s datastore has not been backed up for more than 2 days description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST. - eval_time: 90h alertname: PlatformGiteaBackupTooOld exp_alerts: [] - eval_time: 90h alertname: PlatformHostBackupMetricsAbsent exp_alerts: [] - interval: 1m input_series: - series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}' values: '0x11700' alert_rule_test: - eval_time: 194h alertname: PlatformGiteaBackupTooOld exp_alerts: - exp_labels: {severity: warning} exp_annotations: summary: Gitea has not been backed up for more than 8 days description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST. # A failed run is reported with its backup label. - interval: 1m input_series: - series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}' values: '0x30' - series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}' values: '1x30' alert_rule_test: - eval_time: 15m alertname: PlatformHostBackupFailed exp_alerts: - exp_labels: {severity: warning, backup: gitea} exp_annotations: summary: "The last gitea backup run failed" description: "Check `journalctl -u platform-gitea-backup` on desktop." # Metrics never written (timers or textfile collector missing). - interval: 1h input_series: - series: 'up{job="prometheus-node-exporter"}' values: '1x30' alert_rule_test: - eval_time: 25h alertname: PlatformHostBackupMetricsAbsent exp_alerts: - exp_labels: {severity: warning, backup: k3s-datastore} exp_annotations: summary: Host backup metrics are missing description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. - exp_labels: {severity: warning, backup: gitea} exp_annotations: summary: Host backup metrics are missing description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. # Drift for 7h fires; drift 0 does not. - interval: 1m input_series: - series: 'platform_live_drift_objects' values: '3x500' - series: 'platform_live_drift_last_run_timestamp_seconds' values: '0x500' - series: 'platform_live_drift_check_success' values: '1x500' alert_rule_test: - eval_time: 5h alertname: PlatformLiveDriftDetected exp_alerts: [] - eval_time: 7h alertname: PlatformLiveDriftDetected exp_alerts: - exp_labels: {severity: warning} exp_annotations: summary: "3 live objects differ from platform-core origin/main" description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them. - eval_time: 7h alertname: PlatformLiveDriftCheckStale exp_alerts: [] # Certificates: 50 days left -> warning only; 10 days left -> warning and critical. - interval: 1m input_series: - series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}' values: '4320000x200' - series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}' values: '864000x200' - series: 'platform_k3s_certificate_check_timestamp_seconds' values: '0x200' alert_rule_test: - eval_time: 2h alertname: PlatformK3sServingCertificateExpiry exp_alerts: - exp_labels: {severity: warning, certificate: apiserver} exp_annotations: summary: "k3s apiserver serving certificate expires in less than 100 days" description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). - exp_labels: {severity: warning, certificate: kubelet} exp_annotations: summary: "k3s kubelet serving certificate expires in less than 100 days" description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). - exp_labels: {severity: critical, certificate: kubelet} exp_annotations: summary: "k3s kubelet serving certificate expires in less than 14 days" description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`. - eval_time: 2h alertname: PlatformK3sCertificateCheckStale exp_alerts: [] EOF docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml echo "PLATFORM HOST CHECK ALERTS PASS"