Backups, the daily drift check and the k3s serving certificates have no alert. k3s renews certificates only when it starts within 120 days of expiry, and the two kube-prometheus-stack kubelet certificate alerts can never fire because the k3s kubelet exports no kubelet_certificate_manager_* metric. Add a PrometheusRule (platform-host-checks) for backup age, failed runs, missing metrics, drift left for 6h, a stale drift check, and serving certificate expiry (warning under 100 days, critical under 14). Add the two remaining host timers (certificate expiry via openssl on 6443 and 10250; the drift check against origin/main) and an installer that copies the scripts to a root-owned path, creates the textfile and work directories and enables all four timers. Disable the two dead kubelet certificate rules in kube-prometheus-stack. Risk: the metrics need the node-exporter textfile collector, a one-line change in W1's node-exporter values that was asked of W1. Until then PlatformHostBackupMetricsAbsent fires after a day, which is the point. Verification (desktop): - test-platform-host-checks-alerts.sh: promtool check and unit tests pass; with the k3s threshold changed from 2 to 5 days they fail. - render-observability-core.sh passes; the render has no Kubelet* certificate alerts and still has both KubeClientCertificateExpiration. - k3s-cert-expiry.sh as the desktop user reads both certificates. - install-host-checks.sh --check: systemd-analyze verify is clean.
152 lines
6.9 KiB
Bash
152 lines
6.9 KiB
Bash
#!/usr/bin/env bash
|
|
# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml.
|
|
# Needs docker and python3 (PyYAML). No cluster access.
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
# Same pinned image as test-platform-observability-rules.sh.
|
|
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
|
|
|
work="$(mktemp -d)"
|
|
trap 'rm -rf -- "$work"' EXIT
|
|
python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY'
|
|
import sys, yaml
|
|
rule = yaml.safe_load(open(sys.argv[1]))
|
|
yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False)
|
|
PY
|
|
|
|
# promtool evaluates from Unix time 0, so "now" in each test is the eval_time.
|
|
cat >"$work/tests.yaml" <<'EOF'
|
|
rule_files:
|
|
- rules.yaml
|
|
evaluation_interval: 1m
|
|
tests:
|
|
# k3s datastore: last success at t=0; stale after 2 days, alert after 1h more.
|
|
- interval: 1m
|
|
input_series:
|
|
- series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}'
|
|
values: '0x5500'
|
|
- series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}'
|
|
values: '0x5500'
|
|
- series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}'
|
|
values: '0x5500'
|
|
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
|
values: '0x5500'
|
|
alert_rule_test:
|
|
- eval_time: 24h
|
|
alertname: PlatformK3sDatastoreBackupTooOld
|
|
exp_alerts: []
|
|
- eval_time: 50h
|
|
alertname: PlatformK3sDatastoreBackupTooOld
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning}
|
|
exp_annotations:
|
|
summary: k3s datastore has not been backed up for more than 2 days
|
|
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
|
|
- eval_time: 90h
|
|
alertname: PlatformGiteaBackupTooOld
|
|
exp_alerts: []
|
|
- eval_time: 90h
|
|
alertname: PlatformHostBackupMetricsAbsent
|
|
exp_alerts: []
|
|
- interval: 1m
|
|
input_series:
|
|
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
|
values: '0x11700'
|
|
alert_rule_test:
|
|
- eval_time: 194h
|
|
alertname: PlatformGiteaBackupTooOld
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning}
|
|
exp_annotations:
|
|
summary: Gitea has not been backed up for more than 8 days
|
|
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
|
|
# A failed run is reported with its backup label.
|
|
- interval: 1m
|
|
input_series:
|
|
- series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}'
|
|
values: '0x30'
|
|
- series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}'
|
|
values: '1x30'
|
|
alert_rule_test:
|
|
- eval_time: 15m
|
|
alertname: PlatformHostBackupFailed
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning, backup: gitea}
|
|
exp_annotations:
|
|
summary: "The last gitea backup run failed"
|
|
description: "Check `journalctl -u platform-gitea-backup` on desktop."
|
|
# Metrics never written (timers or textfile collector missing).
|
|
- interval: 1h
|
|
input_series:
|
|
- series: 'up{job="prometheus-node-exporter"}'
|
|
values: '1x30'
|
|
alert_rule_test:
|
|
- eval_time: 25h
|
|
alertname: PlatformHostBackupMetricsAbsent
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning, backup: k3s-datastore}
|
|
exp_annotations:
|
|
summary: Host backup metrics are missing
|
|
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
|
- exp_labels: {severity: warning, backup: gitea}
|
|
exp_annotations:
|
|
summary: Host backup metrics are missing
|
|
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
|
# Drift for 7h fires; drift 0 does not.
|
|
- interval: 1m
|
|
input_series:
|
|
- series: 'platform_live_drift_objects'
|
|
values: '3x500'
|
|
- series: 'platform_live_drift_last_run_timestamp_seconds'
|
|
values: '0x500'
|
|
- series: 'platform_live_drift_check_success'
|
|
values: '1x500'
|
|
alert_rule_test:
|
|
- eval_time: 5h
|
|
alertname: PlatformLiveDriftDetected
|
|
exp_alerts: []
|
|
- eval_time: 7h
|
|
alertname: PlatformLiveDriftDetected
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning}
|
|
exp_annotations:
|
|
summary: "3 live objects differ from platform-core origin/main"
|
|
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
|
|
- eval_time: 7h
|
|
alertname: PlatformLiveDriftCheckStale
|
|
exp_alerts: []
|
|
# Certificates: 50 days left -> warning only; 10 days left -> warning and critical.
|
|
- interval: 1m
|
|
input_series:
|
|
- series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}'
|
|
values: '4320000x200'
|
|
- series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}'
|
|
values: '864000x200'
|
|
- series: 'platform_k3s_certificate_check_timestamp_seconds'
|
|
values: '0x200'
|
|
alert_rule_test:
|
|
- eval_time: 2h
|
|
alertname: PlatformK3sServingCertificateExpiry
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning, certificate: apiserver}
|
|
exp_annotations:
|
|
summary: "k3s apiserver serving certificate expires in less than 100 days"
|
|
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
|
- exp_labels: {severity: warning, certificate: kubelet}
|
|
exp_annotations:
|
|
summary: "k3s kubelet serving certificate expires in less than 100 days"
|
|
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
|
- exp_labels: {severity: critical, certificate: kubelet}
|
|
exp_annotations:
|
|
summary: "k3s kubelet serving certificate expires in less than 14 days"
|
|
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
|
|
- eval_time: 2h
|
|
alertname: PlatformK3sCertificateCheckStale
|
|
exp_alerts: []
|
|
EOF
|
|
|
|
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml
|
|
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml
|
|
echo "PLATFORM HOST CHECK ALERTS PASS"
|