apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-host-checks namespace: observability labels: observability.hyeonworks.com/instance: home app.kubernetes.io/part-of: platform spec: groups: - name: platform-host-backups rules: - alert: PlatformK3sDatastoreBackupTooOld expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}) > 2 * 86400 for: 1h labels: severity: warning annotations: summary: k3s datastore has not been backed up for more than 2 days description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST. - alert: PlatformGiteaBackupTooOld expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="gitea"}) > 8 * 86400 for: 1h labels: severity: warning annotations: summary: Gitea has not been backed up for more than 8 days description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST. - alert: PlatformHostBackupFailed expr: max by (backup) (platform_backup_last_run_success) == 0 for: 10m labels: severity: warning annotations: summary: "The last {{ $labels.backup }} backup run failed" description: "Check `journalctl -u platform-{{ $labels.backup }}-backup` on desktop." - alert: PlatformHostBackupMetricsAbsent expr: absent(platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}) or absent(platform_backup_last_run_timestamp_seconds{backup="gitea"}) for: 1d labels: severity: warning annotations: summary: Host backup metrics are missing description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running. - name: platform-host-drift rules: - alert: PlatformLiveDriftDetected expr: max(platform_live_drift_objects) > 0 for: 6h labels: severity: warning annotations: summary: "{{ $value }} live objects differ from platform-core origin/main" description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them. - alert: PlatformLiveDriftCheckStale expr: time() - max(platform_live_drift_last_run_timestamp_seconds) > 2 * 86400 or max(platform_live_drift_check_success) == 0 for: 1h labels: severity: warning annotations: summary: The daily live drift check has not completed description: Check `journalctl -u platform-live-drift-check` on desktop. - name: platform-k3s-certificates rules: - alert: PlatformK3sServingCertificateExpiry expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 100 * 86400 for: 1h labels: severity: warning annotations: summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 100 days" description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks). - alert: PlatformK3sServingCertificateExpiry expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 14 * 86400 for: 10m labels: severity: critical annotations: summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 14 days" description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`. - alert: PlatformK3sCertificateCheckStale expr: time() - max(platform_k3s_certificate_check_timestamp_seconds) > 2 * 86400 for: 1h labels: severity: warning annotations: summary: The daily k3s certificate expiry check has not run description: Check `journalctl -u platform-k3s-cert-expiry` on desktop.