Files
platform-core/services/observability/kube-prometheus-stack/values/home.yaml
T
DongHyeonka 012452cf78 feat(observability): alert on host backups, drift and k3s serving certs
Backups, the daily drift check and the k3s serving certificates have no
alert. k3s renews certificates only when it starts within 120 days of
expiry, and the two kube-prometheus-stack kubelet certificate alerts
can never fire because the k3s kubelet exports no
kubelet_certificate_manager_* metric.

Add a PrometheusRule (platform-host-checks) for backup age, failed
runs, missing metrics, drift left for 6h, a stale drift check, and
serving certificate expiry (warning under 100 days, critical under 14).
Add the two remaining host timers (certificate expiry via openssl on
6443 and 10250; the drift check against origin/main) and an installer
that copies the scripts to a root-owned path, creates the textfile and
work directories and enables all four timers. Disable the two dead
kubelet certificate rules in kube-prometheus-stack.

Risk: the metrics need the node-exporter textfile collector, a one-line
change in W1's node-exporter values that was asked of W1. Until then
PlatformHostBackupMetricsAbsent fires after a day, which is the point.

Verification (desktop):
- test-platform-host-checks-alerts.sh: promtool check and unit tests
  pass; with the k3s threshold changed from 2 to 5 days they fail.
- render-observability-core.sh passes; the render has no Kubelet*
  certificate alerts and still has both KubeClientCertificateExpiration.
- k3s-cert-expiry.sh as the desktop user reads both certificates.
- install-host-checks.sh --check: systemd-analyze verify is clean.
2026-11-12 01:13:02 +09:00

290 lines
6.6 KiB
YAML

commonLabels:
observability.hyeonworks.com/instance: home
crds:
enabled: false
grafana:
enabled: false
nodeExporter:
enabled: false
kubeApiServer:
enabled: true
kubelet:
enabled: true
coreDns:
enabled: true
kubeControllerManager:
enabled: false
kubeScheduler:
enabled: false
kubeEtcd:
enabled: false
kubeProxy:
enabled: false
defaultRules:
disabled:
Watchdog: true
# k3s kubelet exposes no kubelet_certificate_manager_* metrics, so these two
# can never fire. Serving certificates are watched by platform-host-checks
# (PlatformK3sServingCertificateExpiry) instead. Review 2026-11-11 A11.
KubeletClientCertificateExpiration: true
KubeletServerCertificateExpiration: true
rules:
etcd: false
kubeControllerManager: false
kubeProxy: false
kubeSchedulerAlerting: false
kubeSchedulerRecording: false
windows: false
prometheusOperator:
admissionWebhooks:
enabled: false
patch:
enabled: false
tls:
enabled: false
image:
registry: quay.io
repository: prometheus-operator/prometheus-operator
tag: v0.93.0
sha: 64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 256Mi
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
seccompProfile:
type: RuntimeDefault
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
serviceAccount:
automountServiceAccountToken: true
prometheusConfigReloader:
image:
registry: quay.io
repository: prometheus-operator/prometheus-config-reloader
tag: v0.93.0
sha: 65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
kubeStateMetrics:
enabled: true
kube-state-metrics:
image:
registry: registry.k8s.io
repository: kube-state-metrics/kube-state-metrics
tag: v2.19.1
sha: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 256Mi
securityContext:
enabled: true
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
seccompProfile:
type: RuntimeDefault
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
automountServiceAccountToken: true
collectors:
- certificatesigningrequests
- configmaps
- cronjobs
- daemonsets
- deployments
- endpointslices
- horizontalpodautoscalers
- ingresses
- jobs
- leases
- limitranges
- mutatingwebhookconfigurations
- namespaces
- networkpolicies
- nodes
- persistentvolumeclaims
- persistentvolumes
- poddisruptionbudgets
- pods
- replicasets
- replicationcontrollers
- resourcequotas
- services
- statefulsets
- storageclasses
- validatingwebhookconfigurations
- volumeattachments
prometheus:
monitor:
additionalLabels:
observability.hyeonworks.com/instance: home
alertmanager:
enabled: true
config:
global:
resolve_timeout: 5m
route:
group_by:
- namespace
group_wait: 30s
group_interval: 5m
repeat_interval: 12h
receiver: "null"
routes: []
receivers:
- name: "null"
serviceAccount:
automountServiceAccountToken: false
alertmanagerSpec:
replicas: 1
retention: 120h
automountServiceAccountToken: false
alertmanagerConfiguration:
name: platform-alertmanager
image:
registry: quay.io
repository: prometheus/alertmanager
tag: v0.33.1
sha: a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 2000
fsGroup: 2000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
persistentVolumeClaimRetentionPolicy:
whenDeleted: Retain
whenScaled: Retain
storage:
volumeClaimTemplate:
spec:
storageClassName: ssd-local-observability-alertmanager-retain
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
prometheus:
enabled: true
prometheusSpec:
replicas: 1
retention: 15d
retentionSize: 16GB
scrapeInterval: 30s
evaluationInterval: 30s
walCompression: true
tsdb:
outOfOrderTimeWindow: 10h
automountServiceAccountToken: true
image:
registry: quay.io
repository: prometheus/prometheus
tag: v3.13.2-distroless
sha: ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
resources:
requests:
cpu: 750m
memory: 2Gi
limits:
cpu: "2"
memory: 3Gi
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 2000
fsGroup: 2000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
persistentVolumeClaimRetentionPolicy:
whenDeleted: Retain
whenScaled: Retain
storageSpec:
volumeClaimTemplate:
spec:
storageClassName: ssd-local-observability-prometheus-retain
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
serviceMonitorSelectorNilUsesHelmValues: false
podMonitorSelectorNilUsesHelmValues: false
probeSelectorNilUsesHelmValues: false
ruleSelectorNilUsesHelmValues: false
serviceMonitorNamespaceSelector:
matchLabels:
observability.hyeonworks.com/enabled: "true"
podMonitorNamespaceSelector:
matchLabels:
observability.hyeonworks.com/enabled: "true"
probeNamespaceSelector:
matchLabels:
observability.hyeonworks.com/enabled: "true"
ruleNamespaceSelector:
matchLabels:
observability.hyeonworks.com/enabled: "true"
serviceMonitorSelector:
matchLabels:
observability.hyeonworks.com/instance: home
podMonitorSelector:
matchLabels:
observability.hyeonworks.com/instance: home
probeSelector:
matchLabels:
observability.hyeonworks.com/instance: home
ruleSelector:
matchLabels:
observability.hyeonworks.com/instance: home