Backups, the daily drift check and the k3s serving certificates have no alert. k3s renews certificates only when it starts within 120 days of expiry, and the two kube-prometheus-stack kubelet certificate alerts can never fire because the k3s kubelet exports no kubelet_certificate_manager_* metric. Add a PrometheusRule (platform-host-checks) for backup age, failed runs, missing metrics, drift left for 6h, a stale drift check, and serving certificate expiry (warning under 100 days, critical under 14). Add the two remaining host timers (certificate expiry via openssl on 6443 and 10250; the drift check against origin/main) and an installer that copies the scripts to a root-owned path, creates the textfile and work directories and enables all four timers. Disable the two dead kubelet certificate rules in kube-prometheus-stack. Risk: the metrics need the node-exporter textfile collector, a one-line change in W1's node-exporter values that was asked of W1. Until then PlatformHostBackupMetricsAbsent fires after a day, which is the point. Verification (desktop): - test-platform-host-checks-alerts.sh: promtool check and unit tests pass; with the k3s threshold changed from 2 to 5 days they fail. - render-observability-core.sh passes; the render has no Kubelet* certificate alerts and still has both KubeClientCertificateExpiration. - k3s-cert-expiry.sh as the desktop user reads both certificates. - install-host-checks.sh --check: systemd-analyze verify is clean.
290 lines
6.6 KiB
YAML
290 lines
6.6 KiB
YAML
commonLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
|
|
crds:
|
|
enabled: false
|
|
|
|
grafana:
|
|
enabled: false
|
|
|
|
nodeExporter:
|
|
enabled: false
|
|
|
|
kubeApiServer:
|
|
enabled: true
|
|
|
|
kubelet:
|
|
enabled: true
|
|
|
|
coreDns:
|
|
enabled: true
|
|
|
|
kubeControllerManager:
|
|
enabled: false
|
|
|
|
kubeScheduler:
|
|
enabled: false
|
|
|
|
kubeEtcd:
|
|
enabled: false
|
|
|
|
kubeProxy:
|
|
enabled: false
|
|
|
|
defaultRules:
|
|
disabled:
|
|
Watchdog: true
|
|
# k3s kubelet exposes no kubelet_certificate_manager_* metrics, so these two
|
|
# can never fire. Serving certificates are watched by platform-host-checks
|
|
# (PlatformK3sServingCertificateExpiry) instead. Review 2026-11-11 A11.
|
|
KubeletClientCertificateExpiration: true
|
|
KubeletServerCertificateExpiration: true
|
|
rules:
|
|
etcd: false
|
|
kubeControllerManager: false
|
|
kubeProxy: false
|
|
kubeSchedulerAlerting: false
|
|
kubeSchedulerRecording: false
|
|
windows: false
|
|
|
|
prometheusOperator:
|
|
admissionWebhooks:
|
|
enabled: false
|
|
patch:
|
|
enabled: false
|
|
tls:
|
|
enabled: false
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus-operator/prometheus-operator
|
|
tag: v0.93.0
|
|
sha: 64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containerSecurityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
readOnlyRootFilesystem: true
|
|
serviceAccount:
|
|
automountServiceAccountToken: true
|
|
prometheusConfigReloader:
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus-operator/prometheus-config-reloader
|
|
tag: v0.93.0
|
|
sha: 65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
|
|
kubeStateMetrics:
|
|
enabled: true
|
|
|
|
kube-state-metrics:
|
|
image:
|
|
registry: registry.k8s.io
|
|
repository: kube-state-metrics/kube-state-metrics
|
|
tag: v2.19.1
|
|
sha: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
securityContext:
|
|
enabled: true
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containerSecurityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
readOnlyRootFilesystem: true
|
|
automountServiceAccountToken: true
|
|
collectors:
|
|
- certificatesigningrequests
|
|
- configmaps
|
|
- cronjobs
|
|
- daemonsets
|
|
- deployments
|
|
- endpointslices
|
|
- horizontalpodautoscalers
|
|
- ingresses
|
|
- jobs
|
|
- leases
|
|
- limitranges
|
|
- mutatingwebhookconfigurations
|
|
- namespaces
|
|
- networkpolicies
|
|
- nodes
|
|
- persistentvolumeclaims
|
|
- persistentvolumes
|
|
- poddisruptionbudgets
|
|
- pods
|
|
- replicasets
|
|
- replicationcontrollers
|
|
- resourcequotas
|
|
- services
|
|
- statefulsets
|
|
- storageclasses
|
|
- validatingwebhookconfigurations
|
|
- volumeattachments
|
|
prometheus:
|
|
monitor:
|
|
additionalLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
|
|
alertmanager:
|
|
enabled: true
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
route:
|
|
group_by:
|
|
- namespace
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 12h
|
|
receiver: "null"
|
|
routes: []
|
|
receivers:
|
|
- name: "null"
|
|
serviceAccount:
|
|
automountServiceAccountToken: false
|
|
alertmanagerSpec:
|
|
replicas: 1
|
|
retention: 120h
|
|
automountServiceAccountToken: false
|
|
alertmanagerConfiguration:
|
|
name: platform-alertmanager
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus/alertmanager
|
|
tag: v0.33.1
|
|
sha: a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 2000
|
|
fsGroup: 2000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
persistentVolumeClaimRetentionPolicy:
|
|
whenDeleted: Retain
|
|
whenScaled: Retain
|
|
storage:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ssd-local-observability-alertmanager-retain
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
|
|
prometheus:
|
|
enabled: true
|
|
prometheusSpec:
|
|
replicas: 1
|
|
retention: 15d
|
|
retentionSize: 16GB
|
|
scrapeInterval: 30s
|
|
evaluationInterval: 30s
|
|
walCompression: true
|
|
tsdb:
|
|
outOfOrderTimeWindow: 10h
|
|
automountServiceAccountToken: true
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus/prometheus
|
|
tag: v3.13.2-distroless
|
|
sha: ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
|
|
resources:
|
|
requests:
|
|
cpu: 750m
|
|
memory: 2Gi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 3Gi
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 2000
|
|
fsGroup: 2000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
persistentVolumeClaimRetentionPolicy:
|
|
whenDeleted: Retain
|
|
whenScaled: Retain
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ssd-local-observability-prometheus-retain
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
probeSelectorNilUsesHelmValues: false
|
|
ruleSelectorNilUsesHelmValues: false
|
|
serviceMonitorNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
podMonitorNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
probeNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
ruleNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
serviceMonitorSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
podMonitorSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
probeSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
ruleSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|