On 2026-11-11 the desktop booted with the RTC read as UTC while it held KST, so the clock ran 9 hours ahead until NTP pulled it back. Prometheus had written one hour of samples stamped 18:36-19:36Z into its TSDB head during that window. With outOfOrderTimeWindow at the chart default of 0s, every later scrape was older than the head's valid range and was dropped as "out of bounds". From 11:21Z no metric was stored, no rule could fire on real data, and KubeAPIDown (an absent() rule) fired falsely. It would have stayed blind until the wall clock passed the bogus head time, about 7 hours later. Set prometheusSpec.tsdb.outOfOrderTimeWindow to 10h. Samples up to 10 hours behind the head are now accepted, which covers the +9h KST-as-UTC misread. The README records why. Rejected: deleting the head WAL from the Retain PV. It works at once but removes persisted data and needs the pod stopped. Rejected: waiting for the clock to catch up. It leaves monitoring blind for hours during the period when the cause of the incident is still being found. Risk: out-of-order samples use an extra in-memory head and vertical compaction only while they arrive; in normal operation none arrive. A genuinely late exporter could now backfill up to 10h, which is harmless on this single-node Prometheus. Preventing the clock jump itself is the host change in infrastructure/host, not this commit. Verification: the values file parses and holds prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow=10h. The live Prometheus CR already carried tsdb.outOfOrderTimeWindow=0s from this chart key, so the key maps to spec.tsdb. The same value was patched onto the live CR: the operator wrote out_of_order_time_window: 10h, the config reloaded at 13:04:37Z, "too old" warnings dropped to 0, all 34 targets report up, and KubeAPIDown resolved. The pinned renderer (scripts/validate/render-observability-core.sh) was not run: it requires Helm v3.19.4 and kustomize v5.8.1, which are not installed on the laptop or the desktop.
285 lines
6.3 KiB
YAML
285 lines
6.3 KiB
YAML
commonLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
|
|
crds:
|
|
enabled: false
|
|
|
|
grafana:
|
|
enabled: false
|
|
|
|
nodeExporter:
|
|
enabled: false
|
|
|
|
kubeApiServer:
|
|
enabled: true
|
|
|
|
kubelet:
|
|
enabled: true
|
|
|
|
coreDns:
|
|
enabled: true
|
|
|
|
kubeControllerManager:
|
|
enabled: false
|
|
|
|
kubeScheduler:
|
|
enabled: false
|
|
|
|
kubeEtcd:
|
|
enabled: false
|
|
|
|
kubeProxy:
|
|
enabled: false
|
|
|
|
defaultRules:
|
|
disabled:
|
|
Watchdog: true
|
|
rules:
|
|
etcd: false
|
|
kubeControllerManager: false
|
|
kubeProxy: false
|
|
kubeSchedulerAlerting: false
|
|
kubeSchedulerRecording: false
|
|
windows: false
|
|
|
|
prometheusOperator:
|
|
admissionWebhooks:
|
|
enabled: false
|
|
patch:
|
|
enabled: false
|
|
tls:
|
|
enabled: false
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus-operator/prometheus-operator
|
|
tag: v0.93.0
|
|
sha: 64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containerSecurityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
readOnlyRootFilesystem: true
|
|
serviceAccount:
|
|
automountServiceAccountToken: true
|
|
prometheusConfigReloader:
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus-operator/prometheus-config-reloader
|
|
tag: v0.93.0
|
|
sha: 65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
|
|
kubeStateMetrics:
|
|
enabled: true
|
|
|
|
kube-state-metrics:
|
|
image:
|
|
registry: registry.k8s.io
|
|
repository: kube-state-metrics/kube-state-metrics
|
|
tag: v2.19.1
|
|
sha: sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
securityContext:
|
|
enabled: true
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containerSecurityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
readOnlyRootFilesystem: true
|
|
automountServiceAccountToken: true
|
|
collectors:
|
|
- certificatesigningrequests
|
|
- configmaps
|
|
- cronjobs
|
|
- daemonsets
|
|
- deployments
|
|
- endpointslices
|
|
- horizontalpodautoscalers
|
|
- ingresses
|
|
- jobs
|
|
- leases
|
|
- limitranges
|
|
- mutatingwebhookconfigurations
|
|
- namespaces
|
|
- networkpolicies
|
|
- nodes
|
|
- persistentvolumeclaims
|
|
- persistentvolumes
|
|
- poddisruptionbudgets
|
|
- pods
|
|
- replicasets
|
|
- replicationcontrollers
|
|
- resourcequotas
|
|
- services
|
|
- statefulsets
|
|
- storageclasses
|
|
- validatingwebhookconfigurations
|
|
- volumeattachments
|
|
prometheus:
|
|
monitor:
|
|
additionalLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
|
|
alertmanager:
|
|
enabled: true
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
route:
|
|
group_by:
|
|
- namespace
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 12h
|
|
receiver: "null"
|
|
routes: []
|
|
receivers:
|
|
- name: "null"
|
|
serviceAccount:
|
|
automountServiceAccountToken: false
|
|
alertmanagerSpec:
|
|
replicas: 1
|
|
retention: 120h
|
|
automountServiceAccountToken: false
|
|
alertmanagerConfiguration:
|
|
name: platform-alertmanager
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus/alertmanager
|
|
tag: v0.33.1
|
|
sha: a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 2000
|
|
fsGroup: 2000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
persistentVolumeClaimRetentionPolicy:
|
|
whenDeleted: Retain
|
|
whenScaled: Retain
|
|
storage:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ssd-local-observability-alertmanager-retain
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
|
|
prometheus:
|
|
enabled: true
|
|
prometheusSpec:
|
|
replicas: 1
|
|
retention: 15d
|
|
retentionSize: 16GB
|
|
scrapeInterval: 30s
|
|
evaluationInterval: 30s
|
|
walCompression: true
|
|
tsdb:
|
|
outOfOrderTimeWindow: 10h
|
|
automountServiceAccountToken: true
|
|
image:
|
|
registry: quay.io
|
|
repository: prometheus/prometheus
|
|
tag: v3.13.2-distroless
|
|
sha: ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
|
|
resources:
|
|
requests:
|
|
cpu: 750m
|
|
memory: 2Gi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 3Gi
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 2000
|
|
fsGroup: 2000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
persistentVolumeClaimRetentionPolicy:
|
|
whenDeleted: Retain
|
|
whenScaled: Retain
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: ssd-local-observability-prometheus-retain
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
probeSelectorNilUsesHelmValues: false
|
|
ruleSelectorNilUsesHelmValues: false
|
|
serviceMonitorNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
podMonitorNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
probeNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
ruleNamespaceSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/enabled: "true"
|
|
serviceMonitorSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
podMonitorSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
probeSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|
|
ruleSelector:
|
|
matchLabels:
|
|
observability.hyeonworks.com/instance: home
|