From 7fe38428e332c22441d168fb0933930d8779df35 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Wed, 11 Nov 2026 21:51:59 +0900 Subject: [PATCH] feat(observability): alert on the root disk before kubelet acts The kubelet starts deleting unused images at 85% root use and evicts pods at 90%. The root disk warning fired at 85% and the critical at 90%, so both arrived when the kubelet was already acting. On 2026-11-09 the warning fired for 33 hours, then an eviction stopped the tech-log pods, the image GC deleted their imagePullPolicy: Never images, and production went down. Move the warning to 80% for 10m and the critical to 85% for 5m, and add PlatformRootFilesystemFillingFast (critical, for 15m): the 6h linear trend of free bytes falls below 10% of the disk, the eviction point, within 6 hours. On 2026-11-07 the disk lost 17G in 9 hours; this forecast would have fired hours before the 80% warning. The info alert at 75% stays. Add scripts/validate/test-platform-disk-alerts.sh, promtool unit tests for the usage bands and the forecast with the pinned Prometheus image. The existing rules test needs live metric inventories, so this one runs offline. Risk: the live rules change only when the observability access apply runs; this commit does not touch the cluster. The forecast can fire on a large build that the BuildKit GC removes minutes later; the 15m hold and the 6h window damp that. Verification: the new test failed against the old rules (no alert at 81%, 86% or for a 1-per-minute fill) and passes after the change; promtool check rules finds 9 rules. test-observability-alerting.sh passes its static contract, then stops at the helm render because helm v3.19.4 is not installed on the desktop. --- scripts/validate/test-platform-disk-alerts.sh | 153 ++++++++++++++++++ .../rules/platform/core-rules.yaml | 36 +++-- 2 files changed, 180 insertions(+), 9 deletions(-) create mode 100755 scripts/validate/test-platform-disk-alerts.sh diff --git a/scripts/validate/test-platform-disk-alerts.sh b/scripts/validate/test-platform-disk-alerts.sh new file mode 100755 index 0000000..ee20253 --- /dev/null +++ b/scripts/validate/test-platform-disk-alerts.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Unit tests for the root filesystem alerts in core-rules.yaml. +# +# The kubelet starts deleting unused images at 85% use and evicts pods at +# 90% (nodefs.available < 10%). These tests pin the alerts in front of those +# two points: warning at 80%, critical at 85%, and a forecast that fires when +# the 6h trend reaches the eviction point within 6 hours. +# +# Needs docker, jq and kubectl (client only, no cluster access). +set -euo pipefail + +readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml" +# Same pinned image as test-platform-observability-rules.sh. +readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b" +readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md" + +temp_dir='' +cleanup() { + if [[ -n "$temp_dir" && -d "$temp_dir" ]]; then + rm -rf -- "$temp_dir" + fi +} +trap cleanup EXIT + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +for command in docker jq kubectl; do + command -v "$command" >/dev/null || fail "missing command: ${command}" +done + +temp_dir="$(mktemp -d)" +chmod 0755 "$temp_dir" + +kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json | + jq '.spec' >"${temp_dir}/core.rules.json" + +# Every core alert keeps the runbook the other platform rules use. +jq -e --arg url "$RUNBOOK_URL" ' + all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url) +' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url" + +# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter. +readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"' +readonly ALERT_LABELS=' job: prometheus-node-exporter + instance: 10.42.0.95:9100 + device: /dev/sdc1 + fstype: ext4 + mountpoint: /' + +usage_case() { + # name, used percent, eval minute, expected alerts (YAML list body or "[]") + local name=$1 used=$2 eval_minute=$3 expected=$4 + local avail=$((100 - used)) + cat <"${temp_dir}/disk.test.yaml" + +chmod 0444 "${temp_dir}"/* +docker run --rm \ + --network none \ + --volume "${temp_dir}:/work:ro" \ + --entrypoint /bin/promtool \ + "$PROMETHEUS_IMAGE" check rules /work/core.rules.json +docker run --rm \ + --network none \ + --volume "${temp_dir}:/work:ro" \ + --entrypoint /bin/promtool \ + "$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml +printf 'PASS: root filesystem usage and forecast alerts\n' diff --git a/services/observability/rules/platform/core-rules.yaml b/services/observability/rules/platform/core-rules.yaml index 846b484..0a363e2 100644 --- a/services/observability/rules/platform/core-rules.yaml +++ b/services/observability/rules/platform/core-rules.yaml @@ -41,14 +41,14 @@ spec: node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} / node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} - ) > 85 - for: 15m + ) > 80 + for: 10m labels: severity: warning - threshold: "85" + threshold: "80" annotations: - summary: 루트 파일시스템 사용률이 85%를 초과함 - description: 노드 루트 파일시스템 사용률이 경고 임계치인 85%를 15분간 초과했습니다. + summary: 루트 파일시스템 사용률이 80%를 초과함 + description: 노드 루트 파일시스템 사용률이 80%를 10분간 초과했습니다. 85%에서 kubelet이 이미지를 지우기 시작합니다. runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md - alert: PlatformRootFilesystemUsage expr: >- @@ -56,14 +56,32 @@ spec: node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} / node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} - ) > 90 + ) > 85 for: 5m labels: severity: critical - threshold: "90" + threshold: "85" annotations: - summary: 루트 파일시스템 사용률이 90%를 초과함 - description: 노드 루트 파일시스템 사용률이 심각 임계치인 90%를 5분간 초과했습니다. + summary: 루트 파일시스템 사용률이 85%를 초과함 + description: 노드 루트 파일시스템 사용률이 85%를 5분간 초과했습니다. kubelet이 안 쓰는 이미지를 지우는 중이고 90%에서 파드를 내보냅니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + # kubelet evicts pods when nodefs.available drops below 10%. Fire when + # the 6h trend says that happens within the next 6 hours, so a fast + # fill (17G in 9h on 2026-11-07) is caught before the 80% warning. + - alert: PlatformRootFilesystemFillingFast + expr: >- + predict_linear( + node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}[6h], + 6 * 3600 + ) + < + 0.10 * node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"} + for: 15m + labels: + severity: critical + annotations: + summary: 루트 파일시스템이 6시간 안에 축출 문턱에 닿을 것으로 예측됨 + description: 최근 6시간 추세대로면 루트 파일시스템 여유가 6시간 안에 10% 아래로 내려갑니다. 10% 아래에서 kubelet이 파드를 내보냅니다. runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md - alert: PlatformRootFilesystemInodes expr: >-