feat(observability): alert on the root disk before kubelet acts
The kubelet starts deleting unused images at 85% root use and evicts pods at 90%. The root disk warning fired at 85% and the critical at 90%, so both arrived when the kubelet was already acting. On 2026-11-09 the warning fired for 33 hours, then an eviction stopped the tech-log pods, the image GC deleted their imagePullPolicy: Never images, and production went down. Move the warning to 80% for 10m and the critical to 85% for 5m, and add PlatformRootFilesystemFillingFast (critical, for 15m): the 6h linear trend of free bytes falls below 10% of the disk, the eviction point, within 6 hours. On 2026-11-07 the disk lost 17G in 9 hours; this forecast would have fired hours before the 80% warning. The info alert at 75% stays. Add scripts/validate/test-platform-disk-alerts.sh, promtool unit tests for the usage bands and the forecast with the pinned Prometheus image. The existing rules test needs live metric inventories, so this one runs offline. Risk: the live rules change only when the observability access apply runs; this commit does not touch the cluster. The forecast can fire on a large build that the BuildKit GC removes minutes later; the 15m hold and the 6h window damp that. Verification: the new test failed against the old rules (no alert at 81%, 86% or for a 1-per-minute fill) and passes after the change; promtool check rules finds 9 rules. test-observability-alerting.sh passes its static contract, then stops at the helm render because helm v3.19.4 is not installed on the desktop.
This commit is contained in:
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for the root filesystem alerts in core-rules.yaml.
|
||||
#
|
||||
# The kubelet starts deleting unused images at 85% use and evicts pods at
|
||||
# 90% (nodefs.available < 10%). These tests pin the alerts in front of those
|
||||
# two points: warning at 80%, critical at 85%, and a forecast that fires when
|
||||
# the 6h trend reaches the eviction point within 6 hours.
|
||||
#
|
||||
# Needs docker, jq and kubectl (client only, no cluster access).
|
||||
set -euo pipefail
|
||||
|
||||
readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml"
|
||||
# Same pinned image as test-platform-observability-rules.sh.
|
||||
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
||||
readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"
|
||||
|
||||
temp_dir=''
|
||||
cleanup() {
|
||||
if [[ -n "$temp_dir" && -d "$temp_dir" ]]; then
|
||||
rm -rf -- "$temp_dir"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() {
|
||||
printf 'FAIL: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
for command in docker jq kubectl; do
|
||||
command -v "$command" >/dev/null || fail "missing command: ${command}"
|
||||
done
|
||||
|
||||
temp_dir="$(mktemp -d)"
|
||||
chmod 0755 "$temp_dir"
|
||||
|
||||
kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json |
|
||||
jq '.spec' >"${temp_dir}/core.rules.json"
|
||||
|
||||
# Every core alert keeps the runbook the other platform rules use.
|
||||
jq -e --arg url "$RUNBOOK_URL" '
|
||||
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
||||
' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url"
|
||||
|
||||
# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter.
|
||||
readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"'
|
||||
readonly ALERT_LABELS=' job: prometheus-node-exporter
|
||||
instance: 10.42.0.95:9100
|
||||
device: /dev/sdc1
|
||||
fstype: ext4
|
||||
mountpoint: /'
|
||||
|
||||
usage_case() {
|
||||
# name, used percent, eval minute, expected alerts (YAML list body or "[]")
|
||||
local name=$1 used=$2 eval_minute=$3 expected=$4
|
||||
local avail=$((100 - used))
|
||||
cat <<EOF
|
||||
- name: ${name}
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'node_filesystem_avail_bytes{${SERIES_LABELS}}'
|
||||
values: '${avail}+0x40'
|
||||
- series: 'node_filesystem_size_bytes{${SERIES_LABELS}}'
|
||||
values: '100+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: ${eval_minute}m
|
||||
alertname: PlatformRootFilesystemUsage
|
||||
exp_alerts:${expected}
|
||||
EOF
|
||||
}
|
||||
|
||||
usage_alert() {
|
||||
local severity=$1 threshold=$2 summary=$3 description=$4
|
||||
cat <<EOF
|
||||
|
||||
- exp_labels:
|
||||
${ALERT_LABELS}
|
||||
severity: ${severity}
|
||||
threshold: "${threshold}"
|
||||
exp_annotations:
|
||||
summary: ${summary}
|
||||
description: ${description}
|
||||
runbook_url: ${RUNBOOK_URL}
|
||||
EOF
|
||||
}
|
||||
|
||||
readonly WARNING_ALERT="$(usage_alert warning 80 \
|
||||
'루트 파일시스템 사용률이 80%를 초과함' \
|
||||
'노드 루트 파일시스템 사용률이 80%를 10분간 초과했습니다. 85%에서 kubelet이 이미지를 지우기 시작합니다.')"
|
||||
readonly CRITICAL_ALERT="$(usage_alert critical 85 \
|
||||
'루트 파일시스템 사용률이 85%를 초과함' \
|
||||
'노드 루트 파일시스템 사용률이 85%를 5분간 초과했습니다. kubelet이 안 쓰는 이미지를 지우는 중이고 90%에서 파드를 내보냅니다.')"
|
||||
|
||||
forecast_case() {
|
||||
# name, starting avail, avail change per minute, eval minute, expected alerts
|
||||
local name=$1 start=$2 step=$3 eval_minute=$4 expected=$5
|
||||
cat <<EOF
|
||||
- name: ${name}
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'node_filesystem_avail_bytes{${SERIES_LABELS}}'
|
||||
values: '${start}${step}x60'
|
||||
- series: 'node_filesystem_size_bytes{${SERIES_LABELS}}'
|
||||
values: '1000+0x60'
|
||||
alert_rule_test:
|
||||
- eval_time: ${eval_minute}m
|
||||
alertname: PlatformRootFilesystemFillingFast
|
||||
exp_alerts:${expected}
|
||||
EOF
|
||||
}
|
||||
|
||||
readonly FORECAST_ALERT="
|
||||
- exp_labels:
|
||||
${ALERT_LABELS}
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: 루트 파일시스템이 6시간 안에 축출 문턱에 닿을 것으로 예측됨
|
||||
description: 최근 6시간 추세대로면 루트 파일시스템 여유가 6시간 안에 10% 아래로 내려갑니다. 10% 아래에서 kubelet이 파드를 내보냅니다.
|
||||
runbook_url: ${RUNBOOK_URL}"
|
||||
|
||||
{
|
||||
printf '%s\n' \
|
||||
'rule_files:' \
|
||||
' - /work/core.rules.json' \
|
||||
'evaluation_interval: 1m' \
|
||||
'tests:'
|
||||
usage_case 'warning stays quiet at 79 percent' 79 11 ' []'
|
||||
usage_case 'warning fires at 81 percent after 10 minutes' 81 11 "$WARNING_ALERT"
|
||||
usage_case 'warning is still pending at 81 percent after 9 minutes' 81 9 ' []'
|
||||
usage_case 'critical fires at 86 percent after 5 minutes' 86 6 "$CRITICAL_ALERT"
|
||||
usage_case 'warning and critical both fire at 86 percent after 10 minutes' 86 11 "${WARNING_ALERT}${CRITICAL_ALERT}"
|
||||
# 1000 bytes, 400 free, losing 1 per minute: the 6h trend reaches the
|
||||
# 100-byte eviction point in 300 minutes.
|
||||
forecast_case 'forecast fires for a fast fill after 15 minutes' 400 '-1' 16 "$FORECAST_ALERT"
|
||||
forecast_case 'forecast is still pending before 15 minutes' 400 '-1' 14 ' []'
|
||||
# Losing 0.1 per minute takes far longer than 6 hours to reach 100.
|
||||
forecast_case 'forecast stays quiet for a slow fill' 400 '-0.1' 30 ' []'
|
||||
forecast_case 'forecast stays quiet for flat usage' 400 '+0' 30 ' []'
|
||||
} >"${temp_dir}/disk.test.yaml"
|
||||
|
||||
chmod 0444 "${temp_dir}"/*
|
||||
docker run --rm \
|
||||
--network none \
|
||||
--volume "${temp_dir}:/work:ro" \
|
||||
--entrypoint /bin/promtool \
|
||||
"$PROMETHEUS_IMAGE" check rules /work/core.rules.json
|
||||
docker run --rm \
|
||||
--network none \
|
||||
--volume "${temp_dir}:/work:ro" \
|
||||
--entrypoint /bin/promtool \
|
||||
"$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml
|
||||
printf 'PASS: root filesystem usage and forecast alerts\n'
|
||||
@@ -41,14 +41,14 @@ spec:
|
||||
node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
|
||||
/
|
||||
node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
|
||||
) > 85
|
||||
for: 15m
|
||||
) > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
threshold: "85"
|
||||
threshold: "80"
|
||||
annotations:
|
||||
summary: 루트 파일시스템 사용률이 85%를 초과함
|
||||
description: 노드 루트 파일시스템 사용률이 경고 임계치인 85%를 15분간 초과했습니다.
|
||||
summary: 루트 파일시스템 사용률이 80%를 초과함
|
||||
description: 노드 루트 파일시스템 사용률이 80%를 10분간 초과했습니다. 85%에서 kubelet이 이미지를 지우기 시작합니다.
|
||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||
- alert: PlatformRootFilesystemUsage
|
||||
expr: >-
|
||||
@@ -56,14 +56,32 @@ spec:
|
||||
node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
|
||||
/
|
||||
node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
|
||||
) > 90
|
||||
) > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
threshold: "90"
|
||||
threshold: "85"
|
||||
annotations:
|
||||
summary: 루트 파일시스템 사용률이 90%를 초과함
|
||||
description: 노드 루트 파일시스템 사용률이 심각 임계치인 90%를 5분간 초과했습니다.
|
||||
summary: 루트 파일시스템 사용률이 85%를 초과함
|
||||
description: 노드 루트 파일시스템 사용률이 85%를 5분간 초과했습니다. kubelet이 안 쓰는 이미지를 지우는 중이고 90%에서 파드를 내보냅니다.
|
||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||
# kubelet evicts pods when nodefs.available drops below 10%. Fire when
|
||||
# the 6h trend says that happens within the next 6 hours, so a fast
|
||||
# fill (17G in 9h on 2026-11-07) is caught before the 80% warning.
|
||||
- alert: PlatformRootFilesystemFillingFast
|
||||
expr: >-
|
||||
predict_linear(
|
||||
node_filesystem_avail_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}[6h],
|
||||
6 * 3600
|
||||
)
|
||||
<
|
||||
0.10 * node_filesystem_size_bytes{job="prometheus-node-exporter",mountpoint="/",fstype!~"rootfs|tmpfs|ramfs|squashfs|overlay|nsfs|tracefs|debugfs|securityfs|proc|sysfs|cgroup2?"}
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 루트 파일시스템이 6시간 안에 축출 문턱에 닿을 것으로 예측됨
|
||||
description: 최근 6시간 추세대로면 루트 파일시스템 여유가 6시간 안에 10% 아래로 내려갑니다. 10% 아래에서 kubelet이 파드를 내보냅니다.
|
||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||
- alert: PlatformRootFilesystemInodes
|
||||
expr: >-
|
||||
|
||||
Reference in New Issue
Block a user