The kubelet starts deleting unused images at 85% root use and evicts pods at 90%. The root disk warning fired at 85% and the critical at 90%, so both arrived when the kubelet was already acting. On 2026-11-09 the warning fired for 33 hours, then an eviction stopped the tech-log pods, the image GC deleted their imagePullPolicy: Never images, and production went down. Move the warning to 80% for 10m and the critical to 85% for 5m, and add PlatformRootFilesystemFillingFast (critical, for 15m): the 6h linear trend of free bytes falls below 10% of the disk, the eviction point, within 6 hours. On 2026-11-07 the disk lost 17G in 9 hours; this forecast would have fired hours before the 80% warning. The info alert at 75% stays. Add scripts/validate/test-platform-disk-alerts.sh, promtool unit tests for the usage bands and the forecast with the pinned Prometheus image. The existing rules test needs live metric inventories, so this one runs offline. Risk: the live rules change only when the observability access apply runs; this commit does not touch the cluster. The forecast can fire on a large build that the BuildKit GC removes minutes later; the 15m hold and the 6h window damp that. Verification: the new test failed against the old rules (no alert at 81%, 86% or for a 1-per-minute fill) and passes after the change; promtool check rules finds 9 rules. test-observability-alerting.sh passes its static contract, then stops at the helm render because helm v3.19.4 is not installed on the desktop.
154 lines
6.0 KiB
Bash
Executable File
154 lines
6.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Unit tests for the root filesystem alerts in core-rules.yaml.
|
|
#
|
|
# The kubelet starts deleting unused images at 85% use and evicts pods at
|
|
# 90% (nodefs.available < 10%). These tests pin the alerts in front of those
|
|
# two points: warning at 80%, critical at 85%, and a forecast that fires when
|
|
# the 6h trend reaches the eviction point within 6 hours.
|
|
#
|
|
# Needs docker, jq and kubectl (client only, no cluster access).
|
|
set -euo pipefail
|
|
|
|
readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml"
|
|
# Same pinned image as test-platform-observability-rules.sh.
|
|
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
|
readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"
|
|
|
|
temp_dir=''
|
|
cleanup() {
|
|
if [[ -n "$temp_dir" && -d "$temp_dir" ]]; then
|
|
rm -rf -- "$temp_dir"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
for command in docker jq kubectl; do
|
|
command -v "$command" >/dev/null || fail "missing command: ${command}"
|
|
done
|
|
|
|
temp_dir="$(mktemp -d)"
|
|
chmod 0755 "$temp_dir"
|
|
|
|
kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json |
|
|
jq '.spec' >"${temp_dir}/core.rules.json"
|
|
|
|
# Every core alert keeps the runbook the other platform rules use.
|
|
jq -e --arg url "$RUNBOOK_URL" '
|
|
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
|
' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url"
|
|
|
|
# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter.
|
|
readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"'
|
|
readonly ALERT_LABELS=' job: prometheus-node-exporter
|
|
instance: 10.42.0.95:9100
|
|
device: /dev/sdc1
|
|
fstype: ext4
|
|
mountpoint: /'
|
|
|
|
usage_case() {
|
|
# name, used percent, eval minute, expected alerts (YAML list body or "[]")
|
|
local name=$1 used=$2 eval_minute=$3 expected=$4
|
|
local avail=$((100 - used))
|
|
cat <<EOF
|
|
- name: ${name}
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'node_filesystem_avail_bytes{${SERIES_LABELS}}'
|
|
values: '${avail}+0x40'
|
|
- series: 'node_filesystem_size_bytes{${SERIES_LABELS}}'
|
|
values: '100+0x40'
|
|
alert_rule_test:
|
|
- eval_time: ${eval_minute}m
|
|
alertname: PlatformRootFilesystemUsage
|
|
exp_alerts:${expected}
|
|
EOF
|
|
}
|
|
|
|
usage_alert() {
|
|
local severity=$1 threshold=$2 summary=$3 description=$4
|
|
cat <<EOF
|
|
|
|
- exp_labels:
|
|
${ALERT_LABELS}
|
|
severity: ${severity}
|
|
threshold: "${threshold}"
|
|
exp_annotations:
|
|
summary: ${summary}
|
|
description: ${description}
|
|
runbook_url: ${RUNBOOK_URL}
|
|
EOF
|
|
}
|
|
|
|
readonly WARNING_ALERT="$(usage_alert warning 80 \
|
|
'루트 파일시스템 사용률이 80%를 초과함' \
|
|
'노드 루트 파일시스템 사용률이 80%를 10분간 초과했습니다. 85%에서 kubelet이 이미지를 지우기 시작합니다.')"
|
|
readonly CRITICAL_ALERT="$(usage_alert critical 85 \
|
|
'루트 파일시스템 사용률이 85%를 초과함' \
|
|
'노드 루트 파일시스템 사용률이 85%를 5분간 초과했습니다. kubelet이 안 쓰는 이미지를 지우는 중이고 90%에서 파드를 내보냅니다.')"
|
|
|
|
forecast_case() {
|
|
# name, starting avail, avail change per minute, eval minute, expected alerts
|
|
local name=$1 start=$2 step=$3 eval_minute=$4 expected=$5
|
|
cat <<EOF
|
|
- name: ${name}
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'node_filesystem_avail_bytes{${SERIES_LABELS}}'
|
|
values: '${start}${step}x60'
|
|
- series: 'node_filesystem_size_bytes{${SERIES_LABELS}}'
|
|
values: '1000+0x60'
|
|
alert_rule_test:
|
|
- eval_time: ${eval_minute}m
|
|
alertname: PlatformRootFilesystemFillingFast
|
|
exp_alerts:${expected}
|
|
EOF
|
|
}
|
|
|
|
readonly FORECAST_ALERT="
|
|
- exp_labels:
|
|
${ALERT_LABELS}
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: 루트 파일시스템이 6시간 안에 축출 문턱에 닿을 것으로 예측됨
|
|
description: 최근 6시간 추세대로면 루트 파일시스템 여유가 6시간 안에 10% 아래로 내려갑니다. 10% 아래에서 kubelet이 파드를 내보냅니다.
|
|
runbook_url: ${RUNBOOK_URL}"
|
|
|
|
{
|
|
printf '%s\n' \
|
|
'rule_files:' \
|
|
' - /work/core.rules.json' \
|
|
'evaluation_interval: 1m' \
|
|
'tests:'
|
|
usage_case 'warning stays quiet at 79 percent' 79 11 ' []'
|
|
usage_case 'warning fires at 81 percent after 10 minutes' 81 11 "$WARNING_ALERT"
|
|
usage_case 'warning is still pending at 81 percent after 9 minutes' 81 9 ' []'
|
|
usage_case 'critical fires at 86 percent after 5 minutes' 86 6 "$CRITICAL_ALERT"
|
|
usage_case 'warning and critical both fire at 86 percent after 10 minutes' 86 11 "${WARNING_ALERT}${CRITICAL_ALERT}"
|
|
# 1000 bytes, 400 free, losing 1 per minute: the 6h trend reaches the
|
|
# 100-byte eviction point in 300 minutes.
|
|
forecast_case 'forecast fires for a fast fill after 15 minutes' 400 '-1' 16 "$FORECAST_ALERT"
|
|
forecast_case 'forecast is still pending before 15 minutes' 400 '-1' 14 ' []'
|
|
# Losing 0.1 per minute takes far longer than 6 hours to reach 100.
|
|
forecast_case 'forecast stays quiet for a slow fill' 400 '-0.1' 30 ' []'
|
|
forecast_case 'forecast stays quiet for flat usage' 400 '+0' 30 ' []'
|
|
} >"${temp_dir}/disk.test.yaml"
|
|
|
|
chmod 0444 "${temp_dir}"/*
|
|
docker run --rm \
|
|
--network none \
|
|
--volume "${temp_dir}:/work:ro" \
|
|
--entrypoint /bin/promtool \
|
|
"$PROMETHEUS_IMAGE" check rules /work/core.rules.json
|
|
docker run --rm \
|
|
--network none \
|
|
--volume "${temp_dir}:/work:ro" \
|
|
--entrypoint /bin/promtool \
|
|
"$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml
|
|
printf 'PASS: root filesystem usage and forecast alerts\n'
|