The observability core renderer rejects any alert or record whose name contains "backup" (no_unscoped_monitors_rules): backup monitoring is outside the observability core by design.f19ce54put PlatformPostgresBackupMissing and PlatformPostgresWALArchiveBacklog in core-rules.yaml, so the pinned renderer failed with "out-of-scope monitor or rule is present". Move both alerts unchanged into services/platform-postgres-backup/prometheus-rule.yaml in platform-data, which Prometheus reads because the namespace has observability.hyeonworks.com/enabled=true and the rule has observability.hyeonworks.com/instance=home. They now point their runbook_url at the backup README and are applied with the ScheduledBackup. Renaming them to dodge the filter was rejected: it would keep backup rules inside a scope that excludes them on purpose. Verification: test-platform-disk-alerts.sh loads both rule files and passes; promtool checks 13 rules. render-observability-core.sh and test-render-observability-core.sh pass, and kubectl apply --dry-run=server accepts the backup kustomization with the rule. scan-platform-sensitive-source.sh stops at its observability-core renderer with RC=23 here and on an origin/main snapshot alike, so it says nothing about this change. Fixes:f19ce54740("feat(postgres): back up platform-postgres to AIStor every day")
264 lines
11 KiB
Bash
Executable File
264 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Unit tests for the alerts that guard the desktop host: core-rules.yaml and
|
|
# services/platform-postgres-backup/prometheus-rule.yaml.
|
|
#
|
|
# Root disk: the kubelet starts deleting unused images at 85% use and evicts
|
|
# pods at 90% (nodefs.available < 10%). The alerts sit in front of those two
|
|
# points: warning at 80%, critical at 85%, and a forecast that fires when the
|
|
# 6h trend reaches the eviction point within 6 hours.
|
|
# PostgreSQL backups: no successful base backup for 26 hours, or WAL waiting
|
|
# to be archived. Monitoring: ingestion stopped, and the always-firing
|
|
# heartbeat that an external dead man's switch watches.
|
|
#
|
|
# Needs docker, jq and kubectl (client only, no cluster access).
|
|
set -euo pipefail
|
|
|
|
readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml"
|
|
readonly BACKUP_RULE_FILE="${ROOT_DIR}/services/platform-postgres-backup/prometheus-rule.yaml"
|
|
# Same pinned image as test-platform-observability-rules.sh.
|
|
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
|
readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"
|
|
readonly BACKUP_RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md"
|
|
|
|
temp_dir=''
|
|
cleanup() {
|
|
if [[ -n "$temp_dir" && -d "$temp_dir" ]]; then
|
|
rm -rf -- "$temp_dir"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
for command in docker jq kubectl; do
|
|
command -v "$command" >/dev/null || fail "missing command: ${command}"
|
|
done
|
|
|
|
temp_dir="$(mktemp -d)"
|
|
chmod 0755 "$temp_dir"
|
|
|
|
kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json |
|
|
jq '.spec' >"${temp_dir}/core.rules.json"
|
|
kubectl create --dry-run=client --validate=false -f "$BACKUP_RULE_FILE" -o json |
|
|
jq '.spec' >"${temp_dir}/backup.rules.json"
|
|
|
|
# Every core alert keeps the runbook the other platform rules use.
|
|
jq -e --arg url "$RUNBOOK_URL" '
|
|
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
|
' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url"
|
|
jq -e --arg url "$BACKUP_RUNBOOK_URL" '
|
|
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
|
' "${temp_dir}/backup.rules.json" >/dev/null || fail "backup alert without the backup README runbook_url"
|
|
|
|
# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter.
|
|
readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"'
|
|
readonly PG_LABELS='job="platform-postgres",namespace="platform-data",pod="platform-postgres-1",instance="10.42.0.130:9187"'
|
|
readonly ALERT_LABELS=' job: prometheus-node-exporter
|
|
instance: 10.42.0.95:9100
|
|
device: /dev/sdc1
|
|
fstype: ext4
|
|
mountpoint: /'
|
|
|
|
usage_case() {
|
|
# name, used percent, eval minute, expected alerts (YAML list body or "[]")
|
|
local name=$1 used=$2 eval_minute=$3 expected=$4
|
|
local avail=$((100 - used))
|
|
cat <<EOF
|
|
- name: ${name}
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'node_filesystem_avail_bytes{${SERIES_LABELS}}'
|
|
values: '${avail}+0x40'
|
|
- series: 'node_filesystem_size_bytes{${SERIES_LABELS}}'
|
|
values: '100+0x40'
|
|
alert_rule_test:
|
|
- eval_time: ${eval_minute}m
|
|
alertname: PlatformRootFilesystemUsage
|
|
exp_alerts:${expected}
|
|
EOF
|
|
}
|
|
|
|
usage_alert() {
|
|
local severity=$1 threshold=$2 summary=$3 description=$4
|
|
cat <<EOF
|
|
|
|
- exp_labels:
|
|
${ALERT_LABELS}
|
|
severity: ${severity}
|
|
threshold: "${threshold}"
|
|
exp_annotations:
|
|
summary: ${summary}
|
|
description: ${description}
|
|
runbook_url: ${RUNBOOK_URL}
|
|
EOF
|
|
}
|
|
|
|
readonly WARNING_ALERT="$(usage_alert warning 80 \
|
|
'루트 파일시스템 사용률이 80%를 초과함' \
|
|
'노드 루트 파일시스템 사용률이 80%를 10분간 초과했습니다. 85%에서 kubelet이 이미지를 지우기 시작합니다.')"
|
|
readonly CRITICAL_ALERT="$(usage_alert critical 85 \
|
|
'루트 파일시스템 사용률이 85%를 초과함' \
|
|
'노드 루트 파일시스템 사용률이 85%를 5분간 초과했습니다. kubelet이 안 쓰는 이미지를 지우는 중이고 90%에서 파드를 내보냅니다.')"
|
|
|
|
forecast_case() {
|
|
# name, starting avail, avail change per minute, eval minute, expected alerts
|
|
local name=$1 start=$2 step=$3 eval_minute=$4 expected=$5
|
|
cat <<EOF
|
|
- name: ${name}
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'node_filesystem_avail_bytes{${SERIES_LABELS}}'
|
|
values: '${start}${step}x60'
|
|
- series: 'node_filesystem_size_bytes{${SERIES_LABELS}}'
|
|
values: '1000+0x60'
|
|
alert_rule_test:
|
|
- eval_time: ${eval_minute}m
|
|
alertname: PlatformRootFilesystemFillingFast
|
|
exp_alerts:${expected}
|
|
EOF
|
|
}
|
|
|
|
readonly FORECAST_ALERT="
|
|
- exp_labels:
|
|
${ALERT_LABELS}
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: 루트 파일시스템이 6시간 안에 축출 문턱에 닿을 것으로 예측됨
|
|
description: 최근 6시간 추세대로면 루트 파일시스템 여유가 6시간 안에 10% 아래로 내려갑니다. 10% 아래에서 kubelet이 파드를 내보냅니다.
|
|
runbook_url: ${RUNBOOK_URL}"
|
|
|
|
{
|
|
printf '%s\n' \
|
|
'rule_files:' \
|
|
' - /work/core.rules.json' \
|
|
' - /work/backup.rules.json' \
|
|
'evaluation_interval: 1m' \
|
|
'tests:'
|
|
usage_case 'warning stays quiet at 79 percent' 79 11 ' []'
|
|
usage_case 'warning fires at 81 percent after 10 minutes' 81 11 "$WARNING_ALERT"
|
|
usage_case 'warning is still pending at 81 percent after 9 minutes' 81 9 ' []'
|
|
usage_case 'critical fires at 86 percent after 5 minutes' 86 6 "$CRITICAL_ALERT"
|
|
usage_case 'warning and critical both fire at 86 percent after 10 minutes' 86 11 "${WARNING_ALERT}${CRITICAL_ALERT}"
|
|
# 1000 bytes, 400 free, losing 1 per minute: the 6h trend reaches the
|
|
# 100-byte eviction point in 300 minutes.
|
|
forecast_case 'forecast fires for a fast fill after 15 minutes' 400 '-1' 16 "$FORECAST_ALERT"
|
|
forecast_case 'forecast is still pending before 15 minutes' 400 '-1' 14 ' []'
|
|
# Losing 0.1 per minute takes far longer than 6 hours to reach 100.
|
|
forecast_case 'forecast stays quiet for a slow fill' 400 '-0.1' 30 ' []'
|
|
forecast_case 'forecast stays quiet for flat usage' 400 '+0' 30 ' []'
|
|
|
|
# PostgreSQL backups. Timestamps are seconds since the test epoch.
|
|
cat <<EOF
|
|
# The test clock starts at 0, so "27.8 hours ago" is a negative timestamp.
|
|
# In production a cluster without any backup reports 0 and fires the same way.
|
|
- name: backup alert fires when the last backup is older than 26 hours
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'cnpg_collector_last_available_backup_timestamp{${PG_LABELS}}'
|
|
values: '-100000+0x40'
|
|
alert_rule_test:
|
|
- eval_time: 31m
|
|
alertname: PlatformPostgresBackupMissing
|
|
exp_alerts:
|
|
- exp_labels:
|
|
job: platform-postgres
|
|
namespace: platform-data
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
|
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
|
runbook_url: ${BACKUP_RUNBOOK_URL}
|
|
- name: backup alert stays quiet after a fresh backup
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'cnpg_collector_last_available_backup_timestamp{${PG_LABELS}}'
|
|
values: '60+0x40'
|
|
alert_rule_test:
|
|
- eval_time: 31m
|
|
alertname: PlatformPostgresBackupMissing
|
|
exp_alerts: []
|
|
- name: WAL backlog fires after 15 minutes
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'cnpg_collector_pg_wal_archive_status{${PG_LABELS},value="ready"}'
|
|
values: '11+0x30'
|
|
- series: 'cnpg_collector_pg_wal_archive_status{${PG_LABELS},value="done"}'
|
|
values: '500+0x30'
|
|
alert_rule_test:
|
|
- eval_time: 16m
|
|
alertname: PlatformPostgresWALArchiveBacklog
|
|
exp_alerts:
|
|
- exp_labels:
|
|
job: platform-postgres
|
|
namespace: platform-data
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
|
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
|
runbook_url: ${BACKUP_RUNBOOK_URL}
|
|
- name: WAL backlog stays quiet at 10 ready files
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'cnpg_collector_pg_wal_archive_status{${PG_LABELS},value="ready"}'
|
|
values: '10+0x30'
|
|
alert_rule_test:
|
|
- eval_time: 20m
|
|
alertname: PlatformPostgresWALArchiveBacklog
|
|
exp_alerts: []
|
|
- name: ingestion alert fires when node-exporter samples stop
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'up{job="prometheus-node-exporter",instance="10.42.0.95:9100"}'
|
|
values: '1x5 _x20'
|
|
alert_rule_test:
|
|
- eval_time: 20m
|
|
alertname: PlatformMetricsIngestionStopped
|
|
exp_alerts:
|
|
- exp_labels:
|
|
job: prometheus-node-exporter
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Prometheus에 새 지표가 저장되지 않음
|
|
description: 5분 넘게 node-exporter의 up 샘플이 저장되지 않았습니다. Prometheus가 샘플을 버리고 있으면(예 시계가 되돌아감) 다른 지표 경보도 울리지 않습니다.
|
|
runbook_url: ${RUNBOOK_URL}
|
|
- name: ingestion alert stays quiet while samples arrive
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'up{job="prometheus-node-exporter",instance="10.42.0.95:9100"}'
|
|
values: '1x30'
|
|
alert_rule_test:
|
|
- eval_time: 20m
|
|
alertname: PlatformMetricsIngestionStopped
|
|
exp_alerts: []
|
|
- name: heartbeat always fires
|
|
interval: 1m
|
|
alert_rule_test:
|
|
- eval_time: 1m
|
|
alertname: PlatformHeartbeat
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: none
|
|
exp_annotations:
|
|
summary: 경보 경로 heartbeat
|
|
description: 항상 울린다. Alertmanager가 이 경보를 외부 dead man's switch로 5분마다 보내고, 끊기면 외부 서비스가 알린다.
|
|
runbook_url: ${RUNBOOK_URL}
|
|
EOF
|
|
} >"${temp_dir}/disk.test.yaml"
|
|
|
|
chmod 0444 "${temp_dir}"/*
|
|
docker run --rm \
|
|
--network none \
|
|
--volume "${temp_dir}:/work:ro" \
|
|
--entrypoint /bin/promtool \
|
|
"$PROMETHEUS_IMAGE" check rules /work/core.rules.json /work/backup.rules.json
|
|
docker run --rm \
|
|
--network none \
|
|
--volume "${temp_dir}:/work:ro" \
|
|
--entrypoint /bin/promtool \
|
|
"$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml
|
|
printf 'PASS: root filesystem, PostgreSQL backup, ingestion and heartbeat alerts\n'
|