#!/usr/bin/env bash # Unit tests for core-rules.yaml alerts that guard the desktop host. # # Root disk: the kubelet starts deleting unused images at 85% use and evicts # pods at 90% (nodefs.available < 10%). The alerts sit in front of those two # points: warning at 80%, critical at 85%, and a forecast that fires when the # 6h trend reaches the eviction point within 6 hours. # PostgreSQL backups: no successful base backup for 26 hours, or WAL waiting # to be archived. # # Needs docker, jq and kubectl (client only, no cluster access). set -euo pipefail readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml" # Same pinned image as test-platform-observability-rules.sh. readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b" readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md" temp_dir='' cleanup() { if [[ -n "$temp_dir" && -d "$temp_dir" ]]; then rm -rf -- "$temp_dir" fi } trap cleanup EXIT fail() { printf 'FAIL: %s\n' "$*" >&2 exit 1 } for command in docker jq kubectl; do command -v "$command" >/dev/null || fail "missing command: ${command}" done temp_dir="$(mktemp -d)" chmod 0755 "$temp_dir" kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json | jq '.spec' >"${temp_dir}/core.rules.json" # Every core alert keeps the runbook the other platform rules use. jq -e --arg url "$RUNBOOK_URL" ' all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url) ' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url" # node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter. readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"' readonly PG_LABELS='job="platform-postgres",namespace="platform-data",pod="platform-postgres-1",instance="10.42.0.130:9187"' readonly ALERT_LABELS=' job: prometheus-node-exporter instance: 10.42.0.95:9100 device: /dev/sdc1 fstype: ext4 mountpoint: /' usage_case() { # name, used percent, eval minute, expected alerts (YAML list body or "[]") local name=$1 used=$2 eval_minute=$3 expected=$4 local avail=$((100 - used)) cat <"${temp_dir}/disk.test.yaml" chmod 0444 "${temp_dir}"/* docker run --rm \ --network none \ --volume "${temp_dir}:/work:ro" \ --entrypoint /bin/promtool \ "$PROMETHEUS_IMAGE" check rules /work/core.rules.json docker run --rm \ --network none \ --volume "${temp_dir}:/work:ro" \ --entrypoint /bin/promtool \ "$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml printf 'PASS: root filesystem and PostgreSQL backup alerts\n'