diff --git a/scripts/validate/test-platform-disk-alerts.sh b/scripts/validate/test-platform-disk-alerts.sh index b2c95e5..184b5dc 100755 --- a/scripts/validate/test-platform-disk-alerts.sh +++ b/scripts/validate/test-platform-disk-alerts.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Unit tests for core-rules.yaml alerts that guard the desktop host. +# Unit tests for the alerts that guard the desktop host: core-rules.yaml and +# services/platform-postgres-backup/prometheus-rule.yaml. # # Root disk: the kubelet starts deleting unused images at 85% use and evicts # pods at 90% (nodefs.available < 10%). The alerts sit in front of those two @@ -14,9 +15,11 @@ set -euo pipefail readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml" +readonly BACKUP_RULE_FILE="${ROOT_DIR}/services/platform-postgres-backup/prometheus-rule.yaml" # Same pinned image as test-platform-observability-rules.sh. readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b" readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md" +readonly BACKUP_RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md" temp_dir='' cleanup() { @@ -40,11 +43,16 @@ chmod 0755 "$temp_dir" kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json | jq '.spec' >"${temp_dir}/core.rules.json" +kubectl create --dry-run=client --validate=false -f "$BACKUP_RULE_FILE" -o json | + jq '.spec' >"${temp_dir}/backup.rules.json" # Every core alert keeps the runbook the other platform rules use. jq -e --arg url "$RUNBOOK_URL" ' all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url) ' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url" +jq -e --arg url "$BACKUP_RUNBOOK_URL" ' + all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url) +' "${temp_dir}/backup.rules.json" >/dev/null || fail "backup alert without the backup README runbook_url" # node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter. readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"' @@ -127,6 +135,7 @@ ${ALERT_LABELS} printf '%s\n' \ 'rule_files:' \ ' - /work/core.rules.json' \ + ' - /work/backup.rules.json' \ 'evaluation_interval: 1m' \ 'tests:' usage_case 'warning stays quiet at 79 percent' 79 11 ' []' @@ -162,7 +171,7 @@ ${ALERT_LABELS} exp_annotations: summary: platform-postgres에 26시간 안에 끝난 백업이 없음 description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요. - runbook_url: ${RUNBOOK_URL} + runbook_url: ${BACKUP_RUNBOOK_URL} - name: backup alert stays quiet after a fresh backup interval: 1m input_series: @@ -190,7 +199,7 @@ ${ALERT_LABELS} exp_annotations: summary: platform-postgres의 WAL이 보관소로 올라가지 않음 description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다. - runbook_url: ${RUNBOOK_URL} + runbook_url: ${BACKUP_RUNBOOK_URL} - name: WAL backlog stays quiet at 10 ready files interval: 1m input_series: @@ -245,7 +254,7 @@ docker run --rm \ --network none \ --volume "${temp_dir}:/work:ro" \ --entrypoint /bin/promtool \ - "$PROMETHEUS_IMAGE" check rules /work/core.rules.json + "$PROMETHEUS_IMAGE" check rules /work/core.rules.json /work/backup.rules.json docker run --rm \ --network none \ --volume "${temp_dir}:/work:ro" \ diff --git a/services/observability/rules/platform/core-rules.yaml b/services/observability/rules/platform/core-rules.yaml index 0370606..f56f76a 100644 --- a/services/observability/rules/platform/core-rules.yaml +++ b/services/observability/rules/platform/core-rules.yaml @@ -113,36 +113,6 @@ spec: summary: 루트 파일시스템 inode 사용률이 90%를 초과함 description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다. runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md - - name: platform.backup - rules: - # cnpg_collector_last_available_backup_timestamp is 0 until the first - # base backup, so a cluster without backups fires as well. - - alert: PlatformPostgresBackupMissing - expr: >- - time() - max by (job, namespace) ( - cnpg_collector_last_available_backup_timestamp{job="platform-postgres",namespace="platform-data"} - ) > 26 * 3600 - for: 30m - labels: - severity: critical - annotations: - summary: platform-postgres에 26시간 안에 끝난 백업이 없음 - description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요. - runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md - # A failing archive_command keeps WAL in pg_wal, which then fills the - # PostgreSQL volume. - - alert: PlatformPostgresWALArchiveBacklog - expr: >- - max by (job, namespace) ( - cnpg_collector_pg_wal_archive_status{job="platform-postgres",namespace="platform-data",value="ready"} - ) > 10 - for: 15m - labels: - severity: critical - annotations: - summary: platform-postgres의 WAL이 보관소로 올라가지 않음 - description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다. - runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md - name: platform.monitoring rules: # When the TSDB rejects every sample (2026-11-11: the clock went back diff --git a/services/platform-postgres-backup/README.md b/services/platform-postgres-backup/README.md index 0901aa5..71621d0 100644 --- a/services/platform-postgres-backup/README.md +++ b/services/platform-postgres-backup/README.md @@ -22,6 +22,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain) | `scheduled-backup.yaml` | 매일 18:00 UTC(03:00 KST) base backup. 만들자마자 한 번 | | `object-storage-counterpart.yaml` | `object-storage` 쪽 ingress: platform-data의 PostgreSQL 파드 → AIStor 9000 | | `bucket-policy.json` | 백업 사용자가 자기 bucket만 읽고 쓰는 정책 | +| `prometheus-rule.yaml` | 백업 경보 둘(아래). 관측 core 렌더러가 백업 규칙을 core 범위 밖으로 두므로 여기 둔다 | | `restore-drill/cluster.yaml` | 복원 리허설용 임시 Cluster | | `../../scripts/bootstrap/create-postgres-backup-credentials.sh` | bucket·정책·사용자·Secret 생성 | @@ -84,7 +85,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain) 4. 매일 백업을 건다. 만들자마자 첫 base backup이 돈다. ```bash - kubectl apply -k services/platform-postgres-backup + kubectl apply -k services/platform-postgres-backup # ScheduledBackup, AIStor ingress, 경보 kubectl -n platform-data get backup -w ``` @@ -93,7 +94,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain) 5. 아래 「복원 리허설」을 한 번 한다. -경보(`services/observability/rules/platform/core-rules.yaml`의 `platform.backup`): +경보(`prometheus-rule.yaml`, 4단계에서 같이 들어간다. platform-data는 Prometheus가 규칙을 읽는 namespace다): - `PlatformPostgresBackupMissing`: 26시간 안에 끝난 base backup이 없다. 위 4단계 전까지는 계속 울린다(백업이 정말 없으니까). @@ -104,7 +105,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain) ### 되돌리기 ```bash -kubectl delete -k services/platform-postgres-backup # ScheduledBackup과 ingress +kubectl delete -k services/platform-postgres-backup # ScheduledBackup, ingress, 경보 kubectl -n platform-data patch cluster platform-postgres --type json -p '[{"op":"remove","path":"/spec/backup"}]' ``` diff --git a/services/platform-postgres-backup/kustomization.yaml b/services/platform-postgres-backup/kustomization.yaml index 0a85c50..ac4ed59 100644 --- a/services/platform-postgres-backup/kustomization.yaml +++ b/services/platform-postgres-backup/kustomization.yaml @@ -8,6 +8,7 @@ kind: Kustomization resources: - object-storage-counterpart.yaml - scheduled-backup.yaml + - prometheus-rule.yaml labels: - pairs: diff --git a/services/platform-postgres-backup/prometheus-rule.yaml b/services/platform-postgres-backup/prometheus-rule.yaml new file mode 100644 index 0000000..5ab183d --- /dev/null +++ b/services/platform-postgres-backup/prometheus-rule.yaml @@ -0,0 +1,46 @@ +# Backup alerts for platform-postgres. They live with the backup and not in +# the observability core, whose renderer keeps backup rules out of its scope +# (scripts/validate/render-observability-core.sh, no_unscoped_monitors_rules). +# Prometheus loads PrometheusRules from namespaces labelled +# observability.hyeonworks.com/enabled=true, which platform-data is. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: platform-postgres-backup + namespace: platform-data + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: backup + observability.hyeonworks.com/instance: home +spec: + groups: + - name: platform.backup + rules: + # cnpg_collector_last_available_backup_timestamp is 0 until the first + # base backup, so a cluster without backups fires as well. + - alert: PlatformPostgresBackupMissing + expr: >- + time() - max by (job, namespace) ( + cnpg_collector_last_available_backup_timestamp{job="platform-postgres",namespace="platform-data"} + ) > 26 * 3600 + for: 30m + labels: + severity: critical + annotations: + summary: platform-postgres에 26시간 안에 끝난 백업이 없음 + description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md + # A failing archive_command keeps WAL in pg_wal, which then fills the + # PostgreSQL volume. + - alert: PlatformPostgresWALArchiveBacklog + expr: >- + max by (job, namespace) ( + cnpg_collector_pg_wal_archive_status{job="platform-postgres",namespace="platform-data",value="ready"} + ) > 10 + for: 15m + labels: + severity: critical + annotations: + summary: platform-postgres의 WAL이 보관소로 올라가지 않음 + description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md