refactor(postgres): keep the backup alerts with the backup
The observability core renderer rejects any alert or record whose name contains "backup" (no_unscoped_monitors_rules): backup monitoring is outside the observability core by design.f19ce54put PlatformPostgresBackupMissing and PlatformPostgresWALArchiveBacklog in core-rules.yaml, so the pinned renderer failed with "out-of-scope monitor or rule is present". Move both alerts unchanged into services/platform-postgres-backup/prometheus-rule.yaml in platform-data, which Prometheus reads because the namespace has observability.hyeonworks.com/enabled=true and the rule has observability.hyeonworks.com/instance=home. They now point their runbook_url at the backup README and are applied with the ScheduledBackup. Renaming them to dodge the filter was rejected: it would keep backup rules inside a scope that excludes them on purpose. Verification: test-platform-disk-alerts.sh loads both rule files and passes; promtool checks 13 rules. render-observability-core.sh and test-render-observability-core.sh pass, and kubectl apply --dry-run=server accepts the backup kustomization with the rule. scan-platform-sensitive-source.sh stops at its observability-core renderer with RC=23 here and on an origin/main snapshot alike, so it says nothing about this change. Fixes:f19ce54740("feat(postgres): back up platform-postgres to AIStor every day")
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Unit tests for core-rules.yaml alerts that guard the desktop host.
|
# Unit tests for the alerts that guard the desktop host: core-rules.yaml and
|
||||||
|
# services/platform-postgres-backup/prometheus-rule.yaml.
|
||||||
#
|
#
|
||||||
# Root disk: the kubelet starts deleting unused images at 85% use and evicts
|
# Root disk: the kubelet starts deleting unused images at 85% use and evicts
|
||||||
# pods at 90% (nodefs.available < 10%). The alerts sit in front of those two
|
# pods at 90% (nodefs.available < 10%). The alerts sit in front of those two
|
||||||
@@ -14,9 +15,11 @@ set -euo pipefail
|
|||||||
|
|
||||||
readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml"
|
readonly RULE_FILE="${ROOT_DIR}/services/observability/rules/platform/core-rules.yaml"
|
||||||
|
readonly BACKUP_RULE_FILE="${ROOT_DIR}/services/platform-postgres-backup/prometheus-rule.yaml"
|
||||||
# Same pinned image as test-platform-observability-rules.sh.
|
# Same pinned image as test-platform-observability-rules.sh.
|
||||||
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
||||||
readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"
|
readonly RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"
|
||||||
|
readonly BACKUP_RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md"
|
||||||
|
|
||||||
temp_dir=''
|
temp_dir=''
|
||||||
cleanup() {
|
cleanup() {
|
||||||
@@ -40,11 +43,16 @@ chmod 0755 "$temp_dir"
|
|||||||
|
|
||||||
kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json |
|
kubectl create --dry-run=client --validate=false -f "$RULE_FILE" -o json |
|
||||||
jq '.spec' >"${temp_dir}/core.rules.json"
|
jq '.spec' >"${temp_dir}/core.rules.json"
|
||||||
|
kubectl create --dry-run=client --validate=false -f "$BACKUP_RULE_FILE" -o json |
|
||||||
|
jq '.spec' >"${temp_dir}/backup.rules.json"
|
||||||
|
|
||||||
# Every core alert keeps the runbook the other platform rules use.
|
# Every core alert keeps the runbook the other platform rules use.
|
||||||
jq -e --arg url "$RUNBOOK_URL" '
|
jq -e --arg url "$RUNBOOK_URL" '
|
||||||
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
||||||
' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url"
|
' "${temp_dir}/core.rules.json" >/dev/null || fail "core alert without the shared runbook_url"
|
||||||
|
jq -e --arg url "$BACKUP_RUNBOOK_URL" '
|
||||||
|
all(.groups[].rules[] | select(has("alert")); .annotations.runbook_url == $url)
|
||||||
|
' "${temp_dir}/backup.rules.json" >/dev/null || fail "backup alert without the backup README runbook_url"
|
||||||
|
|
||||||
# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter.
|
# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter.
|
||||||
readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"'
|
readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"'
|
||||||
@@ -127,6 +135,7 @@ ${ALERT_LABELS}
|
|||||||
printf '%s\n' \
|
printf '%s\n' \
|
||||||
'rule_files:' \
|
'rule_files:' \
|
||||||
' - /work/core.rules.json' \
|
' - /work/core.rules.json' \
|
||||||
|
' - /work/backup.rules.json' \
|
||||||
'evaluation_interval: 1m' \
|
'evaluation_interval: 1m' \
|
||||||
'tests:'
|
'tests:'
|
||||||
usage_case 'warning stays quiet at 79 percent' 79 11 ' []'
|
usage_case 'warning stays quiet at 79 percent' 79 11 ' []'
|
||||||
@@ -162,7 +171,7 @@ ${ALERT_LABELS}
|
|||||||
exp_annotations:
|
exp_annotations:
|
||||||
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
||||||
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
||||||
runbook_url: ${RUNBOOK_URL}
|
runbook_url: ${BACKUP_RUNBOOK_URL}
|
||||||
- name: backup alert stays quiet after a fresh backup
|
- name: backup alert stays quiet after a fresh backup
|
||||||
interval: 1m
|
interval: 1m
|
||||||
input_series:
|
input_series:
|
||||||
@@ -190,7 +199,7 @@ ${ALERT_LABELS}
|
|||||||
exp_annotations:
|
exp_annotations:
|
||||||
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
||||||
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
||||||
runbook_url: ${RUNBOOK_URL}
|
runbook_url: ${BACKUP_RUNBOOK_URL}
|
||||||
- name: WAL backlog stays quiet at 10 ready files
|
- name: WAL backlog stays quiet at 10 ready files
|
||||||
interval: 1m
|
interval: 1m
|
||||||
input_series:
|
input_series:
|
||||||
@@ -245,7 +254,7 @@ docker run --rm \
|
|||||||
--network none \
|
--network none \
|
||||||
--volume "${temp_dir}:/work:ro" \
|
--volume "${temp_dir}:/work:ro" \
|
||||||
--entrypoint /bin/promtool \
|
--entrypoint /bin/promtool \
|
||||||
"$PROMETHEUS_IMAGE" check rules /work/core.rules.json
|
"$PROMETHEUS_IMAGE" check rules /work/core.rules.json /work/backup.rules.json
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
--network none \
|
--network none \
|
||||||
--volume "${temp_dir}:/work:ro" \
|
--volume "${temp_dir}:/work:ro" \
|
||||||
|
|||||||
@@ -113,36 +113,6 @@ spec:
|
|||||||
summary: 루트 파일시스템 inode 사용률이 90%를 초과함
|
summary: 루트 파일시스템 inode 사용률이 90%를 초과함
|
||||||
description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다.
|
description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다.
|
||||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||||
- name: platform.backup
|
|
||||||
rules:
|
|
||||||
# cnpg_collector_last_available_backup_timestamp is 0 until the first
|
|
||||||
# base backup, so a cluster without backups fires as well.
|
|
||||||
- alert: PlatformPostgresBackupMissing
|
|
||||||
expr: >-
|
|
||||||
time() - max by (job, namespace) (
|
|
||||||
cnpg_collector_last_available_backup_timestamp{job="platform-postgres",namespace="platform-data"}
|
|
||||||
) > 26 * 3600
|
|
||||||
for: 30m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
|
||||||
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
|
||||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
|
||||||
# A failing archive_command keeps WAL in pg_wal, which then fills the
|
|
||||||
# PostgreSQL volume.
|
|
||||||
- alert: PlatformPostgresWALArchiveBacklog
|
|
||||||
expr: >-
|
|
||||||
max by (job, namespace) (
|
|
||||||
cnpg_collector_pg_wal_archive_status{job="platform-postgres",namespace="platform-data",value="ready"}
|
|
||||||
) > 10
|
|
||||||
for: 15m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
|
||||||
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
|
||||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
|
||||||
- name: platform.monitoring
|
- name: platform.monitoring
|
||||||
rules:
|
rules:
|
||||||
# When the TSDB rejects every sample (2026-11-11: the clock went back
|
# When the TSDB rejects every sample (2026-11-11: the clock went back
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain)
|
|||||||
| `scheduled-backup.yaml` | 매일 18:00 UTC(03:00 KST) base backup. 만들자마자 한 번 |
|
| `scheduled-backup.yaml` | 매일 18:00 UTC(03:00 KST) base backup. 만들자마자 한 번 |
|
||||||
| `object-storage-counterpart.yaml` | `object-storage` 쪽 ingress: platform-data의 PostgreSQL 파드 → AIStor 9000 |
|
| `object-storage-counterpart.yaml` | `object-storage` 쪽 ingress: platform-data의 PostgreSQL 파드 → AIStor 9000 |
|
||||||
| `bucket-policy.json` | 백업 사용자가 자기 bucket만 읽고 쓰는 정책 |
|
| `bucket-policy.json` | 백업 사용자가 자기 bucket만 읽고 쓰는 정책 |
|
||||||
|
| `prometheus-rule.yaml` | 백업 경보 둘(아래). 관측 core 렌더러가 백업 규칙을 core 범위 밖으로 두므로 여기 둔다 |
|
||||||
| `restore-drill/cluster.yaml` | 복원 리허설용 임시 Cluster |
|
| `restore-drill/cluster.yaml` | 복원 리허설용 임시 Cluster |
|
||||||
| `../../scripts/bootstrap/create-postgres-backup-credentials.sh` | bucket·정책·사용자·Secret 생성 |
|
| `../../scripts/bootstrap/create-postgres-backup-credentials.sh` | bucket·정책·사용자·Secret 생성 |
|
||||||
|
|
||||||
@@ -84,7 +85,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain)
|
|||||||
4. 매일 백업을 건다. 만들자마자 첫 base backup이 돈다.
|
4. 매일 백업을 건다. 만들자마자 첫 base backup이 돈다.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
kubectl apply -k services/platform-postgres-backup
|
kubectl apply -k services/platform-postgres-backup # ScheduledBackup, AIStor ingress, 경보
|
||||||
kubectl -n platform-data get backup -w
|
kubectl -n platform-data get backup -w
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -93,7 +94,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain)
|
|||||||
|
|
||||||
5. 아래 「복원 리허설」을 한 번 한다.
|
5. 아래 「복원 리허설」을 한 번 한다.
|
||||||
|
|
||||||
경보(`services/observability/rules/platform/core-rules.yaml`의 `platform.backup`):
|
경보(`prometheus-rule.yaml`, 4단계에서 같이 들어간다. platform-data는 Prometheus가 규칙을 읽는 namespace다):
|
||||||
|
|
||||||
- `PlatformPostgresBackupMissing`: 26시간 안에 끝난 base backup이 없다. 위 4단계 전까지는 계속
|
- `PlatformPostgresBackupMissing`: 26시간 안에 끝난 base backup이 없다. 위 4단계 전까지는 계속
|
||||||
울린다(백업이 정말 없으니까).
|
울린다(백업이 정말 없으니까).
|
||||||
@@ -104,7 +105,7 @@ AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain)
|
|||||||
### 되돌리기
|
### 되돌리기
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
kubectl delete -k services/platform-postgres-backup # ScheduledBackup과 ingress
|
kubectl delete -k services/platform-postgres-backup # ScheduledBackup, ingress, 경보
|
||||||
kubectl -n platform-data patch cluster platform-postgres --type json -p '[{"op":"remove","path":"/spec/backup"}]'
|
kubectl -n platform-data patch cluster platform-postgres --type json -p '[{"op":"remove","path":"/spec/backup"}]'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ kind: Kustomization
|
|||||||
resources:
|
resources:
|
||||||
- object-storage-counterpart.yaml
|
- object-storage-counterpart.yaml
|
||||||
- scheduled-backup.yaml
|
- scheduled-backup.yaml
|
||||||
|
- prometheus-rule.yaml
|
||||||
|
|
||||||
labels:
|
labels:
|
||||||
- pairs:
|
- pairs:
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Backup alerts for platform-postgres. They live with the backup and not in
|
||||||
|
# the observability core, whose renderer keeps backup rules out of its scope
|
||||||
|
# (scripts/validate/render-observability-core.sh, no_unscoped_monitors_rules).
|
||||||
|
# Prometheus loads PrometheusRules from namespaces labelled
|
||||||
|
# observability.hyeonworks.com/enabled=true, which platform-data is.
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: platform-postgres-backup
|
||||||
|
namespace: platform-data
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: platform-postgres
|
||||||
|
app.kubernetes.io/component: backup
|
||||||
|
observability.hyeonworks.com/instance: home
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: platform.backup
|
||||||
|
rules:
|
||||||
|
# cnpg_collector_last_available_backup_timestamp is 0 until the first
|
||||||
|
# base backup, so a cluster without backups fires as well.
|
||||||
|
- alert: PlatformPostgresBackupMissing
|
||||||
|
expr: >-
|
||||||
|
time() - max by (job, namespace) (
|
||||||
|
cnpg_collector_last_available_backup_timestamp{job="platform-postgres",namespace="platform-data"}
|
||||||
|
) > 26 * 3600
|
||||||
|
for: 30m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
||||||
|
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
||||||
|
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md
|
||||||
|
# A failing archive_command keeps WAL in pg_wal, which then fills the
|
||||||
|
# PostgreSQL volume.
|
||||||
|
- alert: PlatformPostgresWALArchiveBacklog
|
||||||
|
expr: >-
|
||||||
|
max by (job, namespace) (
|
||||||
|
cnpg_collector_pg_wal_archive_status{job="platform-postgres",namespace="platform-data",value="ready"}
|
||||||
|
) > 10
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
||||||
|
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
||||||
|
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/platform-core/src/branch/main/services/platform-postgres-backup/README.md
|
||||||
Reference in New Issue
Block a user