feat(postgres): back up platform-postgres to AIStor every day
platform-postgres holds the Gitea, Keycloak and tech-log databases and had no backup: no spec.backup, no ScheduledBackup, and its only copy is a local PV on the root SSD. A lost SSD or a bad migration had no way back. Add services/platform-postgres-backup: a merge patch that gives the live Cluster barman-cloud WAL archiving and base backups to s3://platform-postgres-backups on the AIStor ObjectStore (HDD sdb3) with 30-day retention and gzip, a daily 03:00 KST ScheduledBackup that also runs once at creation, the object-storage ingress for the PostgreSQL pods, the bucket policy, and a restore drill Cluster that only reads the bucket. scripts/bootstrap/create-postgres-backup- credentials.sh creates the bucket (100GiB quota), policy, user and the platform-data Secret through a restricted AIStor client Pod, proves the user can write its bucket and cannot list loki, and prints no key. Add PlatformPostgresBackupMissing (no base backup in 26h) and PlatformPostgresWALArchiveBacklog (more than 10 WAL files waiting for 15m; a stuck archive fills the SSD volume). The native barmanObjectStore is deprecated since CNPG 1.26 and the operator warns it is removed in 1.31.0. The Barman Cloud Plugin needs cert-manager, which this cluster does not have; installing two operators before the first backup is slower and riskier. The bucket layout is the same, so the plugin can take over before CNPG 1.31. Volume snapshots do not work on the static local PV. Risk: the live Cluster has pg_hba lines for techlog that the Phase 1 source lacks, so re-applying services/platform-postgres would cut the tech-log backend off. The README applies only spec.backup with kubectl patch. Backups stay on the same desktop; an off-machine copy is not part of this change. Nothing was applied to the cluster. Verification: kubectl apply --dry-run=server accepts the ScheduledBackup, the NetworkPolicy and the drill Cluster; kubectl patch --dry-run=server keeps both techlog pg_hba lines and adds spec.backup. The row-count comparison from the drill steps ran read-only on production (keycloak 100, techlog 44, gitea 116 tables). mc command syntax was checked with the pinned AIStor client offline. The two new alert tests failed against the old rules and pass now. bash -n passes for the script; shellcheck is not installed on the laptop or the desktop.
This commit is contained in:
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create the AIStor bucket, policy and user that CloudNativePG uses to back up
|
||||
# platform-postgres, and the platform-data/platform-postgres-backup-s3 Secret.
|
||||
#
|
||||
# Run once, by the owner, from a platform-core checkout on the desktop. The
|
||||
# script refuses to run when the Secret already exists; it never rotates or
|
||||
# deletes a bucket, policy, user or object. No credential value is printed.
|
||||
set -Eeuo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
readonly EXPECTED_CONTEXT=default
|
||||
readonly OS_NAMESPACE=object-storage
|
||||
readonly PG_NAMESPACE=platform-data
|
||||
readonly ROOT_SECRET=aistor-root-configuration
|
||||
readonly TARGET_SECRET=platform-postgres-backup-s3
|
||||
readonly BUCKET=platform-postgres-backups
|
||||
readonly POLICY=platform-postgres-backups
|
||||
readonly QUOTA=100GiB
|
||||
readonly S3_ENDPOINT=http://minio.object-storage.svc.cluster.local
|
||||
# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh.
|
||||
readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b"
|
||||
readonly POLICY_FILE="${ROOT}/services/platform-postgres-backup/bucket-policy.json"
|
||||
readonly PASS_LINE=PLATFORM_POSTGRES_BACKUP_BOOTSTRAP_PASS
|
||||
|
||||
fail() {
|
||||
printf 'FAIL: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
for command in kubectl jq openssl; do
|
||||
command -v "$command" >/dev/null || fail "missing command: ${command}"
|
||||
done
|
||||
[[ -f "$POLICY_FILE" ]] || fail "missing ${POLICY_FILE}"
|
||||
jq -e . "$POLICY_FILE" >/dev/null || fail "bucket policy is not valid JSON"
|
||||
|
||||
[[ "$(kubectl config current-context)" == "$EXPECTED_CONTEXT" ]] ||
|
||||
fail "kube context must be ${EXPECTED_CONTEXT}"
|
||||
if kubectl --namespace "$PG_NAMESPACE" get secret "$TARGET_SECRET" >/dev/null 2>&1; then
|
||||
fail "${PG_NAMESPACE}/${TARGET_SECRET} already exists; this script does not rotate it"
|
||||
fi
|
||||
kubectl --namespace "$OS_NAMESPACE" get secret "$ROOT_SECRET" >/dev/null ||
|
||||
fail "${OS_NAMESPACE}/${ROOT_SECRET} is missing"
|
||||
|
||||
work="$(mktemp -d)"
|
||||
pod="pg-backup-bootstrap-$(date +%H%M%S)-$$"
|
||||
handoff_secret="${pod}-keys"
|
||||
policy_map="${pod}-policy"
|
||||
|
||||
cleanup() {
|
||||
kubectl --namespace "$OS_NAMESPACE" delete pod "$pod" \
|
||||
--ignore-not-found --wait=false >/dev/null 2>&1 || true
|
||||
kubectl --namespace "$OS_NAMESPACE" delete secret "$handoff_secret" \
|
||||
--ignore-not-found >/dev/null 2>&1 || true
|
||||
kubectl --namespace "$OS_NAMESPACE" delete configmap "$policy_map" \
|
||||
--ignore-not-found >/dev/null 2>&1 || true
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
printf 'pgbackup%s' "$(openssl rand -hex 8)" >"${work}/ACCESS_KEY_ID"
|
||||
openssl rand -hex 32 | tr -d '\n' >"${work}/ACCESS_SECRET_KEY"
|
||||
|
||||
printf '%s\n' \
|
||||
"Bucket: ${BUCKET} (quota ${QUOTA}) on AIStor ${S3_ENDPOINT}" \
|
||||
"Policy: ${POLICY} (${POLICY_FILE#"${ROOT}/"})" \
|
||||
"User: a new generated access key attached to ${POLICY}" \
|
||||
"Secret: ${PG_NAMESPACE}/${TARGET_SECRET} (ACCESS_KEY_ID, ACCESS_SECRET_KEY)" \
|
||||
"Temporary: one restricted client Pod, one Secret and one ConfigMap in ${OS_NAMESPACE}" \
|
||||
'No credential value will be printed.'
|
||||
printf 'Type APPLY to create them: '
|
||||
IFS= read -r confirmation
|
||||
[[ "$confirmation" == "APPLY" ]] || fail "confirmation did not match APPLY"
|
||||
|
||||
kubectl --namespace "$OS_NAMESPACE" create secret generic "$handoff_secret" \
|
||||
--from-file=ACCESS_KEY_ID="${work}/ACCESS_KEY_ID" \
|
||||
--from-file=ACCESS_SECRET_KEY="${work}/ACCESS_SECRET_KEY" >/dev/null
|
||||
kubectl --namespace "$OS_NAMESPACE" create configmap "$policy_map" \
|
||||
--from-file=policy.json="$POLICY_FILE" >/dev/null
|
||||
|
||||
# Every mc call is silenced so neither key reaches the Pod log.
|
||||
bootstrap_command='set -eu
|
||||
. /run/aistor-root/config.env
|
||||
access_key="$(cat /run/backup/ACCESS_KEY_ID)"
|
||||
secret_key="$(cat /run/backup/ACCESS_SECRET_KEY)"
|
||||
mc alias set root "'"$S3_ENDPOINT"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null
|
||||
mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null
|
||||
mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null
|
||||
mc admin policy create root "'"$POLICY"'" /run/policy/policy.json >/dev/null
|
||||
mc admin user add root "$access_key" "$secret_key" >/dev/null
|
||||
mc admin policy attach root "'"$POLICY"'" --user "$access_key" >/dev/null
|
||||
mc alias set backup "'"$S3_ENDPOINT"'" "$access_key" "$secret_key" --api S3v4 >/dev/null
|
||||
probe=".bootstrap-probe-$(date +%s)"
|
||||
printf "probe" | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null
|
||||
[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = "probe" ]
|
||||
mc rm "backup/'"$BUCKET"'/$probe" >/dev/null
|
||||
if mc ls backup/loki >/dev/null 2>&1; then
|
||||
echo "the backup user can list another bucket" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf "%s\n" "'"$PASS_LINE"'"'
|
||||
|
||||
overrides="$(
|
||||
jq -nc --arg command "$bootstrap_command" --arg image "$MC_IMAGE" \
|
||||
--arg root "$ROOT_SECRET" --arg keys "$handoff_secret" --arg policy "$policy_map" '{
|
||||
spec: {
|
||||
automountServiceAccountToken: false,
|
||||
securityContext: {
|
||||
runAsNonRoot: true,
|
||||
runAsUser: 1000,
|
||||
runAsGroup: 1000,
|
||||
fsGroup: 1000,
|
||||
seccompProfile: {type: "RuntimeDefault"}
|
||||
},
|
||||
containers: [{
|
||||
name: "pg-backup-bootstrap",
|
||||
image: $image,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/bin/sh", "-ec"],
|
||||
args: [$command],
|
||||
securityContext: {
|
||||
allowPrivilegeEscalation: false,
|
||||
capabilities: {drop: ["ALL"]},
|
||||
readOnlyRootFilesystem: true
|
||||
},
|
||||
env: [{name: "MC_CONFIG_DIR", value: "/tmp/mc"}],
|
||||
volumeMounts: [
|
||||
{name: "root-config", mountPath: "/run/aistor-root", readOnly: true},
|
||||
{name: "backup-keys", mountPath: "/run/backup", readOnly: true},
|
||||
{name: "policy", mountPath: "/run/policy", readOnly: true},
|
||||
{name: "tmp", mountPath: "/tmp"}
|
||||
]
|
||||
}],
|
||||
volumes: [
|
||||
{name: "root-config", secret: {secretName: $root, defaultMode: 256}},
|
||||
{name: "backup-keys", secret: {secretName: $keys, defaultMode: 256}},
|
||||
{name: "policy", configMap: {name: $policy}},
|
||||
{name: "tmp", emptyDir: {}}
|
||||
]
|
||||
}
|
||||
}'
|
||||
)"
|
||||
|
||||
kubectl --namespace "$OS_NAMESPACE" run "$pod" \
|
||||
--image="$MC_IMAGE" \
|
||||
--restart=Never \
|
||||
--labels='platform.hyeonworks.com/aistor-client=true,app.kubernetes.io/name=pg-backup-bootstrap' \
|
||||
--overrides="$overrides" >/dev/null
|
||||
|
||||
phase=''
|
||||
for _ in $(seq 1 90); do
|
||||
phase="$(kubectl --namespace "$OS_NAMESPACE" get pod "$pod" --output=jsonpath='{.status.phase}')"
|
||||
[[ "$phase" == "Succeeded" || "$phase" == "Failed" ]] && break
|
||||
sleep 2
|
||||
done
|
||||
logs="$(kubectl --namespace "$OS_NAMESPACE" logs "$pod" 2>&1 || true)"
|
||||
if [[ "$phase" != "Succeeded" || "$logs" != *"$PASS_LINE"* ]]; then
|
||||
printf '%s\n' "$logs" | tail -20 >&2
|
||||
fail "bootstrap Pod ended in phase '${phase}'; the platform-data Secret was not created"
|
||||
fi
|
||||
|
||||
kubectl --namespace "$PG_NAMESPACE" create secret generic "$TARGET_SECRET" \
|
||||
--from-file=ACCESS_KEY_ID="${work}/ACCESS_KEY_ID" \
|
||||
--from-file=ACCESS_SECRET_KEY="${work}/ACCESS_SECRET_KEY" >/dev/null
|
||||
kubectl --namespace "$PG_NAMESPACE" label secret "$TARGET_SECRET" \
|
||||
app.kubernetes.io/part-of=platform app.kubernetes.io/component=platform-postgres-backup >/dev/null
|
||||
|
||||
printf '%s\n' "$PASS_LINE" \
|
||||
"Created bucket ${BUCKET}, policy ${POLICY}, one user and ${PG_NAMESPACE}/${TARGET_SECRET}."
|
||||
@@ -1,10 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Unit tests for the root filesystem alerts in core-rules.yaml.
|
||||
# Unit tests for core-rules.yaml alerts that guard the desktop host.
|
||||
#
|
||||
# The kubelet starts deleting unused images at 85% use and evicts pods at
|
||||
# 90% (nodefs.available < 10%). These tests pin the alerts in front of those
|
||||
# two points: warning at 80%, critical at 85%, and a forecast that fires when
|
||||
# the 6h trend reaches the eviction point within 6 hours.
|
||||
# Root disk: the kubelet starts deleting unused images at 85% use and evicts
|
||||
# pods at 90% (nodefs.available < 10%). The alerts sit in front of those two
|
||||
# points: warning at 80%, critical at 85%, and a forecast that fires when the
|
||||
# 6h trend reaches the eviction point within 6 hours.
|
||||
# PostgreSQL backups: no successful base backup for 26 hours, or WAL waiting
|
||||
# to be archived.
|
||||
#
|
||||
# Needs docker, jq and kubectl (client only, no cluster access).
|
||||
set -euo pipefail
|
||||
@@ -45,6 +47,7 @@ jq -e --arg url "$RUNBOOK_URL" '
|
||||
|
||||
# node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter.
|
||||
readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"'
|
||||
readonly PG_LABELS='job="platform-postgres",namespace="platform-data",pod="platform-postgres-1",instance="10.42.0.130:9187"'
|
||||
readonly ALERT_LABELS=' job: prometheus-node-exporter
|
||||
instance: 10.42.0.95:9100
|
||||
device: /dev/sdc1
|
||||
@@ -137,6 +140,66 @@ ${ALERT_LABELS}
|
||||
# Losing 0.1 per minute takes far longer than 6 hours to reach 100.
|
||||
forecast_case 'forecast stays quiet for a slow fill' 400 '-0.1' 30 ' []'
|
||||
forecast_case 'forecast stays quiet for flat usage' 400 '+0' 30 ' []'
|
||||
|
||||
# PostgreSQL backups. Timestamps are seconds since the test epoch.
|
||||
cat <<EOF
|
||||
# The test clock starts at 0, so "27.8 hours ago" is a negative timestamp.
|
||||
# In production a cluster without any backup reports 0 and fires the same way.
|
||||
- name: backup alert fires when the last backup is older than 26 hours
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'cnpg_collector_last_available_backup_timestamp{${PG_LABELS}}'
|
||||
values: '-100000+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 31m
|
||||
alertname: PlatformPostgresBackupMissing
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
job: platform-postgres
|
||||
namespace: platform-data
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
||||
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
||||
runbook_url: ${RUNBOOK_URL}
|
||||
- name: backup alert stays quiet after a fresh backup
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'cnpg_collector_last_available_backup_timestamp{${PG_LABELS}}'
|
||||
values: '60+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 31m
|
||||
alertname: PlatformPostgresBackupMissing
|
||||
exp_alerts: []
|
||||
- name: WAL backlog fires after 15 minutes
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'cnpg_collector_pg_wal_archive_status{${PG_LABELS},value="ready"}'
|
||||
values: '11+0x30'
|
||||
- series: 'cnpg_collector_pg_wal_archive_status{${PG_LABELS},value="done"}'
|
||||
values: '500+0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 16m
|
||||
alertname: PlatformPostgresWALArchiveBacklog
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
job: platform-postgres
|
||||
namespace: platform-data
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
||||
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
||||
runbook_url: ${RUNBOOK_URL}
|
||||
- name: WAL backlog stays quiet at 10 ready files
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'cnpg_collector_pg_wal_archive_status{${PG_LABELS},value="ready"}'
|
||||
values: '10+0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 20m
|
||||
alertname: PlatformPostgresWALArchiveBacklog
|
||||
exp_alerts: []
|
||||
EOF
|
||||
} >"${temp_dir}/disk.test.yaml"
|
||||
|
||||
chmod 0444 "${temp_dir}"/*
|
||||
@@ -150,4 +213,4 @@ docker run --rm \
|
||||
--volume "${temp_dir}:/work:ro" \
|
||||
--entrypoint /bin/promtool \
|
||||
"$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml
|
||||
printf 'PASS: root filesystem usage and forecast alerts\n'
|
||||
printf 'PASS: root filesystem and PostgreSQL backup alerts\n'
|
||||
|
||||
@@ -113,6 +113,36 @@ spec:
|
||||
summary: 루트 파일시스템 inode 사용률이 90%를 초과함
|
||||
description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다.
|
||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||
- name: platform.backup
|
||||
rules:
|
||||
# cnpg_collector_last_available_backup_timestamp is 0 until the first
|
||||
# base backup, so a cluster without backups fires as well.
|
||||
- alert: PlatformPostgresBackupMissing
|
||||
expr: >-
|
||||
time() - max by (job, namespace) (
|
||||
cnpg_collector_last_available_backup_timestamp{job="platform-postgres",namespace="platform-data"}
|
||||
) > 26 * 3600
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: platform-postgres에 26시간 안에 끝난 백업이 없음
|
||||
description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요.
|
||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||
# A failing archive_command keeps WAL in pg_wal, which then fills the
|
||||
# PostgreSQL volume.
|
||||
- alert: PlatformPostgresWALArchiveBacklog
|
||||
expr: >-
|
||||
max by (job, namespace) (
|
||||
cnpg_collector_pg_wal_archive_status{job="platform-postgres",namespace="platform-data",value="ready"}
|
||||
) > 10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: platform-postgres의 WAL이 보관소로 올라가지 않음
|
||||
description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다.
|
||||
runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md
|
||||
- name: platform.workloads
|
||||
rules:
|
||||
- alert: PlatformPodRestartBurst
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
# 플랫폼 PostgreSQL 백업
|
||||
|
||||
`platform-data/platform-postgres` 하나에 Gitea, Keycloak, tech-log의 DB가 같이 있다
|
||||
(2026-11-11: gitea 608MB, keycloak 14MB, techlog 14MB). 2026-11-11까지 백업 설정도,
|
||||
ScheduledBackup도 없었다. 그 PV는 루트 SSD 하나에만 있었다.
|
||||
|
||||
이 폴더는 매일 base backup과 WAL 연속 보관을 AIStor(HDD `sdb3`)에 두고, 30일 안의
|
||||
아무 시점으로나 되돌릴 수 있게 한다.
|
||||
|
||||
## 무엇이 어디로 가나
|
||||
|
||||
```text
|
||||
platform-postgres-1 (SSD)
|
||||
├─ WAL 파일이 찰 때마다, 그리고 5분마다 ─▶ s3://platform-postgres-backups/platform-postgres/wals/
|
||||
└─ 매일 03:00 KST base backup ─────────▶ s3://platform-postgres-backups/platform-postgres/base/
|
||||
AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain)
|
||||
```
|
||||
|
||||
| 파일 | 하는 일 |
|
||||
| --- | --- |
|
||||
| `cluster-backup-patch.yaml` | live Cluster에 `spec.backup`(보관소, 30일 보존, gzip)을 더한다 |
|
||||
| `scheduled-backup.yaml` | 매일 18:00 UTC(03:00 KST) base backup. 만들자마자 한 번 |
|
||||
| `object-storage-counterpart.yaml` | `object-storage` 쪽 ingress: platform-data의 PostgreSQL 파드 → AIStor 9000 |
|
||||
| `bucket-policy.json` | 백업 사용자가 자기 bucket만 읽고 쓰는 정책 |
|
||||
| `restore-drill/cluster.yaml` | 복원 리허설용 임시 Cluster |
|
||||
| `../../scripts/bootstrap/create-postgres-backup-credentials.sh` | bucket·정책·사용자·Secret 생성 |
|
||||
|
||||
## 고른 방법
|
||||
|
||||
- **내장 `barmanObjectStore`.** CloudNativePG 1.26부터 deprecated이고 Barman Cloud Plugin이
|
||||
권장이다. 하지만 plugin은 cert-manager가 필요한데 이 클러스터에는 없다(2026-11-11 CRD 없음).
|
||||
백업이 하나도 없는 지금은 새 operator 두 개(cert-manager, plugin)를 먼저 세우는 것보다 지금
|
||||
operator(1.30.0)가 바로 지원하는 쪽이 빠르고 위험이 작다. bucket 구조가 같아 plugin으로
|
||||
옮겨도 백업을 그대로 쓴다.
|
||||
- **기한:** operator 경고는 「1.31.0에서 완전히 제거」라고 한다(문서는 1.32.0). 그래서
|
||||
**CNPG를 1.31로 올리기 전에** Barman Cloud Plugin으로 옮긴다.
|
||||
- **보관소는 AIStor.** 같은 desktop이지만 SSD가 아닌 HDD(`sdb3`, 900Gi)다. SSD가 죽어도
|
||||
남는다. desktop 자체가 죽으면 같이 잃는다. 다른 기계·외부로 복사하는 일은 남은 과제다.
|
||||
- **보존 30일, quota 100GiB.** 압축한 base backup이 하루 수백 MB, WAL은 5분마다 한 파일
|
||||
(`archive_timeout` 기본값)이라 30일에 수십 GB 안쪽이다.
|
||||
- **볼륨 스냅숏은 쓰지 않는다.** `ssd-local-postgres-retain`은 정적 local PV라 CSI 스냅숏이 없다.
|
||||
|
||||
## 적용 (소유자, 순서대로)
|
||||
|
||||
전제: 이 PR이 병합된 platform-core checkout, desktop의 kube context `default`.
|
||||
|
||||
**주의: `services/platform-postgres`를 통째로 다시 적용하지 않는다.** live Cluster의
|
||||
`pg_hba`에는 Phase 1 원본에 없는 `host techlog techlog all scram-sha-256`과
|
||||
`host all techlog all reject`가 있다. 원본을 다시 적용하면 이 두 줄이 지워져 tech-log 운영
|
||||
백엔드가 DB에 붙지 못한다. 그래서 백업 설정만 `kubectl patch`로 더한다.
|
||||
|
||||
1. bucket, 정책, 사용자, Secret을 만든다. 화면에 자격 증명이 나오지 않는다.
|
||||
|
||||
```bash
|
||||
bash scripts/bootstrap/create-postgres-backup-credentials.sh
|
||||
```
|
||||
|
||||
마지막 줄에 `PLATFORM_POSTGRES_BACKUP_BOOTSTRAP_PASS`가 나와야 한다. 새 사용자가 자기
|
||||
bucket에 쓰고 읽고 지울 수 있고, `loki` bucket은 못 보는 것까지 확인한다.
|
||||
|
||||
2. AIStor 쪽 ingress를 연다. 그 전에는 PostgreSQL이 9000에 닿지 못한다.
|
||||
|
||||
```bash
|
||||
kubectl apply -f services/platform-postgres-backup/object-storage-counterpart.yaml
|
||||
```
|
||||
|
||||
3. live Cluster에 백업 설정을 더한다. PostgreSQL은 재시작하지 않는다.
|
||||
|
||||
```bash
|
||||
kubectl -n platform-data patch cluster platform-postgres --type merge \
|
||||
--patch-file services/platform-postgres-backup/cluster-backup-patch.yaml --dry-run=server -o name
|
||||
kubectl -n platform-data patch cluster platform-postgres --type merge \
|
||||
--patch-file services/platform-postgres-backup/cluster-backup-patch.yaml
|
||||
kubectl -n platform-data get cluster platform-postgres -o jsonpath='{.spec.postgresql.pg_hba}{"\n"}'
|
||||
```
|
||||
|
||||
마지막 줄에 `techlog` 두 줄이 그대로 있어야 한다. 몇 분 뒤 WAL이 올라가는지 본다.
|
||||
|
||||
```bash
|
||||
kubectl -n platform-data get cluster platform-postgres \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}{" "}{.status.firstRecoverabilityPoint}{"\n"}'
|
||||
```
|
||||
|
||||
4. 매일 백업을 건다. 만들자마자 첫 base backup이 돈다.
|
||||
|
||||
```bash
|
||||
kubectl apply -k services/platform-postgres-backup
|
||||
kubectl -n platform-data get backup -w
|
||||
```
|
||||
|
||||
`PHASE`가 `completed`가 되면 된다. `failed`면
|
||||
`kubectl -n platform-data describe backup <이름>`의 마지막 줄을 본다.
|
||||
|
||||
5. 아래 「복원 리허설」을 한 번 한다.
|
||||
|
||||
경보(`services/observability/rules/platform/core-rules.yaml`의 `platform.backup`):
|
||||
|
||||
- `PlatformPostgresBackupMissing`: 26시간 안에 끝난 base backup이 없다. 위 4단계 전까지는 계속
|
||||
울린다(백업이 정말 없으니까).
|
||||
- `PlatformPostgresWALArchiveBacklog`: 보관을 기다리는 WAL이 10개를 넘은 채 15분.
|
||||
보관이 막히면 WAL이 SSD 볼륨에 쌓여 PostgreSQL이 멈춘다. 2·3단계의 순서가 바뀌었거나
|
||||
Secret이 틀리면 이것이 울린다.
|
||||
|
||||
### 되돌리기
|
||||
|
||||
```bash
|
||||
kubectl delete -k services/platform-postgres-backup # ScheduledBackup과 ingress
|
||||
kubectl -n platform-data patch cluster platform-postgres --type json -p '[{"op":"remove","path":"/spec/backup"}]'
|
||||
```
|
||||
|
||||
bucket, 사용자, Secret, 이미 올린 백업은 지우지 않는다. 지울 때는 소유자가 `mc`로 직접 한다.
|
||||
|
||||
## 복원 리허설
|
||||
|
||||
백업이 실제로 복원되는지는 복원해 봐야 안다. 첫 base backup이 끝난 뒤 한 번, 그 뒤로는 CNPG나
|
||||
PostgreSQL을 올릴 때마다 한다. 운영 Cluster는 건드리지 않는다. 임시 Cluster는 bucket을 읽기만 한다
|
||||
(`backup` 칸이 없다).
|
||||
|
||||
1. 임시 Cluster를 만든다. local-path 5Gi(루트 SSD)를 잠깐 쓴다.
|
||||
|
||||
```bash
|
||||
kubectl apply -f services/platform-postgres-backup/restore-drill/cluster.yaml
|
||||
kubectl -n platform-data wait cluster/platform-postgres-restore-drill \
|
||||
--for=condition=Ready --timeout=20m
|
||||
```
|
||||
|
||||
2. 두 쪽의 표마다 행 수를 비교한다. 운영은 그 사이에도 쓰이므로 활동이 많은 표(Gitea의
|
||||
`action`, `notice` 등)는 조금 다를 수 있다. 표가 없거나 크게 다르면 실패다.
|
||||
|
||||
```bash
|
||||
count_rows() { # $1 = cluster, $2 = database
|
||||
kubectl -n platform-data exec "$1-1" -c postgres -- psql -At -d "$2" -c \
|
||||
"select string_agg(format('select %L, count(*) from %I.%I', schemaname||'.'||relname, schemaname, relname), ' union all ')
|
||||
from pg_stat_user_tables" |
|
||||
kubectl -n platform-data exec -i "$1-1" -c postgres -- psql -At -d "$2" | sort
|
||||
}
|
||||
for db in gitea keycloak techlog; do
|
||||
diff <(count_rows platform-postgres "$db") <(count_rows platform-postgres-restore-drill "$db") \
|
||||
&& echo "$db: same" || echo "$db: differs (see above)"
|
||||
done
|
||||
```
|
||||
|
||||
3. 복원된 시점을 기록한다.
|
||||
|
||||
```bash
|
||||
kubectl -n platform-data exec platform-postgres-restore-drill-1 -c postgres -- \
|
||||
psql -At -c "select now(), pg_last_xact_replay_timestamp()"
|
||||
```
|
||||
|
||||
4. 임시 Cluster를 지운다. PVC도 같이 지워진다.
|
||||
|
||||
```bash
|
||||
kubectl -n platform-data delete cluster platform-postgres-restore-drill
|
||||
```
|
||||
|
||||
결과(날짜, 복원된 시점, 표 비교, 걸린 시간)는 아래 「기록」에 남긴다.
|
||||
|
||||
## 기록
|
||||
|
||||
- 2026-11-11: 설계와 원본만. bucket·Secret·적용·리허설은 하지 않았다(소유자 단계).
|
||||
`kubectl apply --dry-run=server`로 ScheduledBackup, ingress, 리허설 Cluster가 admission을
|
||||
통과했고, `kubectl patch --dry-run=server`로 patch 뒤에도 `techlog` pg_hba 두 줄이 남는 것을 봤다.
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::platform-postgres-backups"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::platform-postgres-backups/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# Merge patch for the live Cluster. Apply it with kubectl patch, not by
|
||||
# re-applying services/platform-postgres: the live Cluster carries pg_hba lines
|
||||
# for techlog that the Phase 1 source does not have (README.md, "적용").
|
||||
spec:
|
||||
backup:
|
||||
# Base backups and WAL older than the newest point that still lets a
|
||||
# restore reach 30 days back are deleted by the operator.
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
# AIStor ObjectStore on the HDD (sdb3, aistor-local-xfs-retain), not the
|
||||
# root SSD that holds the database itself.
|
||||
destinationPath: s3://platform-postgres-backups/
|
||||
endpointURL: http://minio.object-storage.svc.cluster.local
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: platform-postgres-backup-s3
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: platform-postgres-backup-s3
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
maxParallel: 2
|
||||
data:
|
||||
compression: gzip
|
||||
jobs: 2
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
# Applied after platform-postgres (Phase 1) and AIStor (Phase 2) exist and
|
||||
# after scripts/bootstrap/create-postgres-backup-credentials.sh has created
|
||||
# the bucket and the platform-data/platform-postgres-backup-s3 Secret.
|
||||
# cluster-backup-patch.yaml and restore-drill/ are applied by hand; see README.md.
|
||||
resources:
|
||||
- object-storage-counterpart.yaml
|
||||
- scheduled-backup.yaml
|
||||
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: platform
|
||||
app.kubernetes.io/managed-by: kustomize
|
||||
includeSelectors: false
|
||||
@@ -0,0 +1,30 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: object-storage-allow-platform-postgres-backup
|
||||
namespace: object-storage
|
||||
labels:
|
||||
app.kubernetes.io/component: platform-postgres-backup
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
aistor.min.io/objectStore: minio-aistor
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
# barman-cloud in the PostgreSQL pods writes base backups and WAL; the
|
||||
# restore drill cluster reads them.
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: platform-data
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: cnpg.io/cluster
|
||||
operator: In
|
||||
values:
|
||||
- platform-postgres
|
||||
- platform-postgres-restore-drill
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 9000
|
||||
@@ -0,0 +1,49 @@
|
||||
# Restore drill: a throwaway Cluster bootstrapped from the platform-postgres
|
||||
# backups. It has no backup section, so it never writes to the bucket.
|
||||
# Apply, compare, delete; see ../README.md ("복원 리허설").
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: platform-postgres-restore-drill
|
||||
namespace: platform-data
|
||||
labels:
|
||||
app.kubernetes.io/name: platform-postgres-restore-drill
|
||||
app.kubernetes.io/component: restore-drill
|
||||
spec:
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:17.9-standard-trixie
|
||||
storage:
|
||||
# Dynamic local-path volume on the root SSD, deleted with the Cluster.
|
||||
storageClass: local-path
|
||||
size: 5Gi
|
||||
bootstrap:
|
||||
recovery:
|
||||
source: platform-postgres
|
||||
externalClusters:
|
||||
- name: platform-postgres
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://platform-postgres-backups/
|
||||
endpointURL: http://minio.object-storage.svc.cluster.local
|
||||
serverName: platform-postgres
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: platform-postgres-backup-s3
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: platform-postgres-backup-s3
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
maxParallel: 2
|
||||
managed:
|
||||
services:
|
||||
disabledDefaultServices:
|
||||
- ro
|
||||
- r
|
||||
affinity:
|
||||
enablePodAntiAffinity: false
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: ScheduledBackup
|
||||
metadata:
|
||||
name: platform-postgres-daily
|
||||
namespace: platform-data
|
||||
labels:
|
||||
app.kubernetes.io/name: platform-postgres
|
||||
app.kubernetes.io/component: backup
|
||||
spec:
|
||||
# Seconds field first. The operator runs in UTC: 18:00 UTC is 03:00 KST.
|
||||
schedule: "0 0 18 * * *"
|
||||
# Take the first base backup as soon as this object is created.
|
||||
immediate: true
|
||||
backupOwnerReference: self
|
||||
method: barmanObjectStore
|
||||
cluster:
|
||||
name: platform-postgres
|
||||
Reference in New Issue
Block a user