diff --git a/scripts/bootstrap/create-postgres-backup-credentials.sh b/scripts/bootstrap/create-postgres-backup-credentials.sh new file mode 100755 index 0000000..67adb92 --- /dev/null +++ b/scripts/bootstrap/create-postgres-backup-credentials.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Create the AIStor bucket, policy and user that CloudNativePG uses to back up +# platform-postgres, and the platform-data/platform-postgres-backup-s3 Secret. +# +# Run once, by the owner, from a platform-core checkout on the desktop. The +# script refuses to run when the Secret already exists; it never rotates or +# deletes a bucket, policy, user or object. No credential value is printed. +set -Eeuo pipefail +set +x +umask 077 + +readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +readonly EXPECTED_CONTEXT=default +readonly OS_NAMESPACE=object-storage +readonly PG_NAMESPACE=platform-data +readonly ROOT_SECRET=aistor-root-configuration +readonly TARGET_SECRET=platform-postgres-backup-s3 +readonly BUCKET=platform-postgres-backups +readonly POLICY=platform-postgres-backups +readonly QUOTA=100GiB +readonly S3_ENDPOINT=http://minio.object-storage.svc.cluster.local +# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh. +readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b" +readonly POLICY_FILE="${ROOT}/services/platform-postgres-backup/bucket-policy.json" +readonly PASS_LINE=PLATFORM_POSTGRES_BACKUP_BOOTSTRAP_PASS + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +for command in kubectl jq openssl; do + command -v "$command" >/dev/null || fail "missing command: ${command}" +done +[[ -f "$POLICY_FILE" ]] || fail "missing ${POLICY_FILE}" +jq -e . "$POLICY_FILE" >/dev/null || fail "bucket policy is not valid JSON" + +[[ "$(kubectl config current-context)" == "$EXPECTED_CONTEXT" ]] || + fail "kube context must be ${EXPECTED_CONTEXT}" +if kubectl --namespace "$PG_NAMESPACE" get secret "$TARGET_SECRET" >/dev/null 2>&1; then + fail "${PG_NAMESPACE}/${TARGET_SECRET} already exists; this script does not rotate it" +fi +kubectl --namespace "$OS_NAMESPACE" get secret "$ROOT_SECRET" >/dev/null || + fail "${OS_NAMESPACE}/${ROOT_SECRET} is missing" + +work="$(mktemp -d)" +pod="pg-backup-bootstrap-$(date +%H%M%S)-$$" +handoff_secret="${pod}-keys" +policy_map="${pod}-policy" + +cleanup() { + kubectl --namespace "$OS_NAMESPACE" delete pod "$pod" \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + kubectl --namespace "$OS_NAMESPACE" delete secret "$handoff_secret" \ + --ignore-not-found >/dev/null 2>&1 || true + kubectl --namespace "$OS_NAMESPACE" delete configmap "$policy_map" \ + --ignore-not-found >/dev/null 2>&1 || true + rm -rf -- "$work" +} +trap cleanup EXIT INT TERM + +printf 'pgbackup%s' "$(openssl rand -hex 8)" >"${work}/ACCESS_KEY_ID" +openssl rand -hex 32 | tr -d '\n' >"${work}/ACCESS_SECRET_KEY" + +printf '%s\n' \ + "Bucket: ${BUCKET} (quota ${QUOTA}) on AIStor ${S3_ENDPOINT}" \ + "Policy: ${POLICY} (${POLICY_FILE#"${ROOT}/"})" \ + "User: a new generated access key attached to ${POLICY}" \ + "Secret: ${PG_NAMESPACE}/${TARGET_SECRET} (ACCESS_KEY_ID, ACCESS_SECRET_KEY)" \ + "Temporary: one restricted client Pod, one Secret and one ConfigMap in ${OS_NAMESPACE}" \ + 'No credential value will be printed.' +printf 'Type APPLY to create them: ' +IFS= read -r confirmation +[[ "$confirmation" == "APPLY" ]] || fail "confirmation did not match APPLY" + +kubectl --namespace "$OS_NAMESPACE" create secret generic "$handoff_secret" \ + --from-file=ACCESS_KEY_ID="${work}/ACCESS_KEY_ID" \ + --from-file=ACCESS_SECRET_KEY="${work}/ACCESS_SECRET_KEY" >/dev/null +kubectl --namespace "$OS_NAMESPACE" create configmap "$policy_map" \ + --from-file=policy.json="$POLICY_FILE" >/dev/null + +# Every mc call is silenced so neither key reaches the Pod log. +bootstrap_command='set -eu +. /run/aistor-root/config.env +access_key="$(cat /run/backup/ACCESS_KEY_ID)" +secret_key="$(cat /run/backup/ACCESS_SECRET_KEY)" +mc alias set root "'"$S3_ENDPOINT"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null +mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null +mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null +mc admin policy create root "'"$POLICY"'" /run/policy/policy.json >/dev/null +mc admin user add root "$access_key" "$secret_key" >/dev/null +mc admin policy attach root "'"$POLICY"'" --user "$access_key" >/dev/null +mc alias set backup "'"$S3_ENDPOINT"'" "$access_key" "$secret_key" --api S3v4 >/dev/null +probe=".bootstrap-probe-$(date +%s)" +printf "probe" | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null +[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = "probe" ] +mc rm "backup/'"$BUCKET"'/$probe" >/dev/null +if mc ls backup/loki >/dev/null 2>&1; then + echo "the backup user can list another bucket" >&2 + exit 1 +fi +printf "%s\n" "'"$PASS_LINE"'"' + +overrides="$( + jq -nc --arg command "$bootstrap_command" --arg image "$MC_IMAGE" \ + --arg root "$ROOT_SECRET" --arg keys "$handoff_secret" --arg policy "$policy_map" '{ + spec: { + automountServiceAccountToken: false, + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + fsGroup: 1000, + seccompProfile: {type: "RuntimeDefault"} + }, + containers: [{ + name: "pg-backup-bootstrap", + image: $image, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec"], + args: [$command], + securityContext: { + allowPrivilegeEscalation: false, + capabilities: {drop: ["ALL"]}, + readOnlyRootFilesystem: true + }, + env: [{name: "MC_CONFIG_DIR", value: "/tmp/mc"}], + volumeMounts: [ + {name: "root-config", mountPath: "/run/aistor-root", readOnly: true}, + {name: "backup-keys", mountPath: "/run/backup", readOnly: true}, + {name: "policy", mountPath: "/run/policy", readOnly: true}, + {name: "tmp", mountPath: "/tmp"} + ] + }], + volumes: [ + {name: "root-config", secret: {secretName: $root, defaultMode: 256}}, + {name: "backup-keys", secret: {secretName: $keys, defaultMode: 256}}, + {name: "policy", configMap: {name: $policy}}, + {name: "tmp", emptyDir: {}} + ] + } + }' +)" + +kubectl --namespace "$OS_NAMESPACE" run "$pod" \ + --image="$MC_IMAGE" \ + --restart=Never \ + --labels='platform.hyeonworks.com/aistor-client=true,app.kubernetes.io/name=pg-backup-bootstrap' \ + --overrides="$overrides" >/dev/null + +phase='' +for _ in $(seq 1 90); do + phase="$(kubectl --namespace "$OS_NAMESPACE" get pod "$pod" --output=jsonpath='{.status.phase}')" + [[ "$phase" == "Succeeded" || "$phase" == "Failed" ]] && break + sleep 2 +done +logs="$(kubectl --namespace "$OS_NAMESPACE" logs "$pod" 2>&1 || true)" +if [[ "$phase" != "Succeeded" || "$logs" != *"$PASS_LINE"* ]]; then + printf '%s\n' "$logs" | tail -20 >&2 + fail "bootstrap Pod ended in phase '${phase}'; the platform-data Secret was not created" +fi + +kubectl --namespace "$PG_NAMESPACE" create secret generic "$TARGET_SECRET" \ + --from-file=ACCESS_KEY_ID="${work}/ACCESS_KEY_ID" \ + --from-file=ACCESS_SECRET_KEY="${work}/ACCESS_SECRET_KEY" >/dev/null +kubectl --namespace "$PG_NAMESPACE" label secret "$TARGET_SECRET" \ + app.kubernetes.io/part-of=platform app.kubernetes.io/component=platform-postgres-backup >/dev/null + +printf '%s\n' "$PASS_LINE" \ + "Created bucket ${BUCKET}, policy ${POLICY}, one user and ${PG_NAMESPACE}/${TARGET_SECRET}." diff --git a/scripts/validate/test-platform-disk-alerts.sh b/scripts/validate/test-platform-disk-alerts.sh index ee20253..c192247 100755 --- a/scripts/validate/test-platform-disk-alerts.sh +++ b/scripts/validate/test-platform-disk-alerts.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -# Unit tests for the root filesystem alerts in core-rules.yaml. +# Unit tests for core-rules.yaml alerts that guard the desktop host. # -# The kubelet starts deleting unused images at 85% use and evicts pods at -# 90% (nodefs.available < 10%). These tests pin the alerts in front of those -# two points: warning at 80%, critical at 85%, and a forecast that fires when -# the 6h trend reaches the eviction point within 6 hours. +# Root disk: the kubelet starts deleting unused images at 85% use and evicts +# pods at 90% (nodefs.available < 10%). The alerts sit in front of those two +# points: warning at 80%, critical at 85%, and a forecast that fires when the +# 6h trend reaches the eviction point within 6 hours. +# PostgreSQL backups: no successful base backup for 26 hours, or WAL waiting +# to be archived. # # Needs docker, jq and kubectl (client only, no cluster access). set -euo pipefail @@ -45,6 +47,7 @@ jq -e --arg url "$RUNBOOK_URL" ' # node-exporter labels as scraped by serviceMonitor/observability-agent/node-exporter. readonly SERIES_LABELS='job="prometheus-node-exporter",instance="10.42.0.95:9100",device="/dev/sdc1",fstype="ext4",mountpoint="/"' +readonly PG_LABELS='job="platform-postgres",namespace="platform-data",pod="platform-postgres-1",instance="10.42.0.130:9187"' readonly ALERT_LABELS=' job: prometheus-node-exporter instance: 10.42.0.95:9100 device: /dev/sdc1 @@ -137,6 +140,66 @@ ${ALERT_LABELS} # Losing 0.1 per minute takes far longer than 6 hours to reach 100. forecast_case 'forecast stays quiet for a slow fill' 400 '-0.1' 30 ' []' forecast_case 'forecast stays quiet for flat usage' 400 '+0' 30 ' []' + + # PostgreSQL backups. Timestamps are seconds since the test epoch. + cat <"${temp_dir}/disk.test.yaml" chmod 0444 "${temp_dir}"/* @@ -150,4 +213,4 @@ docker run --rm \ --volume "${temp_dir}:/work:ro" \ --entrypoint /bin/promtool \ "$PROMETHEUS_IMAGE" test rules /work/disk.test.yaml -printf 'PASS: root filesystem usage and forecast alerts\n' +printf 'PASS: root filesystem and PostgreSQL backup alerts\n' diff --git a/services/observability/rules/platform/core-rules.yaml b/services/observability/rules/platform/core-rules.yaml index 0a363e2..0f37b15 100644 --- a/services/observability/rules/platform/core-rules.yaml +++ b/services/observability/rules/platform/core-rules.yaml @@ -113,6 +113,36 @@ spec: summary: 루트 파일시스템 inode 사용률이 90%를 초과함 description: 노드 루트 파일시스템 inode 사용률이 심각 임계치인 90%를 15분간 초과했습니다. runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + - name: platform.backup + rules: + # cnpg_collector_last_available_backup_timestamp is 0 until the first + # base backup, so a cluster without backups fires as well. + - alert: PlatformPostgresBackupMissing + expr: >- + time() - max by (job, namespace) ( + cnpg_collector_last_available_backup_timestamp{job="platform-postgres",namespace="platform-data"} + ) > 26 * 3600 + for: 30m + labels: + severity: critical + annotations: + summary: platform-postgres에 26시간 안에 끝난 백업이 없음 + description: platform-postgres의 마지막 성공한 base backup이 26시간보다 오래됐거나 없습니다. 매일 03:00 KST 백업(ScheduledBackup platform-postgres-daily)을 확인하세요. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md + # A failing archive_command keeps WAL in pg_wal, which then fills the + # PostgreSQL volume. + - alert: PlatformPostgresWALArchiveBacklog + expr: >- + max by (job, namespace) ( + cnpg_collector_pg_wal_archive_status{job="platform-postgres",namespace="platform-data",value="ready"} + ) > 10 + for: 15m + labels: + severity: critical + annotations: + summary: platform-postgres의 WAL이 보관소로 올라가지 않음 + description: 보관을 기다리는 WAL 파일이 10개를 넘은 채 15분이 지났습니다. 보관이 막히면 WAL이 PostgreSQL 볼륨에 쌓여 디스크가 찹니다. + runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md - name: platform.workloads rules: - alert: PlatformPodRestartBurst diff --git a/services/platform-postgres-backup/README.md b/services/platform-postgres-backup/README.md new file mode 100644 index 0000000..0901aa5 --- /dev/null +++ b/services/platform-postgres-backup/README.md @@ -0,0 +1,162 @@ +# 플랫폼 PostgreSQL 백업 + +`platform-data/platform-postgres` 하나에 Gitea, Keycloak, tech-log의 DB가 같이 있다 +(2026-11-11: gitea 608MB, keycloak 14MB, techlog 14MB). 2026-11-11까지 백업 설정도, +ScheduledBackup도 없었다. 그 PV는 루트 SSD 하나에만 있었다. + +이 폴더는 매일 base backup과 WAL 연속 보관을 AIStor(HDD `sdb3`)에 두고, 30일 안의 +아무 시점으로나 되돌릴 수 있게 한다. + +## 무엇이 어디로 가나 + +```text +platform-postgres-1 (SSD) + ├─ WAL 파일이 찰 때마다, 그리고 5분마다 ─▶ s3://platform-postgres-backups/platform-postgres/wals/ + └─ 매일 03:00 KST base backup ─────────▶ s3://platform-postgres-backups/platform-postgres/base/ +AIStor ObjectStore minio-aistor (HDD sdb3, aistor-local-xfs-retain) +``` + +| 파일 | 하는 일 | +| --- | --- | +| `cluster-backup-patch.yaml` | live Cluster에 `spec.backup`(보관소, 30일 보존, gzip)을 더한다 | +| `scheduled-backup.yaml` | 매일 18:00 UTC(03:00 KST) base backup. 만들자마자 한 번 | +| `object-storage-counterpart.yaml` | `object-storage` 쪽 ingress: platform-data의 PostgreSQL 파드 → AIStor 9000 | +| `bucket-policy.json` | 백업 사용자가 자기 bucket만 읽고 쓰는 정책 | +| `restore-drill/cluster.yaml` | 복원 리허설용 임시 Cluster | +| `../../scripts/bootstrap/create-postgres-backup-credentials.sh` | bucket·정책·사용자·Secret 생성 | + +## 고른 방법 + +- **내장 `barmanObjectStore`.** CloudNativePG 1.26부터 deprecated이고 Barman Cloud Plugin이 + 권장이다. 하지만 plugin은 cert-manager가 필요한데 이 클러스터에는 없다(2026-11-11 CRD 없음). + 백업이 하나도 없는 지금은 새 operator 두 개(cert-manager, plugin)를 먼저 세우는 것보다 지금 + operator(1.30.0)가 바로 지원하는 쪽이 빠르고 위험이 작다. bucket 구조가 같아 plugin으로 + 옮겨도 백업을 그대로 쓴다. + - **기한:** operator 경고는 「1.31.0에서 완전히 제거」라고 한다(문서는 1.32.0). 그래서 + **CNPG를 1.31로 올리기 전에** Barman Cloud Plugin으로 옮긴다. +- **보관소는 AIStor.** 같은 desktop이지만 SSD가 아닌 HDD(`sdb3`, 900Gi)다. SSD가 죽어도 + 남는다. desktop 자체가 죽으면 같이 잃는다. 다른 기계·외부로 복사하는 일은 남은 과제다. +- **보존 30일, quota 100GiB.** 압축한 base backup이 하루 수백 MB, WAL은 5분마다 한 파일 + (`archive_timeout` 기본값)이라 30일에 수십 GB 안쪽이다. +- **볼륨 스냅숏은 쓰지 않는다.** `ssd-local-postgres-retain`은 정적 local PV라 CSI 스냅숏이 없다. + +## 적용 (소유자, 순서대로) + +전제: 이 PR이 병합된 platform-core checkout, desktop의 kube context `default`. + +**주의: `services/platform-postgres`를 통째로 다시 적용하지 않는다.** live Cluster의 +`pg_hba`에는 Phase 1 원본에 없는 `host techlog techlog all scram-sha-256`과 +`host all techlog all reject`가 있다. 원본을 다시 적용하면 이 두 줄이 지워져 tech-log 운영 +백엔드가 DB에 붙지 못한다. 그래서 백업 설정만 `kubectl patch`로 더한다. + +1. bucket, 정책, 사용자, Secret을 만든다. 화면에 자격 증명이 나오지 않는다. + + ```bash + bash scripts/bootstrap/create-postgres-backup-credentials.sh + ``` + + 마지막 줄에 `PLATFORM_POSTGRES_BACKUP_BOOTSTRAP_PASS`가 나와야 한다. 새 사용자가 자기 + bucket에 쓰고 읽고 지울 수 있고, `loki` bucket은 못 보는 것까지 확인한다. + +2. AIStor 쪽 ingress를 연다. 그 전에는 PostgreSQL이 9000에 닿지 못한다. + + ```bash + kubectl apply -f services/platform-postgres-backup/object-storage-counterpart.yaml + ``` + +3. live Cluster에 백업 설정을 더한다. PostgreSQL은 재시작하지 않는다. + + ```bash + kubectl -n platform-data patch cluster platform-postgres --type merge \ + --patch-file services/platform-postgres-backup/cluster-backup-patch.yaml --dry-run=server -o name + kubectl -n platform-data patch cluster platform-postgres --type merge \ + --patch-file services/platform-postgres-backup/cluster-backup-patch.yaml + kubectl -n platform-data get cluster platform-postgres -o jsonpath='{.spec.postgresql.pg_hba}{"\n"}' + ``` + + 마지막 줄에 `techlog` 두 줄이 그대로 있어야 한다. 몇 분 뒤 WAL이 올라가는지 본다. + + ```bash + kubectl -n platform-data get cluster platform-postgres \ + -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}{" "}{.status.firstRecoverabilityPoint}{"\n"}' + ``` + +4. 매일 백업을 건다. 만들자마자 첫 base backup이 돈다. + + ```bash + kubectl apply -k services/platform-postgres-backup + kubectl -n platform-data get backup -w + ``` + + `PHASE`가 `completed`가 되면 된다. `failed`면 + `kubectl -n platform-data describe backup <이름>`의 마지막 줄을 본다. + +5. 아래 「복원 리허설」을 한 번 한다. + +경보(`services/observability/rules/platform/core-rules.yaml`의 `platform.backup`): + +- `PlatformPostgresBackupMissing`: 26시간 안에 끝난 base backup이 없다. 위 4단계 전까지는 계속 + 울린다(백업이 정말 없으니까). +- `PlatformPostgresWALArchiveBacklog`: 보관을 기다리는 WAL이 10개를 넘은 채 15분. + 보관이 막히면 WAL이 SSD 볼륨에 쌓여 PostgreSQL이 멈춘다. 2·3단계의 순서가 바뀌었거나 + Secret이 틀리면 이것이 울린다. + +### 되돌리기 + +```bash +kubectl delete -k services/platform-postgres-backup # ScheduledBackup과 ingress +kubectl -n platform-data patch cluster platform-postgres --type json -p '[{"op":"remove","path":"/spec/backup"}]' +``` + +bucket, 사용자, Secret, 이미 올린 백업은 지우지 않는다. 지울 때는 소유자가 `mc`로 직접 한다. + +## 복원 리허설 + +백업이 실제로 복원되는지는 복원해 봐야 안다. 첫 base backup이 끝난 뒤 한 번, 그 뒤로는 CNPG나 +PostgreSQL을 올릴 때마다 한다. 운영 Cluster는 건드리지 않는다. 임시 Cluster는 bucket을 읽기만 한다 +(`backup` 칸이 없다). + +1. 임시 Cluster를 만든다. local-path 5Gi(루트 SSD)를 잠깐 쓴다. + + ```bash + kubectl apply -f services/platform-postgres-backup/restore-drill/cluster.yaml + kubectl -n platform-data wait cluster/platform-postgres-restore-drill \ + --for=condition=Ready --timeout=20m + ``` + +2. 두 쪽의 표마다 행 수를 비교한다. 운영은 그 사이에도 쓰이므로 활동이 많은 표(Gitea의 + `action`, `notice` 등)는 조금 다를 수 있다. 표가 없거나 크게 다르면 실패다. + + ```bash + count_rows() { # $1 = cluster, $2 = database + kubectl -n platform-data exec "$1-1" -c postgres -- psql -At -d "$2" -c \ + "select string_agg(format('select %L, count(*) from %I.%I', schemaname||'.'||relname, schemaname, relname), ' union all ') + from pg_stat_user_tables" | + kubectl -n platform-data exec -i "$1-1" -c postgres -- psql -At -d "$2" | sort + } + for db in gitea keycloak techlog; do + diff <(count_rows platform-postgres "$db") <(count_rows platform-postgres-restore-drill "$db") \ + && echo "$db: same" || echo "$db: differs (see above)" + done + ``` + +3. 복원된 시점을 기록한다. + + ```bash + kubectl -n platform-data exec platform-postgres-restore-drill-1 -c postgres -- \ + psql -At -c "select now(), pg_last_xact_replay_timestamp()" + ``` + +4. 임시 Cluster를 지운다. PVC도 같이 지워진다. + + ```bash + kubectl -n platform-data delete cluster platform-postgres-restore-drill + ``` + +결과(날짜, 복원된 시점, 표 비교, 걸린 시간)는 아래 「기록」에 남긴다. + +## 기록 + +- 2026-11-11: 설계와 원본만. bucket·Secret·적용·리허설은 하지 않았다(소유자 단계). + `kubectl apply --dry-run=server`로 ScheduledBackup, ingress, 리허설 Cluster가 admission을 + 통과했고, `kubectl patch --dry-run=server`로 patch 뒤에도 `techlog` pg_hba 두 줄이 남는 것을 봤다. diff --git a/services/platform-postgres-backup/bucket-policy.json b/services/platform-postgres-backup/bucket-policy.json new file mode 100644 index 0000000..c446d6f --- /dev/null +++ b/services/platform-postgres-backup/bucket-policy.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads" + ], + "Resource": [ + "arn:aws:s3:::platform-postgres-backups" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::platform-postgres-backups/*" + ] + } + ] +} diff --git a/services/platform-postgres-backup/cluster-backup-patch.yaml b/services/platform-postgres-backup/cluster-backup-patch.yaml new file mode 100644 index 0000000..a8d7d54 --- /dev/null +++ b/services/platform-postgres-backup/cluster-backup-patch.yaml @@ -0,0 +1,26 @@ +# Merge patch for the live Cluster. Apply it with kubectl patch, not by +# re-applying services/platform-postgres: the live Cluster carries pg_hba lines +# for techlog that the Phase 1 source does not have (README.md, "적용"). +spec: + backup: + # Base backups and WAL older than the newest point that still lets a + # restore reach 30 days back are deleted by the operator. + retentionPolicy: "30d" + barmanObjectStore: + # AIStor ObjectStore on the HDD (sdb3, aistor-local-xfs-retain), not the + # root SSD that holds the database itself. + destinationPath: s3://platform-postgres-backups/ + endpointURL: http://minio.object-storage.svc.cluster.local + s3Credentials: + accessKeyId: + name: platform-postgres-backup-s3 + key: ACCESS_KEY_ID + secretAccessKey: + name: platform-postgres-backup-s3 + key: ACCESS_SECRET_KEY + wal: + compression: gzip + maxParallel: 2 + data: + compression: gzip + jobs: 2 diff --git a/services/platform-postgres-backup/kustomization.yaml b/services/platform-postgres-backup/kustomization.yaml new file mode 100644 index 0000000..0a85c50 --- /dev/null +++ b/services/platform-postgres-backup/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Applied after platform-postgres (Phase 1) and AIStor (Phase 2) exist and +# after scripts/bootstrap/create-postgres-backup-credentials.sh has created +# the bucket and the platform-data/platform-postgres-backup-s3 Secret. +# cluster-backup-patch.yaml and restore-drill/ are applied by hand; see README.md. +resources: + - object-storage-counterpart.yaml + - scheduled-backup.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: platform + app.kubernetes.io/managed-by: kustomize + includeSelectors: false diff --git a/services/platform-postgres-backup/object-storage-counterpart.yaml b/services/platform-postgres-backup/object-storage-counterpart.yaml new file mode 100644 index 0000000..4e94191 --- /dev/null +++ b/services/platform-postgres-backup/object-storage-counterpart.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: object-storage-allow-platform-postgres-backup + namespace: object-storage + labels: + app.kubernetes.io/component: platform-postgres-backup +spec: + podSelector: + matchLabels: + aistor.min.io/objectStore: minio-aistor + policyTypes: + - Ingress + ingress: + # barman-cloud in the PostgreSQL pods writes base backups and WAL; the + # restore drill cluster reads them. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform-data + podSelector: + matchExpressions: + - key: cnpg.io/cluster + operator: In + values: + - platform-postgres + - platform-postgres-restore-drill + ports: + - protocol: TCP + port: 9000 diff --git a/services/platform-postgres-backup/restore-drill/cluster.yaml b/services/platform-postgres-backup/restore-drill/cluster.yaml new file mode 100644 index 0000000..6e13de3 --- /dev/null +++ b/services/platform-postgres-backup/restore-drill/cluster.yaml @@ -0,0 +1,49 @@ +# Restore drill: a throwaway Cluster bootstrapped from the platform-postgres +# backups. It has no backup section, so it never writes to the bucket. +# Apply, compare, delete; see ../README.md ("복원 리허설"). +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: platform-postgres-restore-drill + namespace: platform-data + labels: + app.kubernetes.io/name: platform-postgres-restore-drill + app.kubernetes.io/component: restore-drill +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:17.9-standard-trixie + storage: + # Dynamic local-path volume on the root SSD, deleted with the Cluster. + storageClass: local-path + size: 5Gi + bootstrap: + recovery: + source: platform-postgres + externalClusters: + - name: platform-postgres + barmanObjectStore: + destinationPath: s3://platform-postgres-backups/ + endpointURL: http://minio.object-storage.svc.cluster.local + serverName: platform-postgres + s3Credentials: + accessKeyId: + name: platform-postgres-backup-s3 + key: ACCESS_KEY_ID + secretAccessKey: + name: platform-postgres-backup-s3 + key: ACCESS_SECRET_KEY + wal: + maxParallel: 2 + managed: + services: + disabledDefaultServices: + - ro + - r + affinity: + enablePodAntiAffinity: false + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi diff --git a/services/platform-postgres-backup/scheduled-backup.yaml b/services/platform-postgres-backup/scheduled-backup.yaml new file mode 100644 index 0000000..8a71acb --- /dev/null +++ b/services/platform-postgres-backup/scheduled-backup.yaml @@ -0,0 +1,17 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: platform-postgres-daily + namespace: platform-data + labels: + app.kubernetes.io/name: platform-postgres + app.kubernetes.io/component: backup +spec: + # Seconds field first. The operator runs in UTC: 18:00 UTC is 03:00 KST. + schedule: "0 0 18 * * *" + # Take the first base backup as soon as this object is created. + immediate: true + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: platform-postgres