feat(postgres): back up platform-postgres to AIStor every day
platform-postgres holds the Gitea, Keycloak and tech-log databases and had no backup: no spec.backup, no ScheduledBackup, and its only copy is a local PV on the root SSD. A lost SSD or a bad migration had no way back. Add services/platform-postgres-backup: a merge patch that gives the live Cluster barman-cloud WAL archiving and base backups to s3://platform-postgres-backups on the AIStor ObjectStore (HDD sdb3) with 30-day retention and gzip, a daily 03:00 KST ScheduledBackup that also runs once at creation, the object-storage ingress for the PostgreSQL pods, the bucket policy, and a restore drill Cluster that only reads the bucket. scripts/bootstrap/create-postgres-backup- credentials.sh creates the bucket (100GiB quota), policy, user and the platform-data Secret through a restricted AIStor client Pod, proves the user can write its bucket and cannot list loki, and prints no key. Add PlatformPostgresBackupMissing (no base backup in 26h) and PlatformPostgresWALArchiveBacklog (more than 10 WAL files waiting for 15m; a stuck archive fills the SSD volume). The native barmanObjectStore is deprecated since CNPG 1.26 and the operator warns it is removed in 1.31.0. The Barman Cloud Plugin needs cert-manager, which this cluster does not have; installing two operators before the first backup is slower and riskier. The bucket layout is the same, so the plugin can take over before CNPG 1.31. Volume snapshots do not work on the static local PV. Risk: the live Cluster has pg_hba lines for techlog that the Phase 1 source lacks, so re-applying services/platform-postgres would cut the tech-log backend off. The README applies only spec.backup with kubectl patch. Backups stay on the same desktop; an off-machine copy is not part of this change. Nothing was applied to the cluster. Verification: kubectl apply --dry-run=server accepts the ScheduledBackup, the NetworkPolicy and the drill Cluster; kubectl patch --dry-run=server keeps both techlog pg_hba lines and adds spec.backup. The row-count comparison from the drill steps ran read-only on production (keycloak 100, techlog 44, gitea 116 tables). mc command syntax was checked with the pinned AIStor client offline. The two new alert tests failed against the old rules and pass now. bash -n passes for the script; shellcheck is not installed on the laptop or the desktop.
This commit is contained in:
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create the AIStor bucket, policy and user that CloudNativePG uses to back up
|
||||
# platform-postgres, and the platform-data/platform-postgres-backup-s3 Secret.
|
||||
#
|
||||
# Run once, by the owner, from a platform-core checkout on the desktop. The
|
||||
# script refuses to run when the Secret already exists; it never rotates or
|
||||
# deletes a bucket, policy, user or object. No credential value is printed.
|
||||
set -Eeuo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
readonly EXPECTED_CONTEXT=default
|
||||
readonly OS_NAMESPACE=object-storage
|
||||
readonly PG_NAMESPACE=platform-data
|
||||
readonly ROOT_SECRET=aistor-root-configuration
|
||||
readonly TARGET_SECRET=platform-postgres-backup-s3
|
||||
readonly BUCKET=platform-postgres-backups
|
||||
readonly POLICY=platform-postgres-backups
|
||||
readonly QUOTA=100GiB
|
||||
readonly S3_ENDPOINT=http://minio.object-storage.svc.cluster.local
|
||||
# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh.
|
||||
readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b"
|
||||
readonly POLICY_FILE="${ROOT}/services/platform-postgres-backup/bucket-policy.json"
|
||||
readonly PASS_LINE=PLATFORM_POSTGRES_BACKUP_BOOTSTRAP_PASS
|
||||
|
||||
fail() {
|
||||
printf 'FAIL: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
for command in kubectl jq openssl; do
|
||||
command -v "$command" >/dev/null || fail "missing command: ${command}"
|
||||
done
|
||||
[[ -f "$POLICY_FILE" ]] || fail "missing ${POLICY_FILE}"
|
||||
jq -e . "$POLICY_FILE" >/dev/null || fail "bucket policy is not valid JSON"
|
||||
|
||||
[[ "$(kubectl config current-context)" == "$EXPECTED_CONTEXT" ]] ||
|
||||
fail "kube context must be ${EXPECTED_CONTEXT}"
|
||||
if kubectl --namespace "$PG_NAMESPACE" get secret "$TARGET_SECRET" >/dev/null 2>&1; then
|
||||
fail "${PG_NAMESPACE}/${TARGET_SECRET} already exists; this script does not rotate it"
|
||||
fi
|
||||
kubectl --namespace "$OS_NAMESPACE" get secret "$ROOT_SECRET" >/dev/null ||
|
||||
fail "${OS_NAMESPACE}/${ROOT_SECRET} is missing"
|
||||
|
||||
work="$(mktemp -d)"
|
||||
pod="pg-backup-bootstrap-$(date +%H%M%S)-$$"
|
||||
handoff_secret="${pod}-keys"
|
||||
policy_map="${pod}-policy"
|
||||
|
||||
cleanup() {
|
||||
kubectl --namespace "$OS_NAMESPACE" delete pod "$pod" \
|
||||
--ignore-not-found --wait=false >/dev/null 2>&1 || true
|
||||
kubectl --namespace "$OS_NAMESPACE" delete secret "$handoff_secret" \
|
||||
--ignore-not-found >/dev/null 2>&1 || true
|
||||
kubectl --namespace "$OS_NAMESPACE" delete configmap "$policy_map" \
|
||||
--ignore-not-found >/dev/null 2>&1 || true
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
printf 'pgbackup%s' "$(openssl rand -hex 8)" >"${work}/ACCESS_KEY_ID"
|
||||
openssl rand -hex 32 | tr -d '\n' >"${work}/ACCESS_SECRET_KEY"
|
||||
|
||||
printf '%s\n' \
|
||||
"Bucket: ${BUCKET} (quota ${QUOTA}) on AIStor ${S3_ENDPOINT}" \
|
||||
"Policy: ${POLICY} (${POLICY_FILE#"${ROOT}/"})" \
|
||||
"User: a new generated access key attached to ${POLICY}" \
|
||||
"Secret: ${PG_NAMESPACE}/${TARGET_SECRET} (ACCESS_KEY_ID, ACCESS_SECRET_KEY)" \
|
||||
"Temporary: one restricted client Pod, one Secret and one ConfigMap in ${OS_NAMESPACE}" \
|
||||
'No credential value will be printed.'
|
||||
printf 'Type APPLY to create them: '
|
||||
IFS= read -r confirmation
|
||||
[[ "$confirmation" == "APPLY" ]] || fail "confirmation did not match APPLY"
|
||||
|
||||
kubectl --namespace "$OS_NAMESPACE" create secret generic "$handoff_secret" \
|
||||
--from-file=ACCESS_KEY_ID="${work}/ACCESS_KEY_ID" \
|
||||
--from-file=ACCESS_SECRET_KEY="${work}/ACCESS_SECRET_KEY" >/dev/null
|
||||
kubectl --namespace "$OS_NAMESPACE" create configmap "$policy_map" \
|
||||
--from-file=policy.json="$POLICY_FILE" >/dev/null
|
||||
|
||||
# Every mc call is silenced so neither key reaches the Pod log.
|
||||
bootstrap_command='set -eu
|
||||
. /run/aistor-root/config.env
|
||||
access_key="$(cat /run/backup/ACCESS_KEY_ID)"
|
||||
secret_key="$(cat /run/backup/ACCESS_SECRET_KEY)"
|
||||
mc alias set root "'"$S3_ENDPOINT"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null
|
||||
mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null
|
||||
mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null
|
||||
mc admin policy create root "'"$POLICY"'" /run/policy/policy.json >/dev/null
|
||||
mc admin user add root "$access_key" "$secret_key" >/dev/null
|
||||
mc admin policy attach root "'"$POLICY"'" --user "$access_key" >/dev/null
|
||||
mc alias set backup "'"$S3_ENDPOINT"'" "$access_key" "$secret_key" --api S3v4 >/dev/null
|
||||
probe=".bootstrap-probe-$(date +%s)"
|
||||
printf "probe" | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null
|
||||
[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = "probe" ]
|
||||
mc rm "backup/'"$BUCKET"'/$probe" >/dev/null
|
||||
if mc ls backup/loki >/dev/null 2>&1; then
|
||||
echo "the backup user can list another bucket" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf "%s\n" "'"$PASS_LINE"'"'
|
||||
|
||||
overrides="$(
|
||||
jq -nc --arg command "$bootstrap_command" --arg image "$MC_IMAGE" \
|
||||
--arg root "$ROOT_SECRET" --arg keys "$handoff_secret" --arg policy "$policy_map" '{
|
||||
spec: {
|
||||
automountServiceAccountToken: false,
|
||||
securityContext: {
|
||||
runAsNonRoot: true,
|
||||
runAsUser: 1000,
|
||||
runAsGroup: 1000,
|
||||
fsGroup: 1000,
|
||||
seccompProfile: {type: "RuntimeDefault"}
|
||||
},
|
||||
containers: [{
|
||||
name: "pg-backup-bootstrap",
|
||||
image: $image,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/bin/sh", "-ec"],
|
||||
args: [$command],
|
||||
securityContext: {
|
||||
allowPrivilegeEscalation: false,
|
||||
capabilities: {drop: ["ALL"]},
|
||||
readOnlyRootFilesystem: true
|
||||
},
|
||||
env: [{name: "MC_CONFIG_DIR", value: "/tmp/mc"}],
|
||||
volumeMounts: [
|
||||
{name: "root-config", mountPath: "/run/aistor-root", readOnly: true},
|
||||
{name: "backup-keys", mountPath: "/run/backup", readOnly: true},
|
||||
{name: "policy", mountPath: "/run/policy", readOnly: true},
|
||||
{name: "tmp", mountPath: "/tmp"}
|
||||
]
|
||||
}],
|
||||
volumes: [
|
||||
{name: "root-config", secret: {secretName: $root, defaultMode: 256}},
|
||||
{name: "backup-keys", secret: {secretName: $keys, defaultMode: 256}},
|
||||
{name: "policy", configMap: {name: $policy}},
|
||||
{name: "tmp", emptyDir: {}}
|
||||
]
|
||||
}
|
||||
}'
|
||||
)"
|
||||
|
||||
kubectl --namespace "$OS_NAMESPACE" run "$pod" \
|
||||
--image="$MC_IMAGE" \
|
||||
--restart=Never \
|
||||
--labels='platform.hyeonworks.com/aistor-client=true,app.kubernetes.io/name=pg-backup-bootstrap' \
|
||||
--overrides="$overrides" >/dev/null
|
||||
|
||||
phase=''
|
||||
for _ in $(seq 1 90); do
|
||||
phase="$(kubectl --namespace "$OS_NAMESPACE" get pod "$pod" --output=jsonpath='{.status.phase}')"
|
||||
[[ "$phase" == "Succeeded" || "$phase" == "Failed" ]] && break
|
||||
sleep 2
|
||||
done
|
||||
logs="$(kubectl --namespace "$OS_NAMESPACE" logs "$pod" 2>&1 || true)"
|
||||
if [[ "$phase" != "Succeeded" || "$logs" != *"$PASS_LINE"* ]]; then
|
||||
printf '%s\n' "$logs" | tail -20 >&2
|
||||
fail "bootstrap Pod ended in phase '${phase}'; the platform-data Secret was not created"
|
||||
fi
|
||||
|
||||
kubectl --namespace "$PG_NAMESPACE" create secret generic "$TARGET_SECRET" \
|
||||
--from-file=ACCESS_KEY_ID="${work}/ACCESS_KEY_ID" \
|
||||
--from-file=ACCESS_SECRET_KEY="${work}/ACCESS_SECRET_KEY" >/dev/null
|
||||
kubectl --namespace "$PG_NAMESPACE" label secret "$TARGET_SECRET" \
|
||||
app.kubernetes.io/part-of=platform app.kubernetes.io/component=platform-postgres-backup >/dev/null
|
||||
|
||||
printf '%s\n' "$PASS_LINE" \
|
||||
"Created bucket ${BUCKET}, policy ${POLICY}, one user and ${PG_NAMESPACE}/${TARGET_SECRET}."
|
||||
Reference in New Issue
Block a user