Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb78ccdb0b | ||
|
|
9e9ea94461 | ||
|
|
ba3dddc139 | ||
|
|
012452cf78 | ||
|
|
77daa39a41 | ||
|
|
b484110b6b | ||
|
|
3eafb7e1d8 | ||
|
|
ebdf33f072 | ||
|
|
c0749ec8a4 | ||
|
|
c2a3f75bc3 | ||
|
|
7580eb7f75 |
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Back up Gitea (gitea dump --skip-db) to AIStor (platform-host-backups)
|
||||
Documentation=file:///usr/local/lib/platform-backup/README.md
|
||||
After=k3s.service docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/gitea-backup.sh
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
TimeoutStartSec=1h
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Weekly Gitea backup
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 04:00:00
|
||||
RandomizedDelaySec=10min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Record the k3s API server and kubelet serving certificate expiry
|
||||
After=k3s.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=donghyeon
|
||||
ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/k3s-cert-expiry.sh
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Daily k3s certificate expiry check
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 05:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Back up the k3s datastore and token to AIStor (platform-host-backups)
|
||||
Documentation=file:///usr/local/lib/platform-backup/README.md
|
||||
After=k3s.service docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/bash /usr/local/lib/platform-backup/backup/k3s-datastore-backup.sh
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily k3s datastore backup
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:30:00
|
||||
RandomizedDelaySec=10min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Compare the cluster with platform-core origin/main (kubectl diff, server dry-run)
|
||||
After=k3s.service network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=donghyeon
|
||||
Environment=PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm
|
||||
Environment=PLATFORM_CHECKOUT=/home/donghyeon/workspace/platform
|
||||
ExecStart=/bin/bash /usr/local/lib/platform-backup/validate/live-drift-check-main.sh
|
||||
# Exit 1 means "drift found"; the metric and its alert report that. Only 2 (the
|
||||
# check itself failed) marks the unit failed.
|
||||
SuccessExitStatus=1
|
||||
Nice=10
|
||||
TimeoutStartSec=30min
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily live drift check
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 05:15:00
|
||||
RandomizedDelaySec=10min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,38 @@
|
||||
# keycloak namespace NetworkPolicy
|
||||
|
||||
2026-11-11 점검 전 `keycloak` namespace에는 Keycloak Operator가 만든 ingress 정책 하나뿐이었다.
|
||||
Keycloak 파드와 Operator 파드의 egress는 막혀 있지 않았다. 다른 플랫폼 namespace처럼 기본 차단에
|
||||
필요한 것만 연다.
|
||||
|
||||
| 정책 | 대상 | 허용 |
|
||||
| --- | --- | --- |
|
||||
| `keycloak-default-deny` | 모든 파드 | ingress·egress 기본 차단 |
|
||||
| `keycloak-allow-dns` | 모든 파드 | kube-dns 53 |
|
||||
| `keycloak-allow-platform-postgres` | 모든 파드(서버와 realm import Job) | `platform-data` Postgres 5432 |
|
||||
| `keycloak-allow-cluster-peers` | Keycloak 파드 | 같은 Keycloak 파드 JGroups 7800·57800 |
|
||||
| `keycloak-operator-allow-egress` | Operator | API 서버(`10.43.0.1:443`, `192.168.0.107:6443`), Keycloak 8080·9000 |
|
||||
|
||||
- Keycloak 파드의 **ingress**는 계속 Keycloak CR `spec.networkPolicy`로 Operator가 만드는
|
||||
`keycloak-network-policy`가 소유한다(Traefik 8080, 관리 9000, JGroups).
|
||||
- 바깥(인터넷)으로 나가는 길은 없다. 2026-11-12에 repo와 Keycloak CR에 SMTP·외부 IdP 설정이
|
||||
없고, Keycloak 파드의 열린 TCP 연결이 Postgres뿐인 것을 확인했다. SMTP나 외부 IdP를 붙이면
|
||||
여기에 egress를 더해야 한다(안 더하면 메일·외부 로그인만 조용히 실패한다).
|
||||
- kubelet probe는 노드에서 오므로 막히지 않는다.
|
||||
|
||||
## 적용과 확인
|
||||
|
||||
```sh
|
||||
bash scripts/validate/render-tenants-and-policies.sh
|
||||
kubectl kustomize infrastructure/networking/keycloak | kubectl diff -f -
|
||||
kubectl apply -k infrastructure/networking/keycloak
|
||||
```
|
||||
|
||||
적용 뒤 확인(모두 통과해야 한다):
|
||||
|
||||
```sh
|
||||
kubectl -n keycloak get pods
|
||||
curl -fsS -o /dev/null -w '%{http_code}\n' https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration
|
||||
kubectl -n keycloak logs deploy/keycloak-operator --since=5m | grep -i -E 'error|exception|timeout' || echo no-operator-errors
|
||||
```
|
||||
|
||||
되돌리기: `kubectl delete -k infrastructure/networking/keycloak`(Operator 정책은 남는다).
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: keycloak
|
||||
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: platform
|
||||
includeSelectors: false
|
||||
|
||||
resources:
|
||||
- network-policies.yaml
|
||||
@@ -0,0 +1,117 @@
|
||||
# keycloak namespace: 기본 차단 + 필요한 것만 허용.
|
||||
# Keycloak 파드의 ingress는 Keycloak CR(spec.networkPolicy)로 Operator가 만드는
|
||||
# keycloak-network-policy가 계속 소유한다. 여기서는 기본 차단과 egress만 정한다.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-dns
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- port: 53
|
||||
protocol: UDP
|
||||
- port: 53
|
||||
protocol: TCP
|
||||
---
|
||||
# Keycloak 서버와 realm import Job이 공용 PostgreSQL에 붙는다.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-platform-postgres
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: platform-data
|
||||
podSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: platform-postgres
|
||||
ports:
|
||||
- port: 5432
|
||||
protocol: TCP
|
||||
---
|
||||
# 같은 Keycloak 인스턴스끼리의 JGroups(7800, 57800). Operator ingress 정책과 짝이다.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-cluster-peers
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: keycloak
|
||||
app.kubernetes.io/instance: keycloak
|
||||
app.kubernetes.io/managed-by: keycloak-operator
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: keycloak
|
||||
app.kubernetes.io/instance: keycloak
|
||||
app.kubernetes.io/managed-by: keycloak-operator
|
||||
ports:
|
||||
- port: 7800
|
||||
protocol: TCP
|
||||
- port: 57800
|
||||
protocol: TCP
|
||||
---
|
||||
# Operator: Kubernetes API(ClusterIP와 실제 API 서버 주소)와 Keycloak 관리 포트.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-operator-allow-egress
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak-operator
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 10.43.0.1/32
|
||||
ports:
|
||||
- port: 443
|
||||
protocol: TCP
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 192.168.0.107/32
|
||||
ports:
|
||||
- port: 6443
|
||||
protocol: TCP
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: keycloak
|
||||
app.kubernetes.io/instance: keycloak
|
||||
app.kubernetes.io/managed-by: keycloak-operator
|
||||
ports:
|
||||
- port: 8080
|
||||
protocol: TCP
|
||||
- port: 9000
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# Back up Gitea (repositories, packages, attachments, Actions data, app.ini) to
|
||||
# AIStor (HDD sdb3). Root only. Run by platform-gitea-backup.timer.
|
||||
#
|
||||
# `gitea dump --skip-db` runs inside the live Gitea container and streams a
|
||||
# tar.gz to stdout (logs go to stderr). The database is backed up separately by
|
||||
# CloudNativePG (services/platform-postgres-backup). The dump contains app.ini,
|
||||
# whose secrets are needed to restore, so the archive is sensitive.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
|
||||
|
||||
readonly NAME=gitea_backup
|
||||
readonly KCFG=/etc/rancher/k3s/k3s.yaml
|
||||
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root"
|
||||
hb_load_aistor_env
|
||||
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig "$KCFG" "$@"; }
|
||||
|
||||
started="$(date +%s)"
|
||||
previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)"
|
||||
on_error() {
|
||||
hb_write_metrics "$NAME" <<PROM
|
||||
platform_backup_last_run_success{backup="gitea"} 0
|
||||
platform_backup_last_run_timestamp_seconds{backup="gitea"} $(date +%s)
|
||||
platform_backup_last_success_timestamp_seconds{backup="gitea"} ${previous_success:-0}
|
||||
PROM
|
||||
}
|
||||
trap on_error ERR
|
||||
|
||||
work="$(mktemp -d "$HB_WORK_ROOT/gitea.XXXXXX")"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
archive="gitea-dump-$ts.tar.gz"
|
||||
|
||||
kubectl -n gitea exec deploy/gitea -c gitea -- \
|
||||
gitea dump --skip-db --type tar.gz --file - >"$work/$archive" 2>"$work/dump.log"
|
||||
tar -tzf "$work/$archive" >"$work/list.txt"
|
||||
grep -qx 'app.ini' "$work/list.txt" || hb_fail "the dump has no app.ini"
|
||||
repos="$(grep -c -E '^repos/[^/]+/[^/]+\.git/HEAD$' "$work/list.txt" || true)"
|
||||
(( repos > 0 )) || hb_fail "the dump has no repositories"
|
||||
sha256sum "$work/$archive" | awk '{print $1}' >"$work/$archive.sha256"
|
||||
size="$(stat -c %s "$work/$archive")"
|
||||
# What a restore must match: restore with the same Gitea version (docs/runbooks/gitea-restore.md).
|
||||
{
|
||||
echo "created=$ts"
|
||||
echo "gitea_image=$(kubectl -n gitea get deploy gitea -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
echo "gitea_version=$(kubectl -n gitea exec deploy/gitea -c gitea -- gitea --version 2>/dev/null | awk '{print $3}')"
|
||||
echo "repositories=$repos"
|
||||
} >"$work/$archive.meta"
|
||||
|
||||
hb_mc "$work" cp --quiet "/work/$archive" "/work/$archive.sha256" "/work/$archive.meta" "backup/$AISTOR_BUCKET/gitea/" >/dev/null
|
||||
hb_mc "$work" stat "backup/$AISTOR_BUCKET/gitea/$archive" >/dev/null
|
||||
|
||||
trap - ERR
|
||||
hb_write_metrics "$NAME" <<PROM
|
||||
# HELP platform_backup_last_success_timestamp_seconds Unix time of the last successful backup.
|
||||
# TYPE platform_backup_last_success_timestamp_seconds gauge
|
||||
platform_backup_last_success_timestamp_seconds{backup="gitea"} $(date +%s)
|
||||
platform_backup_last_run_success{backup="gitea"} 1
|
||||
platform_backup_last_run_timestamp_seconds{backup="gitea"} $(date +%s)
|
||||
platform_backup_last_size_bytes{backup="gitea"} $size
|
||||
platform_backup_last_duration_seconds{backup="gitea"} $(( $(date +%s) - started ))
|
||||
# HELP platform_backup_gitea_repositories Repositories in the last successful Gitea dump.
|
||||
# TYPE platform_backup_gitea_repositories gauge
|
||||
platform_backup_gitea_repositories $repos
|
||||
PROM
|
||||
hb_log "gitea backup $archive ($size bytes, $repos repositories) uploaded"
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# Back up the k3s datastore (sqlite) and the server token to AIStor (HDD sdb3).
|
||||
# Root only. Run by platform-k3s-datastore-backup.timer (infrastructure/host/backup).
|
||||
#
|
||||
# k3s docs (datastore/backup-restore, SQLite): copy server/db and server/token;
|
||||
# the token is required to read the datastore back. `sqlite3 .backup` takes a
|
||||
# consistent copy of the live database.
|
||||
#
|
||||
# Off-host copy (prepared, off by default): set in /etc/platform-backup/aistor.env
|
||||
# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc
|
||||
# OFFHOST_RCLONE_TARGET=<rclone remote:path>
|
||||
# The archive is then encrypted to that public key and copied with rclone.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
|
||||
|
||||
readonly SERVER=/var/lib/rancher/k3s/server
|
||||
readonly NAME=k3s_datastore_backup
|
||||
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root"
|
||||
command -v sqlite3 >/dev/null || hb_fail "sqlite3 is required"
|
||||
hb_load_aistor_env
|
||||
|
||||
started="$(date +%s)"
|
||||
on_error() {
|
||||
hb_write_metrics "$NAME" <<PROM
|
||||
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
|
||||
# TYPE platform_backup_last_run_success gauge
|
||||
platform_backup_last_run_success{backup="k3s-datastore"} 0
|
||||
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
||||
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} ${previous_success:-0}
|
||||
PROM
|
||||
}
|
||||
previous_success="$(awk '/^platform_backup_last_success_timestamp_seconds/ {print $2}' "$TEXTFILE_DIR/$NAME.prom" 2>/dev/null || echo 0)"
|
||||
trap on_error ERR
|
||||
|
||||
work="$(mktemp -d "$HB_WORK_ROOT/k3s.XXXXXX")"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
stage="$work/k3s-datastore-$ts"
|
||||
mkdir -m 0700 "$stage"
|
||||
|
||||
sqlite3 "$SERVER/db/state.db" ".backup '$stage/state.db'"
|
||||
check="$(sqlite3 "$stage/state.db" 'PRAGMA integrity_check;')"
|
||||
[[ "$check" == "ok" ]] || hb_fail "integrity_check on the copy returned: $check"
|
||||
install -m 0600 "$SERVER/token" "$stage/token"
|
||||
{
|
||||
echo "created=$ts"
|
||||
echo "k3s_version=$(/usr/local/bin/k3s --version | head -1)"
|
||||
echo "host=$(hostname)"
|
||||
echo "kine_rows=$(sqlite3 "$stage/state.db" 'SELECT COUNT(*) FROM kine;')"
|
||||
} >"$stage/META"
|
||||
tar -C "$work" -czf "$work/k3s-datastore-$ts.tar.gz" "k3s-datastore-$ts"
|
||||
sha256sum "$work/k3s-datastore-$ts.tar.gz" | awk '{print $1}' >"$work/k3s-datastore-$ts.tar.gz.sha256"
|
||||
size="$(stat -c %s "$work/k3s-datastore-$ts.tar.gz")"
|
||||
|
||||
hb_mc "$work" cp --quiet "/work/k3s-datastore-$ts.tar.gz" "/work/k3s-datastore-$ts.tar.gz.sha256" "backup/$AISTOR_BUCKET/k3s-datastore/" >/dev/null
|
||||
hb_mc "$work" stat "backup/$AISTOR_BUCKET/k3s-datastore/k3s-datastore-$ts.tar.gz" >/dev/null
|
||||
|
||||
# Local copy outside AIStor on the same HDD (sdb3). AIStor runs inside the
|
||||
# cluster this archive restores, so after losing the cluster the AIStor copy
|
||||
# cannot be read until k3s is back; this copy can. The directory sits next to
|
||||
# AIStor's drive directory (data/), so AIStor does not see it as a bucket.
|
||||
readonly LOCAL_DIR=/srv/k3s/aistor/host-backups/k3s-datastore
|
||||
install -d -m 0700 -o root -g root "$(dirname "$LOCAL_DIR")" "$LOCAL_DIR"
|
||||
install -m 0600 "$work/k3s-datastore-$ts.tar.gz" "$work/k3s-datastore-$ts.tar.gz.sha256" "$LOCAL_DIR/"
|
||||
ls -1t "$LOCAL_DIR"/k3s-datastore-*.tar.gz | tail -n +8 | while read -r old; do rm -f -- "$old" "$old.sha256"; done
|
||||
local_copies="$(ls -1 "$LOCAL_DIR"/k3s-datastore-*.tar.gz | wc -l)"
|
||||
|
||||
offhost=0
|
||||
if [[ -n "${OFFHOST_GPG_RECIPIENT_FILE:-}" && -n "${OFFHOST_RCLONE_TARGET:-}" ]]; then
|
||||
command -v rclone >/dev/null || hb_fail "OFFHOST_RCLONE_TARGET is set but rclone is not installed"
|
||||
gpg --batch --yes --trust-model always --recipient-file "$OFFHOST_GPG_RECIPIENT_FILE" \
|
||||
--output "$work/k3s-datastore-$ts.tar.gz.gpg" --encrypt "$work/k3s-datastore-$ts.tar.gz"
|
||||
rclone copyto "$work/k3s-datastore-$ts.tar.gz.gpg" "$OFFHOST_RCLONE_TARGET/k3s-datastore-$ts.tar.gz.gpg"
|
||||
offhost=1
|
||||
fi
|
||||
|
||||
trap - ERR
|
||||
hb_write_metrics "$NAME" <<PROM
|
||||
# HELP platform_backup_last_success_timestamp_seconds Unix time of the last successful backup.
|
||||
# TYPE platform_backup_last_success_timestamp_seconds gauge
|
||||
platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
||||
# HELP platform_backup_last_run_success 1 when the last run of this backup succeeded.
|
||||
# TYPE platform_backup_last_run_success gauge
|
||||
platform_backup_last_run_success{backup="k3s-datastore"} 1
|
||||
platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
||||
# HELP platform_backup_last_size_bytes Size of the last successful backup archive.
|
||||
# TYPE platform_backup_last_size_bytes gauge
|
||||
platform_backup_last_size_bytes{backup="k3s-datastore"} $size
|
||||
platform_backup_last_duration_seconds{backup="k3s-datastore"} $(( $(date +%s) - started ))
|
||||
# HELP platform_backup_local_copies Archives kept outside AIStor on the HDD.
|
||||
# TYPE platform_backup_local_copies gauge
|
||||
platform_backup_local_copies{backup="k3s-datastore"} $local_copies
|
||||
# HELP platform_backup_offhost_copy 1 when the last run also wrote the encrypted off-host copy.
|
||||
# TYPE platform_backup_offhost_copy gauge
|
||||
platform_backup_offhost_copy{backup="k3s-datastore"} $offhost
|
||||
PROM
|
||||
hb_log "k3s datastore backup k3s-datastore-$ts.tar.gz ($size bytes) uploaded; off-host=$offhost"
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create the AIStor bucket, lifecycle rules, policy and user that the host backup
|
||||
# timers use, and the root-only /etc/platform-backup/aistor.env.
|
||||
#
|
||||
# sudo bash scripts/bootstrap/create-host-backup-credentials.sh
|
||||
#
|
||||
# Run once, by the owner, on the desktop. Refuses to run when the env file exists;
|
||||
# never rotates or deletes a bucket, policy, user or object. No key is printed.
|
||||
set -Eeuo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$ROOT/scripts/lib/host-backup.sh"
|
||||
readonly BUCKET=platform-host-backups
|
||||
readonly POLICY=platform-host-backups
|
||||
readonly QUOTA=50GiB
|
||||
readonly POLICY_FILE="$ROOT/services/platform-host-checks/bucket-policy.json"
|
||||
readonly PASS_LINE=PLATFORM_HOST_BACKUP_BOOTSTRAP_PASS
|
||||
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; }
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || hb_fail "run with sudo"
|
||||
for c in docker jq openssl; do command -v "$c" >/dev/null || hb_fail "missing command: $c"; done
|
||||
jq -e . "$POLICY_FILE" >/dev/null || hb_fail "bucket policy is not valid JSON"
|
||||
[[ ! -e "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV already exists; this script does not rotate it"
|
||||
|
||||
ip="$(kubectl -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')"
|
||||
[[ -n "$ip" ]] || hb_fail "object-storage/minio Service not found"
|
||||
endpoint="http://${ip}:80"
|
||||
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
kubectl -n object-storage get secret aistor-root-configuration -o jsonpath='{.data.config\.env}' | base64 -d >"$work/root.env"
|
||||
grep -q '^export MINIO_ROOT_USER=' "$work/root.env" || hb_fail "aistor-root-configuration has an unexpected format"
|
||||
printf 'pcbackup%s' "$(openssl rand -hex 8)" >"$work/ACCESS_KEY"
|
||||
openssl rand -hex 32 | tr -d '\n' >"$work/SECRET_KEY"
|
||||
cp "$POLICY_FILE" "$work/policy.json"
|
||||
|
||||
printf '%s\n' \
|
||||
"Bucket: $BUCKET (quota $QUOTA; expire gitea/ after 56 days, k3s-datastore/ after 30 days) on $endpoint" \
|
||||
"Policy: $POLICY (${POLICY_FILE#"$ROOT/"})" \
|
||||
"User: a new generated access key attached to $POLICY" \
|
||||
"File: $HOST_BACKUP_ENV (root, 0600)" \
|
||||
'No key value will be printed.'
|
||||
printf 'Type APPLY to create them: '
|
||||
IFS= read -r confirmation
|
||||
[[ "$confirmation" == "APPLY" ]] || hb_fail "confirmation did not match APPLY"
|
||||
|
||||
# Every mc call is silenced so no key reaches the terminal.
|
||||
docker run --rm --network host -v "$work:/run/b:ro" --tmpfs /tmp -e MC_CONFIG_DIR=/tmp/mc \
|
||||
--entrypoint /bin/sh "$MC_IMAGE" -ec '
|
||||
. /run/b/root.env
|
||||
ak="$(cat /run/b/ACCESS_KEY)"; sk="$(cat /run/b/SECRET_KEY)"
|
||||
mc alias set root "'"$endpoint"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null
|
||||
mc mb --ignore-existing "root/'"$BUCKET"'" >/dev/null
|
||||
mc quota set "root/'"$BUCKET"'" --size "'"$QUOTA"'" >/dev/null
|
||||
mc ilm rule add --prefix gitea/ --expire-days 56 "root/'"$BUCKET"'" >/dev/null
|
||||
mc ilm rule add --prefix k3s-datastore/ --expire-days 30 "root/'"$BUCKET"'" >/dev/null
|
||||
mc admin policy create root "'"$POLICY"'" /run/b/policy.json >/dev/null
|
||||
mc admin user add root "$ak" "$sk" >/dev/null
|
||||
mc admin policy attach root "'"$POLICY"'" --user "$ak" >/dev/null
|
||||
mc alias set backup "'"$endpoint"'" "$ak" "$sk" --api S3v4 >/dev/null
|
||||
probe=".bootstrap-probe-$(date +%s)"
|
||||
printf probe | mc pipe "backup/'"$BUCKET"'/$probe" >/dev/null
|
||||
[ "$(mc cat "backup/'"$BUCKET"'/$probe")" = probe ]
|
||||
mc rm "backup/'"$BUCKET"'/$probe" >/dev/null
|
||||
if mc ls backup/loki >/dev/null 2>&1; then echo "the backup user can list another bucket" >&2; exit 1; fi
|
||||
echo "'"$PASS_LINE"'"' | grep -qx "$PASS_LINE" || hb_fail "bootstrap did not finish; $HOST_BACKUP_ENV was not written"
|
||||
|
||||
install -d -m 0700 -o root -g root "$(dirname "$HOST_BACKUP_ENV")"
|
||||
{
|
||||
echo "AISTOR_BUCKET=$BUCKET"
|
||||
echo "AISTOR_ACCESS_KEY=$(cat "$work/ACCESS_KEY")"
|
||||
echo "AISTOR_SECRET_KEY=$(cat "$work/SECRET_KEY")"
|
||||
echo "# Off-host encrypted copy of the k3s datastore archive (off until both are set):"
|
||||
echo "# OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc"
|
||||
echo "# OFFHOST_RCLONE_TARGET=remote:path"
|
||||
} >"$HOST_BACKUP_ENV"
|
||||
chmod 0600 "$HOST_BACKUP_ENV"
|
||||
printf '%s\n' "$PASS_LINE" "Created bucket $BUCKET, policy $POLICY, one user and $HOST_BACKUP_ENV."
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the host backup and check timers (infrastructure/host/backup).
|
||||
#
|
||||
# bash scripts/bootstrap/install-host-checks.sh --check # no root, changes nothing
|
||||
# sudo bash scripts/bootstrap/install-host-checks.sh --execute # install, enable, first runs
|
||||
#
|
||||
# Scripts are copied to /usr/local/lib/platform-backup (root-owned) so the root
|
||||
# timers never execute files the desktop user can edit.
|
||||
set -Eeuo pipefail
|
||||
umask 022
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
readonly LIB=/usr/local/lib/platform-backup
|
||||
readonly UNIT_DIR=/etc/systemd/system
|
||||
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
|
||||
readonly WORK_ROOT=/var/lib/platform-backup/work
|
||||
readonly USER_NAME=donghyeon
|
||||
readonly -a SCRIPTS=(
|
||||
scripts/lib/host-backup.sh:lib/host-backup.sh
|
||||
scripts/backup/k3s-datastore-backup.sh:backup/k3s-datastore-backup.sh
|
||||
scripts/backup/gitea-backup.sh:backup/gitea-backup.sh
|
||||
scripts/validate/live-drift-check-main.sh:validate/live-drift-check-main.sh
|
||||
scripts/validate/k3s-cert-expiry.sh:validate/k3s-cert-expiry.sh
|
||||
services/platform-host-checks/README.md:README.md
|
||||
)
|
||||
readonly -a TIMERS=(platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check platform-k3s-cert-expiry)
|
||||
|
||||
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
mode="${1:-}"
|
||||
[[ "$#" -eq 1 && ( "$mode" == "--check" || "$mode" == "--execute" ) ]] || { sed -n '2,8p' "$0" >&2; exit 2; }
|
||||
|
||||
echo "== check"
|
||||
for pair in "${SCRIPTS[@]}"; do
|
||||
src="${pair%%:*}"
|
||||
[[ -f "$ROOT/$src" ]] || fail "missing $src"
|
||||
[[ "$src" != *.sh ]] || bash -n "$ROOT/$src" || fail "syntax error in $src"
|
||||
done
|
||||
for t in "${TIMERS[@]}"; do
|
||||
for u in service timer; do [[ -f "$ROOT/infrastructure/host/backup/$t.$u" ]] || fail "missing $t.$u"; done
|
||||
done
|
||||
if command -v systemd-analyze >/dev/null; then
|
||||
verify_dir="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$verify_dir"' EXIT
|
||||
cp "$ROOT"/infrastructure/host/backup/*.service "$ROOT"/infrastructure/host/backup/*.timer "$verify_dir/"
|
||||
# ExecStart paths do not exist before installation; only syntax and keys are checked.
|
||||
out="$(cd "$verify_dir" && systemd-analyze verify ./*.timer ./*.service 2>&1 | grep -v -E 'is not executable|No such file|Command .* is not executable' || true)"
|
||||
[[ -z "$out" ]] || { printf '%s\n' "$out" >&2; fail "systemd-analyze verify reported problems"; }
|
||||
echo " units: systemd-analyze verify clean"
|
||||
fi
|
||||
echo "== plan"
|
||||
for pair in "${SCRIPTS[@]}"; do echo " install ${pair%%:*} -> $LIB/${pair##*:}"; done
|
||||
for t in "${TIMERS[@]}"; do echo " install $UNIT_DIR/$t.service, $UNIT_DIR/$t.timer; enable --now $t.timer"; done
|
||||
echo " mkdir $TEXTFILE_DIR (root:$USER_NAME 0775), $WORK_ROOT (root 0700)"
|
||||
echo " first runs: k3s-cert-expiry, k3s-datastore-backup, gitea-backup, live-drift-check"
|
||||
[[ "$mode" == "--execute" ]] || { echo "CHECK ONLY: nothing changed"; exit 0; }
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || fail "--execute needs root (sudo)"
|
||||
[[ -f /etc/platform-backup/aistor.env ]] || fail "run scripts/bootstrap/create-host-backup-credentials.sh first"
|
||||
echo "== install"
|
||||
for pair in "${SCRIPTS[@]}"; do
|
||||
install -D -o root -g root -m 0755 "$ROOT/${pair%%:*}" "$LIB/${pair##*:}"
|
||||
done
|
||||
install -d -o root -g "$USER_NAME" -m 0775 "$TEXTFILE_DIR"
|
||||
install -d -o root -g root -m 0700 "$WORK_ROOT"
|
||||
for t in "${TIMERS[@]}"; do
|
||||
install -o root -g root -m 0644 "$ROOT/infrastructure/host/backup/$t.service" "$ROOT/infrastructure/host/backup/$t.timer" "$UNIT_DIR/"
|
||||
done
|
||||
systemctl daemon-reload
|
||||
for t in "${TIMERS[@]}"; do systemctl enable --now "$t.timer"; done
|
||||
|
||||
echo "== first runs"
|
||||
for t in platform-k3s-cert-expiry platform-k3s-datastore-backup platform-gitea-backup platform-live-drift-check; do
|
||||
if systemctl start "$t.service"; then echo " $t: ok"; else echo " $t: FAILED (journalctl -u $t)"; fi
|
||||
done
|
||||
echo "== metrics"
|
||||
grep -h -E '^platform_' "$TEXTFILE_DIR"/*.prom || echo " no metrics yet"
|
||||
systemctl list-timers 'platform-*' --no-pager
|
||||
echo "HOST CHECKS INSTALLED"
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared helpers for the host backup and host check scripts. Source only.
|
||||
# shellcheck shell=bash
|
||||
|
||||
readonly HOST_BACKUP_ENV=/etc/platform-backup/aistor.env
|
||||
readonly TEXTFILE_DIR=/var/lib/node_exporter/textfile_collector
|
||||
# Backup staging (root 0700, same SSD). Not /var/tmp: dockerd must see the path.
|
||||
readonly HB_WORK_ROOT=/var/lib/platform-backup/work
|
||||
# Same pinned AIStor client as scripts/validate/aistor-s3-smoke.sh.
|
||||
readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b"
|
||||
|
||||
hb_log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; }
|
||||
hb_fail() { hb_log "FAIL: $*" >&2; exit 1; }
|
||||
|
||||
# Write Prometheus textfile metrics atomically: hb_write_metrics NAME <<'PROM' ... PROM
|
||||
hb_write_metrics() {
|
||||
local name="$1" tmp
|
||||
[[ -d "$TEXTFILE_DIR" ]] || { hb_log "textfile dir $TEXTFILE_DIR missing; metrics not written"; cat >/dev/null; return 0; }
|
||||
tmp="$(mktemp "$TEXTFILE_DIR/.${name}.XXXXXX")"
|
||||
cat >"$tmp"
|
||||
chmod 0644 "$tmp"
|
||||
mv -f -- "$tmp" "$TEXTFILE_DIR/${name}.prom"
|
||||
}
|
||||
|
||||
# Load the AIStor endpoint and the backup user's keys (root-only file).
|
||||
hb_load_aistor_env() {
|
||||
[[ -f "$HOST_BACKUP_ENV" ]] || hb_fail "$HOST_BACKUP_ENV is missing; run scripts/bootstrap/create-host-backup-credentials.sh"
|
||||
[[ "$(stat -c '%u %a' "$HOST_BACKUP_ENV")" == "0 600" ]] || hb_fail "$HOST_BACKUP_ENV must be owned by root with mode 0600"
|
||||
# shellcheck disable=SC1090
|
||||
. "$HOST_BACKUP_ENV"
|
||||
: "${AISTOR_BUCKET:?}" "${AISTOR_ACCESS_KEY:?}" "${AISTOR_SECRET_KEY:?}"
|
||||
# The host cannot resolve cluster DNS, so use the Service ClusterIP (port 80 -> 9000).
|
||||
if [[ -z "${AISTOR_ENDPOINT:-}" ]]; then
|
||||
local ip
|
||||
ip="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml -n object-storage \
|
||||
get service minio -o jsonpath='{.spec.clusterIP}')"
|
||||
[[ -n "$ip" ]] || hb_fail "cannot find the object-storage/minio Service"
|
||||
AISTOR_ENDPOINT="http://${ip}:80"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run the pinned mc client on the host network with the backup user's keys.
|
||||
# The keys go through a 0600 env file, never the command line (ps would show it).
|
||||
# Usage: hb_mc <dir mounted at /work> <mc args...>
|
||||
hb_mc() {
|
||||
local work="$1"; shift
|
||||
local envfile="$work/.mc.env"
|
||||
if [[ ! -f "$envfile" ]]; then
|
||||
( umask 077; printf 'MC_HOST_backup=%s\nMC_CONFIG_DIR=/tmp/mc\n' \
|
||||
"${AISTOR_ENDPOINT/:\/\//://${AISTOR_ACCESS_KEY}:${AISTOR_SECRET_KEY}@}" >"$envfile" )
|
||||
fi
|
||||
docker run --rm --network host --user "$(id -u):$(id -g)" \
|
||||
--env-file "$envfile" -v "$work:/work" --tmpfs /tmp \
|
||||
"$MC_IMAGE" "$@"
|
||||
}
|
||||
@@ -559,3 +559,50 @@ parser와 세 mode의 fixture 회귀는 live cluster에 연결하지 않고 다
|
||||
```sh
|
||||
bash scripts/validate/test-k3s-secret-encryption-status.sh
|
||||
```
|
||||
|
||||
## 테넌트·우선순위·keycloak 정책 렌더 검사
|
||||
|
||||
`render-tenants-and-policies.sh`는 클러스터에 닿지 않고 세 빌드 루트를 렌더해 불변식을
|
||||
확인한다.
|
||||
|
||||
- `services/tenants/techlog`: tech-log 테넌트의 플랫폼 쪽 객체 11개(목록이 정확히 같아야
|
||||
한다), `hyeonworks-prod` PSA 라벨, techlog DatabaseRole 권한 플래그, 대시보드 이름
|
||||
- `services/priority-classes`: `platform-critical`(0보다 크고 시스템 클래스보다 작다),
|
||||
`platform-low`(0보다 작고 선점하지 않는다), 둘 다 global default가 아니다
|
||||
- `infrastructure/networking/keycloak`: namespace 전체 기본 차단, egress 규칙마다 포트,
|
||||
허용 CIDR은 API 서버 둘뿐
|
||||
|
||||
```sh
|
||||
bash scripts/validate/render-tenants-and-policies.sh
|
||||
```
|
||||
|
||||
## 라이브 drift 검사
|
||||
|
||||
`live-drift-check.sh`는 이 저장소의 라이브 빌드 루트를 모두 렌더하고(Phase 1·2, 관리 도구,
|
||||
관측 core, Traefik trust overlay, 위 세 루트) Secret을 뺀 뒤 `kubectl diff`(서버 dry-run,
|
||||
저장하지 않음)로 클러스터와 비교한다. 렌더에는 위 render 스크립트의 `--verified-output-dir`
|
||||
출력을 읽기만 하는 용도로 쓴다(그 디렉터리는 스크립트가 끝날 때 지운다).
|
||||
|
||||
```sh
|
||||
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
|
||||
bash scripts/validate/live-drift-check.sh \
|
||||
--accepted scripts/validate/live-drift-accepted.txt
|
||||
```
|
||||
|
||||
- 종료 코드: 0 차이 없음, 1 차이 있음, 2 검사 자체 실패.
|
||||
- `DRIFT <Kind> <ns>/<name> (+추가 -삭제) [입력 파일]`로 객체마다 한 줄을 쓴다.
|
||||
`generation` 줄만 다른 것은 세지 않는다.
|
||||
- `--accepted`: 알고 있는 차이를 기한과 이유를 달아 적는다. 기한이 지나면 다시 drift로 센다.
|
||||
- `--textfile FILE`: `platform_live_drift_objects`, `platform_live_drift_check_success`,
|
||||
`platform_live_drift_last_run_timestamp_seconds`를 Prometheus textfile 형식으로 쓴다.
|
||||
- Secret 값의 차이는 보지 않는다.
|
||||
|
||||
## 호스트 백업·점검 시험
|
||||
|
||||
- `test-platform-host-checks-alerts.sh`: `services/platform-host-checks/prometheus-rules.yaml`의 promtool
|
||||
단위 시험(백업 신선도, 실패, 지표 없음, drift, k3s 인증서). docker 필요, 클러스터 접근 없음.
|
||||
- `k3s-cert-expiry.sh`: 6443·10250 serving 인증서 만료를 textfile 지표로 쓴다(타이머가 매일 돌림).
|
||||
- `live-drift-check-main.sh`: checkout의 작업 트리가 아니라 `origin/main`으로 `live-drift-check.sh`를 돌린다.
|
||||
- `k3s-datastore-restore-drill.sh`, `gitea-backup-restore-drill.sh`: 복원 리허설. 인자 없이 root로 돌리면 AIStor의
|
||||
최신 백업을 쓰고, `--from-dir`·`--from-file`로 로컬 사본을 줄 수 있다.
|
||||
- `test-k3s-datastore-restore-drill.sh`: 운영 데이터 없이 k3s 복원 리허설 경로를 시험한다(일회용 k3s, 시험 Secret 셋).
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# Restore drill for the Gitea backup (file level).
|
||||
#
|
||||
# sudo bash scripts/validate/gitea-backup-restore-drill.sh # latest dump in AIStor
|
||||
# bash scripts/validate/gitea-backup-restore-drill.sh --from-file DUMP.tar.gz
|
||||
#
|
||||
# Extracts the dump into a throwaway directory and checks what a restore needs:
|
||||
# app.ini is present, every repository passes `git fsck`, and the repository
|
||||
# count matches the live Gitea. The database is restored from the CloudNativePG
|
||||
# backup instead (services/platform-postgres-backup); the full restore order is
|
||||
# in docs/runbooks/gitea-restore.md. Nothing is written to the cluster.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$ROOT/scripts/lib/host-backup.sh"
|
||||
readonly NAME=gitea_restore_drill
|
||||
|
||||
from_file=""
|
||||
case "${1:-}" in
|
||||
--from-file) from_file="${2:?--from-file needs a path}" ;;
|
||||
"") ;;
|
||||
*) sed -n '2,11p' "$0" >&2; exit 2 ;;
|
||||
esac
|
||||
if [[ "$(id -u)" -eq 0 ]]; then
|
||||
kubectl() { command /usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml "$@"; }
|
||||
fi
|
||||
|
||||
base="$HB_WORK_ROOT"; [[ -w "$base" ]] || base="$(mktemp -d)"
|
||||
work="$(mktemp -d "$base/gitea-drill.XXXXXX")"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
|
||||
if [[ -z "$from_file" ]]; then
|
||||
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials"
|
||||
hb_load_aistor_env
|
||||
latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/gitea/" | awk '{print $NF}' | grep -E '^gitea-dump-.*\.tar\.gz$' | sort | tail -1)"
|
||||
[[ -n "$latest" ]] || hb_fail "no Gitea dump found"
|
||||
hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/gitea/$latest" "backup/$AISTOR_BUCKET/gitea/$latest.sha256" /work/ >/dev/null
|
||||
[[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest"
|
||||
from_file="$work/$latest"
|
||||
hb_log "drill on $latest"
|
||||
fi
|
||||
|
||||
mkdir "$work/restore"
|
||||
tar -C "$work/restore" -xzf "$from_file"
|
||||
[[ -s "$work/restore/app.ini" ]] || hb_fail "app.ini is missing from the dump"
|
||||
for key in INTERNAL_TOKEN SECRET_KEY; do
|
||||
grep -q "^${key}[[:space:]]*=" "$work/restore/app.ini" || hb_fail "app.ini has no ${key} (needed to restore)"
|
||||
done
|
||||
|
||||
total=0 broken=0
|
||||
while IFS= read -r -d '' repo; do
|
||||
total=$((total + 1))
|
||||
if ! git -c safe.directory='*' --git-dir="$repo" fsck --no-dangling --no-progress >/dev/null 2>"$work/fsck.err"; then
|
||||
broken=$((broken + 1))
|
||||
hb_log "fsck failed: ${repo#"$work/restore/"}: $(head -1 "$work/fsck.err")"
|
||||
fi
|
||||
done < <(find "$work/restore/repos" -mindepth 2 -maxdepth 2 -type d -name '*.git' -print0)
|
||||
hb_log "repositories in dump: $total, fsck failures: $broken"
|
||||
(( total > 0 )) || hb_fail "the dump has no repositories"
|
||||
(( broken == 0 )) || hb_fail "$broken repositories failed git fsck"
|
||||
|
||||
live="$(kubectl -n gitea exec deploy/gitea -c gitea -- sh -c 'find /data/git/gitea-repositories -mindepth 2 -maxdepth 2 -name "*.git" -type d | wc -l' 2>/dev/null || echo "")"
|
||||
if [[ -n "$live" ]]; then
|
||||
hb_log "live Gitea repositories: $live"
|
||||
(( total >= live - 2 )) || hb_fail "the dump has $total repositories, live Gitea has $live"
|
||||
fi
|
||||
|
||||
hb_write_metrics "$NAME" <<PROM
|
||||
# HELP platform_backup_restore_drill_last_success_timestamp_seconds Unix time of the last successful restore drill.
|
||||
# TYPE platform_backup_restore_drill_last_success_timestamp_seconds gauge
|
||||
platform_backup_restore_drill_last_success_timestamp_seconds{backup="gitea"} $(date +%s)
|
||||
PROM
|
||||
echo "GITEA RESTORE DRILL PASS"
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# Write the notAfter time of the k3s API server (6443) and kubelet (10250)
|
||||
# serving certificates as Prometheus textfile metrics. No root needed.
|
||||
# k3s renews these only when it starts and they are within 120 days of expiry
|
||||
# (docs.k3s.io cli/certificate), and the kubelet exposes no certificate metric,
|
||||
# so nothing else in the cluster watches them.
|
||||
set -Eeuo pipefail
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$(dirname -- "${BASH_SOURCE[0]}")/../lib/host-backup.sh"
|
||||
|
||||
lines=""
|
||||
for target in apiserver:6443 kubelet:10250; do
|
||||
name="${target%%:*}" port="${target##*:}"
|
||||
end="$(echo | timeout 10 openssl s_client -connect "127.0.0.1:$port" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//')" || true
|
||||
[[ -n "$end" ]] || hb_fail "could not read the $name certificate on 127.0.0.1:$port"
|
||||
lines+="platform_k3s_certificate_not_after_seconds{certificate=\"$name\"} $(date -d "$end" +%s)"$'\n'
|
||||
hb_log "$name certificate notAfter $end"
|
||||
done
|
||||
hb_write_metrics platform_k3s_certificates <<PROM
|
||||
# HELP platform_k3s_certificate_not_after_seconds notAfter of a k3s serving certificate (Unix time).
|
||||
# TYPE platform_k3s_certificate_not_after_seconds gauge
|
||||
${lines}# HELP platform_k3s_certificate_check_timestamp_seconds Unix time of the last certificate check.
|
||||
# TYPE platform_k3s_certificate_check_timestamp_seconds gauge
|
||||
platform_k3s_certificate_check_timestamp_seconds $(date +%s)
|
||||
PROM
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# Restore drill for the k3s datastore backup.
|
||||
#
|
||||
# sudo bash scripts/validate/k3s-datastore-restore-drill.sh # latest backup in AIStor
|
||||
# bash scripts/validate/k3s-datastore-restore-drill.sh --from-dir DIR [--expect-secrets N]
|
||||
# DIR holds state.db and token (an extracted archive, or a test datastore)
|
||||
#
|
||||
# Starts a throwaway k3s server (same version as production, --network none,
|
||||
# no agent) on a copy of the backup, and checks that the API answers, that the
|
||||
# Secret count matches and that every Secret decrypts (the token unlocks the
|
||||
# encryption config stored in the datastore). Prints no Secret value. The
|
||||
# container and the copy are removed at the end.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
# shellcheck source=../lib/host-backup.sh
|
||||
source "$ROOT/scripts/lib/host-backup.sh"
|
||||
# Same version as the desktop server (k3s --version: v1.36.2+k3s1).
|
||||
readonly K3S_IMAGE="docker.io/rancher/k3s:v1.36.2-k3s1@sha256:6a47cea22c4b834d4ba72c89d291696b79ebe406251f90b446e4dff03513dd87"
|
||||
readonly NAME=k3s_datastore_restore_drill
|
||||
|
||||
from_dir="" expect_secrets=""
|
||||
while (( $# > 0 )); do
|
||||
case "$1" in
|
||||
--from-dir) from_dir="${2:?}"; shift 2 ;;
|
||||
--expect-secrets) expect_secrets="${2:?}"; shift 2 ;;
|
||||
*) sed -n '2,13p' "$0" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
base="${HB_WORK_ROOT}"
|
||||
[[ -w "$base" ]] || base="$(mktemp -d)"
|
||||
work="$(mktemp -d "$base/k3s-drill.XXXXXX")"
|
||||
container="k3s-restore-drill-$$"
|
||||
cleanup() {
|
||||
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||
# The drill container writes as root; remove its files with the same image.
|
||||
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if [[ -z "$from_dir" ]]; then
|
||||
[[ "$(id -u)" -eq 0 ]] || hb_fail "run as root to read the backup credentials"
|
||||
hb_load_aistor_env
|
||||
latest="$(hb_mc "$work" ls "backup/$AISTOR_BUCKET/k3s-datastore/" | awk '{print $NF}' | grep -E '^k3s-datastore-.*\.tar\.gz$' | sort | tail -1)"
|
||||
[[ -n "$latest" ]] || hb_fail "no k3s datastore backup found"
|
||||
hb_mc "$work" cp --quiet "backup/$AISTOR_BUCKET/k3s-datastore/$latest" "backup/$AISTOR_BUCKET/k3s-datastore/$latest.sha256" /work/ >/dev/null
|
||||
[[ "$(sha256sum "$work/$latest" | awk '{print $1}')" == "$(cat "$work/$latest.sha256")" ]] || hb_fail "checksum mismatch for $latest"
|
||||
tar -C "$work" -xzf "$work/$latest"
|
||||
from_dir="$(find "$work" -mindepth 1 -maxdepth 1 -type d -name 'k3s-datastore-*' | head -1)"
|
||||
hb_log "drill on $latest"
|
||||
if [[ -z "$expect_secrets" ]]; then
|
||||
expect_secrets="$(/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get secrets -A --no-headers | wc -l)"
|
||||
hb_log "live cluster has $expect_secrets Secrets (counts may differ by objects created since the backup)"
|
||||
fi
|
||||
fi
|
||||
[[ -f "$from_dir/state.db" && -f "$from_dir/token" ]] || hb_fail "$from_dir must contain state.db and token"
|
||||
|
||||
install -d -m 0700 "$work/server/db"
|
||||
cp "$from_dir/state.db" "$work/server/db/state.db"
|
||||
cp "$from_dir/token" "$work/server/token"
|
||||
[[ "$(sqlite3 "$work/server/db/state.db" 'PRAGMA integrity_check;')" == "ok" ]] || hb_fail "integrity_check failed"
|
||||
|
||||
started="$(date +%s)"
|
||||
docker run -d --name "$container" --network none --tmpfs /run --tmpfs /var/run \
|
||||
-v "$work/server:/var/lib/rancher/k3s/server" \
|
||||
"$K3S_IMAGE" server --disable-agent --secrets-encryption \
|
||||
--token-file /var/lib/rancher/k3s/server/token \
|
||||
--bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \
|
||||
--disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null
|
||||
|
||||
ready=0
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec "$container" kubectl get --raw /readyz >/dev/null 2>&1; then ready=1; break; fi
|
||||
sleep 5
|
||||
done
|
||||
if (( ready == 0 )); then
|
||||
docker logs --tail 20 "$container" 2>&1 | sed 's/^/ k3s: /' >&2
|
||||
hb_fail "restored k3s did not become ready in 5 minutes"
|
||||
fi
|
||||
namespaces="$(docker exec "$container" kubectl get namespaces --no-headers | wc -l)"
|
||||
secrets="$(docker exec "$container" kubectl get secrets -A --no-headers | wc -l)"
|
||||
# -o json decrypts every Secret on the server side; any failure makes kubectl exit non-zero.
|
||||
decrypted="$(docker exec "$container" kubectl get secrets -A -o json | python3 -I -c 'import json,sys; d=json.load(sys.stdin)["items"]; print(sum(1 for s in d if s.get("data")))')"
|
||||
hb_log "restored: namespaces=$namespaces secrets=$secrets with-data=$decrypted in $(( $(date +%s) - started ))s"
|
||||
if [[ -n "$expect_secrets" ]]; then
|
||||
diff=$(( secrets - expect_secrets ))
|
||||
(( ${diff#-} <= 5 )) || hb_fail "Secret count $secrets is far from the expected $expect_secrets"
|
||||
fi
|
||||
(( decrypted > 0 )) || hb_fail "no Secret carried data after restore"
|
||||
|
||||
hb_write_metrics "$NAME" <<PROM
|
||||
# HELP platform_backup_restore_drill_last_success_timestamp_seconds Unix time of the last successful restore drill.
|
||||
# TYPE platform_backup_restore_drill_last_success_timestamp_seconds gauge
|
||||
platform_backup_restore_drill_last_success_timestamp_seconds{backup="k3s-datastore"} $(date +%s)
|
||||
PROM
|
||||
echo "K3S DATASTORE RESTORE DRILL PASS"
|
||||
@@ -0,0 +1,10 @@
|
||||
# 알고 있는 라이브 drift. 형식: <Kind> <namespace 또는 -> <name> <기한 YYYY-MM-DD> <이유>
|
||||
# 기한이 지나면 live-drift-check.sh가 다시 drift로 센다. 해소하면 줄을 지운다.
|
||||
Cluster platform-data platform-postgres 2026-11-30 pg_hba의 techlog 두 줄이 repo에 없다. w1이 CNPG 백업과 함께 넣는다(1c4ef9a).
|
||||
DatabaseRole platform-data platform-postgres-gitea 2026-11-30 렌더가 false 기본값 필드를 명시할 뿐 값은 같다. 다음 Phase 1 적용 때 사라진다.
|
||||
DatabaseRole platform-data platform-postgres-keycloak 2026-11-30 렌더가 false 기본값 필드를 명시할 뿐 값은 같다. 다음 Phase 2 적용 때 사라진다.
|
||||
Deployment keycloak keycloak-operator 2026-11-30 원격 ref 26.7.0 태그 내용이 바뀌어 build-timestamp 라벨만 다르다. 재시작 창에 커밋 SHA 고정과 함께 적용한다.
|
||||
Service keycloak keycloak-operator 2026-11-30 위와 같은 build-timestamp 라벨.
|
||||
ServiceAccount keycloak keycloak-operator 2026-11-30 위와 같은 build-timestamp 라벨.
|
||||
Deployment observability observability-core-kube-pr-operator 2026-11-30 이미지 문자열이 tag@digest와 digest로 다르고 digest는 같다. 적용하면 재시작되므로 재시작 창에.
|
||||
Deployment observability observability-core-kube-state-metrics 2026-11-30 위와 같은 이미지 문자열 차이.
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Daily drift check against origin/main (not the working tree of the checkout),
|
||||
# so local edits or a stale checkout do not hide or invent drift.
|
||||
# Run by platform-live-drift-check.timer as the desktop user.
|
||||
set -Eeuo pipefail
|
||||
readonly CHECKOUT="${PLATFORM_CHECKOUT:-/home/donghyeon/workspace/platform}"
|
||||
readonly TEXTFILE=/var/lib/node_exporter/textfile_collector/platform_live_drift.prom
|
||||
|
||||
git -C "$CHECKOUT" fetch --quiet origin main
|
||||
work="$(mktemp -d /tmp/platform-drift-main.XXXXXX)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
git -C "$CHECKOUT" archive origin/main | tar -x -C "$work"
|
||||
printf 'origin/main %s\n' "$(git -C "$CHECKOUT" rev-parse --short origin/main)"
|
||||
args=(--accepted "$work/scripts/validate/live-drift-accepted.txt")
|
||||
[[ -d "$(dirname "$TEXTFILE")" ]] && args+=(--textfile "$TEXTFILE")
|
||||
bash "$work/scripts/validate/live-drift-check.sh" "${args[@]}"
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
# Render every live build root of this repository and compare it with the
|
||||
# cluster using `kubectl diff` (server-side dry-run: nothing is persisted).
|
||||
#
|
||||
# Usage:
|
||||
# PLATFORM_HELM_BIN=/abs/helm bash scripts/validate/live-drift-check.sh \
|
||||
# [--accepted FILE] [--textfile FILE]
|
||||
#
|
||||
# --accepted FILE known differences, one per line:
|
||||
# `<Kind> <namespace or -> <name> <until YYYY-MM-DD> <reason...>`
|
||||
# Expired entries count as drift again.
|
||||
# --textfile FILE also write Prometheus textfile metrics (atomic rename).
|
||||
#
|
||||
# Exit status: 0 no drift, 1 drift found, 2 the check itself failed.
|
||||
# Secrets are removed from the renders before diffing; Secret drift is out of scope.
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
accepted_file=""
|
||||
textfile=""
|
||||
while (( $# > 0 )); do
|
||||
case "$1" in
|
||||
--accepted) accepted_file="${2:?--accepted needs a file}"; shift 2 ;;
|
||||
--textfile) textfile="${2:?--textfile needs a file}"; shift 2 ;;
|
||||
-h|--help) sed -n '2,16p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
||||
*) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
declare -a made_dirs=()
|
||||
cleanup() {
|
||||
local d
|
||||
for d in "${made_dirs[@]}"; do
|
||||
case "$d" in /tmp/platform-*) rm -rf -- "$d" ;; esac
|
||||
done
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
write_textfile() {
|
||||
local drift="$1" ok="$2"
|
||||
[[ -n "$textfile" ]] || return 0
|
||||
local tmp="${textfile}.tmp.$$"
|
||||
{
|
||||
echo '# HELP platform_live_drift_objects Objects whose live state differs from the repository render.'
|
||||
echo '# TYPE platform_live_drift_objects gauge'
|
||||
echo "platform_live_drift_objects ${drift}"
|
||||
echo '# HELP platform_live_drift_check_success 1 when the drift check itself completed.'
|
||||
echo '# TYPE platform_live_drift_check_success gauge'
|
||||
echo "platform_live_drift_check_success ${ok}"
|
||||
echo '# HELP platform_live_drift_last_run_timestamp_seconds Unix time the drift check finished.'
|
||||
echo '# TYPE platform_live_drift_last_run_timestamp_seconds gauge'
|
||||
echo "platform_live_drift_last_run_timestamp_seconds $(date +%s)"
|
||||
} >"$tmp"
|
||||
mv -f -- "$tmp" "$textfile"
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'DRIFT CHECK ERROR: %s\n' "$*" >&2
|
||||
write_textfile 0 0
|
||||
exit 2
|
||||
}
|
||||
|
||||
work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work")
|
||||
mkdir -p "$work/in" "$work/out"
|
||||
|
||||
render_root() {
|
||||
local script="$1" prefix="$2" dir
|
||||
dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir")
|
||||
bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \
|
||||
>"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; }
|
||||
printf '%s\n' "$dir"
|
||||
}
|
||||
|
||||
phase1="$(render_root render-phase1.sh platform-phase1-apply)"
|
||||
phase2="$(render_root render-phase2.sh platform-phase2-apply)"
|
||||
admin="$(render_root render-admin-services.sh platform-admin-apply)"
|
||||
observability="$(render_root render-observability-core.sh platform-observability-core-apply)"
|
||||
|
||||
# The live Gitea is the OIDC profile; gitea.yaml is the fresh-install baseline only.
|
||||
declare -a inputs=(
|
||||
"$phase1/namespaces.yaml" "$phase1/ssd-local-pv.yaml" "$phase1/cnpg-operator.yaml"
|
||||
"$phase1/platform-postgres.yaml" "$phase1/gitea-oidc.yaml"
|
||||
"$observability/aggregate.yaml"
|
||||
)
|
||||
for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done
|
||||
|
||||
# Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md).
|
||||
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed"
|
||||
inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml")
|
||||
|
||||
n=0
|
||||
for f in "${inputs[@]}"; do
|
||||
n=$((n + 1))
|
||||
python3 -I - "$f" "$work/in/$(printf '%02d' "$n")-$(basename "$f")" <<'PY' || die "could not strip Secrets from $f"
|
||||
import sys, yaml
|
||||
yaml.SafeLoader.add_constructor("tag:yaml.org,2002:value", lambda l, n: l.construct_scalar(n))
|
||||
docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if isinstance(d, dict) and d.get("kind") not in (None, "Secret")]
|
||||
yaml.safe_dump_all(docs, open(sys.argv[2], "w"), sort_keys=False)
|
||||
PY
|
||||
done
|
||||
|
||||
for f in "$work"/in/*.yaml; do
|
||||
set +e
|
||||
KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff -f "$f" >"$work/out/$(basename "$f").diff" 2>"$work/out/$(basename "$f").err"
|
||||
rc=$?
|
||||
set -e
|
||||
if (( rc > 1 )); then
|
||||
sed -n '1,5p' "$work/out/$(basename "$f").err" >&2
|
||||
die "kubectl diff failed for $(basename "$f")"
|
||||
fi
|
||||
done
|
||||
|
||||
set +e
|
||||
python3 -I - "$work/out" "${accepted_file:-}" "$work/drift-count" <<'PY'
|
||||
import sys, pathlib, re, datetime
|
||||
out, accepted_path, count_path = pathlib.Path(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3])
|
||||
today = datetime.date.today()
|
||||
accepted, expired = {}, set()
|
||||
if accepted_path:
|
||||
for line in pathlib.Path(accepted_path).read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
kind, ns, name, until, *reason = line.split()
|
||||
key = (kind, "" if ns == "-" else ns, name)
|
||||
if datetime.date.fromisoformat(until) < today:
|
||||
expired.add(key)
|
||||
else:
|
||||
accepted[key] = " ".join(reason)
|
||||
version = re.compile(r"^v\d+((alpha|beta)\d+)?$")
|
||||
objects = {}
|
||||
for diff in sorted(out.glob("*.diff")):
|
||||
current = None
|
||||
for line in diff.read_text().splitlines():
|
||||
if line.startswith("diff -u -N "):
|
||||
parts = line.split()[3].rsplit("/", 1)[-1].split(".")
|
||||
i = next(i for i, p in enumerate(parts) if version.match(p))
|
||||
current = (parts[i + 1], parts[i + 2], ".".join(parts[i + 3:]))
|
||||
objects.setdefault(current, [0, 0, diff.name])
|
||||
elif current and line[:1] in "+-" and not line.startswith(("+++", "---")):
|
||||
if re.match(r"^[+-]\s+generation:", line):
|
||||
continue
|
||||
objects[current][0 if line[0] == "+" else 1] += 1
|
||||
drift = accepted_seen = 0
|
||||
for key, (plus, minus, src) in sorted(objects.items()):
|
||||
if plus == 0 and minus == 0:
|
||||
continue
|
||||
kind, ns, name = key
|
||||
label = f"{kind} {ns or '-'}/{name} (+{plus} -{minus}) [{src}]"
|
||||
if key in accepted:
|
||||
accepted_seen += 1
|
||||
print(f"ACCEPTED {label}: {accepted[key]}")
|
||||
continue
|
||||
drift += 1
|
||||
print(f"DRIFT {label}" + (" (accepted entry expired)" if key in expired else ""))
|
||||
print(f"SUMMARY drift={drift} accepted={accepted_seen}")
|
||||
count_path.write_text(str(drift))
|
||||
PY
|
||||
rc=$?
|
||||
set -e
|
||||
(( rc == 0 )) || die "could not summarize the diffs"
|
||||
drift_count="$(cat "$work/drift-count")"
|
||||
write_textfile "$drift_count" 1
|
||||
if (( drift_count == 0 )); then
|
||||
echo "LIVE DRIFT CHECK PASS"
|
||||
exit 0
|
||||
fi
|
||||
echo "LIVE DRIFT CHECK FOUND DRIFT"
|
||||
exit 1
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
# Render the tech-log tenant root, the platform PriorityClasses and the keycloak
|
||||
# namespace NetworkPolicies, and check their invariants. Static: no cluster access.
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
|
||||
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
|
||||
fail() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')"
|
||||
[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
|
||||
fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}"
|
||||
|
||||
work="$(mktemp -d "${TMPDIR:-/tmp}/platform-tenants-render.XXXXXX")"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
|
||||
kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/techlog.yaml"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml"
|
||||
|
||||
python3 -I - "$work" <<'PY'
|
||||
import sys, pathlib, yaml
|
||||
|
||||
work = pathlib.Path(sys.argv[1])
|
||||
errors = []
|
||||
|
||||
def load(name):
|
||||
return [d for d in yaml.safe_load_all((work / name).read_text()) if d]
|
||||
|
||||
def keyset(docs):
|
||||
return sorted((d["kind"], d["metadata"].get("namespace", ""), d["metadata"]["name"]) for d in docs)
|
||||
|
||||
for name in ("techlog.yaml", "priority-classes.yaml", "keycloak-network-policies.yaml"):
|
||||
for d in load(name):
|
||||
if d["kind"] == "Secret":
|
||||
errors.append(f"{name}: Secret objects must not be rendered")
|
||||
|
||||
techlog = load("techlog.yaml")
|
||||
expected = sorted([
|
||||
("Namespace", "", "hyeonworks-prod"),
|
||||
("Database", "platform-data", "platform-postgres-techlog"),
|
||||
("DatabaseRole", "platform-data", "platform-postgres-techlog"),
|
||||
("NetworkPolicy", "platform-data", "allow-techlog-backend"),
|
||||
("NetworkPolicy", "object-storage", "allow-techlog-backend"),
|
||||
("Probe", "observability", "techlog-edge"),
|
||||
("Probe", "observability", "techlog-readiness"),
|
||||
("PrometheusRule", "observability", "techlog"),
|
||||
("ConfigMap", "observability", "grafana-dashboard-techlog"),
|
||||
("NetworkPolicy", "observability", "blackbox-allow-techlog-health"),
|
||||
("NetworkPolicy", "observability", "observability-allow-techlog-scrape"),
|
||||
])
|
||||
if keyset(techlog) != expected:
|
||||
errors.append(f"tech-log tenant objects differ: {keyset(techlog)}")
|
||||
for d in techlog:
|
||||
kind = d["kind"]
|
||||
if kind == "Namespace":
|
||||
labels = d["metadata"].get("labels", {})
|
||||
want = {
|
||||
"pod-security.kubernetes.io/enforce": "baseline",
|
||||
"pod-security.kubernetes.io/warn": "restricted",
|
||||
"pod-security.kubernetes.io/audit": "restricted",
|
||||
"observability.hyeonworks.com/enabled": "true",
|
||||
}
|
||||
for k, v in want.items():
|
||||
if labels.get(k) != v:
|
||||
errors.append(f"hyeonworks-prod label {k} must be {v}")
|
||||
if kind == "DatabaseRole":
|
||||
spec = d["spec"]
|
||||
if spec.get("passwordSecret", {}).get("name") != "techlog-db-credentials":
|
||||
errors.append("techlog DatabaseRole must reference techlog-db-credentials")
|
||||
for flag in ("superuser", "createdb", "createrole", "replication", "bypassrls"):
|
||||
if spec.get(flag) is not False:
|
||||
errors.append(f"techlog DatabaseRole {flag} must be false")
|
||||
if kind in ("Database", "DatabaseRole"):
|
||||
policy = d["spec"].get("databaseReclaimPolicy") or d["spec"].get("databaseRoleReclaimPolicy")
|
||||
if policy != "retain":
|
||||
errors.append(f"{kind} must keep the retain reclaim policy")
|
||||
if kind == "ConfigMap":
|
||||
labels = d["metadata"].get("labels", {})
|
||||
if labels.get("grafana_dashboard") != "1":
|
||||
errors.append("tech-log dashboard ConfigMap needs grafana_dashboard=1")
|
||||
if d["metadata"]["name"] != "grafana-dashboard-techlog":
|
||||
errors.append("tech-log dashboard ConfigMap must keep its name (no hash suffix)")
|
||||
|
||||
classes = {d["metadata"]["name"]: d for d in load("priority-classes.yaml")}
|
||||
if sorted(classes) != ["platform-critical", "platform-low"]:
|
||||
errors.append(f"PriorityClasses differ: {sorted(classes)}")
|
||||
else:
|
||||
crit, low = classes["platform-critical"], classes["platform-low"]
|
||||
if not (0 < crit["value"] < 1_000_000_000):
|
||||
errors.append("platform-critical must be above 0 and below the system classes")
|
||||
if not low["value"] < 0:
|
||||
errors.append("platform-low must be below the default priority 0")
|
||||
if low.get("preemptionPolicy") != "Never":
|
||||
errors.append("platform-low must never preempt")
|
||||
for d in classes.values():
|
||||
if d.get("globalDefault") is not False:
|
||||
errors.append(f"{d['metadata']['name']} must not be the global default")
|
||||
|
||||
policies = {d["metadata"]["name"]: d for d in load("keycloak-network-policies.yaml")}
|
||||
deny = policies.get("keycloak-default-deny")
|
||||
if not deny or deny["spec"].get("podSelector") != {} or sorted(deny["spec"].get("policyTypes", [])) != ["Egress", "Ingress"]:
|
||||
errors.append("keycloak needs a namespace-wide default deny for ingress and egress")
|
||||
for d in policies.values():
|
||||
if d["metadata"].get("namespace") != "keycloak":
|
||||
errors.append(f"{d['metadata']['name']} must live in the keycloak namespace")
|
||||
for rule in d["spec"].get("egress", []) or []:
|
||||
for peer in rule.get("to", []):
|
||||
cidr = peer.get("ipBlock", {}).get("cidr")
|
||||
if cidr and cidr not in ("10.43.0.1/32", "192.168.0.107/32"):
|
||||
errors.append(f"{d['metadata']['name']}: unexpected egress CIDR {cidr}")
|
||||
if not rule.get("ports"):
|
||||
errors.append(f"{d['metadata']['name']}: every egress rule must name its ports")
|
||||
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"tech-log tenant: {len(techlog)} objects; PriorityClasses: {len(classes)}; keycloak NetworkPolicies: {len(policies)}")
|
||||
PY
|
||||
|
||||
echo "TENANTS AND POLICIES STATIC RENDER PASS"
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test k3s-datastore-restore-drill.sh without production data: start a fresh,
|
||||
# network-less k3s with secrets encryption, add three test Secrets, stop it,
|
||||
# and run the drill on its state.db and token. Needs docker; no root.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
readonly K3S_IMAGE="$(sed -n 's/^readonly K3S_IMAGE="\(.*\)"$/\1/p' "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh")"
|
||||
[[ -n "$K3S_IMAGE" ]] || { echo "cannot read K3S_IMAGE from the drill script" >&2; exit 1; }
|
||||
work="$(mktemp -d)"
|
||||
seed="w6-k3s-drill-seed-$$"
|
||||
cleanup() {
|
||||
docker rm -f "$seed" >/dev/null 2>&1 || true
|
||||
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c 'rm -rf /w/*' >/dev/null 2>&1 || true
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$work/server" "$work/copy"
|
||||
|
||||
docker run -d --name "$seed" --network none --tmpfs /run --tmpfs /var/run \
|
||||
-v "$work/server:/var/lib/rancher/k3s/server" "$K3S_IMAGE" server --disable-agent --secrets-encryption \
|
||||
--bind-address 127.0.0.1 --advertise-address 127.0.0.1 --node-ip 127.0.0.1 \
|
||||
--disable traefik,servicelb,metrics-server,local-storage,coredns >/dev/null
|
||||
for _ in $(seq 1 60); do docker exec "$seed" kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 5; done
|
||||
for i in 1 2 3; do docker exec "$seed" kubectl create secret generic "drill-test-$i" --from-literal=value="test-$i" >/dev/null; done
|
||||
expected="$(docker exec "$seed" kubectl get secrets -A --no-headers | wc -l)"
|
||||
docker stop -t 30 "$seed" >/dev/null
|
||||
docker run --rm -v "$work:/w" --entrypoint /bin/sh "$K3S_IMAGE" -c \
|
||||
"cp /w/server/db/state.db* /w/server/token /w/copy/ && chown $(id -u):$(id -g) /w/copy/*"
|
||||
# Fold the WAL into state.db, as `sqlite3 .backup` does in the real backup.
|
||||
sqlite3 "$work/copy/state.db" 'PRAGMA wal_checkpoint(TRUNCATE);' >/dev/null
|
||||
rm -f "$work/copy/state.db-wal" "$work/copy/state.db-shm"
|
||||
# The test Secrets must be stored encrypted (k8s:enc:...), so the drill really
|
||||
# proves that the token unlocks the encryption key.
|
||||
encrypted="$(sqlite3 "$work/copy/state.db" "SELECT COUNT(*) FROM kine WHERE name LIKE '/registry/secrets/default/drill-test-%' AND CAST(value AS TEXT) LIKE 'k8s:enc:%';")"
|
||||
echo "seed: $expected Secrets, encrypted test Secret rows=$encrypted"
|
||||
(( encrypted >= 3 )) || { echo "test Secrets are not encrypted at rest in the seed datastore" >&2; exit 1; }
|
||||
|
||||
bash "$ROOT/scripts/validate/k3s-datastore-restore-drill.sh" --from-dir "$work/copy" --expect-secrets "$expected"
|
||||
echo "K3S DATASTORE RESTORE DRILL TEST PASS"
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
# promtool unit tests for services/platform-host-checks/prometheus-rules.yaml.
|
||||
# Needs docker and python3 (PyYAML). No cluster access.
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
||||
# Same pinned image as test-platform-observability-rules.sh.
|
||||
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
||||
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
python3 -I - "$ROOT/services/platform-host-checks/prometheus-rules.yaml" "$work/rules.yaml" <<'PY'
|
||||
import sys, yaml
|
||||
rule = yaml.safe_load(open(sys.argv[1]))
|
||||
yaml.safe_dump({"groups": rule["spec"]["groups"]}, open(sys.argv[2], "w"), sort_keys=False)
|
||||
PY
|
||||
|
||||
# promtool evaluates from Unix time 0, so "now" in each test is the eval_time.
|
||||
cat >"$work/tests.yaml" <<'EOF'
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
# k3s datastore: last success at t=0; stale after 2 days, alert after 1h more.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_run_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x5500'
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x5500'
|
||||
alert_rule_test:
|
||||
- eval_time: 24h
|
||||
alertname: PlatformK3sDatastoreBackupTooOld
|
||||
exp_alerts: []
|
||||
- eval_time: 50h
|
||||
alertname: PlatformK3sDatastoreBackupTooOld
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: k3s datastore has not been backed up for more than 2 days
|
||||
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
|
||||
- eval_time: 90h
|
||||
alertname: PlatformGiteaBackupTooOld
|
||||
exp_alerts: []
|
||||
- eval_time: 90h
|
||||
alertname: PlatformHostBackupMetricsAbsent
|
||||
exp_alerts: []
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_success_timestamp_seconds{backup="gitea"}'
|
||||
values: '0x11700'
|
||||
alert_rule_test:
|
||||
- eval_time: 194h
|
||||
alertname: PlatformGiteaBackupTooOld
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: Gitea has not been backed up for more than 8 days
|
||||
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
|
||||
# A failed run is reported with its backup label.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_backup_last_run_success{backup="gitea",instance="desktop"}'
|
||||
values: '0x30'
|
||||
- series: 'platform_backup_last_run_success{backup="k3s-datastore",instance="desktop"}'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 15m
|
||||
alertname: PlatformHostBackupFailed
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, backup: gitea}
|
||||
exp_annotations:
|
||||
summary: "The last gitea backup run failed"
|
||||
description: "Check `journalctl -u platform-gitea-backup` on desktop."
|
||||
# Metrics never written (timers or textfile collector missing).
|
||||
- interval: 1h
|
||||
input_series:
|
||||
- series: 'up{job="prometheus-node-exporter"}'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 25h
|
||||
alertname: PlatformHostBackupMetricsAbsent
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, backup: k3s-datastore}
|
||||
exp_annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
- exp_labels: {severity: warning, backup: gitea}
|
||||
exp_annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
# Drift for 7h fires; drift 0 does not.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_live_drift_objects'
|
||||
values: '3x500'
|
||||
- series: 'platform_live_drift_last_run_timestamp_seconds'
|
||||
values: '0x500'
|
||||
- series: 'platform_live_drift_check_success'
|
||||
values: '1x500'
|
||||
alert_rule_test:
|
||||
- eval_time: 5h
|
||||
alertname: PlatformLiveDriftDetected
|
||||
exp_alerts: []
|
||||
- eval_time: 7h
|
||||
alertname: PlatformLiveDriftDetected
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning}
|
||||
exp_annotations:
|
||||
summary: "3 live objects differ from platform-core origin/main"
|
||||
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
|
||||
- eval_time: 7h
|
||||
alertname: PlatformLiveDriftCheckStale
|
||||
exp_alerts: []
|
||||
# Certificates: 50 days left -> warning only; 10 days left -> warning and critical.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'platform_k3s_certificate_not_after_seconds{certificate="apiserver"}'
|
||||
values: '4320000x200'
|
||||
- series: 'platform_k3s_certificate_not_after_seconds{certificate="kubelet"}'
|
||||
values: '864000x200'
|
||||
- series: 'platform_k3s_certificate_check_timestamp_seconds'
|
||||
values: '0x200'
|
||||
alert_rule_test:
|
||||
- eval_time: 2h
|
||||
alertname: PlatformK3sServingCertificateExpiry
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, certificate: apiserver}
|
||||
exp_annotations:
|
||||
summary: "k3s apiserver serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- exp_labels: {severity: warning, certificate: kubelet}
|
||||
exp_annotations:
|
||||
summary: "k3s kubelet serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- exp_labels: {severity: critical, certificate: kubelet}
|
||||
exp_annotations:
|
||||
summary: "k3s kubelet serving certificate expires in less than 14 days"
|
||||
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
|
||||
- eval_time: 2h
|
||||
alertname: PlatformK3sCertificateCheckStale
|
||||
exp_alerts: []
|
||||
EOF
|
||||
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" check rules rules.yaml
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$work:/work:ro" -w /work --entrypoint /bin/promtool "$PROMETHEUS_IMAGE" test rules tests.yaml
|
||||
echo "PLATFORM HOST CHECK ALERTS PASS"
|
||||
@@ -34,6 +34,11 @@ kubeProxy:
|
||||
defaultRules:
|
||||
disabled:
|
||||
Watchdog: true
|
||||
# k3s kubelet exposes no kubelet_certificate_manager_* metrics, so these two
|
||||
# can never fire. Serving certificates are watched by platform-host-checks
|
||||
# (PlatformK3sServingCertificateExpiry) instead. Review 2026-11-11 A11.
|
||||
KubeletClientCertificateExpiration: true
|
||||
KubeletServerCertificateExpiration: true
|
||||
rules:
|
||||
etcd: false
|
||||
kubeControllerManager: false
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# 호스트 백업과 점검
|
||||
|
||||
desktop 호스트에서 systemd 타이머로 도는 백업과 점검이다. 결과는 node-exporter textfile 수집기
|
||||
(`/var/lib/node_exporter/textfile_collector/*.prom`)로 Prometheus에 올라가고, 이 폴더의 PrometheusRule이
|
||||
경보를 낸다. 근거와 결정은 `docs/reviews/2026-11-11-k3s-maturity-review.md` A1·A11·A13·A14, 8절 결정 1·6이다.
|
||||
|
||||
## 무엇을 어디에 얼마나
|
||||
|
||||
| 대상 | 방법 | 주기 | 보관 | 위치 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| k3s datastore + `server/token` | `sqlite3 .backup` → tar.gz + sha256 | 매일 03:30 KST | AIStor 30일 + 로컬 최근 7개 | AIStor `platform-host-backups/k3s-datastore/`와 `/srv/k3s/aistor/host-backups/k3s-datastore/`(root 0700). 둘 다 HDD sdb3 |
|
||||
| Gitea (저장소·패키지·첨부·Actions·`app.ini`) | 라이브 파드에서 `gitea dump --skip-db` | 일요일 04:00 KST | 56일(8주) | AIStor `platform-host-backups/gitea/` |
|
||||
| platform-postgres (Gitea·Keycloak·tech-log DB) | CloudNativePG base backup + WAL | 매일 03:00 KST | 30일 | AIStor `platform-postgres-backups/` (W1, `services/platform-postgres-backup`) |
|
||||
| AIStor 데이터(Loki·Tempo) | **백업하지 않는다** | — | — | 8절 결정 6: 잃어도 되는 관측 데이터. 디스크를 잃으면 로그·추적 기록만 사라진다 |
|
||||
| Prometheus·Grafana·Alertmanager PV | 백업하지 않는다 | — | — | 설정은 이 저장소가 정본이고, 지표는 다시 쌓인다 |
|
||||
|
||||
- Gitea DB는 dump에 없다(`--skip-db`). Gitea를 복원하려면 CNPG 복원이 먼저다(`docs/runbooks/gitea-restore.md`).
|
||||
- k3s 백업에는 `token`이 들어 있다. token과 datastore가 같이 있으면 클러스터의 모든 Secret을 풀 수 있다.
|
||||
Gitea dump의 `app.ini`에도 비밀 값이 있다. 그래서 둘 다 AIStor의 전용 사용자만 읽고 쓴다.
|
||||
- AIStor는 같은 desktop의 HDD다. SSD가 죽으면 남지만 desktop 자체를 잃으면 같이 잃는다.
|
||||
- k3s 백업을 AIStor 밖에도 두는 이유: AIStor는 바로 그 k3s 위에서 돈다. 클러스터를 통째로 잃으면 AIStor를
|
||||
다시 띄우기 전에는 AIStor 안의 사본을 꺼낼 수 없다. 로컬 사본은 sdb3를 mount하기만 하면 읽힌다
|
||||
(`docs/runbooks/k3s-datastore-restore.md`). 폴더는 AIStor 드라이브 폴더(`data/`) 옆이라 bucket으로 보이지 않는다.
|
||||
|
||||
## off-host 사본 (준비만, 기본 꺼짐)
|
||||
|
||||
위치는 소유자가 정한다(8절 결정 1). `/etc/platform-backup/aistor.env`에 두 줄을 넣으면 k3s datastore
|
||||
백업이 그 공개 키로 암호화한 사본을 `rclone`으로 보낸다.
|
||||
|
||||
```bash
|
||||
OFFHOST_GPG_RECIPIENT_FILE=/etc/platform-backup/offhost-recipient.asc # 공개 키. 개인 키는 desktop에 두지 않는다
|
||||
OFFHOST_RCLONE_TARGET=<rclone remote>:<path> # rclone 설치·설정 필요
|
||||
```
|
||||
|
||||
지표 `platform_backup_offhost_copy{backup="k3s-datastore"}`가 1이 된다. Gitea dump의 off-host 사본은 아직 없다.
|
||||
|
||||
## 점검
|
||||
|
||||
| 타이머 | 사용자 | 하는 일 | 지표 |
|
||||
| --- | --- | --- | --- |
|
||||
| `platform-k3s-cert-expiry` | donghyeon | 6443·10250 serving 인증서 notAfter | `platform_k3s_certificate_not_after_seconds` |
|
||||
| `platform-live-drift-check` | donghyeon | origin/main 렌더 → `kubectl diff`(서버 dry-run) | `platform_live_drift_objects` |
|
||||
|
||||
## 경보 (`prometheus-rules.yaml`)
|
||||
|
||||
| 경보 | 조건 |
|
||||
| --- | --- |
|
||||
| `PlatformK3sDatastoreBackupTooOld` | 마지막 성공이 2일보다 오래됨 |
|
||||
| `PlatformGiteaBackupTooOld` | 8일보다 오래됨 |
|
||||
| `PlatformHostBackupFailed` | 마지막 실행 실패 |
|
||||
| `PlatformHostBackupMetricsAbsent` | 지표가 하루 동안 없음(타이머나 textfile 수집기가 안 돎) |
|
||||
| `PlatformLiveDriftDetected` | drift가 6시간 넘게 남음 |
|
||||
| `PlatformLiveDriftCheckStale` | drift 검사가 2일 넘게 완료되지 않음 |
|
||||
| `PlatformK3sServingCertificateExpiry` | 100일 미만 warning, 14일 미만 critical |
|
||||
| `PlatformK3sCertificateCheckStale` | 인증서 점검이 2일 넘게 안 돎 |
|
||||
|
||||
`scripts/validate/test-platform-host-checks-alerts.sh`가 promtool 단위 시험으로 각 경보가 울려야 할 때 울리고
|
||||
아닐 때 안 울리는지 본다. 같은 PR에서 kube-prometheus-stack의 `KubeletClientCertificateExpiration`·
|
||||
`KubeletServerCertificateExpiration`을 껐다. k3s kubelet은 그 지표를 내지 않아 울릴 수 없었다.
|
||||
|
||||
## 적용 (소유자, 순서대로)
|
||||
|
||||
전제: node-exporter textfile 수집기(W1, `services/observability/node-exporter/values/home.yaml`의
|
||||
`--collector.textfile.directory=/host/root/var/lib/node_exporter/textfile_collector`)가 들어간 main checkout.
|
||||
|
||||
```bash
|
||||
cd /home/donghyeon/workspace/platform && git pull --ff-only
|
||||
sudo bash scripts/bootstrap/create-host-backup-credentials.sh # bucket·정책·사용자·/etc/platform-backup/aistor.env (APPLY 입력)
|
||||
bash scripts/bootstrap/install-host-checks.sh --check
|
||||
sudo bash scripts/bootstrap/install-host-checks.sh --execute # 타이머 설치·켜기·첫 실행
|
||||
kubectl apply -k services/platform-host-checks # 경보 규칙
|
||||
sudo bash scripts/validate/k3s-datastore-restore-drill.sh # 첫 백업으로 복원 리허설
|
||||
sudo bash scripts/validate/gitea-backup-restore-drill.sh
|
||||
```
|
||||
|
||||
`install-host-checks.sh --execute` 마지막에 `platform_backup_last_run_success{backup="k3s-datastore"} 1`과
|
||||
`{backup="gitea"} 1`이 보여야 한다. 리허설 둘은 각각 `K3S DATASTORE RESTORE DRILL PASS`, `GITEA RESTORE DRILL PASS`로 끝나야 한다.
|
||||
|
||||
## 복원 리허설
|
||||
|
||||
- k3s: 백업을 내려받아 sha256을 맞춘 뒤, 운영과 같은 버전의 k3s를 **네트워크 없는** 일회용 컨테이너로
|
||||
띄워 API가 뜨는지, Secret 수가 라이브와 비슷한지(±5), 모든 Secret이 복호화되는지 본다.
|
||||
`scripts/validate/test-k3s-datastore-restore-drill.sh`는 운영 데이터 없이 같은 경로를 시험한다(새 k3s에
|
||||
시험 Secret 셋 → 암호화돼 저장된 것을 확인 → 복원).
|
||||
- Gitea: dump를 풀어 `app.ini`(INTERNAL_TOKEN·SECRET_KEY), 모든 저장소의 `git fsck`, 라이브와 저장소 수를 본다.
|
||||
- 분기마다 한 번 돌린다. 성공하면 `platform_backup_restore_drill_last_success_timestamp_seconds`가 갱신된다.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::platform-host-backups"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::platform-host-backups/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- prometheus-rules.yaml
|
||||
@@ -0,0 +1,88 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-host-checks
|
||||
namespace: observability
|
||||
labels:
|
||||
observability.hyeonworks.com/instance: home
|
||||
app.kubernetes.io/part-of: platform
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-host-backups
|
||||
rules:
|
||||
- alert: PlatformK3sDatastoreBackupTooOld
|
||||
expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="k3s-datastore"}) > 2 * 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: k3s datastore has not been backed up for more than 2 days
|
||||
description: Check `journalctl -u platform-k3s-datastore-backup` on desktop. Daily timer, 03:30 KST.
|
||||
- alert: PlatformGiteaBackupTooOld
|
||||
expr: time() - max(platform_backup_last_success_timestamp_seconds{backup="gitea"}) > 8 * 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Gitea has not been backed up for more than 8 days
|
||||
description: Check `journalctl -u platform-gitea-backup` on desktop. Weekly timer, Sunday 04:00 KST.
|
||||
- alert: PlatformHostBackupFailed
|
||||
expr: max by (backup) (platform_backup_last_run_success) == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "The last {{ $labels.backup }} backup run failed"
|
||||
description: "Check `journalctl -u platform-{{ $labels.backup }}-backup` on desktop."
|
||||
- alert: PlatformHostBackupMetricsAbsent
|
||||
expr: absent(platform_backup_last_run_timestamp_seconds{backup="k3s-datastore"}) or absent(platform_backup_last_run_timestamp_seconds{backup="gitea"})
|
||||
for: 1d
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Host backup metrics are missing
|
||||
description: The backup timers or the node-exporter textfile collector (/var/lib/node_exporter/textfile_collector) are not running.
|
||||
- name: platform-host-drift
|
||||
rules:
|
||||
- alert: PlatformLiveDriftDetected
|
||||
expr: max(platform_live_drift_objects) > 0
|
||||
for: 6h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $value }} live objects differ from platform-core origin/main"
|
||||
description: Run `scripts/validate/live-drift-check.sh --accepted scripts/validate/live-drift-accepted.txt` on desktop to list them.
|
||||
- alert: PlatformLiveDriftCheckStale
|
||||
expr: time() - max(platform_live_drift_last_run_timestamp_seconds) > 2 * 86400 or max(platform_live_drift_check_success) == 0
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: The daily live drift check has not completed
|
||||
description: Check `journalctl -u platform-live-drift-check` on desktop.
|
||||
- name: platform-k3s-certificates
|
||||
rules:
|
||||
- alert: PlatformK3sServingCertificateExpiry
|
||||
expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 100 * 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 100 days"
|
||||
description: k3s renews certificates only when it starts within 120 days of expiry. Plan a k3s restart window (docs/runbooks).
|
||||
- alert: PlatformK3sServingCertificateExpiry
|
||||
expr: (min by (certificate) (platform_k3s_certificate_not_after_seconds) - time()) < 14 * 86400
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "k3s {{ $labels.certificate }} serving certificate expires in less than 14 days"
|
||||
description: Restart k3s (it renews certificates within 120 days of expiry) or run `k3s certificate rotate`.
|
||||
- alert: PlatformK3sCertificateCheckStale
|
||||
expr: time() - max(platform_k3s_certificate_check_timestamp_seconds) > 2 * 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: The daily k3s certificate expiry check has not run
|
||||
description: Check `journalctl -u platform-k3s-cert-expiry` on desktop.
|
||||
@@ -0,0 +1,20 @@
|
||||
# 플랫폼 PriorityClass
|
||||
|
||||
노드가 한 대라 자원이 모자라면 kubelet이 파드를 내보낸다. 내보낼 순서는 requests를 넘었는지,
|
||||
그다음 우선순위로 정한다. 2026-11-09 디스크 축출 때 플랫폼 파드는 모두 우선순위 0이라 tech-log
|
||||
운영 파드가 Grafana·kube-state-metrics와 같은 순위로 나갔다.
|
||||
|
||||
| 이름 | 값 | 선점 | 붙일 곳 |
|
||||
| --- | --- | --- | --- |
|
||||
| `platform-critical` | 1000000 | 낮은 것을 선점 | platform-postgres, Gitea, Keycloak, tech-log |
|
||||
| `platform-low` | -1000 | 선점 안 함 | Grafana, pgAdmin, Loki, Tempo |
|
||||
|
||||
- 아무것도 안 붙인 파드는 기본값 0이다. 시스템 클래스(`system-*`, 20억)는 그대로 가장 높다.
|
||||
- `globalDefault`는 둘 다 끈다. 새 파드가 모르게 우선순위를 얻지 않게 하려는 것이다.
|
||||
- 클래스를 만드는 것은 아무 파드도 바꾸지 않는다. **붙이는 것은 파드 재시작을 부르므로**
|
||||
k3s 재시작 창에 한다(`infrastructure/host/k3s/README.md`).
|
||||
|
||||
```sh
|
||||
kubectl apply -k services/priority-classes
|
||||
kubectl get priorityclass platform-critical platform-low
|
||||
```
|
||||
@@ -0,0 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- priority-classes.yaml
|
||||
@@ -0,0 +1,25 @@
|
||||
apiVersion: scheduling.k8s.io/v1
|
||||
kind: PriorityClass
|
||||
metadata:
|
||||
name: platform-critical
|
||||
labels:
|
||||
app.kubernetes.io/part-of: platform
|
||||
value: 1000000
|
||||
globalDefault: false
|
||||
preemptionPolicy: PreemptLowerPriority
|
||||
description: >-
|
||||
Stateful platform services and production workloads that other services depend on
|
||||
(platform-postgres, Gitea, Keycloak, tech-log). Evicted after everything else.
|
||||
---
|
||||
apiVersion: scheduling.k8s.io/v1
|
||||
kind: PriorityClass
|
||||
metadata:
|
||||
name: platform-low
|
||||
labels:
|
||||
app.kubernetes.io/part-of: platform
|
||||
value: -1000
|
||||
globalDefault: false
|
||||
preemptionPolicy: Never
|
||||
description: >-
|
||||
Tools whose loss does not stop a user-facing service (Grafana, pgAdmin, Loki, Tempo).
|
||||
Evicted first and never preempts another pod.
|
||||
@@ -0,0 +1,44 @@
|
||||
# tech-log 테넌트 — 플랫폼 쪽 객체
|
||||
|
||||
tech-log(hyeonworks.com 운영)가 플랫폼을 쓰려면 플랫폼 namespace 안에 허용 객체가 있어야 한다.
|
||||
이 빌드 루트가 그 객체를 소유한다. 앱 자체(Deployment·Service·Ingress·ServiceMonitor·앱
|
||||
namespace 안의 NetworkPolicy)는 tech-log 저장소의 `deploy/`가 소유한다.
|
||||
|
||||
2026-11-11 점검(`docs/reviews/2026-11-11-k3s-maturity-review.md` 4.3절) 전까지 이 객체들은
|
||||
`kubectl apply`로만 들어가 있었고 어느 저장소에도 없었다. 처음 내용은 라이브 객체의
|
||||
`last-applied-configuration`을 그대로 옮겼다. 라이브와 다른 것은 `hyeonworks-prod`의 PSA 라벨뿐이다.
|
||||
|
||||
| 파일 | 객체 |
|
||||
| --- | --- |
|
||||
| `namespace.yaml` | Namespace `hyeonworks-prod`: 관측 대상 라벨, PSA `enforce=baseline`·`warn=restricted`·`audit=restricted` |
|
||||
| `platform-data.yaml` | `techlog` Database와 DatabaseRole(로그인만, 연결 30), Postgres로 들어오는 ingress |
|
||||
| `object-storage.yaml` | AIStor 9000으로 들어오는 ingress |
|
||||
| `observability-probes.yaml` | blackbox Probe 둘(공개 HTTPS, 내부 readiness) |
|
||||
| `observability-rules.yaml` | PrometheusRule `techlog` 일곱 개 |
|
||||
| `dashboards/techlog-operations.json` | Grafana 대시보드(`grafana-dashboard-techlog` ConfigMap으로 생성) |
|
||||
| `observability-network-policies.yaml` | blackbox → readiness, Prometheus → management scrape egress |
|
||||
|
||||
## 여기에 없는 것
|
||||
|
||||
- `platform-data/platform-postgres`의 `pg_hba` techlog 두 줄: Cluster 객체는
|
||||
`services/platform-postgres`가 소유한다.
|
||||
- Secret: `platform-data/techlog-db-credentials`(`kubernetes.io/basic-auth`, `username`은 정확히
|
||||
`techlog`)는 DatabaseRole이 참조한다. 앱 쪽 `hyeonworks-prod/hyeonworks-secrets`는 tech-log가
|
||||
소유한다. 둘 다 Git에 두지 않는다.
|
||||
|
||||
## PSA
|
||||
|
||||
`enforce=baseline`은 2026-11-12에 서버 dry-run(`kubectl label --dry-run=server`)으로 위반 0을
|
||||
확인하고 붙였다. `restricted`로 강제하면 지금 파드 셋(redis, backend, frontend)이 모두 걸린다
|
||||
(`allowPrivilegeEscalation`, capabilities, `runAsNonRoot`, seccomp). tech-log 쪽 securityContext가
|
||||
고쳐진 뒤 같은 dry-run이 경고 없이 통과하면 `enforce`를 `restricted`로 올린다.
|
||||
|
||||
## 렌더·적용
|
||||
|
||||
```sh
|
||||
bash scripts/validate/render-tenants-and-policies.sh
|
||||
kubectl kustomize services/tenants/techlog | kubectl diff -f -
|
||||
kubectl apply -k services/tenants/techlog
|
||||
```
|
||||
|
||||
새로 설치할 때는 이 루트를 tech-log `deploy/`보다 먼저 적용한다(namespace를 여기서 만든다).
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,23 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
# tech-log(hyeonworks.com) 테넌트의 플랫폼 쪽 객체. 앱 쪽 매니페스트는 tech-log repo가 소유한다.
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- platform-data.yaml
|
||||
- object-storage.yaml
|
||||
- observability-probes.yaml
|
||||
- observability-rules.yaml
|
||||
- observability-network-policies.yaml
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
labels:
|
||||
app.kubernetes.io/part-of: techlog
|
||||
grafana_dashboard: "1"
|
||||
|
||||
configMapGenerator:
|
||||
- name: grafana-dashboard-techlog
|
||||
namespace: observability
|
||||
files:
|
||||
- dashboards/techlog-operations.json
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: hyeonworks-prod
|
||||
labels:
|
||||
observability.hyeonworks.com/enabled: 'true'
|
||||
pod-security.kubernetes.io/enforce: baseline
|
||||
pod-security.kubernetes.io/enforce-version: latest
|
||||
pod-security.kubernetes.io/warn: restricted
|
||||
pod-security.kubernetes.io/warn-version: latest
|
||||
pod-security.kubernetes.io/audit: restricted
|
||||
pod-security.kubernetes.io/audit-version: latest
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-techlog-backend
|
||||
namespace: object-storage
|
||||
spec:
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hyeonworks-prod
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: techlog-backend
|
||||
ports:
|
||||
- port: 9000
|
||||
protocol: TCP
|
||||
podSelector:
|
||||
matchLabels:
|
||||
aistor.min.io/objectStore: minio-aistor
|
||||
policyTypes:
|
||||
- Ingress
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: blackbox-allow-techlog-health
|
||||
namespace: observability
|
||||
spec:
|
||||
egress:
|
||||
- ports:
|
||||
- port: 9001
|
||||
protocol: TCP
|
||||
to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hyeonworks-prod
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: techlog-backend
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus-blackbox-exporter
|
||||
policyTypes:
|
||||
- Egress
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: observability-allow-techlog-scrape
|
||||
namespace: observability
|
||||
spec:
|
||||
egress:
|
||||
- ports:
|
||||
- port: 9001
|
||||
protocol: TCP
|
||||
to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hyeonworks-prod
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: techlog-backend
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
policyTypes:
|
||||
- Egress
|
||||
@@ -0,0 +1,46 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
labels:
|
||||
observability.hyeonworks.com/instance: home
|
||||
name: techlog-edge
|
||||
namespace: observability
|
||||
spec:
|
||||
interval: 30s
|
||||
jobName: techlog-edge
|
||||
module: http_2xx
|
||||
prober:
|
||||
path: /probe
|
||||
scheme: http
|
||||
url: blackbox-exporter.observability.svc.cluster.local:9115
|
||||
scrapeTimeout: 15s
|
||||
targets:
|
||||
staticConfig:
|
||||
labels:
|
||||
application: techlog
|
||||
static:
|
||||
- https://hyeonworks.com/
|
||||
- https://hyeonworks.com/api/healthcheck
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
labels:
|
||||
observability.hyeonworks.com/instance: home
|
||||
name: techlog-readiness
|
||||
namespace: observability
|
||||
spec:
|
||||
interval: 30s
|
||||
jobName: techlog-readiness
|
||||
module: http_2xx
|
||||
prober:
|
||||
path: /probe
|
||||
scheme: http
|
||||
url: blackbox-exporter.observability.svc.cluster.local:9115
|
||||
scrapeTimeout: 15s
|
||||
targets:
|
||||
staticConfig:
|
||||
labels:
|
||||
application: techlog
|
||||
static:
|
||||
- http://techlog-management.hyeonworks-prod.svc.cluster.local:9001/actuator/health/readiness
|
||||
@@ -0,0 +1,67 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
labels:
|
||||
observability.hyeonworks.com/instance: home
|
||||
name: techlog
|
||||
namespace: observability
|
||||
spec:
|
||||
groups:
|
||||
- name: techlog
|
||||
rules:
|
||||
- alert: TechLogMetricsDown
|
||||
annotations:
|
||||
summary: TechLog metrics target is unreachable
|
||||
expr: up{namespace="hyeonworks-prod",service="techlog-management"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: warning
|
||||
- alert: TechLogReadinessDown
|
||||
annotations:
|
||||
summary: TechLog dependency readiness is DOWN
|
||||
expr: probe_success{job="techlog-readiness"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: critical
|
||||
- alert: TechLogHTTPSDown
|
||||
annotations:
|
||||
summary: TechLog HTTPS or API health is failing
|
||||
expr: probe_success{job="techlog-edge"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: critical
|
||||
- alert: TechLogHTTP5xx
|
||||
annotations:
|
||||
summary: TechLog HTTP 5xx exceeds 5%
|
||||
expr: (sum(rate(http_server_requests_seconds_count{namespace="hyeonworks-prod",status=~"5.."}[5m])) / clamp_min(sum(rate(http_server_requests_seconds_count{namespace="hyeonworks-prod"}[5m])),0.001)) > 0.05 and sum(increase(http_server_requests_seconds_count{namespace="hyeonworks-prod"}[5m])) > 20
|
||||
for: 5m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: warning
|
||||
- alert: TechLogLatencyHigh
|
||||
annotations:
|
||||
summary: TechLog p99 latency exceeds 2 seconds
|
||||
expr: histogram_quantile(0.99,sum by(le)(rate(http_server_requests_seconds_bucket{namespace="hyeonworks-prod",uri!~"/actuator.*"}[5m]))) > 2
|
||||
for: 10m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: warning
|
||||
- alert: TechLogHeapPressure
|
||||
annotations:
|
||||
summary: TechLog heap usage exceeds 85%
|
||||
expr: sum(jvm_memory_used_bytes{namespace="hyeonworks-prod",area="heap"}) / sum(jvm_memory_max_bytes{namespace="hyeonworks-prod",area="heap"}) > 0.85
|
||||
for: 10m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: warning
|
||||
- alert: TechLogDatabasePoolWaiting
|
||||
annotations:
|
||||
summary: TechLog database connections are queued
|
||||
expr: hikaricp_connections_pending{namespace="hyeonworks-prod"} > 0
|
||||
for: 5m
|
||||
labels:
|
||||
application: techlog
|
||||
severity: warning
|
||||
@@ -0,0 +1,55 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Database
|
||||
metadata:
|
||||
name: platform-postgres-techlog
|
||||
namespace: platform-data
|
||||
spec:
|
||||
cluster:
|
||||
name: platform-postgres
|
||||
databaseReclaimPolicy: retain
|
||||
ensure: present
|
||||
name: techlog
|
||||
owner: techlog
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: DatabaseRole
|
||||
metadata:
|
||||
name: platform-postgres-techlog
|
||||
namespace: platform-data
|
||||
spec:
|
||||
bypassrls: false
|
||||
cluster:
|
||||
name: platform-postgres
|
||||
connectionLimit: 30
|
||||
createdb: false
|
||||
createrole: false
|
||||
databaseRoleReclaimPolicy: retain
|
||||
login: true
|
||||
name: techlog
|
||||
passwordSecret:
|
||||
name: techlog-db-credentials
|
||||
replication: false
|
||||
superuser: false
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-techlog-backend
|
||||
namespace: platform-data
|
||||
spec:
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: hyeonworks-prod
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: techlog-backend
|
||||
ports:
|
||||
- port: 5432
|
||||
protocol: TCP
|
||||
podSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: platform-postgres
|
||||
policyTypes:
|
||||
- Ingress
|
||||
Reference in New Issue
Block a user