diff --git a/scripts/validate/README.md b/scripts/validate/README.md index 6c6df6e..b30e14a 100644 --- a/scripts/validate/README.md +++ b/scripts/validate/README.md @@ -559,3 +559,40 @@ parser와 세 mode의 fixture 회귀는 live cluster에 연결하지 않고 다 ```sh bash scripts/validate/test-k3s-secret-encryption-status.sh ``` + +## 테넌트·우선순위·keycloak 정책 렌더 검사 + +`render-tenants-and-policies.sh`는 클러스터에 닿지 않고 세 빌드 루트를 렌더해 불변식을 +확인한다. + +- `services/tenants/techlog`: tech-log 테넌트의 플랫폼 쪽 객체 11개(목록이 정확히 같아야 + 한다), `hyeonworks-prod` PSA 라벨, techlog DatabaseRole 권한 플래그, 대시보드 이름 +- `services/priority-classes`: `platform-critical`(0보다 크고 시스템 클래스보다 작다), + `platform-low`(0보다 작고 선점하지 않는다), 둘 다 global default가 아니다 +- `infrastructure/networking/keycloak`: namespace 전체 기본 차단, egress 규칙마다 포트, + 허용 CIDR은 API 서버 둘뿐 + +```sh +bash scripts/validate/render-tenants-and-policies.sh +``` + +## 라이브 drift 검사 + +`live-drift-check.sh`는 이 저장소의 라이브 빌드 루트를 모두 렌더하고(Phase 1·2, 관리 도구, +관측 core, Traefik trust overlay, 위 세 루트) Secret을 뺀 뒤 `kubectl diff`(서버 dry-run, +저장하지 않음)로 클러스터와 비교한다. 렌더에는 위 render 스크립트의 `--verified-output-dir` +출력을 읽기만 하는 용도로 쓴다(그 디렉터리는 스크립트가 끝날 때 지운다). + +```sh +PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \ + bash scripts/validate/live-drift-check.sh \ + --accepted scripts/validate/live-drift-accepted.txt +``` + +- 종료 코드: 0 차이 없음, 1 차이 있음, 2 검사 자체 실패. +- `DRIFT / (+추가 -삭제) [입력 파일]`로 객체마다 한 줄을 쓴다. + `generation` 줄만 다른 것은 세지 않는다. +- `--accepted`: 알고 있는 차이를 기한과 이유를 달아 적는다. 기한이 지나면 다시 drift로 센다. +- `--textfile FILE`: `platform_live_drift_objects`, `platform_live_drift_check_success`, + `platform_live_drift_last_run_timestamp_seconds`를 Prometheus textfile 형식으로 쓴다. +- Secret 값의 차이는 보지 않는다. diff --git a/scripts/validate/live-drift-accepted.txt b/scripts/validate/live-drift-accepted.txt new file mode 100644 index 0000000..762302b --- /dev/null +++ b/scripts/validate/live-drift-accepted.txt @@ -0,0 +1,10 @@ +# 알고 있는 라이브 drift. 형식: <기한 YYYY-MM-DD> <이유> +# 기한이 지나면 live-drift-check.sh가 다시 drift로 센다. 해소하면 줄을 지운다. +Cluster platform-data platform-postgres 2026-11-30 pg_hba의 techlog 두 줄이 repo에 없다. w1이 CNPG 백업과 함께 넣는다(1c4ef9a). +DatabaseRole platform-data platform-postgres-gitea 2026-11-30 렌더가 false 기본값 필드를 명시할 뿐 값은 같다. 다음 Phase 1 적용 때 사라진다. +DatabaseRole platform-data platform-postgres-keycloak 2026-11-30 렌더가 false 기본값 필드를 명시할 뿐 값은 같다. 다음 Phase 2 적용 때 사라진다. +Deployment keycloak keycloak-operator 2026-11-30 원격 ref 26.7.0 태그 내용이 바뀌어 build-timestamp 라벨만 다르다. 재시작 창에 커밋 SHA 고정과 함께 적용한다. +Service keycloak keycloak-operator 2026-11-30 위와 같은 build-timestamp 라벨. +ServiceAccount keycloak keycloak-operator 2026-11-30 위와 같은 build-timestamp 라벨. +Deployment observability observability-core-kube-pr-operator 2026-11-30 이미지 문자열이 tag@digest와 digest로 다르고 digest는 같다. 적용하면 재시작되므로 재시작 창에. +Deployment observability observability-core-kube-state-metrics 2026-11-30 위와 같은 이미지 문자열 차이. diff --git a/scripts/validate/live-drift-check.sh b/scripts/validate/live-drift-check.sh new file mode 100644 index 0000000..4e85a2d --- /dev/null +++ b/scripts/validate/live-drift-check.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# Render every live build root of this repository and compare it with the +# cluster using `kubectl diff` (server-side dry-run: nothing is persisted). +# +# Usage: +# PLATFORM_HELM_BIN=/abs/helm bash scripts/validate/live-drift-check.sh \ +# [--accepted FILE] [--textfile FILE] +# +# --accepted FILE known differences, one per line: +# ` ` +# Expired entries count as drift again. +# --textfile FILE also write Prometheus textfile metrics (atomic rename). +# +# Exit status: 0 no drift, 1 drift found, 2 the check itself failed. +# Secrets are removed from the renders before diffing; Secret drift is out of scope. +set -Eeuo pipefail + +readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +accepted_file="" +textfile="" +while (( $# > 0 )); do + case "$1" in + --accepted) accepted_file="${2:?--accepted needs a file}"; shift 2 ;; + --textfile) textfile="${2:?--textfile needs a file}"; shift 2 ;; + -h|--help) sed -n '2,16p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;; + esac +done + + +declare -a made_dirs=() +cleanup() { + local d + for d in "${made_dirs[@]}"; do + case "$d" in /tmp/platform-*) rm -rf -- "$d" ;; esac + done +} +trap cleanup EXIT + +write_textfile() { + local drift="$1" ok="$2" + [[ -n "$textfile" ]] || return 0 + local tmp="${textfile}.tmp.$$" + { + echo '# HELP platform_live_drift_objects Objects whose live state differs from the repository render.' + echo '# TYPE platform_live_drift_objects gauge' + echo "platform_live_drift_objects ${drift}" + echo '# HELP platform_live_drift_check_success 1 when the drift check itself completed.' + echo '# TYPE platform_live_drift_check_success gauge' + echo "platform_live_drift_check_success ${ok}" + echo '# HELP platform_live_drift_last_run_timestamp_seconds Unix time the drift check finished.' + echo '# TYPE platform_live_drift_last_run_timestamp_seconds gauge' + echo "platform_live_drift_last_run_timestamp_seconds $(date +%s)" + } >"$tmp" + mv -f -- "$tmp" "$textfile" +} + +die() { + printf 'DRIFT CHECK ERROR: %s\n' "$*" >&2 + write_textfile 0 0 + exit 2 +} + +work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work") +mkdir -p "$work/in" "$work/out" + +render_root() { + local script="$1" prefix="$2" dir + dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir") + bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \ + >"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; } + printf '%s\n' "$dir" +} + +phase1="$(render_root render-phase1.sh platform-phase1-apply)" +phase2="$(render_root render-phase2.sh platform-phase2-apply)" +admin="$(render_root render-admin-services.sh platform-admin-apply)" +observability="$(render_root render-observability-core.sh platform-observability-core-apply)" + +# The live Gitea is the OIDC profile; gitea.yaml is the fresh-install baseline only. +declare -a inputs=( + "$phase1/namespaces.yaml" "$phase1/ssd-local-pv.yaml" "$phase1/cnpg-operator.yaml" + "$phase1/platform-postgres.yaml" "$phase1/gitea-oidc.yaml" + "$observability/aggregate.yaml" +) +for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done + +# Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md). +kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed" +kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed" +kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed" +kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed" +inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml") + +n=0 +for f in "${inputs[@]}"; do + n=$((n + 1)) + python3 -I - "$f" "$work/in/$(printf '%02d' "$n")-$(basename "$f")" <<'PY' || die "could not strip Secrets from $f" +import sys, yaml +yaml.SafeLoader.add_constructor("tag:yaml.org,2002:value", lambda l, n: l.construct_scalar(n)) +docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if isinstance(d, dict) and d.get("kind") not in (None, "Secret")] +yaml.safe_dump_all(docs, open(sys.argv[2], "w"), sort_keys=False) +PY +done + +for f in "$work"/in/*.yaml; do + set +e + KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff -f "$f" >"$work/out/$(basename "$f").diff" 2>"$work/out/$(basename "$f").err" + rc=$? + set -e + if (( rc > 1 )); then + sed -n '1,5p' "$work/out/$(basename "$f").err" >&2 + die "kubectl diff failed for $(basename "$f")" + fi +done + +set +e +python3 -I - "$work/out" "${accepted_file:-}" "$work/drift-count" <<'PY' +import sys, pathlib, re, datetime +out, accepted_path, count_path = pathlib.Path(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3]) +today = datetime.date.today() +accepted, expired = {}, set() +if accepted_path: + for line in pathlib.Path(accepted_path).read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + kind, ns, name, until, *reason = line.split() + key = (kind, "" if ns == "-" else ns, name) + if datetime.date.fromisoformat(until) < today: + expired.add(key) + else: + accepted[key] = " ".join(reason) +version = re.compile(r"^v\d+((alpha|beta)\d+)?$") +objects = {} +for diff in sorted(out.glob("*.diff")): + current = None + for line in diff.read_text().splitlines(): + if line.startswith("diff -u -N "): + parts = line.split()[3].rsplit("/", 1)[-1].split(".") + i = next(i for i, p in enumerate(parts) if version.match(p)) + current = (parts[i + 1], parts[i + 2], ".".join(parts[i + 3:])) + objects.setdefault(current, [0, 0, diff.name]) + elif current and line[:1] in "+-" and not line.startswith(("+++", "---")): + if re.match(r"^[+-]\s+generation:", line): + continue + objects[current][0 if line[0] == "+" else 1] += 1 +drift = accepted_seen = 0 +for key, (plus, minus, src) in sorted(objects.items()): + if plus == 0 and minus == 0: + continue + kind, ns, name = key + label = f"{kind} {ns or '-'}/{name} (+{plus} -{minus}) [{src}]" + if key in accepted: + accepted_seen += 1 + print(f"ACCEPTED {label}: {accepted[key]}") + continue + drift += 1 + print(f"DRIFT {label}" + (" (accepted entry expired)" if key in expired else "")) +print(f"SUMMARY drift={drift} accepted={accepted_seen}") +count_path.write_text(str(drift)) +PY +rc=$? +set -e +(( rc == 0 )) || die "could not summarize the diffs" +drift_count="$(cat "$work/drift-count")" +write_textfile "$drift_count" 1 +if (( drift_count == 0 )); then + echo "LIVE DRIFT CHECK PASS" + exit 0 +fi +echo "LIVE DRIFT CHECK FOUND DRIFT" +exit 1