Files
platform-core/scripts/validate/live-drift-check.sh
T
DongHyeonka 3eafb7e1d8 feat(validate): add a live drift check against the repository renders
The 2026-11-11 review found a pg_hba rule that existed only in the
cluster by rendering every root by hand and running kubectl diff.
Nothing does that regularly, so drift is found by accident.

live-drift-check.sh renders all live roots (Phase 1 and 2, admin
services, observability core, the Traefik trust overlay, the tenant,
PriorityClass and keycloak policy roots), strips Secrets and runs
kubectl diff, which is a server-side dry run. It prints one line per
drifted object, honours a dated accepted-drift list and can write
Prometheus textfile metrics for a later alert.

Risk: read-only. It consumes the render scripts' verified-output-dir
handoff and deletes those directories on exit; the README notes this
second consumer. Secret values are not compared.

Verification (desktop, before this series is applied): 17 drifted
objects, matching the review plus the new roots; exit 1; the textfile
holds platform_live_drift_objects 17.
2026-11-12 00:29:18 +09:00

174 lines
6.9 KiB
Bash

#!/usr/bin/env bash
# Render every live build root of this repository and compare it with the
# cluster using `kubectl diff` (server-side dry-run: nothing is persisted).
#
# Usage:
# PLATFORM_HELM_BIN=/abs/helm bash scripts/validate/live-drift-check.sh \
# [--accepted FILE] [--textfile FILE]
#
# --accepted FILE known differences, one per line:
# `<Kind> <namespace or -> <name> <until YYYY-MM-DD> <reason...>`
# Expired entries count as drift again.
# --textfile FILE also write Prometheus textfile metrics (atomic rename).
#
# Exit status: 0 no drift, 1 drift found, 2 the check itself failed.
# Secrets are removed from the renders before diffing; Secret drift is out of scope.
set -Eeuo pipefail
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
accepted_file=""
textfile=""
while (( $# > 0 )); do
case "$1" in
--accepted) accepted_file="${2:?--accepted needs a file}"; shift 2 ;;
--textfile) textfile="${2:?--textfile needs a file}"; shift 2 ;;
-h|--help) sed -n '2,16p' "${BASH_SOURCE[0]}"; exit 0 ;;
*) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;;
esac
done
declare -a made_dirs=()
cleanup() {
local d
for d in "${made_dirs[@]}"; do
case "$d" in /tmp/platform-*) rm -rf -- "$d" ;; esac
done
}
trap cleanup EXIT
write_textfile() {
local drift="$1" ok="$2"
[[ -n "$textfile" ]] || return 0
local tmp="${textfile}.tmp.$$"
{
echo '# HELP platform_live_drift_objects Objects whose live state differs from the repository render.'
echo '# TYPE platform_live_drift_objects gauge'
echo "platform_live_drift_objects ${drift}"
echo '# HELP platform_live_drift_check_success 1 when the drift check itself completed.'
echo '# TYPE platform_live_drift_check_success gauge'
echo "platform_live_drift_check_success ${ok}"
echo '# HELP platform_live_drift_last_run_timestamp_seconds Unix time the drift check finished.'
echo '# TYPE platform_live_drift_last_run_timestamp_seconds gauge'
echo "platform_live_drift_last_run_timestamp_seconds $(date +%s)"
} >"$tmp"
mv -f -- "$tmp" "$textfile"
}
die() {
printf 'DRIFT CHECK ERROR: %s\n' "$*" >&2
write_textfile 0 0
exit 2
}
work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work")
mkdir -p "$work/in" "$work/out"
render_root() {
local script="$1" prefix="$2" dir
dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir")
bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \
>"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; }
printf '%s\n' "$dir"
}
phase1="$(render_root render-phase1.sh platform-phase1-apply)"
phase2="$(render_root render-phase2.sh platform-phase2-apply)"
admin="$(render_root render-admin-services.sh platform-admin-apply)"
observability="$(render_root render-observability-core.sh platform-observability-core-apply)"
# The live Gitea is the OIDC profile; gitea.yaml is the fresh-install baseline only.
declare -a inputs=(
"$phase1/namespaces.yaml" "$phase1/ssd-local-pv.yaml" "$phase1/cnpg-operator.yaml"
"$phase1/platform-postgres.yaml" "$phase1/gitea-oidc.yaml"
"$observability/aggregate.yaml"
)
for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done
# Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md).
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed"
kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed"
kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed"
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed"
inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml")
n=0
for f in "${inputs[@]}"; do
n=$((n + 1))
python3 -I - "$f" "$work/in/$(printf '%02d' "$n")-$(basename "$f")" <<'PY' || die "could not strip Secrets from $f"
import sys, yaml
yaml.SafeLoader.add_constructor("tag:yaml.org,2002:value", lambda l, n: l.construct_scalar(n))
docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if isinstance(d, dict) and d.get("kind") not in (None, "Secret")]
yaml.safe_dump_all(docs, open(sys.argv[2], "w"), sort_keys=False)
PY
done
for f in "$work"/in/*.yaml; do
set +e
KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff -f "$f" >"$work/out/$(basename "$f").diff" 2>"$work/out/$(basename "$f").err"
rc=$?
set -e
if (( rc > 1 )); then
sed -n '1,5p' "$work/out/$(basename "$f").err" >&2
die "kubectl diff failed for $(basename "$f")"
fi
done
set +e
python3 -I - "$work/out" "${accepted_file:-}" "$work/drift-count" <<'PY'
import sys, pathlib, re, datetime
out, accepted_path, count_path = pathlib.Path(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3])
today = datetime.date.today()
accepted, expired = {}, set()
if accepted_path:
for line in pathlib.Path(accepted_path).read_text().splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
kind, ns, name, until, *reason = line.split()
key = (kind, "" if ns == "-" else ns, name)
if datetime.date.fromisoformat(until) < today:
expired.add(key)
else:
accepted[key] = " ".join(reason)
version = re.compile(r"^v\d+((alpha|beta)\d+)?$")
objects = {}
for diff in sorted(out.glob("*.diff")):
current = None
for line in diff.read_text().splitlines():
if line.startswith("diff -u -N "):
parts = line.split()[3].rsplit("/", 1)[-1].split(".")
i = next(i for i, p in enumerate(parts) if version.match(p))
current = (parts[i + 1], parts[i + 2], ".".join(parts[i + 3:]))
objects.setdefault(current, [0, 0, diff.name])
elif current and line[:1] in "+-" and not line.startswith(("+++", "---")):
if re.match(r"^[+-]\s+generation:", line):
continue
objects[current][0 if line[0] == "+" else 1] += 1
drift = accepted_seen = 0
for key, (plus, minus, src) in sorted(objects.items()):
if plus == 0 and minus == 0:
continue
kind, ns, name = key
label = f"{kind} {ns or '-'}/{name} (+{plus} -{minus}) [{src}]"
if key in accepted:
accepted_seen += 1
print(f"ACCEPTED {label}: {accepted[key]}")
continue
drift += 1
print(f"DRIFT {label}" + (" (accepted entry expired)" if key in expired else ""))
print(f"SUMMARY drift={drift} accepted={accepted_seen}")
count_path.write_text(str(drift))
PY
rc=$?
set -e
(( rc == 0 )) || die "could not summarize the diffs"
drift_count="$(cat "$work/drift-count")"
write_textfile "$drift_count" 1
if (( drift_count == 0 )); then
echo "LIVE DRIFT CHECK PASS"
exit 0
fi
echo "LIVE DRIFT CHECK FOUND DRIFT"
exit 1