#!/usr/bin/env bash # Render every live build root of this repository and compare it with the # cluster using `kubectl diff` (server-side dry-run: nothing is persisted). # # Usage: # PLATFORM_HELM_BIN=/abs/helm bash scripts/validate/live-drift-check.sh \ # [--accepted FILE] [--textfile FILE] # # --accepted FILE known differences, one per line: # ` ` # Expired entries count as drift again. # --textfile FILE also write Prometheus textfile metrics (atomic rename). # # Exit status: 0 no drift, 1 drift found, 2 the check itself failed. # Secrets are removed from the renders before diffing; Secret drift is out of scope. set -Eeuo pipefail readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" accepted_file="" textfile="" while (( $# > 0 )); do case "$1" in --accepted) accepted_file="${2:?--accepted needs a file}"; shift 2 ;; --textfile) textfile="${2:?--textfile needs a file}"; shift 2 ;; -h|--help) sed -n '2,16p' "${BASH_SOURCE[0]}"; exit 0 ;; *) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;; esac done declare -a made_dirs=() cleanup() { local d for d in "${made_dirs[@]}"; do case "$d" in /tmp/platform-*) rm -rf -- "$d" ;; esac done } trap cleanup EXIT write_textfile() { local drift="$1" ok="$2" [[ -n "$textfile" ]] || return 0 local tmp="${textfile}.tmp.$$" { echo '# HELP platform_live_drift_objects Objects whose live state differs from the repository render.' echo '# TYPE platform_live_drift_objects gauge' echo "platform_live_drift_objects ${drift}" echo '# HELP platform_live_drift_check_success 1 when the drift check itself completed.' echo '# TYPE platform_live_drift_check_success gauge' echo "platform_live_drift_check_success ${ok}" echo '# HELP platform_live_drift_last_run_timestamp_seconds Unix time the drift check finished.' echo '# TYPE platform_live_drift_last_run_timestamp_seconds gauge' echo "platform_live_drift_last_run_timestamp_seconds $(date +%s)" } >"$tmp" mv -f -- "$tmp" "$textfile" } die() { printf 'DRIFT CHECK ERROR: %s\n' "$*" >&2 write_textfile 0 0 exit 2 } work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work") mkdir -p "$work/in" "$work/out" # Render from a clean copy when run inside a git working tree. The observability # renderer refuses to run when an ignored chart cache (.helm/) is left over from an # earlier render, which happens in long-lived checkouts. The copy keeps tracked and # untracked-but-not-ignored files, so local edits are still checked. SOURCE_ROOT="$REPOSITORY_ROOT" if git -C "$REPOSITORY_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then SOURCE_ROOT="$work/src" mkdir -p "$SOURCE_ROOT" python3 -I - "$REPOSITORY_ROOT" <<'PY' | tar -C "$REPOSITORY_ROOT" --null -T - -cf - | tar -x -C "$SOURCE_ROOT" || die "could not copy the working tree" import subprocess, sys root = sys.argv[1] def ls(*args): out = subprocess.run(["git", "-C", root, "ls-files", "-z", *args], check=True, capture_output=True).stdout return [p for p in out.decode().split("\0") if p] keep = set(ls("--cached", "--others", "--exclude-standard")) - set(ls("--deleted")) sys.stdout.write("\0".join(sorted(keep)) + "\0") PY fi render_root() { local script="$1" prefix="$2" dir dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir") bash "$SOURCE_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \ >"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; } printf '%s\n' "$dir" } phase1="$(render_root render-phase1.sh platform-phase1-apply)" phase2="$(render_root render-phase2.sh platform-phase2-apply)" admin="$(render_root render-admin-services.sh platform-admin-apply)" observability="$(render_root render-observability-core.sh platform-observability-core-apply)" # The live Gitea is the OIDC profile; gitea.yaml is the fresh-install baseline only. declare -a inputs=( "$phase1/namespaces.yaml" "$phase1/ssd-local-pv.yaml" "$phase1/cnpg-operator.yaml" "$phase1/platform-postgres.yaml" "$phase1/gitea-oidc.yaml" "$observability/aggregate.yaml" ) for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done # Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md). kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed" kubectl kustomize "$SOURCE_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed" kubectl kustomize "$SOURCE_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed" kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed" inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml") n=0 for f in "${inputs[@]}"; do n=$((n + 1)) python3 -I - "$f" "$work/in/$(printf '%02d' "$n")-$(basename "$f")" <<'PY' || die "could not strip Secrets from $f" import sys, yaml yaml.SafeLoader.add_constructor("tag:yaml.org,2002:value", lambda l, n: l.construct_scalar(n)) docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if isinstance(d, dict) and d.get("kind") not in (None, "Secret")] yaml.safe_dump_all(docs, open(sys.argv[2], "w"), sort_keys=False) PY done for f in "$work"/in/*.yaml; do set +e KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff -f "$f" >"$work/out/$(basename "$f").diff" 2>"$work/out/$(basename "$f").err" rc=$? set -e if (( rc > 1 )); then sed -n '1,5p' "$work/out/$(basename "$f").err" >&2 die "kubectl diff failed for $(basename "$f")" fi done set +e python3 -I - "$work/out" "${accepted_file:-}" "$work/drift-count" <<'PY' import sys, pathlib, re, datetime out, accepted_path, count_path = pathlib.Path(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3]) today = datetime.date.today() accepted, expired = {}, set() if accepted_path: for line in pathlib.Path(accepted_path).read_text().splitlines(): line = line.strip() if not line or line.startswith("#"): continue kind, ns, name, until, *reason = line.split() key = (kind, "" if ns == "-" else ns, name) if datetime.date.fromisoformat(until) < today: expired.add(key) else: accepted[key] = " ".join(reason) version = re.compile(r"^v\d+((alpha|beta)\d+)?$") objects = {} for diff in sorted(out.glob("*.diff")): current = None for line in diff.read_text().splitlines(): if line.startswith("diff -u -N "): parts = line.split()[3].rsplit("/", 1)[-1].split(".") i = next(i for i, p in enumerate(parts) if version.match(p)) current = (parts[i + 1], parts[i + 2], ".".join(parts[i + 3:])) objects.setdefault(current, [0, 0, diff.name]) elif current and line[:1] in "+-" and not line.startswith(("+++", "---")): if re.match(r"^[+-]\s+generation:", line): continue objects[current][0 if line[0] == "+" else 1] += 1 drift = accepted_seen = 0 for key, (plus, minus, src) in sorted(objects.items()): if plus == 0 and minus == 0: continue kind, ns, name = key label = f"{kind} {ns or '-'}/{name} (+{plus} -{minus}) [{src}]" if key in accepted: accepted_seen += 1 print(f"ACCEPTED {label}: {accepted[key]}") continue drift += 1 print(f"DRIFT {label}" + (" (accepted entry expired)" if key in expired else "")) print(f"SUMMARY drift={drift} accepted={accepted_seen}") count_path.write_text(str(drift)) PY rc=$? set -e (( rc == 0 )) || die "could not summarize the diffs" drift_count="$(cat "$work/drift-count")" write_textfile "$drift_count" 1 if (( drift_count == 0 )); then echo "LIVE DRIFT CHECK PASS" exit 0 fi echo "LIVE DRIFT CHECK FOUND DRIFT" exit 1