The 2026-11-11 review found a pg_hba rule that existed only in the cluster by rendering every root by hand and running kubectl diff. Nothing does that regularly, so drift is found by accident. live-drift-check.sh renders all live roots (Phase 1 and 2, admin services, observability core, the Traefik trust overlay, the tenant, PriorityClass and keycloak policy roots), strips Secrets and runs kubectl diff, which is a server-side dry run. It prints one line per drifted object, honours a dated accepted-drift list and can write Prometheus textfile metrics for a later alert. Risk: read-only. It consumes the render scripts' verified-output-dir handoff and deletes those directories on exit; the README notes this second consumer. Secret values are not compared. Verification (desktop, before this series is applied): 17 drifted objects, matching the review plus the new roots; exit 1; the textfile holds platform_live_drift_objects 17.
174 lines
6.9 KiB
Bash
174 lines
6.9 KiB
Bash
#!/usr/bin/env bash
|
|
# Render every live build root of this repository and compare it with the
|
|
# cluster using `kubectl diff` (server-side dry-run: nothing is persisted).
|
|
#
|
|
# Usage:
|
|
# PLATFORM_HELM_BIN=/abs/helm bash scripts/validate/live-drift-check.sh \
|
|
# [--accepted FILE] [--textfile FILE]
|
|
#
|
|
# --accepted FILE known differences, one per line:
|
|
# `<Kind> <namespace or -> <name> <until YYYY-MM-DD> <reason...>`
|
|
# Expired entries count as drift again.
|
|
# --textfile FILE also write Prometheus textfile metrics (atomic rename).
|
|
#
|
|
# Exit status: 0 no drift, 1 drift found, 2 the check itself failed.
|
|
# Secrets are removed from the renders before diffing; Secret drift is out of scope.
|
|
set -Eeuo pipefail
|
|
|
|
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
accepted_file=""
|
|
textfile=""
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--accepted) accepted_file="${2:?--accepted needs a file}"; shift 2 ;;
|
|
--textfile) textfile="${2:?--textfile needs a file}"; shift 2 ;;
|
|
-h|--help) sed -n '2,16p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
*) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
|
|
declare -a made_dirs=()
|
|
cleanup() {
|
|
local d
|
|
for d in "${made_dirs[@]}"; do
|
|
case "$d" in /tmp/platform-*) rm -rf -- "$d" ;; esac
|
|
done
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
write_textfile() {
|
|
local drift="$1" ok="$2"
|
|
[[ -n "$textfile" ]] || return 0
|
|
local tmp="${textfile}.tmp.$$"
|
|
{
|
|
echo '# HELP platform_live_drift_objects Objects whose live state differs from the repository render.'
|
|
echo '# TYPE platform_live_drift_objects gauge'
|
|
echo "platform_live_drift_objects ${drift}"
|
|
echo '# HELP platform_live_drift_check_success 1 when the drift check itself completed.'
|
|
echo '# TYPE platform_live_drift_check_success gauge'
|
|
echo "platform_live_drift_check_success ${ok}"
|
|
echo '# HELP platform_live_drift_last_run_timestamp_seconds Unix time the drift check finished.'
|
|
echo '# TYPE platform_live_drift_last_run_timestamp_seconds gauge'
|
|
echo "platform_live_drift_last_run_timestamp_seconds $(date +%s)"
|
|
} >"$tmp"
|
|
mv -f -- "$tmp" "$textfile"
|
|
}
|
|
|
|
die() {
|
|
printf 'DRIFT CHECK ERROR: %s\n' "$*" >&2
|
|
write_textfile 0 0
|
|
exit 2
|
|
}
|
|
|
|
work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work")
|
|
mkdir -p "$work/in" "$work/out"
|
|
|
|
render_root() {
|
|
local script="$1" prefix="$2" dir
|
|
dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir")
|
|
bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \
|
|
>"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; }
|
|
printf '%s\n' "$dir"
|
|
}
|
|
|
|
phase1="$(render_root render-phase1.sh platform-phase1-apply)"
|
|
phase2="$(render_root render-phase2.sh platform-phase2-apply)"
|
|
admin="$(render_root render-admin-services.sh platform-admin-apply)"
|
|
observability="$(render_root render-observability-core.sh platform-observability-core-apply)"
|
|
|
|
# The live Gitea is the OIDC profile; gitea.yaml is the fresh-install baseline only.
|
|
declare -a inputs=(
|
|
"$phase1/namespaces.yaml" "$phase1/ssd-local-pv.yaml" "$phase1/cnpg-operator.yaml"
|
|
"$phase1/platform-postgres.yaml" "$phase1/gitea-oidc.yaml"
|
|
"$observability/aggregate.yaml"
|
|
)
|
|
for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done
|
|
|
|
# Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md).
|
|
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed"
|
|
kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed"
|
|
kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed"
|
|
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed"
|
|
inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml")
|
|
|
|
n=0
|
|
for f in "${inputs[@]}"; do
|
|
n=$((n + 1))
|
|
python3 -I - "$f" "$work/in/$(printf '%02d' "$n")-$(basename "$f")" <<'PY' || die "could not strip Secrets from $f"
|
|
import sys, yaml
|
|
yaml.SafeLoader.add_constructor("tag:yaml.org,2002:value", lambda l, n: l.construct_scalar(n))
|
|
docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if isinstance(d, dict) and d.get("kind") not in (None, "Secret")]
|
|
yaml.safe_dump_all(docs, open(sys.argv[2], "w"), sort_keys=False)
|
|
PY
|
|
done
|
|
|
|
for f in "$work"/in/*.yaml; do
|
|
set +e
|
|
KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff -f "$f" >"$work/out/$(basename "$f").diff" 2>"$work/out/$(basename "$f").err"
|
|
rc=$?
|
|
set -e
|
|
if (( rc > 1 )); then
|
|
sed -n '1,5p' "$work/out/$(basename "$f").err" >&2
|
|
die "kubectl diff failed for $(basename "$f")"
|
|
fi
|
|
done
|
|
|
|
set +e
|
|
python3 -I - "$work/out" "${accepted_file:-}" "$work/drift-count" <<'PY'
|
|
import sys, pathlib, re, datetime
|
|
out, accepted_path, count_path = pathlib.Path(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3])
|
|
today = datetime.date.today()
|
|
accepted, expired = {}, set()
|
|
if accepted_path:
|
|
for line in pathlib.Path(accepted_path).read_text().splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
kind, ns, name, until, *reason = line.split()
|
|
key = (kind, "" if ns == "-" else ns, name)
|
|
if datetime.date.fromisoformat(until) < today:
|
|
expired.add(key)
|
|
else:
|
|
accepted[key] = " ".join(reason)
|
|
version = re.compile(r"^v\d+((alpha|beta)\d+)?$")
|
|
objects = {}
|
|
for diff in sorted(out.glob("*.diff")):
|
|
current = None
|
|
for line in diff.read_text().splitlines():
|
|
if line.startswith("diff -u -N "):
|
|
parts = line.split()[3].rsplit("/", 1)[-1].split(".")
|
|
i = next(i for i, p in enumerate(parts) if version.match(p))
|
|
current = (parts[i + 1], parts[i + 2], ".".join(parts[i + 3:]))
|
|
objects.setdefault(current, [0, 0, diff.name])
|
|
elif current and line[:1] in "+-" and not line.startswith(("+++", "---")):
|
|
if re.match(r"^[+-]\s+generation:", line):
|
|
continue
|
|
objects[current][0 if line[0] == "+" else 1] += 1
|
|
drift = accepted_seen = 0
|
|
for key, (plus, minus, src) in sorted(objects.items()):
|
|
if plus == 0 and minus == 0:
|
|
continue
|
|
kind, ns, name = key
|
|
label = f"{kind} {ns or '-'}/{name} (+{plus} -{minus}) [{src}]"
|
|
if key in accepted:
|
|
accepted_seen += 1
|
|
print(f"ACCEPTED {label}: {accepted[key]}")
|
|
continue
|
|
drift += 1
|
|
print(f"DRIFT {label}" + (" (accepted entry expired)" if key in expired else ""))
|
|
print(f"SUMMARY drift={drift} accepted={accepted_seen}")
|
|
count_path.write_text(str(drift))
|
|
PY
|
|
rc=$?
|
|
set -e
|
|
(( rc == 0 )) || die "could not summarize the diffs"
|
|
drift_count="$(cat "$work/drift-count")"
|
|
write_textfile "$drift_count" 1
|
|
if (( drift_count == 0 )); then
|
|
echo "LIVE DRIFT CHECK PASS"
|
|
exit 0
|
|
fi
|
|
echo "LIVE DRIFT CHECK FOUND DRIFT"
|
|
exit 1
|