From cc5decc538a58b66c6e1d4b5ae775312cb5dce96 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 02:00:20 +0900 Subject: [PATCH 1/2] feat(traefik): re-apply the trust overlay when trust is already live apply-trust.sh is the observe -> trust transition. When the live Traefik is already in the trust phase and only another value changed (PR #3 resources), it stops at "observation phase must not trust forwarded headers" before applying, so the change had to be applied with a bare kubectl apply outside the guarded path. reapply-trust.sh checks that the runtime is in the trust phase with the reviewed CIDR and that forwarded-header trust works (a request through Host Nginx is logged with the forwarded client, not the proxy address), saves the live HelmChartConfig, shows the change with kubectl diff and does nothing when there is none, asks for "APPLY RETRUST ", applies, and repeats the checks plus the NodePort, Gitea health and site-manifest checks. Any failure after the apply re-applies the saved HelmChartConfig, so the trusted CIDR is never dropped. Risk: the apply path was not run against the cluster (the live values already match). The restore path reuses the same wait and health helpers as apply-trust.sh. Verification (desktop): - Live (already trust, values match): "Forwarded-header trust in effect (ClientHost 127.0.0.1, proxy 10.42.0.1)", then "already matches the trust overlay. Nothing to do.", exit 0. - Copy with memory 64Mi changed to 65Mi, answering "no": the diff shows only that line, "ERROR: cancelled", exit 1; live still 64Mi. - The ClientHost check of apply-trust.sh would have failed here (127.0.0.1 is expected in the trust phase), which is why the check is inverted. --- infrastructure/networking/traefik/README.md | 15 +++ .../traefik/scripts/reapply-trust.sh | 126 ++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100755 infrastructure/networking/traefik/scripts/reapply-trust.sh diff --git a/infrastructure/networking/traefik/README.md b/infrastructure/networking/traefik/README.md index 3901fa9..e61a5fc 100644 --- a/infrastructure/networking/traefik/README.md +++ b/infrastructure/networking/traefik/README.md @@ -94,6 +94,7 @@ infrastructure/networking/traefik/ ├── apply-observe.sh ├── observe-client-host.sh ├── apply-trust.sh + ├── reapply-trust.sh ├── rollback-to-observe.sh └── validate.sh ``` @@ -238,6 +239,20 @@ bash infrastructure/networking/traefik/scripts/observe-client-host.sh `apply-trust.sh`를 실행한다. Chart, 이미지, context, API server, NodePort 경계 또는 재관측 값이 기대와 다르면 스크립트가 적용을 중단한다. +이미 trust인 상태에서 trust overlay의 다른 값(예: `resources`)만 바꿨을 때는 `apply-trust.sh`가 +「observation phase must not trust forwarded headers」로 멈춘다(observe → trust 전환 전용이다). 그때는 +`reapply-trust.sh`를 쓴다. + +```sh +bash infrastructure/networking/traefik/scripts/reapply-trust.sh --observed-client-host 10.42.0.1 --execute +# 승인: APPLY <현재-context> RETRUST 10.42.0.1/32 +``` + +- 적용 전에 runtime이 trust인지, 전달 헤더 신뢰가 실제로 작동하는지(Host Nginx 경유 요청의 ClientHost가 + 프록시 주소가 아님)를 본다. 바꿀 것을 `kubectl diff`로 보여 주고, 차이가 없으면 아무것도 하지 않는다. +- 적용 뒤 같은 검사를 다시 하고, 하나라도 실패하면 직전에 라이브였던 HelmChartConfig를 다시 적용한다. + trusted CIDR이 빠지는 일은 없다. + ### 롤백 trust만 제거하고 JSON access log를 남기는 표준 롤백은 observe overlay를 적용한다. diff --git a/infrastructure/networking/traefik/scripts/reapply-trust.sh b/infrastructure/networking/traefik/scripts/reapply-trust.sh new file mode 100755 index 0000000..fcb724b --- /dev/null +++ b/infrastructure/networking/traefik/scripts/reapply-trust.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Re-apply the trust overlay when the live Traefik is already in the trust phase, +# for example after a values change such as resources. apply-trust.sh is only for +# the observe -> trust transition and stops when the runtime already trusts the +# CIDR ("observation phase must not trust forwarded headers"). +# +# bash infrastructure/networking/traefik/scripts/reapply-trust.sh \ +# --observed-client-host 10.42.0.1 --execute +# +# On any failure after the apply it restores the HelmChartConfig that was live +# before, so the trusted CIDR is never dropped. +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +[[ "$#" -eq 3 && "$1" == "--observed-client-host" && "$3" == "--execute" ]] || { + sed -n '2,11p' "${BASH_SOURCE[0]}" >&2 + exit 2 +} +readonly REVIEWED_CLIENT_HOST="$2" + +require_commands kubectl jq rg curl nc python3 awk sort date cmp find bash +reviewed_cidr="$(host_to_exact_cidr "$REVIEWED_CLIENT_HOST")" +declared_cidr="$(normalize_exact_host_cidr "$(source_trusted_proxy_cidr)")" +[[ "$declared_cidr" != "$SENTINEL_TRUSTED_PROXY_CIDR" ]] || \ + fail "trust overlay still contains the non-routable sentinel CIDR" +[[ "$declared_cidr" == "$reviewed_cidr" ]] || \ + fail "declared CIDR ${declared_cidr} does not match ClientHost ${REVIEWED_CLIENT_HOST}" + +bash "${SCRIPT_DIR}/validate.sh" + +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-retrust.XXXXXX")" +restore_required=false +selected_context="" +cleanup() { + local exit_code=$? + trap - EXIT + if [[ "$exit_code" -ne 0 && "$restore_required" == "true" ]]; then + printf '\nRESTORE: re-applying the HelmChartConfig that was live before.\n' >&2 + set +e + kubectl --context "$selected_context" apply --filename "${work_dir}/previous.yaml" + wait_for_runtime trust "$declared_cidr" + kubectl --context "$selected_context" --namespace kube-system \ + rollout status deployment/traefik --timeout=5m + wait_for_nodeport_boundary_and_health + printf 'RESTORE complete. The previous trust values are live again.\n' >&2 + set -e + fi + case "$work_dir" in + /tmp/platform-traefik-retrust.*|"${TMPDIR:-/tmp}"/platform-traefik-retrust.*) rm -rf -- "$work_dir" ;; + esac + exit "$exit_code" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +trust_render="${work_dir}/trust.yaml" +render_overlay "$TRUST_OVERLAY" "$trust_render" + +assert_live_baseline +assert_runtime trust "$declared_cidr" +assert_nodeport_boundary_and_health + +# In the trust phase Traefik logs the forwarded client, not the proxy. Seeing the +# proxy address (the reviewed ClientHost) would mean the trust is not working. +assert_trust_effective() { + local seen + seen="$(observe_host_nginx_client_host)" + [[ "$seen" != "$REVIEWED_CLIENT_HOST" ]] || \ + fail "Traefik logged the proxy ${seen} as ClientHost; forwarded-header trust is not in effect" + printf 'Forwarded-header trust in effect (ClientHost %s, proxy %s)\n' "$seen" "$REVIEWED_CLIENT_HOST" +} +assert_trust_effective + +kubectl --namespace kube-system get helmchartconfig traefik --output=json | + python3 -I -c ' +import json, sys +o = json.load(sys.stdin) +md = o["metadata"] +keep = {k: md[k] for k in ("name", "namespace", "labels", "annotations") if k in md} +keep.get("annotations", {}).pop("kubectl.kubernetes.io/last-applied-configuration", None) +print(json.dumps({"apiVersion": o["apiVersion"], "kind": o["kind"], "metadata": keep, "spec": o["spec"]})) +' >"${work_dir}/previous.yaml" + +printf '\nChange to apply (kubectl diff, server dry-run):\n' +set +e +KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff --filename "$trust_render" | grep -E '^[-+] ' | grep -v -E '^[-+] generation:' +diff_rc=${PIPESTATUS[0]} +set -e +if [[ "$diff_rc" -eq 0 ]]; then + printf 'Live HelmChartConfig already matches the trust overlay. Nothing to do.\n' + exit 0 +fi +(( diff_rc == 1 )) || fail "kubectl diff failed" + +selected_context="$(current_context)" +selected_api_server="$(current_api_server)" +printf '\nKubernetes context: %s\nAPI server: %s\nTrusted CIDR (unchanged): %s\n' \ + "$selected_context" "$selected_api_server" "$declared_cidr" +printf 'Type APPLY %s RETRUST %s to continue: ' "$selected_context" "$declared_cidr" +read -r confirmation +[[ "$confirmation" == "APPLY ${selected_context} RETRUST ${declared_cidr}" ]] || fail "cancelled" +[[ "$(current_context)" == "$selected_context" ]] || fail "kubectl context changed after confirmation" +[[ "$(current_api_server)" == "$selected_api_server" ]] || fail "Kubernetes API server changed after confirmation" +assert_runtime trust "$declared_cidr" + +restore_required=true +kubectl apply --filename "$trust_render" +wait_for_runtime trust "$declared_cidr" +kubectl --namespace kube-system rollout status deployment/traefik --timeout=5m +assert_runtime trust "$declared_cidr" +assert_live_hcc_matches_overlay "$TRUST_OVERLAY" "${work_dir}/post-apply-trust.yaml" +assert_live_baseline +wait_for_nodeport_boundary_and_health +assert_manifest_https +assert_trust_effective +restore_required=false + +printf '\nTRUST PHASE RE-APPLIED\n' +printf 'Traefik web trusted CIDR: %s\n' "$declared_cidr" +printf 'Gitea health and HTTPS site-manifest checks: PASS\n' From 050a4de23508cb40b204e946a8a7c486e1642d19 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 02:00:20 +0900 Subject: [PATCH 2/2] fix(validate): render the drift check from a clean working-tree copy In a long-lived checkout the observability renderer finds an ignored .helm/charts/- cache left by an earlier render and refuses ("exact chart cache already exists"), so live-drift-check.sh fails with exit 2 in /home/donghyeon/workspace/platform, the path the runbooks use. When run inside a git working tree, copy the tracked and untracked (not ignored, not deleted) files to the drift work directory and render from there. Local edits are still checked; ignored caches are not carried over. Outside a git tree (the daily timer renders a git archive of origin/main) nothing changes. Also accept the one drift this exposed: the kube-prometheus-stack PrometheusRule still has the two Kubelet certificate alerts that PR #6 disabled in values; it goes away when the observability core is re-applied in the restart window (until 2026-11-30). Risk: read-only, as before. The copy is removed with the work directory. Verification (desktop, fresh clone of main 8b019d2 with an ignored services/observability/tempo/.helm/charts/tempo-2.2.3 created): - old script: "REJECT: exact chart cache already exists", exit 2. - new script as an uncommitted edit in that clone: SUMMARY drift=1 accepted=12 (the PrometheusRule above), exit 1. - new script from a git archive export, as the timer runs it: same summary. --- scripts/validate/live-drift-accepted.txt | 1 + scripts/validate/live-drift-check.sh | 29 ++++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/validate/live-drift-accepted.txt b/scripts/validate/live-drift-accepted.txt index cd416e9..4ae432c 100644 --- a/scripts/validate/live-drift-accepted.txt +++ b/scripts/validate/live-drift-accepted.txt @@ -12,3 +12,4 @@ Keycloak keycloak keycloak 2026-11-30 이미지 digest·startOptimized false·pl Deployment gitea gitea 2026-11-30 priorityClassName platform-critical(Gitea 재시작). 재시작 창에 apply-gitea-oidc.sh로 적용한다. StatefulSet observability loki 2026-11-30 priorityClassName platform-low(Loki 재시작). 재시작 창에 관측 core 적용으로. StatefulSet observability tempo 2026-11-30 priorityClassName platform-low(Tempo 재시작). 재시작 창에 관측 core 적용으로. +PrometheusRule observability observability-core-kube-pr-kubernetes-system-kubelet 2026-11-30 Kubelet 인증서 경보 둘을 values에서 껐다(PR #6). 관측 core를 다시 적용하면 사라진다(재시작 창 2단계). diff --git a/scripts/validate/live-drift-check.sh b/scripts/validate/live-drift-check.sh index 4e85a2d..6334d59 100644 --- a/scripts/validate/live-drift-check.sh +++ b/scripts/validate/live-drift-check.sh @@ -64,10 +64,29 @@ die() { work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work") mkdir -p "$work/in" "$work/out" +# Render from a clean copy when run inside a git working tree. The observability +# renderer refuses to run when an ignored chart cache (.helm/) is left over from an +# earlier render, which happens in long-lived checkouts. The copy keeps tracked and +# untracked-but-not-ignored files, so local edits are still checked. +SOURCE_ROOT="$REPOSITORY_ROOT" +if git -C "$REPOSITORY_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + SOURCE_ROOT="$work/src" + mkdir -p "$SOURCE_ROOT" + python3 -I - "$REPOSITORY_ROOT" <<'PY' | tar -C "$REPOSITORY_ROOT" --null -T - -cf - | tar -x -C "$SOURCE_ROOT" || die "could not copy the working tree" +import subprocess, sys +root = sys.argv[1] +def ls(*args): + out = subprocess.run(["git", "-C", root, "ls-files", "-z", *args], check=True, capture_output=True).stdout + return [p for p in out.decode().split("\0") if p] +keep = set(ls("--cached", "--others", "--exclude-standard")) - set(ls("--deleted")) +sys.stdout.write("\0".join(sorted(keep)) + "\0") +PY +fi + render_root() { local script="$1" prefix="$2" dir dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir") - bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \ + bash "$SOURCE_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \ >"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; } printf '%s\n' "$dir" } @@ -86,10 +105,10 @@ declare -a inputs=( for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done # Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md). -kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed" -kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed" -kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed" -kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed" +kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed" +kubectl kustomize "$SOURCE_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed" +kubectl kustomize "$SOURCE_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed" +kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed" inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml") n=0