diff --git a/infrastructure/networking/traefik/README.md b/infrastructure/networking/traefik/README.md index 3901fa9..e61a5fc 100644 --- a/infrastructure/networking/traefik/README.md +++ b/infrastructure/networking/traefik/README.md @@ -94,6 +94,7 @@ infrastructure/networking/traefik/ ├── apply-observe.sh ├── observe-client-host.sh ├── apply-trust.sh + ├── reapply-trust.sh ├── rollback-to-observe.sh └── validate.sh ``` @@ -238,6 +239,20 @@ bash infrastructure/networking/traefik/scripts/observe-client-host.sh `apply-trust.sh`를 실행한다. Chart, 이미지, context, API server, NodePort 경계 또는 재관측 값이 기대와 다르면 스크립트가 적용을 중단한다. +이미 trust인 상태에서 trust overlay의 다른 값(예: `resources`)만 바꿨을 때는 `apply-trust.sh`가 +「observation phase must not trust forwarded headers」로 멈춘다(observe → trust 전환 전용이다). 그때는 +`reapply-trust.sh`를 쓴다. + +```sh +bash infrastructure/networking/traefik/scripts/reapply-trust.sh --observed-client-host 10.42.0.1 --execute +# 승인: APPLY <현재-context> RETRUST 10.42.0.1/32 +``` + +- 적용 전에 runtime이 trust인지, 전달 헤더 신뢰가 실제로 작동하는지(Host Nginx 경유 요청의 ClientHost가 + 프록시 주소가 아님)를 본다. 바꿀 것을 `kubectl diff`로 보여 주고, 차이가 없으면 아무것도 하지 않는다. +- 적용 뒤 같은 검사를 다시 하고, 하나라도 실패하면 직전에 라이브였던 HelmChartConfig를 다시 적용한다. + trusted CIDR이 빠지는 일은 없다. + ### 롤백 trust만 제거하고 JSON access log를 남기는 표준 롤백은 observe overlay를 적용한다. diff --git a/infrastructure/networking/traefik/scripts/reapply-trust.sh b/infrastructure/networking/traefik/scripts/reapply-trust.sh new file mode 100755 index 0000000..fcb724b --- /dev/null +++ b/infrastructure/networking/traefik/scripts/reapply-trust.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Re-apply the trust overlay when the live Traefik is already in the trust phase, +# for example after a values change such as resources. apply-trust.sh is only for +# the observe -> trust transition and stops when the runtime already trusts the +# CIDR ("observation phase must not trust forwarded headers"). +# +# bash infrastructure/networking/traefik/scripts/reapply-trust.sh \ +# --observed-client-host 10.42.0.1 --execute +# +# On any failure after the apply it restores the HelmChartConfig that was live +# before, so the trusted CIDR is never dropped. +set -Eeuo pipefail +set +x +umask 077 + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +[[ "$#" -eq 3 && "$1" == "--observed-client-host" && "$3" == "--execute" ]] || { + sed -n '2,11p' "${BASH_SOURCE[0]}" >&2 + exit 2 +} +readonly REVIEWED_CLIENT_HOST="$2" + +require_commands kubectl jq rg curl nc python3 awk sort date cmp find bash +reviewed_cidr="$(host_to_exact_cidr "$REVIEWED_CLIENT_HOST")" +declared_cidr="$(normalize_exact_host_cidr "$(source_trusted_proxy_cidr)")" +[[ "$declared_cidr" != "$SENTINEL_TRUSTED_PROXY_CIDR" ]] || \ + fail "trust overlay still contains the non-routable sentinel CIDR" +[[ "$declared_cidr" == "$reviewed_cidr" ]] || \ + fail "declared CIDR ${declared_cidr} does not match ClientHost ${REVIEWED_CLIENT_HOST}" + +bash "${SCRIPT_DIR}/validate.sh" + +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-retrust.XXXXXX")" +restore_required=false +selected_context="" +cleanup() { + local exit_code=$? + trap - EXIT + if [[ "$exit_code" -ne 0 && "$restore_required" == "true" ]]; then + printf '\nRESTORE: re-applying the HelmChartConfig that was live before.\n' >&2 + set +e + kubectl --context "$selected_context" apply --filename "${work_dir}/previous.yaml" + wait_for_runtime trust "$declared_cidr" + kubectl --context "$selected_context" --namespace kube-system \ + rollout status deployment/traefik --timeout=5m + wait_for_nodeport_boundary_and_health + printf 'RESTORE complete. The previous trust values are live again.\n' >&2 + set -e + fi + case "$work_dir" in + /tmp/platform-traefik-retrust.*|"${TMPDIR:-/tmp}"/platform-traefik-retrust.*) rm -rf -- "$work_dir" ;; + esac + exit "$exit_code" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +trust_render="${work_dir}/trust.yaml" +render_overlay "$TRUST_OVERLAY" "$trust_render" + +assert_live_baseline +assert_runtime trust "$declared_cidr" +assert_nodeport_boundary_and_health + +# In the trust phase Traefik logs the forwarded client, not the proxy. Seeing the +# proxy address (the reviewed ClientHost) would mean the trust is not working. +assert_trust_effective() { + local seen + seen="$(observe_host_nginx_client_host)" + [[ "$seen" != "$REVIEWED_CLIENT_HOST" ]] || \ + fail "Traefik logged the proxy ${seen} as ClientHost; forwarded-header trust is not in effect" + printf 'Forwarded-header trust in effect (ClientHost %s, proxy %s)\n' "$seen" "$REVIEWED_CLIENT_HOST" +} +assert_trust_effective + +kubectl --namespace kube-system get helmchartconfig traefik --output=json | + python3 -I -c ' +import json, sys +o = json.load(sys.stdin) +md = o["metadata"] +keep = {k: md[k] for k in ("name", "namespace", "labels", "annotations") if k in md} +keep.get("annotations", {}).pop("kubectl.kubernetes.io/last-applied-configuration", None) +print(json.dumps({"apiVersion": o["apiVersion"], "kind": o["kind"], "metadata": keep, "spec": o["spec"]})) +' >"${work_dir}/previous.yaml" + +printf '\nChange to apply (kubectl diff, server dry-run):\n' +set +e +KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff --filename "$trust_render" | grep -E '^[-+] ' | grep -v -E '^[-+] generation:' +diff_rc=${PIPESTATUS[0]} +set -e +if [[ "$diff_rc" -eq 0 ]]; then + printf 'Live HelmChartConfig already matches the trust overlay. Nothing to do.\n' + exit 0 +fi +(( diff_rc == 1 )) || fail "kubectl diff failed" + +selected_context="$(current_context)" +selected_api_server="$(current_api_server)" +printf '\nKubernetes context: %s\nAPI server: %s\nTrusted CIDR (unchanged): %s\n' \ + "$selected_context" "$selected_api_server" "$declared_cidr" +printf 'Type APPLY %s RETRUST %s to continue: ' "$selected_context" "$declared_cidr" +read -r confirmation +[[ "$confirmation" == "APPLY ${selected_context} RETRUST ${declared_cidr}" ]] || fail "cancelled" +[[ "$(current_context)" == "$selected_context" ]] || fail "kubectl context changed after confirmation" +[[ "$(current_api_server)" == "$selected_api_server" ]] || fail "Kubernetes API server changed after confirmation" +assert_runtime trust "$declared_cidr" + +restore_required=true +kubectl apply --filename "$trust_render" +wait_for_runtime trust "$declared_cidr" +kubectl --namespace kube-system rollout status deployment/traefik --timeout=5m +assert_runtime trust "$declared_cidr" +assert_live_hcc_matches_overlay "$TRUST_OVERLAY" "${work_dir}/post-apply-trust.yaml" +assert_live_baseline +wait_for_nodeport_boundary_and_health +assert_manifest_https +assert_trust_effective +restore_required=false + +printf '\nTRUST PHASE RE-APPLIED\n' +printf 'Traefik web trusted CIDR: %s\n' "$declared_cidr" +printf 'Gitea health and HTTPS site-manifest checks: PASS\n' diff --git a/scripts/validate/live-drift-accepted.txt b/scripts/validate/live-drift-accepted.txt index cd416e9..4ae432c 100644 --- a/scripts/validate/live-drift-accepted.txt +++ b/scripts/validate/live-drift-accepted.txt @@ -12,3 +12,4 @@ Keycloak keycloak keycloak 2026-11-30 이미지 digest·startOptimized false·pl Deployment gitea gitea 2026-11-30 priorityClassName platform-critical(Gitea 재시작). 재시작 창에 apply-gitea-oidc.sh로 적용한다. StatefulSet observability loki 2026-11-30 priorityClassName platform-low(Loki 재시작). 재시작 창에 관측 core 적용으로. StatefulSet observability tempo 2026-11-30 priorityClassName platform-low(Tempo 재시작). 재시작 창에 관측 core 적용으로. +PrometheusRule observability observability-core-kube-pr-kubernetes-system-kubelet 2026-11-30 Kubelet 인증서 경보 둘을 values에서 껐다(PR #6). 관측 core를 다시 적용하면 사라진다(재시작 창 2단계). diff --git a/scripts/validate/live-drift-check.sh b/scripts/validate/live-drift-check.sh index 4e85a2d..6334d59 100644 --- a/scripts/validate/live-drift-check.sh +++ b/scripts/validate/live-drift-check.sh @@ -64,10 +64,29 @@ die() { work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work") mkdir -p "$work/in" "$work/out" +# Render from a clean copy when run inside a git working tree. The observability +# renderer refuses to run when an ignored chart cache (.helm/) is left over from an +# earlier render, which happens in long-lived checkouts. The copy keeps tracked and +# untracked-but-not-ignored files, so local edits are still checked. +SOURCE_ROOT="$REPOSITORY_ROOT" +if git -C "$REPOSITORY_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + SOURCE_ROOT="$work/src" + mkdir -p "$SOURCE_ROOT" + python3 -I - "$REPOSITORY_ROOT" <<'PY' | tar -C "$REPOSITORY_ROOT" --null -T - -cf - | tar -x -C "$SOURCE_ROOT" || die "could not copy the working tree" +import subprocess, sys +root = sys.argv[1] +def ls(*args): + out = subprocess.run(["git", "-C", root, "ls-files", "-z", *args], check=True, capture_output=True).stdout + return [p for p in out.decode().split("\0") if p] +keep = set(ls("--cached", "--others", "--exclude-standard")) - set(ls("--deleted")) +sys.stdout.write("\0".join(sorted(keep)) + "\0") +PY +fi + render_root() { local script="$1" prefix="$2" dir dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir") - bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \ + bash "$SOURCE_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \ >"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; } printf '%s\n' "$dir" } @@ -86,10 +105,10 @@ declare -a inputs=( for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done # Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md). -kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed" -kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed" -kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed" -kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed" +kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed" +kubectl kustomize "$SOURCE_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed" +kubectl kustomize "$SOURCE_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed" +kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed" inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml") n=0