fix(validate): re-apply Traefik trust and render drift from a clean copy #9
@@ -94,6 +94,7 @@ infrastructure/networking/traefik/
|
||||
├── apply-observe.sh
|
||||
├── observe-client-host.sh
|
||||
├── apply-trust.sh
|
||||
├── reapply-trust.sh
|
||||
├── rollback-to-observe.sh
|
||||
└── validate.sh
|
||||
```
|
||||
@@ -238,6 +239,20 @@ bash infrastructure/networking/traefik/scripts/observe-client-host.sh
|
||||
`apply-trust.sh`를 실행한다. Chart, 이미지, context, API server, NodePort 경계 또는
|
||||
재관측 값이 기대와 다르면 스크립트가 적용을 중단한다.
|
||||
|
||||
이미 trust인 상태에서 trust overlay의 다른 값(예: `resources`)만 바꿨을 때는 `apply-trust.sh`가
|
||||
「observation phase must not trust forwarded headers」로 멈춘다(observe → trust 전환 전용이다). 그때는
|
||||
`reapply-trust.sh`를 쓴다.
|
||||
|
||||
```sh
|
||||
bash infrastructure/networking/traefik/scripts/reapply-trust.sh --observed-client-host 10.42.0.1 --execute
|
||||
# 승인: APPLY <현재-context> RETRUST 10.42.0.1/32
|
||||
```
|
||||
|
||||
- 적용 전에 runtime이 trust인지, 전달 헤더 신뢰가 실제로 작동하는지(Host Nginx 경유 요청의 ClientHost가
|
||||
프록시 주소가 아님)를 본다. 바꿀 것을 `kubectl diff`로 보여 주고, 차이가 없으면 아무것도 하지 않는다.
|
||||
- 적용 뒤 같은 검사를 다시 하고, 하나라도 실패하면 직전에 라이브였던 HelmChartConfig를 다시 적용한다.
|
||||
trusted CIDR이 빠지는 일은 없다.
|
||||
|
||||
### 롤백
|
||||
|
||||
trust만 제거하고 JSON access log를 남기는 표준 롤백은 observe overlay를 적용한다.
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-apply the trust overlay when the live Traefik is already in the trust phase,
|
||||
# for example after a values change such as resources. apply-trust.sh is only for
|
||||
# the observe -> trust transition and stops when the runtime already trusts the
|
||||
# CIDR ("observation phase must not trust forwarded headers").
|
||||
#
|
||||
# bash infrastructure/networking/traefik/scripts/reapply-trust.sh \
|
||||
# --observed-client-host 10.42.0.1 --execute
|
||||
#
|
||||
# On any failure after the apply it restores the HelmChartConfig that was live
|
||||
# before, so the trusted CIDR is never dropped.
|
||||
set -Eeuo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
# shellcheck source=common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
[[ "$#" -eq 3 && "$1" == "--observed-client-host" && "$3" == "--execute" ]] || {
|
||||
sed -n '2,11p' "${BASH_SOURCE[0]}" >&2
|
||||
exit 2
|
||||
}
|
||||
readonly REVIEWED_CLIENT_HOST="$2"
|
||||
|
||||
require_commands kubectl jq rg curl nc python3 awk sort date cmp find bash
|
||||
reviewed_cidr="$(host_to_exact_cidr "$REVIEWED_CLIENT_HOST")"
|
||||
declared_cidr="$(normalize_exact_host_cidr "$(source_trusted_proxy_cidr)")"
|
||||
[[ "$declared_cidr" != "$SENTINEL_TRUSTED_PROXY_CIDR" ]] || \
|
||||
fail "trust overlay still contains the non-routable sentinel CIDR"
|
||||
[[ "$declared_cidr" == "$reviewed_cidr" ]] || \
|
||||
fail "declared CIDR ${declared_cidr} does not match ClientHost ${REVIEWED_CLIENT_HOST}"
|
||||
|
||||
bash "${SCRIPT_DIR}/validate.sh"
|
||||
|
||||
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-traefik-retrust.XXXXXX")"
|
||||
restore_required=false
|
||||
selected_context=""
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
trap - EXIT
|
||||
if [[ "$exit_code" -ne 0 && "$restore_required" == "true" ]]; then
|
||||
printf '\nRESTORE: re-applying the HelmChartConfig that was live before.\n' >&2
|
||||
set +e
|
||||
kubectl --context "$selected_context" apply --filename "${work_dir}/previous.yaml"
|
||||
wait_for_runtime trust "$declared_cidr"
|
||||
kubectl --context "$selected_context" --namespace kube-system \
|
||||
rollout status deployment/traefik --timeout=5m
|
||||
wait_for_nodeport_boundary_and_health
|
||||
printf 'RESTORE complete. The previous trust values are live again.\n' >&2
|
||||
set -e
|
||||
fi
|
||||
case "$work_dir" in
|
||||
/tmp/platform-traefik-retrust.*|"${TMPDIR:-/tmp}"/platform-traefik-retrust.*) rm -rf -- "$work_dir" ;;
|
||||
esac
|
||||
exit "$exit_code"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
trust_render="${work_dir}/trust.yaml"
|
||||
render_overlay "$TRUST_OVERLAY" "$trust_render"
|
||||
|
||||
assert_live_baseline
|
||||
assert_runtime trust "$declared_cidr"
|
||||
assert_nodeport_boundary_and_health
|
||||
|
||||
# In the trust phase Traefik logs the forwarded client, not the proxy. Seeing the
|
||||
# proxy address (the reviewed ClientHost) would mean the trust is not working.
|
||||
assert_trust_effective() {
|
||||
local seen
|
||||
seen="$(observe_host_nginx_client_host)"
|
||||
[[ "$seen" != "$REVIEWED_CLIENT_HOST" ]] || \
|
||||
fail "Traefik logged the proxy ${seen} as ClientHost; forwarded-header trust is not in effect"
|
||||
printf 'Forwarded-header trust in effect (ClientHost %s, proxy %s)\n' "$seen" "$REVIEWED_CLIENT_HOST"
|
||||
}
|
||||
assert_trust_effective
|
||||
|
||||
kubectl --namespace kube-system get helmchartconfig traefik --output=json |
|
||||
python3 -I -c '
|
||||
import json, sys
|
||||
o = json.load(sys.stdin)
|
||||
md = o["metadata"]
|
||||
keep = {k: md[k] for k in ("name", "namespace", "labels", "annotations") if k in md}
|
||||
keep.get("annotations", {}).pop("kubectl.kubernetes.io/last-applied-configuration", None)
|
||||
print(json.dumps({"apiVersion": o["apiVersion"], "kind": o["kind"], "metadata": keep, "spec": o["spec"]}))
|
||||
' >"${work_dir}/previous.yaml"
|
||||
|
||||
printf '\nChange to apply (kubectl diff, server dry-run):\n'
|
||||
set +e
|
||||
KUBECTL_EXTERNAL_DIFF="diff -u -N" kubectl diff --filename "$trust_render" | grep -E '^[-+] ' | grep -v -E '^[-+] generation:'
|
||||
diff_rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
if [[ "$diff_rc" -eq 0 ]]; then
|
||||
printf 'Live HelmChartConfig already matches the trust overlay. Nothing to do.\n'
|
||||
exit 0
|
||||
fi
|
||||
(( diff_rc == 1 )) || fail "kubectl diff failed"
|
||||
|
||||
selected_context="$(current_context)"
|
||||
selected_api_server="$(current_api_server)"
|
||||
printf '\nKubernetes context: %s\nAPI server: %s\nTrusted CIDR (unchanged): %s\n' \
|
||||
"$selected_context" "$selected_api_server" "$declared_cidr"
|
||||
printf 'Type APPLY %s RETRUST %s to continue: ' "$selected_context" "$declared_cidr"
|
||||
read -r confirmation
|
||||
[[ "$confirmation" == "APPLY ${selected_context} RETRUST ${declared_cidr}" ]] || fail "cancelled"
|
||||
[[ "$(current_context)" == "$selected_context" ]] || fail "kubectl context changed after confirmation"
|
||||
[[ "$(current_api_server)" == "$selected_api_server" ]] || fail "Kubernetes API server changed after confirmation"
|
||||
assert_runtime trust "$declared_cidr"
|
||||
|
||||
restore_required=true
|
||||
kubectl apply --filename "$trust_render"
|
||||
wait_for_runtime trust "$declared_cidr"
|
||||
kubectl --namespace kube-system rollout status deployment/traefik --timeout=5m
|
||||
assert_runtime trust "$declared_cidr"
|
||||
assert_live_hcc_matches_overlay "$TRUST_OVERLAY" "${work_dir}/post-apply-trust.yaml"
|
||||
assert_live_baseline
|
||||
wait_for_nodeport_boundary_and_health
|
||||
assert_manifest_https
|
||||
assert_trust_effective
|
||||
restore_required=false
|
||||
|
||||
printf '\nTRUST PHASE RE-APPLIED\n'
|
||||
printf 'Traefik web trusted CIDR: %s\n' "$declared_cidr"
|
||||
printf 'Gitea health and HTTPS site-manifest checks: PASS\n'
|
||||
@@ -12,3 +12,4 @@ Keycloak keycloak keycloak 2026-11-30 이미지 digest·startOptimized false·pl
|
||||
Deployment gitea gitea 2026-11-30 priorityClassName platform-critical(Gitea 재시작). 재시작 창에 apply-gitea-oidc.sh로 적용한다.
|
||||
StatefulSet observability loki 2026-11-30 priorityClassName platform-low(Loki 재시작). 재시작 창에 관측 core 적용으로.
|
||||
StatefulSet observability tempo 2026-11-30 priorityClassName platform-low(Tempo 재시작). 재시작 창에 관측 core 적용으로.
|
||||
PrometheusRule observability observability-core-kube-pr-kubernetes-system-kubelet 2026-11-30 Kubelet 인증서 경보 둘을 values에서 껐다(PR #6). 관측 core를 다시 적용하면 사라진다(재시작 창 2단계).
|
||||
|
||||
@@ -64,10 +64,29 @@ die() {
|
||||
work="$(mktemp -d /tmp/platform-drift.XXXXXX)"; made_dirs+=("$work")
|
||||
mkdir -p "$work/in" "$work/out"
|
||||
|
||||
# Render from a clean copy when run inside a git working tree. The observability
|
||||
# renderer refuses to run when an ignored chart cache (.helm/) is left over from an
|
||||
# earlier render, which happens in long-lived checkouts. The copy keeps tracked and
|
||||
# untracked-but-not-ignored files, so local edits are still checked.
|
||||
SOURCE_ROOT="$REPOSITORY_ROOT"
|
||||
if git -C "$REPOSITORY_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
SOURCE_ROOT="$work/src"
|
||||
mkdir -p "$SOURCE_ROOT"
|
||||
python3 -I - "$REPOSITORY_ROOT" <<'PY' | tar -C "$REPOSITORY_ROOT" --null -T - -cf - | tar -x -C "$SOURCE_ROOT" || die "could not copy the working tree"
|
||||
import subprocess, sys
|
||||
root = sys.argv[1]
|
||||
def ls(*args):
|
||||
out = subprocess.run(["git", "-C", root, "ls-files", "-z", *args], check=True, capture_output=True).stdout
|
||||
return [p for p in out.decode().split("\0") if p]
|
||||
keep = set(ls("--cached", "--others", "--exclude-standard")) - set(ls("--deleted"))
|
||||
sys.stdout.write("\0".join(sorted(keep)) + "\0")
|
||||
PY
|
||||
fi
|
||||
|
||||
render_root() {
|
||||
local script="$1" prefix="$2" dir
|
||||
dir="$(mktemp -d "/tmp/${prefix}.XXXXXX")"; made_dirs+=("$dir")
|
||||
bash "$REPOSITORY_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \
|
||||
bash "$SOURCE_ROOT/scripts/validate/${script}" --verified-output-dir "$dir" \
|
||||
>"$work/${script}.log" 2>&1 || { tail -5 "$work/${script}.log" >&2; die "${script} failed"; }
|
||||
printf '%s\n' "$dir"
|
||||
}
|
||||
@@ -86,10 +105,10 @@ declare -a inputs=(
|
||||
for f in "$phase2"/*.yaml "$admin"/*.yaml; do inputs+=("$f"); done
|
||||
|
||||
# Live Traefik is the trust overlay (infrastructure/networking/traefik/README.md).
|
||||
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed"
|
||||
kubectl kustomize "$REPOSITORY_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed"
|
||||
kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/traefik/overlays/trust" >"$work/traefik-trust.yaml" || die "traefik render failed"
|
||||
kubectl kustomize "$SOURCE_ROOT/services/tenants/techlog" >"$work/tenant-techlog.yaml" || die "tenant render failed"
|
||||
kubectl kustomize "$SOURCE_ROOT/services/priority-classes" >"$work/priority-classes.yaml" || die "priority class render failed"
|
||||
kubectl kustomize "$SOURCE_ROOT/infrastructure/networking/keycloak" >"$work/keycloak-network-policies.yaml" || die "keycloak policy render failed"
|
||||
inputs+=("$work/traefik-trust.yaml" "$work/tenant-techlog.yaml" "$work/priority-classes.yaml" "$work/keycloak-network-policies.yaml")
|
||||
|
||||
n=0
|
||||
|
||||
Reference in New Issue
Block a user