feat: 2홉 구성 진행

.gitignore

@@ -12,3 +12,7 @@ frontend/dist/
 
 bff/target
 token-mediator/target
+
+# lab cloud-init contains a console password; keep the filled copy local
+deploy/lab/cloud-init/kc-lab.yaml
+deploy/lab/cloud-init/kc-lab-*.yaml

backend/src/main/java/com/example/keycloakpattern/ApiController.java

@@ -1,6 +1,8 @@
 package com.example.keycloakpattern;
 
+import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.springframework.security.core.annotation.AuthenticationPrincipal;
@@ -9,6 +11,8 @@ import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import jakarta.servlet.http.HttpServletRequest;
+
 @RestController
 @RequestMapping("/api")
 public class ApiController {
@@ -18,6 +22,35 @@ public class ApiController {
         return Map.of("status", "ok", "service", "keycloak-pattern-api");
     }
 
+    /**
+     * Reflects what actually reached the application after the proxy chain.
+     *
+     * <p>The reverse proxy contract is defined in {@code docs/reverse-proxy-headers.md}
+     * for a single nginx hop. The lab runs {@code nginx -> Traefik -> pod}, so this
+     * endpoint exists to measure the two-hop result instead of assuming it.
+     *
+     * <p>{@code scheme}, {@code secure} and {@code requestUrl} are the values Keycloak
+     * uses to build the {@code iss} claim and redirect URLs. If forwarded headers are
+     * lost or rewritten, the mismatch shows up here first.
+     */
+    @GetMapping("/echo")
+    public Map<String, Object> echo(HttpServletRequest request) {
+        Map<String, List<String>> headers = new LinkedHashMap<>();
+        for (String name : Collections.list(request.getHeaderNames())) {
+            headers.put(name.toLowerCase(), Collections.list(request.getHeaders(name)));
+        }
+
+        Map<String, Object> response = new LinkedHashMap<>();
+        response.put("headers", headers);
+        response.put("remoteAddr", request.getRemoteAddr());
+        response.put("scheme", request.getScheme());
+        response.put("secure", request.isSecure());
+        response.put("serverName", request.getServerName());
+        response.put("serverPort", request.getServerPort());
+        response.put("requestUrl", request.getRequestURL().toString());
+        return response;
+    }
+
     @GetMapping("/me")
     public Map<String, Object> currentUser(@AuthenticationPrincipal Jwt jwt) {
         Map<String, Object> response = new LinkedHashMap<>();

backend/src/main/java/com/example/keycloakpattern/SecurityConfig.java

@@ -17,7 +17,8 @@ public class SecurityConfig {
             .sessionManagement(session ->
                 session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
             .authorizeHttpRequests(authorize -> authorize
-                .requestMatchers("/actuator/health", "/actuator/health/**", "/api/public")
+                .requestMatchers("/actuator/health", "/actuator/health/**", "/api/public",
+                    "/api/echo")
                 .permitAll()
                 .anyRequest()
                 .authenticated())

backend/src/main/resources/application.yml

@@ -1,5 +1,10 @@
 server:
   port: ${SERVER_PORT:8081}
+  # Spring ignores X-Forwarded-* unless this is set, so scheme/secure/requestUrl
+  # report the raw connection by default. Keycloak has the same opt-in as
+  # KC_PROXY_HEADERS. Flipping this to "native" is what the two-hop measurement
+  # compares against.
+  forward-headers-strategy: ${SERVER_FORWARD_HEADERS_STRATEGY:none}
 
 spring:
   application:

backend/src/test/java/com/example/keycloakpattern/ApiSecurityTest.java

@@ -25,6 +25,18 @@ class ApiSecurityTest {
             .andExpect(jsonPath("$.status").value("ok"));
     }
 
+    @Test
+    void echoEndpointReflectsForwardedHeadersWithoutAuthentication() throws Exception {
+        mockMvc.perform(get("/api/echo")
+                .header("X-Forwarded-Proto", "https")
+                .header("X-Forwarded-Host", "app1.example.test"))
+            .andExpect(status().isOk())
+            .andExpect(jsonPath("$.headers['x-forwarded-proto'][0]").value("https"))
+            .andExpect(jsonPath("$.headers['x-forwarded-host'][0]").value("app1.example.test"))
+            .andExpect(jsonPath("$.requestUrl").exists())
+            .andExpect(jsonPath("$.remoteAddr").exists());
+    }
+
     @Test
     void protectedEndpointRejectsAnonymousRequests() throws Exception {
         mockMvc.perform(get("/api/me"))

deploy/lab/README.md

@@ -0,0 +1,130 @@
+# Session store lab
+
+세션 저장소·refresh token 경쟁·장애 복구를 검증하는 2노드 k3s 실험대.
+네 인증 패턴(AP1~AP4)을 가로지르는 공통층이므로 별도 축으로 관리한다.
+
+개념 설명은 [`docs/session-lab-concepts.md`](../../docs/session-lab-concepts.md)에
+누적한다. 이 문서는 절차만 담는다.
+
+## 토폴로지
+
+```
+  브라우저 / SSH  (tailnet)
+        │  https://{auth,app1,app2}.hyeonworks.com → 100.83.212.4
+        ▼
+  lab host  ── nginx :443  TLS 종료 · X-Forwarded-* 주입
+        │      nginx :80   301 → https
+        │
+        │  virbr0  192.168.122.0/24 (libvirt NAT)
+        ├──▶ kc-lab-1  .11  k3s server  Traefik :80
+        └──▶ kc-lab-2  .12  k3s agent   Traefik :80
+                            └──▶ Pod
+```
+
+`nginx → Traefik` **2홉**이 운영 구조와 같다는 점이 이 배치의 핵심이다.
+L7 프록시가 두 겹인 이유는 역할이 다르기 때문이다 — nginx는 바깥세상과의
+접점(TLS·인증서·헤더)을, Traefik은 클러스터 내부의 동적 라우팅을 맡는다.
+
+## 구성 요소
+
+| 경로 | 역할 |
+|---|---|
+| `cloud-init/kc-lab.yaml.example` | 게스트 부트스트랩 템플릿 |
+| `host/nginx-keycloak-lab.conf` | lab host의 `sites-available/keycloak-lab` |
+| `k8s/echo.yaml` | 2홉 헤더 계약 측정용 워크로드 |
+| `scripts/rebuild-seed.sh` | cloud-init 시드 ISO 재생성 + 풀 업로드 |
+| `scripts/build-and-import.sh` | 이미지 빌드 → 각 노드 containerd 반입 |
+| `scripts/measure-proxy-headers.sh` | 헤더 계약 실측 |
+| `scripts/verify-lab.sh` | 인프라 상태 점검 |
+
+## 상태 점검
+
+```bash
+./deploy/lab/scripts/verify-lab.sh      # lab host 에서
+```
+
+**`404`가 성공 신호다.** TLS가 종료되고 Traefik까지 도달했으나 매칭되는
+Ingress 규칙이 없다는 뜻이다. `502`나 연결 거부면 체인이 끊긴 것이다.
+
+## 첫 실험 — 2홉 헤더 계약
+
+[`docs/reverse-proxy-headers.md`](../../docs/reverse-proxy-headers.md)의 계약은
+nginx **1홉**을 가정하고 쓰였다. 실제 배치는 2홉이므로, nginx가 세팅한
+`X-Forwarded-*`를 Traefik이 그대로 넘기는지 덮어쓰는지 **측정해서 확인한다.**
+
+이 결론이 뒤의 모든 실험에 깔린다. Keycloak의 `iss` 클레임, redirect URL,
+쿠키 도메인 검증이 전부 이 헤더에 의존하기 때문이다.
+
+```bash
+# 워크스테이션: 이미지 빌드 후 두 노드에 반입
+./deploy/lab/scripts/build-and-import.sh
+
+# lab host: 배포
+kubectl apply -f deploy/lab/k8s/echo.yaml
+kubectl -n header-lab rollout status deployment/echo
+
+# 어디서든: 실측
+./deploy/lab/scripts/measure-proxy-headers.sh
+```
+
+관측 대상은 넷이다.
+
+1. `X-Forwarded-For` — Traefik이 **덧붙이는가 덮어쓰는가**
+2. `X-Forwarded-Proto` / `-Host` / `-Port` — 그대로 전달되는가
+3. **위조 내성** — 클라이언트가 직접 넣은 `X-Forwarded-*`가 앱까지 도달하는가
+4. `scheme` / `secure` / `requestUrl` — Keycloak이 URL을 만들 때 쓰는 값
+
+3번이 신뢰 경계의 핵심이다. 이 헤더들은 누구나 위조할 수 있는 평범한 HTTP
+헤더이므로, 신뢰 경계에 선 프록시가 **반드시 덮어써야** 한다.
+
+## 이미지 배포 경로
+
+k3s는 containerd를 쓰고 이 실험대에는 레지스트리가 없다.
+
+```
+워크스테이션  docker build → docker save
+      │ ssh (lab host 경유)
+      ▼
+게스트        sudo k3s ctr images import
+매니페스트    imagePullPolicy: Never
+```
+
+**두 노드 모두에 반입해야 한다.** 스케줄러가 어느 노드에 배치할지 모른다.
+Keycloak·PostgreSQL·Redis는 공식 이미지를 그대로 당겨오므로 이 경로가
+필요한 것은 자체 빌드 이미지뿐이다.
+
+**lab host에 Docker를 설치하지 않는다.** k3s의 containerd와 이미지 저장소가
+갈려서 `docker build`한 이미지를 k3s가 보지 못하게 된다.
+
+## 게스트 재생성
+
+파괴적 실험 후 초기화하는 경로다.
+
+```bash
+virsh destroy kc-lab-1
+virsh undefine kc-lab-1                       # --remove-all-storage 는 시드 ISO 까지 지운다
+virsh vol-delete --pool default kc-lab-1.qcow2
+
+./deploy/lab/scripts/rebuild-seed.sh 1        # user-data 를 고쳤을 때만
+
+virt-install --name kc-lab-1 --memory 3584 --vcpus 2 \
+  --disk size=20,backing_store=/var/lib/libvirt/images/base.qcow2 \
+  --disk vol=default/seed-kc-lab-1.iso,device=disk,bus=virtio,readonly=on \
+  --network network=default,mac=52:54:00:aa:bb:11 \
+  --import --os-variant debian12 --noautoconsole
+```
+
+시드는 **virtio 디스크**로 붙인다. `virt-install --cloud-init`은 시드를 SATA
+CD-ROM으로 붙이는데, Debian `genericcloud` 이미지는 크기를 줄이려고 물리
+하드웨어 드라이버를 제외해서 **AHCI 장치를 보지 못한다.** 그러면 cloud-init이
+데이터소스를 찾지 못하고 아무 오류도 남기지 않은 채 종료한다. 증상은
+hostname이 `localhost`로 남고 SSH가 `Permission denied (publickey)`로 거부되는
+것뿐이다.
+
+게스트에 들어갈 수 없을 때는 화면을 직접 뜬다.
+
+```bash
+virsh screenshot kc-lab-1 /tmp/kc1.ppm    # 확장자와 무관하게 PNG 로 저장된다
+```
+
+`localhost login:`이면 cloud-init 미실행, `kc-lab-1 login:`이면 실행된 것이다.

deploy/lab/cloud-init/kc-lab.yaml.example

@@ -0,0 +1,37 @@
+#cloud-config
+# Template for both lab guests. scripts/rebuild-seed.sh substitutes __NODE__
+# and bakes this into a CIDATA seed image.
+#
+# Copy to kc-lab.yaml and fill the two placeholders. The real file is ignored by
+# git because plain_text_passwd is a credential, however disposable.
+#
+# Indentation is spaces only. YAML forbids tabs, and cloud-init fails silently
+# on a parse error: the guest boots as "localhost" with no user and no way in.
+hostname: kc-lab-__NODE__
+fqdn: kc-lab-__NODE__
+manage_etc_hosts: true
+
+users:
+  - name: donghyeon
+    groups: [sudo]
+    shell: /bin/bash
+    # NOPASSWD is required: the k3s installer and the fault-injection scripts
+    # run non-interactively and would block on a password prompt.
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    # Console-only escape hatch. Without it, a cloud-init failure leaves a guest
+    # that cannot be logged into at all, so its own failure log is unreadable.
+    # ssh_pwauth stays false, so this never widens SSH exposure.
+    lock_passwd: false
+    plain_text_passwd: CHANGE_ME
+    ssh_authorized_keys:
+      # Lab host key: needed because automation runs from the lab host, where
+      # agent forwarding is not available.
+      - CHANGE_ME_LAB_HOST_PUBLIC_KEY
+      # Workstation key: lets ProxyJump reach the guest directly.
+      - CHANGE_ME_WORKSTATION_PUBLIC_KEY
+
+ssh_pwauth: false
+package_update: true
+packages:
+  - curl
+  - nftables

deploy/lab/host/nginx-keycloak-lab.conf

@@ -0,0 +1,56 @@
+# Lab entry point. Deployed on the lab host as
+#   /etc/nginx/sites-available/keycloak-lab
+# and symlinked from sites-enabled/.
+#
+# Arch does not ship the Debian sites-available convention, so nginx.conf needs
+#   include /etc/nginx/sites-enabled/*;
+# inside its http { } block before this file has any effect.
+#
+# This is the outer of two L7 hops. It terminates TLS and hands plain HTTP to
+# the Traefik instance running on each k3s node.
+
+upstream k3s_traefik {
+    # Sticky-session switch. Keycloak recommends affinity on AUTH_SESSION_ID;
+    # ip_hash is the cheap stand-in for a single-browser lab. Leaving it off is
+    # the interesting case: Infinispan still routes correctly, only slower.
+    # ip_hash;
+    server 192.168.122.11:80;
+    server 192.168.122.12:80;
+}
+
+server {
+    listen 80 default_server;
+    server_name _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen 443 ssl default_server;
+    http2 on;
+    server_name _;
+
+    # fullchain.pem, never cert.pem: omitting the intermediates passes on
+    # desktop browsers and fails on mobile and curl.
+    ssl_certificate     /etc/letsencrypt/live/auth.hyeonworks.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/auth.hyeonworks.com/privkey.pem;
+    ssl_protocols       TLSv1.2 TLSv1.3;
+
+    location / {
+        proxy_pass http://k3s_traefik;
+        proxy_http_version 1.1;
+
+        proxy_set_header Host              $host;
+        proxy_set_header X-Forwarded-Host  $host;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_set_header X-Forwarded-Port  443;
+
+        # $remote_addr, not $proxy_add_x_forwarded_for. This is the trust
+        # boundary: a client-supplied X-Forwarded-For must be discarded, not
+        # extended, or nothing downstream can rely on the value.
+        proxy_set_header X-Forwarded-For   $remote_addr;
+        proxy_set_header X-Real-IP         $remote_addr;
+
+        proxy_read_timeout 3600s;
+        proxy_send_timeout 3600s;
+    }
+}

deploy/lab/k8s/echo.yaml

@@ -0,0 +1,113 @@
+# Header echo workload for the two-hop proxy contract measurement.
+#
+#   browser -> host nginx (TLS termination) -> Traefik -> this pod
+#
+# The image is built from backend/ and imported straight into each node's
+# containerd, so imagePullPolicy must stay Never. See scripts/build-and-import.sh.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: header-lab
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: echo
+  namespace: header-lab
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: echo
+  template:
+    metadata:
+      labels:
+        app: echo
+    spec:
+      # One replica per node so the sticky-session switch on the host nginx
+      # upstream has something observable to route between.
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app: echo
+      containers:
+        - name: echo
+          image: keycloak-pattern-api:lab
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8081
+              name: http
+          env:
+            - name: SERVER_PORT
+              value: "8081"
+            # "none" makes the app report the raw connection, so scheme/secure/
+            # requestUrl show what arrives without any forwarded-header handling.
+            # Set to "native" and redeploy to see the same request interpreted
+            # with X-Forwarded-* honoured. Keycloak's KC_PROXY_HEADERS is the
+            # same opt-in, which is why measuring both sides matters here.
+            - name: SERVER_FORWARD_HEADERS_STRATEGY
+              value: "none"
+            # The JVM sizes its heap from the container limit, not the host.
+            - name: JAVA_TOOL_OPTIONS
+              value: "-XX:MaxRAMPercentage=70"
+            # /api/echo is permitAll, so the JWT decoder is never exercised.
+            # These stay pointed at the future Keycloak service name.
+            - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI
+              value: "https://auth.hyeonworks.com/realms/keycloak-patterns"
+            - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI
+              value: "https://auth.hyeonworks.com/realms/keycloak-patterns/protocol/openid-connect/certs"
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: http
+            initialDelaySeconds: 45
+            periodSeconds: 15
+          resources:
+            requests:
+              memory: 320Mi
+              cpu: 100m
+            limits:
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echo
+  namespace: header-lab
+spec:
+  selector:
+    app: echo
+  ports:
+    - port: 8081
+      targetPort: http
+      name: http
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: echo
+  namespace: header-lab
+spec:
+  # k3s ships Traefik as the default ingress controller. Keeping it is what
+  # makes this lab a faithful two-hop replica.
+  ingressClassName: traefik
+  rules:
+    - host: app1.hyeonworks.com
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: echo
+                port:
+                  number: 8081

deploy/lab/scripts/build-and-import.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Build the API image on this workstation and import it into each lab node's
+# containerd.
+#
+# k3s does not run Docker and the lab has no registry, so images are shipped as
+# a stream: docker save -> ssh through the lab host -> k3s ctr images import.
+# Every node needs its own copy because the scheduler may place the pod anywhere.
+#
+#   ./deploy/lab/scripts/build-and-import.sh
+#   IMAGE=keycloak-pattern-api:lab NODES="kc-lab-1" ./deploy/lab/scripts/build-and-import.sh
+set -euo pipefail
+
+IMAGE="${IMAGE:-keycloak-pattern-api:lab}"
+NODES="${NODES:-kc-lab-1 kc-lab-2}"
+LAB_HOST="${LAB_HOST:-test-server}"
+CONTEXT="${CONTEXT:-backend}"
+
+repo_root="$(git rev-parse --show-toplevel)"
+cd "$repo_root"
+
+echo "==> building ${IMAGE} from ${CONTEXT}/"
+docker build -t "$IMAGE" "$CONTEXT"
+
+for node in $NODES; do
+  echo "==> importing into ${node}"
+  # Nested ssh: the workstation cannot reach the guests directly because they
+  # sit behind the lab host's libvirt NAT. The lab host's ~/.ssh/config holds
+  # the kc-lab-* aliases.
+  docker save "$IMAGE" \
+    | ssh "$LAB_HOST" "ssh ${node} 'sudo k3s ctr images import -'"
+done
+
+echo "==> verifying"
+for node in $NODES; do
+  printf '  %-10s ' "$node"
+  ssh "$LAB_HOST" "ssh ${node} 'sudo k3s ctr images ls -q'" \
+    | grep -c "$IMAGE" \
+    | xargs -I{} echo "{} match(es)"
+done
+
+echo
+echo "next: kubectl rollout restart -n header-lab deployment/echo"

deploy/lab/scripts/measure-proxy-headers.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Measure what the nginx -> Traefik chain actually delivers to the application.
+#
+# docs/reverse-proxy-headers.md documents a single-hop nginx contract. The lab
+# runs two hops, so the forwarded headers are measured rather than assumed.
+# Run from anywhere that can resolve the lab hostnames.
+#
+#   ./deploy/lab/scripts/measure-proxy-headers.sh
+set -euo pipefail
+
+HOST="${HOST:-app1.hyeonworks.com}"
+URL="https://${HOST}/api/echo"
+
+jqf() {
+  if command -v jq >/dev/null 2>&1; then jq "$@"; else python3 -m json.tool; fi
+}
+
+echo "=== 1. baseline: what the app sees for a normal request ==="
+curl -s "$URL" | jqf '{
+  scheme, secure, serverName, serverPort, requestUrl, remoteAddr,
+  forwarded: .headers | with_entries(select(.key | startswith("x-forwarded") or . == "x-real-ip" or . == "forwarded"))
+}' 2>/dev/null || curl -s "$URL"
+
+echo
+echo "=== 2. spoof test: client sends its own X-Forwarded-* ==="
+echo "    a trusted boundary must overwrite these, not append to them"
+curl -s "$URL" \
+  -H 'X-Forwarded-For: 1.2.3.4' \
+  -H 'X-Forwarded-Proto: http' \
+  -H 'X-Forwarded-Host: evil.example.com' \
+  -H 'X-Real-IP: 1.2.3.4' \
+  | jqf '.headers | with_entries(select(.key | startswith("x-forwarded") or . == "x-real-ip"))' 2>/dev/null
+
+echo
+echo "=== 3. which pod answered (host nginx upstream distribution) ==="
+for _ in 1 2 3 4; do
+  curl -s "$URL" | jqf -r '.headers["x-forwarded-server"] // "n/a"' 2>/dev/null
+done
+
+echo
+echo "=== 4. plain HTTP is redirected, not proxied ==="
+curl -s -o /dev/null -w '  http  -> %{http_code} %{redirect_url}\n' "http://${HOST}/api/echo"

deploy/lab/scripts/rebuild-seed.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Rebuild a guest's cloud-init seed image and publish it into the libvirt pool.
+# Run on the lab host.
+#
+#   ./rebuild-seed.sh 1
+#
+# The same content lives in three places: the source YAML, the ISO, and the
+# uploaded pool volume. Editing the YAML alone changes nothing, which is why
+# this is a script and not a set of remembered commands.
+#
+# A rebuilt seed only takes effect on a freshly created VM. cloud-init runs its
+# per-instance modules once per instance-id, so an existing guest ignores it.
+set -euo pipefail
+
+N="${1:?usage: rebuild-seed.sh <1|2>}"
+CLOUD_DIR="${CLOUD_DIR:-$HOME/workspace/cloud}"
+POOL="${POOL:-default}"
+export LIBVIRT_DEFAULT_URI="${LIBVIRT_DEFAULT_URI:-qemu:///system}"
+
+cd "$CLOUD_DIR"
+src="kc-lab-${N}.yaml"
+iso="seed-kc-lab-${N}.iso"
+meta="meta-kc-lab-${N}"
+
+[ -f "$src" ] || { echo "missing $CLOUD_DIR/$src" >&2; exit 1; }
+
+# A fresh instance-id makes cloud-init treat the guest as new and re-run the
+# per-instance modules.
+printf 'instance-id: kc-lab-%s-%s\nlocal-hostname: kc-lab-%s\n' \
+  "$N" "$(date +%s)" "$N" > "$meta"
+
+# NoCloud looks for a volume labelled cidata holding files named exactly
+# user-data and meta-data. -graft-points renames them inside the image so no
+# staging directory is needed.
+xorrisofs -quiet -output "$iso" -volid CIDATA -joliet -rock -graft-points \
+  "/user-data=${src}" "/meta-data=${meta}"
+
+size="$(stat -c%s "$iso")"
+virsh vol-delete --pool "$POOL" "$iso" >/dev/null 2>&1 || true
+virsh vol-create-as "$POOL" "$iso" "$size" --format raw >/dev/null
+virsh vol-upload --pool "$POOL" "$iso" "$iso"
+
+echo "$iso published to pool '$POOL' ($size bytes)"
+echo "attach it as a virtio disk, not a SATA cdrom:"
+echo "  --disk vol=${POOL}/${iso},device=disk,bus=virtio,readonly=on"
+echo "Debian genericcloud images carry no AHCI driver, so a SATA cdrom is invisible"
+echo "to the guest and cloud-init fails with no error anywhere."

deploy/lab/scripts/verify-lab.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Confirm the lab infrastructure is intact. Run on the lab host.
+#
+# A 404 from the HTTPS entry point is the success signal: TLS terminated and the
+# request reached Traefik, which simply had no matching ingress rule. A 502 or a
+# refused connection means the chain is broken somewhere.
+set -uo pipefail
+
+export LIBVIRT_DEFAULT_URI="${LIBVIRT_DEFAULT_URI:-qemu:///system}"
+HOSTS="${HOSTS:-auth.hyeonworks.com app1.hyeonworks.com app2.hyeonworks.com}"
+NODE_IPS="${NODE_IPS:-192.168.122.11 192.168.122.12}"
+fail=0
+
+check() { # description, expected, actual
+  if [ "$2" = "$3" ]; then printf '  ok    %-34s %s\n' "$1" "$3"
+  else printf '  FAIL  %-34s got %s, want %s\n' "$1" "$3" "$2"; fail=1; fi
+}
+
+echo "== guests =="
+for name in kc-lab-1 kc-lab-2; do
+  check "$name" running "$(virsh domstate "$name" 2>/dev/null || echo absent)"
+done
+
+echo "== k3s =="
+ready="$(kubectl get nodes --no-headers 2>/dev/null | grep -c ' Ready ')"
+check "nodes Ready" 2 "$ready"
+lb="$(kubectl -n kube-system get svc traefik \
+      -o jsonpath='{.status.loadBalancer.ingress[*].ip}' 2>/dev/null | wc -w)"
+check "traefik node IPs" 2 "$lb"
+
+echo "== host nginx =="
+check "service" active "$(systemctl is-active nginx)"
+check "cert renew timer" active "$(systemctl is-active certbot-renew.timer)"
+for ip in $NODE_IPS; do
+  check "traefik $ip" 404 "$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://${ip}/")"
+done
+
+echo "== public entry point =="
+for h in $HOSTS; do
+  check "https://$h" 404 "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "https://${h}/")"
+  check "tls verify $h" 0 "$(curl -s -o /dev/null -w '%{ssl_verify_result}' --max-time 8 "https://${h}/")"
+done
+check "http redirect" 301 "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "http://${HOSTS%% *}/")"
+
+echo
+[ "$fail" -eq 0 ] && echo "lab is healthy" || echo "lab has failures"
+exit "$fail"