chore: 실행 환경 구성 문서 추가 및 수정
This commit is contained in:
DongHyeonka
+18
-6
@@ -18,15 +18,23 @@
|
||||
브라우저 / SSH (tailnet)
|
||||
│ https://{auth,app1,app2}.hyeonworks.com → 100.83.212.4
|
||||
▼
|
||||
lab host ── nginx :443 TLS 종료 · X-Forwarded-* 주입
|
||||
│ nginx :80 301 → https
|
||||
lab host ── nftables DNAT :80,:443 → 192.168.122.10
|
||||
│ (물리 호스트가 실험대를 위해 하는 일의 전부)
|
||||
│
|
||||
│ virbr0 192.168.122.0/24 (libvirt NAT)
|
||||
├──▶ kc-lab-1 .11 k3s server Traefik :80
|
||||
└──▶ kc-lab-2 .12 k3s agent Traefik :80
|
||||
└──▶ Pod
|
||||
├──▶ kc-lab-edge .10 nginx :443 TLS 종료 · X-Forwarded-* 주입
|
||||
│ │ nginx :80 301 → https
|
||||
│ │ certbot · 갱신 타이머 · deploy 훅
|
||||
│ ├──▶ kc-lab-1 .11 Traefik :80 ──▶ Pod
|
||||
│ └──▶ kc-lab-2 .12 Traefik :80 ──▶ Pod
|
||||
├──▶ kc-lab-1 .11 k3s server
|
||||
└──▶ kc-lab-2 .12 k3s agent
|
||||
```
|
||||
|
||||
**L7 홉은 두 겹 그대로다**(엣지 nginx → Traefik). 앞에 늘어난 것은 커널이
|
||||
하는 L4 전달 한 번뿐이고, 그 대가로 **인증서·nginx 설정·certbot 이 전부
|
||||
일회용 게스트 안**으로 들어갔다.
|
||||
|
||||
`nginx → Traefik` **2홉**이 운영 구조와 같다는 점이 이 배치의 핵심이다.
|
||||
L7 프록시가 두 겹인 이유는 역할이 다르기 때문이다 — nginx는 바깥세상과의
|
||||
접점(TLS·인증서·헤더)을, Traefik은 클러스터 내부의 동적 라우팅을 맡는다.
|
||||
@@ -36,7 +44,11 @@ L7 프록시가 두 겹인 이유는 역할이 다르기 때문이다 — nginx
|
||||
| 경로 | 역할 |
|
||||
|---|---|
|
||||
| `cloud-init/kc-lab.yaml.example` | 게스트 부트스트랩 템플릿 |
|
||||
| `host/nginx-keycloak-lab.conf` | lab host의 `sites-available/keycloak-lab` |
|
||||
| `edge/nginx-keycloak-lab.conf` | `kc-lab-edge` 의 `sites-available/keycloak-lab` |
|
||||
| `edge/reload-nginx.sh` | certbot deploy 훅. 없으면 갱신이 서빙에 반영되지 않는다 (D-4) |
|
||||
| `edge/lab-edge-dnat.nft` | 물리 호스트의 유일한 트래픽 규칙 |
|
||||
| `edge/lab-edge-dnat.service` | 위 규칙을 부팅 때 적용 |
|
||||
| `scripts/migrate-to-edge.sh` | 엣지 계층을 호스트에서 게스트로 옮긴다 |
|
||||
| `k8s/echo.yaml` | 2홉 헤더 계약 측정용 워크로드 |
|
||||
| `scripts/rebuild-seed.sh` | cloud-init 시드 ISO 재생성 + 풀 업로드 |
|
||||
| `scripts/build-and-import.sh` | 이미지 빌드 → 각 노드 containerd 반입 |
|
||||
|
||||
@@ -17,7 +17,12 @@ users:
|
||||
shell: /bin/bash
|
||||
# NOPASSWD is required: the k3s installer and the fault-injection scripts
|
||||
# run non-interactively and would block on a password prompt.
|
||||
sudo: ['ALL=(ALL) NOPASSWD:ALL']
|
||||
#
|
||||
# A string, not a list. The list form still boots, but `cloud-init schema -c`
|
||||
# (22.4.2 on the guests) rejects it and prints the whole users.0 block with
|
||||
# "is not valid under any of the given schemas" — naming no key. That makes
|
||||
# the guide's own validation step look broken when it is not.
|
||||
sudo: "ALL=(ALL) NOPASSWD:ALL"
|
||||
# Console-only escape hatch. Without it, a cloud-init failure leaves a guest
|
||||
# that cannot be logged into at all, so its own failure log is unreadable.
|
||||
# ssh_pwauth stays false, so this never widens SSH exposure.
|
||||
@@ -35,3 +40,9 @@ package_update: true
|
||||
packages:
|
||||
- curl
|
||||
- nftables
|
||||
# kc-lab-edge only. The k3s nodes do not need these, and the edge does not need
|
||||
# anything else — nginx terminates TLS and certbot renews the certificate, both
|
||||
# inside this disposable guest.
|
||||
# - nginx
|
||||
# - certbot
|
||||
# - python3-certbot-dns-cloudflare
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/sbin/nft -f
|
||||
# Forward the tailnet entry point to the edge guest.
|
||||
#
|
||||
# This is the ONLY lab traffic rule the physical host carries. Everything else
|
||||
# that used to live here — nginx config, certificates, certbot, the deploy hook
|
||||
# — now lives on kc-lab-edge and is destroyed with it.
|
||||
#
|
||||
# DNAT only, never SNAT. The guests' default route is the host, so replies come
|
||||
# back through here and conntrack reverses the translation on its own. Adding a
|
||||
# masquerade would rewrite the source and the edge would see 192.168.122.1 for
|
||||
# every client — which would silently invalidate the X-Forwarded-For contract
|
||||
# that this lab measures.
|
||||
#
|
||||
# PREROUTING nat runs before the routing decision, so this wins over any local
|
||||
# socket on :80/:443. That makes the cutover atomic and the rollback a single
|
||||
# `nft delete table ip lab_edge`.
|
||||
|
||||
table ip lab_edge
|
||||
delete table ip lab_edge
|
||||
|
||||
table ip lab_edge {
|
||||
chain prerouting {
|
||||
type nat hook prerouting priority dstnat; policy accept;
|
||||
iifname "tailscale0" tcp dport { 80, 443 } dnat to 192.168.122.10
|
||||
}
|
||||
|
||||
# libvirt's own forward rules accept RELATED,ESTABLISHED into the guest
|
||||
# subnet but not a NEW inbound connection. This runs ahead of them.
|
||||
chain forward {
|
||||
type filter hook forward priority filter - 10; policy accept;
|
||||
ip daddr 192.168.122.10 tcp dport { 80, 443 } ct state new accept
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Lab edge DNAT (tailnet :80/:443 -> kc-lab-edge)
|
||||
After=network-online.target libvirtd.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/sbin/nft -f /etc/nftables.d/lab-edge-dnat.nft
|
||||
ExecStop=/usr/sbin/nft delete table ip lab_edge
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -25,8 +25,10 @@ server {
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl default_server;
|
||||
http2 on;
|
||||
# The http2 parameter of listen, not the separate `http2 on;` directive:
|
||||
# that directive needs nginx >= 1.25.1 and the edge guest is Debian 12
|
||||
# (nginx 1.22). This form works on both and is what the lab actually runs.
|
||||
listen 443 ssl http2 default_server;
|
||||
server_name _;
|
||||
|
||||
# fullchain.pem, never cert.pem: omitting the intermediates passes on
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
# certbot deploy hook. Install as
|
||||
# /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh (chmod +x)
|
||||
#
|
||||
# deploy/ runs only when a certificate was actually renewed (RENEWED_LINEAGE is
|
||||
# set). post/ would run twice a day whether or not anything changed, reloading
|
||||
# nginx for nothing.
|
||||
#
|
||||
# Without this, D-4 measured the failure exactly: the renewal succeeds, the
|
||||
# timer reports SUCCESS, and the old certificate keeps being served for 38m25s
|
||||
# — with no error anywhere.
|
||||
nginx -t && nginx -s reload
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# Remove the lab's host layer from test-server. Packages stay.
|
||||
#
|
||||
# sudo bash deploy/lab/host/teardown-host.sh
|
||||
#
|
||||
# The host's sudo asks for a password, so run this in a terminal — not over a
|
||||
# non-interactive ssh, where sudo fails silently into an empty result.
|
||||
#
|
||||
# ★ Certificates are BACKED UP, NOT DELETED. Let's Encrypt allows 5 duplicate
|
||||
# certificates per week for the same name set, and this lab's names resolve to
|
||||
# a tailnet address (100.64.0.0/10, not routable from the public internet), so
|
||||
# an HTTP-01 reissue cannot be validated from here. Deleting the files turns a
|
||||
# free restore into a problem that has to be solved first. Restoring is:
|
||||
#
|
||||
# sudo tar xzf ~/letsencrypt-backup-<stamp>.tgz -C /etc
|
||||
set -u
|
||||
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
HOME_DIR="${SUDO_USER:+/home/$SUDO_USER}"
|
||||
HOME_DIR="${HOME_DIR:-$HOME}"
|
||||
|
||||
echo "===== 1) 인증서 백업 (지우지 않는다) ====="
|
||||
if [ -d /etc/letsencrypt ]; then
|
||||
out="$HOME_DIR/letsencrypt-backup-$STAMP.tgz"
|
||||
tar czf "$out" -C /etc letsencrypt
|
||||
chown "${SUDO_USER:-root}" "$out"
|
||||
echo "백업: $out ($(du -h "$out" | cut -f1))"
|
||||
echo "현재 인증서:"
|
||||
certbot certificates 2>/dev/null | grep -E "Certificate Name|Domains|Expiry Date" || true
|
||||
echo "검증 방식 (재발급이 되는지의 답):"
|
||||
grep -H authenticator /etc/letsencrypt/renewal/*.conf 2>/dev/null || echo " (renewal 설정 없음)"
|
||||
else
|
||||
echo "/etc/letsencrypt 없음 — 건너뜀"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "===== 2) nginx 실험대 설정 제거 ====="
|
||||
if [ -f /etc/nginx/sites-available/keycloak-lab ]; then
|
||||
cp /etc/nginx/sites-available/keycloak-lab "$HOME_DIR/keycloak-lab.nginx.$STAMP.bak"
|
||||
echo "백업: $HOME_DIR/keycloak-lab.nginx.$STAMP.bak"
|
||||
fi
|
||||
rm -fv /etc/nginx/sites-enabled/keycloak-lab
|
||||
rm -fv /etc/nginx/sites-available/keycloak-lab
|
||||
systemctl disable --now nginx
|
||||
echo
|
||||
|
||||
echo "===== 3) certbot 갱신 타이머 정지 ====="
|
||||
# 인증서 파일은 남기지만, 갱신 시도는 멈춘다. 지금 DNS 로는 HTTP-01 검증이
|
||||
# 실패하고, 실패가 로그에만 쌓이면서 「왜 안 되지」의 원인이 된다.
|
||||
systemctl disable --now certbot-renew.timer 2>/dev/null || true
|
||||
echo
|
||||
|
||||
echo "===== 4) 엣지 DNAT (있으면) ====="
|
||||
systemctl disable --now lab-edge-dnat.service 2>/dev/null || true
|
||||
rm -fv /etc/systemd/system/lab-edge-dnat.service /etc/nftables.d/lab-edge-dnat.nft
|
||||
systemctl daemon-reload
|
||||
nft delete table ip lab_edge 2>/dev/null || true
|
||||
echo
|
||||
|
||||
echo "===== 5) 확인 ====="
|
||||
echo "-- nginx: $(systemctl is-active nginx) / $(systemctl is-enabled nginx 2>&1)"
|
||||
echo "-- certbot timer: $(systemctl is-active certbot-renew.timer 2>&1) / $(systemctl is-enabled certbot-renew.timer 2>&1)"
|
||||
echo "-- 80/443 리스너:"; ss -tlnp | grep -E ':(80|443) ' || echo " (없음 — 정상)"
|
||||
echo "-- sites-enabled:"; ls -A /etc/nginx/sites-enabled 2>/dev/null || echo " (비었음 — 정상)"
|
||||
echo "-- letsencrypt:"; ls /etc/letsencrypt/live 2>/dev/null || echo " (없음)"
|
||||
echo "-- libvirt 도메인:"; virsh list --all 2>/dev/null | tail -n +3 | grep -v '^$' || echo " (없음 — 정상)"
|
||||
echo
|
||||
echo "완료. 패키지(nginx · libvirt · qemu · certbot · kubectl)와 base.qcow2 는 남아 있다."
|
||||
Reference in New Issue
Block a user