docs(keycloak-session-store): import the session-storage lab as a new project
The keycloak project ended with four open questions that design could not
settle. A two-VM lab was built to answer them by measurement, and this is
that material: 26 experiments, 125 raw command outputs, 22 browser captures.
Follows the import procedure in README.md.
source/ the originating repository verbatim — 78 documents, 28 SVGs,
8 manifests, plus .source-revision recording the commit
final/ the SSOT
document.md 729 lines written from the 29 experiment documents, not
concatenated: what was predicted, what was measured, and
where the measurement itself was wrong
evidence/raw 125 outputs, flattened to <experiment>__<file> because
the originals collided (01-baseline.txt appeared three
times) and the audit only globs the top level
evidence/meta one per raw file; command and exitCode are null and the
README says why rather than inventing them
evidence/browser 22 captures
assets/ three diagrams through techviz
.techviz/ their VizSpecs
A separate project rather than an addition to keycloak: the B-layer answers
that project's four questions, but the A, C and D layers are about cluster
failure, SSO and operations, and one document.md should hold one subject.
The four question records there can point here through 관계.
Recorded rather than papered over: only three of the 28 diagrams were
remade. The repository forbids hand-drawn SVG and forbids titles inside the
canvas; all 28 originals carry both, so converting them is redrawing, not
reformatting. They stay in source/ and the gap is written into the document.
verify-pipeline.py passes. audit-records.py reports no issues.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
co-authored by
Claude Opus 5
Claude Opus 5
parent
43bccd08a8
commit
b2963105a8
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the API image on this workstation and import it into each lab node's
|
||||
# containerd.
|
||||
#
|
||||
# k3s does not run Docker and the lab has no registry, so images are shipped as
|
||||
# a stream: docker save -> ssh through the lab host -> k3s ctr images import.
|
||||
# Every node needs its own copy because the scheduler may place the pod anywhere.
|
||||
#
|
||||
# ./deploy/lab/scripts/build-and-import.sh
|
||||
# IMAGE=keycloak-pattern-api:lab NODES="kc-lab-1" ./deploy/lab/scripts/build-and-import.sh
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${IMAGE:-keycloak-pattern-api:lab}"
|
||||
NODES="${NODES:-kc-lab-1 kc-lab-2}"
|
||||
LAB_HOST="${LAB_HOST:-test-server}"
|
||||
CONTEXT="${CONTEXT:-backend}"
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
cd "$repo_root"
|
||||
|
||||
echo "==> building ${IMAGE} from ${CONTEXT}/"
|
||||
docker build -t "$IMAGE" "$CONTEXT"
|
||||
|
||||
for node in $NODES; do
|
||||
echo "==> importing into ${node}"
|
||||
# Nested ssh: the workstation cannot reach the guests directly because they
|
||||
# sit behind the lab host's libvirt NAT. The lab host's ~/.ssh/config holds
|
||||
# the kc-lab-* aliases.
|
||||
docker save "$IMAGE" \
|
||||
| ssh "$LAB_HOST" "ssh ${node} 'sudo k3s ctr images import -'"
|
||||
done
|
||||
|
||||
echo "==> verifying"
|
||||
for node in $NODES; do
|
||||
printf ' %-10s ' "$node"
|
||||
ssh "$LAB_HOST" "ssh ${node} 'sudo k3s ctr images ls -q'" \
|
||||
| grep -c "$IMAGE" \
|
||||
| xargs -I{} echo "{} match(es)"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "next: kubectl rollout restart -n header-lab deployment/echo"
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Experiment 0c — where does a session entry actually live?
|
||||
#
|
||||
# Experiment 0b showed keycloak-1's session cache never moved when keycloak-0
|
||||
# handled a login. That leaves two explanations:
|
||||
#
|
||||
# (a) a DISTRIBUTED cache with owners=1 — entries are spread across nodes by
|
||||
# consistent hashing, and this one happened to land on keycloak-0;
|
||||
# (b) a LOCAL cache — each node only ever caches what it handled itself.
|
||||
#
|
||||
# They are distinguished by driving logins at the OTHER node. Under (a) the
|
||||
# entries would keep landing on both nodes regardless of who was asked. Under
|
||||
# (b) the count rises only on the node that received the request.
|
||||
set -uo pipefail
|
||||
|
||||
NS="${NS:-keycloak-lab}"
|
||||
N="${N:-5}"
|
||||
K0_IP=$(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.status.podIP}')
|
||||
K1_IP=$(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.status.podIP}')
|
||||
ADMIN_PW=$(kubectl -n "$NS" get secret keycloak-lab-secrets \
|
||||
-o jsonpath='{.data.KC_BOOTSTRAP_ADMIN_PASSWORD}' | base64 -d)
|
||||
|
||||
echo "수집 시각: $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
echo " keycloak-0 = $K0_IP ($(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.spec.nodeName}'))"
|
||||
echo " keycloak-1 = $K1_IP ($(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.spec.nodeName}'))"
|
||||
echo
|
||||
|
||||
kubectl -n "$NS" run kc-own --rm -i --restart=Never \
|
||||
--image=curlimages/curl:8.11.1 --quiet --command -- sh -c "
|
||||
O=/tmp/o; : > \$O
|
||||
ent() {
|
||||
curl -s --retry 3 --max-time 20 http://\$1:9000/metrics \
|
||||
| grep -E '^vendor_statistics_approximate_entries_unique.cache=.sessions' \
|
||||
| awk '{print \$NF}'
|
||||
}
|
||||
login() { i=0; while [ \$i -lt $N ]; do
|
||||
curl -s -o /dev/null -X POST http://\$1:8080/realms/master/protocol/openid-connect/token \
|
||||
-d grant_type=password -d client_id=admin-cli \
|
||||
-d username=admin -d 'password=$ADMIN_PW'
|
||||
i=\$((i+1)); done; sleep 5; }
|
||||
{
|
||||
printf '%-32s %12s %12s\n' '단계' 'k0 entries' 'k1 entries'
|
||||
printf '%-32s %12s %12s\n' '시작' \"\$(ent $K0_IP)\" \"\$(ent $K1_IP)\"
|
||||
login $K1_IP
|
||||
printf '%-32s %12s %12s\n' 'keycloak-1 에 로그인 ${N}회' \"\$(ent $K0_IP)\" \"\$(ent $K1_IP)\"
|
||||
login $K0_IP
|
||||
printf '%-32s %12s %12s\n' 'keycloak-0 에 로그인 ${N}회' \"\$(ent $K0_IP)\" \"\$(ent $K1_IP)\"
|
||||
} >> \$O
|
||||
cat \$O
|
||||
" 2>&1 | grep -v '^pod .* deleted$'
|
||||
|
||||
echo
|
||||
echo "=== 대조: PostgreSQL 에는 몇 건인가 ==="
|
||||
kubectl -n "$NS" exec deploy/postgres -- psql -U keycloak -d keycloak -tAc \
|
||||
"select count(*) from offline_user_session where offline_flag='0'" 2>/dev/null | sed 's/^/ online 세션 /'
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
# Experiment 0b — does the Infinispan cache itself replicate, or do both nodes
|
||||
# merely agree because they read the same database?
|
||||
#
|
||||
# Experiment 0 proved the two nodes give the same answers. That alone does NOT
|
||||
# prove Infinispan replicated anything: with persistent-user-sessions (the
|
||||
# Keycloak 26 default) the session is written to PostgreSQL, so two nodes reading
|
||||
# one database would agree even with the cache disabled entirely.
|
||||
#
|
||||
# This script separates the two by measuring the cache counters on BOTH nodes
|
||||
# around a single login. If the write on keycloak-0 shows up as cache activity
|
||||
# on keycloak-1, the replication is real and not a database artifact.
|
||||
set -uo pipefail
|
||||
|
||||
NS="${NS:-keycloak-lab}"
|
||||
K0_IP=$(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.status.podIP}')
|
||||
K1_IP=$(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.status.podIP}')
|
||||
ADMIN_PW=$(kubectl -n "$NS" get secret keycloak-lab-secrets \
|
||||
-o jsonpath='{.data.KC_BOOTSTRAP_ADMIN_PASSWORD}' | base64 -d)
|
||||
|
||||
echo "수집 시각: $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
echo
|
||||
|
||||
# 파드 출력을 스트리밍으로 받으면 조각이 유실된다. 실제로 첫 시도에서
|
||||
# keycloak-1 의 스냅샷과 그 다음 마커가 통째로 사라져 델타가 0 으로 보였다.
|
||||
# 파드 안에서 파일로 모았다가 마지막에 한 번만 내보낸다.
|
||||
kubectl -n "$NS" run kc-delta --rm -i --restart=Never \
|
||||
--image=curlimages/curl:8.11.1 --quiet --command -- sh -c "
|
||||
set -u
|
||||
K0='http://$K0_IP'; K1='http://$K1_IP'
|
||||
O=/tmp/o.txt; : > \$O
|
||||
snap() {
|
||||
curl -s --retry 3 --retry-connrefused --max-time 20 \$1:9000/metrics \
|
||||
| grep -E '^vendor_(statistics_(stores|hits|misses|approximate_entries_unique)|rpc_manager_replication_count)\{cache=\"(sessions|clientSessions)\"' \
|
||||
| sed 's/,cache_manager=\"keycloak\"//; s/,node=\"[^\"]*\"//' >> \$O
|
||||
}
|
||||
echo '###BEFORE_K0' >> \$O; snap \$K0
|
||||
echo '###BEFORE_K1' >> \$O; snap \$K1
|
||||
echo '###LOGIN' >> \$O
|
||||
curl -s -o /dev/null -w 'http_code=%{http_code}\n' -X POST \
|
||||
\"\$K0:8080/realms/master/protocol/openid-connect/token\" \
|
||||
-d grant_type=password -d client_id=admin-cli \
|
||||
-d username=admin -d 'password=$ADMIN_PW' >> \$O
|
||||
sleep 5
|
||||
echo '###AFTER_K0' >> \$O; snap \$K0
|
||||
echo '###AFTER_K1' >> \$O; snap \$K1
|
||||
echo '###END' >> \$O
|
||||
cat \$O
|
||||
" 2>&1 | grep -v '^pod .* deleted$' > /tmp/cache-delta.txt
|
||||
|
||||
python3 - /tmp/cache-delta.txt <<'PY'
|
||||
import re, sys
|
||||
raw = open(sys.argv[1]).read()
|
||||
blocks, cur = {}, None
|
||||
for line in raw.splitlines():
|
||||
if line.startswith('###'):
|
||||
cur = line[3:]; blocks[cur] = {}
|
||||
elif cur and '{' in line:
|
||||
m = re.match(r'(\S+?)\{cache="(\w+)"\}\s+(\S+)', line)
|
||||
if m:
|
||||
blocks[cur][(m.group(1), m.group(2))] = float(m.group(3))
|
||||
|
||||
print('=== 로그인은 keycloak-0 에만 보냈다 ===')
|
||||
code = [l for l in raw.splitlines() if l.startswith('http_code=')]
|
||||
print(' 로그인 응답: ' + (code[0] if code else '없음'))
|
||||
for n in ('BEFORE_K0','BEFORE_K1','AFTER_K0','AFTER_K1'):
|
||||
if not blocks.get(n):
|
||||
print(f' !! {n} 스냅샷이 비었다 — 델타를 신뢰할 수 없다')
|
||||
print()
|
||||
hdr = f" {'계수기':<42} {'캐시':<15} {'전':>8} {'후':>8} {'증가':>7}"
|
||||
for node in ('K0', 'K1'):
|
||||
who = 'keycloak-0 (로그인을 받은 노드)' if node == 'K0' else 'keycloak-1 (아무 요청도 받지 않은 노드)'
|
||||
print(f'=== {who} ===')
|
||||
print(hdr)
|
||||
b, a = blocks.get(f'BEFORE_{node}', {}), blocks.get(f'AFTER_{node}', {})
|
||||
for k in sorted(set(b) | set(a)):
|
||||
before, after = b.get(k[0:2], 0.0), a.get(k[0:2], 0.0)
|
||||
d = after - before
|
||||
mark = ' ←' if d else ''
|
||||
name = k[0].replace('vendor_statistics_', '').replace('vendor_rpc_manager_', 'rpc.')
|
||||
print(f" {name:<42} {k[1]:<15} {before:>8.0f} {after:>8.0f} {d:>+7.0f}{mark}")
|
||||
print()
|
||||
PY
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env bash
|
||||
# Experiment 0d — capture the actual SQL that the OTHER node runs.
|
||||
#
|
||||
# Experiments 0b/0c showed that session entries never appear in keycloak-1's
|
||||
# memory, yet keycloak-1 can use a session keycloak-0 created. The conclusion
|
||||
# "keycloak-1 reads it from PostgreSQL" was an inference, not an observation.
|
||||
#
|
||||
# This script turns on statement logging in PostgreSQL for a few seconds, sends
|
||||
# ONE refresh request to keycloak-1 for a session born on keycloak-0, and greps
|
||||
# the database log for that session id. If the inference is right, the SQL is
|
||||
# there, issued from keycloak-1's pod IP.
|
||||
#
|
||||
# It also checks whether serving that request makes keycloak-1 cache the session
|
||||
# — which sharpens "each node caches what it handled" from "what it logged in"
|
||||
# to "what it touched".
|
||||
set -uo pipefail
|
||||
|
||||
NS="${NS:-keycloak-lab}"
|
||||
PSQL="kubectl -n $NS exec deploy/postgres -- psql -U keycloak -d keycloak -tAc"
|
||||
|
||||
K0_IP=$(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.status.podIP}')
|
||||
K1_IP=$(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.status.podIP}')
|
||||
ADMIN_PW=$(kubectl -n "$NS" get secret keycloak-lab-secrets \
|
||||
-o jsonpath='{.data.KC_BOOTSTRAP_ADMIN_PASSWORD}' | base64 -d)
|
||||
|
||||
echo "수집 시각: $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
echo " keycloak-0 = $K0_IP (세션을 만드는 노드)"
|
||||
echo " keycloak-1 = $K1_IP (읽기만 하는 노드)"
|
||||
echo
|
||||
|
||||
# %h 를 넣어야 어느 파드가 보낸 질의인지 로그에서 구분된다.
|
||||
echo "=== PostgreSQL 문장 로깅을 켠다 ==="
|
||||
$PSQL "alter system set log_statement='all'" >/dev/null 2>&1
|
||||
$PSQL "alter system set log_line_prefix='%m [%p] %h '" >/dev/null 2>&1
|
||||
$PSQL "select pg_reload_conf()" >/dev/null 2>&1
|
||||
echo " log_statement = $($PSQL 'show log_statement' 2>/dev/null)"
|
||||
echo " log_line_prefix = $($PSQL 'show log_line_prefix' 2>/dev/null)"
|
||||
echo
|
||||
|
||||
# 로그 커서를 잡아둔다. 이 줄 수 이후만 본다.
|
||||
LOG_BEFORE=$(kubectl -n "$NS" logs deploy/postgres --tail=-1 2>/dev/null | wc -l)
|
||||
|
||||
RESULT=$(kubectl -n "$NS" run kc-readpath --rm -i --restart=Never \
|
||||
--image=curlimages/curl:8.11.1 --quiet --command -- sh -c "
|
||||
O=/tmp/o; : > \$O
|
||||
TOKEN_EP='/realms/master/protocol/openid-connect/token'
|
||||
jget() { sed -n \"s/.*\\\"\$1\\\":\\\"\\([^\\\"]*\\)\\\".*/\\1/p\"; }
|
||||
ent() {
|
||||
curl -s --retry 3 --max-time 20 http://\$1:9000/metrics \
|
||||
| grep -E '^vendor_statistics_approximate_entries_unique.cache=.sessions' | awk '{print \$NF}'
|
||||
}
|
||||
# keycloak-0 에서 로그인한다
|
||||
L=\$(curl -s -X POST \"http://$K0_IP:8080\$TOKEN_EP\" -d grant_type=password \
|
||||
-d client_id=admin-cli -d username=admin -d 'password=$ADMIN_PW')
|
||||
SID=\$(echo \"\$L\" | jget access_token | cut -d. -f2 | sed 's/\$/==/' | base64 -d 2>/dev/null | jget sid)
|
||||
RT=\$(echo \"\$L\" | jget refresh_token)
|
||||
echo \"SID=\$SID\" >> \$O
|
||||
echo \"K1_ENTRIES_BEFORE=\$(ent $K1_IP)\" >> \$O
|
||||
sleep 2
|
||||
# 반대편 노드에 refresh 를 딱 한 번 보낸다
|
||||
# 인용을 한 겹 더 쌓으면 curl 이 URL 을 통째로 못 읽는다. 실제로 000 이 나왔다.
|
||||
CODE=\$(curl -s -o /dev/null -w '%{http_code}' -X POST \
|
||||
\"http://$K1_IP:8080\$TOKEN_EP\" \
|
||||
-d grant_type=refresh_token -d client_id=admin-cli -d \"refresh_token=\$RT\")
|
||||
echo \"REFRESH_ON_K1=\$CODE\" >> \$O
|
||||
sleep 3
|
||||
echo \"K1_ENTRIES_AFTER=\$(ent $K1_IP)\" >> \$O
|
||||
cat \$O
|
||||
" 2>&1 | grep -v '^pod .* deleted$')
|
||||
|
||||
echo "=== 요청 ==="
|
||||
echo "$RESULT" | sed 's/^/ /'
|
||||
SID=$(echo "$RESULT" | sed -n 's/^SID=//p')
|
||||
|
||||
echo
|
||||
echo "=== PostgreSQL 문장 로깅을 끈다 ==="
|
||||
$PSQL "alter system reset log_statement" >/dev/null 2>&1
|
||||
$PSQL "alter system reset log_line_prefix" >/dev/null 2>&1
|
||||
$PSQL "select pg_reload_conf()" >/dev/null 2>&1
|
||||
echo " log_statement = $($PSQL 'show log_statement' 2>/dev/null)"
|
||||
|
||||
echo
|
||||
echo "=== keycloak-1 이 실제로 보낸 SQL 문장 ==="
|
||||
echo " (파라미터가 \$1 로 묶여 있어, sid 는 바로 아래 DETAIL 줄에 있다)"
|
||||
echo
|
||||
kubectl -n "$NS" logs deploy/postgres --tail=-1 2>/dev/null \
|
||||
| tail -n +$((LOG_BEFORE + 1)) \
|
||||
| grep -F "$K1_IP" | grep -E "LOG: execute" \
|
||||
| sed 's/.*execute [^:]*: //' | sed 's/^/ /' | head -12
|
||||
echo
|
||||
echo "=== 그 sid 를 언급한 SQL — 누가 보냈는가 ==="
|
||||
echo " 찾는 sid: $SID"
|
||||
echo
|
||||
kubectl -n "$NS" logs deploy/postgres --tail=-1 2>/dev/null \
|
||||
| tail -n +$((LOG_BEFORE + 1)) \
|
||||
| grep -F "$SID" \
|
||||
| sed -e "s/$K0_IP/[keycloak-0]/g" -e "s/$K1_IP/[keycloak-1]/g" \
|
||||
| cut -c1-220 \
|
||||
| head -20
|
||||
|
||||
echo
|
||||
echo "=== 요약: 파드별 질의 건수 ==="
|
||||
kubectl -n "$NS" logs deploy/postgres --tail=-1 2>/dev/null \
|
||||
| tail -n +$((LOG_BEFORE + 1)) \
|
||||
| grep -F "$SID" \
|
||||
| grep -oE "^[0-9-]+ [0-9:.]+ [A-Z]+ \[[0-9]+\] [0-9.]+" \
|
||||
| awk '{print $NF}' | sort | uniq -c \
|
||||
| sed -e "s/$K0_IP/[keycloak-0]/" -e "s/$K1_IP/[keycloak-1]/" -e 's/^/ /'
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env bash
|
||||
# Experiment 0 — is a session created on one Keycloak node usable on the other?
|
||||
#
|
||||
# Forming a cluster is not the same as sharing session state. The Infinispan log
|
||||
# says "cluster view (2)", but that only proves the members found each other.
|
||||
#
|
||||
# Design notes, learned the hard way:
|
||||
#
|
||||
# * Every probe has a CONTROL. A result from the far node means nothing unless
|
||||
# the same call against the issuing node is also measured. The first version
|
||||
# of this script reported "403 on keycloak-1" as if it were a replication
|
||||
# failure; the issuing node returned 403 too, and the cause was a missing
|
||||
# openid scope. Measure both, always.
|
||||
#
|
||||
# * Sessions are tracked by SID, not by count. Both the test login and the
|
||||
# admin API calls create sessions for the same user, so counts are noisy.
|
||||
# A specific session id either appears in a node's answer or it does not.
|
||||
#
|
||||
# * The probe is the REFRESH TOKEN grant, not userinfo. userinfo only validates
|
||||
# a signature and can succeed on a node that knows nothing about the session.
|
||||
# Refreshing requires the node to find the session, check it is alive, and
|
||||
# write back a new refresh time — it actually touches the session store.
|
||||
#
|
||||
# Talks to pod IPs directly: going through nginx/Traefik would hide which node
|
||||
# handled each request, which is the entire question.
|
||||
#
|
||||
# ./deploy/lab/scripts/experiment-session-replication.sh
|
||||
set -uo pipefail
|
||||
|
||||
NS="${NS:-keycloak-lab}"
|
||||
OUT="${OUT:-/tmp/session-replication}"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
PSQL="kubectl -n $NS exec deploy/postgres -- psql -U keycloak -d keycloak -tAc"
|
||||
|
||||
echo "수집 시각: $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
echo
|
||||
|
||||
K0_IP=$(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.status.podIP}')
|
||||
K1_IP=$(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.status.podIP}')
|
||||
K0_NODE=$(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.spec.nodeName}')
|
||||
K1_NODE=$(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.spec.nodeName}')
|
||||
ADMIN_PW=$(kubectl -n "$NS" get secret keycloak-lab-secrets \
|
||||
-o jsonpath='{.data.KC_BOOTSTRAP_ADMIN_PASSWORD}' | base64 -d)
|
||||
|
||||
echo "=== 대상 ==="
|
||||
printf ' keycloak-0 %-14s %s\n' "$K0_IP" "$K0_NODE"
|
||||
printf ' keycloak-1 %-14s %s\n' "$K1_IP" "$K1_NODE"
|
||||
echo
|
||||
|
||||
echo "=== [0] 실험 전 DB 세션 ==="
|
||||
$PSQL "select offline_flag, count(*) from offline_user_session group by offline_flag" 2>/dev/null \
|
||||
| sed 's/^/ offline_flag=/' || echo " (없음)"
|
||||
echo
|
||||
|
||||
# 파드 하나 안에서 전 단계를 실행한다. 단계마다 파드를 새로 띄우면 토큰을
|
||||
# 단계 사이로 넘길 수 없다.
|
||||
kubectl -n "$NS" run kc-probe --rm -i --restart=Never \
|
||||
--image=curlimages/curl:8.11.1 --quiet --command -- sh -c "
|
||||
set -u
|
||||
K0='http://$K0_IP:8080'; K1='http://$K1_IP:8080'
|
||||
TOKEN_EP='/realms/master/protocol/openid-connect/token'
|
||||
jget() { sed -n \"s/.*\\\"\$1\\\":\\\"\\([^\\\"]*\\)\\\".*/\\1/p\"; }
|
||||
|
||||
# ── [1] keycloak-0 에서 로그인. 이 노드가 세션의 출생지다 ──────────────────
|
||||
LOGIN=\$(curl -s -X POST \"\$K0\$TOKEN_EP\" \
|
||||
-d grant_type=password -d client_id=admin-cli \
|
||||
-d username=admin -d 'password=$ADMIN_PW')
|
||||
echo '###STEP1_LOGIN'; echo \"\$LOGIN\"
|
||||
|
||||
AT=\$(echo \"\$LOGIN\" | jget access_token)
|
||||
RT=\$(echo \"\$LOGIN\" | jget refresh_token)
|
||||
|
||||
# ── [2] 관리 API 조회용 토큰. 세션 오염을 피하려고 따로 하나만 더 만든다 ──
|
||||
ADMTOK=\$(curl -s -X POST \"\$K0\$TOKEN_EP\" \
|
||||
-d grant_type=password -d client_id=admin-cli \
|
||||
-d username=admin -d 'password=$ADMIN_PW' | jget access_token)
|
||||
CID=\$(curl -s -H \"Authorization: Bearer \$ADMTOK\" \
|
||||
\"\$K0/admin/realms/master/clients?clientId=admin-cli\" | jget id | head -1)
|
||||
|
||||
# ── [3] 두 노드에 같은 질문을 한다: admin-cli 의 세션 목록 ────────────────
|
||||
echo '###STEP3_SESSIONS_K0'
|
||||
curl -s -H \"Authorization: Bearer \$ADMTOK\" \
|
||||
\"\$K0/admin/realms/master/clients/\$CID/user-sessions?max=100\"
|
||||
echo
|
||||
echo '###STEP3_SESSIONS_K1'
|
||||
curl -s -H \"Authorization: Bearer \$ADMTOK\" \
|
||||
\"\$K1/admin/realms/master/clients/\$CID/user-sessions?max=100\"
|
||||
echo
|
||||
|
||||
# ── [4] 대조군: keycloak-0 이 발급한 refresh token 을 keycloak-0 에 쓴다 ──
|
||||
# 먼저 반대편에 써야 하므로 여기서는 쓰지 않고, 순서를 [5] 뒤로 미룬다.
|
||||
# refresh token 은 회전(rotation)되므로 한 번 쓰면 옛 것이 무효가 된다.
|
||||
# 따라서 '반대편 먼저'가 유일하게 의미 있는 순서다.
|
||||
|
||||
# ── [5] 시험군: keycloak-0 이 발급한 refresh token 을 keycloak-1 에 쓴다 ──
|
||||
echo '###STEP5_REFRESH_ON_K1'
|
||||
curl -s -w '\nhttp_code=%{http_code}\n' -X POST \"\$K1\$TOKEN_EP\" \
|
||||
-d grant_type=refresh_token -d client_id=admin-cli -d \"refresh_token=\$RT\"
|
||||
|
||||
RT2=\$(curl -s -X POST \"\$K1\$TOKEN_EP\" \
|
||||
-d grant_type=refresh_token -d client_id=admin-cli -d \"refresh_token=\$RT\" \
|
||||
| jget refresh_token)
|
||||
|
||||
# ── [6] 무효화가 반대 방향으로도 전파되는가 ───────────────────────────────
|
||||
# keycloak-1 에서 로그아웃시키고, keycloak-0 에서 갱신을 시도한다.
|
||||
echo '###STEP6_LOGOUT_VIA_K1'
|
||||
curl -s -o /dev/null -w 'http_code=%{http_code}\n' -X POST \"\$K1/realms/master/protocol/openid-connect/logout\" \
|
||||
-d client_id=admin-cli -d \"refresh_token=\$RT2\"
|
||||
|
||||
echo '###STEP7_REFRESH_ON_K0_AFTER_LOGOUT'
|
||||
curl -s -w '\nhttp_code=%{http_code}\n' -X POST \"\$K0\$TOKEN_EP\" \
|
||||
-d grant_type=refresh_token -d client_id=admin-cli -d \"refresh_token=\$RT2\"
|
||||
echo '###END'
|
||||
" > "$OUT/raw.txt" 2>&1
|
||||
|
||||
sed -i '/^pod .* deleted$/d' "$OUT/raw.txt"
|
||||
|
||||
python3 - "$OUT/raw.txt" <<'PY' | tee "$OUT/report.txt"
|
||||
import base64, json, sys
|
||||
|
||||
raw = open(sys.argv[1]).read()
|
||||
blocks, cur = {}, None
|
||||
for line in raw.splitlines():
|
||||
if line.startswith('###'):
|
||||
cur = line[3:]; blocks[cur] = []
|
||||
elif cur is not None:
|
||||
blocks[cur].append(line)
|
||||
get = lambda k: '\n'.join(blocks.get(k, [])).strip()
|
||||
|
||||
def j(s):
|
||||
try: return json.JSONDecoder().raw_decode(s.strip())[0]
|
||||
except Exception: return None
|
||||
|
||||
def claims(tok):
|
||||
p = tok.split('.')[1]; p += '=' * (-len(p) % 4)
|
||||
return json.loads(base64.urlsafe_b64decode(p))
|
||||
|
||||
login = j(get('STEP1_LOGIN'))
|
||||
if not login or 'access_token' not in login:
|
||||
print('로그인 실패:', get('STEP1_LOGIN')[:300]); sys.exit(1)
|
||||
|
||||
ac = claims(login['access_token'])
|
||||
rc = claims(login['refresh_token'])
|
||||
SID = ac['sid']
|
||||
print('=== [1] keycloak-0 에서 로그인 ===')
|
||||
print(f" sid {SID}")
|
||||
print(f" sub {ac.get('sub')}")
|
||||
print(f" iss {ac.get('iss')}")
|
||||
print(f" access 수명 {ac['exp']-ac['iat']}초")
|
||||
print(f" refresh 수명 {rc['exp']-rc['iat']}초 typ={rc.get('typ')}")
|
||||
print(f" refresh jti {rc.get('jti')}")
|
||||
|
||||
print()
|
||||
print('=== [3] 같은 sid 가 두 노드 모두에서 보이는가 ===')
|
||||
for step, who in (('STEP3_SESSIONS_K0', 'keycloak-0 (발급 노드)'),
|
||||
('STEP3_SESSIONS_K1', 'keycloak-1 (반대편)')):
|
||||
d = j(get(step))
|
||||
if d is None:
|
||||
print(f' {who:24} 파싱 실패: {get(step)[:120]}'); continue
|
||||
ids = [s.get('id') for s in d]
|
||||
mark = '보임 ✔' if SID in ids else '없음 ✘'
|
||||
print(f' {who:24} 세션 {len(ids)}개 중 대상 sid → {mark}')
|
||||
for s in d:
|
||||
if s.get('id') == SID:
|
||||
print(f" ipAddress={s.get('ipAddress')} start={s.get('start')} lastAccess={s.get('lastAccess')}")
|
||||
|
||||
def show(step, title, expect):
|
||||
print(); print(f'=== {title} ===')
|
||||
body = get(step)
|
||||
code = [l for l in body.splitlines() if l.startswith('http_code=')]
|
||||
code = code[0].split('=')[1] if code else '?'
|
||||
d = j(body)
|
||||
ok = '기대대로' if code == expect else f'기대({expect})와 다름'
|
||||
print(f' HTTP {code} ← {ok}')
|
||||
if d and 'access_token' in d:
|
||||
c = claims(d['access_token'])
|
||||
same = '동일 ✔' if c.get('sid') == SID else f"다름 ✘ ({c.get('sid')})"
|
||||
print(f' 새 토큰의 sid → {same}')
|
||||
elif d:
|
||||
print(f" error {d.get('error')}")
|
||||
print(f" error_description {d.get('error_description')}")
|
||||
|
||||
show('STEP5_REFRESH_ON_K1',
|
||||
'[5] keycloak-0 이 발급한 refresh token 을 keycloak-1 에 사용', '200')
|
||||
|
||||
print(); print('=== [6] keycloak-1 을 통해 로그아웃 ===')
|
||||
print(' ' + get('STEP6_LOGOUT_VIA_K1').strip())
|
||||
|
||||
show('STEP7_REFRESH_ON_K0_AFTER_LOGOUT',
|
||||
'[7] 로그아웃 후 keycloak-0 에서 갱신 시도 (무효화 전파)', '400')
|
||||
|
||||
open('/tmp/session-replication/sid.txt','w').write(SID)
|
||||
PY
|
||||
|
||||
SID=$(cat /tmp/session-replication/sid.txt 2>/dev/null)
|
||||
echo
|
||||
echo "=== [8] PostgreSQL 에서 그 sid 를 직접 확인 ==="
|
||||
echo " 대상 sid: $SID"
|
||||
$PSQL "select user_session_id, offline_flag, created_on, last_session_refresh
|
||||
from offline_user_session where user_session_id='$SID'" 2>/dev/null \
|
||||
| sed 's/^/ /' | grep -q . \
|
||||
&& $PSQL "select user_session_id||' | flag='||offline_flag||' | created='||created_on||' | refresh='||last_session_refresh
|
||||
from offline_user_session where user_session_id='$SID'" 2>/dev/null | sed 's/^/ /' \
|
||||
|| echo " 행 없음 — 로그아웃으로 삭제되었다"
|
||||
echo
|
||||
echo " 전체 세션 수: $($PSQL 'select count(*) from offline_user_session' 2>/dev/null)"
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# Measure what the nginx -> Traefik chain actually delivers to the application.
|
||||
#
|
||||
# docs/reverse-proxy-headers.md documents a single-hop nginx contract. The lab
|
||||
# runs two hops, so the forwarded headers are measured rather than assumed.
|
||||
# Run from anywhere that can resolve the lab hostnames.
|
||||
#
|
||||
# ./deploy/lab/scripts/measure-proxy-headers.sh
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${HOST:-app1.hyeonworks.com}"
|
||||
URL="https://${HOST}/api/echo"
|
||||
|
||||
jqf() {
|
||||
if command -v jq >/dev/null 2>&1; then jq "$@"; else python3 -m json.tool; fi
|
||||
}
|
||||
|
||||
echo "=== 1. baseline: what the app sees for a normal request ==="
|
||||
curl -s "$URL" | jqf '{
|
||||
scheme, secure, serverName, serverPort, requestUrl, remoteAddr,
|
||||
forwarded: .headers | with_entries(select(.key | startswith("x-forwarded") or . == "x-real-ip" or . == "forwarded"))
|
||||
}' 2>/dev/null || curl -s "$URL"
|
||||
|
||||
echo
|
||||
echo "=== 2. spoof test: client sends its own X-Forwarded-* ==="
|
||||
echo " a trusted boundary must overwrite these, not append to them"
|
||||
curl -s "$URL" \
|
||||
-H 'X-Forwarded-For: 1.2.3.4' \
|
||||
-H 'X-Forwarded-Proto: http' \
|
||||
-H 'X-Forwarded-Host: evil.example.com' \
|
||||
-H 'X-Real-IP: 1.2.3.4' \
|
||||
| jqf '.headers | with_entries(select(.key | startswith("x-forwarded") or . == "x-real-ip"))' 2>/dev/null
|
||||
|
||||
echo
|
||||
echo "=== 3. which pod answered (host nginx upstream distribution) ==="
|
||||
for _ in 1 2 3 4; do
|
||||
curl -s "$URL" | jqf -r '.headers["x-forwarded-server"] // "n/a"' 2>/dev/null
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== 4. plain HTTP is redirected, not proxied ==="
|
||||
curl -s -o /dev/null -w ' http -> %{http_code} %{redirect_url}\n' "http://${HOST}/api/echo"
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Rebuild a guest's cloud-init seed image and publish it into the libvirt pool.
|
||||
# Run on the lab host.
|
||||
#
|
||||
# ./rebuild-seed.sh 1
|
||||
#
|
||||
# The same content lives in three places: the source YAML, the ISO, and the
|
||||
# uploaded pool volume. Editing the YAML alone changes nothing, which is why
|
||||
# this is a script and not a set of remembered commands.
|
||||
#
|
||||
# A rebuilt seed only takes effect on a freshly created VM. cloud-init runs its
|
||||
# per-instance modules once per instance-id, so an existing guest ignores it.
|
||||
set -euo pipefail
|
||||
|
||||
N="${1:?usage: rebuild-seed.sh <1|2>}"
|
||||
CLOUD_DIR="${CLOUD_DIR:-$HOME/workspace/cloud}"
|
||||
POOL="${POOL:-default}"
|
||||
export LIBVIRT_DEFAULT_URI="${LIBVIRT_DEFAULT_URI:-qemu:///system}"
|
||||
|
||||
cd "$CLOUD_DIR"
|
||||
src="kc-lab-${N}.yaml"
|
||||
iso="seed-kc-lab-${N}.iso"
|
||||
meta="meta-kc-lab-${N}"
|
||||
|
||||
[ -f "$src" ] || { echo "missing $CLOUD_DIR/$src" >&2; exit 1; }
|
||||
|
||||
# A fresh instance-id makes cloud-init treat the guest as new and re-run the
|
||||
# per-instance modules.
|
||||
printf 'instance-id: kc-lab-%s-%s\nlocal-hostname: kc-lab-%s\n' \
|
||||
"$N" "$(date +%s)" "$N" > "$meta"
|
||||
|
||||
# NoCloud looks for a volume labelled cidata holding files named exactly
|
||||
# user-data and meta-data. -graft-points renames them inside the image so no
|
||||
# staging directory is needed.
|
||||
xorrisofs -quiet -output "$iso" -volid CIDATA -joliet -rock -graft-points \
|
||||
"/user-data=${src}" "/meta-data=${meta}"
|
||||
|
||||
size="$(stat -c%s "$iso")"
|
||||
virsh vol-delete --pool "$POOL" "$iso" >/dev/null 2>&1 || true
|
||||
virsh vol-create-as "$POOL" "$iso" "$size" --format raw >/dev/null
|
||||
virsh vol-upload --pool "$POOL" "$iso" "$iso"
|
||||
|
||||
echo "$iso published to pool '$POOL' ($size bytes)"
|
||||
echo "attach it as a virtio disk, not a SATA cdrom:"
|
||||
echo " --disk vol=${POOL}/${iso},device=disk,bus=virtio,readonly=on"
|
||||
echo "Debian genericcloud images carry no AHCI driver, so a SATA cdrom is invisible"
|
||||
echo "to the guest and cloud-init fails with no error anywhere."
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Confirm the lab infrastructure is intact. Run on the lab host.
|
||||
#
|
||||
# A 404 from the HTTPS entry point is the success signal: TLS terminated and the
|
||||
# request reached Traefik, which simply had no matching ingress rule. A 502 or a
|
||||
# refused connection means the chain is broken somewhere.
|
||||
set -uo pipefail
|
||||
|
||||
export LIBVIRT_DEFAULT_URI="${LIBVIRT_DEFAULT_URI:-qemu:///system}"
|
||||
HOSTS="${HOSTS:-auth.hyeonworks.com app1.hyeonworks.com app2.hyeonworks.com}"
|
||||
NODE_IPS="${NODE_IPS:-192.168.122.11 192.168.122.12}"
|
||||
fail=0
|
||||
|
||||
check() { # description, expected, actual
|
||||
if [ "$2" = "$3" ]; then printf ' ok %-34s %s\n' "$1" "$3"
|
||||
else printf ' FAIL %-34s got %s, want %s\n' "$1" "$3" "$2"; fail=1; fi
|
||||
}
|
||||
|
||||
echo "== guests =="
|
||||
for name in kc-lab-1 kc-lab-2; do
|
||||
check "$name" running "$(virsh domstate "$name" 2>/dev/null || echo absent)"
|
||||
done
|
||||
|
||||
echo "== k3s =="
|
||||
ready="$(kubectl get nodes --no-headers 2>/dev/null | grep -c ' Ready ')"
|
||||
check "nodes Ready" 2 "$ready"
|
||||
lb="$(kubectl -n kube-system get svc traefik \
|
||||
-o jsonpath='{.status.loadBalancer.ingress[*].ip}' 2>/dev/null | wc -w)"
|
||||
check "traefik node IPs" 2 "$lb"
|
||||
|
||||
echo "== host nginx =="
|
||||
check "service" active "$(systemctl is-active nginx)"
|
||||
check "cert renew timer" active "$(systemctl is-active certbot-renew.timer)"
|
||||
for ip in $NODE_IPS; do
|
||||
check "traefik $ip" 404 "$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://${ip}/")"
|
||||
done
|
||||
|
||||
echo "== public entry point =="
|
||||
for h in $HOSTS; do
|
||||
check "https://$h" 404 "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "https://${h}/")"
|
||||
check "tls verify $h" 0 "$(curl -s -o /dev/null -w '%{ssl_verify_result}' --max-time 8 "https://${h}/")"
|
||||
done
|
||||
check "http redirect" 301 "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "http://${HOSTS%% *}/")"
|
||||
|
||||
echo
|
||||
[ "$fail" -eq 0 ] && echo "lab is healthy" || echo "lab has failures"
|
||||
exit "$fail"
|
||||
Reference in New Issue
Block a user