docs: prove sessions are shared by PostgreSQL, not Infinispan replication

Experiment 0 with three probes: cross-node refresh/logout, cache counter deltas around a single login, and cache entry ownership. Each node caches only what it handled; cache totals sum exactly to the database count.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

deploy/lab/scripts/experiment-cache-replication-delta.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Experiment 0b — does the Infinispan cache itself replicate, or do both nodes
+# merely agree because they read the same database?
+#
+# Experiment 0 proved the two nodes give the same answers. That alone does NOT
+# prove Infinispan replicated anything: with persistent-user-sessions (the
+# Keycloak 26 default) the session is written to PostgreSQL, so two nodes reading
+# one database would agree even with the cache disabled entirely.
+#
+# This script separates the two by measuring the cache counters on BOTH nodes
+# around a single login. If the write on keycloak-0 shows up as cache activity
+# on keycloak-1, the replication is real and not a database artifact.
+set -uo pipefail
+
+NS="${NS:-keycloak-lab}"
+K0_IP=$(kubectl -n "$NS" get pod keycloak-0 -o jsonpath='{.status.podIP}')
+K1_IP=$(kubectl -n "$NS" get pod keycloak-1 -o jsonpath='{.status.podIP}')
+ADMIN_PW=$(kubectl -n "$NS" get secret keycloak-lab-secrets \
+             -o jsonpath='{.data.KC_BOOTSTRAP_ADMIN_PASSWORD}' | base64 -d)
+
+echo "수집 시각: $(date '+%Y-%m-%d %H:%M:%S %Z')"
+echo
+
+# 파드 출력을 스트리밍으로 받으면 조각이 유실된다. 실제로 첫 시도에서
+# keycloak-1 의 스냅샷과 그 다음 마커가 통째로 사라져 델타가 0 으로 보였다.
+# 파드 안에서 파일로 모았다가 마지막에 한 번만 내보낸다.
+kubectl -n "$NS" run kc-delta --rm -i --restart=Never \
+  --image=curlimages/curl:8.11.1 --quiet --command -- sh -c "
+set -u
+K0='http://$K0_IP'; K1='http://$K1_IP'
+O=/tmp/o.txt; : > \$O
+snap() {
+  curl -s --retry 3 --retry-connrefused --max-time 20 \$1:9000/metrics \
+    | grep -E '^vendor_(statistics_(stores|hits|misses|approximate_entries_unique)|rpc_manager_replication_count)\{cache=\"(sessions|clientSessions)\"' \
+    | sed 's/,cache_manager=\"keycloak\"//; s/,node=\"[^\"]*\"//' >> \$O
+}
+echo '###BEFORE_K0' >> \$O; snap \$K0
+echo '###BEFORE_K1' >> \$O; snap \$K1
+echo '###LOGIN' >> \$O
+curl -s -o /dev/null -w 'http_code=%{http_code}\n' -X POST \
+  \"\$K0:8080/realms/master/protocol/openid-connect/token\" \
+  -d grant_type=password -d client_id=admin-cli \
+  -d username=admin -d 'password=$ADMIN_PW' >> \$O
+sleep 5
+echo '###AFTER_K0' >> \$O; snap \$K0
+echo '###AFTER_K1' >> \$O; snap \$K1
+echo '###END' >> \$O
+cat \$O
+" 2>&1 | grep -v '^pod .* deleted$' > /tmp/cache-delta.txt
+
+python3 - /tmp/cache-delta.txt <<'PY'
+import re, sys
+raw = open(sys.argv[1]).read()
+blocks, cur = {}, None
+for line in raw.splitlines():
+    if line.startswith('###'):
+        cur = line[3:]; blocks[cur] = {}
+    elif cur and '{' in line:
+        m = re.match(r'(\S+?)\{cache="(\w+)"\}\s+(\S+)', line)
+        if m:
+            blocks[cur][(m.group(1), m.group(2))] = float(m.group(3))
+
+print('=== 로그인은 keycloak-0 에만 보냈다 ===')
+code = [l for l in raw.splitlines() if l.startswith('http_code=')]
+print('  로그인 응답: ' + (code[0] if code else '없음'))
+for n in ('BEFORE_K0','BEFORE_K1','AFTER_K0','AFTER_K1'):
+    if not blocks.get(n):
+        print(f'  !! {n} 스냅샷이 비었다 — 델타를 신뢰할 수 없다')
+print()
+hdr = f"  {'계수기':<42} {'캐시':<15} {'전':>8} {'후':>8} {'증가':>7}"
+for node in ('K0', 'K1'):
+    who = 'keycloak-0 (로그인을 받은 노드)' if node == 'K0' else 'keycloak-1 (아무 요청도 받지 않은 노드)'
+    print(f'=== {who} ===')
+    print(hdr)
+    b, a = blocks.get(f'BEFORE_{node}', {}), blocks.get(f'AFTER_{node}', {})
+    for k in sorted(set(b) | set(a)):
+        before, after = b.get(k[0:2], 0.0), a.get(k[0:2], 0.0)
+        d = after - before
+        mark = '  ←' if d else ''
+        name = k[0].replace('vendor_statistics_', '').replace('vendor_rpc_manager_', 'rpc.')
+        print(f"  {name:<42} {k[1]:<15} {before:>8.0f} {after:>8.0f} {d:>+7.0f}{mark}")
+    print()
+PY