donghyeon.kang/platform-core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
main
platform-core/scripts/validate/test-observability-smoke.sh
T

1112 lines
69 KiB
Bash
Raw Permalink Blame History

#!/usr/bin/env bash
# Focused tests for the observability acceptance state machine. The production
# artifact is executed for CLI behavior and sourced for pure validators and
# lifecycle orchestration; no live command, sudo, Secret, or mutation is used.
set -Eeuo pipefail
set +x
umask 077
readonly TEST_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly SMOKE="$TEST_ROOT/scripts/validate/observability-smoke.sh"
TEST_WORK=''
RAW_DELETE_WORK=''
ASSERTIONS=0
fail() { printf 'OBSERVABILITY SMOKE TEST FAILURE: %s\n' "$*" >&2; exit 1; }
pass() { ASSERTIONS=$((ASSERTIONS + 1)); printf 'PASS: %s\n' "$1"; }
assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected=[$1] actual=[$2]"; }
assert_contains() { [[ "$1" == *"$2"* ]] || fail "$3"; }
assert_not_contains() { [[ "$1" != *"$2"* ]] || fail "$3"; }
expect_failure() { local rc=0; "$@" >/dev/null 2>&1 || rc=$?; (( rc != 0 )) || fail "expected failure: $*"; }
cleanup() {
trap - EXIT HUP INT TERM
case "$RAW_DELETE_WORK" in /tmp/platform-observability-smoke.????????) rm -rf -- "$RAW_DELETE_WORK" ;; esac
case "$TEST_WORK" in /tmp/platform-observability-smoke-test.??????) rm -rf -- "$TEST_WORK" ;; esac
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
[[ -f "$SMOKE" && ! -L "$SMOKE" ]] ||
fail 'production smoke is missing (RED: create observability-smoke.sh)'
TEST_WORK="$(mktemp -d /tmp/platform-observability-smoke-test.XXXXXX)"
chmod 0700 "$TEST_WORK"
readonly EXPECTED_DRY_RUN=$'OBSERVABILITY_SMOKE_DRY_RUN=PASS\nCHECK_01=Grafana public/private DNS exactness\nCHECK_02=Grafana certificate chain,hostname,one-SAN,expiry\nCHECK_03=LAN,Tailscale,denied-source,public-edge HTTPS status\nCHECK_04=Gitea,Grafana,Keycloak public metrics denial\nCHECK_05=unknown-SNI and Traefik NodePort boundary\nCHECK_06=exact Prometheus target allowlist and health\nCHECK_07=Gitea,Keycloak,CNPG,AIStor,Traefik metric contracts\nCHECK_08=untrusted-Pod AIStor and blackbox denial\nCHECK_09=public,private,internal,TLS blackbox series\nCHECK_10=Grafana datasource health and Loki-to-Tempo trace link\nCHECK_11=AIStor quota and recording-rule series\nCHECK_12=Grafana,Alertmanager restart persistence\nCHECK_13=Certbot dry-run deploy-hook and Nginx health\nHUMAN_OIDC=admin,viewer,denied,break-glass,session-revoke,relogin-denied\nHUMAN_SLACK=firing,resolved\nHUMAN_EXTERNAL_CLIENT=required\nTEMPORARY_OBJECTS=NetworkPolicy,Pod,AlertmanagerSilence,PrometheusRule\nPRESERVE=Secret,PVC,preexisting-Keycloak-membership\nMUTATION=NOT_REQUESTED'
# Mutation caught: moving a command check or external read before the no-arg
# branch makes the command trace non-empty and breaks the exact output.
mkdir -m 0700 "$TEST_WORK/bin"
for command_name in dirname kubectl curl dig openssl nc sudo; do
printf '#!/usr/bin/env bash\nprintf %s\\n "%s" >>"%s"\nexit 97\n' \
"'%s'" "$command_name" "$TEST_WORK/external.log" >"$TEST_WORK/bin/$command_name"
chmod 0755 "$TEST_WORK/bin/$command_name"
done
: >"$TEST_WORK/external.log"
dry_output="$(PATH="$TEST_WORK/bin:/usr/bin:/bin" bash "$SMOKE")"
assert_eq "$EXPECTED_DRY_RUN" "$dry_output" 'default exact plan'
[[ ! -s "$TEST_WORK/external.log" ]] || fail 'default invocation touched an external boundary'
pass 'default invocation is exact and no-contact'
# Mutation caught: accepting an extra/duplicated flag could accidentally enter
# an execute path the operator did not request.
for argument in --wrong --execute=1 --context default; do
rc=0; PATH="$TEST_WORK/bin:/usr/bin:/bin" bash "$SMOKE" $argument >/dev/null 2>&1 || rc=$?
assert_eq 1 "$rc" "unsupported argument status: $argument"
done
[[ ! -s "$TEST_WORK/external.log" ]] || fail 'bad CLI touched an external boundary'
pass 'CLI rejects every non-contract argument before contact'
override_rc=0
override_output="$(PLATFORM_OBSERVABILITY_SMOKE_KUBECTL=/tmp/not-kubectl bash "$SMOKE" 2>&1)" || override_rc=$?
assert_eq 1 "$override_rc" 'production override status'
assert_contains "$override_output" 'production boundary override is forbidden' 'production override message'
source_only_rc=0
source_only_output="$(PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 bash "$SMOKE" 2>&1)" || source_only_rc=$?
assert_eq 1 "$source_only_rc" 'source-only environment bypass status'
assert_contains "$source_only_output" 'production boundary override is forbidden' 'source-only environment bypass message'
pass 'production invocation rejects test seams'
# Source-only mode exposes pure validators without running main.
PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$SMOKE"
# Production command constants must themselves satisfy the non-symlink
# executable boundary used by execute preflight on this host.
declare -F validate_command_boundaries >/dev/null || fail 'command-boundary validator is missing'
validate_command_boundaries
pass 'production command boundaries resolve to fixed regular executables'
mkdir -m 0700 "$TEST_WORK/hostile-bin"
cat >"$TEST_WORK/hostile-bin/stat" <<'SH'
#!/usr/bin/env bash
printf 'hostile-stat\n' >>"${OBS_SMOKE_HOSTILE_PATH_LOG:?}"
exec /usr/bin/stat "$@"
SH
chmod 0755 "$TEST_WORK/hostile-bin/stat"
: >"$TEST_WORK/hostile-path.log"
RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)"
chmod 0700 "$RAW_DELETE_WORK"
: >"$RAW_DELETE_WORK/file"; chmod 0600 "$RAW_DELETE_WORK/file"
(
WORK="$RAW_DELETE_WORK"
PATH="$TEST_WORK/hostile-bin:/usr/bin:/bin" OBS_SMOKE_HOSTILE_PATH_LOG="$TEST_WORK/hostile-path.log" \
safe_work_file "$WORK/file"
) || fail 'fixed utility boundary setup'
[[ ! -s "$TEST_WORK/hostile-path.log" ]] || fail 'security utility resolved through hostile PATH'
rm -rf -- "$RAW_DELETE_WORK"; RAW_DELETE_WORK=''
pass 'security utility boundaries ignore hostile PATH'
# A signal after the long-lived API proxy PID is assigned but before its
# process-start identity is captured must be deferred. Otherwise EXIT cleanup
# cannot prove ownership and the proxy is leaked.
cat >"$TEST_WORK/bin/kubectl-proxy-signal" <<'SH'
#!/usr/bin/env bash
trap 'exit 0' TERM INT
while :; do /usr/bin/sleep 1; done
SH
chmod 0755 "$TEST_WORK/bin/kubectl-proxy-signal"
RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)"
chmod 0700 "$RAW_DELETE_WORK"
rm -f -- "$TEST_WORK/proxy-signal.marker" "$TEST_WORK/proxy-signal.pid"
proxy_signal_rc=0
(
WORK="$RAW_DELETE_WORK"; KUBECTL_BIN="$TEST_WORK/bin/kubectl-proxy-signal"
API_PROXY_PID=''; API_PROXY_START=''; API_PROXY_SOCKET=''
MUTATION_CRITICAL_DEPTH=0; DEFERRED_SIGNAL_RC=0
OBS_SMOKE_PROXY_SIGNAL_TARGET=$BASHPID
OBS_SMOKE_PROXY_SIGNAL_MARKER="$TEST_WORK/proxy-signal.marker"
export OBS_SMOKE_PROXY_SIGNAL_TARGET OBS_SMOKE_PROXY_SIGNAL_MARKER
process_start_time() {
if [[ ! -e "$OBS_SMOKE_PROXY_SIGNAL_MARKER" ]]; then
: >"$OBS_SMOKE_PROXY_SIGNAL_MARKER"
/bin/kill -INT "$OBS_SMOKE_PROXY_SIGNAL_TARGET"
fi
printf '12345'
}
trap on_exit EXIT
trap 'handle_termination_signal 130' INT
start_api_proxy
) || proxy_signal_rc=$?
assert_eq 130 "$proxy_signal_rc" 'API proxy deferred signal status'
[[ -f "$TEST_WORK/proxy-signal.marker" ]] || fail 'API proxy signal fixture did not reach identity capture'
proxy_pid="$(pgrep -f "^bash $TEST_WORK/bin/kubectl-proxy-signal" | head -n1 || true)"
if [[ -n "$proxy_pid" ]] && /bin/kill -0 "$proxy_pid" 2>/dev/null; then
/bin/kill -KILL "$proxy_pid" 2>/dev/null || true
fail 'API proxy process leaked across the signal ownership window'
fi
RAW_DELETE_WORK=''
pass 'API proxy signal window closes after PID start-time ownership capture'
declare -F validate_commit_environment >/dev/null || fail 'full repeat commit environment gate is missing'
(
EXPECTED_NODE_UID=''
NODE_FIXTURE_UID='node-uid-one'
validate_command_boundaries() { printf 'commands\n' >>"$TEST_WORK/commit-gate.log"; }
kubectl_bounded() {
printf '%q ' "$@" >>"$TEST_WORK/commit-gate.log"; printf '\n' >>"$TEST_WORK/commit-gate.log"
case "$1 $2" in
'config current-context') printf 'default' ;;
'config view') printf 'https://127.0.0.1:6443' ;;
'get --raw=/readyz') : ;;
'get node') jq -n --arg name "$EXPECTED_NODE" --arg uid "$NODE_FIXTURE_UID" '{metadata:{name:$name,uid:$uid},status:{conditions:[{type:"Ready",status:"True"}]}}' ;;
'auth can-i') printf 'yes' ;;
*) return 91 ;;
esac
}
: >"$TEST_WORK/commit-gate.log"
validate_commit_environment
validate_commit_environment
NODE_FIXTURE_UID='node-uid-replacement'
expect_failure validate_commit_environment
) || fail 'full repeat commit environment gate flow'
assert_eq 3 "$(grep -c '^commands$' "$TEST_WORK/commit-gate.log")" 'command boundary repeat count'
[[ "$(grep -c '^auth can-i ' "$TEST_WORK/commit-gate.log")" -gt 20 ]] || fail 'authority set was not repeated'
pass 'commit gate repeats commands, context, node identity, API and authority'
declare -F kubectl_wait_90_bounded >/dev/null || fail '90-second Kubernetes convergence wrapper is missing'
declare -F kubectl_wait_180_bounded >/dev/null || fail '180-second Kubernetes convergence wrapper is missing'
cat >"$TEST_WORK/bin/kubectl-convergence" <<'SH'
#!/usr/bin/env bash
set -Eeuo pipefail
/usr/bin/tr '\0' ' ' <"/proc/$PPID/cmdline" >>"${OBS_SMOKE_CONVERGENCE_LOG:?}"
printf '\n' >>"$OBS_SMOKE_CONVERGENCE_LOG"
printf '%q ' "$@" >>"$OBS_SMOKE_CONVERGENCE_LOG"; printf '\n' >>"$OBS_SMOKE_CONVERGENCE_LOG"
SH
chmod 0755 "$TEST_WORK/bin/kubectl-convergence"
: >"$TEST_WORK/convergence.log"
KUBECTL_BIN="$TEST_WORK/bin/kubectl-convergence" OBS_SMOKE_CONVERGENCE_LOG="$TEST_WORK/convergence.log" \
kubectl_wait_90_bounded -n observability wait --timeout=90s pod/example
KUBECTL_BIN="$TEST_WORK/bin/kubectl-convergence" OBS_SMOKE_CONVERGENCE_LOG="$TEST_WORK/convergence.log" \
kubectl_wait_180_bounded -n observability rollout status deployment/grafana --timeout=180s
convergence_log="$(<"$TEST_WORK/convergence.log")"
assert_contains "$convergence_log" '--kill-after=2s 100s' '90-second outer convergence deadline'
assert_contains "$convergence_log" '--request-timeout=95s -n observability wait --timeout=90s pod/example' '90-second request deadline'
assert_contains "$convergence_log" '--kill-after=2s 190s' '180-second outer convergence deadline'
assert_contains "$convergence_log" '--request-timeout=185s -n observability rollout status deployment/grafana --timeout=180s' '180-second request deadline'
KUBECTL_BIN="$KUBECTL_DEFAULT"
pass 'Kubernetes convergence honors declared waits within fixed outer deadlines'
# A second-resolution timestamp cannot identify ownership under concurrent
# runs. Production must add a cryptographically random, non-secret nonce.
declare -F new_smoke_run_id >/dev/null || fail 'unique smoke run-id generator is missing'
run_id_one="$(new_smoke_run_id network)"
run_id_two="$(new_smoke_run_id network)"
[[ "$run_id_one" =~ ^network-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}$ ]] || fail 'first smoke run ID is not exact'
[[ "$run_id_two" =~ ^network-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}$ ]] || fail 'second smoke run ID is not exact'
[[ "$run_id_one" != "$run_id_two" ]] || fail 'concurrent smoke run IDs collided'
pass 'temporary object identity uses a unique non-secret nonce'
# The full run ID is safe as a label value but, when prefixed again, exceeds
# Kubernetes' 63-character DNS label limit. Resource names must retain the
# full 128-bit nonce while remaining valid exact DNS labels.
declare -F kubernetes_smoke_name >/dev/null || fail 'bounded Kubernetes smoke-name helper is missing'
kubernetes_name="$(kubernetes_smoke_name "$run_id_one")"
[[ "$kubernetes_name" =~ ^observability-smoke-[0-9a-f]{32}$ ]] || fail 'Kubernetes smoke name is not nonce-bound'
(( ${#kubernetes_name} <= 63 )) || fail 'Kubernetes smoke name exceeds DNS label length'
expect_failure kubernetes_smoke_name 'network-20260812t091011z-not-a-nonce'
pass 'temporary Kubernetes names retain ownership entropy within DNS limits'
# Mutation caught: accepting public/private DNS drift or more than one SAN.
validate_dns_contract '' '' 192.168.0.107 '' 100.92.240.34 '' 192.168.0.107 ''
expect_failure validate_dns_contract 203.0.113.10 '' 192.168.0.107 '' 100.92.240.34 '' 192.168.0.107 ''
expect_failure validate_dns_contract '' '' 192.168.0.108 '' 100.92.240.34 '' 192.168.0.107 ''
validate_certificate_contract $'DNS:grafana.learn.hyeonworks.com' 0 0
expect_failure validate_certificate_contract $'DNS:grafana.learn.hyeonworks.com\nDNS:extra.example' 0 0
expect_failure validate_certificate_contract 'DNS:grafana.learn.hyeonworks.com' 1 0
expect_failure validate_certificate_contract 'DNS:grafana.learn.hyeonworks.com' 0 1
declare -F validate_certificate_extension_contract >/dev/null ||
fail 'full certificate GeneralNames validator is missing'
validate_certificate_extension_contract \
$'X509v3 Subject Alternative Name:\n DNS:grafana.learn.hyeonworks.com' 0 0
for extra_name in \
'IP Address:192.168.0.107' \
'email:admin@learn.hyeonworks.com' \
'URI:https://grafana.learn.hyeonworks.com/' \
'othername: UPN::grafana'; do
expect_failure validate_certificate_extension_contract \
"X509v3 Subject Alternative Name:
DNS:grafana.learn.hyeonworks.com, $extra_name" 0 0
done
pass 'DNS and certificate validators reject drift'
# Production machine leaf: exercise its real bounded dig argv construction,
# normalization, and response-to-contract flow at the command boundary.
cat >"$TEST_WORK/bin/dig-leaf" <<'SH'
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%q ' "$@" >>"${OBS_SMOKE_LEAF_LOG:?}"
printf '\n' >>"${OBS_SMOKE_LEAF_LOG:?}"
[[ "$1" == +time=2 && "$2" == +tries=1 && "$3" == +short && "$6" == grafana.learn.hyeonworks.com ]]
resolver=${4#@}; record=$5
if [[ "${OBS_SMOKE_LEAF_DRIFT:-0}" == 1 && "$resolver" == 192.168.0.107 && "$record" == A ]]; then
printf '192.168.0.108\n'
elif [[ "$resolver" == 192.168.0.107 && "$record" == A ]]; then
printf '\n192.168.0.107\n192.168.0.107\n'
elif [[ "$resolver" == 100.92.240.34 && "$record" == A ]]; then
printf '100.92.240.34\n'
fi
SH
chmod 0755 "$TEST_WORK/bin/dig-leaf"
: >"$TEST_WORK/dig-leaf.log"
DIG_BIN="$TEST_WORK/bin/dig-leaf"
OBS_SMOKE_LEAF_LOG="$TEST_WORK/dig-leaf.log" check_dns_host >/dev/null
assert_eq 6 "$(wc -l <"$TEST_WORK/dig-leaf.log" | tr -d '[:space:]')" 'DNS leaf command count'
assert_contains "$(<"$TEST_WORK/dig-leaf.log")" '+time=2 +tries=1 +short @1.1.1.1 A grafana.learn.hyeonworks.com' 'public A argv contract'
assert_contains "$(<"$TEST_WORK/dig-leaf.log")" '+time=2 +tries=1 +short @192.168.0.107 AAAA grafana.learn.hyeonworks.com' 'LAN AAAA argv contract'
assert_contains "$(<"$TEST_WORK/dig-leaf.log")" '+time=2 +tries=1 +short @100.92.240.34 A grafana.learn.hyeonworks.com' 'Tailscale A argv contract'
expect_failure env OBS_SMOKE_LEAF_LOG="$TEST_WORK/dig-leaf.log" OBS_SMOKE_LEAF_DRIFT=1 \
bash -c 'PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"; DIG_BIN="$2"; check_dns_host' \
bash "$SMOKE" "$TEST_WORK/bin/dig-leaf"
DIG_BIN="$DIG_DEFAULT"
pass 'production DNS leaf uses bounded exact argv and rejects parsed drift'
cat >"$TEST_WORK/traefik-service.json" <<'JSON'
{"apiVersion":"v1","kind":"Service","metadata":{"name":"traefik","namespace":"kube-system","uid":"traefik-service-uid"},"spec":{"type":"NodePort","externalTrafficPolicy":"Cluster","selector":{"app.kubernetes.io/name":"traefik","app.kubernetes.io/instance":"traefik-kube-system"},"ports":[{"name":"web","port":80,"protocol":"TCP","targetPort":"web","nodePort":30080},{"name":"websecure","port":443,"protocol":"TCP","targetPort":"websecure","nodePort":30443}]}}
JSON
chmod 0600 "$TEST_WORK/traefik-service.json"
declare -F check_nodeport_identity >/dev/null || fail 'Traefik NodePort identity leaf is missing'
(
WORK="$TEST_WORK"
kubectl_bounded() { cat -- "$TEST_WORK/traefik-service.json"; }
http_status() {
printf '%q ' "$@" >"$TEST_WORK/nodeport-http.argv"
printf '404'
}
check_nodeport_identity >/dev/null
) || fail 'exact Traefik NodePort identity flow'
assert_contains "$(<"$TEST_WORK/nodeport-http.argv")" '--header Host:\ observability-smoke.invalid http://127.0.0.1:30080/' 'NodePort exact HTTP argv'
expect_failure bash -c '
PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"
WORK="$(dirname -- "$2")"
SERVICE_FIXTURE=$2
kubectl_bounded() { cat -- "$SERVICE_FIXTURE"; }
http_status() { printf 200; }
check_nodeport_identity
' bash "$SMOKE" "$TEST_WORK/traefik-service.json"
jq '.spec.selector["app.kubernetes.io/name"]="foreign"' "$TEST_WORK/traefik-service.json" >"$TEST_WORK/traefik-service-foreign.json"
expect_failure bash -c '
PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"
WORK="$(dirname -- "$2")"
SERVICE_FIXTURE=$2
kubectl_bounded() { cat -- "$SERVICE_FIXTURE"; }
http_status() { printf 404; }
check_nodeport_identity
' bash "$SMOKE" "$TEST_WORK/traefik-service-foreign.json"
pass 'loopback NodePort proves exact Traefik Service and HTTP response'
# A hostile ambient curlrc must not influence either public HTTPS calls or the
# private Unix-socket API client. `--disable` has to be curl's first argument.
mkdir -m 0700 "$TEST_WORK/curl-home"
cat >"$TEST_WORK/curl-home/.curlrc" <<EOF
trace-ascii = "$TEST_WORK/hostile-curl.trace"
header = "X-Ambient-Curlrc: forbidden"
EOF
chmod 0600 "$TEST_WORK/curl-home/.curlrc"
HOME="$TEST_WORK/curl-home" CURL_HOME="$TEST_WORK/curl-home" \
http_status 'file:///etc/hosts' >/dev/null
[[ ! -e "$TEST_WORK/hostile-curl.trace" ]] || fail 'http_status loaded hostile curlrc'
RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)"
chmod 0700 "$RAW_DELETE_WORK"
proxy_rc=0
(
WORK="$RAW_DELETE_WORK"
API_PROXY_SOCKET="$WORK/nonexistent.sock"
HOME="$TEST_WORK/curl-home" CURL_HOME="$TEST_WORK/curl-home" \
proxy_request GET '/api/v1/namespaces/observability/pods' "$WORK/output.json"
) >/dev/null 2>&1 || proxy_rc=$?
assert_eq 1 "$proxy_rc" 'failed-socket proxy status'
[[ ! -e "$TEST_WORK/hostile-curl.trace" ]] || fail 'proxy_request loaded hostile curlrc'
rm -rf -- "$RAW_DELETE_WORK"
RAW_DELETE_WORK=''
pass 'curl boundaries ignore hostile ambient curlrc'
cat >"$TEST_WORK/targets.json" <<'JSON'
{"status":"success","data":{"activeTargets":[
{"scrapePool":"podMonitor/platform-data/platform-postgres/0","health":"up"},
{"scrapePool":"probe/observability/platform-private-edge","health":"up"},
{"scrapePool":"probe/observability/platform-private-edge","health":"up"},
{"scrapePool":"probe/observability/platform-private-edge","health":"up"},
{"scrapePool":"probe/observability/platform-private-internal","health":"up"},
{"scrapePool":"probe/observability/platform-private-internal","health":"up"},
{"scrapePool":"probe/observability/platform-private-internal","health":"up"},
{"scrapePool":"probe/observability/platform-public-edge","health":"up"},
{"scrapePool":"probe/observability/platform-public-edge","health":"up"},
{"scrapePool":"serviceMonitor/gitea/gitea/0","health":"up"},
{"scrapePool":"serviceMonitor/keycloak/keycloak/0","health":"up"},
{"scrapePool":"serviceMonitor/kube-system/traefik/0","health":"up"},
{"scrapePool":"serviceMonitor/object-storage/aistor-bucket-usage/0","health":"up"},
{"scrapePool":"serviceMonitor/observability-agent/alloy/0","health":"up"},
{"scrapePool":"serviceMonitor/observability-agent/node-exporter/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/blackbox-exporter/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/grafana/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/loki/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-apiserver/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-coredns/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-kubelet/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-kubelet/1","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-kubelet/2","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-operator/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-prometheus/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-pr-prometheus/1","health":"up"},
{"scrapePool":"serviceMonitor/observability/observability-core-kube-state-metrics/0","health":"up"},
{"scrapePool":"serviceMonitor/observability/tempo/0","health":"up"}
]}}
JSON
python3 -I -S - "$TEST_WORK/targets.json" <<'PY'
import json,pathlib,sys
path=pathlib.Path(sys.argv[1]); item=json.loads(path.read_text())
urls={
'podMonitor/platform-data/platform-postgres/0':'http://10.42.0.10:9187/metrics',
'serviceMonitor/gitea/gitea/0':'http://10.42.0.11:3000/metrics',
'serviceMonitor/keycloak/keycloak/0':'http://10.42.0.12:9000/metrics',
'serviceMonitor/kube-system/traefik/0':'http://10.42.0.13:9100/metrics',
'serviceMonitor/object-storage/aistor-bucket-usage/0':'http://10.42.0.14:9000/minio/metrics/v3/cluster/usage/buckets',
'serviceMonitor/observability-agent/alloy/0':'http://10.42.0.21:12345/metrics',
'serviceMonitor/observability-agent/node-exporter/0':'http://192.168.0.107:9100/metrics',
'serviceMonitor/observability/blackbox-exporter/0':'http://10.42.0.22:9115/metrics',
'serviceMonitor/observability/grafana/0':'http://10.42.0.23:3000/metrics',
'serviceMonitor/observability/loki/0':'http://10.42.0.24:3100/metrics',
'serviceMonitor/observability/observability-core-kube-pr-alertmanager/0':'http://10.42.0.25:9093/metrics',
'serviceMonitor/observability/observability-core-kube-pr-alertmanager/1':'http://10.42.0.25:8080/metrics',
'serviceMonitor/observability/observability-core-kube-pr-apiserver/0':'https://192.168.0.107:6443/metrics',
'serviceMonitor/observability/observability-core-kube-pr-coredns/0':'http://10.42.0.26:9153/metrics',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/0':'https://192.168.0.107:10250/metrics',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/1':'https://192.168.0.107:10250/metrics/cadvisor',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/2':'https://192.168.0.107:10250/metrics/probes',
'serviceMonitor/observability/observability-core-kube-pr-operator/0':'http://10.42.0.27:8080/metrics',
'serviceMonitor/observability/observability-core-kube-pr-prometheus/0':'http://10.42.0.28:9090/metrics',
'serviceMonitor/observability/observability-core-kube-pr-prometheus/1':'http://10.42.0.28:8080/metrics',
'serviceMonitor/observability/observability-core-kube-state-metrics/0':'http://10.42.0.29:8080/metrics',
'serviceMonitor/observability/tempo/0':'http://10.42.0.30:3200/metrics',
}
jobs={
'podMonitor/platform-data/platform-postgres/0':'platform-postgres',
'serviceMonitor/gitea/gitea/0':'gitea',
'serviceMonitor/keycloak/keycloak/0':'keycloak-service',
'serviceMonitor/kube-system/traefik/0':'traefik',
'serviceMonitor/object-storage/aistor-bucket-usage/0':'minio-aistor',
'serviceMonitor/observability-agent/alloy/0':'alloy',
'serviceMonitor/observability-agent/node-exporter/0':'prometheus-node-exporter',
'serviceMonitor/observability/blackbox-exporter/0':'blackbox-exporter',
'serviceMonitor/observability/grafana/0':'grafana',
'serviceMonitor/observability/loki/0':'observability/loki',
'serviceMonitor/observability/observability-core-kube-pr-alertmanager/0':'observability-core-kube-pr-alertmanager',
'serviceMonitor/observability/observability-core-kube-pr-alertmanager/1':'observability-core-kube-pr-alertmanager',
'serviceMonitor/observability/observability-core-kube-pr-apiserver/0':'apiserver',
'serviceMonitor/observability/observability-core-kube-pr-coredns/0':'coredns',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/0':'kubelet',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/1':'kubelet',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/2':'kubelet',
'serviceMonitor/observability/observability-core-kube-pr-operator/0':'observability-core-kube-pr-operator',
'serviceMonitor/observability/observability-core-kube-pr-prometheus/0':'observability-core-kube-pr-prometheus',
'serviceMonitor/observability/observability-core-kube-pr-prometheus/1':'observability-core-kube-pr-prometheus',
'serviceMonitor/observability/observability-core-kube-state-metrics/0':'kube-state-metrics',
'serviceMonitor/observability/tempo/0':'tempo',
}
namespaces={pool:pool.split('/')[1] for pool in urls}
namespaces.update({
'serviceMonitor/observability/observability-core-kube-pr-apiserver/0':'default',
'serviceMonitor/observability/observability-core-kube-pr-coredns/0':'kube-system',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/0':'kube-system',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/1':'kube-system',
'serviceMonitor/observability/observability-core-kube-pr-kubelet/2':'kube-system',
})
probe_targets={
'probe/observability/platform-private-edge':iter([
('https://db-admin.learn.hyeonworks.com/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F','blackbox-private-edge','private-edge'),
('https://grafana.learn.hyeonworks.com/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F','blackbox-private-edge','private-edge'),
('https://storage-admin.learn.hyeonworks.com/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F','blackbox-private-edge','private-edge')]),
'probe/observability/platform-private-internal':iter([
('http://pgadmin.platform-admin.svc.cluster.local/misc/ping','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping','blackbox-private-internal','private-internal'),
('http://grafana.observability.svc.cluster.local/api/health','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth','blackbox-private-internal','private-internal'),
('http://minio-aistor-console.object-storage.svc.cluster.local:9090/','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F','blackbox-private-internal','private-internal')]),
'probe/observability/platform-public-edge':iter([
('https://git.learn.hyeonworks.com/api/healthz','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz','blackbox-public-edge','public-edge'),
('https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration','http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration','blackbox-public-edge','public-edge')]),
}
for target in item['data']['activeTargets']:
pool=target['scrapePool']
if pool.startswith('probe/'):
instance,url,job,group=next(probe_targets[pool])
target['scrapeUrl']=url
target['labels']={'instance':instance,'job':job,'namespace':'observability','observability.hyeonworks.com/probe-group':group}
else:
from urllib.parse import urlsplit
target['scrapeUrl']=target['globalUrl']=urls[pool]
parsed=urlsplit(target['scrapeUrl'])
target['discoveredLabels']={'__address__':parsed.netloc,'__scheme__':parsed.scheme,'__metrics_path__':parsed.path}
target['labels']={'instance':parsed.netloc,'job':jobs[pool],'namespace':namespaces[pool]}
path.write_text(json.dumps(item,separators=(',',':'))+'\n')
PY
chmod 0600 "$TEST_WORK/targets.json"
validate_target_contract "$TEST_WORK/targets.json"
jq '(.data.activeTargets[0].health)="down"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-down.json"
expect_failure validate_target_contract "$TEST_WORK/targets-down.json"
jq '.data.activeTargets += [{"scrapePool":"serviceMonitor/kube-system/kube-scheduler/0","health":"up"}]' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-extra.json"
expect_failure validate_target_contract "$TEST_WORK/targets-extra.json"
jq '(.data.activeTargets[]|select(.scrapePool=="serviceMonitor/keycloak/keycloak/0").scrapeUrl)="http://10.42.0.12:8080/metrics"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-wrong-port.json"
expect_failure validate_target_contract "$TEST_WORK/targets-wrong-port.json"
jq '(.data.activeTargets[]|select(.scrapePool=="serviceMonitor/observability/grafana/0").scrapeUrl)="http://10.42.0.99:9998/metrics"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-core-wrong-port.json"
expect_failure validate_target_contract "$TEST_WORK/targets-core-wrong-port.json"
jq '(.data.activeTargets[]|select(.scrapePool=="serviceMonitor/observability/grafana/0")) |= (.scrapeUrl="http://10.42.0.99:3000/metrics" | .globalUrl=.scrapeUrl | .discoveredLabels.__address__="10.42.0.99:3000")' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-same-port-substitution.json"
expect_failure validate_target_contract "$TEST_WORK/targets-same-port-substitution.json"
jq '(.data.activeTargets[]|select(.scrapePool=="probe/observability/platform-public-edge").labels.instance)="https://git.learn.hyeonworks.com/api/healthz"' "$TEST_WORK/targets.json" >"$TEST_WORK/targets-substituted-instance.json"
expect_failure validate_target_contract "$TEST_WORK/targets-substituted-instance.json"
pass 'target validator enforces exact 30-target/25-pool allowlist'
cat >"$TEST_WORK/series.json" <<'JSON'
{"status":"success","data":{"resultType":"vector","result":[
{"metric":{"bucket":"loki","job":"minio-aistor","namespace":"object-storage","instance":"10.42.0.4:9000"},"value":[1,"1"]},
{"metric":{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage","instance":"10.42.0.4:9000"},"value":[1,"1"]}
]}}
JSON
chmod 0600 "$TEST_WORK/series.json"
validate_series_contract "$TEST_WORK/series.json" 2 'bucket,instance,job,namespace' 'loki,tempo' bucket
expect_failure validate_series_contract "$TEST_WORK/series.json" 1 'bucket,instance,job,namespace' 'loki,tempo' bucket
jq 'del(.data.result[0].metric.instance)' "$TEST_WORK/series.json" >"$TEST_WORK/series-bad.json"
expect_failure validate_series_contract "$TEST_WORK/series-bad.json" 2 'bucket,instance,job,namespace' 'loki,tempo' bucket
declare -F validate_series_identity_contract >/dev/null || fail 'exact series identity validator is missing'
validate_series_identity_contract "$TEST_WORK/series.json" \
'[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \
'^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$'
jq '(.data.result[0].metric.job)="wrong"' "$TEST_WORK/series.json" >"$TEST_WORK/series-wrong-job.json"
expect_failure validate_series_identity_contract "$TEST_WORK/series-wrong-job.json" \
'[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \
'^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$'
pass 'metric-series validator enforces count, labels, and exact values'
: >"$TEST_WORK/metric-query-contract.log"
(
check_one_query() { printf '%s\037' "$@" >>"$TEST_WORK/metric-query-contract.log"; printf '\n' >>"$TEST_WORK/metric-query-contract.log"; }
check_metric_contracts >/dev/null
) || fail 'metric query contract renderer'
cnpg_contract="$(grep '^cnpg' "$TEST_WORK/metric-query-contract.log")"
keycloak_contract="$(grep '^keycloak' "$TEST_WORK/metric-query-contract.log")"
assert_contains "$keycloak_contract" 'process_uptime_seconds{job="keycloak-service",namespace="keycloak"}' 'Keycloak query live job identity'
assert_contains "$keycloak_contract" '[{"job":"keycloak-service","namespace":"keycloak"}]' 'Keycloak result live job identity'
assert_contains "$cnpg_contract" 'cnpg_collector_up{job="platform-postgres",namespace="platform-data",cluster="platform-postgres"}' 'CNPG query cluster identity'
assert_contains "$cnpg_contract" '[{"cluster":"platform-postgres","job":"platform-postgres","namespace":"platform-data"}]' 'CNPG result cluster identity'
pass 'CNPG metric query binds the exact cluster identity'
cat >"$TEST_WORK/datasources.json" <<'JSON'
{"prometheus":{"status":"OK"},"loki":{"status":"OK"},"tempo":{"status":"OK"},
"loki_config":{"uid":"loki","jsonData":{"derivedFields":[{"name":"trace_id","datasourceUid":"tempo","url":"${__value.raw}"}]}},
"tempo_config":{"uid":"tempo","jsonData":{"tracesToLogsV2":{"datasourceUid":"loki","filterByTraceID":true}}},
"selected_trace":"0123456789abcdef0123456789abcdef",
"tempo_search":{"traces":[{"traceID":"0123456789abcdef0123456789abcdef","rootServiceName":"platform-smoke"}]},
"loki_query":{"status":"success","data":{"result":[{"values":[["1","{\"trace_id\":\"0123456789abcdef0123456789abcdef\"}"]]}]}},
"tempo_query":{"batches":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"platform-smoke"}}]},"scopeSpans":[{"spans":[{"traceId":"ASNFZ4mrze8BI0VniavN7w=="}]}]}]}}
JSON
chmod 0600 "$TEST_WORK/datasources.json"
validate_datasource_contract "$TEST_WORK/datasources.json"
jq '.tempo.status="ERROR"' "$TEST_WORK/datasources.json" >"$TEST_WORK/datasources-bad.json"
expect_failure validate_datasource_contract "$TEST_WORK/datasources-bad.json"
jq '.loki_query.data.result=[]' "$TEST_WORK/datasources.json" >"$TEST_WORK/datasources-no-link.json"
expect_failure validate_datasource_contract "$TEST_WORK/datasources-no-link.json"
declare -F synthetic_loki_query >/dev/null || fail 'core synthetic Loki selector builder is missing'
assert_eq '{cluster="home",namespace="observability"} | json | trace_id="0123456789abcdef0123456789abcdef"' \
"$(synthetic_loki_query 0123456789abcdef0123456789abcdef)" 'core synthetic Loki selector'
expect_failure synthetic_loki_query 'bad"trace'
jq '(.tempo_query.batches[0].resource.attributes[0].value.stringValue)="wrong-service"' \
"$TEST_WORK/datasources.json" >"$TEST_WORK/datasources-wrong-synthetic.json"
expect_failure validate_datasource_contract "$TEST_WORK/datasources-wrong-synthetic.json"
pass 'datasource validator requires health and a live Loki-to-Tempo link'
# Mutation caught: incomplete human evidence must never produce full PASS.
complete_tokens=$'OIDC_ADMIN_CONFIRMED\nOIDC_VIEWER_CONFIRMED\nOIDC_DENIED_CONFIRMED\nBREAK_GLASS_CONFIRMED\nREMOVAL_VIEWER_CONFIRMED\nSESSION_REVOKED\nRELOGIN_DENIED\nFIRING\nRESOLVED\nEXTERNAL_CLIENT_VERIFIED'
validate_human_tokens <<<"$complete_tokens"
expect_failure bash -c 'PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"; validate_human_tokens' bash "$SMOKE" <<<"${complete_tokens%$'\nEXTERNAL_CLIENT_VERIFIED'}"
pass 'human acceptance requires every exact token'
declare -F external_attestation_token >/dev/null || fail 'external-client bound attestation token is missing'
assert_eq 'EXTERNAL_CLIENT_VERIFIED public_ip=203.0.113.10 status=403 source=outside-lan-tailscale proxy=disabled' \
"$(external_attestation_token 203.0.113.10)" 'external attestation fields'
expect_failure external_attestation_token '203.0.113.10;touch /tmp/no'
external_instructions="$(render_external_attestation 203.0.113.10)"
assert_contains "$external_instructions" "--resolve $GRAFANA_HOST:443:203.0.113.10" 'external exact resolve command'
assert_contains "$external_instructions" 'Expected exact result: HTTP 403' 'external exact expected result'
assert_contains "$external_instructions" 'outside both LAN and Tailscale' 'external source requirement'
pass 'external-client evidence binds exact command, source and result fields'
declare -F confirmation_remaining_seconds >/dev/null || fail 'shared confirmation deadline calculator is missing'
assert_eq 300 "$(confirmation_remaining_seconds 1300 1000)" 'shared deadline initial window'
assert_eq 1 "$(confirmation_remaining_seconds 1300 1299)" 'shared deadline last second'
expect_failure confirmation_remaining_seconds 1300 1300
expect_failure confirmation_remaining_seconds 1300 1301
pass 'session revoke and relogin confirmations share one five-minute deadline'
declare -F capture_keycloak_absent_baseline >/dev/null || fail 'stable full Keycloak membership baseline helper is missing'
declare -F add_keycloak_viewer_membership >/dev/null || fail 'response-safe Keycloak membership add helper is missing'
(
WORK="$TEST_WORK"; KEYCLOAK_USER_ID='user-owned-id'; KEYCLOAK_VIEWER_GROUP_ID='viewer-owned-id'
KEYCLOAK_MEMBERSHIP_BEFORE="$TEST_WORK/keycloak-full-baseline.json"
: >"$TEST_WORK/keycloak-pagination.log"
keycloak_request() {
local output=$3 first
printf '%s\n' "$2" >>"$TEST_WORK/keycloak-pagination.log"
first="$(sed -n 's/.*[?&]first=\([0-9]*\).*/\1/p' <<<"$2")"
if [[ "$first" == 0 ]]; then
jq -n '[range(0;100)|{id:("group-"+tostring),name:("group-"+tostring),path:("/group-"+tostring)}]' >"$output"
else
jq -n '[{id:"viewer-owned-id",name:"platform-observability-viewers",path:"/platform-observability-viewers"}]' >"$output"
fi
chmod 0600 -- "$output"; printf 200
}
expect_failure capture_keycloak_absent_baseline admin-owned-id
[[ "$(wc -l <"$TEST_WORK/keycloak-pagination.log" | tr -d '[:space:]')" -ge 2 ]]
) || fail 'full Keycloak pagination detects a viewer membership after page one'
keycloak_signal_rc=0
(
WORK="$TEST_WORK"; KEYCLOAK_USER_ID='user-owned-id'; KEYCLOAK_VIEWER_GROUP_ID='viewer-owned-id'
KEYCLOAK_MEMBERSHIP_BEFORE="$TEST_WORK/keycloak-signal-baseline.json"
printf '[]\n' >"$KEYCLOAK_MEMBERSHIP_BEFORE"; chmod 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE"
KEYCLOAK_TEST_ADDED=false; KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=false
MUTATION_CRITICAL_DEPTH=0; DEFERRED_SIGNAL_RC=0; SIGNAL_TARGET=$BASHPID
trap 'handle_termination_signal 130' INT
trap 'printf "%s|%s\n" "$KEYCLOAK_TEST_ADDED" "$KEYCLOAK_MEMBERSHIP_PENDING" >"$TEST_WORK/keycloak-signal.capture"' EXIT
keycloak_request() {
local method=$1 output=$3
if [[ "$method" == PUT ]]; then
: >"$output"; chmod 0600 -- "$output"; /bin/kill -INT "$SIGNAL_TARGET"; return 28
fi
return 91
}
keycloak_memberships() {
jq -n '[{id:"viewer-owned-id",name:"platform-observability-viewers",path:"/platform-observability-viewers"}]' >"$1"
chmod 0600 -- "$1"
}
add_keycloak_viewer_membership
) || keycloak_signal_rc=$?
assert_eq 130 "$keycloak_signal_rc" 'deferred Keycloak membership signal status'
assert_eq 'true|false' "$(<"$TEST_WORK/keycloak-signal.capture")" 'signal-safe Keycloak membership ownership capture'
pass 'Keycloak membership add uses full baseline and response-safe ownership capture'
# Production Keycloak cleanup leaf: a preexisting membership is never touched;
# a test-added membership is removed by exact user/group IDs and verified
# absent even when the DELETE response was lost.
(
WORK="$TEST_WORK"
KEYCLOAK_USER_ID='user-owned-id'; KEYCLOAK_VIEWER_GROUP_ID='viewer-owned-id'
KEYCLOAK_MEMBERSHIP_BEFORE="$TEST_WORK/keycloak-cleanup-baseline.json"
printf '[]\n' >"$KEYCLOAK_MEMBERSHIP_BEFORE"; chmod 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE"
KEYCLOAK_TEST_ADDED=false; KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=false
keycloak_request() { printf 'unexpected\n' >>"$TEST_WORK/keycloak-cleanup.log"; return 1; }
: >"$TEST_WORK/keycloak-cleanup.log"
cleanup_keycloak_membership
[[ ! -s "$TEST_WORK/keycloak-cleanup.log" ]]
KEYCLOAK_TEST_ADDED=true
rm -f -- "$TEST_WORK/keycloak-deleted.marker"
keycloak_request() {
printf '%s|%s\n' "$1" "$2" >>"$TEST_WORK/keycloak-cleanup.log"
: >"$TEST_WORK/keycloak-deleted.marker"
return 28
}
keycloak_memberships() {
if [[ ! -e "$TEST_WORK/keycloak-deleted.marker" ]]; then
jq -n '[{id:"viewer-owned-id",name:"platform-observability-viewers",path:"/platform-observability-viewers"}]' >"$1"
else
printf '[]\n' >"$1"
fi
chmod 0600 -- "$1"
}
cleanup_keycloak_membership
[[ "$KEYCLOAK_MEMBERSHIP_REMOVED" == true ]]
) || fail 'test-added Keycloak membership cleanup flow'
assert_eq 'DELETE|/admin/realms/hyeonworks/users/user-owned-id/groups/viewer-owned-id' \
"$(<"$TEST_WORK/keycloak-cleanup.log")" 'exact Keycloak membership cleanup path'
pass 'Keycloak cleanup touches only test-added exact membership'
# Production Slack lifecycle/render leaf: firing proof precedes confirmation,
# exact owned rule cleanup precedes resolved confirmation, and the alert carries
# every grouping/runbook label without a webhook payload.
cat >"$TEST_WORK/bin/date-slack" <<'SH'
#!/usr/bin/env bash
set -Eeuo pipefail
[[ " $* " == *' +%Y%m%dT%H%M%SZ '* ]]
printf '20260812T091011Z\n'
SH
chmod 0755 "$TEST_WORK/bin/date-slack"
(
WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-slack"
new_smoke_run_id() { printf 'alert-20260812t091011z-00000000000000000000000000000001'; }
CONFIRMATION_QUEUE=(FIRING RESOLVED); CONFIRMATION_INDEX=0; TEST_MODE=true
create_owned_from_file() { cp -- "$1" "$TEST_WORK/slack-rule.capture"; printf 'create\n' >>"$TEST_WORK/slack-order.log"; }
wait_for_smoke_alert() { printf 'wait-%s\n' "$2" >>"$TEST_WORK/slack-order.log"; }
require_confirmation() { printf 'confirm-%s\n' "$1" >>"$TEST_WORK/slack-order.log"; [[ "${CONFIRMATION_QUEUE[$CONFIRMATION_INDEX]}" == "$1" ]] && CONFIRMATION_INDEX=$((CONFIRMATION_INDEX + 1)); }
cleanup_owned_objects() { printf 'cleanup\n' >>"$TEST_WORK/slack-order.log"; }
: >"$TEST_WORK/slack-order.log"
run_slack_acceptance >"$TEST_WORK/slack-output.capture"
) || fail 'Slack lifecycle render flow'
assert_eq $'create\nwait-present\nconfirm-FIRING\ncleanup\nconfirm-RESOLVED\nwait-absent' \
"$(<"$TEST_WORK/slack-order.log")" 'Slack lifecycle order'
jq -e --arg runbook "$RUNBOOK_URL" '
.kind=="PrometheusRule" and .metadata.name=="observability-smoke-00000000000000000000000000000001" and
.metadata.namespace=="observability" and
.metadata.labels["platform.hyeonworks.com/smoke-run"]=="alert-20260812t091011z-00000000000000000000000000000001" and
(.spec.groups|length)==1 and (.spec.groups[0].rules|length)==1 and
.spec.groups[0].rules[0].alert=="PlatformObservabilitySmoke" and
.spec.groups[0].rules[0].expr=="vector(1)" and
.spec.groups[0].rules[0].labels=={"cluster":"home","namespace":"observability","instance":"alert-20260812t091011z-00000000000000000000000000000001","severity":"info"} and
.spec.groups[0].rules[0].annotations.runbook_url==$runbook
' "$TEST_WORK/slack-rule.capture" >/dev/null || fail 'Slack rule exact payload contract'
pass 'Slack lifecycle renders and cleans only one exact temporary rule'
# Production network transaction leaf: first prove the exact deployed ingress
# policies, then explicitly allow the test Pod's egress to both targets. A
# failed connect can no longer be manufactured by a test-owned egress deny.
cat >"$TEST_WORK/fixture-aistor-ingress-policy.json" <<'JSON'
{"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"object-storage-allow-prometheus-metrics","namespace":"object-storage","uid":"aistor-policy-uid","resourceVersion":"11"},"spec":{"podSelector":{"matchLabels":{"aistor.min.io/objectStore":"minio-aistor"}},"policyTypes":["Ingress"],"ingress":[{"from":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"observability"}},"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus","app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}}],"ports":[{"protocol":"TCP","port":9000}]}]}}
JSON
cat >"$TEST_WORK/fixture-blackbox-ingress-policy.json" <<'JSON'
{"apiVersion":"networking.k8s.io/v1","kind":"NetworkPolicy","metadata":{"name":"observability-allow-prometheus-to-blackbox","namespace":"observability","uid":"blackbox-policy-uid","resourceVersion":"12"},"spec":{"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus-blackbox-exporter","app.kubernetes.io/instance":"blackbox-exporter"}},"policyTypes":["Ingress"],"ingress":[{"from":[{"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus","app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}}],"ports":[{"protocol":"TCP","port":9115}]}]}}
JSON
chmod 0600 "$TEST_WORK/fixture-aistor-ingress-policy.json" "$TEST_WORK/fixture-blackbox-ingress-policy.json"
declare -F validate_deployed_ingress_policy_contracts >/dev/null ||
fail 'deployed ingress policy exact validator is missing'
validate_deployed_ingress_policy_contracts "$TEST_WORK/fixture-aistor-ingress-policy.json" "$TEST_WORK/fixture-blackbox-ingress-policy.json"
jq '.spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/name"]="foreign"' \
"$TEST_WORK/fixture-aistor-ingress-policy.json" >"$TEST_WORK/aistor-ingress-policy-drift.json"
expect_failure validate_deployed_ingress_policy_contracts \
"$TEST_WORK/aistor-ingress-policy-drift.json" "$TEST_WORK/fixture-blackbox-ingress-policy.json"
(
WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-slack"
new_smoke_run_id() { printf 'network-20260812t091011z-00000000000000000000000000000001'; }
kubectl_bounded() {
if [[ "$1 $2 $3 $4 $5" == '-n object-storage get networkpolicy object-storage-allow-prometheus-metrics' ]]; then
printf 'precondition-aistor\n' >>"$TEST_WORK/network-order.log"; cat -- "$TEST_WORK/fixture-aistor-ingress-policy.json"; return 0
fi
if [[ "$1 $2 $3 $4 $5" == '-n observability get networkpolicy observability-allow-prometheus-to-blackbox' ]]; then
printf 'precondition-blackbox\n' >>"$TEST_WORK/network-order.log"; cat -- "$TEST_WORK/fixture-blackbox-ingress-policy.json"; return 0
fi
if [[ "$1 $2 $3 $4" == '-n object-storage get service' && "$5" == minio ]]; then printf '10.43.0.41'; return 0; fi
if [[ "$1 $2 $3 $4" == '-n observability get service' && "$5" == blackbox-exporter ]]; then printf '10.43.0.42'; return 0; fi
if [[ "$1 $2" == '-n observability' && "$3" == wait ]]; then printf 'wait\n' >>"$TEST_WORK/network-order.log"; return 0; fi
if [[ "$1 $2" == '-n observability' && "$3" == logs ]]; then printf 'OBSERVABILITY_NETWORK_BOUNDARY_PASS\n'; return 0; fi
return 91
}
kubectl_wait_90_bounded() { kubectl_bounded "$@"; }
create_owned_from_file() {
case "$2" in
NetworkPolicy) cp -- "$1" "$TEST_WORK/network-policy.capture" ;;
Pod) cp -- "$1" "$TEST_WORK/network-pod.capture" ;;
esac
printf 'create-%s\n' "$2" >>"$TEST_WORK/network-order.log"
}
cleanup_owned_objects() { printf 'cleanup\n' >>"$TEST_WORK/network-order.log"; }
: >"$TEST_WORK/network-order.log"
run_network_boundary_transaction >"$TEST_WORK/network-output.capture"
) || fail "network transaction render flow: $(<"$TEST_WORK/network-order.log")"
assert_eq $'precondition-aistor\nprecondition-blackbox\ncreate-NetworkPolicy\ncreate-Pod\nwait\ncleanup' "$(<"$TEST_WORK/network-order.log")" 'network transaction order'
jq -e '
.kind=="NetworkPolicy" and .metadata.namespace=="observability" and
.spec.policyTypes==["Egress"] and (.spec.egress|length)==3 and
.spec.egress[0].ports==[{"protocol":"UDP","port":53},{"protocol":"TCP","port":53}] and
(.spec.egress[0].to|length)==1 and
.spec.egress[0].to[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"]=="kube-system" and
.spec.egress[0].to[0].podSelector.matchLabels["k8s-app"]=="kube-dns" and
.spec.egress[1]=={"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"object-storage"}},"podSelector":{"matchLabels":{"aistor.min.io/objectStore":"minio-aistor"}}}],"ports":[{"protocol":"TCP","port":9000}]} and
.spec.egress[2]=={"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"observability"}},"podSelector":{"matchLabels":{"app.kubernetes.io/name":"prometheus-blackbox-exporter","app.kubernetes.io/instance":"blackbox-exporter"}}}],"ports":[{"protocol":"TCP","port":9115}]}
' "$TEST_WORK/network-policy.capture" >/dev/null || fail 'network-policy explicit target egress contract'
jq -e --arg image "$BUSYBOX_IMAGE" '
.kind=="Pod" and .spec.automountServiceAccountToken==false and .spec.restartPolicy=="Never" and
(.spec.containers|length)==1 and .spec.containers[0].image==$image and
.spec.containers[0].securityContext.allowPrivilegeEscalation==false and
.spec.containers[0].securityContext.readOnlyRootFilesystem==true and
.spec.containers[0].securityContext.capabilities.drop==["ALL"] and
(.spec.containers[0].command[2]|contains("nc -z -w 4 10.43.0.41 9000")) and
(.spec.containers[0].command[2]|contains("nc -z -w 4 10.43.0.42 9115"))
' "$TEST_WORK/network-pod.capture" >/dev/null || fail 'network Pod exact restriction contract'
pass 'network transaction proves deployed ingress denial with explicit test egress'
unbounded_timeout_lines="$(rg -n '\"\$TIMEOUT_BIN\"(?!.*--kill-after)' "$SMOKE" --pcre2 | grep -v '\"\$SUDO_BIN\".*\"\$DATE_BIN\"' || true)"
[[ -z "$unbounded_timeout_lines" ]] || fail "timeout without kill-after remains: $unbounded_timeout_lines"
raw_silence_date_lines="$(rg -n 'start=\"\$\(\"\$DATE_BIN\"|end=\"\$\(\"\$DATE_BIN\"' "$SMOKE" || true)"
[[ -z "$raw_silence_date_lines" ]] || fail "raw silence date subprocess remains: $raw_silence_date_lines"
pass 'all fixed subprocess boundaries include terminal kill deadlines'
# Production Certbot leaf: exact bounded dry-run includes deploy hooks and only
# succeeds when Nginx remains active.
cat >"$TEST_WORK/bin/sudo-certbot" <<'SH'
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%q ' "$@" >>"${OBS_SMOKE_CERTBOT_LOG:?}"; printf '\n' >>"${OBS_SMOKE_CERTBOT_LOG:?}"
[[ "${1:-}" == -v ]] && exit 0
if [[ "${1:-}" == -n && "${2:-}" == /usr/bin/true ]]; then exit 0; fi
if [[ "${1:-}" == -n && "${2:-}" == /usr/bin/systemctl && "${3:-}" == is-active && "${4:-}" == nginx ]]; then printf 'active\n'; exit 0; fi
[[ " $* " == *' /snap/bin/certbot renew --dry-run --run-deploy-hooks '* ]]
SH
chmod 0755 "$TEST_WORK/bin/sudo-certbot"
: >"$TEST_WORK/certbot.log"
SUDO_BIN="$TEST_WORK/bin/sudo-certbot" OBS_SMOKE_CERTBOT_LOG="$TEST_WORK/certbot.log" run_certbot_acceptance >/dev/null
assert_contains "$(<"$TEST_WORK/certbot.log")" '-n /usr/bin/timeout --signal=TERM --kill-after=5s 600s /snap/bin/certbot renew --dry-run --run-deploy-hooks' 'Certbot bounded deploy-hook argv'
SUDO_BIN="$SUDO_DEFAULT"
pass 'Certbot leaf runs exact dry-run deploy-hook and Nginx health gate'
cat >"$TEST_WORK/bin/sudo-stall-unless-bounded" <<'SH'
#!/usr/bin/env bash
set -Eeuo pipefail
parent="$(/usr/bin/tr '\0' ' ' <"/proc/$PPID/cmdline")"
if [[ "$parent" == *'/usr/bin/timeout'* ]]; then exit 17; fi
/usr/bin/sleep 5
SH
chmod 0755 "$TEST_WORK/bin/sudo-stall-unless-bounded"
stall_rc=0
/usr/bin/timeout --signal=TERM --kill-after=1s 1s bash -c '
PLATFORM_OBSERVABILITY_SMOKE_SOURCE_ONLY=1 source "$1"
SUDO_BIN="$2"
run_certbot_acceptance
' bash "$SMOKE" "$TEST_WORK/bin/sudo-stall-unless-bounded" >/dev/null 2>&1 || stall_rc=$?
[[ "$stall_rc" != 124 ]] || fail 'privileged subprocess escaped its internal timeout'
pass 'privileged Certbot and systemd subprocesses are internally bounded'
# Regression RED: silence create/delete is response-outcome safe. A lost create
# response is recovered only from one exact unique-run silence; a false 204
# cannot clear ownership while that exact silence remains.
cat >"$TEST_WORK/bin/date-silence" <<'SH'
#!/usr/bin/env bash
set -Eeuo pipefail
case " $* " in
*' +%Y%m%dT%H%M%SZ '*) printf '20260812T091011Z\n' ;;
*" -d +20 minutes +%Y-%m-%dT%H:%M:%SZ "*) printf '2026-08-12T09:30:11Z\n' ;;
*' +%Y-%m-%dT%H:%M:%SZ '*) printf '2026-08-12T09:10:11Z\n' ;;
*) exit 91 ;;
esac
SH
chmod 0755 "$TEST_WORK/bin/date-silence"
(
WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-silence"
new_smoke_run_id() { printf 'persistence-20260812t091011z-00000000000000000000000000000001'; }
SILENCE_ID=''; SILENCE_RUN=''; SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0
MANUAL_RECOVERY=false
rm -f -- "$TEST_WORK/silence-post-attempted"
proxy_request() {
local method=$1 output=$3
case "$method" in
POST)
cp -- "$4" "$TEST_WORK/silence-create-body.capture"
: >"$output"; chmod 0600 -- "$output"
: >"$TEST_WORK/silence-post-attempted"
return 28
;;
GET)
if [[ ! -f "$TEST_WORK/silence-post-attempted" ]]; then printf '[]\n' >"$output"; else jq -n --arg run 'persistence-20260812t091011z-00000000000000000000000000000001' '[{
id:"silence-owned-id",status:{state:"active"},createdBy:"platform-observability-smoke",
comment:("Temporary restart-persistence acceptance "+$run),
matchers:[
{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},
{name:"instance",value:$run,isRegex:false,isEqual:true}
]
}]' >"$output"; fi
chmod 0600 -- "$output"; printf '200'
;;
esac
}
create_persistence_silence
printf '%s|%s\n' "$SILENCE_ID" "$SILENCE_RUN" >"$TEST_WORK/silence-create-recovered.capture"
) || fail 'lost-response silence create was not reconciled'
assert_eq 'silence-owned-id|persistence-20260812t091011z-00000000000000000000000000000001' "$(<"$TEST_WORK/silence-create-recovered.capture")" 'recovered silence identity'
jq -e '
.createdBy=="platform-observability-smoke" and
.comment=="Temporary restart-persistence acceptance persistence-20260812t091011z-00000000000000000000000000000001" and
([.matchers[]|select(.name=="instance" and .value=="persistence-20260812t091011z-00000000000000000000000000000001" and .isRegex==false and .isEqual==true)]|length)==1
' "$TEST_WORK/silence-create-body.capture" >/dev/null || fail 'unique silence create payload'
false_delete_rc=0
(
WORK="$TEST_WORK"; SILENCE_ID='silence-owned-id'; SILENCE_RUN='persistence-20260812t091011z-00000000000000000000000000000001'
SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0; MANUAL_RECOVERY=false
emit_manual_recovery() { printf 'manual\n' >"$TEST_WORK/silence-delete-manual.capture"; MANUAL_RECOVERY=true; }
proxy_request() {
local method=$1 output=$3
if [[ "$method" == DELETE ]]; then : >"$output"; chmod 0600 -- "$output"; printf '204'; return 0; fi
jq -n --arg run "$SILENCE_RUN" '[{
id:"silence-owned-id",status:{state:"active"},createdBy:"platform-observability-smoke",
comment:("Temporary restart-persistence acceptance "+$run),
matchers:[{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},{name:"instance",value:$run,isRegex:false,isEqual:true}]
}]' >"$output"; chmod 0600 -- "$output"; printf '200'
}
delete_persistence_silence
) || false_delete_rc=$?
assert_eq 1 "$false_delete_rc" 'false-204 silence delete status'
[[ -f "$TEST_WORK/silence-delete-manual.capture" ]] || fail 'false-204 silence delete omitted manual recovery'
expired_delete_rc=0
(
WORK="$TEST_WORK"; SILENCE_ID='silence-owned-id'; SILENCE_RUN='persistence-20260812t091011z-00000000000000000000000000000001'
SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0; MANUAL_RECOVERY=false
proxy_request() {
local method=$1 output=$3
if [[ "$method" == DELETE ]]; then : >"$output"; chmod 0600 -- "$output"; printf '204'; return 0; fi
jq -n --arg run "$SILENCE_RUN" '[{
id:"silence-owned-id",status:{state:"expired"},createdBy:"platform-observability-smoke",
comment:("Temporary restart-persistence acceptance "+$run),
matchers:[{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},{name:"instance",value:$run,isRegex:false,isEqual:true}]
}]' >"$output"; chmod 0600 -- "$output"; printf '200'
}
delete_persistence_silence
) || expired_delete_rc=$?
assert_eq 0 "$expired_delete_rc" 'stable expired silence delete outcome'
pass 'silence lifecycle reconciles response loss and rejects false delete success'
signal_silence_rc=0
(
WORK="$TEST_WORK"; DATE_BIN="$TEST_WORK/bin/date-silence"
new_smoke_run_id() { printf 'persistence-20260812t091011z-00000000000000000000000000000002'; }
SILENCE_ID=''; SILENCE_RUN=''; SILENCE_POLL_ATTEMPTS=2; SILENCE_POLL_DELAY=0
MUTATION_CRITICAL_DEPTH=0; DEFERRED_SIGNAL_RC=0
SIGNAL_TARGET=$BASHPID
trap 'handle_termination_signal 130' INT
trap 'printf "%s|%s\n" "$SILENCE_ID" "$SILENCE_RUN" >"$TEST_WORK/signal-silence.capture"' EXIT
rm -f -- "$TEST_WORK/signal-silence-posted"
proxy_request() {
local method=$1 output=$3
case "$method" in
POST)
: >"$output"; chmod 0600 -- "$output"
: >"$TEST_WORK/signal-silence-posted"
/bin/kill -INT "$SIGNAL_TARGET"
return 28
;;
GET)
if [[ ! -f "$TEST_WORK/signal-silence-posted" ]]; then
printf '[]\n' >"$output"
else
jq -n --arg run "$SILENCE_RUN" '[{id:"silence-signal-id",status:{state:"active"},createdBy:"platform-observability-smoke",comment:("Temporary restart-persistence acceptance "+$run),matchers:[{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},{name:"instance",value:$run,isRegex:false,isEqual:true}]}]' >"$output"
fi
chmod 0600 -- "$output"; printf '200'
;;
esac
}
create_persistence_silence
) || signal_silence_rc=$?
assert_eq 130 "$signal_silence_rc" 'deferred silence signal status'
assert_eq 'silence-signal-id|persistence-20260812t091011z-00000000000000000000000000000002' \
"$(<"$TEST_WORK/signal-silence.capture")" 'signal-safe silence ownership capture'
pass 'silence signal window closes only after exact ownership capture'
# Production raw delete leaf: keep its real API-path and UID DeleteOptions
# builder while replacing only transport and post-delete reads. Transport loss
# is success only after stable absence; a foreign replacement fails closed.
RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)"
chmod 0700 "$RAW_DELETE_WORK"
(
WORK="$RAW_DELETE_WORK"
proxy_request() {
printf '%s|%s\n' "$1" "$2" >"$WORK/raw-delete-call"
cp -- "$4" "$WORK/raw-delete-body"
: >"$3"
printf '200'
}
owned_object_state() { return 1; }
delete_owned_uid Pod observability smoke-pod uid-pod
)
assert_eq 'DELETE|/api/v1/namespaces/observability/pods/smoke-pod' "$(<"$RAW_DELETE_WORK/raw-delete-call")" 'raw delete API path'
jq -e '.apiVersion=="meta.k8s.io/v1" and .kind=="DeleteOptions" and .propagationPolicy=="Background" and .preconditions=={"uid":"uid-pod"}' "$RAW_DELETE_WORK/raw-delete-body" >/dev/null || fail 'raw UID DeleteOptions contract'
ambiguity_rc=0
(
WORK="$RAW_DELETE_WORK"
proxy_request() { return 28; }
owned_object_state() { return 1; }
delete_owned_uid PrometheusRule observability smoke-rule uid-rule
) || ambiguity_rc=$?
assert_eq 0 "$ambiguity_rc" 'transport ambiguity followed by absence'
foreign_rc=0
(
WORK="$RAW_DELETE_WORK"
proxy_request() { return 28; }
owned_object_state() { printf '%s\n' 'foreign-uid|foreign-run'; return 0; }
delete_owned_uid PrometheusRule observability smoke-rule uid-rule
) || foreign_rc=$?
assert_eq 1 "$foreign_rc" 'foreign replacement raw delete status'
rm -rf -- "$RAW_DELETE_WORK"
RAW_DELETE_WORK=''
pass 'production raw UID delete reconciles ambiguity and preserves replacement'
# Regression RED: an ambiguous create may be followed by an unrelated object
# with copied name/labels. Server defaults are allowed, but every desired field
# and the exact standalone-container boundary must match before ownership.
cat >"$TEST_WORK/owned-desired.json" <<'JSON'
{"apiVersion":"v1","kind":"Pod","metadata":{"name":"smoke-pod","namespace":"observability","labels":{"app.kubernetes.io/managed-by":"platform-observability-smoke","platform.hyeonworks.com/smoke-run":"run-1"}},"spec":{"automountServiceAccountToken":false,"restartPolicy":"Never","containers":[{"name":"network-smoke","image":"busybox@sha256:fixture","command":["sh","-ec","true"],"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}}]}}
JSON
jq '.metadata.uid="uid-pod" | .metadata.resourceVersion="7" | .spec.dnsPolicy="ClusterFirst" | .spec.containers[0].terminationMessagePolicy="File" | .status={"phase":"Pending"}' \
"$TEST_WORK/owned-desired.json" >"$TEST_WORK/owned-live-defaulted.json"
chmod 0600 "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-defaulted.json"
declare -F validate_live_matches_desired >/dev/null ||
fail 'ambiguous-create desired/live ownership validator is missing'
validate_live_matches_desired "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-defaulted.json"
jq '.spec.containers += [{"name":"foreign","image":"busybox:latest"}]' \
"$TEST_WORK/owned-live-defaulted.json" >"$TEST_WORK/owned-live-sidecar.json"
expect_failure validate_live_matches_desired "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-sidecar.json"
jq '.spec.containers[0].command=["sh","-ec","exfiltrate"]' \
"$TEST_WORK/owned-live-defaulted.json" >"$TEST_WORK/owned-live-drift.json"
expect_failure validate_live_matches_desired "$TEST_WORK/owned-desired.json" "$TEST_WORK/owned-live-drift.json"
pass 'ambiguous create ownership requires desired spec, not copied labels'
# Production create leaf: exercise server dry-run, successful identity capture,
# transport-ambiguous re-read, and a copied-label foreign-spec rejection.
RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)"
chmod 0700 "$RAW_DELETE_WORK"
(
WORK="$RAW_DELETE_WORK"
scenario=success
owned_object_state() { return 1; }
kubectl_bounded() {
if [[ "$1" == create && "$2" == --dry-run=server ]]; then return 0; fi
if [[ "$1" == create ]]; then
cat -- "$TEST_WORK/owned-live-defaulted.json"
[[ "$scenario" == success ]] && return 0
return 28
fi
if [[ "$1" == -n && "$3" == get ]]; then
if [[ "$scenario" == foreign ]]; then cat -- "$TEST_WORK/owned-live-sidecar.json"; else cat -- "$TEST_WORK/owned-live-defaulted.json"; fi
return 0
fi
return 91
}
OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=()
create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1
printf '%s|%s|%s|%s|%s\n' "${OWNED_KIND[0]}" "${OWNED_NAMESPACE[0]}" "${OWNED_NAME[0]}" "${OWNED_UID[0]}" "${OWNED_RUN[0]}" >"$TEST_WORK/create-success.capture"
OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=(); scenario=ambiguous
ambiguous_create_rc=0
create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1 || ambiguous_create_rc=$?
printf '%s|%s|%s\n' "$ambiguous_create_rc" "${#OWNED_UID[@]}" "${OWNED_UID[0]:-}" >"$TEST_WORK/create-ambiguous.capture"
OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=(); scenario=foreign
foreign_create_rc=0
emit_manual_recovery() { printf 'manual\n' >"$TEST_WORK/create-foreign-manual.capture"; }
create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1 || foreign_create_rc=$?
printf '%s|%s\n' "$foreign_create_rc" "${#OWNED_UID[@]}" >"$TEST_WORK/create-foreign.capture"
)
assert_eq 'Pod|observability|smoke-pod|uid-pod|run-1' "$(<"$TEST_WORK/create-success.capture")" 'successful create ownership capture'
assert_eq '1|1|uid-pod' "$(<"$TEST_WORK/create-ambiguous.capture")" 'ambiguous create owned-state capture'
assert_eq '1|0' "$(<"$TEST_WORK/create-foreign.capture")" 'foreign create ownership rejection'
[[ -f "$TEST_WORK/create-foreign-manual.capture" ]] || fail 'foreign ambiguous create omitted manual-recovery evidence'
rm -rf -- "$RAW_DELETE_WORK"
RAW_DELETE_WORK=''
pass 'production create leaf fails closed and records only exact owned state'
# A signal delivered after the API commits but before the response is recorded
# must be deferred until the exact created UID is in the cleanup ledger.
declare -F handle_termination_signal >/dev/null || fail 'mutation signal deferral is missing'
RAW_DELETE_WORK="$(mktemp -d /tmp/platform-observability-smoke.XXXXXXXX)"
chmod 0700 "$RAW_DELETE_WORK"
signal_create_rc=0
(
WORK="$RAW_DELETE_WORK"
trap 'handle_termination_signal 130' INT
trap 'printf "%s|%s\n" "${#OWNED_UID[@]}" "${OWNED_UID[0]:-}" >"$TEST_WORK/signal-create.capture"' EXIT
owned_object_state() { return 1; }
kubectl_bounded() {
if [[ "$1" == create && "$2" == --dry-run=server ]]; then return 0; fi
if [[ "$1" == create ]]; then
/bin/kill -INT "$BASHPID"
cat -- "$TEST_WORK/owned-live-defaulted.json"
return 0
fi
return 91
}
OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=()
create_owned_from_file "$TEST_WORK/owned-desired.json" Pod observability smoke-pod run-1
) || signal_create_rc=$?
assert_eq 130 "$signal_create_rc" 'deferred create signal status'
assert_eq '1|uid-pod' "$(<"$TEST_WORK/signal-create.capture")" 'signal-safe UID registration'
rm -rf -- "$RAW_DELETE_WORK"
RAW_DELETE_WORK=''
pass 'create signal window closes only after exact UID registration'
# Mutation caught: cleanup by name or without a matching UID could delete a
# replacement. The fixture exercises owned, absent, and foreign identities.
OWNED_KIND=(Pod NetworkPolicy PrometheusRule)
OWNED_NAMESPACE=(observability observability observability)
OWNED_NAME=(smoke-pod smoke-policy smoke-rule)
OWNED_UID=(uid-pod uid-policy uid-rule)
OWNED_RUN=(run-1 run-1 run-1)
declare -a cleanup_calls=()
owned_object_state() {
case "$3" in
smoke-pod) return 1 ;;
smoke-policy) printf '%s\n' 'uid-policy|run-1'; return 0 ;;
smoke-rule) printf '%s\n' 'foreign-uid|foreign-run'; return 0 ;;
esac
}
delete_owned_uid() { cleanup_calls+=("$1|$2|$3|$4"); [[ "$3" != smoke-rule ]]; }
cleanup_rc=0
cleanup_owned_objects || cleanup_rc=$?
assert_eq 1 "$cleanup_rc" 'foreign cleanup status'
assert_eq 'NetworkPolicy|observability|smoke-policy|uid-policy' "${cleanup_calls[0]}" 'exact UID cleanup call'
assert_eq 1 "${#cleanup_calls[@]}" 'foreign object must not be deleted'
pass 'cleanup preserves absent and foreign objects and UID-deletes only owned state'
# Regression RED: an execute preflight may create the private work/proxy and
# fail halfway. EXIT cleanup must already be armed before that first allocation.
preflight_cleanup_marker="$TEST_WORK/preflight-cleanup.marker"
preflight_rc=0
(
TEST_MODE=true
require_execute_environment() { WORK='/tmp/platform-observability-smoke.fixture00'; return 1; }
cleanup_all() { printf 'cleanup\n' >"$preflight_cleanup_marker"; }
main --execute
) >/dev/null 2>&1 || preflight_rc=$?
assert_eq 1 "$preflight_rc" 'partial preflight failure status'
[[ -f "$preflight_cleanup_marker" ]] || fail 'partial execute preflight was not cleanup-armed'
pass 'execute cleanup is armed before preflight allocation'
# The main execute orchestration is tested with all external work replaced at
# function boundaries. Wrong APPLY stops before mutation; missing external
# evidence returns PARTIAL=2; success performs cleanup and reaches final PASS.
run_orchestration_fixture() {
local confirmations=$1
(
TEST_MODE=true
PUBLIC_EDGE_IP='203.0.113.10'
TEST_CONFIRMATIONS="$confirmations"
mapfile -t CONFIRMATION_QUEUE <<<"$TEST_CONFIRMATIONS"
CONFIRMATION_INDEX=0
require_execute_environment() { :; }
run_read_only_machine_checks() { printf 'machine\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; printf 'READ_ONLY_FIXTURE=PASS\n'; }
repeat_commit_gate() { printf 'commit-gate\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; }
run_network_boundary_transaction() { printf 'network\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; OWNED_KIND+=(Pod); OWNED_NAMESPACE+=(observability); OWNED_NAME+=(fixture-pod); OWNED_UID+=(fixture-uid); OWNED_RUN+=(fixture-run); }
run_restart_persistence_transaction() { printf 'restart\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; }
run_certbot_acceptance() { printf 'certbot\n' >>"${ORCHESTRATION_LOG:-/dev/null}"; }
run_oidc_acceptance() { require_confirmation OIDC_ADMIN_CONFIRMED; require_confirmation OIDC_VIEWER_CONFIRMED; require_confirmation OIDC_DENIED_CONFIRMED; require_confirmation BREAK_GLASS_CONFIRMED; require_confirmation REMOVAL_VIEWER_CONFIRMED; require_confirmation SESSION_REVOKED; require_confirmation RELOGIN_DENIED; printf 'OBSERVABILITY_OIDC_ACCEPTANCE=PASS\n'; }
run_slack_acceptance() { require_confirmation FIRING; require_confirmation RESOLVED; printf 'OBSERVABILITY_SLACK_ACCEPTANCE=PASS\n'; }
owned_object_state() { return 1; }
delete_owned_uid() { :; }
main --execute
)
}
wrong_rc=0
wrong_output="$(run_orchestration_fixture 'WRONG' 2>&1)" || wrong_rc=$?
assert_eq 1 "$wrong_rc" 'wrong APPLY orchestration status'
assert_not_contains "$wrong_output" 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS' 'wrong APPLY reached mutation'
partial_confirmations=$'APPLY default\nOIDC_ADMIN_CONFIRMED\nOIDC_VIEWER_CONFIRMED\nOIDC_DENIED_CONFIRMED\nBREAK_GLASS_CONFIRMED\nREMOVAL_VIEWER_CONFIRMED\nSESSION_REVOKED\nRELOGIN_DENIED\nFIRING\nRESOLVED'
partial_rc=0
partial_output="$(run_orchestration_fixture "$partial_confirmations" 2>&1)" || partial_rc=$?
assert_eq 2 "$partial_rc" 'external partial orchestration status'
assert_contains "$partial_output" 'OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL' 'external partial marker'
assert_not_contains "$partial_output" 'OBSERVABILITY_SMOKE=PASS' 'partial acceptance falsely passed'
full_confirmations="$partial_confirmations"$'\nEXTERNAL_CLIENT_VERIFIED public_ip=203.0.113.10 status=403 source=outside-lan-tailscale proxy=disabled'
: >"$TEST_WORK/orchestration-order.log"
full_output="$(ORCHESTRATION_LOG="$TEST_WORK/orchestration-order.log" run_orchestration_fixture "$full_confirmations")"
assert_contains "$full_output" 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS' 'machine acceptance marker'
assert_contains "$full_output" 'OBSERVABILITY_SMOKE=PASS' 'full acceptance marker'
assert_eq $'machine\ncommit-gate\nnetwork\nrestart\ncertbot\ncommit-gate\nmachine' \
"$(<"$TEST_WORK/orchestration-order.log")" 'post-mutation full machine revalidation order'
pass 'execute orchestration is gated and cannot turn partial into PASS'
bash -n "$SMOKE"
bash -n "$TEST_ROOT/scripts/validate/test-observability-smoke.sh"
printf 'OBSERVABILITY SMOKE TEST PASS (%d assertions)\n' "$ASSERTIONS"
Reference in New Issue View Git Blame Copy Permalink