#!/usr/bin/env bash

# End-to-end acceptance for the private observability access and alerting
# cutover. No-argument mode is intentionally a no-contact plan printer.
set -Eeuo pipefail
set +x
umask 077

readonly EXPECTED_CONTEXT='default'
readonly EXPECTED_API_SERVER='https://127.0.0.1:6443'
readonly EXPECTED_NODE='donghyeon-system-product-name'
readonly NAMESPACE='observability'
readonly GRAFANA_HOST='grafana.learn.hyeonworks.com'
readonly GITEA_HOST='git.learn.hyeonworks.com'
readonly KEYCLOAK_HOST='id.learn.hyeonworks.com'
readonly LAN_IP='192.168.0.107'
readonly TAILSCALE_IP='100.92.240.34'
readonly PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus'
readonly ALERTMANAGER_SERVICE='observability-core-kube-pr-alertmanager'
readonly RUNBOOK_URL='https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md'
readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0'
readonly KUBECTL_DEFAULT='/usr/local/bin/kubectl'
readonly CURL_DEFAULT='/usr/bin/curl'
readonly DIG_DEFAULT='/usr/bin/dig'
readonly OPENSSL_DEFAULT='/usr/bin/openssl'
readonly NC_DEFAULT='/usr/bin/nc.openbsd'
readonly SUDO_DEFAULT='/usr/bin/sudo'
readonly DATE_DEFAULT='/usr/bin/date'
readonly CERTBOT_BIN='/snap/bin/certbot'
readonly CERTBOT_REAL='/usr/bin/snap'
readonly SYSTEMCTL_BIN='/usr/bin/systemctl'
readonly TIMEOUT_BIN='/usr/bin/timeout'
readonly JQ_BIN='/usr/bin/jq'
readonly PYTHON_BIN='/usr/bin/python3.12'
readonly CAT_BIN='/usr/bin/cat'
readonly CHMOD_BIN='/usr/bin/chmod'
readonly CMP_BIN='/usr/bin/cmp'
readonly MKTEMP_BIN='/usr/bin/mktemp'
readonly MV_BIN='/usr/bin/mv'
readonly READLINK_BIN='/usr/bin/readlink'
readonly RM_BIN='/usr/bin/rm'
readonly STAT_BIN='/usr/bin/stat'
readonly REQUEST_TIMEOUT='10s'
readonly PROCESS_TIMEOUT='20s'
readonly HUMAN_TIMEOUT_SECONDS=300

KUBECTL_BIN="$KUBECTL_DEFAULT"
CURL_BIN="$CURL_DEFAULT"
DIG_BIN="$DIG_DEFAULT"
OPENSSL_BIN="$OPENSSL_DEFAULT"
NC_BIN="$NC_DEFAULT"
SUDO_BIN="$SUDO_DEFAULT"
DATE_BIN="$DATE_DEFAULT"
TEST_MODE=false
TEST_CONFIRMATIONS=''
CONFIRMATION_INDEX=0
declare -a CONFIRMATION_QUEUE=()

execute_requested=false
WORK=''
API_PROXY_SOCKET=''
API_PROXY_PID=''
API_PROXY_START=''
EXIT_HANDLER_RUNNING=false
MANUAL_RECOVERY=false
SILENCE_ID=''
SILENCE_RUN=''
SILENCE_POLL_ATTEMPTS=30
SILENCE_POLL_DELAY=1
MUTATION_CRITICAL_DEPTH=0
DEFERRED_SIGNAL_RC=0
EXPECTED_NODE_UID=''
PUBLIC_EDGE_IP=''
KEYCLOAK_TEST_ADDED=false
KEYCLOAK_MEMBERSHIP_PENDING=false
KEYCLOAK_MEMBERSHIP_REMOVED=false
KEYCLOAK_USER_ID=''
KEYCLOAK_VIEWER_GROUP_ID=''
KEYCLOAK_MEMBERSHIP_BEFORE=''
KEYCLOAK_AUTH_CONFIG=''
declare -a OWNED_KIND=()
declare -a OWNED_NAMESPACE=()
declare -a OWNED_NAME=()
declare -a OWNED_UID=()
declare -a OWNED_RUN=()

fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
emit_manual_recovery() {
  [[ "$MANUAL_RECOVERY" == true ]] && return 0
  MANUAL_RECOVERY=true
  printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2
}

usage() {
  "$CAT_BIN" <<'USAGE'
Usage:
  bash scripts/validate/observability-smoke.sh
  bash scripts/validate/observability-smoke.sh --execute

No arguments prints the exact acceptance plan without external reads or
mutations. Execute requires the visible `APPLY default` gate, creates only
short-lived acceptance objects, restarts only Grafana/Alertmanager Pods, and
requires exact human OIDC, Slack, and external-client evidence tokens.
USAGE
}

print_dry_run() {
  printf '%s\n' \
    'OBSERVABILITY_SMOKE_DRY_RUN=PASS' \
    'CHECK_01=Grafana public/private DNS exactness' \
    'CHECK_02=Grafana certificate chain,hostname,one-SAN,expiry' \
    'CHECK_03=LAN,Tailscale,denied-source,public-edge HTTPS status' \
    'CHECK_04=Gitea,Grafana,Keycloak public metrics denial' \
    'CHECK_05=unknown-SNI and Traefik NodePort boundary' \
    'CHECK_06=exact Prometheus target allowlist and health' \
    'CHECK_07=Gitea,Keycloak,CNPG,AIStor,Traefik metric contracts' \
    'CHECK_08=untrusted-Pod AIStor and blackbox denial' \
    'CHECK_09=public,private,internal,TLS blackbox series' \
    'CHECK_10=Grafana datasource health and Loki-to-Tempo trace link' \
    'CHECK_11=AIStor quota and recording-rule series' \
    'CHECK_12=Grafana,Alertmanager restart persistence' \
    'CHECK_13=Certbot dry-run deploy-hook and Nginx health' \
    'HUMAN_OIDC=admin,viewer,denied,break-glass,session-revoke,relogin-denied' \
    'HUMAN_SLACK=firing,resolved' \
    'HUMAN_EXTERNAL_CLIENT=required' \
    'TEMPORARY_OBJECTS=NetworkPolicy,Pod,AlertmanagerSilence,PrometheusRule' \
    'PRESERVE=Secret,PVC,preexisting-Keycloak-membership' \
    'MUTATION=NOT_REQUESTED'
}

parse_arguments() {
  if (( $# == 0 )); then return 0; fi
  if (( $# == 1 )) && [[ "$1" == --execute ]]; then execute_requested=true; return 0; fi
  usage >&2
  fail 'supported arguments are empty or exactly --execute'
}

reject_production_overrides() {
  local variable
  for variable in ${!PLATFORM_OBSERVABILITY_SMOKE_@}; do
    fail "production boundary override is forbidden: $variable"
    return 1
  done
}

validate_dns_contract() {
  (( $# == 8 )) || return 1
  [[ "$1" == '' && "$2" == '' && "$3" == "$LAN_IP" && "$4" == '' &&
     "$5" == "$TAILSCALE_IP" && "$6" == '' && "$7" == "$LAN_IP" && "$8" == '' ]]
}

validate_certificate_contract() {
  local sans=$1 verify_rc=$2 expiry_rc=$3
  [[ "$verify_rc" == 0 && "$expiry_rc" == 0 &&
     "$sans" == "DNS:$GRAFANA_HOST" ]]
}

validate_certificate_extension_contract() {
  local extension=$1 verify_rc=$2 expiry_rc=$3
  [[ "$verify_rc" == 0 && "$expiry_rc" == 0 ]] || return 1
  python_bounded -I -S - "$extension" "$GRAFANA_HOST" <<'PY'
import sys
extension,host=sys.argv[1:]
lines=[line.strip() for line in extension.splitlines() if line.strip()]
if len(lines)<2 or lines[0]!='X509v3 Subject Alternative Name:':
    raise SystemExit(1)
names=[name.strip() for name in ' '.join(lines[1:]).split(',')]
if names != ['DNS:'+host]:
    raise SystemExit(1)
PY
}

validate_target_contract() {
  local file=$1
  python_bounded -I -S - "$file" <<'PY'
import json,pathlib,re,sys
from urllib.parse import urlsplit
expected={
"podMonitor/platform-data/platform-postgres/0":1,
"probe/observability/platform-private-edge":3,
"probe/observability/platform-private-internal":3,
"probe/observability/platform-public-edge":2,
"serviceMonitor/gitea/gitea/0":1,
"serviceMonitor/keycloak/keycloak/0":1,
"serviceMonitor/kube-system/traefik/0":1,
"serviceMonitor/object-storage/aistor-bucket-usage/0":1,
"serviceMonitor/observability-agent/alloy/0":1,
"serviceMonitor/observability-agent/node-exporter/0":1,
"serviceMonitor/observability/blackbox-exporter/0":1,
"serviceMonitor/observability/grafana/0":1,
"serviceMonitor/observability/loki/0":1,
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":1,
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":1,
"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":1,
"serviceMonitor/observability/observability-core-kube-pr-coredns/0":1,
"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":1,
"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":1,
"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":1,
"serviceMonitor/observability/observability-core-kube-pr-operator/0":1,
"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":1,
"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":1,
"serviceMonitor/observability/observability-core-kube-state-metrics/0":1,
"serviceMonitor/observability/tempo/0":1,
}
expected_probe_targets={
"probe/observability/platform-private-edge":{
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","https://db-admin.learn.hyeonworks.com/","blackbox-private-edge","private-edge"),
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","https://grafana.learn.hyeonworks.com/","blackbox-private-edge","private-edge"),
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","https://storage-admin.learn.hyeonworks.com/","blackbox-private-edge","private-edge"),
},
"probe/observability/platform-private-internal":{
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","http://pgadmin.platform-admin.svc.cluster.local/misc/ping","blackbox-private-internal","private-internal"),
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","http://grafana.observability.svc.cluster.local/api/health","blackbox-private-internal","private-internal"),
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","http://minio-aistor-console.object-storage.svc.cluster.local:9090/","blackbox-private-internal","private-internal"),
},
"probe/observability/platform-public-edge":{
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","https://git.learn.hyeonworks.com/api/healthz","blackbox-public-edge","public-edge"),
 ("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","blackbox-public-edge","public-edge"),
},
}
endpoint_patterns={
"podMonitor/platform-data/platform-postgres/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9187/metrics$",
"serviceMonitor/gitea/gitea/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3000/metrics$",
"serviceMonitor/keycloak/keycloak/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9000/metrics$",
"serviceMonitor/kube-system/traefik/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9100/metrics$",
"serviceMonitor/object-storage/aistor-bucket-usage/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9000/minio/metrics/v3/cluster/usage/buckets$",
"serviceMonitor/observability-agent/alloy/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:12345/metrics$",
"serviceMonitor/observability-agent/node-exporter/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9100/metrics$",
"serviceMonitor/observability/blackbox-exporter/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9115/metrics$",
"serviceMonitor/observability/grafana/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3000/metrics$",
"serviceMonitor/observability/loki/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3100/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9093/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:6443/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-coredns/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9153/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics/cadvisor$",
"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics/probes$",
"serviceMonitor/observability/observability-core-kube-pr-operator/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9090/metrics$",
"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
"serviceMonitor/observability/observability-core-kube-state-metrics/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
"serviceMonitor/observability/tempo/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3200/metrics$",
}
target_identities={
"podMonitor/platform-data/platform-postgres/0":("platform-postgres","platform-data"),
"serviceMonitor/gitea/gitea/0":("gitea","gitea"),
"serviceMonitor/keycloak/keycloak/0":("keycloak-service","keycloak"),
"serviceMonitor/kube-system/traefik/0":("traefik","kube-system"),
"serviceMonitor/object-storage/aistor-bucket-usage/0":("minio-aistor","object-storage"),
"serviceMonitor/observability-agent/alloy/0":("alloy","observability-agent"),
"serviceMonitor/observability-agent/node-exporter/0":("prometheus-node-exporter","observability-agent"),
"serviceMonitor/observability/blackbox-exporter/0":("blackbox-exporter","observability"),
"serviceMonitor/observability/grafana/0":("grafana","observability"),
"serviceMonitor/observability/loki/0":("observability/loki","observability"),
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":("observability-core-kube-pr-alertmanager","observability"),
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":("observability-core-kube-pr-alertmanager","observability"),
"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":("apiserver","default"),
"serviceMonitor/observability/observability-core-kube-pr-coredns/0":("coredns","kube-system"),
"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":("kubelet","kube-system"),
"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":("kubelet","kube-system"),
"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":("kubelet","kube-system"),
"serviceMonitor/observability/observability-core-kube-pr-operator/0":("observability-core-kube-pr-operator","observability"),
"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":("observability-core-kube-pr-prometheus","observability"),
"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":("observability-core-kube-pr-prometheus","observability"),
"serviceMonitor/observability/observability-core-kube-state-metrics/0":("kube-state-metrics","observability"),
"serviceMonitor/observability/tempo/0":("tempo","observability"),
}
try: item=json.loads(pathlib.Path(sys.argv[1]).read_text())
except Exception: raise SystemExit(1)
targets=((item.get("data") or {}).get("activeTargets"))
if item.get("status")!="success" or not isinstance(targets,list) or len(targets)!=30: raise SystemExit(1)
actual={}
probe_targets={pool:set() for pool in expected_probe_targets}
for target in targets:
    if target.get("health")!="up" or target.get("lastError","") not in ("",None): raise SystemExit(1)
    pool=target.get("scrapePool")
    url=target.get("scrapeUrl")
    if not isinstance(pool,str) or not isinstance(url,str): raise SystemExit(1)
    if pool in endpoint_patterns and re.fullmatch(endpoint_patterns[pool],url) is None: raise SystemExit(1)
    if pool in target_identities:
        labels=target.get("labels")
        discovered=target.get("discoveredLabels")
        parsed=urlsplit(url)
        expected_job,expected_namespace=target_identities[pool]
        if (not isinstance(labels,dict) or labels.get("job")!=expected_job or
            labels.get("namespace")!=expected_namespace or labels.get("instance")!=parsed.netloc):
            raise SystemExit(1)
        if (target.get("globalUrl")!=url or not isinstance(discovered,dict) or
            discovered.get("__address__")!=parsed.netloc or
            discovered.get("__scheme__")!=parsed.scheme or
            discovered.get("__metrics_path__")!=parsed.path):
            raise SystemExit(1)
    if pool in expected_probe_targets:
        labels=target.get("labels")
        if not isinstance(labels,dict) or set(labels)!={"instance","job","namespace","observability.hyeonworks.com/probe-group"}: raise SystemExit(1)
        identity=(url,labels.get("instance"),labels.get("job"),labels.get("observability.hyeonworks.com/probe-group"))
        if labels.get("namespace")!="observability" or identity in probe_targets[pool]: raise SystemExit(1)
        probe_targets[pool].add(identity)
    actual[pool]=actual.get(pool,0)+1
if actual!=expected or probe_targets!=expected_probe_targets: raise SystemExit(1)
PY
}

validate_series_contract() {
  local file=$1 expected_count=$2 required_csv=$3 expected_values_csv=$4 value_label=$5
  python_bounded -I -S - "$file" "$expected_count" "$required_csv" "$expected_values_csv" "$value_label" <<'PY'
import json,pathlib,sys
path,count,required,values,label=sys.argv[1:]
try: item=json.loads(pathlib.Path(path).read_text())
except Exception: raise SystemExit(1)
result=((item.get("data") or {}).get("result"))
if item.get("status")!="success" or not isinstance(result,list) or len(result)!=int(count): raise SystemExit(1)
required=set(filter(None,required.split(',')))
expected=sorted(filter(None,values.split(',')))
seen=[]
for series in result:
    metric=series.get("metric")
    if not isinstance(metric,dict) or not required.issubset(metric): raise SystemExit(1)
    value=series.get("value")
    if not isinstance(value,list) or len(value)!=2:
        raise SystemExit(1)
    try: float(value[1])
    except (TypeError,ValueError): raise SystemExit(1)
    if label: seen.append(metric.get(label))
if label and (any(not isinstance(x,str) for x in seen) or sorted(seen)!=expected): raise SystemExit(1)
PY
}

validate_series_identity_contract() {
  local file=$1 expected_json=$2 instance_pattern=${3:-}
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$PYTHON_BIN" -I -S - \
    "$file" "$expected_json" "$instance_pattern" <<'PY'
import json,pathlib,re,sys
path,expected_raw,instance_pattern=sys.argv[1:]
try:
    item=json.loads(pathlib.Path(path).read_text())
    expected=json.loads(expected_raw)
except Exception: raise SystemExit(1)
result=((item.get('data') or {}).get('result'))
if item.get('status')!='success' or not isinstance(result,list) or not isinstance(expected,list) or len(result)!=len(expected) or not expected:
    raise SystemExit(1)
if any(not isinstance(value,dict) or not value for value in expected): raise SystemExit(1)
keys=set(expected[0])
if any(set(value)!=keys or any(not isinstance(x,str) for x in value.values()) for value in expected): raise SystemExit(1)
def canonical(value): return json.dumps(value,sort_keys=True,separators=(',',':'))
expected_rows=sorted(canonical(value) for value in expected)
if len(set(expected_rows))!=len(expected_rows): raise SystemExit(1)
actual=[]
for series in result:
    metric=series.get('metric')
    if not isinstance(metric,dict) or any(not isinstance(metric.get(key),str) for key in keys): raise SystemExit(1)
    if instance_pattern:
        instance=metric.get('instance')
        if not isinstance(instance,str) or re.fullmatch(instance_pattern,instance) is None: raise SystemExit(1)
    actual.append(canonical({key:metric[key] for key in keys}))
if sorted(actual)!=expected_rows: raise SystemExit(1)
PY
}

validate_datasource_contract() {
  local file=$1
  python_bounded -I -S - "$file" <<'PY'
import base64,binascii,json,pathlib,re,sys
try: item=json.loads(pathlib.Path(sys.argv[1]).read_text())
except Exception: raise SystemExit(1)
if any((item.get(x) or {}).get("status")!="OK" for x in ("prometheus","loki","tempo")): raise SystemExit(1)
loki=item.get("loki_config") or {}; tempo=item.get("tempo_config") or {}
fields=((loki.get("jsonData") or {}).get("derivedFields")) or []
if not any(x.get("name")=="trace_id" and x.get("datasourceUid")=="tempo" and x.get("url")=="${__value.raw}" for x in fields): raise SystemExit(1)
link=((tempo.get("jsonData") or {}).get("tracesToLogsV2")) or {}
if link.get("datasourceUid")!="loki" or link.get("filterByTraceID") is not True: raise SystemExit(1)
lines=[]
for stream in (((item.get("loki_query") or {}).get("data") or {}).get("result") or []):
    for value in stream.get("values") or []:
        if isinstance(value,list) and len(value)==2 and isinstance(value[1],str): lines.append(value[1])
selected=item.get('selected_trace')
if not isinstance(selected,str) or re.fullmatch(r'[0-9a-f]{32}',selected) is None: raise SystemExit(1)
search=(item.get('tempo_search') or {}).get('traces')
if not isinstance(search,list): raise SystemExit(1)
candidates=[x for x in search if isinstance(x,dict) and x.get('traceID')==selected and x.get('rootServiceName')=='platform-smoke']
if len(candidates)!=1: raise SystemExit(1)
matches=[]
for line in lines: matches.extend(re.findall(r'"trace_id"\s*:\s*"([0-9a-f]{32})"',line))
if not matches or set(matches)!={selected}: raise SystemExit(1)
batches=(item.get("tempo_query") or {}).get("batches")
if not isinstance(batches,list) or not batches: raise SystemExit(1)
seen_span=False
for batch in batches:
    if not isinstance(batch,dict): raise SystemExit(1)
    attrs=((batch.get('resource') or {}).get('attributes'))
    if not isinstance(attrs,list): raise SystemExit(1)
    service=[x for x in attrs if isinstance(x,dict) and x.get('key')=='service.name']
    if len(service)!=1 or ((service[0].get('value') or {}).get('stringValue'))!='platform-smoke': raise SystemExit(1)
    for scope in batch.get('scopeSpans') or []:
        for span in scope.get('spans') or []:
            encoded=span.get('traceId')
            if not isinstance(encoded,str): raise SystemExit(1)
            try: decoded=base64.b64decode(encoded,validate=True).hex()
            except (binascii.Error,ValueError): raise SystemExit(1)
            if decoded!=selected: raise SystemExit(1)
            seen_span=True
if not seen_span: raise SystemExit(1)
PY
}

synthetic_loki_query() {
  local trace=$1
  [[ "$trace" =~ ^[0-9a-f]{32}$ ]] || return 1
  printf '{cluster="home",namespace="observability"} | json | trace_id="%s"' "$trace"
}

validate_human_tokens() {
  local -a expected=(OIDC_ADMIN_CONFIRMED OIDC_VIEWER_CONFIRMED OIDC_DENIED_CONFIRMED
    BREAK_GLASS_CONFIRMED REMOVAL_VIEWER_CONFIRMED SESSION_REVOKED RELOGIN_DENIED
    FIRING RESOLVED EXTERNAL_CLIENT_VERIFIED)
  local -a actual=()
  mapfile -t actual
  (( ${#actual[@]} == ${#expected[@]} )) || return 1
  local index
  for ((index=0; index<${#expected[@]}; index++)); do
    [[ "${actual[$index]}" == "${expected[$index]}" ]] || return 1
  done
}

validate_ipv4() {
  local value=$1 octet
  local -a parts=()
  [[ "$value" =~ ^[0-9]{1,3}([.][0-9]{1,3}){3}$ ]] || return 1
  IFS=. read -r -a parts <<<"$value"
  (( ${#parts[@]} == 4 )) || return 1
  for octet in "${parts[@]}"; do
    (( 10#$octet <= 255 )) || return 1
  done
}

external_attestation_token() {
  local public_ip=$1
  validate_ipv4 "$public_ip" || return 1
  printf 'EXTERNAL_CLIENT_VERIFIED public_ip=%s status=403 source=outside-lan-tailscale proxy=disabled' "$public_ip"
}

render_external_attestation() {
  local public_ip=$1 token
  token="$(external_attestation_token "$public_ip")" || return 1
  printf '%s\n' \
    'Run this from a client outside both LAN and Tailscale, with no HTTP(S) proxy:' \
    "/usr/bin/curl --disable --silent --show-error --output /dev/null --write-out '%{http_code}\\n' --connect-timeout 5 --max-time 15 --noproxy '*' --resolve $GRAFANA_HOST:443:$public_ip https://$GRAFANA_HOST/" \
    'Expected exact result: HTTP 403' \
    "After verifying the client source and result, type: $token"
}

new_smoke_run_id() {
  local prefix=$1 stamp nonce
  [[ "$prefix" =~ ^[a-z][a-z0-9-]{0,14}$ ]] || return 1
  stamp="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1
  nonce="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$PYTHON_BIN" -I -S - <<'PY'
import os
print(os.urandom(16).hex(),end='')
PY
)" || return 1
  [[ "$stamp" =~ ^[0-9]{8}T[0-9]{6}Z$ && "$nonce" =~ ^[0-9a-f]{32}$ ]] || return 1
  printf '%s-%s-%s' "$prefix" "${stamp,,}" "$nonce"
}

kubernetes_smoke_name() {
  local run=$1 nonce
  [[ "$run" =~ ^[a-z][a-z0-9-]{0,14}-[0-9]{8}t[0-9]{6}z-([0-9a-f]{32})$ ]] || return 1
  nonce=${BASH_REMATCH[1]}
  printf 'observability-smoke-%s' "$nonce"
}

handle_termination_signal() {
  local rc=$1
  [[ "$rc" =~ ^(129|130|143)$ ]] || rc=143
  if (( MUTATION_CRITICAL_DEPTH > 0 )); then
    (( DEFERRED_SIGNAL_RC == 0 )) && DEFERRED_SIGNAL_RC=$rc
    return 0
  fi
  exit "$rc"
}

begin_mutation_critical() {
  MUTATION_CRITICAL_DEPTH=$((MUTATION_CRITICAL_DEPTH + 1))
}

end_mutation_critical() {
  (( MUTATION_CRITICAL_DEPTH > 0 )) || return 1
  MUTATION_CRITICAL_DEPTH=$((MUTATION_CRITICAL_DEPTH - 1))
  if (( MUTATION_CRITICAL_DEPTH == 0 && DEFERRED_SIGNAL_RC != 0 )); then
    local rc=$DEFERRED_SIGNAL_RC
    DEFERRED_SIGNAL_RC=0
    exit "$rc"
  fi
}

require_confirmation() {
  local expected=$1 answer=''
  if [[ "$TEST_MODE" == true ]]; then
    (( CONFIRMATION_INDEX < ${#CONFIRMATION_QUEUE[@]} )) || return 1
    answer=${CONFIRMATION_QUEUE[$CONFIRMATION_INDEX]}
    CONFIRMATION_INDEX=$((CONFIRMATION_INDEX + 1))
  else
    [[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; }
    printf 'Type %s: ' "$expected" >&2
    IFS= read -r -t "$HUMAN_TIMEOUT_SECONDS" answer || return 1
  fi
  [[ "$answer" == "$expected" ]]
}

confirmation_remaining_seconds() {
  local deadline=$1 now=$2 remaining
  [[ "$deadline" =~ ^[0-9]+$ && "$now" =~ ^[0-9]+$ ]] || return 1
  remaining=$((deadline - now))
  (( remaining >= 1 && remaining <= HUMAN_TIMEOUT_SECONDS )) || return 1
  printf '%s' "$remaining"
}

require_confirmation_before() {
  local expected=$1 deadline=$2 now remaining answer=''
  if [[ "$TEST_MODE" == true ]]; then require_confirmation "$expected"; return; fi
  now="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%s)" || return 1
  remaining="$(confirmation_remaining_seconds "$deadline" "$now")" || return 1
  [[ -t 0 ]] || return 1
  printf 'Type %s within the shared deadline: ' "$expected" >&2
  IFS= read -r -t "$remaining" answer || return 1
  [[ "$answer" == "$expected" ]]
}

read_identity() {
  local purpose=$1 answer=''
  [[ "$TEST_MODE" == false ]] || return 1
  [[ -t 0 ]] || return 1
  printf 'Enter the existing %s username (not recorded): ' "$purpose" >&2
  IFS= read -r -t "$HUMAN_TIMEOUT_SECONDS" answer || return 1
  [[ "$answer" =~ ^[A-Za-z0-9][A-Za-z0-9._@+-]{0,127}$ ]] || return 1
  printf '%s' "$answer"
}

process_start_time() {
  local pid=$1
  python_bounded -I -S - "$pid" <<'PY'
import pathlib,sys
raw=pathlib.Path('/proc',sys.argv[1],'stat').read_text()
end=raw.rfind(') ')
if end<0: raise SystemExit(1)
fields=raw[end+2:].split()
if len(fields)<20 or not fields[19].isdigit(): raise SystemExit(1)
print(fields[19],end='')
PY
}

stop_owned_process() {
  local pid=$1 expected_start=$2 current attempt
  [[ "$pid" =~ ^[0-9]+$ && "$expected_start" =~ ^[0-9]+$ ]] || return 0
  current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; }
  [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; }
  /bin/kill -TERM "$pid" 2>/dev/null || true
  for ((attempt=0; attempt<40; attempt++)); do
    /bin/kill -0 "$pid" 2>/dev/null || { wait "$pid" 2>/dev/null || true; return 0; }
    /usr/bin/sleep 0.05
  done
  current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; }
  [[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; }
  /bin/kill -KILL "$pid" 2>/dev/null || true
  wait "$pid" 2>/dev/null || true
}

kubectl_bounded() {
  "$TIMEOUT_BIN" --signal=TERM --kill-after=2s "$PROCESS_TIMEOUT" \
    "$KUBECTL_BIN" --request-timeout="$REQUEST_TIMEOUT" "$@"
}

kubectl_wait_90_bounded() {
  "$TIMEOUT_BIN" --signal=TERM --kill-after=2s 100s \
    "$KUBECTL_BIN" --request-timeout=95s "$@"
}

kubectl_wait_180_bounded() {
  "$TIMEOUT_BIN" --signal=TERM --kill-after=2s 190s \
    "$KUBECTL_BIN" --request-timeout=185s "$@"
}

jq_bounded() {
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$JQ_BIN" "$@"
}

python_bounded() {
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$PYTHON_BIN" "$@"
}

safe_work_file() {
  [[ -n "$WORK" && "$WORK" == /tmp/platform-observability-smoke.???????? &&
     "$1" == "$WORK"/* && -f "$1" && ! -L "$1" && -O "$1" &&
     "$("$STAT_BIN" -c '%a:%h' -- "$1")" == 600:1 ]]
}

make_work() {
  WORK="$("$MKTEMP_BIN" -d /tmp/platform-observability-smoke.XXXXXXXX)" || return 1
  [[ "$WORK" == /tmp/platform-observability-smoke.???????? && -d "$WORK" && ! -L "$WORK" ]] || return 1
  "$CHMOD_BIN" 0700 -- "$WORK"
}

start_api_proxy() {
  [[ -n "$WORK" ]] || return 1
  API_PROXY_SOCKET="$WORK/kube-api.sock"
  "$KUBECTL_BIN" --request-timeout="$REQUEST_TIMEOUT" proxy \
    --unix-socket="$API_PROXY_SOCKET" --api-prefix=/ \
    --accept-paths='^/(api|apis)/.*$' >"$WORK/kube-api-proxy.log" 2>&1 &
  API_PROXY_PID=$!
  API_PROXY_START="$(process_start_time "$API_PROXY_PID")" || return 1
  local attempt
  for ((attempt=0; attempt<100; attempt++)); do
    [[ -S "$API_PROXY_SOCKET" ]] && { "$CHMOD_BIN" 0600 -- "$API_PROXY_SOCKET"; return 0; }
    /bin/kill -0 "$API_PROXY_PID" 2>/dev/null || return 1
    /usr/bin/sleep 0.05
  done
  return 1
}

proxy_request() {
  local method=$1 path=$2 output=$3 body=${4:-} extra=${5:-}
  local config status
  [[ "$method" =~ ^(GET|POST|PUT|DELETE)$ && "$path" == /api* &&
     "$path" != *$'\n'* && "$path" != *'"'* && "$path" != *'\\'* ]] || return 1
  config="$("$MKTEMP_BIN" "$WORK/curl.XXXXXXXX")" || return 1
  "$CHMOD_BIN" 0600 -- "$config"
  : >"$output"; "$CHMOD_BIN" 0600 -- "$output"
  {
    printf 'silent\nshow-error\nconnect-timeout = 2\nmax-time = 12\n'
    printf 'unix-socket = "%s"\nrequest = "%s"\n' "$API_PROXY_SOCKET" "$method"
    printf 'url = "http://localhost%s"\noutput = "%s"\nwrite-out = "%%{http_code}"\n' "$path" "$output"
    [[ -z "$body" ]] || printf 'header = "Content-Type: application/json"\ndata-binary = "@%s"\n' "$body"
    [[ -z "$extra" ]] || /bin/cat -- "$extra"
  } >"$config"
  safe_work_file "$config" && safe_work_file "$output" || return 1
  status="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$CURL_BIN" --disable --config "$config")" || return 1
  [[ "$status" =~ ^[0-9]{3}$ ]] || return 1
  printf '%s' "$status"
}

prometheus_path() {
  printf '/api/v1/namespaces/observability/services/http:%s:9090/proxy%s' "$PROMETHEUS_SERVICE" "$1"
}

alertmanager_path() {
  printf '/api/v1/namespaces/observability/services/http:%s:9093/proxy%s' "$ALERTMANAGER_SERVICE" "$1"
}

grafana_path() { printf '/api/v1/namespaces/observability/services/http:grafana:80/proxy%s' "$1"; }
keycloak_path() { printf '/api/v1/namespaces/keycloak/services/http:keycloak-service:8080/proxy%s' "$1"; }

proxy_get_200() {
  local path=$1 output=$2 extra=${3:-} status
  status="$(proxy_request GET "$path" "$output" '' "$extra")" || return 1
  [[ "$status" == 200 && -s "$output" ]]
}

prom_query() {
  local expression=$1 output=$2 encoded
  encoded="$(jq_bounded -nr --arg value "$expression" '$value|@uri')" || return 1
  proxy_get_200 "$(prometheus_path "/api/v1/query?query=$encoded")" "$output"
}

api_path_for_owned() {
  local kind=$1 namespace=$2 name=$3
  [[ "$namespace" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ &&
     "$name" =~ ^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$ ]] || return 1
  case "$kind" in
    Pod) printf '/api/v1/namespaces/%s/pods/%s' "$namespace" "$name" ;;
    NetworkPolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s' "$namespace" "$name" ;;
    PrometheusRule) printf '/apis/monitoring.coreos.com/v1/namespaces/%s/prometheusrules/%s' "$namespace" "$name" ;;
    *) return 1 ;;
  esac
}

kind_resource() {
  case "$1" in
    Pod) printf pod ;;
    NetworkPolicy) printf networkpolicy ;;
    PrometheusRule) printf prometheusrule ;;
    *) return 1 ;;
  esac
}

owned_object_state() {
  local kind=$1 namespace=$2 name=$3 resource file uid run
  resource="$(kind_resource "$kind")" || return 4
  file="$("$MKTEMP_BIN" "$WORK/state.XXXXXXXX")" || return 4
  "$CHMOD_BIN" 0600 -- "$file"
  kubectl_bounded -n "$namespace" get "$resource" "$name" --ignore-not-found -o json >"$file" || return 4
  [[ -s "$file" ]] || return 1
  uid="$(jq_bounded -er '.metadata.uid | select(type=="string" and length>0)' "$file")" || return 4
  run="$(jq_bounded -er '.metadata.labels["platform.hyeonworks.com/smoke-run"] | select(type=="string" and length>0)' "$file")" || return 4
  printf '%s|%s\n' "$uid" "$run"
}

delete_owned_uid() {
  local kind=$1 namespace=$2 name=$3 uid=$4 path body response status state attempt
  [[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
  path="$(api_path_for_owned "$kind" "$namespace" "$name")" || return 1
  body="$("$MKTEMP_BIN" "$WORK/delete.XXXXXXXX")" || return 1
  response="$("$MKTEMP_BIN" "$WORK/delete-response.XXXXXXXX")" || return 1
  "$CHMOD_BIN" 0600 -- "$body" "$response"
  jq_bounded -nc --arg uid "$uid" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid}}' >"$body"
  status="$(proxy_request DELETE "$path" "$response" "$body")" || status=000
  for ((attempt=0; attempt<30; attempt++)); do
    state="$(owned_object_state "$kind" "$namespace" "$name" 2>/dev/null)" && {
      [[ "${state%%|*}" == "$uid" ]] || return 1
    } || {
      [[ $? == 1 ]] && return 0
      return 1
    }
    /usr/bin/sleep 1
  done
  return 1
}

register_owned() {
  OWNED_KIND+=("$1"); OWNED_NAMESPACE+=("$2"); OWNED_NAME+=("$3");
  OWNED_UID+=("$4"); OWNED_RUN+=("$5")
}

cleanup_owned_objects() {
  local index state uid run rc=0
  for ((index=${#OWNED_KIND[@]}-1; index>=0; index--)); do
    state="$(owned_object_state "${OWNED_KIND[$index]}" "${OWNED_NAMESPACE[$index]}" "${OWNED_NAME[$index]}" 2>/dev/null)" || {
      [[ $? == 1 ]] && continue
      emit_manual_recovery; rc=1; continue
    }
    IFS='|' read -r uid run <<<"$state"
    if [[ "$uid" != "${OWNED_UID[$index]}" || "$run" != "${OWNED_RUN[$index]}" ]]; then
      emit_manual_recovery; rc=1; continue
    fi
    delete_owned_uid "${OWNED_KIND[$index]}" "${OWNED_NAMESPACE[$index]}" "${OWNED_NAME[$index]}" "$uid" || {
      emit_manual_recovery; rc=1;
    }
  done
  OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=()
  return "$rc"
}

validate_commit_environment() {
  local permission result context api node_uid
  validate_command_boundaries || return 1
  context="$(kubectl_bounded config current-context)" || return 1
  [[ "$context" == "$EXPECTED_CONTEXT" ]] || { fail 'Kubernetes context is not default'; return 1; }
  api="$(kubectl_bounded config view --minify -o jsonpath='{.clusters[0].cluster.server}')" || return 1
  [[ "$api" == "$EXPECTED_API_SERVER" ]] || { fail 'Kubernetes API server is not the exact local endpoint'; return 1; }
  kubectl_bounded get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; }
  node_uid="$(kubectl_bounded get node "$EXPECTED_NODE" -o json | jq_bounded -er --arg node "$EXPECTED_NODE" '
    select(.metadata.name==$node and (.metadata.uid|type=="string" and length>0) and
      any(.status.conditions[]?; .type=="Ready" and .status=="True")) | .metadata.uid
  ')" || { fail 'target node identity/readiness changed'; return 1; }
  if [[ -z "$EXPECTED_NODE_UID" ]]; then EXPECTED_NODE_UID=$node_uid; fi
  [[ "$node_uid" == "$EXPECTED_NODE_UID" ]] || { fail 'target node UID changed'; return 1; }
  for permission in \
    'get pods observability' 'list pods observability' 'watch pods observability' \
    'create pods observability' 'delete pods observability' 'get pods/log observability' \
    'get services observability' 'get services object-storage' 'get services kube-system' \
    'get services/proxy observability' 'create services/proxy observability' \
    'delete services/proxy observability' \
    'get services/proxy keycloak' 'create services/proxy keycloak' \
    'update services/proxy keycloak' 'delete services/proxy keycloak' \
    'get persistentvolumeclaims observability' 'list persistentvolumeclaims observability' \
    'get configmaps observability' 'list configmaps observability' \
    'get deployments.apps observability' 'watch deployments.apps observability' \
    'get statefulsets.apps observability' 'watch statefulsets.apps observability' \
    'get networkpolicies.networking.k8s.io observability' \
    'create networkpolicies.networking.k8s.io observability' \
    'delete networkpolicies.networking.k8s.io observability' \
    'get prometheusrules.monitoring.coreos.com observability' \
    'create prometheusrules.monitoring.coreos.com observability' \
    'delete prometheusrules.monitoring.coreos.com observability' \
    'get secrets observability' 'get secrets keycloak'; do
    read -r -a _permission_parts <<<"$permission"
    result="$(kubectl_bounded auth can-i "${_permission_parts[0]}" "${_permission_parts[1]}" -n "${_permission_parts[2]}")" || return 1
    [[ "$result" == yes ]] || { fail "missing Kubernetes authority: $permission"; return 1; }
  done
}

require_execute_environment() {
  validate_commit_environment || return 1
  make_work || { fail 'cannot create private work directory'; return 1; }
  start_api_proxy || { fail 'cannot start bounded Kubernetes API proxy'; return 1; }
}

validate_command_boundaries() {
  local path certbot_real metadata
  for path in "$KUBECTL_BIN" "$CURL_BIN" "$DIG_BIN" "$OPENSSL_BIN" "$NC_BIN" \
    "$SUDO_BIN" "$DATE_BIN" "$TIMEOUT_BIN" "$JQ_BIN" "$PYTHON_BIN" \
    "$CAT_BIN" "$CHMOD_BIN" "$CMP_BIN" "$MKTEMP_BIN" "$MV_BIN" \
    "$READLINK_BIN" "$RM_BIN" "$STAT_BIN"; do
    [[ -x "$path" && ! -L "$path" ]] || { fail "required command boundary is unavailable: $path"; return 1; }
  done
  [[ -L "$CERTBOT_BIN" && -x "$CERTBOT_BIN" ]] || {
    fail "Certbot command boundary is unavailable: $CERTBOT_BIN"; return 1;
  }
  certbot_real="$("$READLINK_BIN" -f -- "$CERTBOT_BIN")" || return 1
  [[ "$certbot_real" == "$CERTBOT_REAL" && -x "$CERTBOT_REAL" && ! -L "$CERTBOT_REAL" ]] || {
    fail 'Certbot command boundary does not resolve to the fixed snap executable'; return 1;
  }
  metadata="$("$STAT_BIN" -c '%u:%g:%a:%F' -- "$CERTBOT_REAL")" || return 1
  [[ "$metadata" == '0:0:755:regular file' ]] || {
    fail 'Certbot real executable metadata is unsafe'; return 1;
  }
  [[ -x "$SYSTEMCTL_BIN" && ! -L "$SYSTEMCTL_BIN" &&
     "$("$STAT_BIN" -c '%u:%g:%a:%F' -- "$SYSTEMCTL_BIN")" == '0:0:755:regular file' ]] || {
    fail 'systemctl command boundary is unsafe'; return 1;
  }
}

dig_values() {
  local resolver=$1 record=$2 host=$3
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 8s "$DIG_BIN" +time=2 +tries=1 +short \
    "@$resolver" "$record" "$host" | /usr/bin/sed '/^[[:space:]]*$/d' | LC_ALL=C /usr/bin/sort -u
}

http_status() {
  local -a args=(--disable --silent --show-error --output /dev/null --write-out '%{http_code}'
    --connect-timeout 3 --max-time 12 --noproxy '*')
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$CURL_BIN" "${args[@]}" "$@"
}

check_dns_host() {
  local public_a public_aaaa lan_a lan_aaaa tail_a tail_aaaa
  public_a="$(dig_values 1.1.1.1 A "$GRAFANA_HOST")" || return 1
  public_aaaa="$(dig_values 1.1.1.1 AAAA "$GRAFANA_HOST")" || return 1
  lan_a="$(dig_values "$LAN_IP" A "$GRAFANA_HOST")" || return 1
  lan_aaaa="$(dig_values "$LAN_IP" AAAA "$GRAFANA_HOST")" || return 1
  tail_a="$(dig_values "$TAILSCALE_IP" A "$GRAFANA_HOST")" || return 1
  tail_aaaa="$(dig_values "$TAILSCALE_IP" AAAA "$GRAFANA_HOST")" || return 1
  validate_dns_contract "$public_a" "$public_aaaa" "$lan_a" "$lan_aaaa" \
    "$tail_a" "$tail_aaaa" "$LAN_IP" '' || return 1
  printf 'OBSERVABILITY_MACHINE_DNS_HOST=PASS\n'
}

check_certificate() {
  local raw cert extension verify_rc=0 expiry_rc=0
  raw="$WORK/grafana-sclient.txt"; cert="$WORK/grafana-leaf.pem"
  : >"$raw"; : >"$cert"; "$CHMOD_BIN" 0600 -- "$raw" "$cert"
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$OPENSSL_BIN" s_client \
    -connect "$LAN_IP:443" -servername "$GRAFANA_HOST" -showcerts </dev/null >"$raw" 2>/dev/null || return 1
  /usr/bin/awk '/-----BEGIN CERTIFICATE-----/{copy=1} copy{print} /-----END CERTIFICATE-----/{exit}' "$raw" >"$cert"
  [[ -s "$cert" ]] || return 1
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$OPENSSL_BIN" s_client -connect "$LAN_IP:443" -servername "$GRAFANA_HOST" \
    -verify_hostname "$GRAFANA_HOST" -verify_return_error </dev/null >/dev/null 2>&1 || verify_rc=$?
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$OPENSSL_BIN" x509 -in "$cert" -noout -checkend 604800 >/dev/null 2>&1 || expiry_rc=$?
  extension="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$OPENSSL_BIN" x509 \
    -in "$cert" -noout -ext subjectAltName 2>/dev/null)" || return 1
  validate_certificate_extension_contract "$extension" "$verify_rc" "$expiry_rc" || return 1
  printf 'OBSERVABILITY_MACHINE_CERTIFICATE=PASS\n'
}

check_nodeport_identity() {
  local service="$WORK/traefik-nodeport-service.json"
  kubectl_bounded -n kube-system get service traefik -o json >"$service" || return 1
  "$CHMOD_BIN" 0600 -- "$service"
  jq_bounded -e '
    .apiVersion=="v1" and .kind=="Service" and
    .metadata.name=="traefik" and .metadata.namespace=="kube-system" and
    (.metadata.uid|type=="string" and length>0) and
    .spec.type=="NodePort" and .spec.externalTrafficPolicy=="Cluster" and
    .spec.selector=={
      "app.kubernetes.io/name":"traefik",
      "app.kubernetes.io/instance":"traefik-kube-system"} and
    ([.spec.ports[]|{name,port,protocol,targetPort,nodePort}]|sort_by(.name))==([
      {name:"web",port:80,protocol:"TCP",targetPort:"web",nodePort:30080},
      {name:"websecure",port:443,protocol:"TCP",targetPort:"websecure",nodePort:30443}
    ]|sort_by(.name))
  ' "$service" >/dev/null || return 1
  [[ "$(http_status --header 'Host: observability-smoke.invalid' http://127.0.0.1:30080/)" == 404 ]]
}

check_https_boundaries() {
  local public_ip
  [[ "$(http_status --interface "$LAN_IP" --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/")" =~ ^(200|302)$ ]] || return 1
  [[ "$(http_status --interface "$TAILSCALE_IP" --resolve "$GRAFANA_HOST:443:$TAILSCALE_IP" "https://$GRAFANA_HOST/")" =~ ^(200|302)$ ]] || return 1
  [[ "$(http_status --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/")" == 403 ]] || return 1
  public_ip="$(dig_values 1.1.1.1 A "$GITEA_HOST" | /usr/bin/head -n1)" || return 1
  validate_ipv4 "$public_ip" || return 1
  if [[ -n "$PUBLIC_EDGE_IP" && "$PUBLIC_EDGE_IP" != "$public_ip" ]]; then return 1; fi
  PUBLIC_EDGE_IP=$public_ip
  [[ "$(http_status --resolve "$GRAFANA_HOST:443:$public_ip" "https://$GRAFANA_HOST/")" == 403 ]] || return 1
  [[ "$(http_status --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/metrics")" == 404 ]] || return 1
  [[ "$(http_status "https://$GITEA_HOST/metrics")" == 404 ]] || return 1
  [[ "$(http_status "https://$KEYCLOAK_HOST/metrics")" == 404 ]] || return 1
  [[ "$(http_status "$RUNBOOK_URL")" == 200 ]] || return 1
  if "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 8s "$OPENSSL_BIN" s_client -connect "$LAN_IP:443" \
      -servername unknown-sni.invalid </dev/null >/dev/null 2>&1; then return 1; fi
  ! "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$NC_BIN" -z -w 2 "$LAN_IP" 30080 || return 1
  ! "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$NC_BIN" -z -w 2 "$LAN_IP" 30443 || return 1
  check_nodeport_identity || return 1
  printf 'OBSERVABILITY_MACHINE_EDGE=PASS\n'
}

check_targets() {
  local file="$WORK/prometheus-targets.json"
  proxy_get_200 "$(prometheus_path '/api/v1/targets?state=active')" "$file" || return 1
  validate_target_contract "$file" || return 1
  printf 'OBSERVABILITY_MACHINE_TARGETS=PASS\n'
}

check_one_query() {
  local name=$1 expression=$2 count=$3 labels=$4 values=${5:-} value_label=${6:-}
  local identity_json=${7:-} instance_pattern=${8:-}
  local file="$WORK/query-$name.json"
  prom_query "$expression" "$file" || return 1
  validate_series_contract "$file" "$count" "$labels" "$values" "$value_label" || return 1
  [[ -z "$identity_json" ]] || validate_series_identity_contract "$file" "$identity_json" "$instance_pattern"
}

check_metric_contracts() {
  check_one_query gitea 'gitea_build_info{job="gitea",namespace="gitea"}' 1 'instance,job,namespace' '' '' \
    '[{"job":"gitea","namespace":"gitea"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:3000$' || return 1
  check_one_query keycloak 'process_uptime_seconds{job="keycloak-service",namespace="keycloak"}' 1 'instance,job,namespace' '' '' \
    '[{"job":"keycloak-service","namespace":"keycloak"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1
  check_one_query cnpg 'cnpg_collector_up{job="platform-postgres",namespace="platform-data",cluster="platform-postgres"}' 1 'cluster,instance,job,namespace' '' '' \
    '[{"cluster":"platform-postgres","job":"platform-postgres","namespace":"platform-data"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9187$' || return 1
  check_one_query traefik 'traefik_config_last_reload_success{job="traefik",namespace="kube-system"}' 1 'instance,job,namespace' '' '' \
    '[{"job":"traefik","namespace":"kube-system"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9100$' || return 1
  check_one_query aistor-usage 'minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}' 2 \
    'bucket,instance,job,namespace' 'loki,tempo' bucket \
    '[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \
    '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1
  check_one_query aistor-quota 'minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"} > 0' 2 \
    'bucket,instance,job,namespace' 'loki,tempo' bucket \
    '[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \
    '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1
  check_one_query quota-rule 'platform:aistor_bucket_quota_usage_percent' 2 'bucket' 'loki,tempo' bucket \
    '[{"bucket":"loki"},{"bucket":"tempo"}]' || return 1
  printf 'OBSERVABILITY_MACHINE_METRICS_QUOTA=PASS\n'
}

check_blackbox_series() {
  local public='[{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"},{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]'
  local private='[{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"},{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"},{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}]'
  local internal='[{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"},{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"},{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}]'
  check_one_query probe-public 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="public-edge"} == 1' 2 \
    'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$public" || return 1
  check_one_query probe-private 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"} == 1' 3 \
    'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$private" || return 1
  check_one_query probe-private-status 'probe_http_status_code{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"} == 403' 3 \
    'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$private" || return 1
  check_one_query probe-internal 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-internal"} == 1' 3 \
    'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$internal" || return 1
  local tls="$WORK/query-probe-tls.json"
  prom_query 'probe_ssl_earliest_cert_expiry{job=~"blackbox-public-edge|blackbox-private-edge"} > time()' "$tls" || return 1
  validate_series_identity_contract "$tls" "$(jq_bounded -cn --argjson public "$public" --argjson private "$private" '$public+$private')" || return 1
  printf 'OBSERVABILITY_MACHINE_BLACKBOX=PASS\n'
}

decode_exact_secret_to_netrc() {
  local secret_json=$1 output=$2
  python_bounded -I -S - "$secret_json" "$output" <<'PY'
import base64,binascii,json,os,pathlib,sys
source,destination=sys.argv[1:]
try: item=json.loads(pathlib.Path(source).read_text())
except Exception: raise SystemExit(1)
meta=item.get('metadata') or {}; data=item.get('data')
if item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='observability' or meta.get('name')!='grafana-admin' or sorted(data or {})!=['admin-password','admin-user']:
    raise SystemExit(1)
values=[]
for key in ('admin-user','admin-password'):
    try: value=base64.b64decode(data[key],validate=True).decode('utf-8')
    except (binascii.Error,UnicodeError,ValueError): raise SystemExit(1)
    if not value or any(ch.isspace() for ch in value) or '\x00' in value: raise SystemExit(1)
    values.append(value)
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
with os.fdopen(fd,'w') as stream: stream.write('machine localhost\nlogin '+values[0]+'\npassword '+values[1]+'\n')
PY
}

prepare_grafana_auth() {
  local secret="$WORK/grafana-admin-secret.json" netrc="$WORK/grafana.netrc" extra="$WORK/grafana-curl.conf"
  if safe_work_file "$netrc" && safe_work_file "$extra"; then
    printf '%s' "$extra"
    return 0
  fi
  kubectl_bounded -n observability get secret grafana-admin -o json >"$secret" || return 1
  "$CHMOD_BIN" 0600 -- "$secret"; safe_work_file "$secret" || return 1
  decode_exact_secret_to_netrc "$secret" "$netrc" || return 1
  printf 'netrc\nnetrc-file = "%s"\n' "$netrc" >"$extra"; "$CHMOD_BIN" 0600 -- "$extra"
  safe_work_file "$netrc" && safe_work_file "$extra" || return 1
  printf '%s' "$extra"
}

grafana_get() {
  local api_path=$1 output=$2 auth=$3
  proxy_get_200 "$(grafana_path "$api_path")" "$output" "$auth"
}

check_datasources() {
  local auth prometheus loki tempo loki_config tempo_config tempo_search loki_query tempo_query combined trace encoded query
  auth="$(prepare_grafana_auth)" || return 1
  prometheus="$WORK/grafana-prometheus-health.json"; loki="$WORK/grafana-loki-health.json"; tempo="$WORK/grafana-tempo-health.json"
  loki_config="$WORK/grafana-loki-config.json"; tempo_config="$WORK/grafana-tempo-config.json"
  tempo_search="$WORK/grafana-tempo-search.json"; loki_query="$WORK/grafana-loki-query.json"
  tempo_query="$WORK/grafana-tempo-query.json"; combined="$WORK/grafana-datasource-contract.json"
  grafana_get '/api/datasources/uid/prometheus/health' "$prometheus" "$auth" || return 1
  grafana_get '/api/datasources/uid/loki/health' "$loki" "$auth" || return 1
  grafana_get '/api/datasources/uid/tempo/health' "$tempo" "$auth" || return 1
  grafana_get '/api/datasources/uid/loki' "$loki_config" "$auth" || return 1
  grafana_get '/api/datasources/uid/tempo' "$tempo_config" "$auth" || return 1
  encoded="$(jq_bounded -nr --arg value 'service.name=platform-smoke' '$value|@uri')" || return 1
  grafana_get "/api/datasources/proxy/uid/tempo/api/search?tags=$encoded&limit=20" "$tempo_search" "$auth" || return 1
  trace="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$PYTHON_BIN" -I -S - "$tempo_search" <<'PY'
import json,pathlib,re,sys
try: item=json.loads(pathlib.Path(sys.argv[1]).read_text())
except Exception: raise SystemExit(1)
traces=item.get('traces')
if not isinstance(traces,list): raise SystemExit(1)
candidates=[]
for trace in traces:
    if not isinstance(trace,dict) or trace.get('rootServiceName')!='platform-smoke': continue
    identifier=trace.get('traceID')
    if isinstance(identifier,str) and re.fullmatch(r'[0-9a-f]{32}',identifier): candidates.append(identifier)
if not candidates: raise SystemExit(1)
print(candidates[0],end='')
PY
)" || return 1
  [[ "$trace" =~ ^[0-9a-f]{32}$ ]] || return 1
  query="$(synthetic_loki_query "$trace")" || return 1
  encoded="$(jq_bounded -nr --arg value "$query" '$value|@uri')" || return 1
  grafana_get "/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=$encoded&limit=20&direction=backward&since=720h" "$loki_query" "$auth" || return 1
  grafana_get "/api/datasources/proxy/uid/tempo/api/traces/$trace" "$tempo_query" "$auth" || return 1
  jq_bounded -n --slurpfile prometheus "$prometheus" --slurpfile loki "$loki" --slurpfile tempo "$tempo" \
    --slurpfile loki_config "$loki_config" --slurpfile tempo_config "$tempo_config" \
    --slurpfile tempo_search "$tempo_search" --slurpfile loki_query "$loki_query" --slurpfile tempo_query "$tempo_query" \
    --arg selected_trace "$trace" \
    '{prometheus:$prometheus[0],loki:$loki[0],tempo:$tempo[0],loki_config:$loki_config[0],tempo_config:$tempo_config[0],tempo_search:$tempo_search[0],selected_trace:$selected_trace,loki_query:$loki_query[0],tempo_query:$tempo_query[0]}' >"$combined" || return 1
  "$CHMOD_BIN" 0600 -- "$combined"
  validate_datasource_contract "$combined" || return 1
  printf 'OBSERVABILITY_MACHINE_DATASOURCES=PASS\n'
}

run_read_only_machine_checks() {
  check_dns_host || { fail 'DNS host acceptance failed'; return 1; }
  check_certificate || { fail 'certificate acceptance failed'; return 1; }
  check_https_boundaries || { fail 'HTTPS/NodePort boundary acceptance failed'; return 1; }
  check_targets || { fail 'Prometheus target acceptance failed'; return 1; }
  check_metric_contracts || { fail 'metric/quota acceptance failed'; return 1; }
  check_blackbox_series || { fail 'blackbox acceptance failed'; return 1; }
  check_datasources || { fail 'Grafana datasource/link acceptance failed'; return 1; }
}

repeat_commit_gate() {
  validate_commit_environment
}

validate_owned_object_json() {
  local file=$1 kind=$2 namespace=$3 name=$4 run=$5 require_uid=${6:-false}
  jq_bounded -e --arg kind "$kind" --arg namespace "$namespace" --arg name "$name" --arg run "$run" --argjson require_uid "$require_uid" '
    .kind==$kind and .metadata.namespace==$namespace and .metadata.name==$name and
    .metadata.labels["app.kubernetes.io/managed-by"]=="platform-observability-smoke" and
    .metadata.labels["platform.hyeonworks.com/smoke-run"]==$run and
    (($require_uid|not) or (.metadata.uid|type=="string" and length>0))
  ' "$file" >/dev/null
}

validate_live_matches_desired() {
  local desired=$1 live=$2
  python_bounded -I -S - "$desired" "$live" <<'PY'
import json,pathlib,sys
try:
    desired=json.loads(pathlib.Path(sys.argv[1]).read_text())
    live=json.loads(pathlib.Path(sys.argv[2]).read_text())
except Exception:
    raise SystemExit(1)

def subset(want,got):
    if isinstance(want,dict):
        return isinstance(got,dict) and all(key in got and subset(value,got[key]) for key,value in want.items())
    if isinstance(want,list):
        return isinstance(got,list) and len(want)==len(got) and all(subset(a,b) for a,b in zip(want,got))
    return want==got

if desired.get('apiVersion')!=live.get('apiVersion') or desired.get('kind')!=live.get('kind'):
    raise SystemExit(1)
dmeta=desired.get('metadata') or {}; lmeta=live.get('metadata') or {}
for field in ('name','namespace'):
    if dmeta.get(field)!=lmeta.get(field): raise SystemExit(1)
if dmeta.get('labels')!=lmeta.get('labels'): raise SystemExit(1)
if not isinstance(lmeta.get('uid'),str) or not lmeta['uid'] or not isinstance(lmeta.get('resourceVersion'),str) or not lmeta['resourceVersion']:
    raise SystemExit(1)
if lmeta.get('deletionTimestamp') or lmeta.get('ownerReferences') or lmeta.get('finalizers'):
    raise SystemExit(1)
if not subset(desired.get('spec'),live.get('spec')): raise SystemExit(1)
if desired.get('kind')=='Pod':
    dspec=desired.get('spec') or {}; lspec=live.get('spec') or {}
    if len(lspec.get('containers') or [])!=len(dspec.get('containers') or []): raise SystemExit(1)
    if lspec.get('initContainers') or lspec.get('ephemeralContainers'): raise SystemExit(1)
PY
}

create_owned_from_file() {
  local file=$1 kind=$2 namespace=$3 name=$4 run=$5 resource live rc=0 uid state outcome=0
  resource="$(kind_resource "$kind")" || return 1
  if state="$(owned_object_state "$kind" "$namespace" "$name" 2>/dev/null)"; then
    fail "temporary object already exists: $kind/$namespace/$name"; return 1
  else [[ $? == 1 ]] || return 1; fi
  validate_owned_object_json "$file" "$kind" "$namespace" "$name" "$run" false || return 1
  kubectl_bounded create --dry-run=server -f "$file" -o json >/dev/null || return 1
  live="$WORK/created-${kind}-${name}.json"
  begin_mutation_critical
  if kubectl_bounded create -f "$file" -o json >"$live"; then rc=0; else rc=$?; fi
  "$CHMOD_BIN" 0600 -- "$live"
  if (( rc != 0 )); then
    if ! kubectl_bounded -n "$namespace" get "$resource" "$name" --ignore-not-found -o json >"$live"; then
      emit_manual_recovery; outcome=1
    fi
  fi
  if (( outcome == 0 )) && [[ ! -s "$live" ]]; then emit_manual_recovery; outcome=1; fi
  if (( outcome == 0 )) && ! validate_owned_object_json "$live" "$kind" "$namespace" "$name" "$run" true; then
    emit_manual_recovery; outcome=1
  fi
  if (( outcome == 0 )); then
    uid="$(jq_bounded -r '.metadata.uid' "$live")" || { emit_manual_recovery; outcome=1; }
  fi
  if (( outcome == 0 && rc == 0 )); then
    # A successful create response is authoritative ownership even if an
    # admission/defaulting drift makes the object unsafe; register its UID so
    # the exit transaction can remove only that exact object.
    register_owned "$kind" "$namespace" "$name" "$uid" "$run"
    validate_live_matches_desired "$file" "$live" || outcome=1
  elif (( outcome == 0 )); then
    # A nonzero create is response-ambiguous. Copied labels are insufficient:
    # only an exact desired-spec match can be classified as ours and cleaned.
    if validate_live_matches_desired "$file" "$live"; then
      register_owned "$kind" "$namespace" "$name" "$uid" "$run"
    else
      emit_manual_recovery
    fi
    outcome=1
  fi
  end_mutation_critical
  return "$outcome"
}

validate_deployed_ingress_policy_contracts() {
  local aistor=$1 blackbox=$2
  jq_bounded -e '
    .apiVersion=="networking.k8s.io/v1" and .kind=="NetworkPolicy" and
    .metadata.name=="object-storage-allow-prometheus-metrics" and
    .metadata.namespace=="object-storage" and
    (.metadata.uid|type=="string" and length>0) and
    (.metadata.resourceVersion|type=="string" and length>0) and
    (.metadata.deletionTimestamp==null) and
    .spec=={
      podSelector:{matchLabels:{"aistor.min.io/objectStore":"minio-aistor"}},
      policyTypes:["Ingress"],
      ingress:[{from:[{
        namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"observability"}},
        podSelector:{matchLabels:{
          "app.kubernetes.io/name":"prometheus",
          "app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}
      }],ports:[{protocol:"TCP",port:9000}]}]
    }
  ' "$aistor" >/dev/null || return 1
  jq_bounded -e '
    .apiVersion=="networking.k8s.io/v1" and .kind=="NetworkPolicy" and
    .metadata.name=="observability-allow-prometheus-to-blackbox" and
    .metadata.namespace=="observability" and
    (.metadata.uid|type=="string" and length>0) and
    (.metadata.resourceVersion|type=="string" and length>0) and
    (.metadata.deletionTimestamp==null) and
    .spec=={
      podSelector:{matchLabels:{
        "app.kubernetes.io/name":"prometheus-blackbox-exporter",
        "app.kubernetes.io/instance":"blackbox-exporter"}},
      policyTypes:["Ingress"],
      ingress:[{from:[{podSelector:{matchLabels:{
        "app.kubernetes.io/name":"prometheus",
        "app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}
      }],ports:[{protocol:"TCP",port:9115}]}]
    }
  ' "$blackbox" >/dev/null
}

run_network_boundary_transaction() {
  local run name policy pod minio_ip blackbox_ip pod_result aistor_ingress blackbox_ingress
  run="$(new_smoke_run_id network)" || return 1
  name="$(kubernetes_smoke_name "$run")" || return 1
  policy="$WORK/network-policy.json"; pod="$WORK/network-pod.json"
  aistor_ingress="$WORK/aistor-ingress-policy.json"
  blackbox_ingress="$WORK/blackbox-ingress-policy.json"
  kubectl_bounded -n object-storage get networkpolicy object-storage-allow-prometheus-metrics -o json >"$aistor_ingress" || return 1
  kubectl_bounded -n observability get networkpolicy observability-allow-prometheus-to-blackbox -o json >"$blackbox_ingress" || return 1
  "$CHMOD_BIN" 0600 -- "$aistor_ingress" "$blackbox_ingress"
  validate_deployed_ingress_policy_contracts "$aistor_ingress" "$blackbox_ingress" || return 1
  minio_ip="$(kubectl_bounded -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" || return 1
  blackbox_ip="$(kubectl_bounded -n observability get service blackbox-exporter -o jsonpath='{.spec.clusterIP}')" || return 1
  [[ "$minio_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$blackbox_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || return 1
  jq_bounded -n --arg name "$name" --arg run "$run" '{
    apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",
    metadata:{name:$name,namespace:"observability",labels:{
      "app.kubernetes.io/managed-by":"platform-observability-smoke",
      "platform.hyeonworks.com/smoke-run":$run}},
    spec:{podSelector:{matchLabels:{"platform.hyeonworks.com/smoke-run":$run}},
      policyTypes:["Egress"],egress:[{to:[{
        namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"kube-system"}},
        podSelector:{matchLabels:{"k8s-app":"kube-dns"}}}],
        ports:[{protocol:"UDP",port:53},{protocol:"TCP",port:53}]},{to:[{
          namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"object-storage"}},
          podSelector:{matchLabels:{"aistor.min.io/objectStore":"minio-aistor"}}}],
        ports:[{protocol:"TCP",port:9000}]},{to:[{
          namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"observability"}},
          podSelector:{matchLabels:{
            "app.kubernetes.io/name":"prometheus-blackbox-exporter",
            "app.kubernetes.io/instance":"blackbox-exporter"}}}],
        ports:[{protocol:"TCP",port:9115}]}]}}
  ' >"$policy" || return 1
  "$CHMOD_BIN" 0600 -- "$policy"
  local command
  command="set -eu
resolved=\$(nslookup $GRAFANA_HOST | awk '/^Address: /{print \$2}' | tail -n1)
[ \"\$resolved\" = $LAN_IP ]
if nc -z -w 4 $minio_ip 9000; then exit 41; fi
if nc -z -w 4 $blackbox_ip 9115; then exit 42; fi
printf '%s\\n' OBSERVABILITY_NETWORK_BOUNDARY_PASS"
  jq_bounded -n --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg command "$command" '{
    apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:"observability",labels:{
      "app.kubernetes.io/managed-by":"platform-observability-smoke",
      "platform.hyeonworks.com/smoke-run":$run}},
    spec:{automountServiceAccountToken:false,restartPolicy:"Never",terminationGracePeriodSeconds:1,
      securityContext:{runAsNonRoot:true,runAsUser:65534,runAsGroup:65534,seccompProfile:{type:"RuntimeDefault"}},
      containers:[{name:"network-smoke",image:$image,imagePullPolicy:"IfNotPresent",
        command:["sh","-ec",$command],securityContext:{allowPrivilegeEscalation:false,
          readOnlyRootFilesystem:true,capabilities:{drop:["ALL"]}}}]}}
  ' >"$pod" || return 1
  "$CHMOD_BIN" 0600 -- "$pod"
  create_owned_from_file "$policy" NetworkPolicy observability "$name" "$run" || return 1
  create_owned_from_file "$pod" Pod observability "$name" "$run" || return 1
  kubectl_wait_90_bounded -n observability wait --for=jsonpath='{.status.phase}'=Succeeded \
    --timeout=90s "pod/$name" >/dev/null || return 1
  pod_result="$(kubectl_bounded -n observability logs "$name" -c network-smoke)" || return 1
  [[ "$pod_result" == OBSERVABILITY_NETWORK_BOUNDARY_PASS ]] || return 1
  cleanup_owned_objects || return 1
  printf 'OBSERVABILITY_MACHINE_NETWORK_POLICY=PASS\n'
}

canonical_object_snapshot() {
  local file=$1
  jq_bounded -S '{metadata:{name:.metadata.name,namespace:.metadata.namespace,uid:.metadata.uid},data:(.data//{}),status:(.status//{})}' "$file"
}

list_one_ready_pod() {
  local selector=$1 output=$2
  kubectl_bounded -n observability get pods -l "$selector" -o json >"$output" || return 1
  "$CHMOD_BIN" 0600 -- "$output"
  jq_bounded -e '.items|length==1 and .[0].metadata.uid and any(.[0].status.conditions[]?; .type=="Ready" and .status=="True")' "$output" >/dev/null
}

delete_pod_uid_for_restart() {
  local name=$1 uid=$2 body response path status attempt current
  body="$("$MKTEMP_BIN" "$WORK/restart-delete.XXXXXXXX")" || return 1
  response="$("$MKTEMP_BIN" "$WORK/restart-response.XXXXXXXX")" || return 1
  "$CHMOD_BIN" 0600 -- "$body" "$response"
  jq_bounded -nc --arg uid "$uid" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid}}' >"$body"
  path="/api/v1/namespaces/observability/pods/$name"
  status="$(proxy_request DELETE "$path" "$response" "$body")" || status=000
  for ((attempt=0; attempt<60; attempt++)); do
    current="$(kubectl_bounded -n observability get pod "$name" --ignore-not-found -o json 2>/dev/null | jq_bounded -r '.metadata.uid//empty')" || return 1
    [[ -z "$current" || "$current" != "$uid" ]] && return 0
    /usr/bin/sleep 1
  done
  return 1
}

fetch_silences() {
  local output=$1 status
  status="$(proxy_request GET "$(alertmanager_path '/api/v2/silences')" "$output")" || return 1
  [[ "$status" == 200 ]] || return 1
  jq_bounded -e 'type=="array"' "$output" >/dev/null
}

silence_inventory_state() {
  local source=$1 run=$2
  python_bounded -I -S - "$source" "$run" <<'PY'
import json,pathlib,re,sys
try: items=json.loads(pathlib.Path(sys.argv[1]).read_text())
except Exception: raise SystemExit(1)
run=sys.argv[2]
if not isinstance(items,list) or re.fullmatch(r'persistence-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}',run) is None: raise SystemExit(1)
related=[]; active=[]; expired=[]; structural=[]
expected={
 ('alertname','PlatformObservabilityPersistenceSmoke',False,True),
 ('instance',run,False,True),
}
for item in items:
    if not isinstance(item,dict): raise SystemExit(1)
    matchers=item.get('matchers')
    if not isinstance(matchers,list): continue
    tuples=[]
    malformed=False
    for matcher in matchers:
        if not isinstance(matcher,dict): malformed=True; break
        value=(matcher.get('name'),matcher.get('value'),matcher.get('isRegex'),matcher.get('isEqual'))
        tuples.append(value)
    if malformed: continue
    if any(value[0]=='instance' and value[1]==run for value in tuples): related.append(item)
    state=((item.get('status') or {}).get('state'))
    if set(tuples)==expected and len(tuples)==2 and item.get('createdBy')=='platform-observability-smoke' and item.get('comment')=='Temporary restart-persistence acceptance '+run:
        identifier=item.get('id')
        if not isinstance(identifier,str) or re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',identifier) is None: raise SystemExit(1)
        structural.append(identifier)
        if state in ('active','pending'): active.append(identifier)
        elif state=='expired': expired.append(identifier)
print(f'{len(related)}|{len(active)}|{len(expired)}|{structural[0] if len(structural)==1 else ""}')
PY
}

read_silence_state() {
  local run=$1 output
  output="$("$MKTEMP_BIN" "$WORK/silence-list.XXXXXXXX")" || return 1
  "$CHMOD_BIN" 0600 -- "$output"
  fetch_silences "$output" || return 1
  silence_inventory_state "$output" "$run"
}

create_persistence_silence() {
  local body="$WORK/silence-create.json" response="$WORK/silence-create-response.json"
  local status='' start end state related active expired observed_id response_id='' request_rc=0 attempt outcome=1 classified=false
  SILENCE_RUN="$(new_smoke_run_id persistence)" || return 1
  state="$(read_silence_state "$SILENCE_RUN")" || return 1
  IFS='|' read -r related active expired observed_id <<<"$state"
  if [[ "$related" != 0 || "$active" != 0 || "$expired" != 0 || -n "$observed_id" ]]; then
    emit_manual_recovery; return 1
  fi
  start="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1
  end="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u -d '+20 minutes' +%Y-%m-%dT%H:%M:%SZ)" || return 1
  jq_bounded -n --arg start "$start" --arg end "$end" --arg run "$SILENCE_RUN" '{
    matchers:[
      {name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},
      {name:"instance",value:$run,isRegex:false,isEqual:true}
    ],startsAt:$start,endsAt:$end,createdBy:"platform-observability-smoke",
    comment:("Temporary restart-persistence acceptance "+$run)
  }' >"$body" || return 1
  "$CHMOD_BIN" 0600 -- "$body"
  begin_mutation_critical
  status="$(proxy_request POST "$(alertmanager_path '/api/v2/silences')" "$response" "$body")" || request_rc=$?
  if (( request_rc == 0 )) && [[ "$status" == 200 ]]; then
    response_id="$(jq_bounded -er '.silenceID|select(type=="string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))' "$response" 2>/dev/null)" || response_id=''
  fi
  for ((attempt=0; attempt<SILENCE_POLL_ATTEMPTS; attempt++)); do
    state="$(read_silence_state "$SILENCE_RUN")" || { (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"; continue; }
    IFS='|' read -r related active expired observed_id <<<"$state"
    if [[ "$related" == 1 && "$active" == 1 && "$expired" == 0 && "$observed_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
      if [[ -n "$response_id" && "$response_id" != "$observed_id" ]]; then
        emit_manual_recovery
      else
        SILENCE_ID=$observed_id
        outcome=0
      fi
      classified=true
      break
    fi
    if [[ "$related" != 0 || "$active" != 0 || "$expired" != 0 ]]; then emit_manual_recovery; classified=true; break; fi
    (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"
  done
  [[ "$classified" == true ]] || emit_manual_recovery
  end_mutation_critical
  return "$outcome"
}

delete_persistence_silence() {
  local response status='' request_rc=0 state related active expired observed_id attempt terminal=0
  if [[ -z "$SILENCE_ID" && -z "$SILENCE_RUN" ]]; then return 0; fi
  if [[ ! "$SILENCE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ || ! "$SILENCE_RUN" =~ ^persistence-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}$ ]]; then
    emit_manual_recovery; return 1
  fi
  response="$WORK/silence-delete-response.json"
  status="$(proxy_request DELETE "$(alertmanager_path "/api/v2/silence/$SILENCE_ID")" "$response")" || request_rc=$?
  for ((attempt=0; attempt<SILENCE_POLL_ATTEMPTS; attempt++)); do
    state="$(read_silence_state "$SILENCE_RUN")" || { terminal=0; (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"; continue; }
    IFS='|' read -r related active expired observed_id <<<"$state"
    if [[ ( "$related" == 0 && "$active" == 0 && "$expired" == 0 && -z "$observed_id" ) ||
          ( "$related" == 1 && "$active" == 0 && "$expired" == 1 && "$observed_id" == "$SILENCE_ID" ) ]]; then
      terminal=$((terminal + 1))
      if (( terminal >= 2 )); then SILENCE_ID=''; SILENCE_RUN=''; return 0; fi
    else
      terminal=0
      if [[ "$related" != 1 || "$active" != 1 || "$expired" != 0 || "$observed_id" != "$SILENCE_ID" ]]; then
        emit_manual_recovery; return 1
      fi
    fi
    (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"
  done
  emit_manual_recovery
  return 1
}

run_restart_persistence_transaction() {
  local grafana_before="$WORK/grafana-pod-before.json" grafana_after="$WORK/grafana-pod-after.json"
  local alert_before="$WORK/alertmanager-pod-before.json" alert_after="$WORK/alertmanager-pod-after.json"
  local pvc_before="$WORK/pvc-before.json" pvc_after="$WORK/pvc-after.json"
  local dashboards_before="$WORK/dashboards-before.json" dashboards_after="$WORK/dashboards-after.json"
  local api_before="$WORK/dashboard-api-before.json" api_after="$WORK/dashboard-api-after.json"
  local silence="$WORK/silence-after-restart.json" auth name uid state related active expired observed_id
  auth="$WORK/grafana-curl.conf"; safe_work_file "$auth" || return 1
  list_one_ready_pod 'app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana' "$grafana_before" || return 1
  list_one_ready_pod 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' "$alert_before" || return 1
  {
    kubectl_bounded -n observability get pvc grafana -o json
    kubectl_bounded -n observability get pvc -l 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' -o json
  } | jq_bounded -s '.[0] as $grafana | .[1] as $am | {grafana:{name:$grafana.metadata.name,uid:$grafana.metadata.uid,phase:$grafana.status.phase},alertmanager:[$am.items[]|{name:.metadata.name,uid:.metadata.uid,phase:.status.phase}]}' >"$pvc_before" || return 1
  kubectl_bounded -n observability get configmaps -l grafana_dashboard=1 -o json | jq_bounded -S \
    '[.items[]|{name:.metadata.name,uid:.metadata.uid,data:.data}]|sort_by(.name)' >"$dashboards_before" || return 1
  grafana_get '/api/search?type=dash-db' "$api_before" "$auth" || return 1
  jq_bounded -S '[.[]|{uid,title,type}]|sort_by(.uid)' "$api_before" >"$api_before.canonical" || return 1
  "$MV_BIN" -- "$api_before.canonical" "$api_before"
  "$CHMOD_BIN" 0600 -- "$pvc_before" "$dashboards_before" "$api_before"
  jq_bounded -e '.grafana.phase=="Bound" and (.alertmanager|length==1) and all(.alertmanager[];.phase=="Bound")' "$pvc_before" >/dev/null || return 1
  jq_bounded -e 'length==5' "$dashboards_before" >/dev/null || return 1
  jq_bounded -e 'length==5 and all(.[];.uid|type=="string" and length>0)' "$api_before" >/dev/null || return 1
  create_persistence_silence || return 1

  name="$(jq_bounded -r '.items[0].metadata.name' "$grafana_before")"; uid="$(jq_bounded -r '.items[0].metadata.uid' "$grafana_before")"
  delete_pod_uid_for_restart "$name" "$uid" || return 1
  kubectl_wait_180_bounded -n observability rollout status deployment/grafana --timeout=180s >/dev/null || return 1
  name="$(jq_bounded -r '.items[0].metadata.name' "$alert_before")"; uid="$(jq_bounded -r '.items[0].metadata.uid' "$alert_before")"
  delete_pod_uid_for_restart "$name" "$uid" || return 1
  kubectl_wait_180_bounded -n observability rollout status statefulset/alertmanager-observability-core-kube-pr-alertmanager --timeout=180s >/dev/null || return 1

  list_one_ready_pod 'app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana' "$grafana_after" || return 1
  list_one_ready_pod 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' "$alert_after" || return 1
  [[ "$(jq_bounded -r '.items[0].metadata.uid' "$grafana_before")" != "$(jq_bounded -r '.items[0].metadata.uid' "$grafana_after")" ]] || return 1
  [[ "$(jq_bounded -r '.items[0].metadata.uid' "$alert_before")" != "$(jq_bounded -r '.items[0].metadata.uid' "$alert_after")" ]] || return 1
  {
    kubectl_bounded -n observability get pvc grafana -o json
    kubectl_bounded -n observability get pvc -l 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' -o json
  } | jq_bounded -s '.[0] as $grafana | .[1] as $am | {grafana:{name:$grafana.metadata.name,uid:$grafana.metadata.uid,phase:$grafana.status.phase},alertmanager:[$am.items[]|{name:.metadata.name,uid:.metadata.uid,phase:.status.phase}]}' >"$pvc_after" || return 1
  kubectl_bounded -n observability get configmaps -l grafana_dashboard=1 -o json | jq_bounded -S \
    '[.items[]|{name:.metadata.name,uid:.metadata.uid,data:.data}]|sort_by(.name)' >"$dashboards_after" || return 1
  grafana_get '/api/search?type=dash-db' "$api_after" "$auth" || return 1
  jq_bounded -S '[.[]|{uid,title,type}]|sort_by(.uid)' "$api_after" >"$api_after.canonical" || return 1
  "$MV_BIN" -- "$api_after.canonical" "$api_after"
  "$CMP_BIN" --silent -- "$pvc_before" "$pvc_after" || return 1
  "$CMP_BIN" --silent -- "$dashboards_before" "$dashboards_after" || return 1
  "$CMP_BIN" --silent -- "$api_before" "$api_after" || return 1
  fetch_silences "$silence" || return 1
  state="$(silence_inventory_state "$silence" "$SILENCE_RUN")" || return 1
  IFS='|' read -r related active expired observed_id <<<"$state"
  [[ "$related" == 1 && "$active" == 1 && "$expired" == 0 && "$observed_id" == "$SILENCE_ID" ]] || return 1
  delete_persistence_silence || return 1
  printf 'OBSERVABILITY_MACHINE_RESTART_PERSISTENCE=PASS\n'
}

run_certbot_acceptance() {
  "$TIMEOUT_BIN" --signal=TERM --kill-after=2s 30s "$SUDO_BIN" -v || return 1
  "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$SUDO_BIN" -n /usr/bin/true || return 1
  "$TIMEOUT_BIN" --signal=TERM --kill-after=5s 610s "$SUDO_BIN" -n "$TIMEOUT_BIN" --signal=TERM --kill-after=5s 600s \
    /snap/bin/certbot renew --dry-run --run-deploy-hooks >/dev/null || return 1
  [[ "$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$SUDO_BIN" -n /usr/bin/systemctl is-active nginx)" == active ]] || return 1
  printf 'OBSERVABILITY_MACHINE_CERTBOT=PASS\n'
}

decode_keycloak_admin_secret() {
  local secret_json=$1 username_file=$2 password_file=$3
  python_bounded -I -S - "$secret_json" "$username_file" "$password_file" <<'PY'
import base64,binascii,json,os,pathlib,sys
source,user_path,password_path=sys.argv[1:]
try: item=json.loads(pathlib.Path(source).read_text())
except Exception: raise SystemExit(1)
meta=item.get('metadata') or {}; data=item.get('data')
if item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='keycloak' or meta.get('name')!='keycloak-initial-admin' or sorted(data or {})!=['password','username']:
    raise SystemExit(1)
for key,destination in [('username',user_path),('password',password_path)]:
    try: value=base64.b64decode(data[key],validate=True)
    except (binascii.Error,ValueError): raise SystemExit(1)
    if not value or b'\x00' in value or b'\r' in value or b'\n' in value: raise SystemExit(1)
    fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
    with os.fdopen(fd,'wb') as stream: stream.write(value)
PY
}

prepare_keycloak_auth() {
  local secret="$WORK/keycloak-admin-secret.json" username="$WORK/keycloak-admin-user" password="$WORK/keycloak-admin-password"
  local form="$WORK/keycloak-token-form" config="$WORK/keycloak-token-curl.conf" response="$WORK/keycloak-token.json" status token
  kubectl_bounded -n keycloak get secret keycloak-initial-admin -o json >"$secret" || return 1
  "$CHMOD_BIN" 0600 -- "$secret"; decode_keycloak_admin_secret "$secret" "$username" "$password" || return 1
  python_bounded -I -S - "$username" "$password" "$form" <<'PY'
import os,pathlib,sys,urllib.parse
user,password,destination=sys.argv[1:]
values={'grant_type':'password','client_id':'admin-cli','username':pathlib.Path(user).read_text(),'password':pathlib.Path(password).read_text()}
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
with os.fdopen(fd,'w') as stream: stream.write(urllib.parse.urlencode(values))
PY
  printf 'header = "Content-Type: application/x-www-form-urlencoded"\ndata-binary = "@%s"\n' "$form" >"$config"; "$CHMOD_BIN" 0600 -- "$config"
  status="$(proxy_request POST "$(keycloak_path '/realms/master/protocol/openid-connect/token')" "$response" '' "$config")" || return 1
  [[ "$status" == 200 ]] || return 1
  token="$(jq_bounded -er '.access_token|select(type=="string" and test("^[A-Za-z0-9._~-]+$"))' "$response")" || return 1
  KEYCLOAK_AUTH_CONFIG="$WORK/keycloak-auth.conf"
  printf 'header = "Authorization: Bearer %s"\n' "$token" >"$KEYCLOAK_AUTH_CONFIG"; "$CHMOD_BIN" 0600 -- "$KEYCLOAK_AUTH_CONFIG"
  safe_work_file "$KEYCLOAK_AUTH_CONFIG"
}

keycloak_request() {
  local method=$1 api_path=$2 output=$3 body=${4:-} status
  safe_work_file "$KEYCLOAK_AUTH_CONFIG" || return 1
  status="$(proxy_request "$method" "$(keycloak_path "$api_path")" "$output" "$body" "$KEYCLOAK_AUTH_CONFIG")" || return 1
  printf '%s' "$status"
}

keycloak_lookup_group() {
  local name=$1 expected_path=$2 output=$3 encoded status
  encoded="$(jq_bounded -nr --arg value "$name" '$value|@uri')" || return 1
  status="$(keycloak_request GET "/admin/realms/hyeonworks/groups?search=$encoded&exact=true&max=2" "$output")" || return 1
  [[ "$status" == 200 ]] || return 1
  jq_bounded -er --arg name "$name" --arg path "$expected_path" '
    [ .[] | select(.name==$name and .path==$path) ] as $items |
    select(($items|length)==1) | $items[0].id | select(type=="string" and test("^[A-Za-z0-9._-]+$"))
  ' "$output"
}

keycloak_lookup_user() {
  local username=$1 output=$2 encoded status
  encoded="$(jq_bounded -nr --arg value "$username" '$value|@uri')" || return 1
  status="$(keycloak_request GET "/admin/realms/hyeonworks/users?username=$encoded&exact=true&max=2" "$output")" || return 1
  [[ "$status" == 200 ]] || return 1
  jq_bounded -er --arg username "$username" '
    [ .[] | select(.username==$username and .enabled==true) ] as $items |
    select(($items|length)==1) | $items[0].id | select(type=="string" and test("^[A-Za-z0-9._-]+$"))
  ' "$output"
}

keycloak_memberships() {
  local output=$1 status page aggregate next count first=0 pages=0
  aggregate="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-all.XXXXXXXX")" || return 1
  printf '[]\n' >"$aggregate"; "$CHMOD_BIN" 0600 -- "$aggregate"
  while (( pages < 100 )); do
    page="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-page.XXXXXXXX")" || return 1
    "$CHMOD_BIN" 0600 -- "$page"
    status="$(keycloak_request GET "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups?briefRepresentation=true&first=$first&max=100" "$page")" || return 1
    [[ "$status" == 200 ]] || return 1
    count="$(jq_bounded -er 'select(type=="array" and length<=100)|length' "$page")" || return 1
    next="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-next.XXXXXXXX")" || return 1
    "$CHMOD_BIN" 0600 -- "$next"
    jq_bounded -s '.[0]+.[1]' "$aggregate" "$page" >"$next" || return 1
    "$MV_BIN" -- "$next" "$aggregate"
    pages=$((pages + 1))
    (( count == 100 )) || break
    first=$((first + 100))
  done
  (( pages < 100 || count < 100 )) || return 1
  jq_bounded -S '
    select(type=="array" and all(.[];
      (.id|type=="string" and test("^[A-Za-z0-9._-]+$")) and
      (.name|type=="string" and length>0) and (.path|type=="string" and startswith("/")))) |
    select(([.[].id]|length)==([.[].id]|unique|length)) |
    sort_by(.id)
  ' "$aggregate" >"$output" || return 1
  "$CHMOD_BIN" 0600 -- "$output"
}

keycloak_membership_state() {
  local memberships=$1 group_id=$2
  jq_bounded -er --arg id "$group_id" '
    [ .[] | select(.id==$id) ] as $items |
    if ($items|length)==0 then "absent"
    elif ($items|length)==1 and $items[0].name=="platform-observability-viewers" and
      $items[0].path=="/platform-observability-viewers" then "present"
    else error("ambiguous viewer membership") end
  ' "$memberships"
}

capture_keycloak_absent_baseline() {
  local admin_group_id=$1 first second state
  [[ "$admin_group_id" =~ ^[A-Za-z0-9._-]+$ && "$KEYCLOAK_VIEWER_GROUP_ID" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
  first="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-before-a.XXXXXXXX")" || return 1
  second="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-before-b.XXXXXXXX")" || return 1
  keycloak_memberships "$first" || return 1
  keycloak_memberships "$second" || return 1
  "$CMP_BIN" --silent -- "$first" "$second" || return 1
  state="$(keycloak_membership_state "$first" "$KEYCLOAK_VIEWER_GROUP_ID")" || return 1
  [[ "$state" == absent ]] || return 1
  jq_bounded -e --arg admin "$admin_group_id" 'all(.[]?; .id!=$admin)' "$first" >/dev/null || return 1
  [[ -n "$KEYCLOAK_MEMBERSHIP_BEFORE" && "$KEYCLOAK_MEMBERSHIP_BEFORE" == "$WORK"/* ]] || return 1
  "$MV_BIN" -- "$first" "$KEYCLOAK_MEMBERSHIP_BEFORE"
  "$CHMOD_BIN" 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE"
}

add_keycloak_viewer_membership() {
  local response memberships status=000 state outcome=1
  [[ -f "$KEYCLOAK_MEMBERSHIP_BEFORE" && ! -L "$KEYCLOAK_MEMBERSHIP_BEFORE" ]] || return 1
  [[ "$(keycloak_membership_state "$KEYCLOAK_MEMBERSHIP_BEFORE" "$KEYCLOAK_VIEWER_GROUP_ID")" == absent ]] || return 1
  response="$WORK/keycloak-membership-add-response.json"
  memberships="$WORK/keycloak-memberships-added.json"
  begin_mutation_critical
  KEYCLOAK_MEMBERSHIP_PENDING=true
  status="$(keycloak_request PUT "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups/$KEYCLOAK_VIEWER_GROUP_ID" "$response")" || status=000
  if keycloak_memberships "$memberships"; then
    state="$(keycloak_membership_state "$memberships" "$KEYCLOAK_VIEWER_GROUP_ID")" || state=ambiguous
    if [[ "$state" == present ]]; then
      KEYCLOAK_TEST_ADDED=true
      KEYCLOAK_MEMBERSHIP_PENDING=false
      outcome=0
    elif [[ "$state" == absent ]]; then
      KEYCLOAK_MEMBERSHIP_PENDING=false
    fi
  fi
  end_mutation_critical
  (( outcome == 0 )) || return 1
  [[ "$status" == 204 || "$status" == 000 ]]
}

cleanup_keycloak_membership() {
  local response status memberships verify state
  [[ ( "$KEYCLOAK_TEST_ADDED" == true || "$KEYCLOAK_MEMBERSHIP_PENDING" == true ) &&
     "$KEYCLOAK_MEMBERSHIP_REMOVED" == false ]] || return 0
  [[ "$KEYCLOAK_USER_ID" =~ ^[A-Za-z0-9._-]+$ && "$KEYCLOAK_VIEWER_GROUP_ID" =~ ^[A-Za-z0-9._-]+$ ]] || {
    emit_manual_recovery; return 1;
  }
  [[ -f "$KEYCLOAK_MEMBERSHIP_BEFORE" && ! -L "$KEYCLOAK_MEMBERSHIP_BEFORE" &&
     "$(keycloak_membership_state "$KEYCLOAK_MEMBERSHIP_BEFORE" "$KEYCLOAK_VIEWER_GROUP_ID" 2>/dev/null)" == absent ]] || {
    emit_manual_recovery; return 1;
  }
  memberships="$WORK/keycloak-cleanup-memberships-before.json"
  keycloak_memberships "$memberships" || { emit_manual_recovery; return 1; }
  state="$(keycloak_membership_state "$memberships" "$KEYCLOAK_VIEWER_GROUP_ID")" || {
    emit_manual_recovery; return 1;
  }
  if [[ "$state" == absent ]]; then
    KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=true
    return 0
  fi
  response="$WORK/keycloak-cleanup-response.json"
  status="$(keycloak_request DELETE "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups/$KEYCLOAK_VIEWER_GROUP_ID" "$response")" || status=000
  verify="$WORK/keycloak-cleanup-memberships-after.json"
  keycloak_memberships "$verify" || { emit_manual_recovery; return 1; }
  [[ "$(keycloak_membership_state "$verify" "$KEYCLOAK_VIEWER_GROUP_ID")" == absent ]] || {
    emit_manual_recovery; return 1;
  }
  KEYCLOAK_MEMBERSHIP_PENDING=false
  KEYCLOAK_MEMBERSHIP_REMOVED=true
}

run_oidc_acceptance() {
  local identity admin_identity viewer_identity denied_identity removal_identity
  local admin_group viewer_group user memberships response status encoded grafana_user grafana_auth deadline now
  printf '%s\n' 'Browser check: organization Admin and server-admin=false.' >&2
  admin_identity="$(read_identity 'OIDC organization Admin')" || return 1
  require_confirmation OIDC_ADMIN_CONFIRMED || return 1
  printf '%s\n' 'Browser check: Viewer and edit/admin actions denied.' >&2
  viewer_identity="$(read_identity 'OIDC Viewer')" || return 1
  require_confirmation OIDC_VIEWER_CONFIRMED || return 1
  printf '%s\n' 'Browser check: no observability groups and login denied.' >&2
  denied_identity="$(read_identity 'OIDC denied-user')" || return 1
  require_confirmation OIDC_DENIED_CONFIRMED || return 1
  printf '%s\n' 'Browser check: local break-glass login and server-admin=true.' >&2
  require_confirmation BREAK_GLASS_CONFIRMED || return 1
  [[ "$admin_identity" != "$viewer_identity" && "$admin_identity" != "$denied_identity" && "$viewer_identity" != "$denied_identity" ]] || return 1

  removal_identity="$(read_identity 'group-removal test')" || return 1
  [[ "$removal_identity" != "$admin_identity" && "$removal_identity" != "$viewer_identity" && "$removal_identity" != "$denied_identity" ]] || return 1
  prepare_keycloak_auth || return 1
  admin_group="$WORK/keycloak-admin-group.json"; viewer_group="$WORK/keycloak-viewer-group.json"; user="$WORK/keycloak-removal-user.json"
  local admin_group_id
  admin_group_id="$(keycloak_lookup_group platform-observability-admins /platform-observability-admins "$admin_group")" || return 1
  KEYCLOAK_VIEWER_GROUP_ID="$(keycloak_lookup_group platform-observability-viewers /platform-observability-viewers "$viewer_group")" || return 1
  KEYCLOAK_USER_ID="$(keycloak_lookup_user "$removal_identity" "$user")" || return 1
  KEYCLOAK_MEMBERSHIP_BEFORE="$WORK/keycloak-memberships-before.json"
  capture_keycloak_absent_baseline "$admin_group_id" || return 1
  add_keycloak_viewer_membership || return 1
  printf '%s\n' 'Browser check: removal-test user is Viewer before membership removal.' >&2
  require_confirmation REMOVAL_VIEWER_CONFIRMED || return 1
  cleanup_keycloak_membership || return 1
  response="$WORK/keycloak-session-revoke.json"
  status="$(keycloak_request POST "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/logout" "$response")" || return 1
  [[ "$status" == 204 ]] || return 1

  grafana_auth="$WORK/grafana-curl.conf"; safe_work_file "$grafana_auth" || return 1
  encoded="$(jq_bounded -nr --arg value "$removal_identity" '$value|@uri')" || return 1
  grafana_user="$WORK/grafana-removal-user.json"
  grafana_get "/api/users/lookup?loginOrEmail=$encoded" "$grafana_user" "$grafana_auth" || return 1
  local grafana_id
  grafana_id="$(jq_bounded -er '.id|select(type=="number" and .>0 and floor==.)' "$grafana_user")" || return 1
  response="$WORK/grafana-session-revoke.json"
  status="$(proxy_request POST "$(grafana_path "/api/admin/users/$grafana_id/logout")" "$response" '' "$grafana_auth")" || return 1
  [[ "$status" == 200 || "$status" == 204 ]] || return 1
  now="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%s)" || return 1
  [[ "$now" =~ ^[0-9]+$ ]] || return 1
  deadline=$((now + HUMAN_TIMEOUT_SECONDS))
  printf '%s\n' 'Within five minutes, verify old session rejection and OIDC relogin denial.' >&2
  require_confirmation_before SESSION_REVOKED "$deadline" || return 1
  require_confirmation_before RELOGIN_DENIED "$deadline" || return 1
  printf 'OBSERVABILITY_OIDC_ACCEPTANCE=PASS\n'
}

wait_for_smoke_alert() {
  local run=$1 expected=$2 attempt query alerts
  for ((attempt=0; attempt<60; attempt++)); do
    query="$WORK/slack-query-$attempt.json"; alerts="$WORK/slack-alerts-$attempt.json"
    prom_query "ALERTS{alertname=\"PlatformObservabilitySmoke\",instance=\"$run\",alertstate=\"firing\"}" "$query" || return 1
    proxy_get_200 "$(alertmanager_path '/api/v2/alerts')" "$alerts" || return 1
    if [[ "$expected" == present ]]; then
      if jq_bounded -e --arg run "$run" --arg runbook "$RUNBOOK_URL" '
        (.status=="success" and (.data.result|length)==1 and
          .data.result[0].metric.alertname=="PlatformObservabilitySmoke" and
          .data.result[0].metric.instance==$run) 
      ' "$query" >/dev/null &&
        jq_bounded -e --arg run "$run" --arg runbook "$RUNBOOK_URL" '
          [.[]|select(.labels.alertname=="PlatformObservabilitySmoke" and .labels.instance==$run)] as $items |
          ($items|length)==1 and $items[0].labels.cluster=="home" and
          $items[0].labels.namespace=="observability" and $items[0].labels.severity=="info" and
          $items[0].annotations.runbook_url==$runbook
        ' "$alerts" >/dev/null; then return 0; fi
    else
      if jq_bounded -e '.status=="success" and (.data.result|length)==0' "$query" >/dev/null &&
        jq_bounded -e --arg run "$run" '[.[]|select(.labels.alertname=="PlatformObservabilitySmoke" and .labels.instance==$run)]|length==0' "$alerts" >/dev/null; then return 0; fi
    fi
    /usr/bin/sleep 5
  done
  return 1
}

run_slack_acceptance() {
  local run name rule
  run="$(new_smoke_run_id alert)" || return 1
  name="$(kubernetes_smoke_name "$run")" || return 1
  rule="$WORK/slack-smoke-rule.json"
  jq_bounded -n --arg name "$name" --arg run "$run" --arg runbook "$RUNBOOK_URL" '{
    apiVersion:"monitoring.coreos.com/v1",kind:"PrometheusRule",
    metadata:{name:$name,namespace:"observability",labels:{
      "app.kubernetes.io/managed-by":"platform-observability-smoke",
      "platform.hyeonworks.com/smoke-run":$run,
      "observability.hyeonworks.com/instance":"home"}},
    spec:{groups:[{name:("platform.observability-smoke."+$run),rules:[{
      alert:"PlatformObservabilitySmoke",expr:"vector(1)",for:"0m",
      labels:{cluster:"home",namespace:"observability",instance:$run,severity:"info"},
      annotations:{summary:"Observability acceptance smoke",runbook_url:$runbook}
    }]}]}}
  ' >"$rule" || return 1
  "$CHMOD_BIN" 0600 -- "$rule"
  create_owned_from_file "$rule" PrometheusRule observability "$name" "$run" || return 1
  wait_for_smoke_alert "$run" present || return 1
  printf '%s\n' 'Confirm the Slack message groups cluster, namespace, alertname and includes severity/runbook.' >&2
  require_confirmation FIRING || return 1
  cleanup_owned_objects || return 1
  printf '%s\n' 'Confirm the matching Slack resolved message.' >&2
  require_confirmation RESOLVED || return 1
  wait_for_smoke_alert "$run" absent || return 1
  printf 'OBSERVABILITY_SLACK_ACCEPTANCE=PASS\n'
}

cleanup_all() {
  local rc=0
  cleanup_owned_objects || rc=1
  cleanup_keycloak_membership || rc=1
  delete_persistence_silence || rc=1
  if [[ -n "$API_PROXY_PID" ]]; then
    stop_owned_process "$API_PROXY_PID" "$API_PROXY_START" || rc=1
    API_PROXY_PID=''; API_PROXY_START=''
  fi
  if [[ -n "$WORK" ]]; then
    if [[ "$WORK" == /tmp/platform-observability-smoke.???????? && -d "$WORK" && ! -L "$WORK" ]]; then
      "$RM_BIN" -rf -- "$WORK"
      WORK=''
    else
      emit_manual_recovery; rc=1
    fi
  fi
  return "$rc"
}

on_exit() {
  local rc=$?
  [[ "$EXIT_HANDLER_RUNNING" == false ]] || exit "$rc"
  EXIT_HANDLER_RUNNING=true
  trap - EXIT HUP INT TERM
  cleanup_all || { (( rc != 0 )) || rc=1; }
  exit "$rc"
}

main() {
  parse_arguments "$@" || return 1
  if [[ "$TEST_MODE" == false ]]; then reject_production_overrides || return 1; fi
  if [[ "$execute_requested" == false ]]; then print_dry_run; return 0; fi

  trap on_exit EXIT
  trap 'handle_termination_signal 129' HUP
  trap 'handle_termination_signal 130' INT
  trap 'handle_termination_signal 143' TERM
  require_execute_environment || { fail 'execute preflight failed'; return 1; }
  run_read_only_machine_checks || return 1
  require_confirmation 'APPLY default' || { fail 'confirmation did not match APPLY default'; return 1; }
  repeat_commit_gate || { fail 'post-confirmation commit gate changed'; return 1; }
  run_network_boundary_transaction || { fail 'network boundary transaction failed'; return 1; }
  run_restart_persistence_transaction || { fail 'restart persistence transaction failed'; return 1; }
  run_certbot_acceptance || { fail 'Certbot/Nginx acceptance failed'; return 1; }
  repeat_commit_gate || { fail 'post-mutation context/API gate changed'; return 1; }
  run_read_only_machine_checks || { fail 'post-mutation machine acceptance failed'; return 1; }
  printf 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS\n'
  run_oidc_acceptance || { fail 'OIDC human/session acceptance failed'; return 1; }
  run_slack_acceptance || { fail 'Slack firing/resolved acceptance failed'; return 1; }
  local external_token
  external_token="$(external_attestation_token "$PUBLIC_EDGE_IP")" || return 1
  render_external_attestation "$PUBLIC_EDGE_IP" >&2 || return 1
  if ! require_confirmation "$external_token"; then
    printf 'OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL\n'
    return 2
  fi
  printf '%s\n' 'OBSERVABILITY_EXTERNAL_BOUNDARY=PASS' 'OBSERVABILITY_SMOKE=PASS'
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
  main "$@"
fi