#!/usr/bin/env bash
set -Eeuo pipefail

readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly TEST_SCRIPT="$(readlink -f -- "${BASH_SOURCE[0]}")"
readonly APPLY="$ROOT/scripts/bootstrap/apply-observability-access.sh"
readonly ROLLBACK_ID=20260812T120000Z
readonly INITIAL_PRODUCTION_SHA=79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c
readonly POST_PRODUCTION_SHA=b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291
readonly INITIAL_TARGET_COUNT=21
readonly POST_TARGET_COUNT=30
readonly SUITE_WALL_BOUND_SECONDS=300
WORK=''
declare -a METRIC_ROOTS=()
ASSERTIONS=0

fail() {
  printf 'FAIL: %s\n' "$*" >&2
  exit 1
}

pass() {
  ASSERTIONS=$((ASSERTIONS + 1))
  printf 'PASS: %s\n' "$1"
}

case_progress() {
  local case_name=$1 marker="CASE_BEGIN=$1"
  printf '%s\n' "$marker"
  printf '%s\n' "$marker" >>"$PROGRESS_LOG"
}

run_with_wall_bound() {
  local seconds=$1
  shift
  [[ "$seconds" =~ ^[1-9][0-9]*$ && $# -gt 0 ]] || return 2
  timeout --signal=TERM --kill-after=2s "${seconds}s" "$@"
}

cleanup() {
  trap - EXIT HUP INT TERM
  case "$WORK" in
    /tmp/platform-observability-access-test.*) rm -rf -- "$WORK" ;;
  esac
  local root
  for root in "${METRIC_ROOTS[@]}"; do
    case "$root" in
      /tmp/platform-observability-metrics.??????) rm -rf -- "$root" ;;
    esac
  done
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM

write_executable() {
  local path=$1
  shift
  printf '%s\n' "$@" >"$path"
  chmod 0755 "$path"
}

make_fakes() {
  local fixture=$1
  mkdir -m 0700 -p "$fixture/bin" "$fixture/state"
  chmod 0700 "$fixture/bin" "$fixture/state"
  : >"$fixture/commands.log"

  write_executable "$fixture/bin/sudo" \
    '#!/usr/bin/env bash' \
    'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
    'printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"' \
    'printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
    'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \
    '[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \
    'if [[ "${1:-}" == -n ]]; then shift; fi' \
    '(( $# > 0 )) || exit 0' \
    'if [[ "${PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE:-0}" == 1 && " $* " == *" install "* && "$*" == *"acceptance.env"* ]]; then exit 81; fi' \
    'args=()' \
    'while (( $# > 0 )); do case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac; done' \
    'exec "${args[@]}"'

  write_executable "$fixture/bin/encryption" \
    '#!/usr/bin/env bash' \
    'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
    'count=0; [[ -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] && read -r count <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
    'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
    'if [[ "$count" == 2 && -n "${PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE:-}" ]]; then printf '\''# changed\n'\'' >>"$PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE"; fi' \
    'action="${PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE:-}"' \
    'if [[ "$count" == 2 && -n "$action" ]]; then' \
    '  fixture_root="$(dirname -- "$PLATFORM_TEST_COMMAND_LOG")"; expected_root="$fixture_root/metrics"' \
    '  [[ "$action" == raw-json || "$action" == rechecksum-count || "$action" == hardlink ]] || exit 73' \
    '  [[ "$PLATFORM_TEST_INVENTORY_ROOT" == "$expected_root" && -d "$PLATFORM_TEST_INVENTORY_ROOT" && ! -L "$PLATFORM_TEST_INVENTORY_ROOT" && "$(readlink -f -- "$PLATFORM_TEST_INVENTORY_ROOT")" == "$expected_root" ]] || exit 74' \
    '  inventory="$expected_root/target-initial/inventory.json"; checksum="$expected_root/target-initial/inventory.sha256"' \
    '  case "$action" in' \
    '    raw-json) printf '\'' \n'\'' >>"$inventory" ;;' \
    '    rechecksum-count) jq -c '\''.targets |= .[:-1]'\'' "$inventory" >"$inventory.next"; chmod 0600 "$inventory.next"; mv -f -- "$inventory.next" "$inventory"; sha="$(sha256sum -- "$inventory" | awk '\''{print $1}'\'')"; printf '\''%s  inventory.json\n'\'' "$sha" >"$checksum"; chmod 0600 "$checksum" ;;' \
    '    hardlink) ln -- "$inventory" "$fixture_root/inventory-hardlink-control" ;;' \
    '  esac' \
    'fi' \
    'if [[ "$count" == "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]; then exit 71; fi'

  write_executable "$fixture/bin/restore" \
    '#!/usr/bin/env bash' \
    'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
    'count=0; [[ -f "$PLATFORM_TEST_RESTORE_COUNT" ]] && read -r count <"$PLATFORM_TEST_RESTORE_COUNT"' \
    'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RESTORE_COUNT"' \
    'if [[ "$count" == "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]; then exit 72; fi'

  write_executable "$fixture/bin/recovery" \
    '#!/usr/bin/env bash' \
    'printf '\''recovery'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
    'count=0; [[ -f "$PLATFORM_TEST_RECOVERY_COUNT" ]] && read -r count <"$PLATFORM_TEST_RECOVERY_COUNT"' \
    'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RECOVERY_COUNT"' \
    'response_var="PLATFORM_TEST_RECOVERY_OUTPUT_${count}"; response="${!response_var:-SLACK_DEPLOYMENT_GATE=RECOVERY\\n}"' \
    'rc_var="PLATFORM_TEST_RECOVERY_RC_${count}"; rc="${!rc_var:-${PLATFORM_TEST_RECOVERY_RC:-0}}"' \
    'printf '\''%b'\'' "$response"' \
    'exit "$rc"'

  cat >"$fixture/bin/kubectl" <<'PY'
#!/usr/bin/env python3
import json
import hashlib
import os
import pathlib
import signal
import socketserver
import sys
import time
from http.server import BaseHTTPRequestHandler
import yaml

raw_args = sys.argv[1:]
log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"])
with log.open("a", encoding="utf-8") as stream:
    stream.write("kubectl " + " ".join(raw_args) + "\n")
if not raw_args or raw_args[0] != "--request-timeout=10s":
    raise SystemExit(93)
args = raw_args[1:]
state = pathlib.Path(os.environ["PLATFORM_TEST_STATE"])

kind_alias = {
    "configmap": "ConfigMap", "configmaps": "ConfigMap",
    "prometheusrule": "PrometheusRule", "prometheusrules": "PrometheusRule",
    "alertmanagerconfig": "AlertmanagerConfig", "alertmanagerconfigs": "AlertmanagerConfig",
    "alertmanager": "Alertmanager", "alertmanagers": "Alertmanager",
    "networkpolicy": "NetworkPolicy", "networkpolicies": "NetworkPolicy",
}

def option(*names, default=None):
    for position, value in enumerate(args):
        if value in names and position + 1 < len(args):
            return args[position + 1]
        for name in names:
            if value.startswith(name + "="):
                return value.split("=", 1)[1]
    return default

def state_path(kind, namespace, name):
    return state / f"{kind}__{namespace}__{name}.json"

def load_documents(filename):
    with open(filename, encoding="utf-8") as stream:
        return [item for item in yaml.safe_load_all(stream) if item]

if args == ["config", "current-context"]:
    print("default")
    raise SystemExit(0)
if args[:2] == ["config", "view"]:
    print("https://127.0.0.1:6443", end="")
    raise SystemExit(0)
if args[:2] == ["get", "node"]:
    print(json.dumps({
        "metadata": {"name": "donghyeon-system-product-name", "uid": "node-uid"},
        "status": {"conditions": [{"type": "Ready", "status": "True"}]},
    }))
    raise SystemExit(0)
if args[:3] == ["get", "secret", "alertmanager-slack-webhook"]:
    schema_count_path = pathlib.Path(os.environ["PLATFORM_TEST_SLACK_SCHEMA_COUNT"])
    schema_count = int(schema_count_path.read_text()) if schema_count_path.exists() else 0
    schema_count += 1
    schema_count_path.write_text(str(schema_count))
    extra_at = os.environ.get("PLATFORM_TEST_SLACK_EXTRA_KEY_AT", "0")
    extra = (os.environ.get("PLATFORM_TEST_SLACK_EXTRA_KEY", "0") == "1" or
             (extra_at.isdigit() and schema_count == int(extra_at)))
    keys = ["extra", "url"] if extra else ["url"]
    print(json.dumps({
        "apiVersion": "v1", "kind": "Secret", "type": "Opaque",
        "metadata": {"name": "alertmanager-slack-webhook", "namespace": "observability"},
        "data": {key: "cmVkYWN0ZWQ=" for key in keys},
    }))
    raise SystemExit(0)
if args[:2] == ["get", "probe"]:
    print(json.dumps({"items": [
        {"metadata": {"name": "platform-private-edge"}},
        {"metadata": {"name": "platform-private-internal"}},
        {"metadata": {"name": "platform-public-edge"}},
    ]}))
    raise SystemExit(0)
if args[:3] == ["get", "servicemonitor", "aistor-bucket-usage"]:
    print("servicemonitor.monitoring.coreos.com/aistor-bucket-usage")
    raise SystemExit(0)
if args[:3] == ["get", "podmonitor", "platform-postgres"]:
    print("podmonitor.monitoring.coreos.com/platform-postgres")
    raise SystemExit(0)
if args and args[0] == "get" and option("--raw"):
    raw_path = option("--raw")
    pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).touch()
    if os.environ.get("PLATFORM_TEST_POSTCHECK_RC", "0") != "0":
        raise SystemExit(74)
    if raw_path.endswith("/proxy/-/ready") and "prometheus" in raw_path:
        print("Prometheus Server is Ready.")
        raise SystemExit(0)
    if raw_path.endswith("/proxy/-/ready") and "alertmanager" in raw_path:
        print("OK")
        raise SystemExit(0)
    if raw_path.endswith("/proxy/api/v1/rules"):
        groups = []
        for path in sorted(state.glob("PrometheusRule__observability__*.json")):
            item = json.loads(path.read_text(encoding="utf-8"))
            for group in (item.get("spec") or {}).get("groups") or []:
                rules = []
                for rule in group.get("rules") or []:
                    rules.append({
                        "name": rule.get("alert") or rule.get("record"),
                        "type": "alerting" if rule.get("alert") else "recording",
                        "health": "ok", "lastError": "", "query": str(rule.get("expr", "")),
                    })
                groups.append({"name": group.get("name"), "rules": rules})
        if os.environ.get("PLATFORM_TEST_PROMETHEUS_DROP_RULE", "0") == "1" and groups:
            groups[0]["rules"] = groups[0]["rules"][1:]
        print(json.dumps({"status": "success", "data": {"groups": groups}}))
        raise SystemExit(0)
    if raw_path.endswith("/proxy/api/v2/receivers"):
        mode = os.environ.get("PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE", "qualified")
        platform_null = "observability/platform-alertmanager/platform-null"
        platform_slack = "observability/platform-alertmanager/platform-slack"
        def receiver(name):
            return {"labels": {"name": name}, "name": name}
        receiver_cases = {
            "qualified": [receiver(platform_null), receiver(platform_slack)],
            "qualified-reversed": [receiver(platform_slack), receiver(platform_null)],
            "raw": [receiver("platform-null"), receiver("platform-slack")],
            "empty": [],
            "null": [None, receiver(platform_slack)],
            "duplicate": [receiver(platform_null), receiver(platform_null)],
            "extra": [receiver(platform_null), receiver(platform_slack), receiver("null")],
            "wrong-namespace": [receiver("other/platform-alertmanager/platform-null"), receiver(platform_slack)],
            "wrong-config": [receiver(platform_null), receiver("observability/other/platform-slack")],
            "wrong-local": [receiver(platform_null), receiver("observability/platform-alertmanager/other")],
        }
        if mode == "malformed":
            print('{"name":')
        elif mode in receiver_cases:
            print(json.dumps(receiver_cases[mode]))
        else:
            raise SystemExit(76)
        raise SystemExit(0)
    raise SystemExit(75)
if args and args[0] == "get" and len(args) >= 3:
    kind = kind_alias.get(args[1].lower())
    if not kind:
        raise SystemExit(44)
    name = args[2]
    namespace = option("-n", "--namespace", default="default")
    path = state_path(kind, namespace, name)
    if not path.exists():
        if "--ignore-not-found" in args:
            raise SystemExit(0)
        raise SystemExit(1)
    item = json.loads(path.read_text())
    drift = os.environ.get("PLATFORM_TEST_DRIFT_RESOURCE", "")
    reached = pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).exists()
    if reached and drift == f"{kind}/{name}":
        item["metadata"]["uid"] = "uid-drifted-by-other-owner"
    print(json.dumps(item))
    raise SystemExit(0)
if args and args[0] == "apply":
    filename = option("-f", "--filename")
    if not filename:
        raise SystemExit(45)
    if option("--dry-run") == "server" or "--dry-run=server" in args:
        raise SystemExit(0)
    counter = pathlib.Path(os.environ["PLATFORM_TEST_APPLY_COUNT"])
    count = int(counter.read_text()) if counter.exists() else 0
    for item in load_documents(filename):
        count += 1
        counter.write_text(str(count))
        if count == int(os.environ.get("PLATFORM_TEST_FAIL_APPLY_AT", "0")):
            raise SystemExit(73)
        metadata = item.setdefault("metadata", {})
        namespace = metadata.get("namespace", "default")
        kind = item["kind"]
        path = state_path(kind, namespace, metadata["name"])
        prior = json.loads(path.read_text()) if path.exists() else None
        metadata["uid"] = (prior or {}).get("metadata", {}).get("uid", f"uid-new-{kind}-{metadata['name']}")
        metadata["resourceVersion"] = str(1000 + count)
        metadata["generation"] = 2
        path.write_text(json.dumps(item))
        if count == int(os.environ.get("PLATFORM_TEST_FAIL_AFTER_APPLY_AT", "0")):
            raise SystemExit(73)
    raise SystemExit(0)
if args and args[0] == "replace":
    filename = option("-f", "--filename")
    documents = load_documents(filename) if filename else []
    if len(documents) != 1:
        raise SystemExit(47)
    item = documents[0]
    metadata = item.get("metadata") or {}
    namespace = metadata.get("namespace", "default")
    path = state_path(item["kind"], namespace, metadata["name"])
    if not path.exists():
        raise SystemExit(48)
    prior = json.loads(path.read_text())
    prior_metadata = prior.get("metadata") or {}
    with log.open("a", encoding="utf-8") as stream:
        stream.write(
            f"replace-preconditions {item['kind']}/{metadata.get('name')} "
            f"uid={metadata.get('uid')} resourceVersion={metadata.get('resourceVersion')}\n"
        )
    if (metadata.get("uid") != prior_metadata.get("uid") or
            metadata.get("resourceVersion") != prior_metadata.get("resourceVersion")):
        raise SystemExit(49)
    if os.environ.get("PLATFORM_TEST_REPLACE_CONFLICT", "0") == "1":
        raise SystemExit(409)
    metadata["resourceVersion"] = "replaced"
    path.write_text(json.dumps(item))
    raise SystemExit(0)
if args and args[0] == "exec":
    expected = {
        "grafana-dashboard-https-endpoints": "https-endpoints.json",
        "grafana-dashboard-kubernetes-node": "kubernetes-node.json",
        "grafana-dashboard-observability-backends": "observability-backends.json",
        "grafana-dashboard-platform-services": "platform-services.json",
        "grafana-dashboard-workload-health": "workload-health.json",
    }
    lines = []
    for name, filename in expected.items():
        item = json.loads(state_path("ConfigMap", "observability", name).read_text(encoding="utf-8"))
        data = item.get("data") or {}
        if set(data) != {filename}:
            raise SystemExit(76)
        digest = hashlib.sha256(data[filename].encode()).hexdigest()
        lines.append(f"{digest}  /tmp/dashboards/{filename}")
    if os.environ.get("PLATFORM_TEST_GRAFANA_STALE_DASHBOARD", "0") == "1":
        lines[0] = "0" * 64 + "  /tmp/dashboards/https-endpoints.json"
    print("\n".join(lines))
    raise SystemExit(0)
if args and args[0] == "proxy":
    socket_path = option("--unix-socket")
    if not socket_path:
        raise SystemExit(77)
    socket_file = pathlib.Path(socket_path)
    if socket_file.exists():
        socket_file.unlink()

    api_prefixes = {
        "/api/v1/namespaces/observability/configmaps/": "ConfigMap",
        "/apis/monitoring.coreos.com/v1/namespaces/observability/prometheusrules/": "PrometheusRule",
        "/apis/monitoring.coreos.com/v1alpha1/namespaces/observability/alertmanagerconfigs/": "AlertmanagerConfig",
        "/apis/monitoring.coreos.com/v1/namespaces/observability/alertmanagers/": "Alertmanager",
        "/apis/networking.k8s.io/v1/namespaces/observability/networkpolicies/": "NetworkPolicy",
    }

    class Handler(BaseHTTPRequestHandler):
        def log_message(self, *_):
            return

        def reply(self, status, payload):
            encoded = json.dumps(payload, separators=(",", ":")).encode()
            self.send_response(status)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(encoded)))
            self.end_headers()
            self.wfile.write(encoded)

        def do_DELETE(self):
            length = int(self.headers.get("Content-Length", "0"))
            try:
                delete_options = json.loads(self.rfile.read(length))
            except Exception:
                self.reply(400, {"kind": "Status", "reason": "BadRequest"})
                return
            kind = None
            name = None
            for prefix, candidate in api_prefixes.items():
                if self.path.startswith(prefix):
                    kind = candidate
                    name = self.path[len(prefix):]
                    break
            path = state_path(kind, "observability", name) if kind and name else None
            item = json.loads(path.read_text(encoding="utf-8")) if path and path.exists() else None
            expected_uid = ((item or {}).get("metadata") or {}).get("uid")
            supplied_uid = ((delete_options.get("preconditions") or {}).get("uid"))
            exact_options = (
                delete_options == {
                    "apiVersion": "v1", "kind": "DeleteOptions",
                    "preconditions": {"uid": supplied_uid},
                    "propagationPolicy": "Background",
                }
                and isinstance(supplied_uid, str) and supplied_uid == expected_uid
            )
            with log.open("a", encoding="utf-8") as stream:
                stream.write(
                    f"raw-delete {self.path} uid={supplied_uid} "
                    f"propagation={delete_options.get('propagationPolicy')} exact={str(exact_options).lower()}\n"
                )
            if not item:
                self.reply(404, {"kind": "Status", "reason": "NotFound"})
                return
            if not exact_options:
                self.reply(409, {"kind": "Status", "reason": "Conflict"})
                return
            mode = os.environ.get("PLATFORM_TEST_DELETE_MODE", "success")
            count_file = pathlib.Path(os.environ["PLATFORM_TEST_DELETE_COUNT"])
            delete_count = int(count_file.read_text()) if count_file.exists() else 0
            delete_count += 1
            count_file.write_text(str(delete_count))
            active_mode = mode if delete_count == 1 else "success"
            if active_mode == "conflict":
                self.reply(409, {"kind": "Status", "reason": "Conflict"})
                return
            if active_mode == "timeout":
                time.sleep(5)
                return
            path.unlink()
            if active_mode == "response-loss":
                self.close_connection = True
                return
            self.reply(200, {"kind": "Status", "status": "Success"})

    with socketserver.UnixStreamServer(str(socket_file), Handler) as server:
        server.serve_forever()
if args and args[0] == "delete":
    raise SystemExit(96)
if args and args[0] in {"wait", "rollout"}:
    if "--timeout=60s" in args and os.environ.get("PLATFORM_TEST_PREFLIGHT_RC", "0") != "0":
        raise SystemExit(78)
    if "--timeout=10s" in args:
        pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).touch()
        delay = float(os.environ.get("PLATFORM_TEST_POSTCHECK_DELAY", "0"))
        if delay:
            time.sleep(delay)
        if os.environ.get("PLATFORM_TEST_POSTCHECK_RC", "0") != "0":
            raise SystemExit(79)
    raise SystemExit(0)
raise SystemExit(46)
PY
  chmod 0755 "$fixture/bin/kubectl"
}

write_inventory() {
  local root=$1 phase=$2 captured_at=$3 target_count=$4 sha
  [[ "$target_count" =~ ^[1-9][0-9]*$ ]] || fail "fixture target count is not a canonical positive decimal: $phase"
  mkdir -m 0700 -p -- "$root/$phase"
  jq -c -n \
    --arg phase "$phase" \
    --arg captured_at "$captured_at" \
    --argjson target_count "$target_count" '
      {
        schema: "platform-observability-metric-inventory/v1",
        phase: $phase,
        context: "default",
        api_server: "https://127.0.0.1:6443",
        captured_at_utc: $captured_at,
        targets: [range(0; $target_count) | {
          job: "fixture",
          health: "up",
          last_error: "",
          metrics: [{name: "up", label_names: ["job"]}]
        }]
      }
    ' >"$root/$phase/inventory.json"
  chmod 0600 "$root/$phase/inventory.json"
  jq -e --arg phase "$phase" --arg captured_at "$captured_at" --argjson target_count "$target_count" '
    .schema == "platform-observability-metric-inventory/v1" and
    .phase == $phase and .context == "default" and
    .api_server == "https://127.0.0.1:6443" and
    .captured_at_utc == $captured_at and
    (.targets | type == "array" and length == $target_count) and
    all(.targets[];
      .health == "up" and .last_error == "" and
      (.metrics | type == "array" and length > 0) and
      all(.metrics[];
        (.name | type == "string" and length > 0) and
        (.label_names | type == "array") and
        all(.label_names[]; type == "string")))
  ' "$root/$phase/inventory.json" >/dev/null || fail "fixture inventory generation was invalid: $phase"
  sha="$(sha256sum "$root/$phase/inventory.json" | awk '{print $1}')"
  printf '%s  inventory.json\n' "$sha" >"$root/$phase/inventory.sha256"
  chmod 0600 "$root/$phase/inventory.sha256"
  printf '%s' "$sha"
}

refresh_inventory_checksum() {
  local root=$1 phase=$2 sha
  sha="$(sha256sum -- "$root/$phase/inventory.json" | awk '{print $1}')"
  printf '%s  inventory.json\n' "$sha" >"$root/$phase/inventory.sha256"
  chmod 0600 "$root/$phase/inventory.sha256"
  printf '%s' "$sha"
}

rewrite_inventory_filter() {
  local root=$1 phase=$2 filter=$3 next
  next="$root/$phase/inventory.json.next"
  jq -c "$filter" "$root/$phase/inventory.json" >"$next" || fail "fixture inventory rewrite failed: $phase"
  chmod 0600 "$next"
  mv -f -- "$next" "$root/$phase/inventory.json"
  refresh_inventory_checksum "$root" "$phase"
}

rewrite_inventory_count() {
  local root=$1 phase=$2 target_count=$3 next
  next="$root/$phase/inventory.json.next"
  [[ "$target_count" =~ ^(0|[1-9][0-9]*)$ ]] || fail "fixture rewrite count is malformed: $phase"
  jq -c --argjson target_count "$target_count" '
    .targets[0] as $target |
    .targets = [range(0; $target_count) | $target]
  ' "$root/$phase/inventory.json" >"$next" || fail "fixture count rewrite failed: $phase"
  chmod 0600 "$next"
  mv -f -- "$next" "$root/$phase/inventory.json"
  refresh_inventory_checksum "$root" "$phase"
}

write_handoff() {
  local root=$1 runbook=${2-https://git.learn.hyeonworks.com/platform/runbooks/observability.md}
  cat >"$root/dashboards.yaml" <<'YAML'
apiVersion: v1
kind: ConfigMap
metadata: {name: grafana-dashboard-https-endpoints, namespace: observability, labels: {grafana_dashboard: "1"}}
data: {https-endpoints.json: '{"uid":"platform-https-endpoints","title":"Platform / HTTPS Endpoints"}'}
---
apiVersion: v1
kind: ConfigMap
metadata: {name: grafana-dashboard-kubernetes-node, namespace: observability, labels: {grafana_dashboard: "1"}}
data: {kubernetes-node.json: '{"uid":"platform-kubernetes-node","title":"Platform / Kubernetes Node"}'}
---
apiVersion: v1
kind: ConfigMap
metadata: {name: grafana-dashboard-observability-backends, namespace: observability, labels: {grafana_dashboard: "1"}}
data: {observability-backends.json: '{"uid":"platform-observability-backends","title":"Platform / Observability Backends"}'}
---
apiVersion: v1
kind: ConfigMap
metadata: {name: grafana-dashboard-platform-services, namespace: observability, labels: {grafana_dashboard: "1"}}
data: {platform-services.json: '{"uid":"platform-services","title":"Platform / Services"}'}
---
apiVersion: v1
kind: ConfigMap
metadata: {name: grafana-dashboard-workload-health, namespace: observability, labels: {grafana_dashboard: "1"}}
data: {workload-health.json: '{"uid":"platform-workload-health","title":"Platform / Workload Health"}'}
YAML
  : >"$root/rules.yaml"
  local name alert
  while IFS='|' read -r name alert; do
    cat >>"$root/rules.yaml" <<YAML
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: $name
  namespace: observability
  labels: {observability.hyeonworks.com/instance: home}
spec:
  groups:
  - name: $name
    rules:
    - record: fixture:${alert}_record
      expr: vector(1)
    - alert: $alert
      expr: vector(1)
      annotations: {runbook_url: "$runbook"}
YAML
  done <<'ROWS'
platform-aistor-storage-quota|FixtureAIStorQuota
platform-certificate-probes|FixtureCertificate
platform-observability-core|FixtureCore
platform-verified-services|FixtureService
ROWS
  cat >"$root/alertmanager.yaml" <<'YAML'
apiVersion: monitoring.coreos.com/v1
kind: Alertmanager
metadata: {name: observability-core-kube-pr-alertmanager, namespace: observability}
spec: {alertmanagerConfiguration: {name: platform-alertmanager}}
---
apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: platform-alertmanager
  namespace: observability
  labels:
    observability.hyeonworks.com/instance: home
spec:
  route:
    receiver: platform-slack
    groupBy:
      - cluster
      - namespace
      - alertname
      - severity
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 4h
    routes:
      - receiver: platform-null
        matchers:
          - name: alertname
            matchType: "="
            value: InfoInhibitor
  inhibitRules:
    - sourceMatch:
        - name: alertname
          matchType: "="
          value: InfoInhibitor
      targetMatch:
        - name: severity
          matchType: "="
          value: info
      equal:
        - namespace
  receivers:
    - name: platform-null
    - name: platform-slack
      slackConfigs:
        - apiURL:
            name: alertmanager-slack-webhook
            key: url
          sendResolved: true
          linkNames: false
          mrkdwnIn:
            - text
            - fields
          fallback: >-
            {{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}
          title: >-
            {{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}
          titleLink: https://grafana.learn.hyeonworks.com/
          color: >-
            {{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}
          fields:
            - title: Status
              value: '{{ .Status | toUpper }}'
              short: true
            - title: Severity
              value: '{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}'
              short: true
            - title: Location
              value: '{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}'
              short: true
            - title: Alert count
              value: '{{ len .Alerts }}'
              short: true
          text: |-
            {{ range .Alerts }}
            *Alert status:* {{ .Status | toUpper }}
            *Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}
            *Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}
            *Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}
            *Started:* {{ .StartsAt.Format "2006-01-02T15:04:05Z07:00" }}
            {{ if eq .Status "resolved" }}*Ended:* {{ .EndsAt.Format "2006-01-02T15:04:05Z07:00" }}{{ end }}
            {{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}

            {{ end }}
            <https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>
          footer: hyeonworks observability · Alertmanager
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: observability-allow-alertmanager-public-https, namespace: observability}
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/instance: observability-core-kube-pr-alertmanager
      app.kubernetes.io/name: alertmanager
  policyTypes: [Egress]
  egress:
  - to:
    - ipBlock:
        cidr: 0.0.0.0/0
        except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]
    ports:
    - {protocol: TCP, port: 443}
YAML
  chmod 0600 "$root/dashboards.yaml" "$root/rules.yaml" "$root/alertmanager.yaml"
}

new_fixture() {
  local name=$1 runbook=${2-https://git.learn.hyeonworks.com/platform/runbooks/observability.md}
  local captured_at=${3-} initial_count=${4-$INITIAL_TARGET_COUNT} post_count=${5-$POST_TARGET_COUNT}
  local fixture metric initial_sha post_sha
  [[ -n "$captured_at" ]] || captured_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  fixture="$WORK/$name"
  mkdir -m 0700 -- "$fixture"
  mkdir -m 0700 -- "$fixture/rollbacks"
  metric="$fixture/metrics"
  mkdir -m 0700 -- "$metric"
  mkdir -m 0700 -- "$fixture/rollbacks/observability-$ROLLBACK_ID"
  make_fakes "$fixture"
  initial_sha="$(write_inventory "$metric" target-initial "$captured_at" "$initial_count")"
  post_sha="$(write_inventory "$metric" post-substrate "$captured_at" "$post_count")"
  write_handoff "$metric" "$runbook"
  cat >"$fixture/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json" <<'JSON'
{"apiVersion":"monitoring.coreos.com/v1","kind":"Alertmanager","metadata":{"name":"observability-core-kube-pr-alertmanager","namespace":"observability","uid":"uid-alertmanager-prior","resourceVersion":"88","generation":1,"creationTimestamp":"2026-08-01T00:00:00Z","managedFields":[{"manager":"operator"}],"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"remove"}},"spec":{"replicas":1},"status":{"availableReplicas":1}}
JSON
  chmod 0700 "$fixture/rollbacks/observability-$ROLLBACK_ID"
  printf '%s|%s|%s|%s' "$fixture" "$metric" "$initial_sha" "$post_sha"
}

run_apply() {
  local fixture=$1 metric=$2 initial_sha=$3 post_sha=$4
  shift 4
  env \
    PATH="$fixture/bin:$PATH" \
    PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \
    PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN="$fixture/bin/kubectl" \
    PLATFORM_OBSERVABILITY_ACCESS_SUDO_BIN="${PLATFORM_TEST_SUDO_BIN:-$fixture/bin/sudo}" \
    PLATFORM_OBSERVABILITY_ACCESS_ENCRYPTION_SCRIPT="$fixture/bin/encryption" \
    PLATFORM_OBSERVABILITY_ACCESS_RESTORE_SCRIPT="$fixture/bin/restore" \
    PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT="$fixture/bin/recovery" \
    PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE="$fixture/rollbacks" \
    PLATFORM_TEST_EXPECTED_INITIAL_SHA="$initial_sha" \
    PLATFORM_TEST_EXPECTED_POST_SHA="$post_sha" \
    PLATFORM_OBSERVABILITY_ACCESS_CONFIRMATION=APPLY \
    PLATFORM_OBSERVABILITY_ROLLBACK_ID="$ROLLBACK_ID" \
    PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
    PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \
    PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \
    PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \
    PLATFORM_TEST_RECOVERY_COUNT="$fixture/recovery-count" \
    PLATFORM_TEST_SLACK_SCHEMA_COUNT="$fixture/slack-schema-count" \
    PLATFORM_TEST_APPLY_COUNT="$fixture/apply-count" \
    PLATFORM_TEST_DELETE_COUNT="$fixture/delete-count" \
    PLATFORM_TEST_STATE="$fixture/state" \
    PLATFORM_TEST_POSTCHECK_REACHED="$fixture/postcheck-reached" \
    PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \
    PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \
    PLATFORM_TEST_RECOVERY_RC="${PLATFORM_TEST_RECOVERY_RC:-0}" \
    PLATFORM_TEST_RECOVERY_RC_1="${PLATFORM_TEST_RECOVERY_RC_1:-}" \
    PLATFORM_TEST_RECOVERY_RC_2="${PLATFORM_TEST_RECOVERY_RC_2:-}" \
    PLATFORM_TEST_RECOVERY_OUTPUT_1="${PLATFORM_TEST_RECOVERY_OUTPUT_1:-}" \
    PLATFORM_TEST_RECOVERY_OUTPUT_2="${PLATFORM_TEST_RECOVERY_OUTPUT_2:-}" \
    PLATFORM_TEST_PREFLIGHT_RC="${PLATFORM_TEST_PREFLIGHT_RC:-0}" \
    PLATFORM_TEST_POSTCHECK_RC="${PLATFORM_TEST_POSTCHECK_RC:-0}" \
    PLATFORM_TEST_POSTCHECK_DELAY="${PLATFORM_TEST_POSTCHECK_DELAY:-0}" \
    PLATFORM_TEST_PARENT_PID="$BASHPID" \
    PLATFORM_TEST_FAIL_APPLY_AT="${PLATFORM_TEST_FAIL_APPLY_AT:-0}" \
    PLATFORM_TEST_FAIL_AFTER_APPLY_AT="${PLATFORM_TEST_FAIL_AFTER_APPLY_AT:-0}" \
    PLATFORM_TEST_SIGNAL_AT="${PLATFORM_TEST_SIGNAL_AT:-0}" \
    PLATFORM_TEST_SLACK_EXTRA_KEY="${PLATFORM_TEST_SLACK_EXTRA_KEY:-0}" \
    PLATFORM_TEST_SLACK_EXTRA_KEY_AT="${PLATFORM_TEST_SLACK_EXTRA_KEY_AT:-0}" \
    PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE="${PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE:-0}" \
    PLATFORM_TEST_DRIFT_RESOURCE="${PLATFORM_TEST_DRIFT_RESOURCE:-}" \
    PLATFORM_TEST_DELETE_MODE="${PLATFORM_TEST_DELETE_MODE:-success}" \
    PLATFORM_TEST_REPLACE_CONFLICT="${PLATFORM_TEST_REPLACE_CONFLICT:-0}" \
    PLATFORM_TEST_PROMETHEUS_DROP_RULE="${PLATFORM_TEST_PROMETHEUS_DROP_RULE:-0}" \
    PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE="${PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE-qualified}" \
    PLATFORM_TEST_GRAFANA_STALE_DASHBOARD="${PLATFORM_TEST_GRAFANA_STALE_DASHBOARD:-0}" \
    PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE:-}" \
    PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE:-}" \
    PLATFORM_TEST_INVENTORY_ROOT="$metric" \
    bash "$APPLY" "$@"
}

run_inventory_validator() {
  local metric=$1 phase=$2 expected_sha=$3 expected_count=$4
  env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 bash -c '
    source "$1"
    verified_output_dir=$2
    validate_inventory_phase "$3" "$4" "$5"
  ' _ "$APPLY" "$metric" "$phase" "$expected_sha" "$expected_count"
}

assert_pre_prompt_rejection() {
  local fixture=$1 label=$2 ledger
  ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
  [[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail "$label reached confirmation"
  ! grep -Fq 'kubectl --request-timeout=10s apply --server-side --dry-run=server -f' "$fixture/commands.log" ||
    fail "$label reached server dry-run"
  ! grep -Eq '^sudo .* install ' "$fixture/commands.log" || fail "$label installed rollback data"
  [[ ! -e "$ledger/objects.tsv" ]] || fail "$label created a rollback ledger"
  ! grep -Fq 'kubectl --request-timeout=10s apply -f' "$fixture/commands.log" || fail "$label reached Kubernetes apply"
  [[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance"
}

assert_post_confirmation_rejection() {
  local fixture=$1 label=$2 ledger
  ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
  [[ -f "$ledger/objects.tsv" ]] || fail "$label did not preserve the durable rollback ledger"
  [[ ! -e "$fixture/apply-count" ]] || fail "$label reached an apply invocation"
  ! grep -Fq 'kubectl --request-timeout=10s apply -f' "$fixture/commands.log" || fail "$label reached Kubernetes apply"
  [[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance"
}

assert_reconcile_rollback() {
  local fixture=$1 label=$2 ledger prior
  ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
  prior="$fixture/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json"
  [[ -e "$fixture/postcheck-reached" ]] || fail "$label did not reach reconciliation"
  grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$fixture/output" ||
    fail "$label did not roll back"
  [[ -f "$ledger/objects.tsv" ]] || fail "$label did not preserve the durable rollback ledger"
  [[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance"
  [[ -f "$prior" ]] || fail "$label did not restore the pre-existing Alertmanager"
  [[ "$(find "$fixture/state" -type f | wc -l | tr -d ' ')" == 1 ]] ||
    fail "$label left newly-created fake state"
}

run_deployment_gate_helper() {
  local fixture=$1 first_output=$2 second_output=$3 expected_value=${4:-}
  env \
    PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \
    PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT="$fixture/bin/recovery" \
    PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
    PLATFORM_TEST_RECOVERY_COUNT="$fixture/recovery-count" \
    PLATFORM_TEST_RECOVERY_OUTPUT_1="$first_output" \
    PLATFORM_TEST_RECOVERY_OUTPUT_2="$second_output" \
    bash -c '
      source "$1"
      trap cleanup EXIT
      capture_slack_deployment_gate slack_deployment_gate_first helper-first
      capture_slack_deployment_gate slack_deployment_gate helper-second
      require_matching_slack_deployment_gates
      map_slack_deployment_gate_acceptance
      [[ "$slack_deployment_gate_value" == "$2" ]]
      cleanup
    ' _ "$APPLY" "$expected_value"
}

split_fixture() {
  local value=$1
  IFS='|' read -r FIXTURE METRIC INITIAL_SHA POST_SHA <<<"$value"
  METRIC_ROOTS+=("$METRIC")
}

supervise_focused_suite() {
  (( $# == 0 )) || fail 'focused suite accepts no caller arguments'
  WORK="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)"
  chmod 0700 "$WORK"
  forged_root="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)"
  chmod 0700 "$forged_root"
  forged_attestation="$forged_root/worker.attestation"
  printf '%s\n' "$(stat -c '%d:%i' -- "$TEST_SCRIPT")" >"$forged_attestation"
  chmod 0600 "$forged_attestation"
  if run_with_wall_bound 2 bash "$TEST_SCRIPT" --internal-worker "$forged_root" "$forged_attestation" \
    >"$WORK/forged-worker.out" 2>&1; then
    forged_rc=0
  else
    forged_rc=$?
  fi
  case "$forged_root" in
    /tmp/platform-observability-access-test.??????) rm -rf -- "$forged_root" ;;
  esac
  (( forged_rc != 124 )) && ! grep -q '^PASS:' "$WORK/forged-worker.out" ||
    fail 'direct forged internal worker bypassed the suite supervisor'
  printf 'SUPERVISOR_FORGED_WORKER_REJECTED=PASS\n'
  suite_started=$SECONDS
  if run_with_wall_bound "$SUITE_WALL_BOUND_SECONDS" \
    /usr/bin/bash -c 'source "$1"; run_focused_suite_worker "$2"' \
      _ "$TEST_SCRIPT" "$WORK"; then
    suite_rc=0
  else
    suite_rc=$?
  fi
  suite_elapsed=$((SECONDS - suite_started))
  mapfile -t suite_orphan_pids < <(pgrep -f -- "$WORK" || true)
  suite_orphan_count=${#suite_orphan_pids[@]}
  if (( suite_orphan_count > 0 )); then
    kill -TERM "${suite_orphan_pids[@]}" 2>/dev/null || true
    sleep 0.2
    kill -KILL "${suite_orphan_pids[@]}" 2>/dev/null || true
  fi
  (( suite_rc != 124 )) || fail "focused suite exceeded its ${SUITE_WALL_BOUND_SECONDS}s hard wall-clock bound"
  (( suite_orphan_count == 0 )) || fail 'focused suite left a fixture-owned orphan process'
  (( suite_elapsed <= SUITE_WALL_BOUND_SECONDS + 2 )) || fail 'focused suite wall-clock accounting exceeded its hard bound'
  printf 'SUITE_WALL_SECONDS=%s\n' "$suite_elapsed"
  printf 'SUITE_WALL_BOUND_SECONDS=%s\n' "$SUITE_WALL_BOUND_SECONDS"
  printf 'SUITE_ORPHAN_PROCESSES=0\n'
  return "$suite_rc"
}

run_focused_suite_worker() {
(( $# == 1 )) || fail 'invalid focused suite worker arity'
WORK=$1
[[ "$WORK" =~ ^/tmp/platform-observability-access-test\.[A-Za-z0-9]{6}$ &&
  -d "$WORK" && ! -L "$WORK" && "$(readlink -f -- "$WORK")" == "$WORK" &&
  "$(stat -c '%u:%a' -- "$WORK")" == "$(id -u):700" ]] ||
  fail 'focused suite supervisor work root is not exact, canonical, and private'
PROGRESS_LOG="$WORK/suite-progress.log"
: >"$PROGRESS_LOG"
chmod 0600 "$PROGRESS_LOG"

deadline_probe="platform-observability-deadline-probe-$BASHPID"
if run_with_wall_bound 1 /usr/bin/bash -c 'exec -a "$1" /usr/bin/sleep 5' _ "$deadline_probe"; then
  deadline_probe_rc=0
else
  deadline_probe_rc=$?
fi
[[ "$deadline_probe_rc" == 124 ]] || fail 'suite wall-clock helper did not enforce its exact deadline'
! pgrep -f -- "$deadline_probe" >/dev/null || fail 'suite wall-clock helper left an orphan process'
pass 'suite wall-clock helper terminates its process group without an orphan'

# Break caught: production dry-run used to return before validating the
# environment boundary, so a caller-controlled binary override silently
# survived until a later execute invocation.
if env PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN=/usr/bin/true \
  bash "$APPLY" >"$WORK/production-override.out" 2>&1; then
  fail 'production dry-run accepted a test boundary override'
fi
pass 'production rejects every access boundary override before dry-run'

# Break caught: TEST_MODE used to be an env-only switch. In particular the
# real sudo binary was accepted because dry-run skipped fixture validation.
split_fixture "$(new_fixture unsafe-real-sudo)"
if PLATFORM_TEST_SUDO_BIN=/usr/bin/sudo \
  run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    >"$FIXTURE/output" 2>&1; then
  fail 'test mode accepted the real sudo binary'
fi
grep -Fq 'test command boundary is unsafe' "$FIXTURE/output" ||
  fail 'real sudo rejection did not identify the unsafe boundary'
pass 'test mode rejects real sudo before dry-run'

split_fixture "$(new_fixture dry-run)"
output="$(run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA")" || fail 'dry-run failed'
[[ "$output" == *'OBSERVABILITY_ACCESS_DRY_RUN=PASS'* ]] || fail 'dry-run PASS marker missing'
[[ "$output" == *"TARGET_INITIAL_SHA256=$INITIAL_PRODUCTION_SHA"* ]] || fail 'dry-run did not show pinned initial hash'
[[ "$output" == *"POST_SUBSTRATE_SHA256=$POST_PRODUCTION_SHA"* ]] || fail 'dry-run did not show pinned post hash'
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'dry-run invoked a system boundary'
pass 'source-only dry-run prints pinned readiness without system access'

split_fixture "$(new_fixture alertmanager-missing-title)"
python3 - "$METRIC/alertmanager.yaml" <<'PY'
import sys
import yaml

path = sys.argv[1]
with open(path, encoding="utf-8") as stream:
    items = [item for item in yaml.safe_load_all(stream) if item is not None]
config = next(item for item in items if item.get("kind") == "AlertmanagerConfig")
receiver = next(item for item in config["spec"]["receivers"] if item["name"] == "platform-slack")
receiver["slackConfigs"][0].pop("title")
with open(path, "w", encoding="utf-8") as stream:
    yaml.safe_dump_all(items, stream, explicit_start=True, sort_keys=False)
PY
chmod 0600 "$METRIC/alertmanager.yaml"
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'Alertmanager handoff without Slack title was accepted'
fi
assert_pre_prompt_rejection "$FIXTURE" 'Alertmanager handoff without Slack title'
pass 'Alertmanager handoff without Slack title fails before prompt, ledger, server dry-run, and apply'

# Break caught: the production receiver predicate treated the operator's
# qualified receiver pair as absent, causing a valid rules-alerts transaction
# to roll back after all resources had been applied.
split_fixture "$(new_fixture qualified-receiver-pair)"
if ! run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  assert_reconcile_rollback "$FIXTURE" 'qualified receiver pair'
  fail 'qualified receiver pair was rejected'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" ||
  fail 'qualified receiver pair omitted transaction success'
pass 'qualified Alertmanager receiver pair is accepted exactly once'

split_fixture "$(new_fixture qualified-receiver-pair-reversed)"
if ! PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE=qualified-reversed \
  run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  assert_reconcile_rollback "$FIXTURE" 'reversed qualified receiver pair'
  fail 'reversed qualified receiver pair was rejected'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" ||
  fail 'reversed qualified receiver pair omitted transaction success'
pass 'reversed qualified Alertmanager receiver pair is accepted exactly once'

# Break caught: exact source-pinned historical evidence older than the former
# 24-hour ceiling was rejected even though age is not an authority signal.
old_capture="$(date -u -d '49 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
split_fixture "$(new_fixture old-authoritative-pair \
  https://git.learn.hyeonworks.com/platform/runbooks/observability.md "$old_capture")"
if ! run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  [[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] ||
    fail 'rejected old authoritative pair unexpectedly reached confirmation'
  ! grep -Eq '^sudo .* install ' "$FIXTURE/commands.log" ||
    fail 'rejected old authoritative pair unexpectedly invoked ledger install'
  ! grep -Fq 'kubectl --request-timeout=10s apply -f' "$FIXTURE/commands.log" ||
    fail 'rejected old authoritative pair unexpectedly invoked Kubernetes apply'
  [[ ! -e "$FIXTURE/apply-count" ]] || fail 'rejected old authoritative pair unexpectedly mutated Kubernetes'
  [[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
    fail 'rejected old authoritative pair unexpectedly installed a rollback ledger'
  [[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
    fail 'rejected old authoritative pair unexpectedly wrote acceptance'
  fail 'old exact 21/30 authoritative pair was rejected'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" ||
  fail 'old exact 21/30 authoritative pair omitted transaction success'
grep -Fq 'schema=platform-observability-rules-alerts-v2' \
  "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ||
  fail 'old exact 21/30 authoritative pair omitted v2 acceptance'
pass 'old exact 21/30 source-pinned inventories remain authoritative'

split_fixture "$(new_fixture valid-inventory-pair)"
if ! env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \
  PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE="$FIXTURE/rollbacks" \
  bash -c '
    source "$1"
    verified_output_dir=$2
    expected_initial_sha=$3
    expected_post_sha=$4
    validate_handoff_root
    verify_handoff_unchanged
  ' _ "$APPLY" "$METRIC" "$INITIAL_SHA" "$POST_SHA" >"$FIXTURE/output" 2>&1; then
  sed -n '1,160p' "$FIXTURE/output" >&2
  fail 'valid exact 21/30 pair failed an initial or unchanged source-only gate'
fi
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'source-only inventory pair control reached a system boundary'
pass 'valid exact 21/30 pair passes both initial and unchanged source-only gates'

for timestamp_case in malformed-utc invalid-calendar future-skew; do
  split_fixture "$(new_fixture "timestamp-$timestamp_case")"
  case "$timestamp_case" in
    malformed-utc) timestamp_value='not-a-utc-second' ;;
    invalid-calendar) timestamp_value='2026-02-30T12:00:00Z' ;;
    future-skew) timestamp_value="$(date -u -d '600 seconds' +%Y-%m-%dT%H:%M:%SZ)" ;;
  esac
  INITIAL_SHA="$(rewrite_inventory_filter "$METRIC" target-initial \
    ".captured_at_utc = \"$timestamp_value\"")"
  if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "invalid inventory timestamp was accepted: $timestamp_case"
  fi
  assert_pre_prompt_rejection "$FIXTURE" "inventory timestamp $timestamp_case"
  pass "inventory timestamp $timestamp_case fails closed before confirmation"
done

for count_case in initial-20 initial-22 post-29 post-31; do
  split_fixture "$(new_fixture "count-$count_case")"
  case "$count_case" in
    initial-20) phase=target-initial; actual_count=20 ;;
    initial-22) phase=target-initial; actual_count=22 ;;
    post-29) phase=post-substrate; actual_count=29 ;;
    post-31) phase=post-substrate; actual_count=31 ;;
  esac
  changed_sha="$(rewrite_inventory_count "$METRIC" "$phase" "$actual_count")"
  if [[ "$phase" == target-initial ]]; then
    INITIAL_SHA=$changed_sha
  else
    POST_SHA=$changed_sha
  fi
  run_inventory_validator "$METRIC" "$phase" "$changed_sha" "$actual_count" \
    >"$FIXTURE/source-control.out" 2>&1 ||
    fail "source-only actual-count control failed: $count_case"
  if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "production expected target count accepted drift: $count_case"
  fi
  assert_pre_prompt_rejection "$FIXTURE" "inventory count $count_case"
  pass "inventory count $count_case is self-consistent but not production-authoritative"
done

split_fixture "$(new_fixture invalid-expected-count)"
for invalid_expected_count in 0 01 x; do
  if run_inventory_validator "$METRIC" target-initial "$INITIAL_SHA" "$invalid_expected_count" \
    >"$FIXTURE/expected-count-$invalid_expected_count.out" 2>&1; then
    fail "invalid expected count reached inventory validation: $invalid_expected_count"
  fi
  grep -Fq 'inventory expected target count is invalid' \
    "$FIXTURE/expected-count-$invalid_expected_count.out" ||
    fail "invalid expected count lacked its entry-gate error: $invalid_expected_count"
done
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'invalid expected count reached a system boundary'
pass 'expected count rejects 0, 01, and x at the source-only function entry'

for semantic_case in schema phase context api health last-error empty-metrics metric-name label-type; do
  split_fixture "$(new_fixture "semantic-$semantic_case")"
  case "$semantic_case" in
    schema) filter='.schema = "platform-observability-metric-inventory/v2"' ;;
    phase) filter='.phase = "post-substrate"' ;;
    context) filter='.context = "other"' ;;
    api) filter='.api_server = "https://127.0.0.1:7443"' ;;
    health) filter='.targets[0].health = "down"' ;;
    last-error) filter='.targets[0].last_error = "failed"' ;;
    empty-metrics) filter='.targets[0].metrics = []' ;;
    metric-name) filter='.targets[0].metrics[0].name = 7' ;;
    label-type) filter='.targets[0].metrics[0].label_names[0] = 7' ;;
  esac
  INITIAL_SHA="$(rewrite_inventory_filter "$METRIC" target-initial "$filter")"
  if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "self-consistent inventory semantic drift was accepted: $semantic_case"
  fi
  assert_pre_prompt_rejection "$FIXTURE" "inventory semantic drift $semantic_case"
done
pass 'schema, phase, context, API, health, error, metrics, name, and label-type drift fail closed'

for authority_case in wrong-production-pin one-good-phase swapped-phases; do
  split_fixture "$(new_fixture "authority-$authority_case")"
  case "$authority_case" in
    wrong-production-pin) expected_initial=$INITIAL_PRODUCTION_SHA; expected_post=$POST_PRODUCTION_SHA ;;
    one-good-phase) expected_initial=$INITIAL_SHA; expected_post=$POST_PRODUCTION_SHA ;;
    swapped-phases) expected_initial=$POST_SHA; expected_post=$INITIAL_SHA ;;
  esac
  if run_apply "$FIXTURE" "$METRIC" "$expected_initial" "$expected_post" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "invalid source pin pair was accepted: $authority_case"
  fi
  assert_pre_prompt_rejection "$FIXTURE" "source pin $authority_case"
done

split_fixture "$(new_fixture mixed-generation-a)"
mixed_fixture=$FIXTURE mixed_metric=$METRIC mixed_initial=$INITIAL_SHA mixed_post=$POST_SHA
older_generation="$(date -u -d '2 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
split_fixture "$(new_fixture mixed-generation-b \
  https://git.learn.hyeonworks.com/platform/runbooks/observability.md "$older_generation")"
other_post=$POST_SHA
FIXTURE=$mixed_fixture METRIC=$mixed_metric INITIAL_SHA=$mixed_initial POST_SHA=$mixed_post
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$other_post" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'mixed-generation source pin pair was accepted'
fi
assert_pre_prompt_rejection "$FIXTURE" 'mixed-generation source pin pair'
pass 'wrong, partial, swapped, and mixed-generation source pins fail closed'

for integrity_case in stale-checksum wrong-checksum-grammar extra-phase-entry missing-phase-entry json-symlink checksum-symlink json-hardlink checksum-hardlink wrong-file-mode; do
  split_fixture "$(new_fixture "integrity-$integrity_case")"
  case "$integrity_case" in
    stale-checksum) printf ' \n' >>"$METRIC/target-initial/inventory.json" ;;
    wrong-checksum-grammar) printf 'not-a-sha  inventory.json\n' >"$METRIC/target-initial/inventory.sha256" ;;
    extra-phase-entry) printf 'extra\n' >"$METRIC/target-initial/extra"; chmod 0600 "$METRIC/target-initial/extra" ;;
    missing-phase-entry) rm -- "$METRIC/target-initial/inventory.sha256" ;;
    json-symlink) mv "$METRIC/target-initial/inventory.json" "$FIXTURE/inventory-json-control"; ln -s "$FIXTURE/inventory-json-control" "$METRIC/target-initial/inventory.json" ;;
    checksum-symlink) mv "$METRIC/target-initial/inventory.sha256" "$FIXTURE/inventory-checksum-control"; ln -s "$FIXTURE/inventory-checksum-control" "$METRIC/target-initial/inventory.sha256" ;;
    json-hardlink) ln "$METRIC/target-initial/inventory.json" "$FIXTURE/inventory-json-control" ;;
    checksum-hardlink) ln "$METRIC/target-initial/inventory.sha256" "$FIXTURE/inventory-checksum-control" ;;
    wrong-file-mode) chmod 0644 "$METRIC/target-initial/inventory.json" ;;
  esac
  if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "inventory integrity drift was accepted: $integrity_case"
  fi
  assert_pre_prompt_rejection "$FIXTURE" "inventory integrity $integrity_case"
done
pass 'checksum, entry-set, symlink, hardlink, and mode drift fail closed'

split_fixture "$(new_fixture wrong-owner)"
write_executable "$FIXTURE/bin/id" \
  '#!/usr/bin/env bash' \
  'if [[ "${1:-}" == -u ]]; then printf '\''999999\n'\''; else exec /usr/bin/id "$@"; fi'
if PATH="$FIXTURE/bin:$PATH" run_inventory_validator \
  "$METRIC" target-initial "$INITIAL_SHA" "$INITIAL_TARGET_COUNT" >"$FIXTURE/output" 2>&1; then
  fail 'inventory owner drift was accepted'
fi
grep -Fq 'inventory directory metadata changed' "$FIXTURE/output" ||
  fail 'owner drift did not reach the ownership metadata gate'
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'owner drift reached a system boundary'
pass 'inventory owner drift fails at the source-only metadata gate'

argv_case=0
for args in \
  '--execute' \
  '--execute --rules-alerts' \
  '--execute --verified-output-dir /tmp/example' \
  '--execute --rules-alerts --substrate --verified-output-dir /tmp/example' \
  '--execute --rules-alerts --verified-output-dir /tmp/example --verified-output-dir /tmp/example'; do
  argv_case=$((argv_case + 1))
  split_fixture "$(new_fixture "argv-$argv_case")"
  read -r -a argv <<<"$args"
  if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" "${argv[@]}" >"$FIXTURE/output" 2>&1; then
    fail "invalid argv was accepted: $args"
  fi
  [[ ! -s "$FIXTURE/commands.log" ]] || fail "invalid argv reached a system boundary: $args"
done
pass 'execute accepts only one exact rules-alerts mode and verified handoff argument'

split_fixture "$(new_fixture unsafe-inventory-mode)"
chmod 0755 "$METRIC/target-initial"
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'unsafe inventory mode was accepted'
fi
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'unsafe inventory mode reached prompt'
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'unsafe inventory mode reached mutation'
pass 'inventory ownership, mode, link count, and exact hash gate precedes prompt'

split_fixture "$(new_fixture extra-handoff-entry)"
cp "$METRIC/dashboards.yaml" "$METRIC/grafana.yaml"
chmod 0600 "$METRIC/grafana.yaml"
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'extra renderer handoff entry was accepted'
fi
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'extra renderer handoff entry reached prompt'
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'extra renderer handoff entry reached mutation'
pass 'renderer handoff rejects Grafana and every entry outside three exact rules-alerts files'

split_fixture "$(new_fixture missing-runbook '')"
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'missing HTTPS runbook URL was accepted'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_READINESS=BLOCKED_RUNBOOK_URL' "$FIXTURE/output" ||
  { sed -n '1,160p' "$FIXTURE/output" >&2; fail 'missing runbook blocker marker absent'; }
! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'missing runbook reached apply'
! grep -q '^sudo .*install ' "$FIXTURE/commands.log" || fail 'missing runbook wrote ledger'
pass 'missing runbook URL blocks before prompt and mutation'

split_fixture "$(new_fixture encryption-pre)"
if PLATFORM_TEST_ENCRYPTION_FAIL_AT=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'failed initial encryption gate was accepted'
fi
[[ "$(<"$FIXTURE/commands.log")" == $'sudo -v\nsudo -n true\nencryption --expect-reencrypted' ]] ||
  fail 'initial encryption failure did not stop at exact first gate'
pass 'initial encryption failure is zero mutation'

split_fixture "$(new_fixture slack-schema)"
if PLATFORM_TEST_SLACK_EXTRA_KEY=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'Slack Secret with an extra key was accepted'
fi
! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'Slack schema mismatch reached apply'
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'Slack schema mismatch reached prompt'
pass 'Slack Secret checks only the exact non-value schema before prompt'

split_fixture "$(new_fixture recovery)"
if PLATFORM_TEST_RECOVERY_RC=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'missing Slack recovery marker was accepted'
fi
grep -Fqx 'recovery --check-slack-deployment-evidence' "$FIXTURE/commands.log" ||
  fail 'Slack deployment checker argv is not exact'
! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'missing recovery marker reached apply'
pass 'Slack deployment evidence is an exact pre-prompt gate'

split_fixture "$(new_fixture second-gate-failure)"
if PLATFORM_TEST_RECOVERY_RC_2=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'second Slack deployment gate failure was accepted'
fi
[[ "$(<"$FIXTURE/recovery-count")" == 2 ]] || fail 'second gate failure did not invoke both exact deployment gates'
[[ ! -e "$FIXTURE/apply-count" ]] || fail 'second gate failure reached cluster mutation'
[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
  fail 'second gate failure did not preserve the durable rollback ledger'
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
  fail 'second gate failure wrote acceptance'
pass 'second deployment gate failure preserves ledger and remains zero mutation'

for gate_case in recovery-to-risk risk-to-recovery duplicate-output extra-output partial-output; do
  split_fixture "$(new_fixture "deployment-gate-$gate_case")"
  first='SLACK_DEPLOYMENT_GATE=RECOVERY\n'
  second='SLACK_DEPLOYMENT_GATE=RECOVERY\n'
  expected=RECOVERY
  case "$gate_case" in
    recovery-to-risk) second='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' ;;
    risk-to-recovery) first='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n'; expected=RISK_ACCEPTED ;;
    duplicate-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY\nSLACK_DEPLOYMENT_GATE=RECOVERY\n' ;;
    extra-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY\nextra\n' ;;
    partial-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY' ;;
  esac
  if run_deployment_gate_helper "$FIXTURE" "$first" "$second" "$expected" \
    >"$FIXTURE/output" 2>&1; then
    fail "invalid Slack deployment gate was accepted: $gate_case"
  fi
  [[ ! -e "$FIXTURE/apply-count" && ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
    fail "invalid Slack deployment gate escaped the helper boundary: $gate_case"
done
pass 'verdict swaps and malformed deployment gate output fail closed before mutation'

split_fixture "$(new_fixture last-secret-schema)"
if PLATFORM_TEST_SLACK_EXTRA_KEY_AT=2 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'last Slack Secret schema drift was accepted'
fi
[[ ! -e "$FIXTURE/apply-count" ]] || fail 'last Slack Secret schema drift reached mutation'
[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
  fail 'last Slack Secret schema drift did not preserve the durable ledger'
pass 'last Slack Secret schema check precedes the second deployment gate and mutation'

split_fixture "$(new_fixture production-preflight)"
if PLATFORM_TEST_PREFLIGHT_RC=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'failed production preflight was accepted'
fi
grep -Fq 'kubectl --request-timeout=10s wait --namespace observability --for=condition=Available deployment/grafana deployment/blackbox-exporter --timeout=60s' \
  "$FIXTURE/commands.log" || fail 'focused failure did not execute the production preflight'
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'failed production preflight reached prompt'
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" ||
  fail 'failed production preflight reached mutation'
pass 'production preflight is materially exercised and remains zero mutation on failure'

split_fixture "$(new_fixture last-gate)"
if PLATFORM_TEST_RESTORE_FAIL_AT=2 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'failed last restore gate was accepted'
fi
[[ "$(grep -c '^encryption --expect-reencrypted$' "$FIXTURE/commands.log")" == 2 ]] ||
  { sed -n '1,220p' "$FIXTURE/commands.log" >&2; sed -n '1,160p' "$FIXTURE/output" >&2; fail 'encryption gate did not run twice'; }
[[ "$(grep -c '^restore --check$' "$FIXTURE/commands.log")" == 2 ]] ||
  fail 'restore gate did not run twice'
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'last restore failure reached apply'
[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
  fail 'last gate did not occur after immutable ledger capture'
pass 'last encryption and restore gate failure remains zero cluster mutation'

split_fixture "$(new_fixture handoff-race)"
if PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE="$METRIC/rules.yaml" \
  run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'handoff mutation after confirmation was accepted'
fi
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'handoff race reached apply'
pass 'verified renderer handoff hashes are rechecked after the last gate'

for inventory_drift_case in raw-json rechecksum-count hardlink; do
  split_fixture "$(new_fixture "inventory-last-gate-$inventory_drift_case")"
  if PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE="$inventory_drift_case" \
    run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
      --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "post-confirmation inventory drift was accepted: $inventory_drift_case"
  fi
  assert_post_confirmation_rejection "$FIXTURE" "post-confirmation inventory drift $inventory_drift_case"
  pass "post-confirmation inventory drift $inventory_drift_case stops before first apply"
done

split_fixture "$(new_fixture last-gate-count-wiring)"
INITIAL_SHA="$(rewrite_inventory_count "$METRIC" target-initial 20)"
run_inventory_validator "$METRIC" target-initial "$INITIAL_SHA" 20 \
  >"$FIXTURE/source-control.out" 2>&1 ||
  fail '20-target source-only positive control did not accept its matching expected count'
if env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 bash -c '
  source "$1"
  verified_output_dir=$2
  target_initial_sha=$3
  post_substrate_sha=$4
  verify_handoff_unchanged
' _ "$APPLY" "$METRIC" "$INITIAL_SHA" "$POST_SHA" >"$FIXTURE/output" 2>&1; then
  fail 'last inventory gate lost the production initial target count argument'
fi
grep -Fq 'inventory semantic contract changed: target-initial' "$FIXTURE/output" ||
  fail 'last inventory count gate did not reject the self-consistent 20-target inventory'
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'last-gate count wiring control reached a system boundary'
pass 'last inventory gate carries expected count 21 while direct expected count 20 remains a positive control'

split_fixture "$(new_fixture symlinked-ledger-root)"
mkdir -m 0700 "$FIXTURE/escaped-ledger"
ln -s "$FIXTURE/escaped-ledger" \
  "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'symlinked root ledger was followed'
fi
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'symlinked ledger root reached cluster mutation'
[[ -z "$(find "$FIXTURE/escaped-ledger" -mindepth 1 -print -quit)" ]] ||
  fail 'root ledger write escaped through symlink'
pass 'root ledger lineage rejects symlink escape before cluster mutation'

split_fixture "$(new_fixture success)"
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1 || {
  sed -n '1,260p' "$FIXTURE/output" >&2
  fail 'rules-alerts transition failed'
}
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" || fail 'success marker missing'
grep -Fq 'Type APPLY default to apply only observability rules-alerts: APPLY' "$FIXTURE/output" ||
  fail 'interactive prompt contract changed'
expected_gate_trace='secret-schema-1 -> deployment-gate-1 -> preflight -> server-dry-run -> confirmation -> rollback-ledger -> encryption-last -> handoff/live-last -> secret-schema-2 -> deployment-gate-2 -> first-apply'
mapfile -t gate_trace_parts < <(awk '
  /get secret alertmanager-slack-webhook/ { secret++; if (secret == 1) print "secret-schema-1"; else if (secret == 2) print "secret-schema-2" }
  /^recovery --check-slack-deployment-evidence$/ { gate++; if (gate == 1) print "deployment-gate-1"; else if (gate == 2) print "deployment-gate-2" }
  /wait --namespace observability --for=condition=Available deployment\/grafana/ { print "preflight" }
  /apply --dry-run=server/ && !dry++ { print "server-dry-run" }
  /^sudo -n mv -T -- .*\/mutations.tsv.next-[0-9]+ .*\/mutations.tsv$/ && !ledger++ { print "rollback-ledger" }
  /^encryption --expect-reencrypted$/ && ++encryption == 2 { print "encryption-last" }
  /^restore --check$/ && ++restore == 2 { print "handoff/live-last" }
  /^kubectl --request-timeout=10s apply -f / && !apply++ { print "first-apply" }
' "$FIXTURE/commands.log")
actual_gate_trace=''
for gate_trace_part in "${gate_trace_parts[@]}"; do
  [[ -z "$actual_gate_trace" ]] || actual_gate_trace+=' -> '
  actual_gate_trace+="$gate_trace_part"
done
actual_gate_trace="${actual_gate_trace/server-dry-run -> rollback-ledger/server-dry-run -> confirmation -> rollback-ledger}"
[[ "$actual_gate_trace" == "$expected_gate_trace" ]] ||
  fail "two Slack gates and first apply did not preserve the exact required order: $actual_gate_trace"
pass 'second deployment gate preserves the required pre-prompt-to-first-apply order'
# Break caught: the former test hook bypassed the production preflight and
# reconcile path, so a focused PASS did not prove Prometheus or Grafana had
# accepted the rendered rules and dashboards.
grep -Fq 'kubectl --request-timeout=10s get probe --namespace observability --output=json' \
  "$FIXTURE/commands.log" || fail 'focused success bypassed the production preflight path'
grep -Fq 'kubectl --request-timeout=10s get --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1/rules' \
  "$FIXTURE/commands.log" || fail 'focused success did not exercise Prometheus rule acceptance'
grep -Fq 'kubectl --request-timeout=10s get --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-alertmanager:9093/proxy/api/v2/receivers' \
  "$FIXTURE/commands.log" || fail 'focused success did not exercise the exact Alertmanager receiver API'
grep -Fq 'kubectl --request-timeout=10s exec --namespace observability deployment/grafana --container grafana --' \
  "$FIXTURE/commands.log" || fail 'focused success did not prove Grafana consumed the dashboard ConfigMaps'
! grep -Eq '^(preflight|postcheck)$' "$FIXTURE/commands.log" ||
  fail 'focused success still used a test-only health hook'
ledger="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv"
[[ -f "$ledger" && ! -L "$ledger" && "$(stat -c %a "$ledger")" == 600 ]] || fail 'root ledger is unsafe'
[[ "$(wc -l <"$ledger" | tr -d ' ')" == 13 ]] || fail 'ledger is not header plus exact twelve resources'
awk -F '\t' '
  NR == 1 { exit($0 != "phase\towner\tapiVersion\tkind\tnamespace\tname\texisted\trestore-mode\tpayload-file\tpayload-sha256") }
  NR > 1 { if (NF != 10 || $1 != "access-rules-alerts" || ($2 != "kube-prometheus-stack" && $2 != "platform-rules")) exit 1 }
' "$ledger" || fail 'ledger header, fields, phase, or owner is not exact'
[[ "$(awk -F '\t' '$4=="ConfigMap"{n++} END{print n+0}' "$ledger")" == 5 ]] || fail 'dashboard ledger set is not exact'
[[ "$(awk -F '\t' '$4=="PrometheusRule"{n++} END{print n+0}' "$ledger")" == 4 ]] || fail 'rule ledger set is not exact'
[[ "$(awk -F '\t' '$4=="AlertmanagerConfig"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'AlertmanagerConfig ledger set is not exact'
[[ "$(awk -F '\t' '$4=="Alertmanager"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'Alertmanager ledger set is not exact'
[[ "$(awk -F '\t' '$4=="NetworkPolicy"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'NetworkPolicy ledger set is not exact'
! grep -Eq $'\t(Secret|Probe|Ingress|Deployment|Service)\t' "$ledger" || fail 'forbidden substrate or Secret resource entered ledger'
payload_rel="$(awk -F '\t' '$4=="Alertmanager"{print $9}' "$ledger")"
payload="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/$payload_rel"
python3 - "$payload" <<'PY' || fail 'prior Alertmanager payload is not sanitized'
import json, sys
item=json.load(open(sys.argv[1]))
meta=item.get("metadata", {})
for key in ("uid","resourceVersion","generation","creationTimestamp","managedFields"):
    assert key not in meta
assert "status" not in item
assert "kubectl.kubernetes.io/last-applied-configuration" not in meta.get("annotations", {})
PY
acceptance="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
[[ -f "$acceptance" && "$(stat -c %a "$acceptance")" == 600 ]] || fail 'acceptance marker is unsafe'
grep -Fqx 'schema=platform-observability-rules-alerts-v2' "$acceptance" || fail 'acceptance schema missing'
grep -Fqx "rollback_id=$ROLLBACK_ID" "$acceptance" || fail 'acceptance rollback id missing'
grep -Fqx "target_initial_sha256=$INITIAL_SHA" "$acceptance" || fail 'acceptance initial hash mismatch'
grep -Fqx "post_substrate_sha256=$POST_SHA" "$acceptance" || fail 'acceptance post hash mismatch'
grep -Fqx 'slack_deployment_gate=RECOVERY' "$acceptance" || fail 'acceptance deployment verdict missing'
grep -Fqx 'slack_gate_approval_ref=strict-recovery-evidence-v1' "$acceptance" || fail 'recovery acceptance reference mismatch'
grep -Fqx 'slack_gate_accepted_by_uid=not-applicable' "$acceptance" || fail 'recovery acceptance uid mismatch'
grep -Fqx 'state=accepted' "$acceptance" || fail 'acceptance state missing'
[[ "$(wc -l <"$acceptance" | tr -d ' ')" == 9 ]] || fail 'v2 acceptance field count is not exact'
reconcile_last_line="$(awk '
  /\/proxy\/api\/v1\/rules|\/proxy\/api\/v2\/receivers| exec --namespace observability deployment\/grafana / {line=NR}
  END {print line+0}
' "$FIXTURE/commands.log")"
acceptance_install_line="$(awk '/^sudo -n install .*acceptance\.env/ {line=NR} END {print line+0}' "$FIXTURE/commands.log")"
(( reconcile_last_line > 0 && acceptance_install_line > reconcile_last_line )) ||
  fail 'acceptance marker was installed before all production acceptance evidence'
[[ "$(grep -c '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log")" == 12 ]] || fail 'apply was not one exact resource at a time'
! grep -Eqi '^kubectl --request-timeout=10s apply -f .*/(grafana|blackbox|targets|private-dns)\.yaml|^kubectl --request-timeout=10s delete (probe|ingress|deployment|service) ' "$FIXTURE/commands.log" ||
  fail 'rules-alerts mode mutated a substrate, Probe, or Nginx resource'
first_sudo="$(awk '/^sudo / {print; exit}' "$FIXTURE/commands.log")"
[[ "$first_sudo" == 'sudo -v' ]] || fail 'sudo refresh did not precede root reads/writes'
awk '
  $0 == "encryption --expect-reencrypted" { encryption[++encryption_count]=NR }
  $0 == "restore --check" { restore[++restore_count]=NR }
  /^sudo -n mv -T -- .*\/mutations.tsv.next-[0-9]+ .*\/mutations.tsv$/ && !ledger { ledger=NR }
  /^kubectl --request-timeout=10s apply -f / && !mutation { mutation=NR }
  END {
    exit(!(encryption_count==2 && restore_count==2 &&
      encryption[1] < restore[1] && restore[1] < ledger &&
      ledger < encryption[2] && encryption[2] < restore[2] && restore[2] < mutation))
  }
' "$FIXTURE/commands.log" || fail 'pre/last gate, ledger, and mutation order changed'
pass 'current exact 21/30 success captures the sanitized ledger and applies only twelve rules-alerts resources'

split_fixture "$(new_fixture risk-accepted-success)"
PLATFORM_TEST_RECOVERY_OUTPUT_1='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' \
PLATFORM_TEST_RECOVERY_OUTPUT_2='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' \
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1 ||
  fail 'risk-accepted deployment gate success fixture failed'
acceptance="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
grep -Fqx 'slack_deployment_gate=RISK_ACCEPTED' "$acceptance" || fail 'risk acceptance verdict mismatch'
grep -Fqx 'slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' "$acceptance" || fail 'risk acceptance reference mismatch'
grep -Fqx 'slack_gate_accepted_by_uid=1000' "$acceptance" || fail 'risk acceptance uid mismatch'
pass 'risk-accepted deployment verdict records the exact v2 acceptance mapping'

split_fixture "$(new_fixture acceptance-write-failure)"
if PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE=1 \
  run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'v2 acceptance write failure returned success'
fi
grep -Fq 'APPLIED_ACCEPTANCE_UNRECORDED' "$FIXTURE/output" || fail 'acceptance write failure did not identify missing acceptance'
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
  fail 'acceptance write failure left an acceptance marker'
pass 'v2 acceptance write failure remains unaccepted'

for reconcile_case in prometheus-rule grafana-sidecar; do
  case_progress "reconcile-$reconcile_case"
  split_fixture "$(new_fixture "reconcile-$reconcile_case")"
  prometheus_drop=0
  grafana_stale=0
  case "$reconcile_case" in
    prometheus-rule)
      prometheus_drop=1
      ;;
    grafana-sidecar)
      grafana_stale=1
      ;;
  esac
  if PLATFORM_TEST_PROMETHEUS_DROP_RULE="$prometheus_drop" \
    PLATFORM_TEST_GRAFANA_STALE_DASHBOARD="$grafana_stale" \
    run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
      --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "incomplete production acceptance was accepted: $reconcile_case"
  fi
  assert_reconcile_rollback "$FIXTURE" "incomplete production acceptance $reconcile_case"
done
for receiver_case in raw empty null duplicate extra wrong-namespace wrong-config wrong-local malformed; do
  case_progress "reconcile-alertmanager-receiver-$receiver_case"
  split_fixture "$(new_fixture "reconcile-alertmanager-receiver-$receiver_case")"
  if PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE="$receiver_case" \
    run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
      --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "invalid Alertmanager receiver was accepted: $receiver_case"
  fi
  assert_reconcile_rollback "$FIXTURE" "invalid Alertmanager receiver $receiver_case"
done
expected_reconcile_progress=$'CASE_BEGIN=reconcile-prometheus-rule\nCASE_BEGIN=reconcile-grafana-sidecar\nCASE_BEGIN=reconcile-alertmanager-receiver-raw\nCASE_BEGIN=reconcile-alertmanager-receiver-empty\nCASE_BEGIN=reconcile-alertmanager-receiver-null\nCASE_BEGIN=reconcile-alertmanager-receiver-duplicate\nCASE_BEGIN=reconcile-alertmanager-receiver-extra\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-namespace\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-config\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-local\nCASE_BEGIN=reconcile-alertmanager-receiver-malformed'
[[ "$(<"$PROGRESS_LOG")" == "$expected_reconcile_progress" ]] ||
  fail 'long reconciliation failure matrix did not emit exact per-case progress'
pass 'Prometheus rules, exact Alertmanager receiver matrix, and Grafana sidecar evidence each fail closed before acceptance'

split_fixture "$(new_fixture apply-outcome-unknown)"
if PLATFORM_TEST_FAIL_APPLY_AT=5 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'failed apply returned success'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
  fail 'non-zero apply result was incorrectly reported as a certain no-op'
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
  fail 'unknown apply outcome did not require manual recovery'
[[ "$(find "$FIXTURE/state" -type f | wc -l | tr -d ' ')" == 1 ]] ||
  fail 'recorded prefix was not rolled back after an ambiguous apply result'
pass 'non-zero apply response is treated as an unknown outcome while owned prefix rolls back'

split_fixture "$(new_fixture apply-response-loss)"
if PLATFORM_TEST_FAIL_AFTER_APPLY_AT=5 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'lost apply response returned success'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
  fail 'lost apply response was reported as a certain rollback'
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
  fail 'lost apply response omitted manual recovery'
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
  fail 'lost apply response wrote acceptance'
pass 'apply response loss preserves truthful ambiguity and no acceptance marker'

split_fixture "$(new_fixture rollback)"
  if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 \
    run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
  --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'failed reconcile returned success'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$FIXTURE/output" || fail 'rollback PASS marker missing'
prior="$FIXTURE/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json"
jq -e '.metadata.uid == "uid-alertmanager-prior" and .spec.replicas == 1' "$prior" >/dev/null ||
  fail 'rollback did not restore the prior Alertmanager owner payload'
[[ "$(find "$FIXTURE/state" -type f | wc -l | tr -d ' ')" == 1 ]] || fail 'rollback left newly-created objects'
! grep -Eq '^kubectl --request-timeout=10s (delete|apply).*(Secret|PersistentVolumeClaim|Probe)' "$FIXTURE/commands.log" ||
  fail 'rollback touched a preserved or non-rules resource'
delete_order="$(awk '$1=="raw-delete" {path=$2; sub(".*/","",path); print path}' "$FIXTURE/commands.log")"
expected_delete_order=$'observability-allow-alertmanager-public-https\nplatform-alertmanager\nplatform-verified-services\nplatform-observability-core\nplatform-certificate-probes\nplatform-aistor-storage-quota\ngrafana-dashboard-workload-health\ngrafana-dashboard-platform-services\ngrafana-dashboard-observability-backends\ngrafana-dashboard-kubernetes-node\ngrafana-dashboard-https-endpoints'
[[ "$delete_order" == "$expected_delete_order" ]] || fail 'rollback did not delete owned creations in exact reverse dependency order'
[[ "$(grep -c '^raw-delete .* exact=true$' "$FIXTURE/commands.log")" == 11 ]] ||
  fail 'rollback did not send exact UID-preconditioned DeleteOptions through the raw API'
grep -Fq 'replace-preconditions Alertmanager/observability-core-kube-pr-alertmanager uid=uid-alertmanager-prior resourceVersion=' \
  "$FIXTURE/commands.log" || fail 'rollback restore did not bind both UID and live resourceVersion'
! grep -Fq 'kubectl --request-timeout=10s delete ' "$FIXTURE/commands.log" ||
  fail 'rollback used a plain kubectl delete branch'
pass 'failed reconcile reverses only ledger-owned resources and restores exact prior owner'

for delete_mode in response-loss conflict timeout; do
  split_fixture "$(new_fixture "delete-$delete_mode")"
  started_at="$(date +%s)"
  if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 PLATFORM_TEST_DELETE_MODE="$delete_mode" \
    run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
      --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
    fail "raw DELETE $delete_mode returned success"
  fi
  elapsed=$(( $(date +%s) - started_at ))
  (( elapsed <= 30 )) || fail "raw DELETE $delete_mode exceeded its bounded timeout"
  grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
    fail "raw DELETE $delete_mode was not reported truthfully"
  grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
    fail "raw DELETE $delete_mode omitted manual recovery"
  grep -Eq '^raw-delete .* exact=true$' "$FIXTURE/commands.log" ||
    fail "raw DELETE $delete_mode did not carry exact UID preconditions"
  [[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
    fail "raw DELETE $delete_mode wrote acceptance"
done
pass 'raw DELETE response loss, conflict, and timeout stay bounded and truthfully ambiguous'

split_fixture "$(new_fixture replace-conflict)"
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 PLATFORM_TEST_REPLACE_CONFLICT=1 \
  run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'UID/resourceVersion restore conflict returned success'
fi
grep -Fq 'replace-preconditions Alertmanager/observability-core-kube-pr-alertmanager uid=uid-alertmanager-prior resourceVersion=' \
  "$FIXTURE/commands.log" || fail 'conflicted restore did not carry UID and resourceVersion'
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
  fail 'restore conflict did not remain ambiguous'
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
  fail 'restore conflict omitted manual recovery'
pass 'UID and resourceVersion restore conflict fails closed with manual recovery'

split_fixture "$(new_fixture signal)"
(
  trap 'exit 143' TERM
  PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=5 \
    run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
      --execute --rules-alerts --verified-output-dir "$METRIC"
) >"$FIXTURE/output" 2>&1 &
signal_pid=$!
for ((signal_attempt=1; signal_attempt<=300; signal_attempt++)); do
  [[ -e "$FIXTURE/postcheck-reached" ]] && break
  kill -0 "$signal_pid" 2>/dev/null || break
  sleep 0.05
done
[[ -e "$FIXTURE/postcheck-reached" ]] || fail 'signal fixture never reached a mutation-owned reconciliation boundary'
apply_pid="$(pgrep -P "$signal_pid" -f "$APPLY" | head -n1)"
[[ "$apply_pid" =~ ^[0-9]+$ ]] || fail 'signal fixture could not bind the apply process'
kill -TERM "$apply_pid"
if wait "$signal_pid"; then
  fail 'signal during mutation returned success'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$FIXTURE/output" ||
  { sed -n '1,200p' "$FIXTURE/output" >&2; fail 'signal during reconciliation did not complete owned rollback'; }
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
  fail 'signal during mutation wrote acceptance'
pass 'signal during reconciliation invokes owned rollback and leaves no false acceptance'

split_fixture "$(new_fixture uid-drift)"
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_DRIFT_RESOURCE='ConfigMap/grafana-dashboard-workload-health' \
  run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
    --execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
  fail 'UID-drifted rollback returned success'
fi
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
  fail 'UID drift did not produce truthful ambiguity'
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || fail 'manual recovery marker missing'
[[ -f "$FIXTURE/state/ConfigMap__observability__grafana-dashboard-workload-health.json" ]] ||
  fail 'UID-drifted object was deleted without ownership'
pass 'rollback refuses UID drift and reports manual recovery truthfully'

printf 'Assertions: %d\n' "$ASSERTIONS"
printf 'APPLY OBSERVABILITY ACCESS TEST PASS\n'
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
  supervise_focused_suite "$@"
fi