#!/usr/bin/env bash
set -euo pipefail

readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
readonly RULE_DIR="${ROOT_DIR}/services/observability/rules/platform"
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
readonly EXPECTED_RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"

runbook_url_blocked=0
temp_dir=''

# Each raw metric in the three Task 7 rule resources is bound to the
# inventory phase and scrape pool that supplied it. Probe metrics select a
# pool from their explicit probe-group matcher below.
declare -A new_rule_metric_phase=(
  [minio_cluster_usage_buckets_total_bytes]=target-initial
  [minio_cluster_usage_buckets_quota_total_bytes]=target-initial
  [cnpg_collector_up]=target-initial
  [alloy_config_last_load_successful]=target-initial
  [loki_write_batch_retries_total]=target-initial
  [loki_write_dropped_entries_total]=target-initial
  [loki_runtime_config_last_reload_successful]=target-initial
  [loki_ingester_wal_disk_usage_percent]=target-initial
  [probe_success]=post-substrate
  [probe_ssl_earliest_cert_expiry]=post-substrate
)
declare -A new_rule_metric_pool=(
  [minio_cluster_usage_buckets_total_bytes]=serviceMonitor/object-storage/aistor-bucket-usage/0
  [minio_cluster_usage_buckets_quota_total_bytes]=serviceMonitor/object-storage/aistor-bucket-usage/0
  [cnpg_collector_up]=podMonitor/platform-data/platform-postgres/0
  [alloy_config_last_load_successful]=serviceMonitor/observability-agent/alloy/0
  [loki_write_batch_retries_total]=serviceMonitor/observability-agent/alloy/0
  [loki_write_dropped_entries_total]=serviceMonitor/observability-agent/alloy/0
  [loki_runtime_config_last_reload_successful]=serviceMonitor/observability/loki/0
  [loki_ingester_wal_disk_usage_percent]=serviceMonitor/observability/loki/0
  [probe_success]=probe-group
  [probe_ssl_earliest_cert_expiry]=probe-group
)
declare -A new_rule_probe_pool=(
  ["probe_success|public-edge"]=probe/observability/platform-public-edge
  ["probe_success|private-edge"]=probe/observability/platform-private-edge
  ["probe_success|private-internal"]=probe/observability/platform-private-internal
  ["probe_ssl_earliest_cert_expiry|public-edge"]=probe/observability/platform-public-edge
  ["probe_ssl_earliest_cert_expiry|private-edge"]=probe/observability/platform-private-edge
)
declare -A new_rule_matcher_contract=(
  [minio_cluster_usage_buckets_total_bytes]=$'bucket\t=~\tloki|tempo'
  [minio_cluster_usage_buckets_quota_total_bytes]=$'bucket\t=~\tloki|tempo'
  [cnpg_collector_up]=$'cluster\t=\tplatform-postgres\nnamespace\t=\tplatform-data'
  [alloy_config_last_load_successful]=$'namespace\t=\tobservability-agent'
  [loki_write_batch_retries_total]=$'namespace\t=\tobservability-agent'
  [loki_write_dropped_entries_total]=$'namespace\t=\tobservability-agent'
  [loki_runtime_config_last_reload_successful]=$'namespace\t=\tobservability'
  [loki_ingester_wal_disk_usage_percent]=$'namespace\t=\tobservability'
  ["probe_success|=|public-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tpublic-edge'
  ["probe_success|=|private-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tprivate-edge'
  ["probe_success|=|private-internal"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tprivate-internal'
  ["probe_ssl_earliest_cert_expiry|=~|public-edge|private-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=~\tpublic-edge|private-edge'
)
declare -A recording_rule_expression=()
declare -A recording_rule_state=()
declare -a selector_matcher_labels=()
declare -A selector_matcher_operator=()
declare -A selector_matcher_value=()

fail() {
  printf 'FAIL: %s\n' "$*" >&2
  exit 1
}

pass() {
  printf 'PASS: %s\n' "$*"
}

cleanup() {
  if [[ -n "$temp_dir" && "$temp_dir" == /tmp/tmp.* && -d "$temp_dir" ]]; then
    rm -rf -- "$temp_dir"
  fi
}

require_command() {
  command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
}

assert_inventory() {
  local inventory_dir=$1
  local expected_phase=$2
  local inventory_file="${inventory_dir}/inventory.json"
  local checksum_file="${inventory_dir}/inventory.sha256"
  local expected_sha actual_sha

  [[ -f "$inventory_file" ]] || fail "missing inventory: ${inventory_file}"
  [[ -f "$checksum_file" ]] || fail "missing inventory checksum: ${checksum_file}"
  expected_sha="$(awk '$2 == "inventory.json" { print $1 }' "$checksum_file")"
  [[ "$expected_sha" =~ ^[0-9a-f]{64}$ ]] || fail "invalid inventory checksum record: ${checksum_file}"
  actual_sha="$(sha256sum "$inventory_file" | awk '{ print $1 }')"
  [[ "$actual_sha" == "$expected_sha" ]] || fail "inventory checksum mismatch: ${expected_phase}"
  jq -e --arg phase "$expected_phase" '
    .schema == "platform-observability-metric-inventory/v1" and
    .phase == $phase and
    (.targets | type == "array" and length > 0) and
    all(.targets[];
      .health == "up" and
      (.metrics | type == "array" and length > 0)
    )
  ' "$inventory_file" >/dev/null || fail "inventory schema/phase mismatch: ${expected_phase}"
}

assert_metric_label() {
  local inventory_file=$1
  local scrape_pool=$2
  local metric_name=$3
  local label_name=$4

  jq -e \
    --arg pool "$scrape_pool" \
    --arg metric "$metric_name" \
    --arg label "$label_name" '
      any(.targets[];
        .scrape_pool == $pool and
        any(.metrics[];
          .name == $metric and
          (.label_names | index($label)) != null
        )
      )
    ' "$inventory_file" >/dev/null ||
    fail "inventory lacks ${scrape_pool} ${metric_name}{${label_name}}"
}

assert_metric() {
  local inventory_file=$1
  local scrape_pool=$2
  local metric_name=$3

  jq -e \
    --arg pool "$scrape_pool" \
    --arg metric "$metric_name" '
      any(.targets[];
        .scrape_pool == $pool and
        any(.metrics[]; .name == $metric)
      )
    ' "$inventory_file" >/dev/null ||
    fail "inventory lacks ${scrape_pool} ${metric_name}"
}

parse_explicit_matchers() {
  local selector=$1
  local context=$2
  local matchers matcher_re match label operator value

  matchers=${selector#*\{}
  matchers=${matchers%\}}
  selector_matcher_labels=()
  selector_matcher_operator=()
  selector_matcher_value=()
  matcher_re='^[[:space:],]*("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)[[:space:]]*(=~|!~|!=|=)[[:space:]]*"(([^"\\]|\\.)*)"'

  while [[ -n "${matchers//[[:space:],]/}" ]]; do
    [[ "$matchers" =~ $matcher_re ]] ||
      fail "unparseable explicit matcher in ${context}: ${matchers}"
    match=${BASH_REMATCH[0]}
    label=${BASH_REMATCH[1]}
    operator=${BASH_REMATCH[2]}
    value=${BASH_REMATCH[3]}
    label=${label#\"}
    label=${label%\"}
    [[ -z ${selector_matcher_operator[$label]+x} ]] ||
      fail "duplicate explicit matcher label ${label} in ${context}"
    selector_matcher_labels+=("$label")
    selector_matcher_operator["$label"]=$operator
    selector_matcher_value["$label"]=$value
    matchers=${matchers:${#match}}
    [[ -z "${matchers//[[:space:]]/}" || "$matchers" =~ ^[[:space:]]*, ]] ||
      fail "unparseable explicit matcher separator in ${context}: ${matchers}"
  done
}

assert_exact_matcher_contract() {
  local metric=$1
  local has_selector=$2
  local pool_binding=$3
  local context=$4
  local contract_key=$metric
  local group_label='observability.hyeonworks.com/probe-group'
  local expected_signature actual_signature label

  (( has_selector == 1 )) ||
    fail "exact matcher contract requires a selector for ${metric} in ${context}"
  if [[ "$pool_binding" == probe-group ]]; then
    [[ -n ${selector_matcher_operator[$group_label]+x} ]] ||
      fail "exact matcher contract requires ${group_label} for ${metric} in ${context}"
    contract_key="${metric}|${selector_matcher_operator[$group_label]}|${selector_matcher_value[$group_label]}"
  fi

  expected_signature=${new_rule_matcher_contract[$contract_key]:-}
  [[ -n "$expected_signature" ]] ||
    fail "exact matcher contract is not mapped for ${contract_key} in ${context}"
  actual_signature="$({
    for label in "${selector_matcher_labels[@]}"; do
      printf '%s\t%s\t%s\n' \
        "$label" \
        "${selector_matcher_operator[$label]}" \
        "${selector_matcher_value[$label]}"
    done
  } | LC_ALL=C sort)"
  [[ "$actual_signature" == "$expected_signature" ]] ||
    fail "exact matcher contract mismatch for ${metric} in ${context}"
}

assert_mapped_raw_metric() {
  local metric=$1
  local has_selector=$2
  local target_inventory=$3
  local substrate_inventory=$4
  local context=$5
  local phase pool_binding inventory pool group_label operator value group probe_key
  local -a pools groups
  local -A seen_pools=()

  phase=${new_rule_metric_phase[$metric]:-}
  pool_binding=${new_rule_metric_pool[$metric]:-}
  [[ -n "$phase" && -n "$pool_binding" ]] ||
    fail "unmapped raw metric: ${metric} in ${context}"
  case "$phase" in
    target-initial) inventory=$target_inventory ;;
    post-substrate) inventory=$substrate_inventory ;;
    *) fail "unmapped inventory phase ${phase} for raw metric ${metric}" ;;
  esac

  pools=()
  if [[ "$pool_binding" == probe-group ]]; then
    (( has_selector == 1 )) ||
      fail "unmapped probe scrape pool without an explicit probe-group matcher: ${metric} in ${context}"
    group_label='observability.hyeonworks.com/probe-group'
    [[ -n ${selector_matcher_operator[$group_label]+x} ]] ||
      fail "unmapped probe scrape pool without ${group_label}: ${metric} in ${context}"
    operator=${selector_matcher_operator[$group_label]}
    value=${selector_matcher_value[$group_label]}
    case "$operator" in
      '=') groups=("$value") ;;
      '=~') IFS='|' read -r -a groups <<<"$value" ;;
      *) fail "unmapped probe-group matcher ${group_label}${operator} in ${context}" ;;
    esac
    for group in "${groups[@]}"; do
      probe_key="${metric}|${group}"
      pool=${new_rule_probe_pool[$probe_key]:-}
      [[ -n "$pool" ]] ||
        fail "unmapped probe scrape pool for ${metric} ${group_label}=${group} in ${context}"
      if [[ -z ${seen_pools[$pool]+x} ]]; then
        pools+=("$pool")
        seen_pools["$pool"]=1
      fi
    done
  else
    pools=("$pool_binding")
  fi

  for pool in "${pools[@]}"; do
    assert_metric "$inventory" "$pool" "$metric"
    if (( has_selector == 1 )); then
      for label in "${selector_matcher_labels[@]}"; do
        assert_metric_label "$inventory" "$pool" "$metric" "$label"
      done
    fi
  done
  assert_exact_matcher_contract "$metric" "$has_selector" "$pool_binding" "$context"
}

validate_raw_selector_provenance() {
  local selector=$1
  local target_inventory=$2
  local substrate_inventory=$3
  local context=$4
  local metric

  metric=$(sed -E 's/[[:space:]]*\{.*$//' <<<"$selector")
  [[ "$metric" =~ ^[A-Za-z_:][A-Za-z0-9_:]*$ ]] ||
    fail "unparseable raw metric selector in ${context}: ${selector}"
  parse_explicit_matchers "$selector" "$context"
  assert_mapped_raw_metric "$metric" 1 "$target_inventory" "$substrate_inventory" "$context"
}

is_promql_syntax_token() {
  case "$1" in
    and|or|unless|bool|by|without|on|ignoring|group_left|group_right|offset|start|end|\
    min|max|sum|avg|group|stddev|stdvar|topk|bottomk|count|count_values|quantile|limitk|limit_ratio|\
    s|m|h|d|w|y|e|NaN|Inf)
      return 0
      ;;
  esac
  return 1
}

is_promql_function_call() {
  local token=$1
  local expression=$2

  grep -Eq "(^|[^[:alnum:]_:])${token}[[:space:]]*\\(" <<<"$expression"
}

load_recording_rule_expressions() {
  local rule_json record encoded_expression expression

  recording_rule_expression=()
  recording_rule_state=()
  for rule_json in "$@"; do
    while IFS=$'\t' read -r record encoded_expression; do
      expression=$(printf '%s' "$encoded_expression" | base64 -d)
      [[ -z ${recording_rule_expression[$record]+x} ]] ||
        fail "duplicate recording rule name ${record} in ${rule_json}"
      recording_rule_expression["$record"]=$expression
    done < <(
      jq -r '
        .spec.groups[].rules[]
        | select(has("record"))
        | [.record, (.expr | @base64)]
        | @tsv
      ' "$rule_json"
    )
  done
}

validate_recording_rule_provenance() {
  local record=$1
  local target_inventory=$2
  local substrate_inventory=$3
  local context=$4
  local state

  [[ -n ${recording_rule_expression[$record]+x} ]] ||
    fail "unresolved recording rule reference: ${record} in ${context}"
  state=${recording_rule_state[$record]:-new}
  case "$state" in
    done) return 0 ;;
    visiting) fail "recording rule dependency cycle: ${record} in ${context}" ;;
  esac
  recording_rule_state["$record"]=visiting
  validate_expression_provenance \
    "${recording_rule_expression[$record]}" \
    "$target_inventory" "$substrate_inventory" \
    "${context} -> recording rule ${record}"
  recording_rule_state["$record"]=done
}

validate_expression_provenance() {
  local expression=$1
  local target_inventory=$2
  local substrate_inventory=$3
  local context=$4
  local flattened selector without_selectors bare_expression token

  flattened=$(tr '\n' ' ' <<<"$expression")
  while IFS= read -r selector; do
    [[ -n "$selector" ]] || continue
    validate_raw_selector_provenance \
      "$selector" "$target_inventory" "$substrate_inventory" "$context"
  done < <(
    grep -oE '[A-Za-z_:][A-Za-z0-9_:]*[[:space:]]*\{[^}]*\}' <<<"$flattened" || true
  )

  without_selectors=$(sed -E 's/[A-Za-z_:][A-Za-z0-9_:]*[[:space:]]*\{[^}]*\}//g' <<<"$flattened")
  bare_expression=$(sed -E \
    -e 's/(by|without|on|ignoring|group_left|group_right)[[:space:]]*\([^)]*\)//g' \
    -e 's/"[^"]*"//g' <<<"$without_selectors")
  while IFS= read -r token; do
    [[ -n "$token" ]] || continue
    if is_promql_syntax_token "$token" || is_promql_function_call "$token" "$bare_expression"; then
      continue
    fi
    if [[ -n ${recording_rule_expression[$token]+x} ]]; then
      validate_recording_rule_provenance \
        "$token" "$target_inventory" "$substrate_inventory" "$context"
    elif [[ "$token" == *:* ]]; then
      validate_recording_rule_provenance \
        "$token" "$target_inventory" "$substrate_inventory" "$context"
    elif [[ -n ${new_rule_metric_phase[$token]+x} ]]; then
      assert_mapped_raw_metric "$token" 0 "$target_inventory" "$substrate_inventory" "$context"
    else
      fail "unmapped raw metric or recording rule reference: ${token} in ${context}"
    fi
  done < <(
    { grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$bare_expression" || true; } | sort -u
  )
}

assert_new_rule_provenance() {
  local target_inventory=$1
  local substrate_inventory=$2
  local rule_json rule_name encoded_expression expression

  shift 2
  load_recording_rule_expressions "$@"
  for rule_json in "$@"; do
    while IFS=$'\t' read -r rule_name encoded_expression; do
      expression=$(printf '%s' "$encoded_expression" | base64 -d)
      validate_expression_provenance \
        "$expression" "$target_inventory" "$substrate_inventory" \
        "${rule_json} rule ${rule_name}"
    done < <(
      jq -r '
        .spec.groups[].rules[]
        | [(.record // .alert), (.expr | @base64)]
        | @tsv
      ' "$rule_json"
    )
  done
}

assert_provenance_rejects() {
  local expected_reason=$1
  shift
  local output

  if output="$( (assert_new_rule_provenance "$@") 2>&1 )"; then
    fail "provenance regression unexpectedly accepted a typo: ${expected_reason}"
  fi
  grep -Fq "$expected_reason" <<<"$output" ||
    fail "provenance regression did not reject the typo for ${expected_reason}: ${output}"
}

assert_new_rule_provenance_regressions() {
  local target_inventory=$1
  local substrate_inventory=$2
  local quota_json=$3
  local certificate_json=$4
  local services_json=$5
  local metric_typo_json="${temp_dir}/provenance-metric-typo.rules.json"
  local matcher_typo_json="${temp_dir}/provenance-matcher-typo.rules.json"
  local quota_matcher_removed_json="${temp_dir}/provenance-quota-matcher-removed.rules.json"
  local quota_matcher_value_typo_json="${temp_dir}/provenance-quota-matcher-value-typo.rules.json"
  local cnpg_matcher_removed_json="${temp_dir}/provenance-cnpg-matcher-removed.rules.json"
  local cnpg_matcher_value_typo_json="${temp_dir}/provenance-cnpg-matcher-value-typo.rules.json"
  local record_typo_json="${temp_dir}/provenance-record-typo.rules.json"

  jq '
    (.spec.groups[].rules[]
      | select(.record == "platform:aistor_bucket_quota_usage_percent")
      | .expr)
    |= sub(
      "minio_cluster_usage_buckets_total_bytes";
      "minio_cluster_usage_buckets_total_typo_bytes"
    )
  ' "$quota_json" >"$metric_typo_json"
  assert_provenance_rejects \
    'unmapped raw metric' \
    "$target_inventory" "$substrate_inventory" \
    "$metric_typo_json" "$certificate_json" "$services_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformCNPGCollectorDown")
      | .expr)
    |= sub("cluster=\"platform-postgres\""; "clustr=\"platform-postgres\"")
  ' "$services_json" >"$matcher_typo_json"
  assert_provenance_rejects \
    'inventory lacks' \
    "$target_inventory" "$substrate_inventory" \
    "$quota_json" "$certificate_json" "$matcher_typo_json"

  jq '
    (.spec.groups[].rules[]
      | select(.record == "platform:aistor_bucket_quota_usage_percent")
      | .expr)
    |= gsub("\\{bucket=~\\\"loki\\|tempo\\\"\\}"; "")
  ' "$quota_json" >"$quota_matcher_removed_json"
  assert_provenance_rejects \
    'exact matcher contract' \
    "$target_inventory" "$substrate_inventory" \
    "$quota_matcher_removed_json" "$certificate_json" "$services_json"

  jq '
    (.spec.groups[].rules[]
      | select(.record == "platform:aistor_bucket_quota_usage_percent")
      | .expr)
    |= sub("loki\\|tempo"; "loki|temop")
  ' "$quota_json" >"$quota_matcher_value_typo_json"
  assert_provenance_rejects \
    'exact matcher contract' \
    "$target_inventory" "$substrate_inventory" \
    "$quota_matcher_value_typo_json" "$certificate_json" "$services_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformCNPGCollectorDown")
      | .expr)
    |= sub("namespace=\\\"platform-data\\\","; "")
  ' "$services_json" >"$cnpg_matcher_removed_json"
  assert_provenance_rejects \
    'exact matcher contract' \
    "$target_inventory" "$substrate_inventory" \
    "$quota_json" "$certificate_json" "$cnpg_matcher_removed_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformCNPGCollectorDown")
      | .expr)
    |= sub("platform-data"; "platform-dtaa")
  ' "$services_json" >"$cnpg_matcher_value_typo_json"
  assert_provenance_rejects \
    'exact matcher contract' \
    "$target_inventory" "$substrate_inventory" \
    "$quota_json" "$certificate_json" "$cnpg_matcher_value_typo_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformAIStorBucketQuotaUsage")
      | .expr)
    |= sub(
      "platform:aistor_bucket_quota_usage_percent";
      "platform:aistor_bucket_quota_usage_typo_percent"
    )
  ' "$quota_json" >"$record_typo_json"
  assert_provenance_rejects \
    'unresolved recording rule reference' \
    "$target_inventory" "$substrate_inventory" \
    "$record_typo_json" "$certificate_json" "$services_json"

  pass 'provenance rejects real new-rule metric, exact matcher, and recording-reference mutations'
}

assert_new_rule_semantics() {
  [[ $# -eq 3 ]] || fail 'new-rule semantic projection requires quota, certificate, and service rule JSON'
  local actual expected canonical_expected

  expected='[
    ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 0, "record", "platform:aistor_bucket_quota_usage_percent", "100 * max by (bucket) ( minio_cluster_usage_buckets_total_bytes{bucket=~\"loki|tempo\"} ) / max by (bucket) ( minio_cluster_usage_buckets_quota_total_bytes{bucket=~\"loki|tempo\"} > 0 )", null, null],
    ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 1, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 70 and platform:aistor_bucket_quota_usage_percent < 85", "15m", "warning"],
    ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 2, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 85 and platform:aistor_bucket_quota_usage_percent < 95", "10m", "critical"],
    ["platform-aistor-storage-quota", "platform.aistor-storage-quota", 3, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 95", "5m", "emergency"],
    ["platform-certificate-probes", "platform.blackbox", 0, "alert", "PlatformPublicEdgeProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"public-edge\" } == 0", "5m", "critical"],
    ["platform-certificate-probes", "platform.blackbox", 1, "alert", "PlatformPrivateEdgeProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"private-edge\" } == 0", "5m", "critical"],
    ["platform-certificate-probes", "platform.blackbox", 2, "alert", "PlatformInternalHealthProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"private-internal\" } == 0", "5m", "critical"],
    ["platform-certificate-probes", "platform.certificates", 0, "alert", "PlatformCertificateExpiry", "( probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() ) >= 14 * 24 * 60 * 60 and ( probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() ) < 30 * 24 * 60 * 60", null, "warning"],
    ["platform-certificate-probes", "platform.certificates", 1, "alert", "PlatformCertificateExpiry", "probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() < 14 * 24 * 60 * 60", null, "critical"],
    ["platform-verified-services", "platform.verified-services", 0, "alert", "PlatformCNPGCollectorDown", "min by (cluster, namespace) ( cnpg_collector_up{ namespace=\"platform-data\", cluster=\"platform-postgres\" } ) == 0", "5m", "critical"],
    ["platform-verified-services", "platform.verified-services", 1, "alert", "PlatformAlloyConfigLoadFailed", "min by (namespace) ( alloy_config_last_load_successful{namespace=\"observability-agent\"} ) == 0", "5m", "critical"],
    ["platform-verified-services", "platform.verified-services", 2, "alert", "PlatformAlloyLogDeliveryRetries", "sum by (namespace) ( increase(loki_write_batch_retries_total{namespace=\"observability-agent\"}[15m]) ) > 0", "5m", "warning"],
    ["platform-verified-services", "platform.verified-services", 3, "alert", "PlatformAlloyLogEntriesDropped", "sum by (namespace) ( increase(loki_write_dropped_entries_total{namespace=\"observability-agent\"}[15m]) ) > 0", "5m", "critical"],
    ["platform-verified-services", "platform.verified-services", 4, "alert", "PlatformLokiRuntimeConfigReloadFailed", "min by (namespace) ( loki_runtime_config_last_reload_successful{namespace=\"observability\"} ) == 0", "5m", "critical"],
    ["platform-verified-services", "platform.verified-services", 5, "alert", "PlatformLokiWALDiskUsageHigh", "max by (namespace) ( loki_ingester_wal_disk_usage_percent{namespace=\"observability\"} ) >= 0.8", "15m", "warning"]
  ]'

  actual="$(jq -c -s '
    def normalized_expr:
      gsub("[[:space:]]+"; " ")
      | sub("^ "; "")
      | sub(" $"; "");
    [.[] as $resource
      | $resource.spec.groups[] as $group
      | $group.rules
      | to_entries[]
      | [
          $resource.metadata.name,
          $group.name,
          .key,
          (if .value.record then "record" else "alert" end),
          (.value.record // .value.alert),
          (.value.expr | normalized_expr),
          (.value.for // null),
          (.value.labels.severity // null)
        ]
    ]
  ' "$@")" || fail 'could not project new-rule semantics'
  canonical_expected="$(jq -c . <<<"$expected")" ||
    fail 'invalid expected new-rule semantic projection'

  [[ "$actual" == "$canonical_expected" ]] ||
    fail 'new-rule semantic projection mismatch'
}

assert_semantics_rejects() {
  local mutation=$1
  shift
  local output

  declare -F assert_new_rule_semantics >/dev/null ||
    fail 'new-rule semantic validator is unavailable'
  if output="$( (assert_new_rule_semantics "$@") 2>&1 )"; then
    fail "semantic regression unexpectedly accepted ${mutation}"
  fi
  grep -Fq 'new-rule semantic projection mismatch' <<<"$output" ||
    fail "semantic regression rejected ${mutation} for the wrong reason: ${output}"
}

assert_new_rule_semantic_regressions() {
  local quota_json=$1
  local certificate_json=$2
  local services_json=$3
  local public_group_drift_json="${temp_dir}/semantic-public-group-drift.rules.json"
  local selector_removed_json="${temp_dir}/semantic-selector-removed.rules.json"
  local duplicate_selector_json="${temp_dir}/semantic-duplicate-selector.rules.json"
  local cnpg_rebound_json="${temp_dir}/semantic-cnpg-rebound.rules.json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformPublicEdgeProbeFailed")
      | .expr)
    |= sub("public-edge"; "private-edge")
  ' "$certificate_json" >"$public_group_drift_json"
  assert_semantics_rejects \
    'PlatformPublicEdgeProbeFailed rebound to the valid private-edge fixture' \
    "$quota_json" "$public_group_drift_json" "$services_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformPublicEdgeProbeFailed")
      | .expr) = "vector(0)"
  ' "$certificate_json" >"$selector_removed_json"
  assert_semantics_rejects \
    'PlatformPublicEdgeProbeFailed with its selector removed' \
    "$quota_json" "$selector_removed_json" "$services_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformPublicEdgeProbeFailed")
      | .expr)
    += " or probe_success{namespace=\"observability\",\"observability.hyeonworks.com/probe-group\"=\"private-edge\"} == 0"
  ' "$certificate_json" >"$duplicate_selector_json"
  assert_semantics_rejects \
    'PlatformPublicEdgeProbeFailed with an extra valid probe selector' \
    "$quota_json" "$duplicate_selector_json" "$services_json"

  jq '
    (.spec.groups[].rules[]
      | select(.alert == "PlatformCNPGCollectorDown")
      | .expr) = "min by (namespace) (alloy_config_last_load_successful{namespace=\"observability-agent\"}) == 0"
  ' "$services_json" >"$cnpg_rebound_json"
  assert_semantics_rejects \
    'PlatformCNPGCollectorDown rebound to the valid Alloy fixture' \
    "$quota_json" "$certificate_json" "$cnpg_rebound_json"

  pass 'semantic projection rejects rule-identity, selector-cardinality, and fixture-routing mutations'
}

render_rule_json() {
  local source_file=$1
  local output_file=$2

  [[ -f "$source_file" ]] || fail "missing rule source: ${source_file}"
  kubectl create --dry-run=client --validate=false -f "$source_file" -o json >"$output_file"
  jq -e '
    .apiVersion == "monitoring.coreos.com/v1" and
    .kind == "PrometheusRule" and
    .metadata.namespace == "observability" and
    .metadata.labels["observability.hyeonworks.com/instance"] == "home" and
    (.spec.groups | type == "array" and length > 0)
  ' "$output_file" >/dev/null || fail "invalid PrometheusRule envelope: ${source_file}"
}

alert_runbooks_are_exact() {
  local rule_json=$1

  jq -e --arg expected "$EXPECTED_RUNBOOK_URL" '
    all(.spec.groups[].rules[] | select(has("alert"));
      .annotations.runbook_url == $expected
    )
  ' "$rule_json" >/dev/null
}

assert_exact_runbook_regression() {
  local rule_json=$1 mutated="${temp_dir}/runbook-url-mutated.rules.json"

  alert_runbooks_are_exact "$rule_json" ||
    fail 'the production rule fixture lacks the exact runbook URL'
  jq '
    (.spec.groups[].rules[] | select(has("alert")) | .annotations.runbook_url) =
      "https://git.learn.hyeonworks.com/wrong/repository/runbook.md"
  ' "$rule_json" >"$mutated"
  if alert_runbooks_are_exact "$mutated"; then
    fail 'a different valid HTTPS runbook URL was accepted'
  fi
  pass 'runbook annotation rejects a different valid HTTPS URL'
}

assert_alert_contract() {
  local rule_json=$1

  jq -e '
    all(.spec.groups[].rules[] | select(has("alert"));
      (.labels | keys | sort) == ["severity"] and
      (.labels.severity | IN("warning", "critical", "emergency")) and
      (.annotations.summary | type == "string" and length > 0) and
      (.annotations.description | type == "string" and length > 0)
    )
  ' "$rule_json" >/dev/null || fail "alert label/summary/description contract failed: ${rule_json}"

  if ! alert_runbooks_are_exact "$rule_json"; then
    printf 'BLOCKED: alert lacks the exact operator-reachable HTTPS runbook_url: %s\n' "$rule_json" >&2
    runbook_url_blocked=1
  fi
}

assert_exact_alert_names() {
  local rule_json=$1
  shift
  local expected actual

  expected="$(printf '%s\n' "$@" | sort)"
  actual="$(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$rule_json" | sort)"
  [[ "$actual" == "$expected" ]] || {
    printf 'Expected alert names:\n%s\nActual alert names:\n%s\n' "$expected" "$actual" >&2
    fail "unexpected alert set: ${rule_json}"
  }
}

run_promtool() {
  local work_dir=$1
  shift

  docker run --rm \
    --network none \
    --volume "${work_dir}:/work:ro" \
    --entrypoint /bin/promtool \
    "$PROMETHEUS_IMAGE" "$@"
}

run_live_promql() {
  local -a rule_jsons=("$@")
  local proxy_path='/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1'
  local now start rule_name encoded_expr expression encoded_query response query_status result_count

  now="$(date +%s)"
  start="$((now - 300))"

  while IFS=$'\t' read -r rule_name encoded_expr; do
    expression="$(printf '%s' "$encoded_expr" | base64 -d)"
    encoded_query="$(jq -rn --arg query "$expression" '$query | @uri')"

    response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${encoded_query}")" ||
      fail "live instant query transport failed: ${rule_name}"
    query_status="$(jq -r '.status' <<<"$response")"
    [[ "$query_status" == success ]] ||
      fail "live instant query rejected: ${rule_name}"
    result_count="$(jq -r '.data.result | length' <<<"$response")"
    printf 'LIVE INSTANT PASS: %s (%s series)\n' "$rule_name" "$result_count"

    response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query_range?query=${encoded_query}&start=${start}&end=${now}&step=60")" ||
      fail "live range query transport failed: ${rule_name}"
    query_status="$(jq -r '.status' <<<"$response")"
    [[ "$query_status" == success ]] ||
      fail "live range query rejected: ${rule_name}"
    result_count="$(jq -r '.data.result | length' <<<"$response")"
    printf 'LIVE RANGE PASS: %s (%s series)\n' "$rule_name" "$result_count"
  done < <(
    jq -r '
      .spec.groups[].rules[] |
      [(.alert // .record), (.expr | @base64)] |
      @tsv
    ' "${rule_jsons[@]}"
  )
}

assert_live_private_403() {
  local proxy_path='/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1'
  local success_query status_query success_response status_response
  local success_set status_set

  success_query="$(jq -rn --arg query 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"}' '$query | @uri')"
  status_query="$(jq -rn --arg query 'probe_http_status_code{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"}' '$query | @uri')"
  success_response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${success_query}")" ||
    fail 'live private-edge probe_success transport failed'
  status_response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${status_query}")" ||
    fail 'live private-edge status transport failed'

  success_set="$(jq -cS '[.data.result[] | select(.value[1] == "1") | .metric.instance] | sort' <<<"$success_response")"
  status_set="$(jq -cS '[.data.result[] | select(.value[1] == "403") | .metric.instance] | sort' <<<"$status_response")"
  [[ "$success_set" == "$status_set" ]] ||
    fail 'live private-edge success and HTTP 403 instance sets differ'
  [[ "$(jq 'length' <<<"$success_set")" -eq 3 ]] ||
    fail 'live private-edge contract must have exactly three successful HTTP 403 endpoints'
  pass 'live private-edge three endpoints are probe_success=1 with HTTP 403'
}

main() {
  [[ $# -eq 1 || ( $# -eq 2 && "$2" == --live-prometheus ) ]] ||
    fail "usage: $0 VERIFIED_OUTPUT_DIR [--live-prometheus]"
  local inventory_root=$1
  local live_prometheus=0
  local target_inventory="${inventory_root}/target-initial/inventory.json"
  local substrate_inventory="${inventory_root}/post-substrate/inventory.json"
  local quota_json certificate_json services_json core_json
  local rendered_names duplicate_names

  [[ ${2:-} == --live-prometheus ]] && live_prometheus=1

  require_command docker
  require_command base64
  require_command jq
  require_command kubectl
  require_command sha256sum

  assert_inventory "${inventory_root}/target-initial" target-initial
  assert_inventory "${inventory_root}/post-substrate" post-substrate

  assert_metric_label "$target_inventory" serviceMonitor/object-storage/aistor-bucket-usage/0 minio_cluster_usage_buckets_total_bytes bucket
  assert_metric_label "$target_inventory" serviceMonitor/object-storage/aistor-bucket-usage/0 minio_cluster_usage_buckets_quota_total_bytes bucket
  assert_metric_label "$target_inventory" podMonitor/platform-data/platform-postgres/0 cnpg_collector_up cluster
  assert_metric_label "$target_inventory" podMonitor/platform-data/platform-postgres/0 cnpg_collector_up namespace
  assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 alloy_config_last_load_successful namespace
  assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 loki_write_batch_retries_total namespace
  assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 loki_write_dropped_entries_total namespace
  assert_metric_label "$target_inventory" serviceMonitor/observability/loki/0 loki_runtime_config_last_reload_successful namespace
  assert_metric_label "$target_inventory" serviceMonitor/observability/loki/0 loki_ingester_wal_disk_usage_percent namespace
  assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success namespace
  assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success observability.hyeonworks.com/probe-group
  assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success instance
  assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry namespace
  assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry observability.hyeonworks.com/probe-group
  assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry instance
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success namespace
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success observability.hyeonworks.com/probe-group
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success instance
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_http_status_code observability.hyeonworks.com/probe-group
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry namespace
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry observability.hyeonworks.com/probe-group
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry instance
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success namespace
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success observability.hyeonworks.com/probe-group
  assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success instance
  pass "inventory hashes, phases, metrics, and labels"

  temp_dir="$(mktemp -d)"
  chmod 0755 "$temp_dir"
  trap cleanup EXIT
  quota_json="${temp_dir}/storage-quota.rules.json"
  certificate_json="${temp_dir}/certificate.rules.json"
  services_json="${temp_dir}/verified-service.rules.json"
  core_json="${temp_dir}/core.rules.json"

  render_rule_json "${RULE_DIR}/storage-quota-rules.yaml" "$quota_json"
  render_rule_json "${RULE_DIR}/certificate-rules.yaml" "$certificate_json"
  render_rule_json "${RULE_DIR}/verified-service-rules.yaml" "$services_json"
  render_rule_json "${RULE_DIR}/core-rules.yaml" "$core_json"
  assert_new_rule_provenance_regressions \
    "$target_inventory" "$substrate_inventory" \
    "$quota_json" "$certificate_json" "$services_json"
  assert_new_rule_semantic_regressions \
    "$quota_json" "$certificate_json" "$services_json"
  assert_exact_runbook_regression "$quota_json"
  assert_new_rule_provenance \
    "$target_inventory" "$substrate_inventory" \
    "$quota_json" "$certificate_json" "$services_json"
  pass 'new-rule raw metrics, matcher labels, and recording dependencies have inventory provenance'
  assert_new_rule_semantics \
    "$quota_json" "$certificate_json" "$services_json"
  pass 'new-rule identities, groups, expressions, durations, and severities have exact semantic binding'
  assert_alert_contract "$quota_json"
  assert_alert_contract "$certificate_json"
  assert_alert_contract "$services_json"
  if ! alert_runbooks_are_exact "$core_json"; then
    printf 'BLOCKED: core alert lacks the exact operator-reachable HTTPS runbook_url: %s\n' "$core_json" >&2
    runbook_url_blocked=1
  fi

  jq -e '
    ([.spec.groups[].rules[] | select(.record == "platform:aistor_bucket_quota_usage_percent")] | length) == 1 and
    ([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage")] | length) == 3 and
    ([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage") | .for] | sort) == ["10m", "15m", "5m"] and
    ([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage") | .labels.severity] | sort) == ["critical", "emergency", "warning"]
  ' "$quota_json" >/dev/null || fail "AIStor quota rule shape is not exact"

  assert_exact_alert_names "$certificate_json" \
    PlatformCertificateExpiry \
    PlatformCertificateExpiry \
    PlatformInternalHealthProbeFailed \
    PlatformPrivateEdgeProbeFailed \
    PlatformPublicEdgeProbeFailed

  assert_exact_alert_names "$services_json" \
    PlatformAlloyConfigLoadFailed \
    PlatformAlloyLogDeliveryRetries \
    PlatformAlloyLogEntriesDropped \
    PlatformCNPGCollectorDown \
    PlatformLokiRuntimeConfigReloadFailed \
    PlatformLokiWALDiskUsageHigh

  duplicate_names="$(
    comm -12 \
      <(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$core_json" | sort -u) \
      <(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$quota_json" "$certificate_json" "$services_json" | sort -u)
  )"
  [[ -z "$duplicate_names" ]] || fail "new rules duplicate core alert names: ${duplicate_names}"
  pass "rule envelopes, bounded labels, annotations, and non-duplicate alert sets"

  rendered_names="$(
    kubectl kustomize "$RULE_DIR" |
      kubectl create --dry-run=client --validate=false -f - -o name |
      sort
  )"
  [[ "$rendered_names" == $'prometheusrule.monitoring.coreos.com/platform-aistor-storage-quota\nprometheusrule.monitoring.coreos.com/platform-certificate-probes\nprometheusrule.monitoring.coreos.com/platform-observability-core\nprometheusrule.monitoring.coreos.com/platform-verified-services' ]] || {
    printf 'Rendered resources:\n%s\n' "$rendered_names" >&2
    fail "platform rule kustomization resource set is not exact"
  }
  pass "platform rule kustomization"

  jq '.spec' "$quota_json" >"${temp_dir}/storage-quota.prometheus.json"
  jq '.spec' "$certificate_json" >"${temp_dir}/certificate.prometheus.json"
  jq '.spec' "$services_json" >"${temp_dir}/verified-service.prometheus.json"
  jq '.spec' "$core_json" >"${temp_dir}/core.prometheus.json"
  chmod 0444 "${temp_dir}"/*.prometheus.json
  run_promtool "$temp_dir" check rules \
    /work/storage-quota.prometheus.json \
    /work/certificate.prometheus.json \
    /work/verified-service.prometheus.json \
    /work/core.prometheus.json
  pass "promtool check rules"

  printf '%s\n' \
    'rule_files:' \
    '  - /work/storage-quota.prometheus.json' \
    '  - /work/certificate.prometheus.json' \
    'evaluation_interval: 1m' \
    'tests:' \
    '  - name: quota warning starts at 70 and excludes 85' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \
    '        values: '\''70+0x20'\''' \
    '      - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \
    '        values: '\''100+0x20'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 16m' \
    '        alertname: PlatformAIStorBucketQuotaUsage' \
    '        exp_alerts:' \
    '          - exp_labels:' \
    '              bucket: loki' \
    '              severity: warning' \
    '            exp_annotations:' \
    '              summary: AIStor bucket quota usage is at warning level' \
    '              description: Bucket loki has remained between 70 and 85 percent used for 15 minutes.' \
    '              runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
    '  - name: quota critical starts at 85 and excludes 95' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="tempo"}'\''' \
    '        values: '\''85+0x20'\''' \
    '      - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="tempo"}'\''' \
    '        values: '\''100+0x20'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 11m' \
    '        alertname: PlatformAIStorBucketQuotaUsage' \
    '        exp_alerts:' \
    '          - exp_labels:' \
    '              bucket: tempo' \
    '              severity: critical' \
    '            exp_annotations:' \
    '              summary: AIStor bucket quota usage is at critical level' \
    '              description: Bucket tempo has remained between 85 and 95 percent used for 10 minutes.' \
    '              runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
    '  - name: quota emergency starts at 95' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \
    '        values: '\''95+0x20'\''' \
    '      - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \
    '        values: '\''100+0x20'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 6m' \
    '        alertname: PlatformAIStorBucketQuotaUsage' \
    '        exp_alerts:' \
    '          - exp_labels:' \
    '              bucket: loki' \
    '              severity: emergency' \
    '            exp_annotations:' \
    '              summary: AIStor bucket quota usage is at emergency level' \
    '              description: Stop the loki ingest path according to the runbook; do not delete objects automatically.' \
    '              runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
    '  - name: zero quota cannot produce a percentage alert' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \
    '        values: '\''100+0x20'\''' \
    '      - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \
    '        values: '\''0+0x20'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 20m' \
    '        alertname: PlatformAIStorBucketQuotaUsage' \
    '        exp_alerts: []' \
    '  - name: successful expected 403 private probe remains healthy' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''probe_success{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \
    '        values: '\''1+0x10'\''' \
    '      - series: '\''probe_http_status_code{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \
    '        values: '\''403+0x10'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 10m' \
    '        alertname: PlatformPrivateEdgeProbeFailed' \
    '        exp_alerts: []' \
    '  - name: failed private probe fires after five minutes' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''probe_success{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \
    '        values: '\''0+0x10'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 6m' \
    '        alertname: PlatformPrivateEdgeProbeFailed' \
    '        exp_alerts:' \
    '          - exp_labels:' \
    '              namespace: observability' \
    '              job: blackbox-private-edge' \
    '              instance: https://db-admin.learn.hyeonworks.com/' \
    '              "observability.hyeonworks.com/probe-group": private-edge' \
    '              severity: critical' \
    '            exp_annotations:' \
    '              summary: Private edge boundary probe failed' \
    '              description: The expected private edge response for https://db-admin.learn.hyeonworks.com/ has failed for 5 minutes; an expected HTTP 403 with probe_success=1 is healthy.' \
    '              runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
    '  - name: TLS warning includes exactly fourteen days' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \
    '        values: '\''1209660+0x2'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 1m' \
    '        alertname: PlatformCertificateExpiry' \
    '        exp_alerts:' \
    '          - exp_labels:' \
    '              namespace: observability' \
    '              instance: https://git.learn.hyeonworks.com/api/healthz' \
    '              "observability.hyeonworks.com/probe-group": public-edge' \
    '              severity: warning' \
    '            exp_annotations:' \
    '              summary: TLS certificate expires within 30 days' \
    '              description: The earliest certificate for https://git.learn.hyeonworks.com/api/healthz expires in fewer than 30 days but not fewer than 14 days.' \
    '              runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
    '  - name: TLS warning excludes exactly thirty days' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \
    '        values: '\''2592060+0x2'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 1m' \
    '        alertname: PlatformCertificateExpiry' \
    '        exp_alerts: []' \
    '  - name: TLS critical covers fewer than fourteen days' \
    '    interval: 1m' \
    '    input_series:' \
    '      - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \
    '        values: '\''864060+0x2'\''' \
    '    alert_rule_test:' \
    '      - eval_time: 1m' \
    '        alertname: PlatformCertificateExpiry' \
    '        exp_alerts:' \
    '          - exp_labels:' \
    '              namespace: observability' \
    '              instance: https://git.learn.hyeonworks.com/api/healthz' \
    '              "observability.hyeonworks.com/probe-group": public-edge' \
    '              severity: critical' \
    '            exp_annotations:' \
    '              summary: TLS certificate expires within 14 days' \
    '              description: The earliest certificate for https://git.learn.hyeonworks.com/api/healthz expires in fewer than 14 days.' \
    '              runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
    >"${temp_dir}/rules.test.yaml"
  chmod 0444 "${temp_dir}/rules.test.yaml"

  run_promtool "$temp_dir" test rules /work/rules.test.yaml
  pass "promtool quota, probe, private-403, and TLS boundaries"

  if (( live_prometheus == 1 )); then
    run_live_promql "$quota_json" "$certificate_json" "$services_json" "$core_json"
    assert_live_private_403
    pass "live Prometheus instant and range queries"
  fi

  (( runbook_url_blocked == 0 )) ||
    fail "operator-reachable HTTPS runbook_url contract is not defined"
}

main "$@"