#!/usr/bin/env bash
set -euo pipefail

ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
DASHBOARD_ROOT="$ROOT/services/observability/dashboards/platform"
METRIC_ROOT=${PLATFORM_OBSERVABILITY_METRIC_ROOT:-/tmp/platform-observability-metrics.VUpsZn}
TARGET_INVENTORY="$METRIC_ROOT/target-initial/inventory.json"
POST_INVENTORY="$METRIC_ROOT/post-substrate/inventory.json"
PROMETHEUS_IMAGE='quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
WORK_DIR=''
PROMETHEUS_BASE_URL=''
PORT_FORWARD_PID=''

fail() {
  printf 'FAIL: %s\n' "$*" >&2
  exit 1
}

pass() {
  printf 'PASS: %s\n' "$*"
}

require_command() {
  command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
}

usage() {
  printf 'Usage: %s [--prometheus-base-url http://127.0.0.1:PORT]\n' "${0##*/}"
}

parse_args() {
  while [[ $# -gt 0 ]]; do
    case "$1" in
      --prometheus-base-url)
        [[ $# -ge 2 ]] || fail '--prometheus-base-url requires a value'
        [[ -z "$PROMETHEUS_BASE_URL" ]] || fail '--prometheus-base-url may be specified only once'
        PROMETHEUS_BASE_URL=$2
        shift 2
        ;;
      --help)
        usage
        exit 0
        ;;
      *)
        fail "unknown argument: $1"
        ;;
    esac
  done

  if [[ -n "$PROMETHEUS_BASE_URL" ]]; then
    [[ "$PROMETHEUS_BASE_URL" =~ ^http://(127\.0\.0\.1|localhost):([1-9][0-9]{0,4})$ ]] \
      || fail '--prometheus-base-url must be an exact loopback HTTP origin without a path'
    (( BASH_REMATCH[2] <= 65535 )) \
      || fail '--prometheus-base-url port exceeds 65535'
  fi
}

cleanup() {
  local rc=$?
  trap - EXIT
  if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
    kill "$PORT_FORWARD_PID" 2>/dev/null || true
    wait "$PORT_FORWARD_PID" 2>/dev/null || true
  fi
  if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then
    rm -rf -- "$WORK_DIR"
  fi
  exit "$rc"
}

verify_inventory() {
  local phase=$1
  local inventory="$METRIC_ROOT/$phase/inventory.json"
  local checksum="$METRIC_ROOT/$phase/inventory.sha256"
  local expected actual

  [[ -f "$inventory" ]] || fail "$phase inventory is absent"
  [[ -f "$checksum" ]] || fail "$phase inventory checksum is absent"
  jq -e --arg phase "$phase" '
    .schema == "platform-observability-metric-inventory/v1"
    and .phase == $phase
    and (.targets | type == "array" and length > 0)
    and all(.targets[]; .health == "up" and (.metrics | type == "array" and length > 0))
  ' "$inventory" >/dev/null || fail "$phase inventory contract is invalid"

  read -r expected checksum_name < "$checksum"
  [[ "$checksum_name" == 'inventory.json' ]] || fail "$phase checksum names an unexpected file"
  actual=$(sha256sum "$inventory" | awk '{print $1}')
  [[ "$actual" == "$expected" ]] || fail "$phase inventory checksum mismatch"
  pass "$phase inventory contract and checksum"
}

inventory_has_metric() {
  local inventory=$1
  local metric=$2
  jq -e --arg metric "$metric" 'any(.targets[].metrics[]; .name == $metric)' "$inventory" >/dev/null
}

inventory_metric_has_label() {
  local inventory=$1
  local metric=$2
  local label=$3
  jq -e --arg metric "$metric" --arg label "$label" '
    any(.targets[].metrics[]; .name == $metric and (.label_names | index($label) != null))
  ' "$inventory" >/dev/null
}

validate_inventory_references() {
  local dashboard=$1
  local inventory=$2
  local expression selector metric matchers matcher label token
  local inventory_metrics referenced_metrics found

  inventory_metrics=$(jq -r '[.targets[].metrics[].name] | unique[]' "$inventory")

  while IFS= read -r expression; do
    [[ -n "$expression" ]] || fail "empty PromQL expression in $dashboard"
    referenced_metrics=''

    while IFS= read -r selector; do
      [[ -n "$selector" ]] || continue
      metric=${selector%%\{*}
      grep -Fxq "$metric" <<<"$inventory_metrics" \
        || fail "$dashboard references absent metric $metric"
      if ! grep -Fxq "$metric" <<<"$referenced_metrics"; then
        referenced_metrics+="${metric}"$'\n'
      fi
      matchers=${selector#*\{}
      matchers=${matchers%\}}
      while IFS= read -r matcher; do
        matcher=${matcher#"${matcher%%[![:space:]]*}"}
        [[ -n "$matcher" ]] || continue
        label=$(sed -E 's/^([A-Za-z_][A-Za-z0-9_]*).*/\1/' <<<"$matcher")
        [[ "$label" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \
          || fail "$dashboard has an unauditable label matcher in $expression"
        inventory_metric_has_label "$inventory" "$metric" "$label" \
          || fail "$dashboard references absent label $metric.$label"
      done < <(tr ',' '\n' <<<"$matchers")
    done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]*\}' <<<"$expression" || true)

    [[ -n "$referenced_metrics" ]] || fail "$dashboard expression has no inventory metric: $expression"

    while IFS= read -r token; do
      [[ -n "$token" ]] || continue
      if grep -Fxq "$token" <<<"$inventory_metrics"; then
        grep -Fxq "$token" <<<"$referenced_metrics" \
          || fail "$dashboard uses inventory metric $token without an auditable selector"
      fi
    done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$expression" | sort -u)

    while IFS= read -r label; do
      [[ -n "$label" ]] || continue
      found=0
      while IFS= read -r metric; do
        [[ -n "$metric" ]] || continue
        if inventory_metric_has_label "$inventory" "$metric" "$label"; then
          found=1
          break
        fi
      done <<<"$referenced_metrics"
      [[ "$found" -eq 1 ]] || fail "$dashboard groups by absent inventory label $label"
    done < <(
      grep -oE '(by|without)[[:space:]]*\([^)]*\)' <<<"$expression" \
        | sed -E 's/^[^(]*\(([^)]*)\)$/\1/' \
        | tr ',' '\n' \
        | sed -E 's/^[[:space:]]+|[[:space:]]+$//g' \
        || true
    )
  done < <(jq -r '.panels[].targets[].expr' "$dashboard")
}

validate_dashboard() {
  local file=$1
  local expected_uid=$2
  local inventory=$3
  local path="$DASHBOARD_ROOT/$file"

  [[ -f "$path" ]] || fail "dashboard source is absent: $file"
  jq -e --arg uid "$expected_uid" '
    type == "object"
    and .uid == $uid
    and (.title | type == "string" and length > 0)
    and .schemaVersion == 42
    and .editable == false
    and .refresh == "30s"
    and (.templating.list == [])
    and (.panels | type == "array" and length > 0)
    and ([.panels[].id] | length == (unique | length))
    and all(.panels[];
      .type != "row"
      and (.title | type == "string" and length > 0)
      and (.gridPos.w > 0 and .gridPos.h > 0)
      and .datasource.type == "prometheus"
      and .datasource.uid == "prometheus"
      and (.targets | type == "array" and length > 0)
      and all(.targets[];
        .datasource.type == "prometheus"
        and .datasource.uid == "prometheus"
        and (.expr | type == "string" and length > 0)
        and (.refId | type == "string" and length > 0)
      )
    )
  ' "$path" >/dev/null || fail "dashboard structure is invalid: $file"

  if jq -r '.title, .tags[], .panels[].title' "$path" \
      | grep -Eiq '(^|[^[:alnum:]_])(spring|jvm|kafka|batch|backup)([^[:alnum:]_]|$)'; then
    fail "dashboard contains a forbidden product area: $file"
  fi
  if jq -r '.panels[].targets[].expr' "$path" \
      | grep -Eiq '(^|[^[:alnum:]_])(spring_|jvm_|kafka_|batch_|backup_)'; then
    fail "dashboard contains a forbidden product query: $file"
  fi
  if jq -r '.. | strings' "$path" \
      | grep -Eiq '(request[_ -]?id|trace[_ -]?id|username|raw[_ -]?url)'; then
    fail "dashboard contains forbidden or high-cardinality content: $file"
  fi

  validate_inventory_references "$path" "$inventory"
  pass "$file structure and inventory references"
}

validate_rendered_configmaps() {
  local rendered=$1
  local file name block
  local -a files=(
    kubernetes-node
    workload-health
    platform-services
    observability-backends
    https-endpoints
  )

  kubectl kustomize "$DASHBOARD_ROOT" > "$rendered"
  [[ $(grep -c '^kind: ConfigMap$' "$rendered") -eq 5 ]] \
    || fail 'kustomization must render exactly five ConfigMaps'

  for file in "${files[@]}"; do
    name="grafana-dashboard-$file"
    block=$(awk -v name="$name" '
      BEGIN { RS="---" }
      $0 ~ "name: " name "([[:space:]]|$)" { print }
    ' "$rendered")
    [[ -n "$block" ]] || fail "rendered ConfigMap is absent: $name"
    grep -Eq 'grafana_dashboard: ("1"|1)$' <<<"$block" \
      || fail "$name lacks grafana_dashboard=1"
    grep -q 'observability.hyeonworks.com/instance: home' <<<"$block" \
      || fail "$name lacks the observability instance label"
    grep -q 'observability.hyeonworks.com/owner: platform-observability' <<<"$block" \
      || fail "$name lacks the observability owner label"
    grep -q "  $file.json:" <<<"$block" \
      || fail "$name does not embed $file.json"
  done
  pass 'kustomization renders five stable labeled dashboard ConfigMaps'
}

validate_inventory_driven_omissions() {
  local workload="$DASHBOARD_ROOT/workload-health.json"
  local services="$DASHBOARD_ROOT/platform-services.json"
  local endpoints="$DASHBOARD_ROOT/https-endpoints.json"

  if inventory_has_metric "$TARGET_INVENTORY" kube_pod_container_status_last_terminated_reason; then
    fail 'the OOM omission contract no longer matches target-initial inventory'
  fi
  if jq -r '.panels[].targets[].expr' "$workload" \
      | grep -Fq 'kube_pod_container_status_last_terminated_reason'; then
    fail 'workload dashboard guessed an OOM metric absent from target-initial inventory'
  fi
  pass 'OOM panel omitted: kube_pod_container_status_last_terminated_reason is absent from target-initial inventory'

  jq -e '
    ([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})") != null)
    and
    ([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})") != null)
  ' "$workload" >/dev/null \
    || fail 'PVC health panel must retain zero-valued Pending and Lost series instead of filtering healthy state away'
  pass 'PVC health panel keeps visible zero-valued Pending and Lost aggregates'

  if jq -e '
    any(.targets[].metrics[];
      (.name | startswith("traefik_"))
      and (.name | endswith("_bucket"))
      and (.name | test("duration|request"))
    )
  ' "$TARGET_INVENTORY" >/dev/null; then
    fail 'the Traefik latency omission contract no longer matches target-initial inventory'
  fi
  if jq -r '.panels[].targets[].expr' "$services" \
      | grep -Eq 'histogram_quantile|traefik_.*_bucket'; then
    fail 'service dashboard guessed a Traefik latency histogram absent from target-initial inventory'
  fi
  pass 'Traefik p95/p99 omitted: no request duration histogram bucket exists in target-initial inventory'

  inventory_has_metric "$POST_INVENTORY" probe_success \
    || fail 'post-substrate inventory lacks probe_success for the platform boundary summary'
  jq -e '
    any(.panels[].targets[];
      .expr == "min by (job) (probe_success{})"
    )
  ' "$services" >/dev/null \
    || fail 'platform service dashboard lacks the inventory-backed boundary health summary'
  pass 'platform service boundary summary uses post-substrate probe_success'

  jq -e '
    any(.panels[].targets[];
      .expr == "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})"
    )
  ' "$endpoints" >/dev/null \
    || fail 'HTTPS dashboard does not isolate the observed private-edge 403 boundary'
  pass 'HTTPS dashboard isolates the observed blackbox-private-edge status boundary'
}

validate_promql_syntax() {
  local rules=$1
  local file expression index=0
  local -a files=(
    kubernetes-node.json
    workload-health.json
    platform-services.json
    observability-backends.json
    https-endpoints.json
  )

  {
    printf 'groups:\n'
    printf '  - name: dashboard.promql.syntax\n'
    printf '    rules:\n'
    for file in "${files[@]}"; do
      while IFS= read -r expression; do
        index=$((index + 1))
        printf '      - record: dashboard_syntax_%d\n' "$index"
        printf '        expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
      done < <(jq -r '.panels[].targets[].expr' "$DASHBOARD_ROOT/$file")
    done
  } > "$rules"

  if [[ -n ${PROMTOOL_BIN:-} ]]; then
    "$PROMTOOL_BIN" check rules "$rules" >/dev/null
  else
    require_command docker
    docker run --rm \
      --entrypoint=/bin/promtool \
      -v "$rules:/tmp/dashboard-rules.yaml:ro" \
      "$PROMETHEUS_IMAGE" check rules /tmp/dashboard-rules.yaml >/dev/null
  fi
  pass "promtool parsed $index dashboard expressions"
}

validate_traefik_low_traffic_ratio() {
  local rules="$WORK_DIR/traefik-ratio-rules.yaml"
  local tests="$WORK_DIR/traefik-ratio-tests.yaml"
  local expression

  expression=$(jq -er '
    .panels[]
    | select(.title == "Traefik 5xx Ratio")
    | .targets[]
    | select(.refId == "A")
    | .expr
  ' "$DASHBOARD_ROOT/platform-services.json") \
    || fail 'Traefik 5xx ratio expression is absent'

  {
    printf 'groups:\n'
    printf '  - name: dashboard.traefik.ratio\n'
    printf '    interval: 1m\n'
    printf '    rules:\n'
    printf '      - record: dashboard_traefik_5xx_ratio_percent\n'
    printf '        expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
  } > "$rules"

  {
    printf 'rule_files:\n'
    printf '  - traefik-ratio-rules.yaml\n'
    printf 'evaluation_interval: 1m\n'
    printf 'tests:\n'
    printf '  - interval: 1m\n'
    printf '    input_series:\n'
    printf '      - series: '\''traefik_entrypoint_requests_total{code="200",entrypoint="websecure"}'\''\n'
    printf '        values: '\''0+1x10'\''\n'
    printf '      - series: '\''traefik_entrypoint_requests_total{code="500",entrypoint="websecure"}'\''\n'
    printf '        values: '\''0+1x10'\''\n'
    printf '    promql_expr_test:\n'
    printf '      - expr: dashboard_traefik_5xx_ratio_percent\n'
    printf '        eval_time: 10m\n'
    printf '        exp_samples:\n'
    printf '          - labels: '\''dashboard_traefik_5xx_ratio_percent{}'\''\n'
    printf '            value: 50\n'
  } > "$tests"

  if [[ -n ${PROMTOOL_BIN:-} ]]; then
    (cd "$WORK_DIR" && "$PROMTOOL_BIN" test rules traefik-ratio-tests.yaml) >/dev/null
  else
    require_command docker
    docker run --rm \
      --entrypoint=/bin/promtool \
      -v "$WORK_DIR:/tmp/dashboard-validation:ro" \
      -w /tmp/dashboard-validation \
      "$PROMETHEUS_IMAGE" test rules traefik-ratio-tests.yaml >/dev/null
  fi
  pass 'Traefik 5xx ratio preserves 50 percent at low request rates'
}

start_prometheus_port_forward() {
  local log="$WORK_DIR/prometheus-port-forward.log"
  local port=''
  local attempt

  kubectl -n observability port-forward \
    --address=127.0.0.1 \
    service/observability-core-kube-pr-prometheus \
    :9090 > "$log" 2>&1 &
  PORT_FORWARD_PID=$!

  for ((attempt = 1; attempt <= 100; attempt++)); do
    if grep -Eq '^Forwarding from 127\.0\.0\.1:[0-9]+ -> 9090$' "$log"; then
      port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> 9090$/\1/p' "$log" | head -n 1)
      break
    fi
    if ! kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
      fail "Prometheus port-forward exited before readiness: $(tr '\n' ' ' < "$log")"
    fi
    sleep 0.1
  done

  [[ "$port" =~ ^[1-9][0-9]*$ ]] \
    || fail 'Prometheus port-forward did not bind a loopback port within 10 seconds'
  PROMETHEUS_BASE_URL="http://127.0.0.1:$port"
}

prometheus_get() {
  local endpoint=$1
  shift
  curl \
    --fail \
    --silent \
    --show-error \
    --connect-timeout 2 \
    --max-time 10 \
    --get \
    "$PROMETHEUS_BASE_URL$endpoint" \
    "$@"
}

wait_for_prometheus() {
  local attempt
  for ((attempt = 1; attempt <= 50; attempt++)); do
    if prometheus_get '/-/ready' >/dev/null 2>&1; then
      return 0
    fi
    sleep 0.1
  done
  fail 'Prometheus did not become ready within 5 seconds'
}

live_query_nonempty() {
  local expression=$1
  local context=$2
  local expected_scalar=${3:-}
  local quiet=${4:-0}
  local response="$WORK_DIR/live-query-response.json"

  if ! prometheus_get '/api/v1/query' \
      --data-urlencode "query=$expression" > "$response"; then
    fail "live Prometheus query request failed: $context"
  fi
  jq -e '
    .status == "success"
    and (.data.resultType == "vector" or .data.resultType == "scalar")
    and (.data.result | type == "array" and length > 0)
  ' "$response" >/dev/null \
    || fail "live Prometheus query returned no vector/scalar result: $context"

  if [[ -n "$expected_scalar" ]] && ! jq -e --arg expected "$expected_scalar" '
      .data.resultType == "scalar"
      and .data.result[1] == $expected
    ' "$response" >/dev/null; then
    if [[ "$quiet" -eq 1 ]]; then
      return 1
    fi
    fail "live Prometheus scalar result was not exact $expected_scalar: $context"
  fi
}

validate_scalar_one_gate_regression() {
  if (live_query_nonempty 'scalar(vector(0))' 'scalar-zero regression fixture' 1 1); then
    fail 'scalar-zero comparison fixture passed the success gate'
  fi
  live_query_nonempty 'scalar(vector(1))' 'scalar-one regression fixture' 1
  pass 'success comparison gate rejects scalar 0 and accepts exact scalar 1'
}

validate_live_dashboard_queries() {
  local dashboard uid panel ref expression
  local count=0

  for dashboard in "$DASHBOARD_ROOT"/*.json; do
    while IFS=$'\t' read -r uid panel ref expression; do
      live_query_nonempty "$expression" "$uid / $panel / $ref"
      count=$((count + 1))
    done < <(
      jq -r '
        .uid as $uid
        | .panels[]
        | .title as $panel
        | .targets[]
        | [$uid, $panel, .refId, .expr]
        | @tsv
      ' "$dashboard"
    )
  done

  [[ "$count" -eq 37 ]] || fail "expected 37 live dashboard queries, got $count"
  pass 'live Prometheus accepted 37 dashboard queries with nonempty vector/scalar results'
}

validate_live_matcher_selectors() {
  local dashboard expression selector
  local selectors="$WORK_DIR/dashboard-matchers.txt"
  local count=0

  : > "$selectors"
  for dashboard in "$DASHBOARD_ROOT"/*.json; do
    while IFS= read -r expression; do
      grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]+\}' <<<"$expression" >> "$selectors" || true
    done < <(jq -r '.panels[].targets[].expr' "$dashboard")
  done
  sort -u -o "$selectors" "$selectors"

  while IFS= read -r selector; do
    [[ -n "$selector" ]] || continue
    live_query_nonempty "$selector" "live matcher selector $selector"
    count=$((count + 1))
  done < "$selectors"

  [[ "$count" -gt 0 ]] || fail 'no dashboard matcher selectors were discovered'
  pass "live Prometheus found series for $count dashboard matcher selectors"
}

validate_live_matcher_values() {
  local -a presence_contracts=(
    'node_cpu_seconds_total{mode="idle"}'
    'node_filesystem_avail_bytes{mountpoint="/"}'
    'node_filesystem_size_bytes{mountpoint="/"}'
    'node_filesystem_files_free{mountpoint="/"}'
    'node_filesystem_files{mountpoint="/"}'
    'kube_persistentvolumeclaim_status_phase{phase="Pending"}'
    'kube_persistentvolumeclaim_status_phase{phase="Lost"}'
  )
  local -a success_contracts=(
    'scalar(count(count by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
    'scalar(count(count by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
    'scalar(min(probe_success{job="blackbox-private-edge"})) == bool 1'
    'scalar(count(probe_http_status_code{job="blackbox-private-edge"})) == bool scalar(count(probe_http_status_code{job="blackbox-private-edge"} == 403))'
  )
  local contract

  for contract in "${presence_contracts[@]}"; do
    live_query_nonempty "$contract" "live matcher-value contract $contract"
  done
  for contract in "${success_contracts[@]}"; do
    live_query_nonempty "$contract" "live matcher-value contract $contract" 1
  done
  pass 'live matcher values cover idle CPU, root filesystems, PVC phases, both buckets, and private-edge 403'
}

validate_live_prometheus() {
  require_command curl
  if [[ -z "$PROMETHEUS_BASE_URL" ]]; then
    start_prometheus_port_forward
  fi
  wait_for_prometheus
  validate_scalar_one_gate_regression
  validate_live_dashboard_queries
  validate_live_matcher_selectors
  validate_live_matcher_values
}

main() {
  local work rendered rules
  parse_args "$@"
  require_command jq
  require_command sha256sum
  require_command kubectl

  verify_inventory target-initial
  verify_inventory post-substrate
  [[ -d "$DASHBOARD_ROOT" ]] || fail 'dashboard source directory is absent'

  validate_dashboard kubernetes-node.json platform-kubernetes-node "$TARGET_INVENTORY"
  validate_dashboard workload-health.json platform-workload-health "$TARGET_INVENTORY"
  validate_dashboard platform-services.json platform-services "$POST_INVENTORY"
  validate_dashboard observability-backends.json platform-observability-backends "$TARGET_INVENTORY"
  validate_dashboard https-endpoints.json platform-https-endpoints "$POST_INVENTORY"
  validate_inventory_driven_omissions

  work=$(mktemp -d)
  WORK_DIR=$work
  chmod 0755 "$WORK_DIR"
  trap cleanup EXIT
  trap 'exit 130' INT
  trap 'exit 143' TERM
  rendered="$work/rendered.yaml"
  rules="$work/dashboard-rules.yaml"
  validate_rendered_configmaps "$rendered"
  validate_promql_syntax "$rules"
  validate_traefik_low_traffic_ratio
  validate_live_prometheus

  pass 'observability dashboard contract'
}

main "$@"