#!/usr/bin/env bash

set -Eeuo pipefail

readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly RENDERER="$ROOT/scripts/validate/render-observability-core.sh"
readonly EXPECTED_REJECTION_STATUS=23

if [[ ! -f "$RENDERER" ]]; then
  printf 'EXPECTED RED: renderer is missing: %s\n' "$RENDERER" >&2
  exit 1
fi

# Sourcing must define assertions without running the renderer or changing shell options.
source "$RENDERER"

fixture_dir="$(mktemp -d /tmp/platform-observability-render-test.XXXXXX)"
handoff_dirs=()
cleanup() {
  local rc=$?
  local directory
  for directory in "${handoff_dirs[@]}"; do
    case "$directory" in
      /tmp/platform-observability-core-apply.*) rm -rf -- "$directory" ;;
    esac
  done
  case "$fixture_dir" in
    /tmp/platform-observability-render-test.*) rm -rf -- "$fixture_dir" ;;
  esac
  exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

pass_count=0

expect_pass() {
  local name="$1"
  local output
  shift
  if ! output="$({ "$@"; } 2>&1)"; then
    printf 'FAIL (expected pass): %s\n%s\n' "$name" "$output" >&2
    exit 1
  fi
  pass_count=$((pass_count + 1))
}

expect_stateful_pass() {
  local name="$1"
  local output_file="$fixture_dir/stateful-output"
  local rc
  shift
  set +e
  "$@" >"$output_file" 2>&1
  rc=$?
  set -e
  if [[ "$rc" != 0 ]]; then
    printf 'FAIL (expected stateful pass): %s\n' "$name" >&2
    sed -n '1,20p' "$output_file" >&2
    exit 1
  fi
  pass_count=$((pass_count + 1))
}

expect_fail() {
  local name="$1"
  local output rc
  shift
  set +e
  output="$({ "$@"; } 2>&1)"
  rc=$?
  set -e
  if [[ "$rc" != "$EXPECTED_REJECTION_STATUS" ]]; then
    printf 'FAIL (expected rejection status %s, got %s): %s\n%s\n' \
      "$EXPECTED_REJECTION_STATUS" "$rc" "$name" "$output" >&2
    exit 1
  fi
  if [[ "$output" != REJECT:* ]]; then
    printf 'FAIL (expected pinned rejection message): %s\n%s\n' "$name" "$output" >&2
    exit 1
  fi
  if [[ "$output" == *'DO-NOT-LEAK-FIXTURE'* ]]; then
    printf 'FAIL (credential leaked): %s\n' "$name" >&2
    exit 1
  fi
  pass_count=$((pass_count + 1))
}

write_file() {
  local path="$1"
  shift
  printf '%s\n' "$@" >"$path"
}

assert_chart_pin_row() {
  local expected="$*"
  local label="$1"
  local actual
  actual="$(chart_pin_rows | awk -F '\t' -v label="$label" '$1 == label { print $1, $2, $3, $4, $5, $6, $7 }')"
  [[ "$actual" == "$expected" ]]
}

assert_image_pin_row() {
  local expected="$*"
  local image="$1"
  local actual
  actual="$(image_pin_rows | awk -F '\t' -v image="$image" '$1 == image { print $1, $2 }')"
  [[ "$actual" == "$expected" ]]
}

expect_pass 'Helm version pin' test "$(observability_expected_helm_version)" = v3.19.4
expect_pass 'Kustomize version pin' test "$(observability_expected_kustomize_version)" = v5.8.1
expect_pass 'Kubernetes render target pin' test "$(observability_helm_kube_version)" = 1.36.2
expected_render_roots=$'infrastructure/namespaces/observability\n'
expected_render_roots+=$'infrastructure/controllers/prometheus-operator-crds\n'
expected_render_roots+=$'infrastructure/storage/observability-local-pv\n'
expected_render_roots+=$'services/observability/kube-prometheus-stack\n'
expected_render_roots+=$'services/observability/loki\n'
expected_render_roots+=$'services/observability/tempo\n'
expected_render_roots+=$'services/observability/alloy\n'
expected_render_roots+=$'services/observability/node-exporter\n'
expected_render_roots+=$'services/observability/grafana\n'
expected_render_roots+=$'services/observability/blackbox-exporter\n'
expected_render_roots+=$'services/observability/network-policies\n'
expected_render_roots+=$'services/observability/targets\n'
expected_render_roots+=$'services/observability/rules/platform\n'
expected_render_roots+=$'services/observability/alerting\n'
expected_render_roots+=$'services/observability/dashboards/platform\n'
expected_render_roots+='services/observability'
expect_pass 'complete observability render root table' test "$(render_root_rows)" = "$expected_render_roots"
expected_chart_table=$'prometheus-operator-crds\thttps://prometheus-community.github.io/helm-charts\tprometheus-operator-crds\t31.0.0\tv0.93.0\t5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd\tinfrastructure/controllers/prometheus-operator-crds\n'
expected_chart_table+=$'kube-prometheus-stack\thttps://prometheus-community.github.io/helm-charts\tkube-prometheus-stack\t88.0.1\tv0.93.0\t0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117\tservices/observability/kube-prometheus-stack\n'
expected_chart_table+=$'loki\thttps://grafana-community.github.io/helm-charts\tloki\t18.7.1\t3.7.4\tde9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d\tservices/observability/loki\n'
expected_chart_table+=$'tempo\thttps://grafana-community.github.io/helm-charts\ttempo\t2.2.3\t2.10.7\t14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe\tservices/observability/tempo\n'
expected_chart_table+=$'alloy\thttps://grafana.github.io/helm-charts\talloy\t1.11.0\tv1.18.0\t11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd\tservices/observability/alloy\n'
expected_chart_table+=$'prometheus-node-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-node-exporter\t4.56.1\t1.12.1\t70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a\tservices/observability/node-exporter'
expected_chart_table+=$'\ngrafana\thttps://grafana-community.github.io/helm-charts\tgrafana\t12.10.1\t13.1.1\tff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8\tservices/observability/grafana'
expected_chart_table+=$'\nprometheus-blackbox-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-blackbox-exporter\t11.16.0\tv0.28.0\t932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b\tservices/observability/blackbox-exporter'
expect_pass 'exact eight-row chart table with no extras' test "$(chart_pin_rows)" = "$expected_chart_table"

expected_image_table=$'quay.io/prometheus-operator/prometheus-operator:v0.93.0\tsha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2\n'
expected_image_table+=$'quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0\tsha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905\n'
expected_image_table+=$'quay.io/prometheus/prometheus:v3.13.2-distroless\tsha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b\n'
expected_image_table+=$'quay.io/prometheus/alertmanager:v0.33.1\tsha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b\n'
expected_image_table+=$'registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1\tsha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4\n'
expected_image_table+=$'quay.io/prometheus/node-exporter:v1.12.1\tsha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798\n'
expected_image_table+=$'docker.io/grafana/loki:3.7.4\tsha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e\n'
expected_image_table+=$'docker.io/grafana/tempo:2.10.7\tsha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c\n'
expected_image_table+=$'docker.io/grafana/alloy:v1.18.0\tsha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17\n'
expected_image_table+=$'ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0\tsha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522\n'
expected_image_table+=$'docker.io/library/busybox:1.37.0\tsha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0\n'
expected_image_table+=$'quay.io/minio/aistor/mc\tsha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b\n'
expected_image_table+=$'docker.io/grafana/grafana:13.1.1\tsha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b\n'
expected_image_table+=$'quay.io/kiwigrid/k8s-sidecar:2.10.0\tsha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3\n'
expected_image_table+=$'quay.io/prometheus/blackbox-exporter:v0.28.0\tsha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68'
expect_pass 'exact fifteen-row image table with no extras' test "$(image_pin_rows)" = "$expected_image_table"
expect_pass 'Prometheus CRD chart pin' assert_chart_pin_row prometheus-operator-crds https://prometheus-community.github.io/helm-charts prometheus-operator-crds 31.0.0 v0.93.0 5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd infrastructure/controllers/prometheus-operator-crds
expect_pass 'KPS chart pin' assert_chart_pin_row kube-prometheus-stack https://prometheus-community.github.io/helm-charts kube-prometheus-stack 88.0.1 v0.93.0 0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117 services/observability/kube-prometheus-stack
expect_pass 'Loki chart pin' assert_chart_pin_row loki https://grafana-community.github.io/helm-charts loki 18.7.1 3.7.4 de9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d services/observability/loki
expect_pass 'Tempo chart pin' assert_chart_pin_row tempo https://grafana-community.github.io/helm-charts tempo 2.2.3 2.10.7 14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe services/observability/tempo
expect_pass 'Alloy chart pin' assert_chart_pin_row alloy https://grafana.github.io/helm-charts alloy 1.11.0 v1.18.0 11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd services/observability/alloy
expect_pass 'node-exporter chart pin' assert_chart_pin_row prometheus-node-exporter https://prometheus-community.github.io/helm-charts prometheus-node-exporter 4.56.1 1.12.1 70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a services/observability/node-exporter
expect_pass 'Grafana chart pin' assert_chart_pin_row grafana https://grafana-community.github.io/helm-charts grafana 12.10.1 13.1.1 ff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8 services/observability/grafana
expect_pass 'Blackbox chart pin' assert_chart_pin_row prometheus-blackbox-exporter https://prometheus-community.github.io/helm-charts prometheus-blackbox-exporter 11.16.0 v0.28.0 932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b services/observability/blackbox-exporter

expect_pass 'operator image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-operator:v0.93.0 sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
expect_pass 'reloader image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0 sha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
expect_pass 'Prometheus image pin' assert_image_pin_row quay.io/prometheus/prometheus:v3.13.2-distroless sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
expect_pass 'Alertmanager image pin' assert_image_pin_row quay.io/prometheus/alertmanager:v0.33.1 sha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
expect_pass 'KSM image pin' assert_image_pin_row registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
expect_pass 'node-exporter image pin' assert_image_pin_row quay.io/prometheus/node-exporter:v1.12.1 sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798
expect_pass 'Loki image pin' assert_image_pin_row docker.io/grafana/loki:3.7.4 sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e
expect_pass 'Tempo image pin' assert_image_pin_row docker.io/grafana/tempo:2.10.7 sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c
expect_pass 'Alloy image pin' assert_image_pin_row docker.io/grafana/alloy:v1.18.0 sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
expect_pass 'telemetrygen image pin' assert_image_pin_row ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0 sha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522
expect_pass 'BusyBox image pin' assert_image_pin_row docker.io/library/busybox:1.37.0 sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0
expect_pass 'AIStor mc image pin' assert_image_pin_row quay.io/minio/aistor/mc sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b
expect_pass 'Grafana image pin' assert_image_pin_row docker.io/grafana/grafana:13.1.1 sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b
expect_pass 'Grafana sidecar image pin' assert_image_pin_row quay.io/kiwigrid/k8s-sidecar:2.10.0 sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3
expect_pass 'Blackbox image pin' assert_image_pin_row quay.io/prometheus/blackbox-exporter:v0.28.0 sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68
expect_pass 'tagged Grafana digest is in the rendered-image allowlist' grep -Fqx \
  docker.io/grafana/grafana:13.1.1@sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b \
  <(_observability_image_allowlist)
expect_pass 'tagged Blackbox digest is in the rendered-image allowlist' grep -Fqx \
  quay.io/prometheus/blackbox-exporter:v0.28.0@sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 \
  <(_observability_image_allowlist)

alertmanager_fixture="$fixture_dir/alertmanager.yaml"
write_file "$alertmanager_fixture" \
  'apiVersion: monitoring.coreos.com/v1' \
  'kind: Alertmanager' \
  'metadata:' \
  '  name: observability-core-kube-pr-alertmanager' \
  '  namespace: observability' \
  'spec:' \
  '  alertmanagerConfiguration:' \
  '    name: platform-alertmanager'
alerting_fixture="$fixture_dir/alerting.yaml"
write_file "$alerting_fixture" '{"apiVersion":"monitoring.coreos.com/v1alpha1","kind":"AlertmanagerConfig","metadata":{"name":"platform-alertmanager","namespace":"observability","labels":{"observability.hyeonworks.com/instance":"home"}},"spec":{"route":{"receiver":"platform-slack","groupBy":["cluster","namespace","alertname","severity"],"groupWait":"30s","groupInterval":"5m","repeatInterval":"4h","routes":[{"receiver":"platform-null","matchers":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}]}]},"inhibitRules":[{"sourceMatch":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}],"targetMatch":[{"name":"severity","matchType":"=","value":"info"}],"equal":["namespace"]}],"receivers":[{"name":"platform-null"},{"name":"platform-slack","slackConfigs":[{"apiURL":{"name":"alertmanager-slack-webhook","key":"url"},"sendResolved":true,"linkNames":false,"mrkdwnIn":["text","fields"],"fallback":"{{ if eq .Status \"firing\" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","title":"{{ if eq .Status \"firing\" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","titleLink":"https://grafana.learn.hyeonworks.com/","color":"{{ if eq .Status \"resolved\" }}good{{ else if or (eq .CommonLabels.severity \"emergency\") (eq .CommonLabels.severity \"critical\") }}danger{{ else if eq .CommonLabels.severity \"warning\" }}warning{{ else }}#439FE0{{ end }}","fields":[{"title":"Status","value":"{{ .Status | toUpper }}","short":true},{"title":"Severity","value":"{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}","short":true},{"title":"Location","value":"{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}","short":true},{"title":"Alert count","value":"{{ len .Alerts }}","short":true}],"text":"{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>","footer":"hyeonworks observability · Alertmanager"}]}]}}' \
  '---' \
  'apiVersion: networking.k8s.io/v1' \
  'kind: NetworkPolicy' \
  'metadata:' \
  '  name: observability-allow-alertmanager-public-https' \
  '  namespace: observability' \
  'spec:' \
  '  podSelector:' \
  '    matchLabels:' \
  '      app.kubernetes.io/name: alertmanager' \
  '      app.kubernetes.io/instance: observability-core-kube-pr-alertmanager' \
  '  policyTypes: [Egress]' \
  '  egress:' \
  '    - to:' \
  '        - ipBlock:' \
  '            cidr: 0.0.0.0/0' \
  '            except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]' \
  '      ports:' \
  '        - protocol: TCP' \
  '          port: 443'
expect_pass 'exact structured Alertmanager routing contract' \
  assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_fixture"

alertmanager_selector_fixture="$fixture_dir/alertmanager-selector.yaml"
sed '/^spec:$/a\  alertmanagerConfigSelector: {}' \
  "$alertmanager_fixture" >"$alertmanager_selector_fixture"
expect_fail 'Alertmanager discovery selector bypasses the global configuration' \
  assert_alertmanager_routing_contract "$alertmanager_selector_fixture" "$alerting_fixture"

alerting_route_fixture="$fixture_dir/alerting-route-drift.yaml"
sed 's/"repeatInterval":"4h"/"repeatInterval":"12h"/' \
  "$alerting_fixture" >"$alerting_route_fixture"
expect_fail 'Alertmanager Slack route drift' \
  assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_route_fixture"

write_pod() {
  local path="$1"
  local namespace="$2"
  local name="$3"
  local pod_fields="$4"
  local container_fields="${5:-}"
  cat >"$path" <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: $name
  namespace: $namespace
spec:
  automountServiceAccountToken: false
$pod_fields
  containers:
    - name: main
      image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
$container_fields
EOF
}

valid_image="$fixture_dir/valid-image.yaml"
write_file "$valid_image" \
  'apiVersion: v1' 'kind: Pod' 'metadata:' '  name: pinned' \
  '  namespace: observability' 'spec:' '  automountServiceAccountToken: false' \
  '  containers:' '    - name: prometheus' \
  '      image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
expect_pass 'pinned image accepted' assert_pinned_images "$valid_image"
expect_pass 'credential assertion positive control' assert_no_credentials "$valid_image"
expect_pass 'observability namespace security positive control' assert_namespace_pod_security "$valid_image"

tag_image="$fixture_dir/tag-image.yaml"
sed 's|@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b|:v3.13.2-distroless|' \
  "$valid_image" >"$tag_image"
expect_fail 'tag-only image' assert_pinned_images "$tag_image"

wrong_digest="$fixture_dir/wrong-digest.yaml"
sed 's/ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' \
  "$valid_image" >"$wrong_digest"
expect_fail 'unpinned digest' assert_pinned_images "$wrong_digest"

secret="$fixture_dir/secret.yaml"
write_file "$secret" \
  'apiVersion: v1' 'kind: Secret' 'metadata:' '  name: forbidden' \
  '  namespace: observability' 'stringData:' '  password: DO-NOT-LEAK-FIXTURE'
expect_fail 'credential Secret' assert_no_credentials "$secret"

literal_credential="$fixture_dir/literal-credential.yaml"
write_file "$literal_credential" \
  'apiVersion: v1' 'kind: Pod' 'metadata:' '  name: forbidden' \
  '  namespace: observability' 'spec:' '  automountServiceAccountToken: false' \
  '  containers:' '    - name: app' '      image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' \
  '      env:' '        - name: ACCESS_KEY' '          value: DO-NOT-LEAK-FIXTURE'
expect_fail 'literal credential payload' assert_no_credentials "$literal_credential"

for field in \
  '      privileged: true' \
  '  hostNetwork: true' \
  '  hostPID: true' \
  '  hostIPC: true'; do
  slug="$(printf '%s' "$field" | tr -cd '[:alpha:]' | tr '[:upper:]' '[:lower:]')"
  pod="$fixture_dir/observability-$slug.yaml"
  if [[ "$field" == *privileged* ]]; then
    container_security="$(printf '      securityContext:\n        privileged: true')"
    write_pod "$pod" observability core '' "$container_security"
  else
    write_pod "$pod" observability core "$field" ''
  fi
  expect_fail "observability forbids $slug" assert_namespace_pod_security "$pod"
done

obs_hostpath="$fixture_dir/observability-hostpath.yaml"
write_pod "$obs_hostpath" observability core \
  $'  volumes:\n    - name: host\n      hostPath:\n        path: /var/log/pods' \
  $'      volumeMounts:\n        - name: host\n          mountPath: /logs\n          readOnly: true'
expect_fail 'observability hostPath' assert_namespace_pod_security "$obs_hostpath"

controller_namespace="$fixture_dir/controller-namespace.yaml"
write_file "$controller_namespace" 'apiVersion: apps/v1' 'kind: Deployment' \
  'metadata:' '  name: controller-security' '  namespace: observability' 'spec:' \
  '  selector:' '    matchLabels:' '      app: controller-security' '  template:' \
  '    metadata:' '      labels:' '        app: controller-security' '    spec:' \
  '      automountServiceAccountToken: false' '      hostPID: true' '      containers:' \
  '        - name: app' \
  '          image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
expect_fail 'controller template inherits namespace security' assert_namespace_pod_security "$controller_namespace"
controller_valid="$fixture_dir/controller-valid.yaml"
sed '/hostPID: true/d' "$controller_namespace" >"$controller_valid"
expect_pass 'controller observability namespace positive control' \
  assert_namespace_pod_security "$controller_valid"
controller_missing_namespace="$fixture_dir/controller-missing-namespace.yaml"
sed '/namespace: observability/d' "$controller_valid" >"$controller_missing_namespace"
expect_fail 'controller namespace missing' \
  assert_namespace_pod_security "$controller_missing_namespace"
controller_foreign_namespace="$fixture_dir/controller-foreign-namespace.yaml"
sed 's/namespace: observability/namespace: default/' \
  "$controller_valid" >"$controller_foreign_namespace"
expect_fail 'controller namespace foreign' \
  assert_namespace_pod_security "$controller_foreign_namespace"

agent_allowed="$fixture_dir/agent-allowed.yaml"
write_pod "$agent_allowed" observability-agent alloy \
  $'  volumes:\n    - name: pod-logs\n      hostPath:\n        path: /var/log/pods' \
  $'      volumeMounts:\n        - name: pod-logs\n          mountPath: /var/log/pods\n          readOnly: true'
expect_pass 'Alloy read-only allowlisted hostPath' assert_namespace_pod_security "$agent_allowed"

missing_namespace="$fixture_dir/missing-namespace.yaml"
sed '/namespace: observability-agent/d' "$agent_allowed" >"$missing_namespace"
expect_fail 'Pod namespace missing' assert_namespace_pod_security "$missing_namespace"

foreign_namespace="$fixture_dir/foreign-namespace.yaml"
sed 's/namespace: observability-agent/namespace: default/' "$agent_allowed" >"$foreign_namespace"
expect_fail 'Pod namespace foreign' assert_namespace_pod_security "$foreign_namespace"

agent_bad_path="$fixture_dir/agent-bad-path.yaml"
sed 's|/var/log/pods|/etc|g' "$agent_allowed" >"$agent_bad_path"
expect_fail 'agent hostPath allowlist' assert_namespace_pod_security "$agent_bad_path"

agent_rw="$fixture_dir/agent-rw.yaml"
sed 's/readOnly: true/readOnly: false/' "$agent_allowed" >"$agent_rw"
expect_fail 'agent hostPath readOnly' assert_namespace_pod_security "$agent_rw"

automount_missing="$fixture_dir/automount-missing.yaml"
sed '/automountServiceAccountToken:/d' "$valid_image" >"$automount_missing"
expect_fail 'automount missing' assert_namespace_pod_security "$automount_missing"

automount_true="$fixture_dir/automount-true.yaml"
sed 's/automountServiceAccountToken: false/automountServiceAccountToken: true/' \
  "$valid_image" >"$automount_true"
expect_fail 'automount true' assert_namespace_pod_security "$automount_true"

for exposure in ingress NodePort LoadBalancer; do
  public="$fixture_dir/public-$exposure.yaml"
  if [[ "$exposure" == ingress ]]; then
    write_file "$public" 'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
      'metadata:' '  name: public' '  namespace: observability' 'spec: {}'
  else
    write_file "$public" 'apiVersion: v1' 'kind: Service' 'metadata:' \
      '  name: public' '  namespace: observability' 'spec:' "  type: $exposure" \
      '  ports:' '    - port: 80'
  fi
  expect_fail "public $exposure" assert_no_public_exposure "$public"
done
grafana_ingress="$fixture_dir/grafana-private-ingress.yaml"
write_file "$grafana_ingress" \
  'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
  'metadata:' '  name: grafana' '  namespace: observability' \
  'spec:' '  ingressClassName: traefik' '  rules:' \
  '    - host: grafana.learn.hyeonworks.com' '      http:' '        paths:' \
  '          - path: /' '            pathType: Prefix' '            backend:' \
  '              service:' '                name: grafana' '                port:' \
  '                  number: 80'
expect_pass 'exact Host-Nginx-fronted Grafana ingress' \
  assert_no_public_exposure "$grafana_ingress"
grafana_public_ingress="$fixture_dir/grafana-public-ingress.yaml"
sed 's/grafana.learn.hyeonworks.com/public.example.com/' \
  "$grafana_ingress" >"$grafana_public_ingress"
expect_fail 'Grafana ingress host drift' \
  assert_no_public_exposure "$grafana_public_ingress"
cluster_ip="$fixture_dir/cluster-ip.yaml"
write_file "$cluster_ip" 'apiVersion: v1' 'kind: Service' 'metadata:' \
  '  name: internal' '  namespace: observability' 'spec:' '  type: ClusterIP' \
  '  ports:' '    - port: 9090'
expect_pass 'public exposure assertion positive control' assert_no_public_exposure "$cluster_ip"

for auxiliary in gateway canary chunks-cache results-cache test minio; do
  loki_aux="$fixture_dir/loki-$auxiliary.yaml"
  write_file "$loki_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
    "  name: loki-$auxiliary" '  namespace: observability' 'spec: {}'
  expect_fail "Loki auxiliary $auxiliary" assert_no_loki_auxiliary "$loki_aux"
done
loki_main="$fixture_dir/loki-main.yaml"
write_file "$loki_main" 'apiVersion: apps/v1' 'kind: StatefulSet' 'metadata:' \
  '  name: loki' '  namespace: observability' 'spec: {}'
expect_pass 'Loki auxiliary assertion positive control' assert_no_loki_auxiliary "$loki_main"

for auxiliary in metrics-generator tempo-query; do
  tempo_aux="$fixture_dir/tempo-$auxiliary.yaml"
  write_file "$tempo_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
    "  name: $auxiliary" '  namespace: observability' 'spec: {}'
  expect_fail "Tempo auxiliary $auxiliary" assert_tempo_topology "$tempo_aux"
done

tempo_http="$fixture_dir/tempo-http.yaml"
write_file "$tempo_http" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
  '  name: tempo' '  namespace: observability' 'data:' '  tempo.yaml: |' \
  '    receivers:' '      otlp:' '        protocols:' '          grpc:' \
  '            endpoint: 0.0.0.0:4317' '          http:' \
  '            endpoint: 0.0.0.0:4318'
expect_fail 'Tempo OTLP HTTP receiver' assert_tempo_topology "$tempo_http"
tempo_grpc="$fixture_dir/tempo-grpc.yaml"
write_file "$tempo_grpc" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
  '  name: tempo' '  namespace: observability' 'data:' '  tempo.yaml: |' \
  '    receivers:' '      otlp:' '        protocols:' '          grpc:' \
  '            endpoint: 0.0.0.0:4317'
expect_pass 'Tempo topology assertion positive control' assert_tempo_topology "$tempo_grpc"

selector_values="$fixture_dir/selector-values.yaml"
write_file "$selector_values" 'prometheus:' '  prometheusSpec:' \
  '    serviceMonitorSelectorNilUsesHelmValues: false' \
  '    podMonitorSelectorNilUsesHelmValues: false' \
  '    probeSelectorNilUsesHelmValues: false' \
  '    ruleSelectorNilUsesHelmValues: false'
expect_pass 'selector values false' assert_selector_values "$selector_values"

selector_true="$fixture_dir/selector-true.yaml"
sed '0,/false/s//true/' "$selector_values" >"$selector_true"
expect_fail 'nilUsesHelmValues true' assert_selector_values "$selector_true"

prometheus_missing="$fixture_dir/prometheus-missing-selector.yaml"
write_file "$prometheus_missing" 'apiVersion: monitoring.coreos.com/v1' \
  'kind: Prometheus' 'metadata:' '  name: home' '  namespace: observability' \
  'spec:' '  serviceMonitorSelector: {}' '  podMonitorSelector: {}' \
  '  probeSelector: {}'
expect_fail 'Prometheus selector missing' assert_prometheus_selectors "$prometheus_missing"
prometheus_valid="$fixture_dir/prometheus-valid.yaml"
write_file "$prometheus_valid" 'apiVersion: monitoring.coreos.com/v1' \
  'kind: Prometheus' 'metadata:' '  name: home' '  namespace: observability' \
  'spec:' '  serviceMonitorSelector: {}' '  podMonitorSelector: {}' \
  '  probeSelector: {}' '  ruleSelector: {}'
expect_pass 'Prometheus selector assertion positive control' assert_prometheus_selectors "$prometheus_valid"

crd="$fixture_dir/kps-crd.yaml"
write_file "$crd" 'apiVersion: apiextensions.k8s.io/v1' \
  'kind: CustomResourceDefinition' 'metadata:' '  name: prometheuses.monitoring.coreos.com'
expect_fail 'CRD in workload render' assert_no_crds "$crd"
expect_pass 'CRD assertion positive control' assert_no_crds "$valid_image"

shared_pv="$fixture_dir/shared-pv.yaml"
write_file "$shared_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
  '  name: prometheus' 'spec:' '  storageClassName: shared-local' '  local:' \
  '    path: /srv/prometheus' '---' 'apiVersion: v1' 'kind: PersistentVolume' \
  'metadata:' '  name: alertmanager' 'spec:' '  storageClassName: shared-local' \
  '  local:' '    path: /srv/alertmanager'
expect_fail 'shared Local PV StorageClass' assert_unique_local_pv_storage_classes "$shared_pv"
single_pv="$fixture_dir/single-pv.yaml"
write_file "$single_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
  '  name: prometheus' 'spec:' '  storageClassName: prometheus-local' '  local:' \
  '    path: /srv/prometheus'
expect_pass 'Local PV assertion positive control' assert_unique_local_pv_storage_classes "$single_pv"

for fake in spring kafka batch backup; do
  fake_monitor="$fixture_dir/fake-$fake.yaml"
  write_file "$fake_monitor" 'apiVersion: monitoring.coreos.com/v1' \
    'kind: ServiceMonitor' 'metadata:' "  name: $fake-monitor" \
    '  namespace: observability' 'spec: {}'
  expect_fail "fake $fake monitor or rule" assert_no_unscoped_monitors_rules "$fake_monitor"
done
scoped_monitor="$fixture_dir/scoped-monitor.yaml"
write_file "$scoped_monitor" 'apiVersion: monitoring.coreos.com/v1' \
  'kind: ServiceMonitor' 'metadata:' '  name: prometheus' \
  '  namespace: observability' 'spec: {}'
expect_pass 'monitor/rule assertion positive control' assert_no_unscoped_monitors_rules "$scoped_monitor"

chart="$fixture_dir/Chart.yaml"
write_file "$chart" 'apiVersion: v2' 'name: loki' 'version: 18.7.1' 'appVersion: 3.7.4'
expect_pass 'chart metadata pin accepted' validate_chart_metadata "$chart" loki 18.7.1 3.7.4
expect_fail 'chart metadata name mismatch' validate_chart_metadata "$chart" tempo 18.7.1 3.7.4
expect_fail 'chart metadata version mismatch' validate_chart_metadata "$chart" loki 18.7.2 3.7.4
expect_fail 'chart metadata appVersion mismatch' validate_chart_metadata "$chart" loki 18.7.1 3.7.5

handoff="$fixture_dir/platform-observability-core-apply.fixture"
mkdir -m 0700 "$handoff"
expect_fail 'handoff prefix restricted to /tmp' validate_handoff_directory "$handoff"
safe_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$safe_handoff")
chmod 0700 "$safe_handoff"
expect_pass 'safe empty handoff' validate_handoff_directory "$safe_handoff"
touch "$safe_handoff/unexpected"
expect_fail 'handoff must be empty' validate_handoff_directory "$safe_handoff"
rm -f -- "$safe_handoff/unexpected"
rmdir -- "$safe_handoff"

empty_suffix='/tmp/platform-observability-core-apply.'
expect_fail 'handoff suffix must be nonempty' validate_handoff_directory "$empty_suffix"

root_fixture="$fixture_dir/render-roots"
mkdir -p "$root_fixture/existing/child"
expect_pass 'physical render roots accepted' validate_render_roots "$root_fixture" existing/child
expect_fail 'missing render root rejected' validate_render_roots "$root_fixture" missing/child
expect_pass 'missing root preflight creates nothing' test ! -e "$root_fixture/missing"
mkdir -p "$root_fixture/physical/child"
ln -s physical "$root_fixture/symlinked"
expect_fail 'symlinked render root lineage rejected' validate_render_roots "$root_fixture" symlinked/child

cache_root="$fixture_dir/cache-root"
mkdir -p "$cache_root/services/component/.helm/charts"
_OBS_ROOT="$cache_root"
_OBS_CREATED_CACHE_DIRS=()
expect_stateful_pass 'exclusive cache directory creation' _observability_create_cache_directory \
  services/component chart 1.0.0
expect_fail 'existing exact cache is no-clobber' _observability_create_cache_directory \
  services/component chart 1.0.0
expect_stateful_pass 'tracked cache cleanup' _observability_cleanup_cache_dirs
expect_pass 'pre-existing .helm preserved' test -d "$cache_root/services/component/.helm"
expect_pass 'pre-existing charts preserved' test -d "$cache_root/services/component/.helm/charts"
expect_pass 'exact created cache removed' test ! -e "$cache_root/services/component/.helm/charts/chart-1.0.0"
mkdir -p "$cache_root/services/missing-parent"
expect_fail 'missing component root rejected before cache mkdir' _observability_create_cache_directory \
  services/missing-parent/component chart 1.0.0
expect_pass 'missing component root leaves no .helm' test ! -e "$cache_root/services/missing-parent/component"
mkdir -p "$cache_root/services/new-component"
_OBS_CREATED_CACHE_DIRS=()
_OBS_CREATED_CACHE_IDENTITIES=()
_OBS_CREATED_CACHE_TYPES=()
expect_stateful_pass 'new cache parents tracked' _observability_create_cache_directory \
  services/new-component chart 1.0.0
expect_stateful_pass 'new cache parents cleaned' _observability_cleanup_cache_dirs
expect_pass 'created .helm parent removed' test ! -e "$cache_root/services/new-component/.helm"

mkdir -p "$cache_root/services/race-component/.helm/charts"
_OBS_CREATED_CACHE_DIRS=()
_OBS_CREATED_CACHE_IDENTITIES=()
_OBS_CREATED_CACHE_TYPES=()
expect_stateful_pass 'race cache created' _observability_create_cache_directory \
  services/race-component chart 1.0.0
race_cache="${_OBS_CREATED_CACHE_DIRS[0]}"
race_identity="${_OBS_CREATED_CACHE_IDENTITIES[0]}"
mkdir "$race_cache/owned-subdirectory"
write_file "$race_cache/owned-subdirectory/owned-marker" 'OWNED-CACHE-CONTENT'
cleanup_race_hook="$fixture_dir/cache-cleanup-race-hook.sh"
write_file "$cleanup_race_hook" '#!/usr/bin/env bash' 'set -Eeuo pipefail' \
  'replacement="$1"' 'mkdir -m 0700 -- "$replacement"' \
  'printf "%s\n" REPLACEMENT-MUST-SURVIVE >"$replacement/replacement-marker"'
chmod 0700 "$cleanup_race_hook"
expect_stateful_pass 'owned cache cleanup is identity-bound across replacement race' \
  _observability_cleanup_owned_cache_directory \
  "$race_cache" "$race_identity" tree "$cleanup_race_hook"
expect_pass 'replacement directory survives cleanup race' test -d "$race_cache"
expect_pass 'replacement content survives cleanup race' test \
  "$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
expect_pass 'owned quarantine is completely cleaned' test -z \
  "$(find "$cache_root/services/race-component/.helm/charts" \
    -mindepth 1 -maxdepth 1 ! -name chart-1.0.0 -print -quit)"

run_cleanup_exit_gate() (
  _OBS_WORK=''
  trap _observability_cleanup EXIT
  :
)
expect_fail 'cleanup identity mismatch fails the successful exit gate' run_cleanup_exit_gate
expect_pass 'exit-gate mismatch still preserves replacement' test \
  "$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
_OBS_CREATED_CACHE_DIRS=()
_OBS_CREATED_CACHE_IDENTITIES=()
_OBS_CREATED_CACHE_TYPES=()

handoff_work="$fixture_dir/handoff-work"
mkdir -m 0700 "$handoff_work"
for handoff_name in namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate; do
  write_file "$handoff_work/$handoff_name.yaml" "fixture-$handoff_name"
  chmod 0600 "$handoff_work/$handoff_name.yaml"
done
write_file "$handoff_work/resource-index.tsv" 'fixture-index'
chmod 0600 "$handoff_work/resource-index.tsv"

published_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$published_handoff")
chmod 0700 "$published_handoff"
published_identity="$(pin_handoff_directory "$published_handoff")"
expect_pass 'atomic no-clobber handoff publication' publish_verified_handoff \
  "$published_handoff" "$published_identity" "$handoff_work"
expect_pass 'exact handoff entry set accepted' validate_handoff_entry_set \
  "$published_handoff" "$published_identity"

ln -s /dev/null "$published_handoff/extra-symlink"
expect_fail 'extra handoff symlink rejected' validate_handoff_entry_set \
  "$published_handoff" "$published_identity"
rm -f -- "$published_handoff/extra-symlink"
mkdir "$published_handoff/extra-directory"
expect_fail 'extra handoff directory rejected' validate_handoff_entry_set \
  "$published_handoff" "$published_identity"
rmdir "$published_handoff/extra-directory"
mv "$published_handoff/aggregate.yaml" "$fixture_dir/aggregate-handoff-saved"
ln -s /dev/null "$published_handoff/aggregate.yaml"
expect_fail 'exact handoff entry symlink rejected' validate_handoff_entry_set \
  "$published_handoff" "$published_identity"
rm -f -- "$published_handoff/aggregate.yaml"
mv "$fixture_dir/aggregate-handoff-saved" "$published_handoff/aggregate.yaml"
mv "$published_handoff/core-rules.yaml" "$fixture_dir/core-rules-handoff-saved"
mkdir "$published_handoff/core-rules.yaml"
expect_fail 'exact handoff entry directory rejected' validate_handoff_entry_set \
  "$published_handoff" "$published_identity"
rmdir "$published_handoff/core-rules.yaml"
mv "$fixture_dir/core-rules-handoff-saved" "$published_handoff/core-rules.yaml"

replaced_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$replaced_handoff")
chmod 0700 "$replaced_handoff"
replaced_identity="$(pin_handoff_directory "$replaced_handoff")"
replaced_original="$replaced_handoff.original"
mv "$replaced_handoff" "$replaced_original"
handoff_dirs+=("$replaced_original")
mkdir -m 0700 "$replaced_handoff"
expect_fail 'replaced handoff identity rejected before writes' publish_verified_handoff \
  "$replaced_handoff" "$replaced_identity" "$handoff_work"
expect_pass 'replacement remains empty' test -z "$(find "$replaced_handoff" -mindepth 1 -maxdepth 1 -print -quit)"

occupied_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$occupied_handoff")
chmod 0700 "$occupied_handoff"
occupied_identity="$(pin_handoff_directory "$occupied_handoff")"
write_file "$occupied_handoff/namespaces.yaml" 'DO-NOT-CLOBBER'
expect_fail 'nonempty handoff rejected immediately before writes' publish_verified_handoff \
  "$occupied_handoff" "$occupied_identity" "$handoff_work"
expect_pass 'existing handoff file is not clobbered' test \
  "$(cat "$occupied_handoff/namespaces.yaml")" = DO-NOT-CLOBBER

child_a="$fixture_dir/child-a.yaml"
child_b="$fixture_dir/child-b.yaml"
aggregate="$fixture_dir/aggregate.yaml"
write_file "$child_a" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
  '  name: a' '  namespace: observability' 'data:' '  value: one'
write_file "$child_b" 'apiVersion: v1' 'kind: Service' 'metadata:' \
  '  name: b' '  namespace: observability' 'spec:' '  type: ClusterIP' \
  '  ports:' '    - port: 80'
{ cat "$child_a"; printf '%s\n' '---'; cat "$child_b"; } >"$aggregate"
index="$fixture_dir/resource-index.tsv"
expect_pass 'ordered resource index' build_resource_index "$index" \
  "kps:$child_a" "loki:$child_b"
expected_index=$'1\tkps\tv1\tConfigMap\tobservability\ta\t'
if [[ "$(sed -n '2p' "$index")" != "$expected_index"* ]]; then
  printf 'FAIL: resource index order or schema differs\n' >&2
  exit 1
fi
pass_count=$((pass_count + 1))
expect_pass 'child aggregate equivalence' assert_child_aggregate_equivalence \
  "$aggregate" "kps:$child_a" "loki:$child_b"

aggregate_missing="$fixture_dir/aggregate-missing.yaml"
cp "$child_a" "$aggregate_missing"
expect_fail 'aggregate missing resource' assert_child_aggregate_equivalence \
  "$aggregate_missing" "kps:$child_a" "loki:$child_b"

aggregate_drift="$fixture_dir/aggregate-drift.yaml"
sed 's/value: one/value: changed/' "$aggregate" >"$aggregate_drift"
expect_fail 'aggregate canonical hash drift' assert_child_aggregate_equivalence \
  "$aggregate_drift" "kps:$child_a" "loki:$child_b"

duplicate_child="$fixture_dir/duplicate-child.yaml"
cp "$child_a" "$duplicate_child"
expect_fail 'duplicate child tuple' assert_child_aggregate_equivalence \
  "$aggregate" "kps:$child_a" "loki:$child_b" "tempo:$duplicate_child"

assert_task2_manifests() {
  local manifest_root="${1:-$ROOT}"
  python3 - "$manifest_root" <<'PY'
import pathlib
import sys

import yaml


root = pathlib.Path(sys.argv[1])
namespace_root = root / "infrastructure/namespaces/observability"
storage_root = root / "infrastructure/storage/observability-local-pv"


def load_documents(path):
    with path.open(encoding="utf-8") as stream:
        return [value for value in yaml.safe_load_all(stream) if value is not None]


namespace_kustomization = yaml.safe_load(
    (namespace_root / "kustomization.yaml").read_text(encoding="utf-8")
)
assert namespace_kustomization["resources"] == [
    "observability.yaml",
    "observability-agent.yaml",
]
namespaces = []
for resource in namespace_kustomization["resources"]:
    namespaces.extend(load_documents(namespace_root / resource))
assert len(namespaces) == 2
assert [item["kind"] for item in namespaces].count("Namespace") == 2
assert len({item["metadata"]["name"] for item in namespaces}) == 2
by_name = {item["metadata"]["name"]: item for item in namespaces}
assert set(by_name) == {"observability", "observability-agent"}
for item in by_name.values():
    assert item["metadata"]["labels"]["observability.hyeonworks.com/enabled"] == "true"

for level in ("enforce", "audit", "warn"):
    labels = by_name["observability"]["metadata"]["labels"]
    assert labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
    assert labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
agent_labels = by_name["observability-agent"]["metadata"]["labels"]
assert agent_labels["pod-security.kubernetes.io/enforce"] == "privileged"
assert agent_labels["pod-security.kubernetes.io/enforce-version"] == "latest"
for level in ("audit", "warn"):
    assert agent_labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
    assert agent_labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"

assert (namespace_root / "discovery-namespaces.txt").read_text(encoding="utf-8") == (
    "kube-system\ngitea\nkeycloak\nplatform-data\nobject-storage\n"
)

storage_kustomization = yaml.safe_load(
    (storage_root / "kustomization.yaml").read_text(encoding="utf-8")
)
assert storage_kustomization["resources"] == [
    "storage-classes.yaml",
    "persistent-volumes.yaml",
]
storage_items = []
for resource in storage_kustomization["resources"]:
    storage_items.extend(load_documents(storage_root / resource))
assert len(storage_items) == 12
assert [item["kind"] for item in storage_items].count("StorageClass") == 6
assert [item["kind"] for item in storage_items].count("PersistentVolume") == 6
assert {item["kind"] for item in storage_items} == {"StorageClass", "PersistentVolume"}
class_items = [item for item in storage_items if item["kind"] == "StorageClass"]
volume_items = [item for item in storage_items if item["kind"] == "PersistentVolume"]
assert len({item["metadata"]["name"] for item in class_items}) == 6
assert len({item["metadata"]["name"] for item in volume_items}) == 6
assert len({item["metadata"]["labels"]["app.kubernetes.io/component"] for item in volume_items}) == 6
classes = {
    item["metadata"]["name"]: item
    for item in class_items
}
volumes = {
    item["metadata"]["labels"]["app.kubernetes.io/component"]: item
    for item in volume_items
}
expected = {
    "prometheus": ("/srv/k3s/ssd/observability/prometheus", "20Gi", "ssd-local-observability-prometheus-retain"),
    "grafana": ("/srv/k3s/ssd/observability/grafana", "2Gi", "ssd-local-observability-grafana-retain"),
    "alertmanager": ("/srv/k3s/ssd/observability/alertmanager", "1Gi", "ssd-local-observability-alertmanager-retain"),
    "alloy": ("/srv/k3s/ssd/observability/alloy", "2Gi", "ssd-local-observability-alloy-retain"),
    "loki": ("/srv/k3s/ssd/observability/loki", "5Gi", "ssd-local-observability-loki-retain"),
    "tempo": ("/srv/k3s/ssd/observability/tempo", "5Gi", "ssd-local-observability-tempo-retain"),
}
assert set(classes) == {row[2] for row in expected.values()}
assert set(volumes) == set(expected)
for owner, (path, capacity, class_name) in expected.items():
    storage_class = classes[class_name]
    assert storage_class["provisioner"] == "kubernetes.io/no-provisioner"
    assert storage_class["reclaimPolicy"] == "Retain"
    assert storage_class["volumeBindingMode"] == "WaitForFirstConsumer"
    assert storage_class["allowVolumeExpansion"] is False
    volume = volumes[owner]
    spec = volume["spec"]
    assert spec["capacity"]["storage"] == capacity
    assert spec["accessModes"] == ["ReadWriteOnce"]
    assert spec["persistentVolumeReclaimPolicy"] == "Retain"
    assert spec["storageClassName"] == class_name
    assert spec["local"]["path"] == path
    expression = spec["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"][0]
    assert expression == {
        "key": "kubernetes.io/hostname",
        "operator": "In",
        "values": ["donghyeon-system-product-name"],
    }
actual_capacity = 0
for volume in volume_items:
    value = volume["spec"]["capacity"]["storage"]
    assert isinstance(value, str) and value.endswith("Gi") and value[:-2].isdigit()
    actual_capacity += int(value[:-2])
assert actual_capacity == 35
PY
}

expect_pass 'Task 2 namespace and 35Gi Local PV matrix' assert_task2_manifests

expect_any_failure() {
  local name="$1"
  shift
  if "$@" >"$fixture_dir/expected-any-failure.out" 2>&1; then
    printf 'FAIL (expected any failure): %s\n' "$name" >&2
    exit 1
  fi
  pass_count=$((pass_count + 1))
}

task2_manifest_fixture() {
  local destination="$1"
  mkdir -p "$destination/infrastructure/namespaces" "$destination/infrastructure/storage"
  cp -R "$ROOT/infrastructure/namespaces/observability" \
    "$destination/infrastructure/namespaces/observability"
  cp -R "$ROOT/infrastructure/storage/observability-local-pv" \
    "$destination/infrastructure/storage/observability-local-pv"
}

duplicate_manifest_root="$fixture_dir/task2-manifest-duplicate"
task2_manifest_fixture "$duplicate_manifest_root"
{
  printf '%s\n' '---'
  sed -n '1,/^---$/p' \
    "$ROOT/infrastructure/storage/observability-local-pv/storage-classes.yaml" | sed '$d'
} >>"$duplicate_manifest_root/infrastructure/storage/observability-local-pv/storage-classes.yaml"
expect_any_failure 'duplicate StorageClass object is rejected' \
  assert_task2_manifests "$duplicate_manifest_root"

extra_kind_root="$fixture_dir/task2-manifest-extra-kind"
task2_manifest_fixture "$extra_kind_root"
write_file "$extra_kind_root/infrastructure/storage/observability-local-pv/extra.yaml" \
  'apiVersion: v1' 'kind: ConfigMap' 'metadata:' '  name: unexpected'
printf '%s\n' '  - extra.yaml' >> \
  "$extra_kind_root/infrastructure/storage/observability-local-pv/kustomization.yaml"
expect_any_failure 'extra manifest kind is rejected' \
  assert_task2_manifests "$extra_kind_root"

capacity_drift_root="$fixture_dir/task2-manifest-capacity-drift"
task2_manifest_fixture "$capacity_drift_root"
sed -i '0,/storage: 20Gi/s//storage: 19Gi/' \
  "$capacity_drift_root/infrastructure/storage/observability-local-pv/persistent-volumes.yaml"
expect_any_failure 'actual rendered capacity drift is rejected' \
  assert_task2_manifests "$capacity_drift_root"

readonly PREPARE_OBSERVABILITY_PATHS="$ROOT/scripts/bootstrap/prepare-observability-local-paths.sh"
source_options_before="$-"
# shellcheck source=/dev/null
source "$PREPARE_OBSERVABILITY_PATHS"
expect_pass 'path preparation script is source-safe' test "$-" = "$source_options_before"

olp_fixture="$fixture_dir/observability-local-paths"
olp_root="$olp_fixture/root"
olp_ssd="$olp_fixture/srv/k3s/ssd"
olp_aistor="$olp_fixture/srv/k3s/aistor"
olp_external="$olp_fixture/external"
mkdir -p "$olp_root" "$olp_ssd/observability" "$olp_aistor" "$olp_external"

olp_available='64424509440'
olp_uses=('0' '0' '0' '0' '0' '0')
olp_context='default'
olp_node='donghyeon-system-product-name'
olp_confirmation='APPLY default'
olp_helper_audit="$olp_fixture/helper-audit.log"
olp_helper_race=''
olp_helper_signal=''
olp_helper_signal_target=''

_olp_root_facts() { printf '%s\t%s\n' '/dev/root' 'ext4'; }
_olp_available_bytes() { printf '%s\n' "$olp_available"; }
_olp_current_context() { printf '%s\n' "$olp_context"; }
_olp_node_names() { printf '%s\n' "$olp_node"; }
_olp_read_confirmation() { printf '%s\n' "$olp_confirmation"; }

_olp_fixture_privileged_helper() {
  if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]] || \
    [[ "$2" != "$olp_root" || "$3" != "$olp_ssd" || "$4" != "$olp_aistor" ]]; then
    return 97
  fi
  local program
  program="$(_olp_helper_program)" || return
  OLP_HELPER_FIXTURE_ROOT="$olp_fixture" \
  OLP_HELPER_FIXTURE_VIRTUAL_AISTOR=1 \
  OLP_HELPER_FIXTURE_OWNERS=1 \
  OLP_HELPER_FIXTURE_AUDIT="$olp_helper_audit" \
  OLP_HELPER_FIXTURE_RACE="$olp_helper_race" \
  OLP_HELPER_FIXTURE_EXTERNAL="$olp_external" \
  OLP_HELPER_FIXTURE_SIGNAL="$olp_helper_signal" \
  OLP_HELPER_FIXTURE_SIGNAL_TARGET="$olp_helper_signal_target" \
    /usr/bin/python3 - "$@" <<<"$program"
}
_olp_privileged_helper() { _olp_fixture_privileged_helper "$@"; }

dry_output="$fixture_dir/observability-dry-run.out"
expect_stateful_pass 'default dry-run emits facts without mutation' \
  observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor"
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor" >"$dry_output"
expect_pass 'dry-run leaves every planned path absent' test -z \
  "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)"
expect_pass 'dry-run prints sanitized root filesystem facts' test "$(sed -n '1,3p' "$dry_output")" = \
  $'Root SSD device: /dev/root\nRoot SSD filesystem: ext4\nRoot SSD available bytes: 64424509440'
expect_pass 'dry-run prints all six exact planned fixture paths' test \
  "$(sed -n '4,9p' "$dry_output")" = \
  "$(for owner in prometheus grafana alertmanager alloy loki tempo; do
    printf 'Planned path: %s/observability/%s\n' "$olp_ssd" "$owner"
  done)"

assert_olp_execute_fails() {
  local output rc
  set +e
  output="$(observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" 2>&1)"
  rc=$?
  set -e
  [[ "$rc" != 0 && "$output" == ERROR:* ]]
}

assert_olp_execute_rejected() {
  assert_olp_execute_fails
  [[ -z "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)" ]]
}

exact_numeric_reply=$'PREFLIGHT\t53687091200\t1073741824\t0\t0\t0\t0\t0'
expect_pass '50GiB available and 1GiB aggregate are inclusive' \
  _olp_validate_preflight_reply "$exact_numeric_reply"
expect_any_failure 'one byte below 50GiB is rejected' \
  _olp_validate_preflight_reply $'PREFLIGHT\t53687091199\t0\t0\t0\t0\t0\t0'
expect_any_failure 'one target byte above 1GiB is rejected before summing' \
  _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t1073741825\t0\t0\t0\t0\t0'
expect_any_failure 'aggregate one byte above 1GiB is rejected' \
  _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t536870913\t536870912\t0\t0\t0\t0'
expect_any_failure 'signed decimal is rejected' \
  _olp_validate_preflight_reply $'PREFLIGHT\t+53687091200\t0\t0\t0\t0\t0\t0'
expect_any_failure 'leading-zero decimal is rejected' \
  _olp_validate_preflight_reply $'PREFLIGHT\t053687091200\t0\t0\t0\t0\t0\t0'
expect_any_failure 'multiline numeric protocol is rejected' \
  _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t0\t0\t0\t0\t0\t0\n0'
expect_any_failure '2^63 available value is rejected' \
  _olp_validate_preflight_reply $'PREFLIGHT\t9223372036854775808\t0\t0\t0\t0\t0\t0'
expect_any_failure 'repeated 2^63 target values fail before mutation' \
  _olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808'

write_file "$olp_external/command-marker" DO-NOT-EXECUTE
for unexpected in sudo install chown rm curl; do
  set +e
  _olp_fixture_privileged_helper "$unexpected" "$olp_root" "$olp_ssd" "$olp_aistor" \
    >"$fixture_dir/unexpected-helper-$unexpected.out" 2>&1
  unexpected_rc=$?
  set -e
  expect_pass "unexpected helper action $unexpected returns 97" test "$unexpected_rc" = 97
  expect_pass "unexpected helper action $unexpected executes nothing" test \
    "$(cat "$olp_external/command-marker")" = DO-NOT-EXECUTE
done
set +e
_olp_fixture_privileged_helper apply "$olp_root" "$olp_ssd" \
  >"$fixture_dir/unexpected-helper-argc.out" 2>&1
unexpected_rc=$?
set -e
expect_pass 'unexpected helper argc returns 97' test "$unexpected_rc" = 97

olp_context='other'
expect_pass 'execute rejects non-default context' assert_olp_execute_rejected
olp_context='default'
olp_node='other-node'
expect_pass 'execute rejects any other node set' assert_olp_execute_rejected
olp_node='donghyeon-system-product-name'
olp_confirmation='apply default'
expect_pass 'execute rejects inexact TTY confirmation' assert_olp_execute_rejected
olp_confirmation='APPLY default'

olp_context_calls="$fixture_dir/observability-context-calls"
write_file "$olp_context_calls" 0
_olp_current_context() {
  local calls
  calls="$(cat "$olp_context_calls")"
  calls=$((calls + 1))
  printf '%s\n' "$calls" >"$olp_context_calls"
  if (( calls == 1 )); then
    printf '%s\n' default
  else
    printf '%s\n' changed-during-confirmation
  fi
}
expect_pass 'execute rechecks context after TTY confirmation' assert_olp_execute_rejected
olp_context='default'
_olp_current_context() { printf '%s\n' "$olp_context"; }

mkdir "$olp_ssd/observability/prometheus"
write_file "$olp_ssd/observability/prometheus/existing" DO-NOT-CLOBBER
expect_pass 'execute rejects nonempty existing target' assert_olp_execute_fails
expect_pass 'nonempty rejection preserves content' test \
  "$(cat "$olp_ssd/observability/prometheus/existing")" = DO-NOT-CLOBBER
rm -f -- "$olp_ssd/observability/prometheus/existing"
rmdir "$olp_ssd/observability/prometheus"
ln -s "$olp_aistor" "$olp_ssd/observability/prometheus"
expect_pass 'execute rejects symlink target' assert_olp_execute_fails
expect_pass 'symlink rejection preserves link' test -L "$olp_ssd/observability/prometheus"
unlink "$olp_ssd/observability/prometheus"

write_file "$olp_helper_audit" START
expect_stateful_pass 'execute prepares all six paths' \
  observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor"
expected_owner_audit="$(printf '%s\n' \
  'set-owner-mode prometheus 1000 2000 0770' \
  'set-owner-mode grafana 472 472 0770' \
  'set-owner-mode alertmanager 1000 2000 0770' \
  'set-owner-mode alloy 473 473 0770' \
  'set-owner-mode loki 10001 10001 0770' \
  'set-owner-mode tempo 10001 10001 0770')"
expect_pass 'descriptor helper uses exact owner group mode matrix' test \
  "$(sed -n '/^set-owner-mode /p' "$olp_helper_audit")" = "$expected_owner_audit"
for owner in prometheus grafana alertmanager alloy loki tempo; do
  expect_pass "execute creates $owner path" test -d "$olp_ssd/observability/$owner"
  expect_pass "execute sets $owner mode" test "$(stat -c %a "$olp_ssd/observability/$owner")" = 770
  rmdir "$olp_ssd/observability/$owner"
done

chmod 0751 "$olp_external"
write_file "$olp_external/race-marker" EXTERNAL-MUST-SURVIVE
mkdir -m 0755 "$olp_ssd/observability/prometheus"
olp_helper_race='ancestor'
set +e
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
  >"$fixture_dir/observability-ancestor-race.out" 2>&1
race_rc=$?
set -e
expect_pass 'ancestor replacement race is rejected' test "$race_rc" != 0
expect_pass 'ancestor replacement leaves external marker untouched' test \
  "$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
expect_pass 'ancestor replacement leaves external metadata untouched' test \
  "$(stat -c %a "$olp_external")" = 751
expect_pass 'ancestor replacement restores pinned pre-existing metadata' test \
  "$(stat -c %a "$olp_ssd/observability.pinned/prometheus")" = 755
unlink "$olp_ssd/observability"
mv "$olp_ssd/observability.pinned" "$olp_ssd/observability"

olp_helper_race='target:prometheus'
set +e
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
  >"$fixture_dir/observability-target-race.out" 2>&1
race_rc=$?
set -e
expect_pass 'target replacement race is rejected' test "$race_rc" != 0
expect_pass 'target replacement leaves external marker untouched' test \
  "$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
expect_pass 'target replacement leaves external metadata untouched' test \
  "$(stat -c %a "$olp_external")" = 751
expect_pass 'target replacement restores pinned target metadata' test \
  "$(stat -c %a "$olp_ssd/observability/prometheus.pinned")" = 755
unlink "$olp_ssd/observability/prometheus"
mv "$olp_ssd/observability/prometheus.pinned" "$olp_ssd/observability/prometheus"
rmdir "$olp_ssd/observability/prometheus"

for signal_case in HUP:prometheus INT:alloy TERM:alloy; do
  olp_helper_race=''
  olp_helper_signal="${signal_case%%:*}"
  olp_helper_signal_target="${signal_case#*:}"
  write_file "$olp_helper_audit" START
  if [[ "$olp_helper_signal_target" == prometheus ]]; then
    rmdir "$olp_ssd/observability"
  else
    mkdir -m 0755 "$olp_ssd/observability/prometheus"
  fi
  set +e
  observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
    >"$fixture_dir/observability-signal-${olp_helper_signal}.out" 2>&1
  signal_rc=$?
  set -e
  expect_pass "$olp_helper_signal post-create signal is rejected" test "$signal_rc" != 0
  for owner in grafana alertmanager alloy loki tempo; do
    expect_pass "$olp_helper_signal leaves no created $owner residue" test \
      ! -e "$olp_ssd/observability/$owner"
  done
  if [[ "$olp_helper_signal_target" == prometheus ]]; then
    expect_pass "$olp_helper_signal leaves no first-target residue" test \
      ! -e "$olp_ssd/observability/prometheus"
    expect_pass "$olp_helper_signal leaves no created parent residue" test \
      ! -e "$olp_ssd/observability"
    mkdir "$olp_ssd/observability"
  else
    expect_pass "$olp_helper_signal restores pre-existing mode" test \
      "$(stat -c %a "$olp_ssd/observability/prometheus")" = 755
    expected_restore="$(printf 'restore-chown prometheus %s %s\nrestore-chmod prometheus 0755' \
      "$(id -u)" "$(id -g)")"
    expect_pass "$olp_helper_signal restores chown then chmod in exact order" test \
      "$(tail -n 2 "$olp_helper_audit")" = "$expected_restore"
    rmdir "$olp_ssd/observability/prometheus"
  fi
done
olp_helper_signal=''
olp_helper_signal_target=''

printf 'Assertions: %s\n' "$pass_count"
printf 'OBSERVABILITY CORE RENDER ASSERTION TEST PASS\n'