#!/usr/bin/env bash set -Eeuo pipefail readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly CAPTURE="$ROOT/scripts/validate/capture-observability-metric-inventory.sh" readonly REJECT_STATUS=23 readonly ORIGINAL_PATH="$PATH" fail() { printf 'TEST FAILURE: %s\n' "$*" >&2 exit 1 } expect_pass() { local name="$1" output shift if ! output="$({ "$@"; } 2>&1)"; then fail "$name (expected success): $output" fi pass_count=$((pass_count + 1)) } expect_reject() { local name="$1" output rc=0 shift output="$({ "$@"; } 2>&1)" || rc=$? [[ "$rc" == "$REJECT_STATUS" ]] || fail "$name (expected $REJECT_STATUS, got $rc): $output" [[ "$output" == REJECT:* ]] || fail "$name (unsanitized rejection): $output" [[ "$output" != *DO-NOT-LEAK-FIXTURE* ]] || fail "$name leaked a fixture value" pass_count=$((pass_count + 1)) } [[ -f "$CAPTURE" ]] || fail "EXPECTED RED: capture command is missing: $CAPTURE" # shellcheck source=/dev/null source "$CAPTURE" fixture_root="$(mktemp -d /tmp/platform-observability-inventory-test.XXXXXX)" test_helper_pids=() cleanup() { local rc=$? local helper_pid for helper_pid in "${test_helper_pids[@]}"; do if [[ "$helper_pid" =~ ^[0-9]+$ ]]; then kill -KILL "$helper_pid" 2>/dev/null || true; fi done if [[ "${KEEP_INVENTORY_TEST_FIXTURE:-0}" == 1 ]]; then printf 'KEPT FIXTURE: %s\n' "$fixture_root" >&2 exit "$rc" fi case "$fixture_root" in /tmp/platform-observability-inventory-test.*) rm -rf -- "$fixture_root" ;; *) printf 'TEST FAILURE: refusing unsafe fixture cleanup\n' >&2; rc=1 ;; esac exit "$rc" } trap cleanup EXIT pass_count=0 mapfile -t exact_pools <<'POOLS' podMonitor/platform-data/platform-postgres/0 serviceMonitor/gitea/gitea/0 serviceMonitor/keycloak/keycloak/0 serviceMonitor/kube-system/traefik/0 serviceMonitor/object-storage/aistor-bucket-usage/0 serviceMonitor/observability-agent/alloy/0 serviceMonitor/observability-agent/node-exporter/0 serviceMonitor/observability/grafana/0 serviceMonitor/observability/loki/0 serviceMonitor/observability/observability-core-kube-pr-alertmanager/0 serviceMonitor/observability/observability-core-kube-pr-alertmanager/1 serviceMonitor/observability/observability-core-kube-pr-apiserver/0 serviceMonitor/observability/observability-core-kube-pr-coredns/0 serviceMonitor/observability/observability-core-kube-pr-kubelet/0 serviceMonitor/observability/observability-core-kube-pr-kubelet/1 serviceMonitor/observability/observability-core-kube-pr-kubelet/2 serviceMonitor/observability/observability-core-kube-pr-operator/0 serviceMonitor/observability/observability-core-kube-pr-prometheus/0 serviceMonitor/observability/observability-core-kube-pr-prometheus/1 serviceMonitor/observability/observability-core-kube-state-metrics/0 serviceMonitor/observability/tempo/0 POOLS expected_target_selectors="$fixture_root/expected-target-selectors" expected_series_selectors="$fixture_root/expected-series-selectors" : >"$expected_target_selectors" : >"$expected_series_selectors" make_targets() { local destination="$1" pool labels selector series_selector index=0 local rows="$fixture_root/target-rows.jsonl" : >"$rows" for pool in "${exact_pools[@]}"; do labels="$(jq -cn --arg instance "10.0.0.$((index + 1)):9090" --arg target "target-$index" \ '{job:"fixture-job",instance:$instance,sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')" selector="{instance=\"10.0.0.$((index + 1)):9090\",job=\"fixture-job\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",target_fixture=\"target-$index\"}" if [[ "$pool" == serviceMonitor/object-storage/aistor-bucket-usage/0 ]]; then labels="$(jq -cn --arg target "target-$index" \ '{job:"minio-aistor",namespace:"object-storage",service:"minio",endpoint:"http-minio", container:"minio",instance:"10.42.7.19:9000",pod:"minio-aistor-pool-0-0", sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')" selector="{container=\"minio\",endpoint=\"http-minio\",instance=\"10.42.7.19:9000\",job=\"minio-aistor\",namespace=\"object-storage\",pod=\"minio-aistor-pool-0-0\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",service=\"minio\",target_fixture=\"target-$index\"}" fi printf '%s\n' "$selector" >>"$expected_target_selectors" series_selector="{__name__=\"fixture_metric\",${selector#\{}" printf '%s\n' "$series_selector" >>"$expected_series_selectors" jq -cn \ --arg pool "$pool" \ --arg url "http://10.42.0.$((index + 1)):9090/metrics" \ --argjson labels "$labels" \ '{discoveredLabels:$labels,labels:$labels,scrapePool:$pool,scrapeUrl:$url, globalUrl:$url,lastError:"",lastScrape:"2026-08-12T00:00:00Z", lastScrapeDuration:0.01,health:"up",scrapeInterval:"30s",scrapeTimeout:"10s"}' \ >>"$rows" index=$((index + 1)) done jq -cs '{status:"success",data:{activeTargets:.,droppedTargets:[]}}' "$rows" >"$destination" } valid_targets="$fixture_root/targets.json" make_targets "$valid_targets" oversized_padding="$fixture_root/oversized-padding" head -c 8388609 /dev/zero | tr '\000' x >"$oversized_padding" oversized_targets="$fixture_root/oversized-targets.json" jq --rawfile padding "$oversized_padding" '. + {padding:$padding}' "$valid_targets" >"$oversized_targets" expect_pass 'exact 21 active pools accepted' validate_targets_json "$valid_targets" post_targets="$fixture_root/post-targets.json" post_probe_rows="$fixture_root/post-probe-rows.json" post_substrate_expected_probe_targets | jq '[.[] as $target | $target + { discoveredLabels:$target.labels,globalUrl:$target.scrapeUrl,lastError:"", lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01, health:"up",scrapeInterval:"30s",scrapeTimeout:"15s"}]' >"$post_probe_rows" jq --slurpfile probes "$post_probe_rows" ' .data.activeTargets += ($probes[0] + [{ discoveredLabels:{}, labels:{job:"blackbox-exporter",namespace:"observability",service:"blackbox-exporter", endpoint:"http",container:"blackbox-exporter",pod:"blackbox-exporter-6d8444ffdb-svsck", instance:"10.42.0.123:9115"}, scrapePool:"serviceMonitor/observability/blackbox-exporter/0", scrapeUrl:"http://10.42.0.123:9115/metrics",globalUrl:"http://10.42.0.123:9115/metrics", lastError:"",lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01, health:"up",scrapeInterval:"30s",scrapeTimeout:"10s" }]) ' "$valid_targets" >"$post_targets" expect_pass 'exact post-substrate 30 targets and multiplicities accepted' \ validate_targets_json "$post_targets" post-substrate expect_reject 'post-substrate targets rejected by target-initial phase' \ validate_targets_json "$post_targets" target-initial post_missing_probe="$fixture_root/post-missing-probe.json" jq 'del(.data.activeTargets[] | select(.scrapePool=="probe/observability/platform-public-edge"))' \ "$post_targets" >"$post_missing_probe" expect_reject 'post-substrate missing Probe target rejected' \ validate_targets_json "$post_missing_probe" post-substrate expect_pass 'Probe query is removed from persisted scrape URL' \ test "$(sanitize_scrape_url 'http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fexample.test' strip-query)" = \ 'http://blackbox-exporter.observability.svc.cluster.local:9115/probe' expect_pass 'UTF-8 Prometheus label key is quoted in selector' \ test "$(prometheus_selector_for_labels '{"job":"blackbox","observability.hyeonworks.com/probe-group":"public-edge"}')" = \ '{job="blackbox","observability.hyeonworks.com/probe-group"="public-edge"}' mutate_and_reject() { local name="$1" filter="$2" destination destination="$fixture_root/${name// /-}.json" jq "$filter" "$valid_targets" >"$destination" expect_reject "$name" validate_targets_json "$destination" } mutate_and_reject 'missing pool rejected' 'del(.data.activeTargets[0])' mutate_and_reject 'extra pool rejected' '.data.activeTargets += [.data.activeTargets[0] | .scrapePool="serviceMonitor/extra/extra/0"]' mutate_and_reject 'duplicate pool rejected' '.data.activeTargets += [.data.activeTargets[0]]' mutate_and_reject 'down target rejected' '.data.activeTargets[0].health="down"' mutate_and_reject 'last error rejected' '.data.activeTargets[0].lastError="DO-NOT-LEAK-FIXTURE"' mutate_and_reject 'blackbox pool rejected' '.data.activeTargets[0].scrapePool="serviceMonitor/observability/blackbox-exporter/0"' mutate_and_reject 'Probe pool rejected' '.data.activeTargets[0].scrapePool="probe/observability/public-edge/0"' dropped_blackbox="$fixture_root/dropped-blackbox.json" jq '.data.droppedTargets = [{ "discoveredLabels": { "__meta_kubernetes_namespace": "observability", "__meta_kubernetes_service_name": "blackbox-exporter", "job": "blackbox-exporter" }, "labels": {"job": "blackbox-exporter"}, "scrapePool": "serviceMonitor/observability/blackbox-exporter/0", "scrapeUrl": "http://blackbox-exporter.observability.svc:9115/metrics", "globalUrl": "http://blackbox-exporter.observability.svc:9115/metrics", "lastError": "", "health": "unknown" }]' "$valid_targets" >"$dropped_blackbox" expect_reject 'dropped blackbox identity rejected' validate_targets_json "$dropped_blackbox" expect_pass 'post-substrate allows blackbox self discovery drops' \ validate_dropped_targets_json "$dropped_blackbox" post-substrate dropped_probe="$fixture_root/dropped-probe.json" jq '.data.droppedTargets = [{ "discoveredLabels": { "__meta_kubernetes_namespace": "observability", "__meta_kubernetes_service_name": "synthetic-exporter", "monitor_kind": "Probe" }, "labels": {"job": "synthetic-checks"}, "scrapePool": "probe/observability/public-edge/0", "scrapeUrl": "http://synthetic-exporter.observability.svc:9115/metrics", "globalUrl": "http://synthetic-exporter.observability.svc:9115/metrics", "lastError": "", "health": "unknown" }]' "$valid_targets" >"$dropped_probe" expect_reject 'dropped Probe identity rejected' validate_targets_json "$dropped_probe" legitimate_probe_word="$fixture_root/legitimate-probe-word.json" jq '.data.droppedTargets = [{ "discoveredLabels": { "__meta_kubernetes_pod_annotation_fixture": "{\"livenessProbe\":true}", "__metrics_path__": "/metrics/probes" }, "labels": {"job": "platform-postgres"}, "scrapePool": "podMonitor/platform-data/platform-postgres/0", "scrapeUrl": "http://10.42.0.1:9187/metrics/probes", "globalUrl": "http://10.42.0.1:9187/metrics/probes", "lastError": "", "health": "unknown" }]' "$valid_targets" >"$legitimate_probe_word" expect_pass 'legitimate dropped metadata containing probe words accepted in active response' \ validate_targets_json "$legitimate_probe_word" expect_pass 'legitimate dropped metadata containing probe words accepted in dropped response' \ validate_dropped_targets_json "$legitimate_probe_word" empty_dropped_targets="$fixture_root/empty-dropped-targets.json" printf '%s\n' '{"status":"success","data":{"activeTargets":[],"droppedTargets":[]}}' >"$empty_dropped_targets" mutate_and_reject 'AIStor wrong job rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.job)="wrong"' mutate_and_reject 'AIStor wrong namespace rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.namespace)="wrong"' mutate_and_reject 'AIStor wrong service rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.service)="wrong"' mutate_and_reject 'AIStor wrong endpoint rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.endpoint)="wrong"' mutate_and_reject 'AIStor wrong container rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.container)="wrong"' expect_pass 'safe HTTP URL accepted' test "$(sanitize_scrape_url 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets')" = 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets' expect_pass 'safe HTTPS URL accepted' test "$(sanitize_scrape_url 'https://metrics.example.test/metrics')" = 'https://metrics.example.test/metrics' for unsafe_url in \ 'http://user:DO-NOT-LEAK-FIXTURE@host/metrics' \ 'http://host/metrics?token=DO-NOT-LEAK-FIXTURE' \ 'http://host/metrics#DO-NOT-LEAK-FIXTURE' \ 'ftp://host/metrics' \ 'http://host/space here'; do expect_reject 'unsafe scrape URL rejected' sanitize_scrape_url "$unsafe_url" done metadata="$fixture_root/metadata.json" printf '%s\n' '{"status":"success","data":{"z_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}],"a_metric":[{"type":"counter","help":"ignored","unit":""}]}}' >"$metadata" expect_pass 'metadata metric names sorted' test "$(extract_metric_names "$metadata")" = $'a_metric\nz_metric' metadata_array="$fixture_root/metadata-array.json" printf '%s\n' '{"status":"success","data":[{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""},{"target":{"job":"fixture"},"metric":"a_metric","type":"counter","help":"ignored","unit":""},{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"duplicate","unit":""}]}' >"$metadata_array" expect_pass 'current Prometheus metadata array yields unique sorted metric names' \ test "$(extract_metric_names "$metadata_array")" = $'a_metric\nz_metric' bad_metadata_array="$fixture_root/bad-metadata-array.json" printf '%s\n' '{"status":"success","data":[{"target":{},"metric":"bad metric","type":"gauge","help":"ignored","unit":""}]}' >"$bad_metadata_array" expect_reject 'invalid current metadata array metric name rejected' extract_metric_names "$bad_metadata_array" expect_pass 'post-substrate Probe pool accepts probe metrics' \ validate_metric_names_for_pool $'probe_duration_seconds\nprobe_success' \ probe/observability/platform-public-edge post-substrate expect_reject 'target-initial rejects probe metrics' \ validate_metric_names_for_pool probe_success serviceMonitor/observability/grafana/0 target-initial expect_reject 'post-substrate non-Probe pool rejects probe metrics' \ validate_metric_names_for_pool probe_success serviceMonitor/observability/blackbox-exporter/0 post-substrate expect_reject 'post-substrate Probe pool requires probe_success descriptor' \ validate_metric_names_for_pool probe_duration_seconds probe/observability/platform-public-edge post-substrate bad_metadata="$fixture_root/bad-metadata.json" printf '%s\n' '{"status":"success","data":{"bad metric":[]}}' >"$bad_metadata" expect_reject 'invalid metadata metric name rejected' extract_metric_names "$bad_metadata" rules="$fixture_root/rules.json" printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"z_metric","query":"DO-NOT-LEAK-FIXTURE"},{"type":"alerting","name":"a_metric"}]}]}}' >"$rules" expect_reject 'recording-rule collision rejected' reject_recording_rule_collisions "$rules" "$metadata" printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"recorded_metric"}]}]}}' >"$rules" expect_pass 'non-colliding recording rule accepted' reject_recording_rule_collisions "$rules" "$metadata" series="$fixture_root/series.json" printf '%s\n' '{"status":"success","data":[{"z_label":"DO-NOT-LEAK-FIXTURE","__name__":"a_metric","a_label":"private-value"},{"__name__":"a_metric","middle":"another-private-value"}]}' >"$series" expect_pass 'series parser emits only sorted label keys' test "$(extract_series_label_names "$series")" = '["__name__","a_label","middle","z_label"]' [[ "$(extract_series_label_names "$series")" != *private-value* ]] || fail 'series parser persisted label values' dotted_series="$fixture_root/dotted-series.json" printf '%s\n' '{"status":"success","data":[{"__name__":"probe_success","instance":"https://example.test/","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]}' >"$dotted_series" expect_pass 'series parser preserves safe quoted Prometheus label names' \ test "$(extract_series_label_names "$dotted_series")" = \ '["__name__","instance","job","namespace","observability.hyeonworks.com/probe-group"]' empty_probe_series="$fixture_root/empty-probe-series.json" printf '%s\n' '{"status":"success","data":[]}' >"$empty_probe_series" expect_pass 'target-initial requires probe series absence' \ validate_probe_series_phase "$empty_probe_series" target-initial expect_reject 'post-substrate rejects probe series absence' \ validate_probe_series_phase "$empty_probe_series" post-substrate expect_pass 'post-substrate accepts observability probe series' \ validate_probe_series_phase "$dotted_series" post-substrate expect_reject 'target-initial rejects existing probe series' \ validate_probe_series_phase "$dotted_series" target-initial post_labels="$fixture_root/post-labels.json" post_substrate_expected_probe_targets | jq '[.[].labels]' >"$post_labels" for query_kind in success status duration tls; do query_fixture="$fixture_root/post-query-$query_kind.json" jq -cn --slurpfile labels "$post_labels" --arg kind "$query_kind" ' ($labels[0] | if $kind=="tls" then map(select(.instance|startswith("https://"))) else . end) as $selected | {status:"success",data:{resultType:"vector",result:[$selected[] | {metric:(. + {__name__:(if $kind=="success" then "probe_success" elif $kind=="status" then "probe_http_status_code" elif $kind=="duration" then "probe_duration_seconds" else "probe_ssl_earliest_cert_expiry" end)}), value:[1770000000,(if $kind=="success" then "1" elif $kind=="status" then (if .job=="blackbox-private-edge" then "403" else "200" end) elif $kind=="duration" then "0.1" else "4102444800" end)]}]}} ' >"$query_fixture" expect_pass "post-substrate $query_kind query contract accepted" \ validate_post_substrate_query_body "$query_fixture" "$query_kind" done large_target_rows="$fixture_root/large-target-rows.jsonl" awk 'BEGIN { for (i = 0; i < 20000; i++) printf "{\"scrape_pool\":\"pool/%d\",\"scrape_url\":\"http://10.42.0.1:9090/metrics\",\"health\":\"up\",\"last_error\":\"\",\"provenance\":\"core\",\"metrics\":[{\"name\":\"fixture_metric\",\"label_names\":[\"__name__\",\"instance\"]}]}\n", i }' >"$large_target_rows" large_inventory_contract() { [[ "$(stat -c %s "$large_target_rows")" -gt 2097152 ]] || return 1 build_inventory_json "$large_target_rows" target-initial 2026-08-12T00:00:00Z | jq -e '.phase=="target-initial" and (.targets|length)==20000' >/dev/null } expect_pass 'inventory builder reads target rows from file beyond argv size limit' large_inventory_contract fake_bin="$fixture_root/bin" mkdir -p "$fake_bin" fake_log="$fixture_root/fake-commands.log" pf_log="$fixture_root/port-forward-pids.log" : >"$fake_log" : >"$pf_log" cat >"$fake_bin/kubectl" <<'FAKE_KUBECTL' #!/usr/bin/env bash set -Eeuo pipefail printf 'kubectl' >>"$FAKE_LOG" printf ' <%s>' "$@" >>"$FAKE_LOG" printf '\n' >>"$FAKE_LOG" args=" $* " if [[ "$args" =~ [[:space:]](apply|delete|patch|replace|exec|rollout|scale)[[:space:]] ]]; then exit 91 fi if [[ "$args" == *' config current-context '* ]]; then printf 'default\n'; exit 0; fi if [[ "$args" == *' config view '* ]]; then printf '%s\n' '{"clusters":[{"cluster":{"server":"https://127.0.0.1:6443"}}]}'; exit 0; fi if [[ "$args" == *' get --raw=/readyz '* ]]; then printf 'ok\n'; exit 0; fi if [[ "$args" == *' auth can-i '* ]]; then printf 'yes\n'; exit 0; fi if [[ "$args" == *' get prometheus '* ]]; then printf '%s\n' '{"apiVersion":"monitoring.coreos.com/v1","kind":"Prometheus","metadata":{"name":"observability-core-kube-pr-prometheus","namespace":"observability"}}'; exit 0; fi if [[ "$args" == *' get pods '* ]]; then printf '%s\n' '{"items":[{"metadata":{"name":"prometheus-observability-core-kube-pr-prometheus-0","namespace":"observability"},"status":{"phase":"Running","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"prometheus","ready":true}]}}]}'; exit 0; fi if [[ "$args" == *' port-forward '* ]]; then printf '%s\n' "$$" >>"$PF_LOG" pf_stderr_path="$(readlink -- "/proc/$$/fd/2")" printf '%s\n' "${pf_stderr_path%/*}" >>"$PF_WORKDIR_LOG" if [[ "${PF_IGNORE_TERM:-0}" == 1 ]]; then trap 'printf "ignored %s TERM\n" "$$" >>"$PF_LOG"' TERM else trap 'printf "stopped %s TERM\n" "$$" >>"$PF_LOG"; exit 0' TERM fi trap 'printf "stopped %s INT\n" "$$" >>"$PF_LOG"; exit 0' INT trap 'printf "stopped %s HUP\n" "$$" >>"$PF_LOG"; exit 0' HUP if [[ "${PF_NO_ANNOUNCEMENT:-0}" != 1 ]]; then printf 'Forwarding from 127.0.0.1:43123 -> 9090\n' >&2 fi while :; do /usr/bin/sleep 0.05; done fi exit 1 FAKE_KUBECTL cat >"$fake_bin/curl" <<'FAKE_CURL' #!/usr/bin/env bash set -Eeuo pipefail [[ "${1-}" == --disable ]] || exit 92 printf 'curl' >>"$FAKE_LOG" printf ' <%s>' "$@" >>"$FAKE_LOG" printf '\n' >>"$FAKE_LOG" destination='' url='' max_filesize='' all_arguments=" $* " declare -a data_values=() declare -a urlencode_values=() while (($#)); do case "$1" in --output) destination="$2"; shift 2 ;; --max-filesize) max_filesize="$2"; shift 2 ;; --data) data_values+=("$2"); shift 2 ;; --data-urlencode) urlencode_values+=("$2"); shift 2 ;; http://*) url="$1"; shift ;; *) shift ;; esac done [[ -n "$destination" && -n "$url" ]] || exit 2 emit_text() { if [[ "$destination" == - ]]; then printf '%s\n' "$1"; else printf '%s\n' "$1" >"$destination"; fi } emit_file() { if [[ "$destination" == - ]]; then cat -- "$1"; else cp -- "$1" "$destination"; fi } case "$url" in */-/ready) emit_text 'ready' ;; */api/v1/targets) [[ "${#data_values[@]}" == 1 ]] || exit 80 case "${data_values[0]}" in state=dropped) if [[ "${FAKE_CURL_MODE:-success}" == dropped_blackbox ]]; then emit_file "$DROPPED_BLACKBOX_FIXTURE" else emit_file "$DROPPED_TARGETS_FIXTURE" fi exit 0 ;; state=active) ;; *) exit 81 ;; esac if [[ "${FAKE_CURL_MODE:-success}" == hang ]]; then trap 'exit 128' INT TERM HUP while :; do /usr/bin/sleep 0.05; done fi if [[ "${FAKE_CURL_MODE:-success}" == error ]]; then exit 22; fi if [[ "${FAKE_CURL_MODE:-success}" == oversized_targets ]]; then [[ "$max_filesize" =~ ^[0-9]+$ ]] || exit 88 if (( $(stat -c %s "$OVERSIZED_TARGETS_FIXTURE") > max_filesize )); then exit 63; fi emit_file "$OVERSIZED_TARGETS_FIXTURE" else emit_file "$TARGETS_FIXTURE" fi ;; */api/v1/rules) emit_text '{"status":"success","data":{"groups":[]}}' ;; */api/v1/targets/metadata) [[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match_target=* ]] || exit 84 [[ "${#data_values[@]}" == 1 && "${data_values[0]}" == 'limit=10000' ]] || exit 85 target_selector="${urlencode_values[0]#match_target=}" grep -Fqx -- "$target_selector" "$EXPECTED_TARGET_SELECTORS" || exit 86 printf '%s\n' "$target_selector" >>"$METADATA_SELECTOR_LOG" emit_text '{"status":"success","data":{"descriptor_only_metric":[{"type":"gauge","help":"registered without a current series","unit":""}],"fixture_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}]}}' ;; */api/v1/series) if [[ "$all_arguments" == *'probe_.+'* ]]; then [[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == 'match[]={__name__=~"probe_.+"}' ]] || exit 87 emit_text '{"status":"success","data":[]}' else [[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match\[\]=* ]] || exit 89 series_selector="${urlencode_values[0]#match[]=}" if [[ "$series_selector" == *'__name__="descriptor_only_metric"'* ]]; then fixture_selector="${series_selector/descriptor_only_metric/fixture_metric}" grep -Fqx -- "$fixture_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90 emit_text '{"status":"success","data":[]}' exit 0 fi grep -Fqx -- "$series_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90 if [[ "${FAKE_CURL_MODE:-success}" == all_series_empty ]]; then emit_text '{"status":"success","data":[]}' exit 0 fi printf '%s\n' "$series_selector" >>"$SERIES_SELECTOR_LOG" if [[ "${FAKE_CURL_MODE:-success}" == hang_series ]]; then printf '%s\n' "$$" >>"$CURL_PID_LOG" : >"$CURL_HANG_MARKER" trap 'exit 128' INT TERM HUP while :; do /usr/bin/sleep 0.05; done fi emit_text '{"status":"success","data":[{"__name__":"fixture_metric","instance":"10.42.0.1:9090","secret_label":"DO-NOT-LEAK-FIXTURE"}]}' fi ;; *) exit 3 ;; esac FAKE_CURL cat >"$fake_bin/ln" <<'FAKE_LN' #!/usr/bin/env bash set -Eeuo pipefail if [[ "${FAIL_SHA_LINK:-0}" == 1 && "${*: -1}" == */inventory.sha256 ]]; then exit 1 fi exec /usr/bin/ln "$@" FAKE_LN chmod 0700 "$fake_bin/kubectl" "$fake_bin/curl" "$fake_bin/ln" export PATH="$fake_bin:$ORIGINAL_PATH" pf_workdir_log="$fixture_root/port-forward-workdirs.log" curl_pid_log="$fixture_root/curl-pids.log" metadata_selector_log="$fixture_root/metadata-selectors.log" series_selector_log="$fixture_root/series-selectors.log" : >"$pf_workdir_log" : >"$curl_pid_log" : >"$metadata_selector_log" : >"$series_selector_log" export FAKE_LOG="$fake_log" PF_LOG="$pf_log" PF_WORKDIR_LOG="$pf_workdir_log" export CURL_PID_LOG="$curl_pid_log" TARGETS_FIXTURE="$valid_targets" OVERSIZED_TARGETS_FIXTURE="$oversized_targets" export DROPPED_TARGETS_FIXTURE="$empty_dropped_targets" DROPPED_BLACKBOX_FIXTURE="$dropped_blackbox" export EXPECTED_TARGET_SELECTORS="$expected_target_selectors" EXPECTED_SERIES_SELECTORS="$expected_series_selectors" export METADATA_SELECTOR_LOG="$metadata_selector_log" SERIES_SELECTOR_LOG="$series_selector_log" run_capture() { local output_dir="$1" shift env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" \ PF_WORKDIR_LOG="$PF_WORKDIR_LOG" CURL_PID_LOG="$CURL_PID_LOG" \ EXPECTED_TARGET_SELECTORS="$EXPECTED_TARGET_SELECTORS" EXPECTED_SERIES_SELECTORS="$EXPECTED_SERIES_SELECTORS" \ METADATA_SELECTOR_LOG="$METADATA_SELECTOR_LOG" SERIES_SELECTOR_LOG="$SERIES_SELECTOR_LOG" \ DROPPED_TARGETS_FIXTURE="$DROPPED_TARGETS_FIXTURE" DROPPED_BLACKBOX_FIXTURE="$DROPPED_BLACKBOX_FIXTURE" \ OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \ FAKE_CURL_MODE="${FAKE_CURL_MODE:-success}" \ bash "$CAPTURE" --execute --phase target-initial --output-dir "$output_dir" "$@" } inventory_kubectl_alias() { inventory_kubectl "$@"; } mutation_log_lines="$(wc -l <"$fake_log")" for wrapper_name in inventory_kubectl inventory_kubectl_alias; do for mutation_verb in apply create delete patch replace exec rollout scale; do expect_reject "$wrapper_name default-denies $mutation_verb" \ "$wrapper_name" "$mutation_verb" fixture done done [[ "$(wc -l <"$fake_log")" == "$mutation_log_lines" ]] || fail 'denied wrapper mutation reached kubectl' ownership_signal_log="$fixture_root/ownership-signals.log" ownership_ready="$fixture_root/ownership-ready" bash -c ' trap '\''printf "TERM\n" >>"$1"; exit 0'\'' TERM : >"$2" while :; do /usr/bin/sleep 0.05; done ' bash "$ownership_signal_log" "$ownership_ready" & ownership_pid=$! test_helper_pids+=("$ownership_pid") for ((attempt=0; attempt<40; attempt++)); do [[ -e "$ownership_ready" ]] && break; sleep 0.025; done [[ -e "$ownership_ready" ]] || fail 'ownership fixture did not start' ownership_snapshot="$(inventory_process_snapshot "$ownership_pid")" || fail 'ownership fixture snapshot failed' INVENTORY_PORT_FORWARD_PID="$ownership_pid" INVENTORY_PORT_FORWARD_START="$((10#${ownership_snapshot#*:} + 1))" expect_pass 'wrong start-time PID is not signaled' inventory_stop_port_forward [[ ! -s "$ownership_signal_log" ]] || fail 'wrong start-time process received TERM' kill -0 "$ownership_pid" 2>/dev/null || fail 'wrong start-time process was killed' kill -TERM "$ownership_pid" 2>/dev/null || true for ((attempt=0; attempt<40; attempt++)); do kill -0 "$ownership_pid" 2>/dev/null || break; sleep 0.025; done cli_dir="$fixture_root/cli" mkdir -m 0700 "$cli_dir" expect_reject 'missing CLI arguments rejected' bash "$CAPTURE" expect_reject 'missing execute rejected' bash "$CAPTURE" --phase target-initial --output-dir "$cli_dir" expect_reject 'duplicate argument rejected' bash "$CAPTURE" --execute --phase target-initial --phase target-initial --output-dir "$cli_dir" expect_reject 'unknown phase rejected' bash "$CAPTURE" --execute --phase future-phase --output-dir "$cli_dir" expect_reject 'unknown argument rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$cli_dir" --unknown relative_dir="${cli_dir#/}" expect_reject 'relative output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$relative_dir" noncanonical="$fixture_root/../$(basename "$fixture_root")/cli" expect_reject 'noncanonical output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$noncanonical" chmod 0755 "$cli_dir" expect_reject 'non-0700 output directory rejected' run_capture "$cli_dir" chmod 0700 "$cli_dir" ln -s "$cli_dir" "$fixture_root/cli-link" expect_reject 'symlink output directory rejected' run_capture "$fixture_root/cli-link" mkdir -m 0700 "$fixture_root/unsafe-parent" mkdir -m 0700 "$fixture_root/real-child" ln -s "$fixture_root/unsafe-parent" "$fixture_root/parent-link" mkdir -m 0700 "$fixture_root/unsafe-parent/child" expect_reject 'symlink path component rejected' run_capture "$fixture_root/parent-link/child" printf 'preserve-me\n' >"$cli_dir/preexisting" expect_reject 'nonempty output directory rejected without clobber' run_capture "$cli_dir" [[ "$(cat "$cli_dir/preexisting")" == preserve-me ]] || fail 'preexisting file was modified' rm "$cli_dir/preexisting" empty_series_dir="$fixture_root/empty-series" mkdir -m 0700 "$empty_series_dir" FAKE_CURL_MODE=all_series_empty export FAKE_CURL_MODE expect_reject 'target with metadata descriptors but no actual series rejected' run_capture "$empty_series_dir" unset FAKE_CURL_MODE [[ -z "$(find "$empty_series_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'empty-series response produced accepted output' : >"$metadata_selector_log" : >"$series_selector_log" success_dir="$fixture_root/success" mkdir -m 0700 "$success_dir" expect_pass 'full capture succeeds with fixture APIs' run_capture "$success_dir" cmp -s <(sort "$expected_target_selectors") <(sort "$metadata_selector_log") || fail 'metadata calls did not cover every exact full-label selector once' cmp -s <(sort "$expected_series_selectors") <(sort "$series_selector_log") || fail 'series calls did not cover every exact metric selector once' mapfile -t output_names < <(find "$success_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort) [[ "${output_names[*]}" == 'inventory.json inventory.sha256' ]] || fail 'capture did not create exactly two files' for output_name in "${output_names[@]}"; do [[ -f "$success_dir/$output_name" && ! -L "$success_dir/$output_name" ]] || fail "$output_name is not a regular non-symlink" [[ "$(stat -c %a "$success_dir/$output_name")" == 600 ]] || fail "$output_name mode is not 0600" done [[ "$(jq -cS . "$success_dir/inventory.json")"$'\n' == "$(cat "$success_dir/inventory.json")"$'\n' ]] || fail 'inventory is not canonical jq -cS JSON' (cd "$success_dir" && sha256sum --check inventory.sha256 >/dev/null) || fail 'inventory hash does not bind JSON bytes' jq -e ' .schema == "platform-observability-metric-inventory/v1" and .phase == "target-initial" and .context == "default" and .api_server == "https://127.0.0.1:6443" and .prometheus == "observability/observability-core-kube-pr-prometheus" and .preexisting_verify_only == ["grafana"] and (.targets | length == 21) and ([.targets[].scrape_pool] == ([.targets[].scrape_pool] | sort)) and (all(.targets[]; .health == "up" and .last_error == "" and .metrics == [{"label_names":["__name__","instance","secret_label"],"name":"fixture_metric"}])) and ([.targets[] | select(.scrape_pool=="serviceMonitor/observability/grafana/0") | .provenance] == ["preexisting-verify-only"]) and ([.targets[] | select(.scrape_pool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .provenance] == ["task6-owned"]) ' "$success_dir/inventory.json" >/dev/null || fail 'inventory schema/provenance/content mismatch' if rg -q 'DO-NOT-LEAK-FIXTURE|private-value|help|query|fragment|exemplar' "$success_dir/inventory.json"; then fail 'inventory persisted a prohibited value or field' fi success_pf_pid="$(head -n 1 "$pf_log")" if kill -0 "$success_pf_pid" 2>/dev/null; then fail 'port-forward survived success'; fi oversized_dir="$fixture_root/oversized-response" mkdir -m 0700 "$oversized_dir" FAKE_CURL_MODE=oversized_targets export FAKE_CURL_MODE expect_reject 'Prometheus response above the byte ceiling rejected' run_capture "$oversized_dir" unset FAKE_CURL_MODE [[ -z "$(find "$oversized_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'oversized response produced accepted output' dropped_live_dir="$fixture_root/dropped-live-response" mkdir -m 0700 "$dropped_live_dir" FAKE_CURL_MODE=dropped_blackbox export FAKE_CURL_MODE expect_reject 'live dropped blackbox response rejected with exact active set' run_capture "$dropped_live_dir" unset FAKE_CURL_MODE [[ -z "$(find "$dropped_live_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'dropped blackbox response produced accepted output' sigkill_dir="$fixture_root/sigkill" mkdir -m 0700 "$sigkill_dir" sigkill_marker="$fixture_root/sigkill-series-started" sigkill_pf_lines="$(wc -l <"$pf_log")" sigkill_workdir_lines="$(wc -l <"$pf_workdir_log")" sigkill_curl_lines="$(wc -l <"$curl_pid_log")" env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" PF_WORKDIR_LOG="$PF_WORKDIR_LOG" \ CURL_PID_LOG="$CURL_PID_LOG" CURL_HANG_MARKER="$sigkill_marker" \ TARGETS_FIXTURE="$TARGETS_FIXTURE" OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \ FAKE_CURL_MODE=hang_series bash "$CAPTURE" --execute --phase target-initial --output-dir "$sigkill_dir" \ >/dev/null 2>&1 & sigkill_capture_pid=$! for ((attempt=0; attempt<200; attempt++)); do [[ -e "$sigkill_marker" ]] && break kill -0 "$sigkill_capture_pid" 2>/dev/null || break sleep 0.025 done [[ -e "$sigkill_marker" ]] || fail 'SIGKILL residue fixture did not reach the series request' sigkill_workdir="$(sed -n "$((sigkill_workdir_lines + 1))p" "$pf_workdir_log")" sigkill_pf_pid="$(sed -n "$((sigkill_pf_lines + 1))p" "$pf_log")" sigkill_curl_pid="$(sed -n "$((sigkill_curl_lines + 1))p" "$curl_pid_log")" [[ "$sigkill_workdir" == /tmp/platform-observability-inventory-capture.* && -d "$sigkill_workdir" && ! -L "$sigkill_workdir" ]] || fail 'SIGKILL fixture work directory was not identified safely' kill -KILL "$sigkill_capture_pid" 2>/dev/null || true wait "$sigkill_capture_pid" 2>/dev/null || true for owned_pid in "$sigkill_curl_pid" "$sigkill_pf_pid"; do [[ "$owned_pid" =~ ^[0-9]+$ ]] || fail 'SIGKILL fixture lost an owned child PID' kill -KILL "$owned_pid" 2>/dev/null || true for ((attempt=0; attempt<40; attempt++)); do kill -0 "$owned_pid" 2>/dev/null || break sleep 0.025 done done raw_residue="$(find "$sigkill_workdir" -type f \( -name 'targets*.json' -o -name 'rules*.json' -o -name 'metadata*.json' -o -name 'series*.json' \) -print)" secret_residue=0 if rg -q 'DO-NOT-LEAK-FIXTURE' "$sigkill_workdir"; then secret_residue=1; fi find "$sigkill_workdir" -depth -delete [[ -z "$raw_residue" ]] || fail "SIGKILL left raw Prometheus response files: $raw_residue" [[ "$secret_residue" == 0 ]] || fail 'SIGKILL residue persisted a raw label/help value' pass_count=$((pass_count + 1)) stubborn_pf_dir="$fixture_root/stubborn-port-forward" mkdir -m 0700 "$stubborn_pf_dir" stubborn_lines="$(wc -l <"$pf_log")" set +e timeout --preserve-status --signal=TERM --kill-after=0.5s 5s \ env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" PF_IGNORE_TERM=1 \ bash "$CAPTURE" --execute --phase target-initial --output-dir "$stubborn_pf_dir" >/dev/null 2>&1 stubborn_rc=$? set -e stubborn_pf_pid="$(sed -n "$((stubborn_lines + 1))p" "$pf_log")" [[ "$stubborn_pf_pid" =~ ^[0-9]+$ ]] || fail 'TERM-ignoring fixture did not start a port-forward' stubborn_survived_production_cleanup=0 if kill -0 "$stubborn_pf_pid" 2>/dev/null; then stubborn_survived_production_cleanup=1 kill -KILL "$stubborn_pf_pid" 2>/dev/null || true for ((attempt=0; attempt<40; attempt++)); do kill -0 "$stubborn_pf_pid" 2>/dev/null || break sleep 0.025 done fi [[ "$stubborn_rc" == 0 ]] || fail "TERM-ignoring port-forward cleanup exceeded its bound (status $stubborn_rc)" grep -Fqx "ignored $stubborn_pf_pid TERM" "$pf_log" || fail 'TERM-ignoring fixture did not receive TERM before escalation' [[ "$stubborn_survived_production_cleanup" == 0 ]] || fail 'TERM-ignoring port-forward survived production cleanup' pass_count=$((pass_count + 1)) silent_pf_dir="$fixture_root/silent-port-forward" mkdir -m 0700 "$silent_pf_dir" PF_NO_ANNOUNCEMENT=1 export PF_NO_ANNOUNCEMENT expect_pass 'readiness does not depend on port-forward status text' run_capture "$silent_pf_dir" unset PF_NO_ANNOUNCEMENT publish_failure_dir="$fixture_root/publish-failure" mkdir -m 0700 "$publish_failure_dir" FAIL_SHA_LINK=1 export FAIL_SHA_LINK expect_reject 'second-file publication failure removes only run-owned output' run_capture "$publish_failure_dir" unset FAIL_SHA_LINK [[ -z "$(find "$publish_failure_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'publication failure left a partial inventory' error_dir="$fixture_root/error" mkdir -m 0700 "$error_dir" error_pf_lines="$(wc -l <"$pf_log")" FAKE_CURL_MODE=error export FAKE_CURL_MODE expect_reject 'ordinary API error is sanitized' run_capture "$error_dir" [[ -z "$(find "$error_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'ordinary error left partial output' error_pf_pid="$(sed -n "$((error_pf_lines + 1))p" "$pf_log")" [[ "$error_pf_pid" =~ ^[0-9]+$ ]] || fail 'ordinary error did not start a port-forward' if kill -0 "$error_pf_pid" 2>/dev/null; then fail 'port-forward survived ordinary error'; fi unset FAKE_CURL_MODE for signal in INT TERM HUP; do signal_dir="$fixture_root/signal-${signal,,}" mkdir -m 0700 "$signal_dir" before_lines="$(wc -l <"$pf_log")" set +e timeout --preserve-status --signal="$signal" --kill-after=2s 0.3s \ env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" FAKE_CURL_MODE=hang \ bash "$CAPTURE" --execute --phase target-initial --output-dir "$signal_dir" >/dev/null 2>&1 signal_rc=$? set -e [[ "$signal_rc" != 0 ]] || fail "$signal unexpectedly succeeded" [[ -z "$(find "$signal_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail "$signal left partial output" signal_pf_pid="$(sed -n "$((before_lines + 1))p" "$pf_log")" [[ "$signal_pf_pid" =~ ^[0-9]+$ ]] || fail "$signal did not start a port-forward" if kill -0 "$signal_pf_pid" 2>/dev/null; then fail "port-forward survived $signal"; fi pass_count=$((pass_count + 1)) done if awk ' /kubectl/ { line=$0 if (line ~ / <(apply|delete|patch|replace|exec|rollout|scale)>/) exit 1 if (line ~ / / && line !~ / /) exit 1 } ' "$fake_log"; then :; else fail 'dynamic command trace contains a mutation verb'; fi # Whole-source proof complements the runtime trace. Removing line boundaries # prevents a backslash-newline, command prefix, or wrapper alias from hiding a # mutation verb. The exact RBAC question is the sole create exception. [[ "$(rg -Fxc " 'create pods/portforward'; do" "$CAPTURE")" == 1 ]] || fail 'port-forward RBAC exception is not pinned exactly once' mutation_free_source="$(sed "s/'create pods\\/portforward'/'RBAC_PORTFORWARD'/" "$CAPTURE")" if rg -q '(^|[^[:alnum:]_])(apply|create|delete|patch|replace|exec|rollout|scale)($|[^[:alnum:]_])' <<<"$mutation_free_source"; then fail 'whole-source proof found a Kubernetes mutation verb' fi [[ "$(rg -c '^[[:space:]]*kubectl --request-timeout=' "$CAPTURE")" == 2 ]] || fail 'kubectl invocation count escaped the wrapper/port-forward allowlist' if rg -q '^[[:space:]]*(alias[[:space:]]+[^=]+=.*kubectl|[A-Za-z_][A-Za-z0-9_]*=.*kubectl)' "$CAPTURE"; then fail 'kubectl executable alias bypasses the default-deny wrapper' fi pass_count=$((pass_count + 1)) printf 'PASS: %d target-initial inventory assertions\n' "$pass_count"