donghyeon.kang/platform-core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
main
platform-core/scripts/validate/test-capture-observability-metric-inventory.sh
T

752 lines
40 KiB
Bash
Raw Permalink Blame History

#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly CAPTURE="$ROOT/scripts/validate/capture-observability-metric-inventory.sh"
readonly REJECT_STATUS=23
readonly ORIGINAL_PATH="$PATH"
fail() {
printf 'TEST FAILURE: %s\n' "$*" >&2
exit 1
}
expect_pass() {
local name="$1" output
shift
if ! output="$({ "$@"; } 2>&1)"; then
fail "$name (expected success): $output"
fi
pass_count=$((pass_count + 1))
}
expect_reject() {
local name="$1" output rc=0
shift
output="$({ "$@"; } 2>&1)" || rc=$?
[[ "$rc" == "$REJECT_STATUS" ]] || fail "$name (expected $REJECT_STATUS, got $rc): $output"
[[ "$output" == REJECT:* ]] || fail "$name (unsanitized rejection): $output"
[[ "$output" != *DO-NOT-LEAK-FIXTURE* ]] || fail "$name leaked a fixture value"
pass_count=$((pass_count + 1))
}
[[ -f "$CAPTURE" ]] || fail "EXPECTED RED: capture command is missing: $CAPTURE"
# shellcheck source=/dev/null
source "$CAPTURE"
fixture_root="$(mktemp -d /tmp/platform-observability-inventory-test.XXXXXX)"
test_helper_pids=()
cleanup() {
local rc=$?
local helper_pid
for helper_pid in "${test_helper_pids[@]}"; do
if [[ "$helper_pid" =~ ^[0-9]+$ ]]; then kill -KILL "$helper_pid" 2>/dev/null || true; fi
done
if [[ "${KEEP_INVENTORY_TEST_FIXTURE:-0}" == 1 ]]; then
printf 'KEPT FIXTURE: %s\n' "$fixture_root" >&2
exit "$rc"
fi
case "$fixture_root" in
/tmp/platform-observability-inventory-test.*) rm -rf -- "$fixture_root" ;;
*) printf 'TEST FAILURE: refusing unsafe fixture cleanup\n' >&2; rc=1 ;;
esac
exit "$rc"
}
trap cleanup EXIT
pass_count=0
mapfile -t exact_pools <<'POOLS'
podMonitor/platform-data/platform-postgres/0
serviceMonitor/gitea/gitea/0
serviceMonitor/keycloak/keycloak/0
serviceMonitor/kube-system/traefik/0
serviceMonitor/object-storage/aistor-bucket-usage/0
serviceMonitor/observability-agent/alloy/0
serviceMonitor/observability-agent/node-exporter/0
serviceMonitor/observability/grafana/0
serviceMonitor/observability/loki/0
serviceMonitor/observability/observability-core-kube-pr-alertmanager/0
serviceMonitor/observability/observability-core-kube-pr-alertmanager/1
serviceMonitor/observability/observability-core-kube-pr-apiserver/0
serviceMonitor/observability/observability-core-kube-pr-coredns/0
serviceMonitor/observability/observability-core-kube-pr-kubelet/0
serviceMonitor/observability/observability-core-kube-pr-kubelet/1
serviceMonitor/observability/observability-core-kube-pr-kubelet/2
serviceMonitor/observability/observability-core-kube-pr-operator/0
serviceMonitor/observability/observability-core-kube-pr-prometheus/0
serviceMonitor/observability/observability-core-kube-pr-prometheus/1
serviceMonitor/observability/observability-core-kube-state-metrics/0
serviceMonitor/observability/tempo/0
POOLS
expected_target_selectors="$fixture_root/expected-target-selectors"
expected_series_selectors="$fixture_root/expected-series-selectors"
: >"$expected_target_selectors"
: >"$expected_series_selectors"
make_targets() {
local destination="$1" pool labels selector series_selector index=0
local rows="$fixture_root/target-rows.jsonl"
: >"$rows"
for pool in "${exact_pools[@]}"; do
labels="$(jq -cn --arg instance "10.0.0.$((index + 1)):9090" --arg target "target-$index" \
'{job:"fixture-job",instance:$instance,sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')"
selector="{instance=\"10.0.0.$((index + 1)):9090\",job=\"fixture-job\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",target_fixture=\"target-$index\"}"
if [[ "$pool" == serviceMonitor/object-storage/aistor-bucket-usage/0 ]]; then
labels="$(jq -cn --arg target "target-$index" \
'{job:"minio-aistor",namespace:"object-storage",service:"minio",endpoint:"http-minio",
container:"minio",instance:"10.42.7.19:9000",pod:"minio-aistor-pool-0-0",
sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')"
selector="{container=\"minio\",endpoint=\"http-minio\",instance=\"10.42.7.19:9000\",job=\"minio-aistor\",namespace=\"object-storage\",pod=\"minio-aistor-pool-0-0\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",service=\"minio\",target_fixture=\"target-$index\"}"
fi
printf '%s\n' "$selector" >>"$expected_target_selectors"
series_selector="{__name__=\"fixture_metric\",${selector#\{}"
printf '%s\n' "$series_selector" >>"$expected_series_selectors"
jq -cn \
--arg pool "$pool" \
--arg url "http://10.42.0.$((index + 1)):9090/metrics" \
--argjson labels "$labels" \
'{discoveredLabels:$labels,labels:$labels,scrapePool:$pool,scrapeUrl:$url,
globalUrl:$url,lastError:"",lastScrape:"2026-08-12T00:00:00Z",
lastScrapeDuration:0.01,health:"up",scrapeInterval:"30s",scrapeTimeout:"10s"}' \
>>"$rows"
index=$((index + 1))
done
jq -cs '{status:"success",data:{activeTargets:.,droppedTargets:[]}}' "$rows" >"$destination"
}
valid_targets="$fixture_root/targets.json"
make_targets "$valid_targets"
oversized_padding="$fixture_root/oversized-padding"
head -c 8388609 /dev/zero | tr '\000' x >"$oversized_padding"
oversized_targets="$fixture_root/oversized-targets.json"
jq --rawfile padding "$oversized_padding" '. + {padding:$padding}' "$valid_targets" >"$oversized_targets"
expect_pass 'exact 21 active pools accepted' validate_targets_json "$valid_targets"
post_targets="$fixture_root/post-targets.json"
post_probe_rows="$fixture_root/post-probe-rows.json"
post_substrate_expected_probe_targets | jq '[.[] as $target | $target + {
discoveredLabels:$target.labels,globalUrl:$target.scrapeUrl,lastError:"",
lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01,
health:"up",scrapeInterval:"30s",scrapeTimeout:"15s"}]' >"$post_probe_rows"
jq --slurpfile probes "$post_probe_rows" '
.data.activeTargets += ($probes[0] + [{
discoveredLabels:{},
labels:{job:"blackbox-exporter",namespace:"observability",service:"blackbox-exporter",
endpoint:"http",container:"blackbox-exporter",pod:"blackbox-exporter-6d8444ffdb-svsck",
instance:"10.42.0.123:9115"},
scrapePool:"serviceMonitor/observability/blackbox-exporter/0",
scrapeUrl:"http://10.42.0.123:9115/metrics",globalUrl:"http://10.42.0.123:9115/metrics",
lastError:"",lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01,
health:"up",scrapeInterval:"30s",scrapeTimeout:"10s"
}])
' "$valid_targets" >"$post_targets"
expect_pass 'exact post-substrate 30 targets and multiplicities accepted' \
validate_targets_json "$post_targets" post-substrate
expect_reject 'post-substrate targets rejected by target-initial phase' \
validate_targets_json "$post_targets" target-initial
post_missing_probe="$fixture_root/post-missing-probe.json"
jq 'del(.data.activeTargets[] | select(.scrapePool=="probe/observability/platform-public-edge"))' \
"$post_targets" >"$post_missing_probe"
expect_reject 'post-substrate missing Probe target rejected' \
validate_targets_json "$post_missing_probe" post-substrate
expect_pass 'Probe query is removed from persisted scrape URL' \
test "$(sanitize_scrape_url 'http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fexample.test' strip-query)" = \
'http://blackbox-exporter.observability.svc.cluster.local:9115/probe'
expect_pass 'UTF-8 Prometheus label key is quoted in selector' \
test "$(prometheus_selector_for_labels '{"job":"blackbox","observability.hyeonworks.com/probe-group":"public-edge"}')" = \
'{job="blackbox","observability.hyeonworks.com/probe-group"="public-edge"}'
mutate_and_reject() {
local name="$1" filter="$2" destination
destination="$fixture_root/${name// /-}.json"
jq "$filter" "$valid_targets" >"$destination"
expect_reject "$name" validate_targets_json "$destination"
}
mutate_and_reject 'missing pool rejected' 'del(.data.activeTargets[0])'
mutate_and_reject 'extra pool rejected' '.data.activeTargets += [.data.activeTargets[0] | .scrapePool="serviceMonitor/extra/extra/0"]'
mutate_and_reject 'duplicate pool rejected' '.data.activeTargets += [.data.activeTargets[0]]'
mutate_and_reject 'down target rejected' '.data.activeTargets[0].health="down"'
mutate_and_reject 'last error rejected' '.data.activeTargets[0].lastError="DO-NOT-LEAK-FIXTURE"'
mutate_and_reject 'blackbox pool rejected' '.data.activeTargets[0].scrapePool="serviceMonitor/observability/blackbox-exporter/0"'
mutate_and_reject 'Probe pool rejected' '.data.activeTargets[0].scrapePool="probe/observability/public-edge/0"'
dropped_blackbox="$fixture_root/dropped-blackbox.json"
jq '.data.droppedTargets = [{
"discoveredLabels": {
"__meta_kubernetes_namespace": "observability",
"__meta_kubernetes_service_name": "blackbox-exporter",
"job": "blackbox-exporter"
},
"labels": {"job": "blackbox-exporter"},
"scrapePool": "serviceMonitor/observability/blackbox-exporter/0",
"scrapeUrl": "http://blackbox-exporter.observability.svc:9115/metrics",
"globalUrl": "http://blackbox-exporter.observability.svc:9115/metrics",
"lastError": "",
"health": "unknown"
}]' "$valid_targets" >"$dropped_blackbox"
expect_reject 'dropped blackbox identity rejected' validate_targets_json "$dropped_blackbox"
expect_pass 'post-substrate allows blackbox self discovery drops' \
validate_dropped_targets_json "$dropped_blackbox" post-substrate
dropped_probe="$fixture_root/dropped-probe.json"
jq '.data.droppedTargets = [{
"discoveredLabels": {
"__meta_kubernetes_namespace": "observability",
"__meta_kubernetes_service_name": "synthetic-exporter",
"monitor_kind": "Probe"
},
"labels": {"job": "synthetic-checks"},
"scrapePool": "probe/observability/public-edge/0",
"scrapeUrl": "http://synthetic-exporter.observability.svc:9115/metrics",
"globalUrl": "http://synthetic-exporter.observability.svc:9115/metrics",
"lastError": "",
"health": "unknown"
}]' "$valid_targets" >"$dropped_probe"
expect_reject 'dropped Probe identity rejected' validate_targets_json "$dropped_probe"
legitimate_probe_word="$fixture_root/legitimate-probe-word.json"
jq '.data.droppedTargets = [{
"discoveredLabels": {
"__meta_kubernetes_pod_annotation_fixture": "{\"livenessProbe\":true}",
"__metrics_path__": "/metrics/probes"
},
"labels": {"job": "platform-postgres"},
"scrapePool": "podMonitor/platform-data/platform-postgres/0",
"scrapeUrl": "http://10.42.0.1:9187/metrics/probes",
"globalUrl": "http://10.42.0.1:9187/metrics/probes",
"lastError": "",
"health": "unknown"
}]' "$valid_targets" >"$legitimate_probe_word"
expect_pass 'legitimate dropped metadata containing probe words accepted in active response' \
validate_targets_json "$legitimate_probe_word"
expect_pass 'legitimate dropped metadata containing probe words accepted in dropped response' \
validate_dropped_targets_json "$legitimate_probe_word"
empty_dropped_targets="$fixture_root/empty-dropped-targets.json"
printf '%s\n' '{"status":"success","data":{"activeTargets":[],"droppedTargets":[]}}' >"$empty_dropped_targets"
mutate_and_reject 'AIStor wrong job rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.job)="wrong"'
mutate_and_reject 'AIStor wrong namespace rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.namespace)="wrong"'
mutate_and_reject 'AIStor wrong service rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.service)="wrong"'
mutate_and_reject 'AIStor wrong endpoint rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.endpoint)="wrong"'
mutate_and_reject 'AIStor wrong container rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.container)="wrong"'
expect_pass 'safe HTTP URL accepted' test "$(sanitize_scrape_url 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets')" = 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets'
expect_pass 'safe HTTPS URL accepted' test "$(sanitize_scrape_url 'https://metrics.example.test/metrics')" = 'https://metrics.example.test/metrics'
for unsafe_url in \
'http://user:DO-NOT-LEAK-FIXTURE@host/metrics' \
'http://host/metrics?token=DO-NOT-LEAK-FIXTURE' \
'http://host/metrics#DO-NOT-LEAK-FIXTURE' \
'ftp://host/metrics' \
'http://host/space here'; do
expect_reject 'unsafe scrape URL rejected' sanitize_scrape_url "$unsafe_url"
done
metadata="$fixture_root/metadata.json"
printf '%s\n' '{"status":"success","data":{"z_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}],"a_metric":[{"type":"counter","help":"ignored","unit":""}]}}' >"$metadata"
expect_pass 'metadata metric names sorted' test "$(extract_metric_names "$metadata")" = $'a_metric\nz_metric'
metadata_array="$fixture_root/metadata-array.json"
printf '%s\n' '{"status":"success","data":[{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""},{"target":{"job":"fixture"},"metric":"a_metric","type":"counter","help":"ignored","unit":""},{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"duplicate","unit":""}]}' >"$metadata_array"
expect_pass 'current Prometheus metadata array yields unique sorted metric names' \
test "$(extract_metric_names "$metadata_array")" = $'a_metric\nz_metric'
bad_metadata_array="$fixture_root/bad-metadata-array.json"
printf '%s\n' '{"status":"success","data":[{"target":{},"metric":"bad metric","type":"gauge","help":"ignored","unit":""}]}' >"$bad_metadata_array"
expect_reject 'invalid current metadata array metric name rejected' extract_metric_names "$bad_metadata_array"
expect_pass 'post-substrate Probe pool accepts probe metrics' \
validate_metric_names_for_pool $'probe_duration_seconds\nprobe_success' \
probe/observability/platform-public-edge post-substrate
expect_reject 'target-initial rejects probe metrics' \
validate_metric_names_for_pool probe_success serviceMonitor/observability/grafana/0 target-initial
expect_reject 'post-substrate non-Probe pool rejects probe metrics' \
validate_metric_names_for_pool probe_success serviceMonitor/observability/blackbox-exporter/0 post-substrate
expect_reject 'post-substrate Probe pool requires probe_success descriptor' \
validate_metric_names_for_pool probe_duration_seconds probe/observability/platform-public-edge post-substrate
bad_metadata="$fixture_root/bad-metadata.json"
printf '%s\n' '{"status":"success","data":{"bad metric":[]}}' >"$bad_metadata"
expect_reject 'invalid metadata metric name rejected' extract_metric_names "$bad_metadata"
rules="$fixture_root/rules.json"
printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"z_metric","query":"DO-NOT-LEAK-FIXTURE"},{"type":"alerting","name":"a_metric"}]}]}}' >"$rules"
expect_reject 'recording-rule collision rejected' reject_recording_rule_collisions "$rules" "$metadata"
printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"recorded_metric"}]}]}}' >"$rules"
expect_pass 'non-colliding recording rule accepted' reject_recording_rule_collisions "$rules" "$metadata"
series="$fixture_root/series.json"
printf '%s\n' '{"status":"success","data":[{"z_label":"DO-NOT-LEAK-FIXTURE","__name__":"a_metric","a_label":"private-value"},{"__name__":"a_metric","middle":"another-private-value"}]}' >"$series"
expect_pass 'series parser emits only sorted label keys' test "$(extract_series_label_names "$series")" = '["__name__","a_label","middle","z_label"]'
[[ "$(extract_series_label_names "$series")" != *private-value* ]] || fail 'series parser persisted label values'
dotted_series="$fixture_root/dotted-series.json"
printf '%s\n' '{"status":"success","data":[{"__name__":"probe_success","instance":"https://example.test/","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]}' >"$dotted_series"
expect_pass 'series parser preserves safe quoted Prometheus label names' \
test "$(extract_series_label_names "$dotted_series")" = \
'["__name__","instance","job","namespace","observability.hyeonworks.com/probe-group"]'
empty_probe_series="$fixture_root/empty-probe-series.json"
printf '%s\n' '{"status":"success","data":[]}' >"$empty_probe_series"
expect_pass 'target-initial requires probe series absence' \
validate_probe_series_phase "$empty_probe_series" target-initial
expect_reject 'post-substrate rejects probe series absence' \
validate_probe_series_phase "$empty_probe_series" post-substrate
expect_pass 'post-substrate accepts observability probe series' \
validate_probe_series_phase "$dotted_series" post-substrate
expect_reject 'target-initial rejects existing probe series' \
validate_probe_series_phase "$dotted_series" target-initial
post_labels="$fixture_root/post-labels.json"
post_substrate_expected_probe_targets | jq '[.[].labels]' >"$post_labels"
for query_kind in success status duration tls; do
query_fixture="$fixture_root/post-query-$query_kind.json"
jq -cn --slurpfile labels "$post_labels" --arg kind "$query_kind" '
($labels[0] | if $kind=="tls" then map(select(.instance|startswith("https://"))) else . end) as $selected |
{status:"success",data:{resultType:"vector",result:[$selected[] |
{metric:(. + {__name__:(if $kind=="success" then "probe_success"
elif $kind=="status" then "probe_http_status_code"
elif $kind=="duration" then "probe_duration_seconds"
else "probe_ssl_earliest_cert_expiry" end)}),
value:[1770000000,(if $kind=="success" then "1"
elif $kind=="status" then (if .job=="blackbox-private-edge" then "403" else "200" end)
elif $kind=="duration" then "0.1" else "4102444800" end)]}]}}
' >"$query_fixture"
expect_pass "post-substrate $query_kind query contract accepted" \
validate_post_substrate_query_body "$query_fixture" "$query_kind"
done
large_target_rows="$fixture_root/large-target-rows.jsonl"
awk 'BEGIN {
for (i = 0; i < 20000; i++)
printf "{\"scrape_pool\":\"pool/%d\",\"scrape_url\":\"http://10.42.0.1:9090/metrics\",\"health\":\"up\",\"last_error\":\"\",\"provenance\":\"core\",\"metrics\":[{\"name\":\"fixture_metric\",\"label_names\":[\"__name__\",\"instance\"]}]}\n", i
}' >"$large_target_rows"
large_inventory_contract() {
[[ "$(stat -c %s "$large_target_rows")" -gt 2097152 ]] || return 1
build_inventory_json "$large_target_rows" target-initial 2026-08-12T00:00:00Z |
jq -e '.phase=="target-initial" and (.targets|length)==20000' >/dev/null
}
expect_pass 'inventory builder reads target rows from file beyond argv size limit' large_inventory_contract
fake_bin="$fixture_root/bin"
mkdir -p "$fake_bin"
fake_log="$fixture_root/fake-commands.log"
pf_log="$fixture_root/port-forward-pids.log"
: >"$fake_log"
: >"$pf_log"
cat >"$fake_bin/kubectl" <<'FAKE_KUBECTL'
#!/usr/bin/env bash
set -Eeuo pipefail
printf 'kubectl' >>"$FAKE_LOG"
printf ' <%s>' "$@" >>"$FAKE_LOG"
printf '\n' >>"$FAKE_LOG"
args=" $* "
if [[ "$args" =~ [[:space:]](apply|delete|patch|replace|exec|rollout|scale)[[:space:]] ]]; then
exit 91
fi
if [[ "$args" == *' config current-context '* ]]; then printf 'default\n'; exit 0; fi
if [[ "$args" == *' config view '* ]]; then printf '%s\n' '{"clusters":[{"cluster":{"server":"https://127.0.0.1:6443"}}]}'; exit 0; fi
if [[ "$args" == *' get --raw=/readyz '* ]]; then printf 'ok\n'; exit 0; fi
if [[ "$args" == *' auth can-i '* ]]; then printf 'yes\n'; exit 0; fi
if [[ "$args" == *' get prometheus '* ]]; then printf '%s\n' '{"apiVersion":"monitoring.coreos.com/v1","kind":"Prometheus","metadata":{"name":"observability-core-kube-pr-prometheus","namespace":"observability"}}'; exit 0; fi
if [[ "$args" == *' get pods '* ]]; then printf '%s\n' '{"items":[{"metadata":{"name":"prometheus-observability-core-kube-pr-prometheus-0","namespace":"observability"},"status":{"phase":"Running","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"prometheus","ready":true}]}}]}'; exit 0; fi
if [[ "$args" == *' port-forward '* ]]; then
printf '%s\n' "$$" >>"$PF_LOG"
pf_stderr_path="$(readlink -- "/proc/$$/fd/2")"
printf '%s\n' "${pf_stderr_path%/*}" >>"$PF_WORKDIR_LOG"
if [[ "${PF_IGNORE_TERM:-0}" == 1 ]]; then
trap 'printf "ignored %s TERM\n" "$$" >>"$PF_LOG"' TERM
else
trap 'printf "stopped %s TERM\n" "$$" >>"$PF_LOG"; exit 0' TERM
fi
trap 'printf "stopped %s INT\n" "$$" >>"$PF_LOG"; exit 0' INT
trap 'printf "stopped %s HUP\n" "$$" >>"$PF_LOG"; exit 0' HUP
if [[ "${PF_NO_ANNOUNCEMENT:-0}" != 1 ]]; then
printf 'Forwarding from 127.0.0.1:43123 -> 9090\n' >&2
fi
while :; do /usr/bin/sleep 0.05; done
fi
exit 1
FAKE_KUBECTL
cat >"$fake_bin/curl" <<'FAKE_CURL'
#!/usr/bin/env bash
set -Eeuo pipefail
[[ "${1-}" == --disable ]] || exit 92
printf 'curl' >>"$FAKE_LOG"
printf ' <%s>' "$@" >>"$FAKE_LOG"
printf '\n' >>"$FAKE_LOG"
destination=''
url=''
max_filesize=''
all_arguments=" $* "
declare -a data_values=()
declare -a urlencode_values=()
while (($#)); do
case "$1" in
--output) destination="$2"; shift 2 ;;
--max-filesize) max_filesize="$2"; shift 2 ;;
--data) data_values+=("$2"); shift 2 ;;
--data-urlencode) urlencode_values+=("$2"); shift 2 ;;
http://*) url="$1"; shift ;;
*) shift ;;
esac
done
[[ -n "$destination" && -n "$url" ]] || exit 2
emit_text() {
if [[ "$destination" == - ]]; then printf '%s\n' "$1"; else printf '%s\n' "$1" >"$destination"; fi
}
emit_file() {
if [[ "$destination" == - ]]; then cat -- "$1"; else cp -- "$1" "$destination"; fi
}
case "$url" in
*/-/ready) emit_text 'ready' ;;
*/api/v1/targets)
[[ "${#data_values[@]}" == 1 ]] || exit 80
case "${data_values[0]}" in
state=dropped)
if [[ "${FAKE_CURL_MODE:-success}" == dropped_blackbox ]]; then
emit_file "$DROPPED_BLACKBOX_FIXTURE"
else
emit_file "$DROPPED_TARGETS_FIXTURE"
fi
exit 0
;;
state=active) ;;
*) exit 81 ;;
esac
if [[ "${FAKE_CURL_MODE:-success}" == hang ]]; then
trap 'exit 128' INT TERM HUP
while :; do /usr/bin/sleep 0.05; done
fi
if [[ "${FAKE_CURL_MODE:-success}" == error ]]; then exit 22; fi
if [[ "${FAKE_CURL_MODE:-success}" == oversized_targets ]]; then
[[ "$max_filesize" =~ ^[0-9]+$ ]] || exit 88
if (( $(stat -c %s "$OVERSIZED_TARGETS_FIXTURE") > max_filesize )); then exit 63; fi
emit_file "$OVERSIZED_TARGETS_FIXTURE"
else
emit_file "$TARGETS_FIXTURE"
fi
;;
*/api/v1/rules) emit_text '{"status":"success","data":{"groups":[]}}' ;;
*/api/v1/targets/metadata)
[[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match_target=* ]] || exit 84
[[ "${#data_values[@]}" == 1 && "${data_values[0]}" == 'limit=10000' ]] || exit 85
target_selector="${urlencode_values[0]#match_target=}"
grep -Fqx -- "$target_selector" "$EXPECTED_TARGET_SELECTORS" || exit 86
printf '%s\n' "$target_selector" >>"$METADATA_SELECTOR_LOG"
emit_text '{"status":"success","data":{"descriptor_only_metric":[{"type":"gauge","help":"registered without a current series","unit":""}],"fixture_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}]}}'
;;
*/api/v1/series)
if [[ "$all_arguments" == *'probe_.+'* ]]; then
[[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == 'match[]={__name__=~"probe_.+"}' ]] || exit 87
emit_text '{"status":"success","data":[]}'
else
[[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match\[\]=* ]] || exit 89
series_selector="${urlencode_values[0]#match[]=}"
if [[ "$series_selector" == *'__name__="descriptor_only_metric"'* ]]; then
fixture_selector="${series_selector/descriptor_only_metric/fixture_metric}"
grep -Fqx -- "$fixture_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90
emit_text '{"status":"success","data":[]}'
exit 0
fi
grep -Fqx -- "$series_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90
if [[ "${FAKE_CURL_MODE:-success}" == all_series_empty ]]; then
emit_text '{"status":"success","data":[]}'
exit 0
fi
printf '%s\n' "$series_selector" >>"$SERIES_SELECTOR_LOG"
if [[ "${FAKE_CURL_MODE:-success}" == hang_series ]]; then
printf '%s\n' "$$" >>"$CURL_PID_LOG"
: >"$CURL_HANG_MARKER"
trap 'exit 128' INT TERM HUP
while :; do /usr/bin/sleep 0.05; done
fi
emit_text '{"status":"success","data":[{"__name__":"fixture_metric","instance":"10.42.0.1:9090","secret_label":"DO-NOT-LEAK-FIXTURE"}]}'
fi
;;
*) exit 3 ;;
esac
FAKE_CURL
cat >"$fake_bin/ln" <<'FAKE_LN'
#!/usr/bin/env bash
set -Eeuo pipefail
if [[ "${FAIL_SHA_LINK:-0}" == 1 && "${*: -1}" == */inventory.sha256 ]]; then
exit 1
fi
exec /usr/bin/ln "$@"
FAKE_LN
chmod 0700 "$fake_bin/kubectl" "$fake_bin/curl" "$fake_bin/ln"
export PATH="$fake_bin:$ORIGINAL_PATH"
pf_workdir_log="$fixture_root/port-forward-workdirs.log"
curl_pid_log="$fixture_root/curl-pids.log"
metadata_selector_log="$fixture_root/metadata-selectors.log"
series_selector_log="$fixture_root/series-selectors.log"
: >"$pf_workdir_log"
: >"$curl_pid_log"
: >"$metadata_selector_log"
: >"$series_selector_log"
export FAKE_LOG="$fake_log" PF_LOG="$pf_log" PF_WORKDIR_LOG="$pf_workdir_log"
export CURL_PID_LOG="$curl_pid_log" TARGETS_FIXTURE="$valid_targets" OVERSIZED_TARGETS_FIXTURE="$oversized_targets"
export DROPPED_TARGETS_FIXTURE="$empty_dropped_targets" DROPPED_BLACKBOX_FIXTURE="$dropped_blackbox"
export EXPECTED_TARGET_SELECTORS="$expected_target_selectors" EXPECTED_SERIES_SELECTORS="$expected_series_selectors"
export METADATA_SELECTOR_LOG="$metadata_selector_log" SERIES_SELECTOR_LOG="$series_selector_log"
run_capture() {
local output_dir="$1"
shift
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" \
PF_WORKDIR_LOG="$PF_WORKDIR_LOG" CURL_PID_LOG="$CURL_PID_LOG" \
EXPECTED_TARGET_SELECTORS="$EXPECTED_TARGET_SELECTORS" EXPECTED_SERIES_SELECTORS="$EXPECTED_SERIES_SELECTORS" \
METADATA_SELECTOR_LOG="$METADATA_SELECTOR_LOG" SERIES_SELECTOR_LOG="$SERIES_SELECTOR_LOG" \
DROPPED_TARGETS_FIXTURE="$DROPPED_TARGETS_FIXTURE" DROPPED_BLACKBOX_FIXTURE="$DROPPED_BLACKBOX_FIXTURE" \
OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \
FAKE_CURL_MODE="${FAKE_CURL_MODE:-success}" \
bash "$CAPTURE" --execute --phase target-initial --output-dir "$output_dir" "$@"
}
inventory_kubectl_alias() { inventory_kubectl "$@"; }
mutation_log_lines="$(wc -l <"$fake_log")"
for wrapper_name in inventory_kubectl inventory_kubectl_alias; do
for mutation_verb in apply create delete patch replace exec rollout scale; do
expect_reject "$wrapper_name default-denies $mutation_verb" \
"$wrapper_name" "$mutation_verb" fixture
done
done
[[ "$(wc -l <"$fake_log")" == "$mutation_log_lines" ]] || fail 'denied wrapper mutation reached kubectl'
ownership_signal_log="$fixture_root/ownership-signals.log"
ownership_ready="$fixture_root/ownership-ready"
bash -c '
trap '\''printf "TERM\n" >>"$1"; exit 0'\'' TERM
: >"$2"
while :; do /usr/bin/sleep 0.05; done
' bash "$ownership_signal_log" "$ownership_ready" &
ownership_pid=$!
test_helper_pids+=("$ownership_pid")
for ((attempt=0; attempt<40; attempt++)); do [[ -e "$ownership_ready" ]] && break; sleep 0.025; done
[[ -e "$ownership_ready" ]] || fail 'ownership fixture did not start'
ownership_snapshot="$(inventory_process_snapshot "$ownership_pid")" || fail 'ownership fixture snapshot failed'
INVENTORY_PORT_FORWARD_PID="$ownership_pid"
INVENTORY_PORT_FORWARD_START="$((10#${ownership_snapshot#*:} + 1))"
expect_pass 'wrong start-time PID is not signaled' inventory_stop_port_forward
[[ ! -s "$ownership_signal_log" ]] || fail 'wrong start-time process received TERM'
kill -0 "$ownership_pid" 2>/dev/null || fail 'wrong start-time process was killed'
kill -TERM "$ownership_pid" 2>/dev/null || true
for ((attempt=0; attempt<40; attempt++)); do kill -0 "$ownership_pid" 2>/dev/null || break; sleep 0.025; done
cli_dir="$fixture_root/cli"
mkdir -m 0700 "$cli_dir"
expect_reject 'missing CLI arguments rejected' bash "$CAPTURE"
expect_reject 'missing execute rejected' bash "$CAPTURE" --phase target-initial --output-dir "$cli_dir"
expect_reject 'duplicate argument rejected' bash "$CAPTURE" --execute --phase target-initial --phase target-initial --output-dir "$cli_dir"
expect_reject 'unknown phase rejected' bash "$CAPTURE" --execute --phase future-phase --output-dir "$cli_dir"
expect_reject 'unknown argument rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$cli_dir" --unknown
relative_dir="${cli_dir#/}"
expect_reject 'relative output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$relative_dir"
noncanonical="$fixture_root/../$(basename "$fixture_root")/cli"
expect_reject 'noncanonical output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$noncanonical"
chmod 0755 "$cli_dir"
expect_reject 'non-0700 output directory rejected' run_capture "$cli_dir"
chmod 0700 "$cli_dir"
ln -s "$cli_dir" "$fixture_root/cli-link"
expect_reject 'symlink output directory rejected' run_capture "$fixture_root/cli-link"
mkdir -m 0700 "$fixture_root/unsafe-parent"
mkdir -m 0700 "$fixture_root/real-child"
ln -s "$fixture_root/unsafe-parent" "$fixture_root/parent-link"
mkdir -m 0700 "$fixture_root/unsafe-parent/child"
expect_reject 'symlink path component rejected' run_capture "$fixture_root/parent-link/child"
printf 'preserve-me\n' >"$cli_dir/preexisting"
expect_reject 'nonempty output directory rejected without clobber' run_capture "$cli_dir"
[[ "$(cat "$cli_dir/preexisting")" == preserve-me ]] || fail 'preexisting file was modified'
rm "$cli_dir/preexisting"
empty_series_dir="$fixture_root/empty-series"
mkdir -m 0700 "$empty_series_dir"
FAKE_CURL_MODE=all_series_empty
export FAKE_CURL_MODE
expect_reject 'target with metadata descriptors but no actual series rejected' run_capture "$empty_series_dir"
unset FAKE_CURL_MODE
[[ -z "$(find "$empty_series_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'empty-series response produced accepted output'
: >"$metadata_selector_log"
: >"$series_selector_log"
success_dir="$fixture_root/success"
mkdir -m 0700 "$success_dir"
expect_pass 'full capture succeeds with fixture APIs' run_capture "$success_dir"
cmp -s <(sort "$expected_target_selectors") <(sort "$metadata_selector_log") || fail 'metadata calls did not cover every exact full-label selector once'
cmp -s <(sort "$expected_series_selectors") <(sort "$series_selector_log") || fail 'series calls did not cover every exact metric selector once'
mapfile -t output_names < <(find "$success_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)
[[ "${output_names[*]}" == 'inventory.json inventory.sha256' ]] || fail 'capture did not create exactly two files'
for output_name in "${output_names[@]}"; do
[[ -f "$success_dir/$output_name" && ! -L "$success_dir/$output_name" ]] || fail "$output_name is not a regular non-symlink"
[[ "$(stat -c %a "$success_dir/$output_name")" == 600 ]] || fail "$output_name mode is not 0600"
done
[[ "$(jq -cS . "$success_dir/inventory.json")"$'\n' == "$(cat "$success_dir/inventory.json")"$'\n' ]] || fail 'inventory is not canonical jq -cS JSON'
(cd "$success_dir" && sha256sum --check inventory.sha256 >/dev/null) || fail 'inventory hash does not bind JSON bytes'
jq -e '
.schema == "platform-observability-metric-inventory/v1" and
.phase == "target-initial" and .context == "default" and
.api_server == "https://127.0.0.1:6443" and
.prometheus == "observability/observability-core-kube-pr-prometheus" and
.preexisting_verify_only == ["grafana"] and
(.targets | length == 21) and
([.targets[].scrape_pool] == ([.targets[].scrape_pool] | sort)) and
(all(.targets[]; .health == "up" and .last_error == "" and
.metrics == [{"label_names":["__name__","instance","secret_label"],"name":"fixture_metric"}])) and
([.targets[] | select(.scrape_pool=="serviceMonitor/observability/grafana/0") | .provenance] == ["preexisting-verify-only"]) and
([.targets[] | select(.scrape_pool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .provenance] == ["task6-owned"])
' "$success_dir/inventory.json" >/dev/null || fail 'inventory schema/provenance/content mismatch'
if rg -q 'DO-NOT-LEAK-FIXTURE|private-value|help|query|fragment|exemplar' "$success_dir/inventory.json"; then
fail 'inventory persisted a prohibited value or field'
fi
success_pf_pid="$(head -n 1 "$pf_log")"
if kill -0 "$success_pf_pid" 2>/dev/null; then fail 'port-forward survived success'; fi
oversized_dir="$fixture_root/oversized-response"
mkdir -m 0700 "$oversized_dir"
FAKE_CURL_MODE=oversized_targets
export FAKE_CURL_MODE
expect_reject 'Prometheus response above the byte ceiling rejected' run_capture "$oversized_dir"
unset FAKE_CURL_MODE
[[ -z "$(find "$oversized_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'oversized response produced accepted output'
dropped_live_dir="$fixture_root/dropped-live-response"
mkdir -m 0700 "$dropped_live_dir"
FAKE_CURL_MODE=dropped_blackbox
export FAKE_CURL_MODE
expect_reject 'live dropped blackbox response rejected with exact active set' run_capture "$dropped_live_dir"
unset FAKE_CURL_MODE
[[ -z "$(find "$dropped_live_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'dropped blackbox response produced accepted output'
sigkill_dir="$fixture_root/sigkill"
mkdir -m 0700 "$sigkill_dir"
sigkill_marker="$fixture_root/sigkill-series-started"
sigkill_pf_lines="$(wc -l <"$pf_log")"
sigkill_workdir_lines="$(wc -l <"$pf_workdir_log")"
sigkill_curl_lines="$(wc -l <"$curl_pid_log")"
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" PF_WORKDIR_LOG="$PF_WORKDIR_LOG" \
CURL_PID_LOG="$CURL_PID_LOG" CURL_HANG_MARKER="$sigkill_marker" \
TARGETS_FIXTURE="$TARGETS_FIXTURE" OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \
FAKE_CURL_MODE=hang_series bash "$CAPTURE" --execute --phase target-initial --output-dir "$sigkill_dir" \
>/dev/null 2>&1 &
sigkill_capture_pid=$!
for ((attempt=0; attempt<200; attempt++)); do
[[ -e "$sigkill_marker" ]] && break
kill -0 "$sigkill_capture_pid" 2>/dev/null || break
sleep 0.025
done
[[ -e "$sigkill_marker" ]] || fail 'SIGKILL residue fixture did not reach the series request'
sigkill_workdir="$(sed -n "$((sigkill_workdir_lines + 1))p" "$pf_workdir_log")"
sigkill_pf_pid="$(sed -n "$((sigkill_pf_lines + 1))p" "$pf_log")"
sigkill_curl_pid="$(sed -n "$((sigkill_curl_lines + 1))p" "$curl_pid_log")"
[[ "$sigkill_workdir" == /tmp/platform-observability-inventory-capture.* && -d "$sigkill_workdir" && ! -L "$sigkill_workdir" ]] || fail 'SIGKILL fixture work directory was not identified safely'
kill -KILL "$sigkill_capture_pid" 2>/dev/null || true
wait "$sigkill_capture_pid" 2>/dev/null || true
for owned_pid in "$sigkill_curl_pid" "$sigkill_pf_pid"; do
[[ "$owned_pid" =~ ^[0-9]+$ ]] || fail 'SIGKILL fixture lost an owned child PID'
kill -KILL "$owned_pid" 2>/dev/null || true
for ((attempt=0; attempt<40; attempt++)); do
kill -0 "$owned_pid" 2>/dev/null || break
sleep 0.025
done
done
raw_residue="$(find "$sigkill_workdir" -type f \( -name 'targets*.json' -o -name 'rules*.json' -o -name 'metadata*.json' -o -name 'series*.json' \) -print)"
secret_residue=0
if rg -q 'DO-NOT-LEAK-FIXTURE' "$sigkill_workdir"; then secret_residue=1; fi
find "$sigkill_workdir" -depth -delete
[[ -z "$raw_residue" ]] || fail "SIGKILL left raw Prometheus response files: $raw_residue"
[[ "$secret_residue" == 0 ]] || fail 'SIGKILL residue persisted a raw label/help value'
pass_count=$((pass_count + 1))
stubborn_pf_dir="$fixture_root/stubborn-port-forward"
mkdir -m 0700 "$stubborn_pf_dir"
stubborn_lines="$(wc -l <"$pf_log")"
set +e
timeout --preserve-status --signal=TERM --kill-after=0.5s 5s \
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" PF_IGNORE_TERM=1 \
bash "$CAPTURE" --execute --phase target-initial --output-dir "$stubborn_pf_dir" >/dev/null 2>&1
stubborn_rc=$?
set -e
stubborn_pf_pid="$(sed -n "$((stubborn_lines + 1))p" "$pf_log")"
[[ "$stubborn_pf_pid" =~ ^[0-9]+$ ]] || fail 'TERM-ignoring fixture did not start a port-forward'
stubborn_survived_production_cleanup=0
if kill -0 "$stubborn_pf_pid" 2>/dev/null; then
stubborn_survived_production_cleanup=1
kill -KILL "$stubborn_pf_pid" 2>/dev/null || true
for ((attempt=0; attempt<40; attempt++)); do
kill -0 "$stubborn_pf_pid" 2>/dev/null || break
sleep 0.025
done
fi
[[ "$stubborn_rc" == 0 ]] || fail "TERM-ignoring port-forward cleanup exceeded its bound (status $stubborn_rc)"
grep -Fqx "ignored $stubborn_pf_pid TERM" "$pf_log" || fail 'TERM-ignoring fixture did not receive TERM before escalation'
[[ "$stubborn_survived_production_cleanup" == 0 ]] || fail 'TERM-ignoring port-forward survived production cleanup'
pass_count=$((pass_count + 1))
silent_pf_dir="$fixture_root/silent-port-forward"
mkdir -m 0700 "$silent_pf_dir"
PF_NO_ANNOUNCEMENT=1
export PF_NO_ANNOUNCEMENT
expect_pass 'readiness does not depend on port-forward status text' run_capture "$silent_pf_dir"
unset PF_NO_ANNOUNCEMENT
publish_failure_dir="$fixture_root/publish-failure"
mkdir -m 0700 "$publish_failure_dir"
FAIL_SHA_LINK=1
export FAIL_SHA_LINK
expect_reject 'second-file publication failure removes only run-owned output' run_capture "$publish_failure_dir"
unset FAIL_SHA_LINK
[[ -z "$(find "$publish_failure_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'publication failure left a partial inventory'
error_dir="$fixture_root/error"
mkdir -m 0700 "$error_dir"
error_pf_lines="$(wc -l <"$pf_log")"
FAKE_CURL_MODE=error
export FAKE_CURL_MODE
expect_reject 'ordinary API error is sanitized' run_capture "$error_dir"
[[ -z "$(find "$error_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'ordinary error left partial output'
error_pf_pid="$(sed -n "$((error_pf_lines + 1))p" "$pf_log")"
[[ "$error_pf_pid" =~ ^[0-9]+$ ]] || fail 'ordinary error did not start a port-forward'
if kill -0 "$error_pf_pid" 2>/dev/null; then fail 'port-forward survived ordinary error'; fi
unset FAKE_CURL_MODE
for signal in INT TERM HUP; do
signal_dir="$fixture_root/signal-${signal,,}"
mkdir -m 0700 "$signal_dir"
before_lines="$(wc -l <"$pf_log")"
set +e
timeout --preserve-status --signal="$signal" --kill-after=2s 0.3s \
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" FAKE_CURL_MODE=hang \
bash "$CAPTURE" --execute --phase target-initial --output-dir "$signal_dir" >/dev/null 2>&1
signal_rc=$?
set -e
[[ "$signal_rc" != 0 ]] || fail "$signal unexpectedly succeeded"
[[ -z "$(find "$signal_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail "$signal left partial output"
signal_pf_pid="$(sed -n "$((before_lines + 1))p" "$pf_log")"
[[ "$signal_pf_pid" =~ ^[0-9]+$ ]] || fail "$signal did not start a port-forward"
if kill -0 "$signal_pf_pid" 2>/dev/null; then fail "port-forward survived $signal"; fi
pass_count=$((pass_count + 1))
done
if awk '
/kubectl/ {
line=$0
if (line ~ / <(apply|delete|patch|replace|exec|rollout|scale)>/) exit 1
if (line ~ / <create>/ && line !~ / <auth> <can-i> <create> <pods\/portforward>/) exit 1
}
' "$fake_log"; then :; else fail 'dynamic command trace contains a mutation verb'; fi
# Whole-source proof complements the runtime trace. Removing line boundaries
# prevents a backslash-newline, command prefix, or wrapper alias from hiding a
# mutation verb. The exact RBAC question is the sole create exception.
[[ "$(rg -Fxc " 'create pods/portforward'; do" "$CAPTURE")" == 1 ]] || fail 'port-forward RBAC exception is not pinned exactly once'
mutation_free_source="$(sed "s/'create pods\\/portforward'/'RBAC_PORTFORWARD'/" "$CAPTURE")"
if rg -q '(^|[^[:alnum:]_])(apply|create|delete|patch|replace|exec|rollout|scale)($|[^[:alnum:]_])' <<<"$mutation_free_source"; then
fail 'whole-source proof found a Kubernetes mutation verb'
fi
[[ "$(rg -c '^[[:space:]]*kubectl --request-timeout=' "$CAPTURE")" == 2 ]] || fail 'kubectl invocation count escaped the wrapper/port-forward allowlist'
if rg -q '^[[:space:]]*(alias[[:space:]]+[^=]+=.*kubectl|[A-Za-z_][A-Za-z0-9_]*=.*kubectl)' "$CAPTURE"; then
fail 'kubectl executable alias bypasses the default-deny wrapper'
fi
pass_count=$((pass_count + 1))
printf 'PASS: %d target-initial inventory assertions\n' "$pass_count"
Reference in New Issue View Git Blame Copy Permalink