#!/usr/bin/env bash # Repository-owned, read-only Prometheus metric inventory capture. This file is # sourceable so its parsers can be exercised without contacting a cluster. readonly INVENTORY_REJECTION_STATUS=23 readonly INVENTORY_SCHEMA='platform-observability-metric-inventory/v1' readonly INVENTORY_CONTEXT='default' readonly INVENTORY_API_SERVER='https://127.0.0.1:6443' readonly INVENTORY_PROMETHEUS_NAMESPACE='observability' readonly INVENTORY_PROMETHEUS_NAME='observability-core-kube-pr-prometheus' readonly INVENTORY_PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus' readonly INVENTORY_MAX_RESPONSE_BYTES=8388608 inventory_reject() { printf 'REJECT: target-initial inventory precondition failed\n' >&2 return "$INVENTORY_REJECTION_STATUS" } inventory_expected_pools() { local phase="${1-target-initial}" [[ "$phase" == target-initial || "$phase" == post-substrate ]] || { inventory_reject return } cat <<'POOLS' podMonitor/platform-data/platform-postgres/0 serviceMonitor/gitea/gitea/0 serviceMonitor/keycloak/keycloak/0 serviceMonitor/kube-system/traefik/0 serviceMonitor/object-storage/aistor-bucket-usage/0 serviceMonitor/observability-agent/alloy/0 serviceMonitor/observability-agent/node-exporter/0 serviceMonitor/observability/grafana/0 serviceMonitor/observability/loki/0 serviceMonitor/observability/observability-core-kube-pr-alertmanager/0 serviceMonitor/observability/observability-core-kube-pr-alertmanager/1 serviceMonitor/observability/observability-core-kube-pr-apiserver/0 serviceMonitor/observability/observability-core-kube-pr-coredns/0 serviceMonitor/observability/observability-core-kube-pr-kubelet/0 serviceMonitor/observability/observability-core-kube-pr-kubelet/1 serviceMonitor/observability/observability-core-kube-pr-kubelet/2 serviceMonitor/observability/observability-core-kube-pr-operator/0 serviceMonitor/observability/observability-core-kube-pr-prometheus/0 serviceMonitor/observability/observability-core-kube-pr-prometheus/1 serviceMonitor/observability/observability-core-kube-state-metrics/0 serviceMonitor/observability/tempo/0 POOLS if [[ "$phase" == post-substrate ]]; then cat <<'POOLS' probe/observability/platform-private-edge probe/observability/platform-private-edge probe/observability/platform-private-edge probe/observability/platform-private-internal probe/observability/platform-private-internal probe/observability/platform-private-internal probe/observability/platform-public-edge probe/observability/platform-public-edge serviceMonitor/observability/blackbox-exporter/0 POOLS fi } post_substrate_expected_probe_targets() { cat <<'JSON' [ {"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}}, {"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","labels":{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}}, {"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}}, {"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","labels":{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}}, {"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","labels":{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}}, {"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","labels":{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}}, {"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","labels":{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}}, {"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","labels":{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}} ] JSON } sanitize_scrape_url() { local url="${1-}" mode="${2-strict}" remainder authority if [[ -z "$url" || "$url" == *[$' \t\r\n']* || "$url" == *'#'* ]]; then inventory_reject return fi if [[ "$mode" == strip-query ]]; then [[ "$url" == *'?'* ]] || { inventory_reject; return; } url="${url%%\?*}" elif [[ "$mode" != strict || "$url" == *'?'* ]]; then inventory_reject return fi case "$url" in http://*|https://*) ;; *) inventory_reject; return ;; esac remainder="${url#*://}" authority="${remainder%%/*}" if [[ -z "$authority" || "$authority" == *'@'* ]]; then inventory_reject return fi printf '%s\n' "$url" } validate_targets_json() { local path="${1--}" phase="${2-target-initial}" expected_json post_probes='[]' expected_json="$(inventory_expected_pools "$phase" | jq -Rsc 'split("\n") | map(select(length > 0)) | sort')" || { inventory_reject return } if [[ "$phase" == post-substrate ]]; then post_probes="$(post_substrate_expected_probe_targets | jq -cS 'sort_by(.scrapePool,.scrapeUrl)')" || { inventory_reject return } fi if ! jq -e --arg phase "$phase" --argjson expected "$expected_json" --argjson postProbes "$post_probes" ' type == "object" and .status == "success" and (.data | type == "object") and (.data.activeTargets | type == "array") and (.data.droppedTargets | type == "array") and ((.data.activeTargets | map(.scrapePool) | sort) == $expected) and all(.data.activeTargets[]; type == "object" and (.scrapePool | type == "string") and (.scrapeUrl | type == "string") and (.labels | type == "object") and (.scrapeUrl | test("^https?://[^/?#@[:space:]]+(?:/[^#[:space:]]*)?$")) and ($phase == "post-substrate" or (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not)) and (.health == "up") and (.lastError == "")) and all(.data.droppedTargets[]?; (.scrapePool | type == "string") and ($phase == "post-substrate" or (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not))) and ([.data.activeTargets[] | select(.scrapePool == "serviceMonitor/object-storage/aistor-bucket-usage/0") | select(.labels.job == "minio-aistor" and .labels.namespace == "object-storage" and .labels.service == "minio" and .labels.endpoint == "http-minio" and .labels.container == "minio")] | length == 1) and ($phase == "target-initial" or ( ([.data.activeTargets[] | select(.scrapePool | startswith("probe/observability/")) | {scrapePool,scrapeUrl,labels}] | sort_by(.scrapePool,.scrapeUrl)) == $postProbes and ([.data.activeTargets[] | select(.scrapePool == "serviceMonitor/observability/blackbox-exporter/0") | select(.labels.job == "blackbox-exporter" and .labels.namespace == "observability" and .labels.service == "blackbox-exporter" and .labels.endpoint == "http" and .labels.container == "blackbox-exporter" and (.labels.pod | test("^blackbox-exporter-[a-z0-9]+-[a-z0-9]+$")) and (.labels.instance | test("^[0-9.]+:9115$")) and (.scrapeUrl | test("^http://[0-9.]+:9115/metrics$")))] | length == 1) )) ' "$path" >/dev/null 2>&1; then inventory_reject return fi } validate_dropped_targets_json() { local path="${1--}" phase="${2-target-initial}" if ! jq -e --arg phase "$phase" ' type == "object" and .status == "success" and (.data | type == "object") and (.data.activeTargets | type == "array") and (.data.droppedTargets | type == "array") and all(.data.droppedTargets[]?; (.scrapePool | type == "string") and (if $phase == "target-initial" then (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not) else (.scrapePool | test("(?:^|/)probe(?:/|$)"; "i") | not) end)) ' "$path" >/dev/null 2>&1; then inventory_reject return fi } validate_probe_series_phase() { local path="${1--}" phase="${2-target-initial}" if ! jq -e --arg phase "$phase" ' type == "object" and .status == "success" and (.data | type == "array") and (if $phase == "target-initial" then (.data | length) == 0 elif $phase == "post-substrate" then (.data | length) > 0 and all(.data[]; type == "object" and (."__name__" | type == "string") and (."__name__" | startswith("probe_")) and .namespace == "observability") else false end) ' "$path" >/dev/null 2>&1; then inventory_reject return fi } extract_metric_names() { local path="${1--}" names if ! names="$(jq -er ' if type == "object" and .status == "success" then if (.data | type == "object") and (.data | length > 0) and all(.data | keys[]; test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) then .data | keys | sort | .[] elif (.data | type == "array") and (.data | length > 0) and all(.data[]; type == "object" and (.target | type == "object") and (.metric | type == "string") and (.metric | test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) and (.type | type == "string") and (.help | type == "string") and (.unit | type == "string")) then [.data[].metric] | unique | sort | .[] else error("invalid metadata") end else error("invalid metadata") end ' "$path" 2>/dev/null)"; then inventory_reject return fi printf '%s\n' "$names" } extract_recording_rule_names() { local path="${1--}" names if ! names="$(jq -er ' if type == "object" and .status == "success" and (.data | type == "object") and (.data.groups | type == "array") and all(.data.groups[]; (.rules | type == "array")) then [.data.groups[].rules[] | select(.type == "recording") | .name] | if all(.[]; type == "string" and test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) then unique | sort | join("\n") else error("invalid recording rule") end else error("invalid rules") end ' "$path" 2>/dev/null)"; then inventory_reject return fi [[ -z "$names" ]] || printf '%s\n' "$names" } reject_recording_rule_collisions() { local rules_path="${1-}" metadata_path="${2-}" rules metrics name if ! rules="$(extract_recording_rule_names "$rules_path")" || ! metrics="$(extract_metric_names "$metadata_path")"; then inventory_reject return fi reject_metric_name_collisions "$rules" "$metrics" } reject_metric_name_collisions() { local rules="${1-}" metrics="${2-}" name while IFS= read -r name; do [[ -z "$name" ]] && continue if grep -Fqx -- "$name" <<<"$rules"; then inventory_reject return fi done <<<"$metrics" } validate_metric_names_for_pool() { local metrics="${1-}" pool="${2-}" phase="${3-target-initial}" if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then grep -Fqx probe_success <<<"$metrics" || { inventory_reject; return; } elif grep -Eq '^probe_' <<<"$metrics"; then inventory_reject return fi } extract_series_label_names() { local path="${1--}" result if ! result="$(jq -ceS ' if type == "object" and .status == "success" and (.data | type == "array") and (.data | length > 0) and all(.data[]; type == "object") and all([.data[] | keys[]][]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$")) then [.data[] | keys[]] | unique | sort else error("invalid series") end ' "$path" 2>/dev/null)"; then inventory_reject return fi printf '%s\n' "$result" } prometheus_selector_for_labels() { local labels_json="${1-}" jq -cer ' if type == "object" and length > 0 and all(keys[]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$")) and all(.[]; type == "string" and (explode | all(. >= 32 and . != 127))) then "{" + (to_entries | sort_by(.key) | map((if (.key | test("^[a-zA-Z_][a-zA-Z0-9_]*$")) then .key else (.key | @json) end) + "=" + (.value | @json)) | join(",")) + "}" else error("invalid target labels") end ' <<<"$labels_json" 2>/dev/null || inventory_reject } inventory_provenance() { case "${1-}" in podMonitor/platform-data/platform-postgres/0|\ serviceMonitor/gitea/gitea/0|\ serviceMonitor/keycloak/keycloak/0|\ serviceMonitor/kube-system/traefik/0|\ serviceMonitor/object-storage/aistor-bucket-usage/0) printf '%s\n' 'task6-owned' ;; serviceMonitor/observability/grafana/0) printf '%s\n' 'preexisting-verify-only' ;; serviceMonitor/observability/blackbox-exporter/0|probe/observability/platform-*) printf '%s\n' 'substrate-owned' ;; *) printf '%s\n' 'core' ;; esac } validate_output_directory() { local directory="${1-}" canonical uid mode current component if [[ -z "$directory" || "$directory" != /* || "$directory" == *[$'\t\r\n']* || ! -d "$directory" || -L "$directory" ]]; then inventory_reject return fi if ! canonical="$(realpath -e -- "$directory" 2>/dev/null)" || [[ "$canonical" != "$directory" ]]; then inventory_reject return fi current='' IFS='/' read -r -a components <<<"${directory#/}" for component in "${components[@]}"; do [[ -n "$component" && "$component" != '.' && "$component" != '..' ]] || { inventory_reject return } current="$current/$component" if [[ -L "$current" ]]; then inventory_reject return fi done if ! uid="$(stat -c %u -- "$directory" 2>/dev/null)" || ! mode="$(stat -c %a -- "$directory" 2>/dev/null)" || [[ "$uid" != "$(id -u)" || "$mode" != 700 ]]; then inventory_reject return fi if [[ -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then inventory_reject return fi } inventory_kubectl() { case "${1-}" in config) case "${2-}" in current-context|view) ;; *) inventory_reject; return ;; esac ;; get) ;; auth) [[ "${2-}" == can-i ]] || { inventory_reject; return; } ;; *) inventory_reject return ;; esac timeout --signal=TERM --kill-after=2s 15s \ kubectl --request-timeout=10s "$@" 2>/dev/null } inventory_process_snapshot() { local pid="${1-}" stat_line remainder state start_time [[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] || return 1 IFS= read -r stat_line <"/proc/$pid/stat" || return 1 remainder="${stat_line##*) }" state="${remainder%% *}" start_time="$(awk '{print $20}' <<<"$remainder")" || return 1 [[ "$state" =~ ^[A-Z]$ && "$start_time" =~ ^[0-9]+$ ]] || return 1 printf '%s:%s\n' "$state" "$start_time" } inventory_port_forward_is_owned() { local pid="${INVENTORY_PORT_FORWARD_PID:-}" expected="${INVENTORY_PORT_FORWARD_START:-}" snapshot [[ "$pid" =~ ^[1-9][0-9]*$ && "$expected" =~ ^[0-9]+$ ]] || return 1 snapshot="$(inventory_process_snapshot "$pid")" || return 1 [[ "${snapshot#*:}" == "$expected" ]] } inventory_stop_port_forward() { local pid="${INVENTORY_PORT_FORWARD_PID:-}" snapshot state attempt [[ -n "$pid" ]] || return 0 if ! inventory_port_forward_is_owned; then INVENTORY_PORT_FORWARD_PID='' INVENTORY_PORT_FORWARD_START='' return 0 fi kill -TERM "$pid" 2>/dev/null || true for ((attempt = 0; attempt < 40; attempt++)); do if ! inventory_port_forward_is_owned; then break; fi snapshot="$(inventory_process_snapshot "$pid")" || break state="${snapshot%%:*}" [[ "$state" == Z || "$state" == X ]] && break sleep 0.025 done if inventory_port_forward_is_owned; then snapshot="$(inventory_process_snapshot "$pid")" || snapshot='' state="${snapshot%%:*}" if [[ "$state" != Z && "$state" != X ]]; then kill -KILL "$pid" 2>/dev/null || true fi fi for ((attempt = 0; attempt < 40; attempt++)); do if ! inventory_port_forward_is_owned; then break; fi snapshot="$(inventory_process_snapshot "$pid")" || break state="${snapshot%%:*}" [[ "$state" == Z || "$state" == X ]] && break sleep 0.025 done if inventory_port_forward_is_owned; then snapshot="$(inventory_process_snapshot "$pid")" || snapshot='' state="${snapshot%%:*}" if [[ "$state" != Z && "$state" != X ]]; then return 1; fi fi INVENTORY_PORT_FORWARD_PID='' INVENTORY_PORT_FORWARD_START='' } build_inventory_json() { local target_rows="${1-}" phase="${2-}" timestamp="${3-}" [[ -n "$target_rows" && -f "$target_rows" && ! -L "$target_rows" && -n "$phase" && -n "$timestamp" ]] || { inventory_reject; return; } jq -cnS --slurpfile targets "$target_rows" \ --arg schema "$INVENTORY_SCHEMA" --arg phase "$phase" --arg captured_at_utc "$timestamp" \ --arg context "$INVENTORY_CONTEXT" --arg api_server "$INVENTORY_API_SERVER" \ --arg prometheus "$INVENTORY_PROMETHEUS_NAMESPACE/$INVENTORY_PROMETHEUS_NAME" ' {schema:$schema,phase:$phase,captured_at_utc:$captured_at_utc,context:$context, api_server:$api_server,prometheus:$prometheus,preexisting_verify_only:["grafana"], targets:($targets | sort_by(.scrape_pool))} ' } validate_post_substrate_query_body() { local path="${1--}" kind="${2-}" expected expected_tls now expected="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels] | sort_by(.job,.instance)')" || { inventory_reject return } expected_tls="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels | select(.instance | startswith("https://"))] | sort_by(.job,.instance)')" || { inventory_reject return } now="$(date -u +%s)" || { inventory_reject; return; } if ! jq -e --arg kind "$kind" --argjson expected "$expected" \ --argjson expectedTls "$expected_tls" --argjson now "$now" ' type == "object" and .status == "success" and .data.resultType == "vector" and (.data.result | type == "array") and (if $kind == "tls" then ([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expectedTls and all(.data.result[]; (.value[1] | tonumber) > $now) else ([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expected and (if $kind == "success" then all(.data.result[]; .value[1] == "1") elif $kind == "duration" then all(.data.result[]; (.value[1] | tonumber) >= 0) elif $kind == "status" then all(.data.result[]; if .metric.job == "blackbox-private-edge" then .value[1] == "403" else .value[1] == "200" end) else false end) end) ' "$path" >/dev/null 2>&1; then inventory_reject return fi } inventory_validate_post_substrate_metrics() { local body spec for spec in \ 'success:probe_success{namespace="observability"}' \ 'status:probe_http_status_code{namespace="observability"}' \ 'duration:probe_duration_seconds{namespace="observability"}' \ 'tls:probe_ssl_earliest_cert_expiry{namespace="observability"}'; do if ! inventory_prometheus_json body '/api/v1/query' --get \ --data-urlencode "query=${spec#*:}" || ! printf '%s' "$body" | validate_post_substrate_query_body - "${spec%%:*}"; then inventory_reject return fi body='' done } inventory_capture_main() { local output_dir phase context api prometheus_json pods_json permission local port='' attempt start_attempt forward_ready=0 timestamp snapshot local targets_raw='' dropped_raw='' rules_raw='' probe_raw='' metadata_raw='' series_raw='' recording_names='' local row pool scrape_url labels selector metric_names metric local metric_labels series_selector series_count metrics_path metrics_json provenance local target_rows counter=0 inventory hash actual_files if (($# != 5)) || [[ "$1" != --execute || "$2" != --phase || ("$3" != target-initial && "$3" != post-substrate) || "$4" != --output-dir || -z "$5" ]]; then inventory_reject return fi phase="$3" output_dir="$5" for permission in jq kubectl curl timeout realpath stat find mktemp sha256sum sync grep head; do command -v "$permission" >/dev/null 2>&1 || { inventory_reject; return; } done validate_output_directory "$output_dir" || return INVENTORY_WORK_DIR="$(mktemp -d /tmp/platform-observability-inventory-capture.XXXXXX)" || { inventory_reject return } chmod 0700 "$INVENTORY_WORK_DIR" || { inventory_reject; return; } INVENTORY_STAGE_JSON='' INVENTORY_STAGE_SHA='' INVENTORY_OUTPUT_DIR="$output_dir" INVENTORY_CREATED_JSON=0 INVENTORY_CREATED_SHA=0 INVENTORY_JSON_ID='' INVENTORY_SHA_ID='' INVENTORY_SUCCESS=0 INVENTORY_PORT_FORWARD_PID='' INVENTORY_PORT_FORWARD_START='' trap inventory_capture_cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM if ! context="$(inventory_kubectl config current-context)" || [[ "$context" != "$INVENTORY_CONTEXT" ]]; then inventory_reject return fi if ! api="$(inventory_kubectl config view --minify -o json | jq -er '.clusters | if length == 1 then .[0].cluster.server else error("cluster count") end' 2>/dev/null)" || [[ "$api" != "$INVENTORY_API_SERVER" ]]; then inventory_reject return fi if ! inventory_kubectl get --raw=/readyz >/dev/null; then inventory_reject; return; fi for permission in \ 'get pods' \ 'get services' \ 'get prometheuses.monitoring.coreos.com' \ 'create pods/portforward'; do if ! inventory_kubectl auth can-i $permission --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" | grep -Fxq yes; then inventory_reject return fi done if ! prometheus_json="$(inventory_kubectl get prometheus "$INVENTORY_PROMETHEUS_NAME" \ --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" -o json)" || ! jq -e --arg namespace "$INVENTORY_PROMETHEUS_NAMESPACE" --arg name "$INVENTORY_PROMETHEUS_NAME" ' .kind == "Prometheus" and .metadata.namespace == $namespace and .metadata.name == $name ' <<<"$prometheus_json" >/dev/null 2>&1; then inventory_reject return fi if ! pods_json="$(inventory_kubectl get pods --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" \ -l "prometheus=$INVENTORY_PROMETHEUS_NAME" -o json)" || ! jq -e ' (.items | length == 1) and .items[0].status.phase == "Running" and any(.items[0].status.conditions[]; .type == "Ready" and .status == "True") and any(.items[0].status.containerStatuses[]; .name == "prometheus" and .ready == true) ' <<<"$pods_json" >/dev/null 2>&1; then inventory_reject return fi : >"$INVENTORY_WORK_DIR/port-forward.log" for ((start_attempt = 0; start_attempt < 10; start_attempt++)); do port=$((20000 + ((RANDOM + $$ + start_attempt * 7919) % 40000))) kubectl --request-timeout=10s -n "$INVENTORY_PROMETHEUS_NAMESPACE" \ port-forward --address 127.0.0.1 \ "service/$INVENTORY_PROMETHEUS_SERVICE" "$port:9090" \ >/dev/null 2>"$INVENTORY_WORK_DIR/port-forward.log" & INVENTORY_PORT_FORWARD_PID=$! snapshot="$(inventory_process_snapshot "$INVENTORY_PORT_FORWARD_PID")" || { inventory_reject return } INVENTORY_PORT_FORWARD_START="${snapshot#*:}" INVENTORY_PROMETHEUS_BASE="http://127.0.0.1:$port" for ((attempt = 0; attempt < 40; attempt++)); do if ! kill -0 "$INVENTORY_PORT_FORWARD_PID" 2>/dev/null; then break; fi if inventory_prometheus_ready; then forward_ready=1 break 2 fi sleep 0.05 done inventory_stop_port_forward || { inventory_reject; return; } done if [[ "$forward_ready" != 1 || -z "$INVENTORY_PORT_FORWARD_PID" ]]; then inventory_reject return fi if ! inventory_prometheus_json targets_raw '/api/v1/targets' --get --data 'state=active' || ! printf '%s' "$targets_raw" | validate_targets_json - "$phase" || ! inventory_prometheus_json dropped_raw '/api/v1/targets' --get --data 'state=dropped' || ! printf '%s' "$dropped_raw" | validate_dropped_targets_json - "$phase" || ! inventory_prometheus_json rules_raw '/api/v1/rules' || ! recording_names="$(printf '%s' "$rules_raw" | extract_recording_rule_names -)" || ! inventory_prometheus_json probe_raw '/api/v1/series' --get \ --data-urlencode 'match[]={__name__=~"probe_.+"}' || ! printf '%s' "$probe_raw" | validate_probe_series_phase - "$phase"; then inventory_reject return fi if [[ "$phase" == post-substrate ]] && ! inventory_validate_post_substrate_metrics; then inventory_reject return fi dropped_raw='' rules_raw='' probe_raw='' target_rows="$INVENTORY_WORK_DIR/target-rows.jsonl" : >"$target_rows" while IFS= read -r row; do pool="$(jq -er '.scrapePool' <<<"$row" 2>/dev/null)" || { inventory_reject; return; } scrape_url="$(jq -er '.scrapeUrl' <<<"$row" 2>/dev/null)" || { inventory_reject; return; } if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then scrape_url="$(sanitize_scrape_url "$scrape_url" strip-query)" || return else scrape_url="$(sanitize_scrape_url "$scrape_url")" || return fi labels="$(jq -ceS '.labels' <<<"$row" 2>/dev/null)" || { inventory_reject; return; } selector="$(prometheus_selector_for_labels "$labels")" || return counter=$((counter + 1)) if ! inventory_prometheus_json metadata_raw '/api/v1/targets/metadata' --get \ --data-urlencode "match_target=$selector" --data 'limit=10000'; then inventory_reject return fi metric_names="$(printf '%s' "$metadata_raw" | extract_metric_names -)" || return metadata_raw='' reject_metric_name_collisions "$recording_names" "$metric_names" || return validate_metric_names_for_pool "$metric_names" "$pool" "$phase" || return metrics_path="$INVENTORY_WORK_DIR/metrics-$counter.jsonl" : >"$metrics_path" while IFS= read -r metric; do series_selector="$(prometheus_selector_for_labels "$( jq -ce --arg metric "$metric" '. + {"__name__":$metric}' <<<"$labels" 2>/dev/null )")" || return if ! inventory_prometheus_json series_raw '/api/v1/series' --get \ --data-urlencode "match[]=$series_selector"; then inventory_reject return fi series_count="$(printf '%s' "$series_raw" | jq -er ' if type == "object" and .status == "success" and (.data | type == "array") and all(.data[]; type == "object") then .data | length else error("invalid series") end ' 2>/dev/null)" || { inventory_reject; return; } if (( series_count == 0 )); then series_raw='' continue fi metric_labels="$(printf '%s' "$series_raw" | extract_series_label_names -)" || return series_raw='' jq -cnS --arg name "$metric" --argjson label_names "$metric_labels" \ '{name:$name,label_names:$label_names}' >>"$metrics_path" || { inventory_reject; return; } done <<<"$metric_names" [[ -s "$metrics_path" ]] || { inventory_reject; return; } metrics_json="$(jq -csS 'sort_by(.name)' "$metrics_path" 2>/dev/null)" || { inventory_reject; return; } provenance="$(inventory_provenance "$pool")" jq -cnS --arg scrape_pool "$pool" --arg scrape_url "$scrape_url" \ --arg provenance "$provenance" --argjson metrics "$metrics_json" \ '{scrape_pool:$scrape_pool,scrape_url:$scrape_url,health:"up",last_error:"", provenance:$provenance,metrics:$metrics}' >>"$target_rows" || { inventory_reject; return; } done < <(printf '%s' "$targets_raw" | jq -c '.data.activeTargets | sort_by(.scrapePool)[]') targets_raw='' timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" || { inventory_reject; return; } inventory="$(build_inventory_json "$target_rows" "$phase" "$timestamp")" || return validate_output_directory "$output_dir" || return INVENTORY_STAGE_JSON="$(mktemp "$output_dir/.inventory.json.tmp.XXXXXX")" || { inventory_reject; return; } chmod 0600 "$INVENTORY_STAGE_JSON" || { inventory_reject; return; } printf '%s\n' "$inventory" >"$INVENTORY_STAGE_JSON" || { inventory_reject; return; } sync -f "$INVENTORY_STAGE_JSON" >/dev/null 2>&1 || { inventory_reject; return; } INVENTORY_JSON_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_JSON")" || { inventory_reject; return; } hash="$(sha256sum -- "$INVENTORY_STAGE_JSON" | awk '{print $1}')" || { inventory_reject; return; } INVENTORY_STAGE_SHA="$(mktemp "$output_dir/.inventory.sha256.tmp.XXXXXX")" || { inventory_reject; return; } chmod 0600 "$INVENTORY_STAGE_SHA" || { inventory_reject; return; } printf '%s inventory.json\n' "$hash" >"$INVENTORY_STAGE_SHA" || { inventory_reject; return; } sync -f "$INVENTORY_STAGE_SHA" >/dev/null 2>&1 || { inventory_reject; return; } INVENTORY_SHA_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_SHA")" || { inventory_reject; return; } if ! ln -- "$INVENTORY_STAGE_JSON" "$output_dir/inventory.json" 2>/dev/null; then inventory_reject; return; fi INVENTORY_CREATED_JSON=1 rm -- "$INVENTORY_STAGE_JSON" || { inventory_reject; return; } INVENTORY_STAGE_JSON='' if ! ln -- "$INVENTORY_STAGE_SHA" "$output_dir/inventory.sha256" 2>/dev/null; then inventory_reject; return; fi INVENTORY_CREATED_SHA=1 rm -- "$INVENTORY_STAGE_SHA" || { inventory_reject; return; } INVENTORY_STAGE_SHA='' sync -f "$output_dir/inventory.json" >/dev/null 2>&1 || { inventory_reject; return; } sync -f "$output_dir/inventory.sha256" >/dev/null 2>&1 || { inventory_reject; return; } sync -f "$output_dir" >/dev/null 2>&1 || { inventory_reject; return; } actual_files="$(find "$output_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" if [[ "$actual_files" != $'inventory.json\ninventory.sha256' ]] || [[ ! -f "$output_dir/inventory.json" || -L "$output_dir/inventory.json" ]] || [[ ! -f "$output_dir/inventory.sha256" || -L "$output_dir/inventory.sha256" ]] || [[ "$(stat -c %a -- "$output_dir/inventory.json")" != 600 ]] || [[ "$(stat -c %a -- "$output_dir/inventory.sha256")" != 600 ]]; then inventory_reject return fi INVENTORY_SUCCESS=1 printf 'OK: %s inventory captured\n' "$phase" } inventory_prometheus_json() { local destination_name="$1" api_path="$2" response local LC_ALL=C local -n destination="$destination_name" shift 2 if ! response="$( set -o pipefail curl --disable --silent --show-error --fail --connect-timeout 3 --max-time 10 \ --max-filesize "$INVENTORY_MAX_RESPONSE_BYTES" --output - \ "$@" "$INVENTORY_PROMETHEUS_BASE$api_path" 2>/dev/null | LC_ALL=C head -c "$((INVENTORY_MAX_RESPONSE_BYTES + 1))" pipeline_status=$? printf '\034' exit "$pipeline_status" )"; then response='' inventory_reject return fi [[ "$response" == *$'\034' ]] || { response=''; inventory_reject; return; } response="${response%?}" if (( ${#response} > INVENTORY_MAX_RESPONSE_BYTES )); then response='' inventory_reject return fi destination="$response" } inventory_prometheus_ready() { curl --disable --silent --show-error --fail --connect-timeout 1 --max-time 1 \ --max-filesize 1024 --output /dev/null "$INVENTORY_PROMETHEUS_BASE/-/ready" 2>/dev/null } inventory_capture_cleanup() { local rc=$? trap - EXIT HUP INT TERM if [[ -n "${INVENTORY_PORT_FORWARD_PID:-}" ]]; then inventory_stop_port_forward || true fi if [[ -n "${INVENTORY_STAGE_JSON:-}" && -f "$INVENTORY_STAGE_JSON" && ! -L "$INVENTORY_STAGE_JSON" ]]; then rm -- "$INVENTORY_STAGE_JSON" fi if [[ -n "${INVENTORY_STAGE_SHA:-}" && -f "$INVENTORY_STAGE_SHA" && ! -L "$INVENTORY_STAGE_SHA" ]]; then rm -- "$INVENTORY_STAGE_SHA" fi if [[ "${INVENTORY_SUCCESS:-0}" != 1 ]]; then if [[ "${INVENTORY_CREATED_SHA:-0}" == 1 ]]; then inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.sha256 "${INVENTORY_SHA_ID:-}" fi if [[ "${INVENTORY_CREATED_JSON:-0}" == 1 ]]; then inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.json "${INVENTORY_JSON_ID:-}" fi if [[ -n "${INVENTORY_OUTPUT_DIR:-}" && -d "$INVENTORY_OUTPUT_DIR" && ! -L "$INVENTORY_OUTPUT_DIR" ]]; then sync -f "$INVENTORY_OUTPUT_DIR" >/dev/null 2>&1 || true fi fi if [[ -n "${INVENTORY_WORK_DIR:-}" && -d "$INVENTORY_WORK_DIR" && ! -L "$INVENTORY_WORK_DIR" ]]; then case "$INVENTORY_WORK_DIR" in /tmp/platform-observability-inventory-capture.*) rm -rf -- "$INVENTORY_WORK_DIR" ;; esac fi exit "$rc" } inventory_remove_created_file() { local directory="${1-}" basename="${2-}" expected_id="${3-}" path actual_id canonical [[ -n "$directory" && -n "$basename" && -n "$expected_id" && -d "$directory" && ! -L "$directory" ]] || return 0 canonical="$(realpath -e -- "$directory" 2>/dev/null)" || return 0 [[ "$canonical" == "$directory" ]] || return 0 path="$directory/$basename" [[ -f "$path" && ! -L "$path" ]] || return 0 actual_id="$(stat -c %d:%i -- "$path" 2>/dev/null)" || return 0 [[ "$actual_id" == "$expected_id" ]] || return 0 rm -- "$path" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then set -Eeuo pipefail umask 077 inventory_capture_main "$@" fi