813 lines
35 KiB
Bash
813 lines
35 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Repository-owned, read-only Prometheus metric inventory capture. This file is
|
|
# sourceable so its parsers can be exercised without contacting a cluster.
|
|
|
|
readonly INVENTORY_REJECTION_STATUS=23
|
|
readonly INVENTORY_SCHEMA='platform-observability-metric-inventory/v1'
|
|
readonly INVENTORY_CONTEXT='default'
|
|
readonly INVENTORY_API_SERVER='https://127.0.0.1:6443'
|
|
readonly INVENTORY_PROMETHEUS_NAMESPACE='observability'
|
|
readonly INVENTORY_PROMETHEUS_NAME='observability-core-kube-pr-prometheus'
|
|
readonly INVENTORY_PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus'
|
|
readonly INVENTORY_MAX_RESPONSE_BYTES=8388608
|
|
|
|
inventory_reject() {
|
|
printf 'REJECT: target-initial inventory precondition failed\n' >&2
|
|
return "$INVENTORY_REJECTION_STATUS"
|
|
}
|
|
|
|
inventory_expected_pools() {
|
|
local phase="${1-target-initial}"
|
|
[[ "$phase" == target-initial || "$phase" == post-substrate ]] || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
cat <<'POOLS'
|
|
podMonitor/platform-data/platform-postgres/0
|
|
serviceMonitor/gitea/gitea/0
|
|
serviceMonitor/keycloak/keycloak/0
|
|
serviceMonitor/kube-system/traefik/0
|
|
serviceMonitor/object-storage/aistor-bucket-usage/0
|
|
serviceMonitor/observability-agent/alloy/0
|
|
serviceMonitor/observability-agent/node-exporter/0
|
|
serviceMonitor/observability/grafana/0
|
|
serviceMonitor/observability/loki/0
|
|
serviceMonitor/observability/observability-core-kube-pr-alertmanager/0
|
|
serviceMonitor/observability/observability-core-kube-pr-alertmanager/1
|
|
serviceMonitor/observability/observability-core-kube-pr-apiserver/0
|
|
serviceMonitor/observability/observability-core-kube-pr-coredns/0
|
|
serviceMonitor/observability/observability-core-kube-pr-kubelet/0
|
|
serviceMonitor/observability/observability-core-kube-pr-kubelet/1
|
|
serviceMonitor/observability/observability-core-kube-pr-kubelet/2
|
|
serviceMonitor/observability/observability-core-kube-pr-operator/0
|
|
serviceMonitor/observability/observability-core-kube-pr-prometheus/0
|
|
serviceMonitor/observability/observability-core-kube-pr-prometheus/1
|
|
serviceMonitor/observability/observability-core-kube-state-metrics/0
|
|
serviceMonitor/observability/tempo/0
|
|
POOLS
|
|
if [[ "$phase" == post-substrate ]]; then
|
|
cat <<'POOLS'
|
|
probe/observability/platform-private-edge
|
|
probe/observability/platform-private-edge
|
|
probe/observability/platform-private-edge
|
|
probe/observability/platform-private-internal
|
|
probe/observability/platform-private-internal
|
|
probe/observability/platform-private-internal
|
|
probe/observability/platform-public-edge
|
|
probe/observability/platform-public-edge
|
|
serviceMonitor/observability/blackbox-exporter/0
|
|
POOLS
|
|
fi
|
|
}
|
|
|
|
post_substrate_expected_probe_targets() {
|
|
cat <<'JSON'
|
|
[
|
|
{"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}},
|
|
{"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","labels":{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}},
|
|
{"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}},
|
|
{"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","labels":{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}},
|
|
{"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","labels":{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}},
|
|
{"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","labels":{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}},
|
|
{"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","labels":{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}},
|
|
{"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","labels":{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}}
|
|
]
|
|
JSON
|
|
}
|
|
|
|
sanitize_scrape_url() {
|
|
local url="${1-}" mode="${2-strict}" remainder authority
|
|
if [[ -z "$url" || "$url" == *[$' \t\r\n']* || "$url" == *'#'* ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if [[ "$mode" == strip-query ]]; then
|
|
[[ "$url" == *'?'* ]] || { inventory_reject; return; }
|
|
url="${url%%\?*}"
|
|
elif [[ "$mode" != strict || "$url" == *'?'* ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
case "$url" in
|
|
http://*|https://*) ;;
|
|
*) inventory_reject; return ;;
|
|
esac
|
|
remainder="${url#*://}"
|
|
authority="${remainder%%/*}"
|
|
if [[ -z "$authority" || "$authority" == *'@'* ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
printf '%s\n' "$url"
|
|
}
|
|
|
|
validate_targets_json() {
|
|
local path="${1--}" phase="${2-target-initial}" expected_json post_probes='[]'
|
|
expected_json="$(inventory_expected_pools "$phase" | jq -Rsc 'split("\n") | map(select(length > 0)) | sort')" || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
if [[ "$phase" == post-substrate ]]; then
|
|
post_probes="$(post_substrate_expected_probe_targets | jq -cS 'sort_by(.scrapePool,.scrapeUrl)')" || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
fi
|
|
if ! jq -e --arg phase "$phase" --argjson expected "$expected_json" --argjson postProbes "$post_probes" '
|
|
type == "object" and .status == "success" and
|
|
(.data | type == "object") and
|
|
(.data.activeTargets | type == "array") and
|
|
(.data.droppedTargets | type == "array") and
|
|
((.data.activeTargets | map(.scrapePool) | sort) == $expected) and
|
|
all(.data.activeTargets[];
|
|
type == "object" and (.scrapePool | type == "string") and
|
|
(.scrapeUrl | type == "string") and (.labels | type == "object") and
|
|
(.scrapeUrl | test("^https?://[^/?#@[:space:]]+(?:/[^#[:space:]]*)?$")) and
|
|
($phase == "post-substrate" or
|
|
(.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not)) and
|
|
(.health == "up") and (.lastError == "")) and
|
|
all(.data.droppedTargets[]?;
|
|
(.scrapePool | type == "string") and
|
|
($phase == "post-substrate" or
|
|
(.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not))) and
|
|
([.data.activeTargets[] |
|
|
select(.scrapePool == "serviceMonitor/object-storage/aistor-bucket-usage/0") |
|
|
select(.labels.job == "minio-aistor" and
|
|
.labels.namespace == "object-storage" and
|
|
.labels.service == "minio" and
|
|
.labels.endpoint == "http-minio" and
|
|
.labels.container == "minio")] | length == 1) and
|
|
($phase == "target-initial" or (
|
|
([.data.activeTargets[] |
|
|
select(.scrapePool | startswith("probe/observability/")) |
|
|
{scrapePool,scrapeUrl,labels}] | sort_by(.scrapePool,.scrapeUrl)) == $postProbes and
|
|
([.data.activeTargets[] |
|
|
select(.scrapePool == "serviceMonitor/observability/blackbox-exporter/0") |
|
|
select(.labels.job == "blackbox-exporter" and
|
|
.labels.namespace == "observability" and
|
|
.labels.service == "blackbox-exporter" and
|
|
.labels.endpoint == "http" and
|
|
.labels.container == "blackbox-exporter" and
|
|
(.labels.pod | test("^blackbox-exporter-[a-z0-9]+-[a-z0-9]+$")) and
|
|
(.labels.instance | test("^[0-9.]+:9115$")) and
|
|
(.scrapeUrl | test("^http://[0-9.]+:9115/metrics$")))] | length == 1)
|
|
))
|
|
' "$path" >/dev/null 2>&1; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
}
|
|
|
|
validate_dropped_targets_json() {
|
|
local path="${1--}" phase="${2-target-initial}"
|
|
if ! jq -e --arg phase "$phase" '
|
|
type == "object" and .status == "success" and
|
|
(.data | type == "object") and
|
|
(.data.activeTargets | type == "array") and
|
|
(.data.droppedTargets | type == "array") and
|
|
all(.data.droppedTargets[]?;
|
|
(.scrapePool | type == "string") and
|
|
(if $phase == "target-initial"
|
|
then (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not)
|
|
else (.scrapePool | test("(?:^|/)probe(?:/|$)"; "i") | not)
|
|
end))
|
|
' "$path" >/dev/null 2>&1; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
}
|
|
|
|
validate_probe_series_phase() {
|
|
local path="${1--}" phase="${2-target-initial}"
|
|
if ! jq -e --arg phase "$phase" '
|
|
type == "object" and .status == "success" and (.data | type == "array") and
|
|
(if $phase == "target-initial" then (.data | length) == 0
|
|
elif $phase == "post-substrate" then
|
|
(.data | length) > 0 and
|
|
all(.data[];
|
|
type == "object" and (."__name__" | type == "string") and
|
|
(."__name__" | startswith("probe_")) and
|
|
.namespace == "observability")
|
|
else false end)
|
|
' "$path" >/dev/null 2>&1; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
}
|
|
|
|
extract_metric_names() {
|
|
local path="${1--}" names
|
|
if ! names="$(jq -er '
|
|
if type == "object" and .status == "success" then
|
|
if (.data | type == "object") and (.data | length > 0) and
|
|
all(.data | keys[]; test("^[a-zA-Z_:][a-zA-Z0-9_:]*$"))
|
|
then .data | keys | sort | .[]
|
|
elif (.data | type == "array") and (.data | length > 0) and
|
|
all(.data[];
|
|
type == "object" and (.target | type == "object") and
|
|
(.metric | type == "string") and
|
|
(.metric | test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) and
|
|
(.type | type == "string") and (.help | type == "string") and
|
|
(.unit | type == "string"))
|
|
then [.data[].metric] | unique | sort | .[]
|
|
else error("invalid metadata") end
|
|
else error("invalid metadata") end
|
|
' "$path" 2>/dev/null)"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
printf '%s\n' "$names"
|
|
}
|
|
|
|
extract_recording_rule_names() {
|
|
local path="${1--}" names
|
|
if ! names="$(jq -er '
|
|
if type == "object" and .status == "success" and
|
|
(.data | type == "object") and (.data.groups | type == "array") and
|
|
all(.data.groups[]; (.rules | type == "array"))
|
|
then [.data.groups[].rules[] | select(.type == "recording") | .name] |
|
|
if all(.[]; type == "string" and test("^[a-zA-Z_:][a-zA-Z0-9_:]*$"))
|
|
then unique | sort | join("\n") else error("invalid recording rule") end
|
|
else error("invalid rules") end
|
|
' "$path" 2>/dev/null)"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
[[ -z "$names" ]] || printf '%s\n' "$names"
|
|
}
|
|
|
|
reject_recording_rule_collisions() {
|
|
local rules_path="${1-}" metadata_path="${2-}" rules metrics name
|
|
if ! rules="$(extract_recording_rule_names "$rules_path")" ||
|
|
! metrics="$(extract_metric_names "$metadata_path")"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
reject_metric_name_collisions "$rules" "$metrics"
|
|
}
|
|
|
|
reject_metric_name_collisions() {
|
|
local rules="${1-}" metrics="${2-}" name
|
|
while IFS= read -r name; do
|
|
[[ -z "$name" ]] && continue
|
|
if grep -Fqx -- "$name" <<<"$rules"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
done <<<"$metrics"
|
|
}
|
|
|
|
validate_metric_names_for_pool() {
|
|
local metrics="${1-}" pool="${2-}" phase="${3-target-initial}"
|
|
if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then
|
|
grep -Fqx probe_success <<<"$metrics" || { inventory_reject; return; }
|
|
elif grep -Eq '^probe_' <<<"$metrics"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
}
|
|
|
|
extract_series_label_names() {
|
|
local path="${1--}" result
|
|
if ! result="$(jq -ceS '
|
|
if type == "object" and .status == "success" and
|
|
(.data | type == "array") and (.data | length > 0) and
|
|
all(.data[]; type == "object") and
|
|
all([.data[] | keys[]][]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$"))
|
|
then [.data[] | keys[]] | unique | sort
|
|
else error("invalid series") end
|
|
' "$path" 2>/dev/null)"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
printf '%s\n' "$result"
|
|
}
|
|
|
|
prometheus_selector_for_labels() {
|
|
local labels_json="${1-}"
|
|
jq -cer '
|
|
if type == "object" and length > 0 and
|
|
all(keys[]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$")) and
|
|
all(.[]; type == "string" and (explode | all(. >= 32 and . != 127)))
|
|
then "{" + (to_entries | sort_by(.key) |
|
|
map((if (.key | test("^[a-zA-Z_][a-zA-Z0-9_]*$"))
|
|
then .key else (.key | @json) end) + "=" + (.value | @json)) | join(",")) + "}"
|
|
else error("invalid target labels") end
|
|
' <<<"$labels_json" 2>/dev/null || inventory_reject
|
|
}
|
|
|
|
inventory_provenance() {
|
|
case "${1-}" in
|
|
podMonitor/platform-data/platform-postgres/0|\
|
|
serviceMonitor/gitea/gitea/0|\
|
|
serviceMonitor/keycloak/keycloak/0|\
|
|
serviceMonitor/kube-system/traefik/0|\
|
|
serviceMonitor/object-storage/aistor-bucket-usage/0)
|
|
printf '%s\n' 'task6-owned'
|
|
;;
|
|
serviceMonitor/observability/grafana/0)
|
|
printf '%s\n' 'preexisting-verify-only'
|
|
;;
|
|
serviceMonitor/observability/blackbox-exporter/0|probe/observability/platform-*)
|
|
printf '%s\n' 'substrate-owned'
|
|
;;
|
|
*)
|
|
printf '%s\n' 'core'
|
|
;;
|
|
esac
|
|
}
|
|
|
|
validate_output_directory() {
|
|
local directory="${1-}" canonical uid mode current component
|
|
if [[ -z "$directory" || "$directory" != /* || "$directory" == *[$'\t\r\n']* ||
|
|
! -d "$directory" || -L "$directory" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if ! canonical="$(realpath -e -- "$directory" 2>/dev/null)" || [[ "$canonical" != "$directory" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
current=''
|
|
IFS='/' read -r -a components <<<"${directory#/}"
|
|
for component in "${components[@]}"; do
|
|
[[ -n "$component" && "$component" != '.' && "$component" != '..' ]] || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
current="$current/$component"
|
|
if [[ -L "$current" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
done
|
|
if ! uid="$(stat -c %u -- "$directory" 2>/dev/null)" ||
|
|
! mode="$(stat -c %a -- "$directory" 2>/dev/null)" ||
|
|
[[ "$uid" != "$(id -u)" || "$mode" != 700 ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if [[ -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
}
|
|
|
|
inventory_kubectl() {
|
|
case "${1-}" in
|
|
config)
|
|
case "${2-}" in current-context|view) ;; *) inventory_reject; return ;; esac
|
|
;;
|
|
get) ;;
|
|
auth)
|
|
[[ "${2-}" == can-i ]] || { inventory_reject; return; }
|
|
;;
|
|
*)
|
|
inventory_reject
|
|
return
|
|
;;
|
|
esac
|
|
timeout --signal=TERM --kill-after=2s 15s \
|
|
kubectl --request-timeout=10s "$@" 2>/dev/null
|
|
}
|
|
|
|
inventory_process_snapshot() {
|
|
local pid="${1-}" stat_line remainder state start_time
|
|
[[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] || return 1
|
|
IFS= read -r stat_line <"/proc/$pid/stat" || return 1
|
|
remainder="${stat_line##*) }"
|
|
state="${remainder%% *}"
|
|
start_time="$(awk '{print $20}' <<<"$remainder")" || return 1
|
|
[[ "$state" =~ ^[A-Z]$ && "$start_time" =~ ^[0-9]+$ ]] || return 1
|
|
printf '%s:%s\n' "$state" "$start_time"
|
|
}
|
|
|
|
inventory_port_forward_is_owned() {
|
|
local pid="${INVENTORY_PORT_FORWARD_PID:-}" expected="${INVENTORY_PORT_FORWARD_START:-}" snapshot
|
|
[[ "$pid" =~ ^[1-9][0-9]*$ && "$expected" =~ ^[0-9]+$ ]] || return 1
|
|
snapshot="$(inventory_process_snapshot "$pid")" || return 1
|
|
[[ "${snapshot#*:}" == "$expected" ]]
|
|
}
|
|
|
|
inventory_stop_port_forward() {
|
|
local pid="${INVENTORY_PORT_FORWARD_PID:-}" snapshot state attempt
|
|
[[ -n "$pid" ]] || return 0
|
|
if ! inventory_port_forward_is_owned; then
|
|
INVENTORY_PORT_FORWARD_PID=''
|
|
INVENTORY_PORT_FORWARD_START=''
|
|
return 0
|
|
fi
|
|
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
for ((attempt = 0; attempt < 40; attempt++)); do
|
|
if ! inventory_port_forward_is_owned; then break; fi
|
|
snapshot="$(inventory_process_snapshot "$pid")" || break
|
|
state="${snapshot%%:*}"
|
|
[[ "$state" == Z || "$state" == X ]] && break
|
|
sleep 0.025
|
|
done
|
|
|
|
if inventory_port_forward_is_owned; then
|
|
snapshot="$(inventory_process_snapshot "$pid")" || snapshot=''
|
|
state="${snapshot%%:*}"
|
|
if [[ "$state" != Z && "$state" != X ]]; then
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
fi
|
|
fi
|
|
for ((attempt = 0; attempt < 40; attempt++)); do
|
|
if ! inventory_port_forward_is_owned; then break; fi
|
|
snapshot="$(inventory_process_snapshot "$pid")" || break
|
|
state="${snapshot%%:*}"
|
|
[[ "$state" == Z || "$state" == X ]] && break
|
|
sleep 0.025
|
|
done
|
|
if inventory_port_forward_is_owned; then
|
|
snapshot="$(inventory_process_snapshot "$pid")" || snapshot=''
|
|
state="${snapshot%%:*}"
|
|
if [[ "$state" != Z && "$state" != X ]]; then return 1; fi
|
|
fi
|
|
INVENTORY_PORT_FORWARD_PID=''
|
|
INVENTORY_PORT_FORWARD_START=''
|
|
}
|
|
|
|
build_inventory_json() {
|
|
local target_rows="${1-}" phase="${2-}" timestamp="${3-}"
|
|
[[ -n "$target_rows" && -f "$target_rows" && ! -L "$target_rows" &&
|
|
-n "$phase" && -n "$timestamp" ]] || { inventory_reject; return; }
|
|
jq -cnS --slurpfile targets "$target_rows" \
|
|
--arg schema "$INVENTORY_SCHEMA" --arg phase "$phase" --arg captured_at_utc "$timestamp" \
|
|
--arg context "$INVENTORY_CONTEXT" --arg api_server "$INVENTORY_API_SERVER" \
|
|
--arg prometheus "$INVENTORY_PROMETHEUS_NAMESPACE/$INVENTORY_PROMETHEUS_NAME" '
|
|
{schema:$schema,phase:$phase,captured_at_utc:$captured_at_utc,context:$context,
|
|
api_server:$api_server,prometheus:$prometheus,preexisting_verify_only:["grafana"],
|
|
targets:($targets | sort_by(.scrape_pool))}
|
|
'
|
|
}
|
|
|
|
validate_post_substrate_query_body() {
|
|
local path="${1--}" kind="${2-}" expected expected_tls now
|
|
expected="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels] | sort_by(.job,.instance)')" || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
expected_tls="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels | select(.instance | startswith("https://"))] | sort_by(.job,.instance)')" || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
now="$(date -u +%s)" || { inventory_reject; return; }
|
|
if ! jq -e --arg kind "$kind" --argjson expected "$expected" \
|
|
--argjson expectedTls "$expected_tls" --argjson now "$now" '
|
|
type == "object" and .status == "success" and .data.resultType == "vector" and
|
|
(.data.result | type == "array") and
|
|
(if $kind == "tls" then
|
|
([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expectedTls and
|
|
all(.data.result[]; (.value[1] | tonumber) > $now)
|
|
else
|
|
([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expected and
|
|
(if $kind == "success" then all(.data.result[]; .value[1] == "1")
|
|
elif $kind == "duration" then all(.data.result[]; (.value[1] | tonumber) >= 0)
|
|
elif $kind == "status" then
|
|
all(.data.result[];
|
|
if .metric.job == "blackbox-private-edge" then .value[1] == "403"
|
|
else .value[1] == "200" end)
|
|
else false end)
|
|
end)
|
|
' "$path" >/dev/null 2>&1; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
}
|
|
|
|
inventory_validate_post_substrate_metrics() {
|
|
local body spec
|
|
for spec in \
|
|
'success:probe_success{namespace="observability"}' \
|
|
'status:probe_http_status_code{namespace="observability"}' \
|
|
'duration:probe_duration_seconds{namespace="observability"}' \
|
|
'tls:probe_ssl_earliest_cert_expiry{namespace="observability"}'; do
|
|
if ! inventory_prometheus_json body '/api/v1/query' --get \
|
|
--data-urlencode "query=${spec#*:}" ||
|
|
! printf '%s' "$body" | validate_post_substrate_query_body - "${spec%%:*}"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
body=''
|
|
done
|
|
}
|
|
|
|
inventory_capture_main() {
|
|
local output_dir phase context api prometheus_json pods_json permission
|
|
local port='' attempt start_attempt forward_ready=0 timestamp snapshot
|
|
local targets_raw='' dropped_raw='' rules_raw='' probe_raw='' metadata_raw='' series_raw='' recording_names=''
|
|
local row pool scrape_url labels selector metric_names metric
|
|
local metric_labels series_selector series_count metrics_path metrics_json provenance
|
|
local target_rows counter=0 inventory hash actual_files
|
|
|
|
if (($# != 5)) || [[ "$1" != --execute || "$2" != --phase ||
|
|
("$3" != target-initial && "$3" != post-substrate) ||
|
|
"$4" != --output-dir || -z "$5" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
phase="$3"
|
|
output_dir="$5"
|
|
|
|
for permission in jq kubectl curl timeout realpath stat find mktemp sha256sum sync grep head; do
|
|
command -v "$permission" >/dev/null 2>&1 || { inventory_reject; return; }
|
|
done
|
|
validate_output_directory "$output_dir" || return
|
|
|
|
INVENTORY_WORK_DIR="$(mktemp -d /tmp/platform-observability-inventory-capture.XXXXXX)" || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
chmod 0700 "$INVENTORY_WORK_DIR" || { inventory_reject; return; }
|
|
INVENTORY_STAGE_JSON=''
|
|
INVENTORY_STAGE_SHA=''
|
|
INVENTORY_OUTPUT_DIR="$output_dir"
|
|
INVENTORY_CREATED_JSON=0
|
|
INVENTORY_CREATED_SHA=0
|
|
INVENTORY_JSON_ID=''
|
|
INVENTORY_SHA_ID=''
|
|
INVENTORY_SUCCESS=0
|
|
INVENTORY_PORT_FORWARD_PID=''
|
|
INVENTORY_PORT_FORWARD_START=''
|
|
|
|
trap inventory_capture_cleanup EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
if ! context="$(inventory_kubectl config current-context)" || [[ "$context" != "$INVENTORY_CONTEXT" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if ! api="$(inventory_kubectl config view --minify -o json | jq -er '.clusters | if length == 1 then .[0].cluster.server else error("cluster count") end' 2>/dev/null)" ||
|
|
[[ "$api" != "$INVENTORY_API_SERVER" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if ! inventory_kubectl get --raw=/readyz >/dev/null; then inventory_reject; return; fi
|
|
|
|
for permission in \
|
|
'get pods' \
|
|
'get services' \
|
|
'get prometheuses.monitoring.coreos.com' \
|
|
'create pods/portforward'; do
|
|
if ! inventory_kubectl auth can-i $permission --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" | grep -Fxq yes; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
done
|
|
|
|
if ! prometheus_json="$(inventory_kubectl get prometheus "$INVENTORY_PROMETHEUS_NAME" \
|
|
--namespace "$INVENTORY_PROMETHEUS_NAMESPACE" -o json)" ||
|
|
! jq -e --arg namespace "$INVENTORY_PROMETHEUS_NAMESPACE" --arg name "$INVENTORY_PROMETHEUS_NAME" '
|
|
.kind == "Prometheus" and .metadata.namespace == $namespace and .metadata.name == $name
|
|
' <<<"$prometheus_json" >/dev/null 2>&1; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if ! pods_json="$(inventory_kubectl get pods --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" \
|
|
-l "prometheus=$INVENTORY_PROMETHEUS_NAME" -o json)" ||
|
|
! jq -e '
|
|
(.items | length == 1) and .items[0].status.phase == "Running" and
|
|
any(.items[0].status.conditions[]; .type == "Ready" and .status == "True") and
|
|
any(.items[0].status.containerStatuses[]; .name == "prometheus" and .ready == true)
|
|
' <<<"$pods_json" >/dev/null 2>&1; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
|
|
: >"$INVENTORY_WORK_DIR/port-forward.log"
|
|
for ((start_attempt = 0; start_attempt < 10; start_attempt++)); do
|
|
port=$((20000 + ((RANDOM + $$ + start_attempt * 7919) % 40000)))
|
|
kubectl --request-timeout=10s -n "$INVENTORY_PROMETHEUS_NAMESPACE" \
|
|
port-forward --address 127.0.0.1 \
|
|
"service/$INVENTORY_PROMETHEUS_SERVICE" "$port:9090" \
|
|
>/dev/null 2>"$INVENTORY_WORK_DIR/port-forward.log" &
|
|
INVENTORY_PORT_FORWARD_PID=$!
|
|
snapshot="$(inventory_process_snapshot "$INVENTORY_PORT_FORWARD_PID")" || {
|
|
inventory_reject
|
|
return
|
|
}
|
|
INVENTORY_PORT_FORWARD_START="${snapshot#*:}"
|
|
INVENTORY_PROMETHEUS_BASE="http://127.0.0.1:$port"
|
|
for ((attempt = 0; attempt < 40; attempt++)); do
|
|
if ! kill -0 "$INVENTORY_PORT_FORWARD_PID" 2>/dev/null; then break; fi
|
|
if inventory_prometheus_ready; then
|
|
forward_ready=1
|
|
break 2
|
|
fi
|
|
sleep 0.05
|
|
done
|
|
inventory_stop_port_forward || { inventory_reject; return; }
|
|
done
|
|
if [[ "$forward_ready" != 1 || -z "$INVENTORY_PORT_FORWARD_PID" ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
|
|
if ! inventory_prometheus_json targets_raw '/api/v1/targets' --get --data 'state=active' ||
|
|
! printf '%s' "$targets_raw" | validate_targets_json - "$phase" ||
|
|
! inventory_prometheus_json dropped_raw '/api/v1/targets' --get --data 'state=dropped' ||
|
|
! printf '%s' "$dropped_raw" | validate_dropped_targets_json - "$phase" ||
|
|
! inventory_prometheus_json rules_raw '/api/v1/rules' ||
|
|
! recording_names="$(printf '%s' "$rules_raw" | extract_recording_rule_names -)" ||
|
|
! inventory_prometheus_json probe_raw '/api/v1/series' --get \
|
|
--data-urlencode 'match[]={__name__=~"probe_.+"}' ||
|
|
! printf '%s' "$probe_raw" | validate_probe_series_phase - "$phase"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
if [[ "$phase" == post-substrate ]] && ! inventory_validate_post_substrate_metrics; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
dropped_raw=''
|
|
rules_raw=''
|
|
probe_raw=''
|
|
|
|
target_rows="$INVENTORY_WORK_DIR/target-rows.jsonl"
|
|
: >"$target_rows"
|
|
while IFS= read -r row; do
|
|
pool="$(jq -er '.scrapePool' <<<"$row" 2>/dev/null)" || { inventory_reject; return; }
|
|
scrape_url="$(jq -er '.scrapeUrl' <<<"$row" 2>/dev/null)" || { inventory_reject; return; }
|
|
if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then
|
|
scrape_url="$(sanitize_scrape_url "$scrape_url" strip-query)" || return
|
|
else
|
|
scrape_url="$(sanitize_scrape_url "$scrape_url")" || return
|
|
fi
|
|
labels="$(jq -ceS '.labels' <<<"$row" 2>/dev/null)" || { inventory_reject; return; }
|
|
selector="$(prometheus_selector_for_labels "$labels")" || return
|
|
counter=$((counter + 1))
|
|
if ! inventory_prometheus_json metadata_raw '/api/v1/targets/metadata' --get \
|
|
--data-urlencode "match_target=$selector" --data 'limit=10000'; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
metric_names="$(printf '%s' "$metadata_raw" | extract_metric_names -)" || return
|
|
metadata_raw=''
|
|
reject_metric_name_collisions "$recording_names" "$metric_names" || return
|
|
validate_metric_names_for_pool "$metric_names" "$pool" "$phase" || return
|
|
|
|
metrics_path="$INVENTORY_WORK_DIR/metrics-$counter.jsonl"
|
|
: >"$metrics_path"
|
|
while IFS= read -r metric; do
|
|
series_selector="$(prometheus_selector_for_labels "$(
|
|
jq -ce --arg metric "$metric" '. + {"__name__":$metric}' <<<"$labels" 2>/dev/null
|
|
)")" || return
|
|
if ! inventory_prometheus_json series_raw '/api/v1/series' --get \
|
|
--data-urlencode "match[]=$series_selector"; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
series_count="$(printf '%s' "$series_raw" | jq -er '
|
|
if type == "object" and .status == "success" and
|
|
(.data | type == "array") and all(.data[]; type == "object")
|
|
then .data | length else error("invalid series") end
|
|
' 2>/dev/null)" || { inventory_reject; return; }
|
|
if (( series_count == 0 )); then
|
|
series_raw=''
|
|
continue
|
|
fi
|
|
metric_labels="$(printf '%s' "$series_raw" | extract_series_label_names -)" || return
|
|
series_raw=''
|
|
jq -cnS --arg name "$metric" --argjson label_names "$metric_labels" \
|
|
'{name:$name,label_names:$label_names}' >>"$metrics_path" || { inventory_reject; return; }
|
|
done <<<"$metric_names"
|
|
[[ -s "$metrics_path" ]] || { inventory_reject; return; }
|
|
metrics_json="$(jq -csS 'sort_by(.name)' "$metrics_path" 2>/dev/null)" || { inventory_reject; return; }
|
|
provenance="$(inventory_provenance "$pool")"
|
|
jq -cnS --arg scrape_pool "$pool" --arg scrape_url "$scrape_url" \
|
|
--arg provenance "$provenance" --argjson metrics "$metrics_json" \
|
|
'{scrape_pool:$scrape_pool,scrape_url:$scrape_url,health:"up",last_error:"",
|
|
provenance:$provenance,metrics:$metrics}' >>"$target_rows" || { inventory_reject; return; }
|
|
done < <(printf '%s' "$targets_raw" | jq -c '.data.activeTargets | sort_by(.scrapePool)[]')
|
|
targets_raw=''
|
|
|
|
timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" || { inventory_reject; return; }
|
|
inventory="$(build_inventory_json "$target_rows" "$phase" "$timestamp")" || return
|
|
|
|
validate_output_directory "$output_dir" || return
|
|
INVENTORY_STAGE_JSON="$(mktemp "$output_dir/.inventory.json.tmp.XXXXXX")" || { inventory_reject; return; }
|
|
chmod 0600 "$INVENTORY_STAGE_JSON" || { inventory_reject; return; }
|
|
printf '%s\n' "$inventory" >"$INVENTORY_STAGE_JSON" || { inventory_reject; return; }
|
|
sync -f "$INVENTORY_STAGE_JSON" >/dev/null 2>&1 || { inventory_reject; return; }
|
|
INVENTORY_JSON_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_JSON")" || { inventory_reject; return; }
|
|
hash="$(sha256sum -- "$INVENTORY_STAGE_JSON" | awk '{print $1}')" || { inventory_reject; return; }
|
|
INVENTORY_STAGE_SHA="$(mktemp "$output_dir/.inventory.sha256.tmp.XXXXXX")" || { inventory_reject; return; }
|
|
chmod 0600 "$INVENTORY_STAGE_SHA" || { inventory_reject; return; }
|
|
printf '%s inventory.json\n' "$hash" >"$INVENTORY_STAGE_SHA" || { inventory_reject; return; }
|
|
sync -f "$INVENTORY_STAGE_SHA" >/dev/null 2>&1 || { inventory_reject; return; }
|
|
INVENTORY_SHA_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_SHA")" || { inventory_reject; return; }
|
|
|
|
if ! ln -- "$INVENTORY_STAGE_JSON" "$output_dir/inventory.json" 2>/dev/null; then inventory_reject; return; fi
|
|
INVENTORY_CREATED_JSON=1
|
|
rm -- "$INVENTORY_STAGE_JSON" || { inventory_reject; return; }
|
|
INVENTORY_STAGE_JSON=''
|
|
if ! ln -- "$INVENTORY_STAGE_SHA" "$output_dir/inventory.sha256" 2>/dev/null; then inventory_reject; return; fi
|
|
INVENTORY_CREATED_SHA=1
|
|
rm -- "$INVENTORY_STAGE_SHA" || { inventory_reject; return; }
|
|
INVENTORY_STAGE_SHA=''
|
|
sync -f "$output_dir/inventory.json" >/dev/null 2>&1 || { inventory_reject; return; }
|
|
sync -f "$output_dir/inventory.sha256" >/dev/null 2>&1 || { inventory_reject; return; }
|
|
sync -f "$output_dir" >/dev/null 2>&1 || { inventory_reject; return; }
|
|
|
|
actual_files="$(find "$output_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)"
|
|
if [[ "$actual_files" != $'inventory.json\ninventory.sha256' ]] ||
|
|
[[ ! -f "$output_dir/inventory.json" || -L "$output_dir/inventory.json" ]] ||
|
|
[[ ! -f "$output_dir/inventory.sha256" || -L "$output_dir/inventory.sha256" ]] ||
|
|
[[ "$(stat -c %a -- "$output_dir/inventory.json")" != 600 ]] ||
|
|
[[ "$(stat -c %a -- "$output_dir/inventory.sha256")" != 600 ]]; then
|
|
inventory_reject
|
|
return
|
|
fi
|
|
INVENTORY_SUCCESS=1
|
|
printf 'OK: %s inventory captured\n' "$phase"
|
|
}
|
|
|
|
inventory_prometheus_json() {
|
|
local destination_name="$1" api_path="$2" response
|
|
local LC_ALL=C
|
|
local -n destination="$destination_name"
|
|
shift 2
|
|
if ! response="$(
|
|
set -o pipefail
|
|
curl --disable --silent --show-error --fail --connect-timeout 3 --max-time 10 \
|
|
--max-filesize "$INVENTORY_MAX_RESPONSE_BYTES" --output - \
|
|
"$@" "$INVENTORY_PROMETHEUS_BASE$api_path" 2>/dev/null |
|
|
LC_ALL=C head -c "$((INVENTORY_MAX_RESPONSE_BYTES + 1))"
|
|
pipeline_status=$?
|
|
printf '\034'
|
|
exit "$pipeline_status"
|
|
)"; then
|
|
response=''
|
|
inventory_reject
|
|
return
|
|
fi
|
|
[[ "$response" == *$'\034' ]] || { response=''; inventory_reject; return; }
|
|
response="${response%?}"
|
|
if (( ${#response} > INVENTORY_MAX_RESPONSE_BYTES )); then
|
|
response=''
|
|
inventory_reject
|
|
return
|
|
fi
|
|
destination="$response"
|
|
}
|
|
|
|
inventory_prometheus_ready() {
|
|
curl --disable --silent --show-error --fail --connect-timeout 1 --max-time 1 \
|
|
--max-filesize 1024 --output /dev/null "$INVENTORY_PROMETHEUS_BASE/-/ready" 2>/dev/null
|
|
}
|
|
|
|
inventory_capture_cleanup() {
|
|
local rc=$?
|
|
trap - EXIT HUP INT TERM
|
|
if [[ -n "${INVENTORY_PORT_FORWARD_PID:-}" ]]; then
|
|
inventory_stop_port_forward || true
|
|
fi
|
|
if [[ -n "${INVENTORY_STAGE_JSON:-}" && -f "$INVENTORY_STAGE_JSON" && ! -L "$INVENTORY_STAGE_JSON" ]]; then
|
|
rm -- "$INVENTORY_STAGE_JSON"
|
|
fi
|
|
if [[ -n "${INVENTORY_STAGE_SHA:-}" && -f "$INVENTORY_STAGE_SHA" && ! -L "$INVENTORY_STAGE_SHA" ]]; then
|
|
rm -- "$INVENTORY_STAGE_SHA"
|
|
fi
|
|
if [[ "${INVENTORY_SUCCESS:-0}" != 1 ]]; then
|
|
if [[ "${INVENTORY_CREATED_SHA:-0}" == 1 ]]; then
|
|
inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.sha256 "${INVENTORY_SHA_ID:-}"
|
|
fi
|
|
if [[ "${INVENTORY_CREATED_JSON:-0}" == 1 ]]; then
|
|
inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.json "${INVENTORY_JSON_ID:-}"
|
|
fi
|
|
if [[ -n "${INVENTORY_OUTPUT_DIR:-}" && -d "$INVENTORY_OUTPUT_DIR" && ! -L "$INVENTORY_OUTPUT_DIR" ]]; then
|
|
sync -f "$INVENTORY_OUTPUT_DIR" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
if [[ -n "${INVENTORY_WORK_DIR:-}" && -d "$INVENTORY_WORK_DIR" && ! -L "$INVENTORY_WORK_DIR" ]]; then
|
|
case "$INVENTORY_WORK_DIR" in
|
|
/tmp/platform-observability-inventory-capture.*) rm -rf -- "$INVENTORY_WORK_DIR" ;;
|
|
esac
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
|
|
inventory_remove_created_file() {
|
|
local directory="${1-}" basename="${2-}" expected_id="${3-}" path actual_id canonical
|
|
[[ -n "$directory" && -n "$basename" && -n "$expected_id" && -d "$directory" && ! -L "$directory" ]] || return 0
|
|
canonical="$(realpath -e -- "$directory" 2>/dev/null)" || return 0
|
|
[[ "$canonical" == "$directory" ]] || return 0
|
|
path="$directory/$basename"
|
|
[[ -f "$path" && ! -L "$path" ]] || return 0
|
|
actual_id="$(stat -c %d:%i -- "$path" 2>/dev/null)" || return 0
|
|
[[ "$actual_id" == "$expected_id" ]] || return 0
|
|
rm -- "$path"
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
inventory_capture_main "$@"
|
|
fi
|