613 lines
21 KiB
Bash
Executable File
613 lines
21 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
|
|
DASHBOARD_ROOT="$ROOT/services/observability/dashboards/platform"
|
|
METRIC_ROOT=${PLATFORM_OBSERVABILITY_METRIC_ROOT:-/tmp/platform-observability-metrics.VUpsZn}
|
|
TARGET_INVENTORY="$METRIC_ROOT/target-initial/inventory.json"
|
|
POST_INVENTORY="$METRIC_ROOT/post-substrate/inventory.json"
|
|
PROMETHEUS_IMAGE='quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
|
|
WORK_DIR=''
|
|
PROMETHEUS_BASE_URL=''
|
|
PORT_FORWARD_PID=''
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
pass() {
|
|
printf 'PASS: %s\n' "$*"
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
|
|
}
|
|
|
|
usage() {
|
|
printf 'Usage: %s [--prometheus-base-url http://127.0.0.1:PORT]\n' "${0##*/}"
|
|
}
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--prometheus-base-url)
|
|
[[ $# -ge 2 ]] || fail '--prometheus-base-url requires a value'
|
|
[[ -z "$PROMETHEUS_BASE_URL" ]] || fail '--prometheus-base-url may be specified only once'
|
|
PROMETHEUS_BASE_URL=$2
|
|
shift 2
|
|
;;
|
|
--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
fail "unknown argument: $1"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -n "$PROMETHEUS_BASE_URL" ]]; then
|
|
[[ "$PROMETHEUS_BASE_URL" =~ ^http://(127\.0\.0\.1|localhost):([1-9][0-9]{0,4})$ ]] \
|
|
|| fail '--prometheus-base-url must be an exact loopback HTTP origin without a path'
|
|
(( BASH_REMATCH[2] <= 65535 )) \
|
|
|| fail '--prometheus-base-url port exceeds 65535'
|
|
fi
|
|
}
|
|
|
|
cleanup() {
|
|
local rc=$?
|
|
trap - EXIT
|
|
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
|
|
kill "$PORT_FORWARD_PID" 2>/dev/null || true
|
|
wait "$PORT_FORWARD_PID" 2>/dev/null || true
|
|
fi
|
|
if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then
|
|
rm -rf -- "$WORK_DIR"
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
|
|
verify_inventory() {
|
|
local phase=$1
|
|
local inventory="$METRIC_ROOT/$phase/inventory.json"
|
|
local checksum="$METRIC_ROOT/$phase/inventory.sha256"
|
|
local expected actual
|
|
|
|
[[ -f "$inventory" ]] || fail "$phase inventory is absent"
|
|
[[ -f "$checksum" ]] || fail "$phase inventory checksum is absent"
|
|
jq -e --arg phase "$phase" '
|
|
.schema == "platform-observability-metric-inventory/v1"
|
|
and .phase == $phase
|
|
and (.targets | type == "array" and length > 0)
|
|
and all(.targets[]; .health == "up" and (.metrics | type == "array" and length > 0))
|
|
' "$inventory" >/dev/null || fail "$phase inventory contract is invalid"
|
|
|
|
read -r expected checksum_name < "$checksum"
|
|
[[ "$checksum_name" == 'inventory.json' ]] || fail "$phase checksum names an unexpected file"
|
|
actual=$(sha256sum "$inventory" | awk '{print $1}')
|
|
[[ "$actual" == "$expected" ]] || fail "$phase inventory checksum mismatch"
|
|
pass "$phase inventory contract and checksum"
|
|
}
|
|
|
|
inventory_has_metric() {
|
|
local inventory=$1
|
|
local metric=$2
|
|
jq -e --arg metric "$metric" 'any(.targets[].metrics[]; .name == $metric)' "$inventory" >/dev/null
|
|
}
|
|
|
|
inventory_metric_has_label() {
|
|
local inventory=$1
|
|
local metric=$2
|
|
local label=$3
|
|
jq -e --arg metric "$metric" --arg label "$label" '
|
|
any(.targets[].metrics[]; .name == $metric and (.label_names | index($label) != null))
|
|
' "$inventory" >/dev/null
|
|
}
|
|
|
|
validate_inventory_references() {
|
|
local dashboard=$1
|
|
local inventory=$2
|
|
local expression selector metric matchers matcher label token
|
|
local inventory_metrics referenced_metrics found
|
|
|
|
inventory_metrics=$(jq -r '[.targets[].metrics[].name] | unique[]' "$inventory")
|
|
|
|
while IFS= read -r expression; do
|
|
[[ -n "$expression" ]] || fail "empty PromQL expression in $dashboard"
|
|
referenced_metrics=''
|
|
|
|
while IFS= read -r selector; do
|
|
[[ -n "$selector" ]] || continue
|
|
metric=${selector%%\{*}
|
|
grep -Fxq "$metric" <<<"$inventory_metrics" \
|
|
|| fail "$dashboard references absent metric $metric"
|
|
if ! grep -Fxq "$metric" <<<"$referenced_metrics"; then
|
|
referenced_metrics+="${metric}"$'\n'
|
|
fi
|
|
matchers=${selector#*\{}
|
|
matchers=${matchers%\}}
|
|
while IFS= read -r matcher; do
|
|
matcher=${matcher#"${matcher%%[![:space:]]*}"}
|
|
[[ -n "$matcher" ]] || continue
|
|
label=$(sed -E 's/^([A-Za-z_][A-Za-z0-9_]*).*/\1/' <<<"$matcher")
|
|
[[ "$label" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \
|
|
|| fail "$dashboard has an unauditable label matcher in $expression"
|
|
inventory_metric_has_label "$inventory" "$metric" "$label" \
|
|
|| fail "$dashboard references absent label $metric.$label"
|
|
done < <(tr ',' '\n' <<<"$matchers")
|
|
done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]*\}' <<<"$expression" || true)
|
|
|
|
[[ -n "$referenced_metrics" ]] || fail "$dashboard expression has no inventory metric: $expression"
|
|
|
|
while IFS= read -r token; do
|
|
[[ -n "$token" ]] || continue
|
|
if grep -Fxq "$token" <<<"$inventory_metrics"; then
|
|
grep -Fxq "$token" <<<"$referenced_metrics" \
|
|
|| fail "$dashboard uses inventory metric $token without an auditable selector"
|
|
fi
|
|
done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$expression" | sort -u)
|
|
|
|
while IFS= read -r label; do
|
|
[[ -n "$label" ]] || continue
|
|
found=0
|
|
while IFS= read -r metric; do
|
|
[[ -n "$metric" ]] || continue
|
|
if inventory_metric_has_label "$inventory" "$metric" "$label"; then
|
|
found=1
|
|
break
|
|
fi
|
|
done <<<"$referenced_metrics"
|
|
[[ "$found" -eq 1 ]] || fail "$dashboard groups by absent inventory label $label"
|
|
done < <(
|
|
grep -oE '(by|without)[[:space:]]*\([^)]*\)' <<<"$expression" \
|
|
| sed -E 's/^[^(]*\(([^)]*)\)$/\1/' \
|
|
| tr ',' '\n' \
|
|
| sed -E 's/^[[:space:]]+|[[:space:]]+$//g' \
|
|
|| true
|
|
)
|
|
done < <(jq -r '.panels[].targets[].expr' "$dashboard")
|
|
}
|
|
|
|
validate_dashboard() {
|
|
local file=$1
|
|
local expected_uid=$2
|
|
local inventory=$3
|
|
local path="$DASHBOARD_ROOT/$file"
|
|
|
|
[[ -f "$path" ]] || fail "dashboard source is absent: $file"
|
|
jq -e --arg uid "$expected_uid" '
|
|
type == "object"
|
|
and .uid == $uid
|
|
and (.title | type == "string" and length > 0)
|
|
and .schemaVersion == 42
|
|
and .editable == false
|
|
and .refresh == "30s"
|
|
and (.templating.list == [])
|
|
and (.panels | type == "array" and length > 0)
|
|
and ([.panels[].id] | length == (unique | length))
|
|
and all(.panels[];
|
|
.type != "row"
|
|
and (.title | type == "string" and length > 0)
|
|
and (.gridPos.w > 0 and .gridPos.h > 0)
|
|
and .datasource.type == "prometheus"
|
|
and .datasource.uid == "prometheus"
|
|
and (.targets | type == "array" and length > 0)
|
|
and all(.targets[];
|
|
.datasource.type == "prometheus"
|
|
and .datasource.uid == "prometheus"
|
|
and (.expr | type == "string" and length > 0)
|
|
and (.refId | type == "string" and length > 0)
|
|
)
|
|
)
|
|
' "$path" >/dev/null || fail "dashboard structure is invalid: $file"
|
|
|
|
if jq -r '.title, .tags[], .panels[].title' "$path" \
|
|
| grep -Eiq '(^|[^[:alnum:]_])(spring|jvm|kafka|batch|backup)([^[:alnum:]_]|$)'; then
|
|
fail "dashboard contains a forbidden product area: $file"
|
|
fi
|
|
if jq -r '.panels[].targets[].expr' "$path" \
|
|
| grep -Eiq '(^|[^[:alnum:]_])(spring_|jvm_|kafka_|batch_|backup_)'; then
|
|
fail "dashboard contains a forbidden product query: $file"
|
|
fi
|
|
if jq -r '.. | strings' "$path" \
|
|
| grep -Eiq '(request[_ -]?id|trace[_ -]?id|username|raw[_ -]?url)'; then
|
|
fail "dashboard contains forbidden or high-cardinality content: $file"
|
|
fi
|
|
|
|
validate_inventory_references "$path" "$inventory"
|
|
pass "$file structure and inventory references"
|
|
}
|
|
|
|
validate_rendered_configmaps() {
|
|
local rendered=$1
|
|
local file name block
|
|
local -a files=(
|
|
kubernetes-node
|
|
workload-health
|
|
platform-services
|
|
observability-backends
|
|
https-endpoints
|
|
)
|
|
|
|
kubectl kustomize "$DASHBOARD_ROOT" > "$rendered"
|
|
[[ $(grep -c '^kind: ConfigMap$' "$rendered") -eq 5 ]] \
|
|
|| fail 'kustomization must render exactly five ConfigMaps'
|
|
|
|
for file in "${files[@]}"; do
|
|
name="grafana-dashboard-$file"
|
|
block=$(awk -v name="$name" '
|
|
BEGIN { RS="---" }
|
|
$0 ~ "name: " name "([[:space:]]|$)" { print }
|
|
' "$rendered")
|
|
[[ -n "$block" ]] || fail "rendered ConfigMap is absent: $name"
|
|
grep -Eq 'grafana_dashboard: ("1"|1)$' <<<"$block" \
|
|
|| fail "$name lacks grafana_dashboard=1"
|
|
grep -q 'observability.hyeonworks.com/instance: home' <<<"$block" \
|
|
|| fail "$name lacks the observability instance label"
|
|
grep -q 'observability.hyeonworks.com/owner: platform-observability' <<<"$block" \
|
|
|| fail "$name lacks the observability owner label"
|
|
grep -q " $file.json:" <<<"$block" \
|
|
|| fail "$name does not embed $file.json"
|
|
done
|
|
pass 'kustomization renders five stable labeled dashboard ConfigMaps'
|
|
}
|
|
|
|
validate_inventory_driven_omissions() {
|
|
local workload="$DASHBOARD_ROOT/workload-health.json"
|
|
local services="$DASHBOARD_ROOT/platform-services.json"
|
|
local endpoints="$DASHBOARD_ROOT/https-endpoints.json"
|
|
|
|
if inventory_has_metric "$TARGET_INVENTORY" kube_pod_container_status_last_terminated_reason; then
|
|
fail 'the OOM omission contract no longer matches target-initial inventory'
|
|
fi
|
|
if jq -r '.panels[].targets[].expr' "$workload" \
|
|
| grep -Fq 'kube_pod_container_status_last_terminated_reason'; then
|
|
fail 'workload dashboard guessed an OOM metric absent from target-initial inventory'
|
|
fi
|
|
pass 'OOM panel omitted: kube_pod_container_status_last_terminated_reason is absent from target-initial inventory'
|
|
|
|
jq -e '
|
|
([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})") != null)
|
|
and
|
|
([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})") != null)
|
|
' "$workload" >/dev/null \
|
|
|| fail 'PVC health panel must retain zero-valued Pending and Lost series instead of filtering healthy state away'
|
|
pass 'PVC health panel keeps visible zero-valued Pending and Lost aggregates'
|
|
|
|
if jq -e '
|
|
any(.targets[].metrics[];
|
|
(.name | startswith("traefik_"))
|
|
and (.name | endswith("_bucket"))
|
|
and (.name | test("duration|request"))
|
|
)
|
|
' "$TARGET_INVENTORY" >/dev/null; then
|
|
fail 'the Traefik latency omission contract no longer matches target-initial inventory'
|
|
fi
|
|
if jq -r '.panels[].targets[].expr' "$services" \
|
|
| grep -Eq 'histogram_quantile|traefik_.*_bucket'; then
|
|
fail 'service dashboard guessed a Traefik latency histogram absent from target-initial inventory'
|
|
fi
|
|
pass 'Traefik p95/p99 omitted: no request duration histogram bucket exists in target-initial inventory'
|
|
|
|
inventory_has_metric "$POST_INVENTORY" probe_success \
|
|
|| fail 'post-substrate inventory lacks probe_success for the platform boundary summary'
|
|
jq -e '
|
|
any(.panels[].targets[];
|
|
.expr == "min by (job) (probe_success{})"
|
|
)
|
|
' "$services" >/dev/null \
|
|
|| fail 'platform service dashboard lacks the inventory-backed boundary health summary'
|
|
pass 'platform service boundary summary uses post-substrate probe_success'
|
|
|
|
jq -e '
|
|
any(.panels[].targets[];
|
|
.expr == "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})"
|
|
)
|
|
' "$endpoints" >/dev/null \
|
|
|| fail 'HTTPS dashboard does not isolate the observed private-edge 403 boundary'
|
|
pass 'HTTPS dashboard isolates the observed blackbox-private-edge status boundary'
|
|
}
|
|
|
|
validate_promql_syntax() {
|
|
local rules=$1
|
|
local file expression index=0
|
|
local -a files=(
|
|
kubernetes-node.json
|
|
workload-health.json
|
|
platform-services.json
|
|
observability-backends.json
|
|
https-endpoints.json
|
|
)
|
|
|
|
{
|
|
printf 'groups:\n'
|
|
printf ' - name: dashboard.promql.syntax\n'
|
|
printf ' rules:\n'
|
|
for file in "${files[@]}"; do
|
|
while IFS= read -r expression; do
|
|
index=$((index + 1))
|
|
printf ' - record: dashboard_syntax_%d\n' "$index"
|
|
printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
|
|
done < <(jq -r '.panels[].targets[].expr' "$DASHBOARD_ROOT/$file")
|
|
done
|
|
} > "$rules"
|
|
|
|
if [[ -n ${PROMTOOL_BIN:-} ]]; then
|
|
"$PROMTOOL_BIN" check rules "$rules" >/dev/null
|
|
else
|
|
require_command docker
|
|
docker run --rm \
|
|
--entrypoint=/bin/promtool \
|
|
-v "$rules:/tmp/dashboard-rules.yaml:ro" \
|
|
"$PROMETHEUS_IMAGE" check rules /tmp/dashboard-rules.yaml >/dev/null
|
|
fi
|
|
pass "promtool parsed $index dashboard expressions"
|
|
}
|
|
|
|
validate_traefik_low_traffic_ratio() {
|
|
local rules="$WORK_DIR/traefik-ratio-rules.yaml"
|
|
local tests="$WORK_DIR/traefik-ratio-tests.yaml"
|
|
local expression
|
|
|
|
expression=$(jq -er '
|
|
.panels[]
|
|
| select(.title == "Traefik 5xx Ratio")
|
|
| .targets[]
|
|
| select(.refId == "A")
|
|
| .expr
|
|
' "$DASHBOARD_ROOT/platform-services.json") \
|
|
|| fail 'Traefik 5xx ratio expression is absent'
|
|
|
|
{
|
|
printf 'groups:\n'
|
|
printf ' - name: dashboard.traefik.ratio\n'
|
|
printf ' interval: 1m\n'
|
|
printf ' rules:\n'
|
|
printf ' - record: dashboard_traefik_5xx_ratio_percent\n'
|
|
printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
|
|
} > "$rules"
|
|
|
|
{
|
|
printf 'rule_files:\n'
|
|
printf ' - traefik-ratio-rules.yaml\n'
|
|
printf 'evaluation_interval: 1m\n'
|
|
printf 'tests:\n'
|
|
printf ' - interval: 1m\n'
|
|
printf ' input_series:\n'
|
|
printf ' - series: '\''traefik_entrypoint_requests_total{code="200",entrypoint="websecure"}'\''\n'
|
|
printf ' values: '\''0+1x10'\''\n'
|
|
printf ' - series: '\''traefik_entrypoint_requests_total{code="500",entrypoint="websecure"}'\''\n'
|
|
printf ' values: '\''0+1x10'\''\n'
|
|
printf ' promql_expr_test:\n'
|
|
printf ' - expr: dashboard_traefik_5xx_ratio_percent\n'
|
|
printf ' eval_time: 10m\n'
|
|
printf ' exp_samples:\n'
|
|
printf ' - labels: '\''dashboard_traefik_5xx_ratio_percent{}'\''\n'
|
|
printf ' value: 50\n'
|
|
} > "$tests"
|
|
|
|
if [[ -n ${PROMTOOL_BIN:-} ]]; then
|
|
(cd "$WORK_DIR" && "$PROMTOOL_BIN" test rules traefik-ratio-tests.yaml) >/dev/null
|
|
else
|
|
require_command docker
|
|
docker run --rm \
|
|
--entrypoint=/bin/promtool \
|
|
-v "$WORK_DIR:/tmp/dashboard-validation:ro" \
|
|
-w /tmp/dashboard-validation \
|
|
"$PROMETHEUS_IMAGE" test rules traefik-ratio-tests.yaml >/dev/null
|
|
fi
|
|
pass 'Traefik 5xx ratio preserves 50 percent at low request rates'
|
|
}
|
|
|
|
start_prometheus_port_forward() {
|
|
local log="$WORK_DIR/prometheus-port-forward.log"
|
|
local port=''
|
|
local attempt
|
|
|
|
kubectl -n observability port-forward \
|
|
--address=127.0.0.1 \
|
|
service/observability-core-kube-pr-prometheus \
|
|
:9090 > "$log" 2>&1 &
|
|
PORT_FORWARD_PID=$!
|
|
|
|
for ((attempt = 1; attempt <= 100; attempt++)); do
|
|
if grep -Eq '^Forwarding from 127\.0\.0\.1:[0-9]+ -> 9090$' "$log"; then
|
|
port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> 9090$/\1/p' "$log" | head -n 1)
|
|
break
|
|
fi
|
|
if ! kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
|
|
fail "Prometheus port-forward exited before readiness: $(tr '\n' ' ' < "$log")"
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
|
|
[[ "$port" =~ ^[1-9][0-9]*$ ]] \
|
|
|| fail 'Prometheus port-forward did not bind a loopback port within 10 seconds'
|
|
PROMETHEUS_BASE_URL="http://127.0.0.1:$port"
|
|
}
|
|
|
|
prometheus_get() {
|
|
local endpoint=$1
|
|
shift
|
|
curl \
|
|
--fail \
|
|
--silent \
|
|
--show-error \
|
|
--connect-timeout 2 \
|
|
--max-time 10 \
|
|
--get \
|
|
"$PROMETHEUS_BASE_URL$endpoint" \
|
|
"$@"
|
|
}
|
|
|
|
wait_for_prometheus() {
|
|
local attempt
|
|
for ((attempt = 1; attempt <= 50; attempt++)); do
|
|
if prometheus_get '/-/ready' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
fail 'Prometheus did not become ready within 5 seconds'
|
|
}
|
|
|
|
live_query_nonempty() {
|
|
local expression=$1
|
|
local context=$2
|
|
local expected_scalar=${3:-}
|
|
local quiet=${4:-0}
|
|
local response="$WORK_DIR/live-query-response.json"
|
|
|
|
if ! prometheus_get '/api/v1/query' \
|
|
--data-urlencode "query=$expression" > "$response"; then
|
|
fail "live Prometheus query request failed: $context"
|
|
fi
|
|
jq -e '
|
|
.status == "success"
|
|
and (.data.resultType == "vector" or .data.resultType == "scalar")
|
|
and (.data.result | type == "array" and length > 0)
|
|
' "$response" >/dev/null \
|
|
|| fail "live Prometheus query returned no vector/scalar result: $context"
|
|
|
|
if [[ -n "$expected_scalar" ]] && ! jq -e --arg expected "$expected_scalar" '
|
|
.data.resultType == "scalar"
|
|
and .data.result[1] == $expected
|
|
' "$response" >/dev/null; then
|
|
if [[ "$quiet" -eq 1 ]]; then
|
|
return 1
|
|
fi
|
|
fail "live Prometheus scalar result was not exact $expected_scalar: $context"
|
|
fi
|
|
}
|
|
|
|
validate_scalar_one_gate_regression() {
|
|
if (live_query_nonempty 'scalar(vector(0))' 'scalar-zero regression fixture' 1 1); then
|
|
fail 'scalar-zero comparison fixture passed the success gate'
|
|
fi
|
|
live_query_nonempty 'scalar(vector(1))' 'scalar-one regression fixture' 1
|
|
pass 'success comparison gate rejects scalar 0 and accepts exact scalar 1'
|
|
}
|
|
|
|
validate_live_dashboard_queries() {
|
|
local dashboard uid panel ref expression
|
|
local count=0
|
|
|
|
for dashboard in "$DASHBOARD_ROOT"/*.json; do
|
|
while IFS=$'\t' read -r uid panel ref expression; do
|
|
live_query_nonempty "$expression" "$uid / $panel / $ref"
|
|
count=$((count + 1))
|
|
done < <(
|
|
jq -r '
|
|
.uid as $uid
|
|
| .panels[]
|
|
| .title as $panel
|
|
| .targets[]
|
|
| [$uid, $panel, .refId, .expr]
|
|
| @tsv
|
|
' "$dashboard"
|
|
)
|
|
done
|
|
|
|
[[ "$count" -eq 37 ]] || fail "expected 37 live dashboard queries, got $count"
|
|
pass 'live Prometheus accepted 37 dashboard queries with nonempty vector/scalar results'
|
|
}
|
|
|
|
validate_live_matcher_selectors() {
|
|
local dashboard expression selector
|
|
local selectors="$WORK_DIR/dashboard-matchers.txt"
|
|
local count=0
|
|
|
|
: > "$selectors"
|
|
for dashboard in "$DASHBOARD_ROOT"/*.json; do
|
|
while IFS= read -r expression; do
|
|
grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]+\}' <<<"$expression" >> "$selectors" || true
|
|
done < <(jq -r '.panels[].targets[].expr' "$dashboard")
|
|
done
|
|
sort -u -o "$selectors" "$selectors"
|
|
|
|
while IFS= read -r selector; do
|
|
[[ -n "$selector" ]] || continue
|
|
live_query_nonempty "$selector" "live matcher selector $selector"
|
|
count=$((count + 1))
|
|
done < "$selectors"
|
|
|
|
[[ "$count" -gt 0 ]] || fail 'no dashboard matcher selectors were discovered'
|
|
pass "live Prometheus found series for $count dashboard matcher selectors"
|
|
}
|
|
|
|
validate_live_matcher_values() {
|
|
local -a presence_contracts=(
|
|
'node_cpu_seconds_total{mode="idle"}'
|
|
'node_filesystem_avail_bytes{mountpoint="/"}'
|
|
'node_filesystem_size_bytes{mountpoint="/"}'
|
|
'node_filesystem_files_free{mountpoint="/"}'
|
|
'node_filesystem_files{mountpoint="/"}'
|
|
'kube_persistentvolumeclaim_status_phase{phase="Pending"}'
|
|
'kube_persistentvolumeclaim_status_phase{phase="Lost"}'
|
|
)
|
|
local -a success_contracts=(
|
|
'scalar(count(count by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
|
|
'scalar(count(count by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
|
|
'scalar(min(probe_success{job="blackbox-private-edge"})) == bool 1'
|
|
'scalar(count(probe_http_status_code{job="blackbox-private-edge"})) == bool scalar(count(probe_http_status_code{job="blackbox-private-edge"} == 403))'
|
|
)
|
|
local contract
|
|
|
|
for contract in "${presence_contracts[@]}"; do
|
|
live_query_nonempty "$contract" "live matcher-value contract $contract"
|
|
done
|
|
for contract in "${success_contracts[@]}"; do
|
|
live_query_nonempty "$contract" "live matcher-value contract $contract" 1
|
|
done
|
|
pass 'live matcher values cover idle CPU, root filesystems, PVC phases, both buckets, and private-edge 403'
|
|
}
|
|
|
|
validate_live_prometheus() {
|
|
require_command curl
|
|
if [[ -z "$PROMETHEUS_BASE_URL" ]]; then
|
|
start_prometheus_port_forward
|
|
fi
|
|
wait_for_prometheus
|
|
validate_scalar_one_gate_regression
|
|
validate_live_dashboard_queries
|
|
validate_live_matcher_selectors
|
|
validate_live_matcher_values
|
|
}
|
|
|
|
main() {
|
|
local work rendered rules
|
|
parse_args "$@"
|
|
require_command jq
|
|
require_command sha256sum
|
|
require_command kubectl
|
|
|
|
verify_inventory target-initial
|
|
verify_inventory post-substrate
|
|
[[ -d "$DASHBOARD_ROOT" ]] || fail 'dashboard source directory is absent'
|
|
|
|
validate_dashboard kubernetes-node.json platform-kubernetes-node "$TARGET_INVENTORY"
|
|
validate_dashboard workload-health.json platform-workload-health "$TARGET_INVENTORY"
|
|
validate_dashboard platform-services.json platform-services "$POST_INVENTORY"
|
|
validate_dashboard observability-backends.json platform-observability-backends "$TARGET_INVENTORY"
|
|
validate_dashboard https-endpoints.json platform-https-endpoints "$POST_INVENTORY"
|
|
validate_inventory_driven_omissions
|
|
|
|
work=$(mktemp -d)
|
|
WORK_DIR=$work
|
|
chmod 0755 "$WORK_DIR"
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
rendered="$work/rendered.yaml"
|
|
rules="$work/dashboard-rules.yaml"
|
|
validate_rendered_configmaps "$rendered"
|
|
validate_promql_syntax "$rules"
|
|
validate_traefik_low_traffic_ratio
|
|
validate_live_prometheus
|
|
|
|
pass 'observability dashboard contract'
|
|
}
|
|
|
|
main "$@"
|