Files
platform-core/scripts/validate/test-observability-dashboards.sh
T

613 lines
21 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
DASHBOARD_ROOT="$ROOT/services/observability/dashboards/platform"
METRIC_ROOT=${PLATFORM_OBSERVABILITY_METRIC_ROOT:-/tmp/platform-observability-metrics.VUpsZn}
TARGET_INVENTORY="$METRIC_ROOT/target-initial/inventory.json"
POST_INVENTORY="$METRIC_ROOT/post-substrate/inventory.json"
PROMETHEUS_IMAGE='quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
WORK_DIR=''
PROMETHEUS_BASE_URL=''
PORT_FORWARD_PID=''
fail() {
printf 'FAIL: %s\n' "$*" >&2
exit 1
}
pass() {
printf 'PASS: %s\n' "$*"
}
require_command() {
command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
}
usage() {
printf 'Usage: %s [--prometheus-base-url http://127.0.0.1:PORT]\n' "${0##*/}"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--prometheus-base-url)
[[ $# -ge 2 ]] || fail '--prometheus-base-url requires a value'
[[ -z "$PROMETHEUS_BASE_URL" ]] || fail '--prometheus-base-url may be specified only once'
PROMETHEUS_BASE_URL=$2
shift 2
;;
--help)
usage
exit 0
;;
*)
fail "unknown argument: $1"
;;
esac
done
if [[ -n "$PROMETHEUS_BASE_URL" ]]; then
[[ "$PROMETHEUS_BASE_URL" =~ ^http://(127\.0\.0\.1|localhost):([1-9][0-9]{0,4})$ ]] \
|| fail '--prometheus-base-url must be an exact loopback HTTP origin without a path'
(( BASH_REMATCH[2] <= 65535 )) \
|| fail '--prometheus-base-url port exceeds 65535'
fi
}
cleanup() {
local rc=$?
trap - EXIT
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
kill "$PORT_FORWARD_PID" 2>/dev/null || true
wait "$PORT_FORWARD_PID" 2>/dev/null || true
fi
if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then
rm -rf -- "$WORK_DIR"
fi
exit "$rc"
}
verify_inventory() {
local phase=$1
local inventory="$METRIC_ROOT/$phase/inventory.json"
local checksum="$METRIC_ROOT/$phase/inventory.sha256"
local expected actual
[[ -f "$inventory" ]] || fail "$phase inventory is absent"
[[ -f "$checksum" ]] || fail "$phase inventory checksum is absent"
jq -e --arg phase "$phase" '
.schema == "platform-observability-metric-inventory/v1"
and .phase == $phase
and (.targets | type == "array" and length > 0)
and all(.targets[]; .health == "up" and (.metrics | type == "array" and length > 0))
' "$inventory" >/dev/null || fail "$phase inventory contract is invalid"
read -r expected checksum_name < "$checksum"
[[ "$checksum_name" == 'inventory.json' ]] || fail "$phase checksum names an unexpected file"
actual=$(sha256sum "$inventory" | awk '{print $1}')
[[ "$actual" == "$expected" ]] || fail "$phase inventory checksum mismatch"
pass "$phase inventory contract and checksum"
}
inventory_has_metric() {
local inventory=$1
local metric=$2
jq -e --arg metric "$metric" 'any(.targets[].metrics[]; .name == $metric)' "$inventory" >/dev/null
}
inventory_metric_has_label() {
local inventory=$1
local metric=$2
local label=$3
jq -e --arg metric "$metric" --arg label "$label" '
any(.targets[].metrics[]; .name == $metric and (.label_names | index($label) != null))
' "$inventory" >/dev/null
}
validate_inventory_references() {
local dashboard=$1
local inventory=$2
local expression selector metric matchers matcher label token
local inventory_metrics referenced_metrics found
inventory_metrics=$(jq -r '[.targets[].metrics[].name] | unique[]' "$inventory")
while IFS= read -r expression; do
[[ -n "$expression" ]] || fail "empty PromQL expression in $dashboard"
referenced_metrics=''
while IFS= read -r selector; do
[[ -n "$selector" ]] || continue
metric=${selector%%\{*}
grep -Fxq "$metric" <<<"$inventory_metrics" \
|| fail "$dashboard references absent metric $metric"
if ! grep -Fxq "$metric" <<<"$referenced_metrics"; then
referenced_metrics+="${metric}"$'\n'
fi
matchers=${selector#*\{}
matchers=${matchers%\}}
while IFS= read -r matcher; do
matcher=${matcher#"${matcher%%[![:space:]]*}"}
[[ -n "$matcher" ]] || continue
label=$(sed -E 's/^([A-Za-z_][A-Za-z0-9_]*).*/\1/' <<<"$matcher")
[[ "$label" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \
|| fail "$dashboard has an unauditable label matcher in $expression"
inventory_metric_has_label "$inventory" "$metric" "$label" \
|| fail "$dashboard references absent label $metric.$label"
done < <(tr ',' '\n' <<<"$matchers")
done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]*\}' <<<"$expression" || true)
[[ -n "$referenced_metrics" ]] || fail "$dashboard expression has no inventory metric: $expression"
while IFS= read -r token; do
[[ -n "$token" ]] || continue
if grep -Fxq "$token" <<<"$inventory_metrics"; then
grep -Fxq "$token" <<<"$referenced_metrics" \
|| fail "$dashboard uses inventory metric $token without an auditable selector"
fi
done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$expression" | sort -u)
while IFS= read -r label; do
[[ -n "$label" ]] || continue
found=0
while IFS= read -r metric; do
[[ -n "$metric" ]] || continue
if inventory_metric_has_label "$inventory" "$metric" "$label"; then
found=1
break
fi
done <<<"$referenced_metrics"
[[ "$found" -eq 1 ]] || fail "$dashboard groups by absent inventory label $label"
done < <(
grep -oE '(by|without)[[:space:]]*\([^)]*\)' <<<"$expression" \
| sed -E 's/^[^(]*\(([^)]*)\)$/\1/' \
| tr ',' '\n' \
| sed -E 's/^[[:space:]]+|[[:space:]]+$//g' \
|| true
)
done < <(jq -r '.panels[].targets[].expr' "$dashboard")
}
validate_dashboard() {
local file=$1
local expected_uid=$2
local inventory=$3
local path="$DASHBOARD_ROOT/$file"
[[ -f "$path" ]] || fail "dashboard source is absent: $file"
jq -e --arg uid "$expected_uid" '
type == "object"
and .uid == $uid
and (.title | type == "string" and length > 0)
and .schemaVersion == 42
and .editable == false
and .refresh == "30s"
and (.templating.list == [])
and (.panels | type == "array" and length > 0)
and ([.panels[].id] | length == (unique | length))
and all(.panels[];
.type != "row"
and (.title | type == "string" and length > 0)
and (.gridPos.w > 0 and .gridPos.h > 0)
and .datasource.type == "prometheus"
and .datasource.uid == "prometheus"
and (.targets | type == "array" and length > 0)
and all(.targets[];
.datasource.type == "prometheus"
and .datasource.uid == "prometheus"
and (.expr | type == "string" and length > 0)
and (.refId | type == "string" and length > 0)
)
)
' "$path" >/dev/null || fail "dashboard structure is invalid: $file"
if jq -r '.title, .tags[], .panels[].title' "$path" \
| grep -Eiq '(^|[^[:alnum:]_])(spring|jvm|kafka|batch|backup)([^[:alnum:]_]|$)'; then
fail "dashboard contains a forbidden product area: $file"
fi
if jq -r '.panels[].targets[].expr' "$path" \
| grep -Eiq '(^|[^[:alnum:]_])(spring_|jvm_|kafka_|batch_|backup_)'; then
fail "dashboard contains a forbidden product query: $file"
fi
if jq -r '.. | strings' "$path" \
| grep -Eiq '(request[_ -]?id|trace[_ -]?id|username|raw[_ -]?url)'; then
fail "dashboard contains forbidden or high-cardinality content: $file"
fi
validate_inventory_references "$path" "$inventory"
pass "$file structure and inventory references"
}
validate_rendered_configmaps() {
local rendered=$1
local file name block
local -a files=(
kubernetes-node
workload-health
platform-services
observability-backends
https-endpoints
)
kubectl kustomize "$DASHBOARD_ROOT" > "$rendered"
[[ $(grep -c '^kind: ConfigMap$' "$rendered") -eq 5 ]] \
|| fail 'kustomization must render exactly five ConfigMaps'
for file in "${files[@]}"; do
name="grafana-dashboard-$file"
block=$(awk -v name="$name" '
BEGIN { RS="---" }
$0 ~ "name: " name "([[:space:]]|$)" { print }
' "$rendered")
[[ -n "$block" ]] || fail "rendered ConfigMap is absent: $name"
grep -Eq 'grafana_dashboard: ("1"|1)$' <<<"$block" \
|| fail "$name lacks grafana_dashboard=1"
grep -q 'observability.hyeonworks.com/instance: home' <<<"$block" \
|| fail "$name lacks the observability instance label"
grep -q 'observability.hyeonworks.com/owner: platform-observability' <<<"$block" \
|| fail "$name lacks the observability owner label"
grep -q " $file.json:" <<<"$block" \
|| fail "$name does not embed $file.json"
done
pass 'kustomization renders five stable labeled dashboard ConfigMaps'
}
validate_inventory_driven_omissions() {
local workload="$DASHBOARD_ROOT/workload-health.json"
local services="$DASHBOARD_ROOT/platform-services.json"
local endpoints="$DASHBOARD_ROOT/https-endpoints.json"
if inventory_has_metric "$TARGET_INVENTORY" kube_pod_container_status_last_terminated_reason; then
fail 'the OOM omission contract no longer matches target-initial inventory'
fi
if jq -r '.panels[].targets[].expr' "$workload" \
| grep -Fq 'kube_pod_container_status_last_terminated_reason'; then
fail 'workload dashboard guessed an OOM metric absent from target-initial inventory'
fi
pass 'OOM panel omitted: kube_pod_container_status_last_terminated_reason is absent from target-initial inventory'
jq -e '
([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})") != null)
and
([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})") != null)
' "$workload" >/dev/null \
|| fail 'PVC health panel must retain zero-valued Pending and Lost series instead of filtering healthy state away'
pass 'PVC health panel keeps visible zero-valued Pending and Lost aggregates'
if jq -e '
any(.targets[].metrics[];
(.name | startswith("traefik_"))
and (.name | endswith("_bucket"))
and (.name | test("duration|request"))
)
' "$TARGET_INVENTORY" >/dev/null; then
fail 'the Traefik latency omission contract no longer matches target-initial inventory'
fi
if jq -r '.panels[].targets[].expr' "$services" \
| grep -Eq 'histogram_quantile|traefik_.*_bucket'; then
fail 'service dashboard guessed a Traefik latency histogram absent from target-initial inventory'
fi
pass 'Traefik p95/p99 omitted: no request duration histogram bucket exists in target-initial inventory'
inventory_has_metric "$POST_INVENTORY" probe_success \
|| fail 'post-substrate inventory lacks probe_success for the platform boundary summary'
jq -e '
any(.panels[].targets[];
.expr == "min by (job) (probe_success{})"
)
' "$services" >/dev/null \
|| fail 'platform service dashboard lacks the inventory-backed boundary health summary'
pass 'platform service boundary summary uses post-substrate probe_success'
jq -e '
any(.panels[].targets[];
.expr == "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})"
)
' "$endpoints" >/dev/null \
|| fail 'HTTPS dashboard does not isolate the observed private-edge 403 boundary'
pass 'HTTPS dashboard isolates the observed blackbox-private-edge status boundary'
}
validate_promql_syntax() {
local rules=$1
local file expression index=0
local -a files=(
kubernetes-node.json
workload-health.json
platform-services.json
observability-backends.json
https-endpoints.json
)
{
printf 'groups:\n'
printf ' - name: dashboard.promql.syntax\n'
printf ' rules:\n'
for file in "${files[@]}"; do
while IFS= read -r expression; do
index=$((index + 1))
printf ' - record: dashboard_syntax_%d\n' "$index"
printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
done < <(jq -r '.panels[].targets[].expr' "$DASHBOARD_ROOT/$file")
done
} > "$rules"
if [[ -n ${PROMTOOL_BIN:-} ]]; then
"$PROMTOOL_BIN" check rules "$rules" >/dev/null
else
require_command docker
docker run --rm \
--entrypoint=/bin/promtool \
-v "$rules:/tmp/dashboard-rules.yaml:ro" \
"$PROMETHEUS_IMAGE" check rules /tmp/dashboard-rules.yaml >/dev/null
fi
pass "promtool parsed $index dashboard expressions"
}
validate_traefik_low_traffic_ratio() {
local rules="$WORK_DIR/traefik-ratio-rules.yaml"
local tests="$WORK_DIR/traefik-ratio-tests.yaml"
local expression
expression=$(jq -er '
.panels[]
| select(.title == "Traefik 5xx Ratio")
| .targets[]
| select(.refId == "A")
| .expr
' "$DASHBOARD_ROOT/platform-services.json") \
|| fail 'Traefik 5xx ratio expression is absent'
{
printf 'groups:\n'
printf ' - name: dashboard.traefik.ratio\n'
printf ' interval: 1m\n'
printf ' rules:\n'
printf ' - record: dashboard_traefik_5xx_ratio_percent\n'
printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
} > "$rules"
{
printf 'rule_files:\n'
printf ' - traefik-ratio-rules.yaml\n'
printf 'evaluation_interval: 1m\n'
printf 'tests:\n'
printf ' - interval: 1m\n'
printf ' input_series:\n'
printf ' - series: '\''traefik_entrypoint_requests_total{code="200",entrypoint="websecure"}'\''\n'
printf ' values: '\''0+1x10'\''\n'
printf ' - series: '\''traefik_entrypoint_requests_total{code="500",entrypoint="websecure"}'\''\n'
printf ' values: '\''0+1x10'\''\n'
printf ' promql_expr_test:\n'
printf ' - expr: dashboard_traefik_5xx_ratio_percent\n'
printf ' eval_time: 10m\n'
printf ' exp_samples:\n'
printf ' - labels: '\''dashboard_traefik_5xx_ratio_percent{}'\''\n'
printf ' value: 50\n'
} > "$tests"
if [[ -n ${PROMTOOL_BIN:-} ]]; then
(cd "$WORK_DIR" && "$PROMTOOL_BIN" test rules traefik-ratio-tests.yaml) >/dev/null
else
require_command docker
docker run --rm \
--entrypoint=/bin/promtool \
-v "$WORK_DIR:/tmp/dashboard-validation:ro" \
-w /tmp/dashboard-validation \
"$PROMETHEUS_IMAGE" test rules traefik-ratio-tests.yaml >/dev/null
fi
pass 'Traefik 5xx ratio preserves 50 percent at low request rates'
}
start_prometheus_port_forward() {
local log="$WORK_DIR/prometheus-port-forward.log"
local port=''
local attempt
kubectl -n observability port-forward \
--address=127.0.0.1 \
service/observability-core-kube-pr-prometheus \
:9090 > "$log" 2>&1 &
PORT_FORWARD_PID=$!
for ((attempt = 1; attempt <= 100; attempt++)); do
if grep -Eq '^Forwarding from 127\.0\.0\.1:[0-9]+ -> 9090$' "$log"; then
port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> 9090$/\1/p' "$log" | head -n 1)
break
fi
if ! kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
fail "Prometheus port-forward exited before readiness: $(tr '\n' ' ' < "$log")"
fi
sleep 0.1
done
[[ "$port" =~ ^[1-9][0-9]*$ ]] \
|| fail 'Prometheus port-forward did not bind a loopback port within 10 seconds'
PROMETHEUS_BASE_URL="http://127.0.0.1:$port"
}
prometheus_get() {
local endpoint=$1
shift
curl \
--fail \
--silent \
--show-error \
--connect-timeout 2 \
--max-time 10 \
--get \
"$PROMETHEUS_BASE_URL$endpoint" \
"$@"
}
wait_for_prometheus() {
local attempt
for ((attempt = 1; attempt <= 50; attempt++)); do
if prometheus_get '/-/ready' >/dev/null 2>&1; then
return 0
fi
sleep 0.1
done
fail 'Prometheus did not become ready within 5 seconds'
}
live_query_nonempty() {
local expression=$1
local context=$2
local expected_scalar=${3:-}
local quiet=${4:-0}
local response="$WORK_DIR/live-query-response.json"
if ! prometheus_get '/api/v1/query' \
--data-urlencode "query=$expression" > "$response"; then
fail "live Prometheus query request failed: $context"
fi
jq -e '
.status == "success"
and (.data.resultType == "vector" or .data.resultType == "scalar")
and (.data.result | type == "array" and length > 0)
' "$response" >/dev/null \
|| fail "live Prometheus query returned no vector/scalar result: $context"
if [[ -n "$expected_scalar" ]] && ! jq -e --arg expected "$expected_scalar" '
.data.resultType == "scalar"
and .data.result[1] == $expected
' "$response" >/dev/null; then
if [[ "$quiet" -eq 1 ]]; then
return 1
fi
fail "live Prometheus scalar result was not exact $expected_scalar: $context"
fi
}
validate_scalar_one_gate_regression() {
if (live_query_nonempty 'scalar(vector(0))' 'scalar-zero regression fixture' 1 1); then
fail 'scalar-zero comparison fixture passed the success gate'
fi
live_query_nonempty 'scalar(vector(1))' 'scalar-one regression fixture' 1
pass 'success comparison gate rejects scalar 0 and accepts exact scalar 1'
}
validate_live_dashboard_queries() {
local dashboard uid panel ref expression
local count=0
for dashboard in "$DASHBOARD_ROOT"/*.json; do
while IFS=$'\t' read -r uid panel ref expression; do
live_query_nonempty "$expression" "$uid / $panel / $ref"
count=$((count + 1))
done < <(
jq -r '
.uid as $uid
| .panels[]
| .title as $panel
| .targets[]
| [$uid, $panel, .refId, .expr]
| @tsv
' "$dashboard"
)
done
[[ "$count" -eq 37 ]] || fail "expected 37 live dashboard queries, got $count"
pass 'live Prometheus accepted 37 dashboard queries with nonempty vector/scalar results'
}
validate_live_matcher_selectors() {
local dashboard expression selector
local selectors="$WORK_DIR/dashboard-matchers.txt"
local count=0
: > "$selectors"
for dashboard in "$DASHBOARD_ROOT"/*.json; do
while IFS= read -r expression; do
grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]+\}' <<<"$expression" >> "$selectors" || true
done < <(jq -r '.panels[].targets[].expr' "$dashboard")
done
sort -u -o "$selectors" "$selectors"
while IFS= read -r selector; do
[[ -n "$selector" ]] || continue
live_query_nonempty "$selector" "live matcher selector $selector"
count=$((count + 1))
done < "$selectors"
[[ "$count" -gt 0 ]] || fail 'no dashboard matcher selectors were discovered'
pass "live Prometheus found series for $count dashboard matcher selectors"
}
validate_live_matcher_values() {
local -a presence_contracts=(
'node_cpu_seconds_total{mode="idle"}'
'node_filesystem_avail_bytes{mountpoint="/"}'
'node_filesystem_size_bytes{mountpoint="/"}'
'node_filesystem_files_free{mountpoint="/"}'
'node_filesystem_files{mountpoint="/"}'
'kube_persistentvolumeclaim_status_phase{phase="Pending"}'
'kube_persistentvolumeclaim_status_phase{phase="Lost"}'
)
local -a success_contracts=(
'scalar(count(count by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
'scalar(count(count by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
'scalar(min(probe_success{job="blackbox-private-edge"})) == bool 1'
'scalar(count(probe_http_status_code{job="blackbox-private-edge"})) == bool scalar(count(probe_http_status_code{job="blackbox-private-edge"} == 403))'
)
local contract
for contract in "${presence_contracts[@]}"; do
live_query_nonempty "$contract" "live matcher-value contract $contract"
done
for contract in "${success_contracts[@]}"; do
live_query_nonempty "$contract" "live matcher-value contract $contract" 1
done
pass 'live matcher values cover idle CPU, root filesystems, PVC phases, both buckets, and private-edge 403'
}
validate_live_prometheus() {
require_command curl
if [[ -z "$PROMETHEUS_BASE_URL" ]]; then
start_prometheus_port_forward
fi
wait_for_prometheus
validate_scalar_one_gate_regression
validate_live_dashboard_queries
validate_live_matcher_selectors
validate_live_matcher_values
}
main() {
local work rendered rules
parse_args "$@"
require_command jq
require_command sha256sum
require_command kubectl
verify_inventory target-initial
verify_inventory post-substrate
[[ -d "$DASHBOARD_ROOT" ]] || fail 'dashboard source directory is absent'
validate_dashboard kubernetes-node.json platform-kubernetes-node "$TARGET_INVENTORY"
validate_dashboard workload-health.json platform-workload-health "$TARGET_INVENTORY"
validate_dashboard platform-services.json platform-services "$POST_INVENTORY"
validate_dashboard observability-backends.json platform-observability-backends "$TARGET_INVENTORY"
validate_dashboard https-endpoints.json platform-https-endpoints "$POST_INVENTORY"
validate_inventory_driven_omissions
work=$(mktemp -d)
WORK_DIR=$work
chmod 0755 "$WORK_DIR"
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
rendered="$work/rendered.yaml"
rules="$work/dashboard-rules.yaml"
validate_rendered_configmaps "$rendered"
validate_promql_syntax "$rules"
validate_traefik_low_traffic_ratio
validate_live_prometheus
pass 'observability dashboard contract'
}
main "$@"