1071 lines
49 KiB
Bash
1071 lines
49 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
readonly RULE_DIR="${ROOT_DIR}/services/observability/rules/platform"
|
|
readonly PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.2-distroless@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b"
|
|
readonly EXPECTED_RUNBOOK_URL="https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md"
|
|
|
|
runbook_url_blocked=0
|
|
temp_dir=''
|
|
|
|
# Each raw metric in the three Task 7 rule resources is bound to the
|
|
# inventory phase and scrape pool that supplied it. Probe metrics select a
|
|
# pool from their explicit probe-group matcher below.
|
|
declare -A new_rule_metric_phase=(
|
|
[minio_cluster_usage_buckets_total_bytes]=target-initial
|
|
[minio_cluster_usage_buckets_quota_total_bytes]=target-initial
|
|
[cnpg_collector_up]=target-initial
|
|
[alloy_config_last_load_successful]=target-initial
|
|
[loki_write_batch_retries_total]=target-initial
|
|
[loki_write_dropped_entries_total]=target-initial
|
|
[loki_runtime_config_last_reload_successful]=target-initial
|
|
[loki_ingester_wal_disk_usage_percent]=target-initial
|
|
[probe_success]=post-substrate
|
|
[probe_ssl_earliest_cert_expiry]=post-substrate
|
|
)
|
|
declare -A new_rule_metric_pool=(
|
|
[minio_cluster_usage_buckets_total_bytes]=serviceMonitor/object-storage/aistor-bucket-usage/0
|
|
[minio_cluster_usage_buckets_quota_total_bytes]=serviceMonitor/object-storage/aistor-bucket-usage/0
|
|
[cnpg_collector_up]=podMonitor/platform-data/platform-postgres/0
|
|
[alloy_config_last_load_successful]=serviceMonitor/observability-agent/alloy/0
|
|
[loki_write_batch_retries_total]=serviceMonitor/observability-agent/alloy/0
|
|
[loki_write_dropped_entries_total]=serviceMonitor/observability-agent/alloy/0
|
|
[loki_runtime_config_last_reload_successful]=serviceMonitor/observability/loki/0
|
|
[loki_ingester_wal_disk_usage_percent]=serviceMonitor/observability/loki/0
|
|
[probe_success]=probe-group
|
|
[probe_ssl_earliest_cert_expiry]=probe-group
|
|
)
|
|
declare -A new_rule_probe_pool=(
|
|
["probe_success|public-edge"]=probe/observability/platform-public-edge
|
|
["probe_success|private-edge"]=probe/observability/platform-private-edge
|
|
["probe_success|private-internal"]=probe/observability/platform-private-internal
|
|
["probe_ssl_earliest_cert_expiry|public-edge"]=probe/observability/platform-public-edge
|
|
["probe_ssl_earliest_cert_expiry|private-edge"]=probe/observability/platform-private-edge
|
|
)
|
|
declare -A new_rule_matcher_contract=(
|
|
[minio_cluster_usage_buckets_total_bytes]=$'bucket\t=~\tloki|tempo'
|
|
[minio_cluster_usage_buckets_quota_total_bytes]=$'bucket\t=~\tloki|tempo'
|
|
[cnpg_collector_up]=$'cluster\t=\tplatform-postgres\nnamespace\t=\tplatform-data'
|
|
[alloy_config_last_load_successful]=$'namespace\t=\tobservability-agent'
|
|
[loki_write_batch_retries_total]=$'namespace\t=\tobservability-agent'
|
|
[loki_write_dropped_entries_total]=$'namespace\t=\tobservability-agent'
|
|
[loki_runtime_config_last_reload_successful]=$'namespace\t=\tobservability'
|
|
[loki_ingester_wal_disk_usage_percent]=$'namespace\t=\tobservability'
|
|
["probe_success|=|public-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tpublic-edge'
|
|
["probe_success|=|private-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tprivate-edge'
|
|
["probe_success|=|private-internal"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=\tprivate-internal'
|
|
["probe_ssl_earliest_cert_expiry|=~|public-edge|private-edge"]=$'namespace\t=\tobservability\nobservability.hyeonworks.com/probe-group\t=~\tpublic-edge|private-edge'
|
|
)
|
|
declare -A recording_rule_expression=()
|
|
declare -A recording_rule_state=()
|
|
declare -a selector_matcher_labels=()
|
|
declare -A selector_matcher_operator=()
|
|
declare -A selector_matcher_value=()
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
pass() {
|
|
printf 'PASS: %s\n' "$*"
|
|
}
|
|
|
|
cleanup() {
|
|
if [[ -n "$temp_dir" && "$temp_dir" == /tmp/tmp.* && -d "$temp_dir" ]]; then
|
|
rm -rf -- "$temp_dir"
|
|
fi
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
|
|
}
|
|
|
|
assert_inventory() {
|
|
local inventory_dir=$1
|
|
local expected_phase=$2
|
|
local inventory_file="${inventory_dir}/inventory.json"
|
|
local checksum_file="${inventory_dir}/inventory.sha256"
|
|
local expected_sha actual_sha
|
|
|
|
[[ -f "$inventory_file" ]] || fail "missing inventory: ${inventory_file}"
|
|
[[ -f "$checksum_file" ]] || fail "missing inventory checksum: ${checksum_file}"
|
|
expected_sha="$(awk '$2 == "inventory.json" { print $1 }' "$checksum_file")"
|
|
[[ "$expected_sha" =~ ^[0-9a-f]{64}$ ]] || fail "invalid inventory checksum record: ${checksum_file}"
|
|
actual_sha="$(sha256sum "$inventory_file" | awk '{ print $1 }')"
|
|
[[ "$actual_sha" == "$expected_sha" ]] || fail "inventory checksum mismatch: ${expected_phase}"
|
|
jq -e --arg phase "$expected_phase" '
|
|
.schema == "platform-observability-metric-inventory/v1" and
|
|
.phase == $phase and
|
|
(.targets | type == "array" and length > 0) and
|
|
all(.targets[];
|
|
.health == "up" and
|
|
(.metrics | type == "array" and length > 0)
|
|
)
|
|
' "$inventory_file" >/dev/null || fail "inventory schema/phase mismatch: ${expected_phase}"
|
|
}
|
|
|
|
assert_metric_label() {
|
|
local inventory_file=$1
|
|
local scrape_pool=$2
|
|
local metric_name=$3
|
|
local label_name=$4
|
|
|
|
jq -e \
|
|
--arg pool "$scrape_pool" \
|
|
--arg metric "$metric_name" \
|
|
--arg label "$label_name" '
|
|
any(.targets[];
|
|
.scrape_pool == $pool and
|
|
any(.metrics[];
|
|
.name == $metric and
|
|
(.label_names | index($label)) != null
|
|
)
|
|
)
|
|
' "$inventory_file" >/dev/null ||
|
|
fail "inventory lacks ${scrape_pool} ${metric_name}{${label_name}}"
|
|
}
|
|
|
|
assert_metric() {
|
|
local inventory_file=$1
|
|
local scrape_pool=$2
|
|
local metric_name=$3
|
|
|
|
jq -e \
|
|
--arg pool "$scrape_pool" \
|
|
--arg metric "$metric_name" '
|
|
any(.targets[];
|
|
.scrape_pool == $pool and
|
|
any(.metrics[]; .name == $metric)
|
|
)
|
|
' "$inventory_file" >/dev/null ||
|
|
fail "inventory lacks ${scrape_pool} ${metric_name}"
|
|
}
|
|
|
|
parse_explicit_matchers() {
|
|
local selector=$1
|
|
local context=$2
|
|
local matchers matcher_re match label operator value
|
|
|
|
matchers=${selector#*\{}
|
|
matchers=${matchers%\}}
|
|
selector_matcher_labels=()
|
|
selector_matcher_operator=()
|
|
selector_matcher_value=()
|
|
matcher_re='^[[:space:],]*("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)[[:space:]]*(=~|!~|!=|=)[[:space:]]*"(([^"\\]|\\.)*)"'
|
|
|
|
while [[ -n "${matchers//[[:space:],]/}" ]]; do
|
|
[[ "$matchers" =~ $matcher_re ]] ||
|
|
fail "unparseable explicit matcher in ${context}: ${matchers}"
|
|
match=${BASH_REMATCH[0]}
|
|
label=${BASH_REMATCH[1]}
|
|
operator=${BASH_REMATCH[2]}
|
|
value=${BASH_REMATCH[3]}
|
|
label=${label#\"}
|
|
label=${label%\"}
|
|
[[ -z ${selector_matcher_operator[$label]+x} ]] ||
|
|
fail "duplicate explicit matcher label ${label} in ${context}"
|
|
selector_matcher_labels+=("$label")
|
|
selector_matcher_operator["$label"]=$operator
|
|
selector_matcher_value["$label"]=$value
|
|
matchers=${matchers:${#match}}
|
|
[[ -z "${matchers//[[:space:]]/}" || "$matchers" =~ ^[[:space:]]*, ]] ||
|
|
fail "unparseable explicit matcher separator in ${context}: ${matchers}"
|
|
done
|
|
}
|
|
|
|
assert_exact_matcher_contract() {
|
|
local metric=$1
|
|
local has_selector=$2
|
|
local pool_binding=$3
|
|
local context=$4
|
|
local contract_key=$metric
|
|
local group_label='observability.hyeonworks.com/probe-group'
|
|
local expected_signature actual_signature label
|
|
|
|
(( has_selector == 1 )) ||
|
|
fail "exact matcher contract requires a selector for ${metric} in ${context}"
|
|
if [[ "$pool_binding" == probe-group ]]; then
|
|
[[ -n ${selector_matcher_operator[$group_label]+x} ]] ||
|
|
fail "exact matcher contract requires ${group_label} for ${metric} in ${context}"
|
|
contract_key="${metric}|${selector_matcher_operator[$group_label]}|${selector_matcher_value[$group_label]}"
|
|
fi
|
|
|
|
expected_signature=${new_rule_matcher_contract[$contract_key]:-}
|
|
[[ -n "$expected_signature" ]] ||
|
|
fail "exact matcher contract is not mapped for ${contract_key} in ${context}"
|
|
actual_signature="$({
|
|
for label in "${selector_matcher_labels[@]}"; do
|
|
printf '%s\t%s\t%s\n' \
|
|
"$label" \
|
|
"${selector_matcher_operator[$label]}" \
|
|
"${selector_matcher_value[$label]}"
|
|
done
|
|
} | LC_ALL=C sort)"
|
|
[[ "$actual_signature" == "$expected_signature" ]] ||
|
|
fail "exact matcher contract mismatch for ${metric} in ${context}"
|
|
}
|
|
|
|
assert_mapped_raw_metric() {
|
|
local metric=$1
|
|
local has_selector=$2
|
|
local target_inventory=$3
|
|
local substrate_inventory=$4
|
|
local context=$5
|
|
local phase pool_binding inventory pool group_label operator value group probe_key
|
|
local -a pools groups
|
|
local -A seen_pools=()
|
|
|
|
phase=${new_rule_metric_phase[$metric]:-}
|
|
pool_binding=${new_rule_metric_pool[$metric]:-}
|
|
[[ -n "$phase" && -n "$pool_binding" ]] ||
|
|
fail "unmapped raw metric: ${metric} in ${context}"
|
|
case "$phase" in
|
|
target-initial) inventory=$target_inventory ;;
|
|
post-substrate) inventory=$substrate_inventory ;;
|
|
*) fail "unmapped inventory phase ${phase} for raw metric ${metric}" ;;
|
|
esac
|
|
|
|
pools=()
|
|
if [[ "$pool_binding" == probe-group ]]; then
|
|
(( has_selector == 1 )) ||
|
|
fail "unmapped probe scrape pool without an explicit probe-group matcher: ${metric} in ${context}"
|
|
group_label='observability.hyeonworks.com/probe-group'
|
|
[[ -n ${selector_matcher_operator[$group_label]+x} ]] ||
|
|
fail "unmapped probe scrape pool without ${group_label}: ${metric} in ${context}"
|
|
operator=${selector_matcher_operator[$group_label]}
|
|
value=${selector_matcher_value[$group_label]}
|
|
case "$operator" in
|
|
'=') groups=("$value") ;;
|
|
'=~') IFS='|' read -r -a groups <<<"$value" ;;
|
|
*) fail "unmapped probe-group matcher ${group_label}${operator} in ${context}" ;;
|
|
esac
|
|
for group in "${groups[@]}"; do
|
|
probe_key="${metric}|${group}"
|
|
pool=${new_rule_probe_pool[$probe_key]:-}
|
|
[[ -n "$pool" ]] ||
|
|
fail "unmapped probe scrape pool for ${metric} ${group_label}=${group} in ${context}"
|
|
if [[ -z ${seen_pools[$pool]+x} ]]; then
|
|
pools+=("$pool")
|
|
seen_pools["$pool"]=1
|
|
fi
|
|
done
|
|
else
|
|
pools=("$pool_binding")
|
|
fi
|
|
|
|
for pool in "${pools[@]}"; do
|
|
assert_metric "$inventory" "$pool" "$metric"
|
|
if (( has_selector == 1 )); then
|
|
for label in "${selector_matcher_labels[@]}"; do
|
|
assert_metric_label "$inventory" "$pool" "$metric" "$label"
|
|
done
|
|
fi
|
|
done
|
|
assert_exact_matcher_contract "$metric" "$has_selector" "$pool_binding" "$context"
|
|
}
|
|
|
|
validate_raw_selector_provenance() {
|
|
local selector=$1
|
|
local target_inventory=$2
|
|
local substrate_inventory=$3
|
|
local context=$4
|
|
local metric
|
|
|
|
metric=$(sed -E 's/[[:space:]]*\{.*$//' <<<"$selector")
|
|
[[ "$metric" =~ ^[A-Za-z_:][A-Za-z0-9_:]*$ ]] ||
|
|
fail "unparseable raw metric selector in ${context}: ${selector}"
|
|
parse_explicit_matchers "$selector" "$context"
|
|
assert_mapped_raw_metric "$metric" 1 "$target_inventory" "$substrate_inventory" "$context"
|
|
}
|
|
|
|
is_promql_syntax_token() {
|
|
case "$1" in
|
|
and|or|unless|bool|by|without|on|ignoring|group_left|group_right|offset|start|end|\
|
|
min|max|sum|avg|group|stddev|stdvar|topk|bottomk|count|count_values|quantile|limitk|limit_ratio|\
|
|
s|m|h|d|w|y|e|NaN|Inf)
|
|
return 0
|
|
;;
|
|
esac
|
|
return 1
|
|
}
|
|
|
|
is_promql_function_call() {
|
|
local token=$1
|
|
local expression=$2
|
|
|
|
grep -Eq "(^|[^[:alnum:]_:])${token}[[:space:]]*\\(" <<<"$expression"
|
|
}
|
|
|
|
load_recording_rule_expressions() {
|
|
local rule_json record encoded_expression expression
|
|
|
|
recording_rule_expression=()
|
|
recording_rule_state=()
|
|
for rule_json in "$@"; do
|
|
while IFS=$'\t' read -r record encoded_expression; do
|
|
expression=$(printf '%s' "$encoded_expression" | base64 -d)
|
|
[[ -z ${recording_rule_expression[$record]+x} ]] ||
|
|
fail "duplicate recording rule name ${record} in ${rule_json}"
|
|
recording_rule_expression["$record"]=$expression
|
|
done < <(
|
|
jq -r '
|
|
.spec.groups[].rules[]
|
|
| select(has("record"))
|
|
| [.record, (.expr | @base64)]
|
|
| @tsv
|
|
' "$rule_json"
|
|
)
|
|
done
|
|
}
|
|
|
|
validate_recording_rule_provenance() {
|
|
local record=$1
|
|
local target_inventory=$2
|
|
local substrate_inventory=$3
|
|
local context=$4
|
|
local state
|
|
|
|
[[ -n ${recording_rule_expression[$record]+x} ]] ||
|
|
fail "unresolved recording rule reference: ${record} in ${context}"
|
|
state=${recording_rule_state[$record]:-new}
|
|
case "$state" in
|
|
done) return 0 ;;
|
|
visiting) fail "recording rule dependency cycle: ${record} in ${context}" ;;
|
|
esac
|
|
recording_rule_state["$record"]=visiting
|
|
validate_expression_provenance \
|
|
"${recording_rule_expression[$record]}" \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"${context} -> recording rule ${record}"
|
|
recording_rule_state["$record"]=done
|
|
}
|
|
|
|
validate_expression_provenance() {
|
|
local expression=$1
|
|
local target_inventory=$2
|
|
local substrate_inventory=$3
|
|
local context=$4
|
|
local flattened selector without_selectors bare_expression token
|
|
|
|
flattened=$(tr '\n' ' ' <<<"$expression")
|
|
while IFS= read -r selector; do
|
|
[[ -n "$selector" ]] || continue
|
|
validate_raw_selector_provenance \
|
|
"$selector" "$target_inventory" "$substrate_inventory" "$context"
|
|
done < <(
|
|
grep -oE '[A-Za-z_:][A-Za-z0-9_:]*[[:space:]]*\{[^}]*\}' <<<"$flattened" || true
|
|
)
|
|
|
|
without_selectors=$(sed -E 's/[A-Za-z_:][A-Za-z0-9_:]*[[:space:]]*\{[^}]*\}//g' <<<"$flattened")
|
|
bare_expression=$(sed -E \
|
|
-e 's/(by|without|on|ignoring|group_left|group_right)[[:space:]]*\([^)]*\)//g' \
|
|
-e 's/"[^"]*"//g' <<<"$without_selectors")
|
|
while IFS= read -r token; do
|
|
[[ -n "$token" ]] || continue
|
|
if is_promql_syntax_token "$token" || is_promql_function_call "$token" "$bare_expression"; then
|
|
continue
|
|
fi
|
|
if [[ -n ${recording_rule_expression[$token]+x} ]]; then
|
|
validate_recording_rule_provenance \
|
|
"$token" "$target_inventory" "$substrate_inventory" "$context"
|
|
elif [[ "$token" == *:* ]]; then
|
|
validate_recording_rule_provenance \
|
|
"$token" "$target_inventory" "$substrate_inventory" "$context"
|
|
elif [[ -n ${new_rule_metric_phase[$token]+x} ]]; then
|
|
assert_mapped_raw_metric "$token" 0 "$target_inventory" "$substrate_inventory" "$context"
|
|
else
|
|
fail "unmapped raw metric or recording rule reference: ${token} in ${context}"
|
|
fi
|
|
done < <(
|
|
{ grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$bare_expression" || true; } | sort -u
|
|
)
|
|
}
|
|
|
|
assert_new_rule_provenance() {
|
|
local target_inventory=$1
|
|
local substrate_inventory=$2
|
|
local rule_json rule_name encoded_expression expression
|
|
|
|
shift 2
|
|
load_recording_rule_expressions "$@"
|
|
for rule_json in "$@"; do
|
|
while IFS=$'\t' read -r rule_name encoded_expression; do
|
|
expression=$(printf '%s' "$encoded_expression" | base64 -d)
|
|
validate_expression_provenance \
|
|
"$expression" "$target_inventory" "$substrate_inventory" \
|
|
"${rule_json} rule ${rule_name}"
|
|
done < <(
|
|
jq -r '
|
|
.spec.groups[].rules[]
|
|
| [(.record // .alert), (.expr | @base64)]
|
|
| @tsv
|
|
' "$rule_json"
|
|
)
|
|
done
|
|
}
|
|
|
|
assert_provenance_rejects() {
|
|
local expected_reason=$1
|
|
shift
|
|
local output
|
|
|
|
if output="$( (assert_new_rule_provenance "$@") 2>&1 )"; then
|
|
fail "provenance regression unexpectedly accepted a typo: ${expected_reason}"
|
|
fi
|
|
grep -Fq "$expected_reason" <<<"$output" ||
|
|
fail "provenance regression did not reject the typo for ${expected_reason}: ${output}"
|
|
}
|
|
|
|
assert_new_rule_provenance_regressions() {
|
|
local target_inventory=$1
|
|
local substrate_inventory=$2
|
|
local quota_json=$3
|
|
local certificate_json=$4
|
|
local services_json=$5
|
|
local metric_typo_json="${temp_dir}/provenance-metric-typo.rules.json"
|
|
local matcher_typo_json="${temp_dir}/provenance-matcher-typo.rules.json"
|
|
local quota_matcher_removed_json="${temp_dir}/provenance-quota-matcher-removed.rules.json"
|
|
local quota_matcher_value_typo_json="${temp_dir}/provenance-quota-matcher-value-typo.rules.json"
|
|
local cnpg_matcher_removed_json="${temp_dir}/provenance-cnpg-matcher-removed.rules.json"
|
|
local cnpg_matcher_value_typo_json="${temp_dir}/provenance-cnpg-matcher-value-typo.rules.json"
|
|
local record_typo_json="${temp_dir}/provenance-record-typo.rules.json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.record == "platform:aistor_bucket_quota_usage_percent")
|
|
| .expr)
|
|
|= sub(
|
|
"minio_cluster_usage_buckets_total_bytes";
|
|
"minio_cluster_usage_buckets_total_typo_bytes"
|
|
)
|
|
' "$quota_json" >"$metric_typo_json"
|
|
assert_provenance_rejects \
|
|
'unmapped raw metric' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$metric_typo_json" "$certificate_json" "$services_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformCNPGCollectorDown")
|
|
| .expr)
|
|
|= sub("cluster=\"platform-postgres\""; "clustr=\"platform-postgres\"")
|
|
' "$services_json" >"$matcher_typo_json"
|
|
assert_provenance_rejects \
|
|
'inventory lacks' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_json" "$certificate_json" "$matcher_typo_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.record == "platform:aistor_bucket_quota_usage_percent")
|
|
| .expr)
|
|
|= gsub("\\{bucket=~\\\"loki\\|tempo\\\"\\}"; "")
|
|
' "$quota_json" >"$quota_matcher_removed_json"
|
|
assert_provenance_rejects \
|
|
'exact matcher contract' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_matcher_removed_json" "$certificate_json" "$services_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.record == "platform:aistor_bucket_quota_usage_percent")
|
|
| .expr)
|
|
|= sub("loki\\|tempo"; "loki|temop")
|
|
' "$quota_json" >"$quota_matcher_value_typo_json"
|
|
assert_provenance_rejects \
|
|
'exact matcher contract' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_matcher_value_typo_json" "$certificate_json" "$services_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformCNPGCollectorDown")
|
|
| .expr)
|
|
|= sub("namespace=\\\"platform-data\\\","; "")
|
|
' "$services_json" >"$cnpg_matcher_removed_json"
|
|
assert_provenance_rejects \
|
|
'exact matcher contract' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_json" "$certificate_json" "$cnpg_matcher_removed_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformCNPGCollectorDown")
|
|
| .expr)
|
|
|= sub("platform-data"; "platform-dtaa")
|
|
' "$services_json" >"$cnpg_matcher_value_typo_json"
|
|
assert_provenance_rejects \
|
|
'exact matcher contract' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_json" "$certificate_json" "$cnpg_matcher_value_typo_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformAIStorBucketQuotaUsage")
|
|
| .expr)
|
|
|= sub(
|
|
"platform:aistor_bucket_quota_usage_percent";
|
|
"platform:aistor_bucket_quota_usage_typo_percent"
|
|
)
|
|
' "$quota_json" >"$record_typo_json"
|
|
assert_provenance_rejects \
|
|
'unresolved recording rule reference' \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$record_typo_json" "$certificate_json" "$services_json"
|
|
|
|
pass 'provenance rejects real new-rule metric, exact matcher, and recording-reference mutations'
|
|
}
|
|
|
|
assert_new_rule_semantics() {
|
|
[[ $# -eq 3 ]] || fail 'new-rule semantic projection requires quota, certificate, and service rule JSON'
|
|
local actual expected canonical_expected
|
|
|
|
expected='[
|
|
["platform-aistor-storage-quota", "platform.aistor-storage-quota", 0, "record", "platform:aistor_bucket_quota_usage_percent", "100 * max by (bucket) ( minio_cluster_usage_buckets_total_bytes{bucket=~\"loki|tempo\"} ) / max by (bucket) ( minio_cluster_usage_buckets_quota_total_bytes{bucket=~\"loki|tempo\"} > 0 )", null, null],
|
|
["platform-aistor-storage-quota", "platform.aistor-storage-quota", 1, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 70 and platform:aistor_bucket_quota_usage_percent < 85", "15m", "warning"],
|
|
["platform-aistor-storage-quota", "platform.aistor-storage-quota", 2, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 85 and platform:aistor_bucket_quota_usage_percent < 95", "10m", "critical"],
|
|
["platform-aistor-storage-quota", "platform.aistor-storage-quota", 3, "alert", "PlatformAIStorBucketQuotaUsage", "platform:aistor_bucket_quota_usage_percent >= 95", "5m", "emergency"],
|
|
["platform-certificate-probes", "platform.blackbox", 0, "alert", "PlatformPublicEdgeProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"public-edge\" } == 0", "5m", "critical"],
|
|
["platform-certificate-probes", "platform.blackbox", 1, "alert", "PlatformPrivateEdgeProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"private-edge\" } == 0", "5m", "critical"],
|
|
["platform-certificate-probes", "platform.blackbox", 2, "alert", "PlatformInternalHealthProbeFailed", "probe_success{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=\"private-internal\" } == 0", "5m", "critical"],
|
|
["platform-certificate-probes", "platform.certificates", 0, "alert", "PlatformCertificateExpiry", "( probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() ) >= 14 * 24 * 60 * 60 and ( probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() ) < 30 * 24 * 60 * 60", null, "warning"],
|
|
["platform-certificate-probes", "platform.certificates", 1, "alert", "PlatformCertificateExpiry", "probe_ssl_earliest_cert_expiry{ namespace=\"observability\", \"observability.hyeonworks.com/probe-group\"=~\"public-edge|private-edge\" } - time() < 14 * 24 * 60 * 60", null, "critical"],
|
|
["platform-verified-services", "platform.verified-services", 0, "alert", "PlatformCNPGCollectorDown", "min by (cluster, namespace) ( cnpg_collector_up{ namespace=\"platform-data\", cluster=\"platform-postgres\" } ) == 0", "5m", "critical"],
|
|
["platform-verified-services", "platform.verified-services", 1, "alert", "PlatformAlloyConfigLoadFailed", "min by (namespace) ( alloy_config_last_load_successful{namespace=\"observability-agent\"} ) == 0", "5m", "critical"],
|
|
["platform-verified-services", "platform.verified-services", 2, "alert", "PlatformAlloyLogDeliveryRetries", "sum by (namespace) ( increase(loki_write_batch_retries_total{namespace=\"observability-agent\"}[15m]) ) > 0", "5m", "warning"],
|
|
["platform-verified-services", "platform.verified-services", 3, "alert", "PlatformAlloyLogEntriesDropped", "sum by (namespace) ( increase(loki_write_dropped_entries_total{namespace=\"observability-agent\"}[15m]) ) > 0", "5m", "critical"],
|
|
["platform-verified-services", "platform.verified-services", 4, "alert", "PlatformLokiRuntimeConfigReloadFailed", "min by (namespace) ( loki_runtime_config_last_reload_successful{namespace=\"observability\"} ) == 0", "5m", "critical"],
|
|
["platform-verified-services", "platform.verified-services", 5, "alert", "PlatformLokiWALDiskUsageHigh", "max by (namespace) ( loki_ingester_wal_disk_usage_percent{namespace=\"observability\"} ) >= 0.8", "15m", "warning"]
|
|
]'
|
|
|
|
actual="$(jq -c -s '
|
|
def normalized_expr:
|
|
gsub("[[:space:]]+"; " ")
|
|
| sub("^ "; "")
|
|
| sub(" $"; "");
|
|
[.[] as $resource
|
|
| $resource.spec.groups[] as $group
|
|
| $group.rules
|
|
| to_entries[]
|
|
| [
|
|
$resource.metadata.name,
|
|
$group.name,
|
|
.key,
|
|
(if .value.record then "record" else "alert" end),
|
|
(.value.record // .value.alert),
|
|
(.value.expr | normalized_expr),
|
|
(.value.for // null),
|
|
(.value.labels.severity // null)
|
|
]
|
|
]
|
|
' "$@")" || fail 'could not project new-rule semantics'
|
|
canonical_expected="$(jq -c . <<<"$expected")" ||
|
|
fail 'invalid expected new-rule semantic projection'
|
|
|
|
[[ "$actual" == "$canonical_expected" ]] ||
|
|
fail 'new-rule semantic projection mismatch'
|
|
}
|
|
|
|
assert_semantics_rejects() {
|
|
local mutation=$1
|
|
shift
|
|
local output
|
|
|
|
declare -F assert_new_rule_semantics >/dev/null ||
|
|
fail 'new-rule semantic validator is unavailable'
|
|
if output="$( (assert_new_rule_semantics "$@") 2>&1 )"; then
|
|
fail "semantic regression unexpectedly accepted ${mutation}"
|
|
fi
|
|
grep -Fq 'new-rule semantic projection mismatch' <<<"$output" ||
|
|
fail "semantic regression rejected ${mutation} for the wrong reason: ${output}"
|
|
}
|
|
|
|
assert_new_rule_semantic_regressions() {
|
|
local quota_json=$1
|
|
local certificate_json=$2
|
|
local services_json=$3
|
|
local public_group_drift_json="${temp_dir}/semantic-public-group-drift.rules.json"
|
|
local selector_removed_json="${temp_dir}/semantic-selector-removed.rules.json"
|
|
local duplicate_selector_json="${temp_dir}/semantic-duplicate-selector.rules.json"
|
|
local cnpg_rebound_json="${temp_dir}/semantic-cnpg-rebound.rules.json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformPublicEdgeProbeFailed")
|
|
| .expr)
|
|
|= sub("public-edge"; "private-edge")
|
|
' "$certificate_json" >"$public_group_drift_json"
|
|
assert_semantics_rejects \
|
|
'PlatformPublicEdgeProbeFailed rebound to the valid private-edge fixture' \
|
|
"$quota_json" "$public_group_drift_json" "$services_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformPublicEdgeProbeFailed")
|
|
| .expr) = "vector(0)"
|
|
' "$certificate_json" >"$selector_removed_json"
|
|
assert_semantics_rejects \
|
|
'PlatformPublicEdgeProbeFailed with its selector removed' \
|
|
"$quota_json" "$selector_removed_json" "$services_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformPublicEdgeProbeFailed")
|
|
| .expr)
|
|
+= " or probe_success{namespace=\"observability\",\"observability.hyeonworks.com/probe-group\"=\"private-edge\"} == 0"
|
|
' "$certificate_json" >"$duplicate_selector_json"
|
|
assert_semantics_rejects \
|
|
'PlatformPublicEdgeProbeFailed with an extra valid probe selector' \
|
|
"$quota_json" "$duplicate_selector_json" "$services_json"
|
|
|
|
jq '
|
|
(.spec.groups[].rules[]
|
|
| select(.alert == "PlatformCNPGCollectorDown")
|
|
| .expr) = "min by (namespace) (alloy_config_last_load_successful{namespace=\"observability-agent\"}) == 0"
|
|
' "$services_json" >"$cnpg_rebound_json"
|
|
assert_semantics_rejects \
|
|
'PlatformCNPGCollectorDown rebound to the valid Alloy fixture' \
|
|
"$quota_json" "$certificate_json" "$cnpg_rebound_json"
|
|
|
|
pass 'semantic projection rejects rule-identity, selector-cardinality, and fixture-routing mutations'
|
|
}
|
|
|
|
render_rule_json() {
|
|
local source_file=$1
|
|
local output_file=$2
|
|
|
|
[[ -f "$source_file" ]] || fail "missing rule source: ${source_file}"
|
|
kubectl create --dry-run=client --validate=false -f "$source_file" -o json >"$output_file"
|
|
jq -e '
|
|
.apiVersion == "monitoring.coreos.com/v1" and
|
|
.kind == "PrometheusRule" and
|
|
.metadata.namespace == "observability" and
|
|
.metadata.labels["observability.hyeonworks.com/instance"] == "home" and
|
|
(.spec.groups | type == "array" and length > 0)
|
|
' "$output_file" >/dev/null || fail "invalid PrometheusRule envelope: ${source_file}"
|
|
}
|
|
|
|
alert_runbooks_are_exact() {
|
|
local rule_json=$1
|
|
|
|
jq -e --arg expected "$EXPECTED_RUNBOOK_URL" '
|
|
all(.spec.groups[].rules[] | select(has("alert"));
|
|
.annotations.runbook_url == $expected
|
|
)
|
|
' "$rule_json" >/dev/null
|
|
}
|
|
|
|
assert_exact_runbook_regression() {
|
|
local rule_json=$1 mutated="${temp_dir}/runbook-url-mutated.rules.json"
|
|
|
|
alert_runbooks_are_exact "$rule_json" ||
|
|
fail 'the production rule fixture lacks the exact runbook URL'
|
|
jq '
|
|
(.spec.groups[].rules[] | select(has("alert")) | .annotations.runbook_url) =
|
|
"https://git.learn.hyeonworks.com/wrong/repository/runbook.md"
|
|
' "$rule_json" >"$mutated"
|
|
if alert_runbooks_are_exact "$mutated"; then
|
|
fail 'a different valid HTTPS runbook URL was accepted'
|
|
fi
|
|
pass 'runbook annotation rejects a different valid HTTPS URL'
|
|
}
|
|
|
|
assert_alert_contract() {
|
|
local rule_json=$1
|
|
|
|
jq -e '
|
|
all(.spec.groups[].rules[] | select(has("alert"));
|
|
(.labels | keys | sort) == ["severity"] and
|
|
(.labels.severity | IN("warning", "critical", "emergency")) and
|
|
(.annotations.summary | type == "string" and length > 0) and
|
|
(.annotations.description | type == "string" and length > 0)
|
|
)
|
|
' "$rule_json" >/dev/null || fail "alert label/summary/description contract failed: ${rule_json}"
|
|
|
|
if ! alert_runbooks_are_exact "$rule_json"; then
|
|
printf 'BLOCKED: alert lacks the exact operator-reachable HTTPS runbook_url: %s\n' "$rule_json" >&2
|
|
runbook_url_blocked=1
|
|
fi
|
|
}
|
|
|
|
assert_exact_alert_names() {
|
|
local rule_json=$1
|
|
shift
|
|
local expected actual
|
|
|
|
expected="$(printf '%s\n' "$@" | sort)"
|
|
actual="$(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$rule_json" | sort)"
|
|
[[ "$actual" == "$expected" ]] || {
|
|
printf 'Expected alert names:\n%s\nActual alert names:\n%s\n' "$expected" "$actual" >&2
|
|
fail "unexpected alert set: ${rule_json}"
|
|
}
|
|
}
|
|
|
|
run_promtool() {
|
|
local work_dir=$1
|
|
shift
|
|
|
|
docker run --rm \
|
|
--network none \
|
|
--volume "${work_dir}:/work:ro" \
|
|
--entrypoint /bin/promtool \
|
|
"$PROMETHEUS_IMAGE" "$@"
|
|
}
|
|
|
|
run_live_promql() {
|
|
local -a rule_jsons=("$@")
|
|
local proxy_path='/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1'
|
|
local now start rule_name encoded_expr expression encoded_query response query_status result_count
|
|
|
|
now="$(date +%s)"
|
|
start="$((now - 300))"
|
|
|
|
while IFS=$'\t' read -r rule_name encoded_expr; do
|
|
expression="$(printf '%s' "$encoded_expr" | base64 -d)"
|
|
encoded_query="$(jq -rn --arg query "$expression" '$query | @uri')"
|
|
|
|
response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${encoded_query}")" ||
|
|
fail "live instant query transport failed: ${rule_name}"
|
|
query_status="$(jq -r '.status' <<<"$response")"
|
|
[[ "$query_status" == success ]] ||
|
|
fail "live instant query rejected: ${rule_name}"
|
|
result_count="$(jq -r '.data.result | length' <<<"$response")"
|
|
printf 'LIVE INSTANT PASS: %s (%s series)\n' "$rule_name" "$result_count"
|
|
|
|
response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query_range?query=${encoded_query}&start=${start}&end=${now}&step=60")" ||
|
|
fail "live range query transport failed: ${rule_name}"
|
|
query_status="$(jq -r '.status' <<<"$response")"
|
|
[[ "$query_status" == success ]] ||
|
|
fail "live range query rejected: ${rule_name}"
|
|
result_count="$(jq -r '.data.result | length' <<<"$response")"
|
|
printf 'LIVE RANGE PASS: %s (%s series)\n' "$rule_name" "$result_count"
|
|
done < <(
|
|
jq -r '
|
|
.spec.groups[].rules[] |
|
|
[(.alert // .record), (.expr | @base64)] |
|
|
@tsv
|
|
' "${rule_jsons[@]}"
|
|
)
|
|
}
|
|
|
|
assert_live_private_403() {
|
|
local proxy_path='/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1'
|
|
local success_query status_query success_response status_response
|
|
local success_set status_set
|
|
|
|
success_query="$(jq -rn --arg query 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"}' '$query | @uri')"
|
|
status_query="$(jq -rn --arg query 'probe_http_status_code{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"}' '$query | @uri')"
|
|
success_response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${success_query}")" ||
|
|
fail 'live private-edge probe_success transport failed'
|
|
status_response="$(kubectl --request-timeout=10s get --raw "${proxy_path}/query?query=${status_query}")" ||
|
|
fail 'live private-edge status transport failed'
|
|
|
|
success_set="$(jq -cS '[.data.result[] | select(.value[1] == "1") | .metric.instance] | sort' <<<"$success_response")"
|
|
status_set="$(jq -cS '[.data.result[] | select(.value[1] == "403") | .metric.instance] | sort' <<<"$status_response")"
|
|
[[ "$success_set" == "$status_set" ]] ||
|
|
fail 'live private-edge success and HTTP 403 instance sets differ'
|
|
[[ "$(jq 'length' <<<"$success_set")" -eq 3 ]] ||
|
|
fail 'live private-edge contract must have exactly three successful HTTP 403 endpoints'
|
|
pass 'live private-edge three endpoints are probe_success=1 with HTTP 403'
|
|
}
|
|
|
|
main() {
|
|
[[ $# -eq 1 || ( $# -eq 2 && "$2" == --live-prometheus ) ]] ||
|
|
fail "usage: $0 VERIFIED_OUTPUT_DIR [--live-prometheus]"
|
|
local inventory_root=$1
|
|
local live_prometheus=0
|
|
local target_inventory="${inventory_root}/target-initial/inventory.json"
|
|
local substrate_inventory="${inventory_root}/post-substrate/inventory.json"
|
|
local quota_json certificate_json services_json core_json
|
|
local rendered_names duplicate_names
|
|
|
|
[[ ${2:-} == --live-prometheus ]] && live_prometheus=1
|
|
|
|
require_command docker
|
|
require_command base64
|
|
require_command jq
|
|
require_command kubectl
|
|
require_command sha256sum
|
|
|
|
assert_inventory "${inventory_root}/target-initial" target-initial
|
|
assert_inventory "${inventory_root}/post-substrate" post-substrate
|
|
|
|
assert_metric_label "$target_inventory" serviceMonitor/object-storage/aistor-bucket-usage/0 minio_cluster_usage_buckets_total_bytes bucket
|
|
assert_metric_label "$target_inventory" serviceMonitor/object-storage/aistor-bucket-usage/0 minio_cluster_usage_buckets_quota_total_bytes bucket
|
|
assert_metric_label "$target_inventory" podMonitor/platform-data/platform-postgres/0 cnpg_collector_up cluster
|
|
assert_metric_label "$target_inventory" podMonitor/platform-data/platform-postgres/0 cnpg_collector_up namespace
|
|
assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 alloy_config_last_load_successful namespace
|
|
assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 loki_write_batch_retries_total namespace
|
|
assert_metric_label "$target_inventory" serviceMonitor/observability-agent/alloy/0 loki_write_dropped_entries_total namespace
|
|
assert_metric_label "$target_inventory" serviceMonitor/observability/loki/0 loki_runtime_config_last_reload_successful namespace
|
|
assert_metric_label "$target_inventory" serviceMonitor/observability/loki/0 loki_ingester_wal_disk_usage_percent namespace
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success namespace
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success observability.hyeonworks.com/probe-group
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_success instance
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry namespace
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry observability.hyeonworks.com/probe-group
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-public-edge probe_ssl_earliest_cert_expiry instance
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success namespace
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success observability.hyeonworks.com/probe-group
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_success instance
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_http_status_code observability.hyeonworks.com/probe-group
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry namespace
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry observability.hyeonworks.com/probe-group
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-edge probe_ssl_earliest_cert_expiry instance
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success namespace
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success observability.hyeonworks.com/probe-group
|
|
assert_metric_label "$substrate_inventory" probe/observability/platform-private-internal probe_success instance
|
|
pass "inventory hashes, phases, metrics, and labels"
|
|
|
|
temp_dir="$(mktemp -d)"
|
|
chmod 0755 "$temp_dir"
|
|
trap cleanup EXIT
|
|
quota_json="${temp_dir}/storage-quota.rules.json"
|
|
certificate_json="${temp_dir}/certificate.rules.json"
|
|
services_json="${temp_dir}/verified-service.rules.json"
|
|
core_json="${temp_dir}/core.rules.json"
|
|
|
|
render_rule_json "${RULE_DIR}/storage-quota-rules.yaml" "$quota_json"
|
|
render_rule_json "${RULE_DIR}/certificate-rules.yaml" "$certificate_json"
|
|
render_rule_json "${RULE_DIR}/verified-service-rules.yaml" "$services_json"
|
|
render_rule_json "${RULE_DIR}/core-rules.yaml" "$core_json"
|
|
assert_new_rule_provenance_regressions \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_json" "$certificate_json" "$services_json"
|
|
assert_new_rule_semantic_regressions \
|
|
"$quota_json" "$certificate_json" "$services_json"
|
|
assert_exact_runbook_regression "$quota_json"
|
|
assert_new_rule_provenance \
|
|
"$target_inventory" "$substrate_inventory" \
|
|
"$quota_json" "$certificate_json" "$services_json"
|
|
pass 'new-rule raw metrics, matcher labels, and recording dependencies have inventory provenance'
|
|
assert_new_rule_semantics \
|
|
"$quota_json" "$certificate_json" "$services_json"
|
|
pass 'new-rule identities, groups, expressions, durations, and severities have exact semantic binding'
|
|
assert_alert_contract "$quota_json"
|
|
assert_alert_contract "$certificate_json"
|
|
assert_alert_contract "$services_json"
|
|
if ! alert_runbooks_are_exact "$core_json"; then
|
|
printf 'BLOCKED: core alert lacks the exact operator-reachable HTTPS runbook_url: %s\n' "$core_json" >&2
|
|
runbook_url_blocked=1
|
|
fi
|
|
|
|
jq -e '
|
|
([.spec.groups[].rules[] | select(.record == "platform:aistor_bucket_quota_usage_percent")] | length) == 1 and
|
|
([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage")] | length) == 3 and
|
|
([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage") | .for] | sort) == ["10m", "15m", "5m"] and
|
|
([.spec.groups[].rules[] | select(.alert == "PlatformAIStorBucketQuotaUsage") | .labels.severity] | sort) == ["critical", "emergency", "warning"]
|
|
' "$quota_json" >/dev/null || fail "AIStor quota rule shape is not exact"
|
|
|
|
assert_exact_alert_names "$certificate_json" \
|
|
PlatformCertificateExpiry \
|
|
PlatformCertificateExpiry \
|
|
PlatformInternalHealthProbeFailed \
|
|
PlatformPrivateEdgeProbeFailed \
|
|
PlatformPublicEdgeProbeFailed
|
|
|
|
assert_exact_alert_names "$services_json" \
|
|
PlatformAlloyConfigLoadFailed \
|
|
PlatformAlloyLogDeliveryRetries \
|
|
PlatformAlloyLogEntriesDropped \
|
|
PlatformCNPGCollectorDown \
|
|
PlatformLokiRuntimeConfigReloadFailed \
|
|
PlatformLokiWALDiskUsageHigh
|
|
|
|
duplicate_names="$(
|
|
comm -12 \
|
|
<(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$core_json" | sort -u) \
|
|
<(jq -r '.spec.groups[].rules[] | select(has("alert")) | .alert' "$quota_json" "$certificate_json" "$services_json" | sort -u)
|
|
)"
|
|
[[ -z "$duplicate_names" ]] || fail "new rules duplicate core alert names: ${duplicate_names}"
|
|
pass "rule envelopes, bounded labels, annotations, and non-duplicate alert sets"
|
|
|
|
rendered_names="$(
|
|
kubectl kustomize "$RULE_DIR" |
|
|
kubectl create --dry-run=client --validate=false -f - -o name |
|
|
sort
|
|
)"
|
|
[[ "$rendered_names" == $'prometheusrule.monitoring.coreos.com/platform-aistor-storage-quota\nprometheusrule.monitoring.coreos.com/platform-certificate-probes\nprometheusrule.monitoring.coreos.com/platform-observability-core\nprometheusrule.monitoring.coreos.com/platform-verified-services' ]] || {
|
|
printf 'Rendered resources:\n%s\n' "$rendered_names" >&2
|
|
fail "platform rule kustomization resource set is not exact"
|
|
}
|
|
pass "platform rule kustomization"
|
|
|
|
jq '.spec' "$quota_json" >"${temp_dir}/storage-quota.prometheus.json"
|
|
jq '.spec' "$certificate_json" >"${temp_dir}/certificate.prometheus.json"
|
|
jq '.spec' "$services_json" >"${temp_dir}/verified-service.prometheus.json"
|
|
jq '.spec' "$core_json" >"${temp_dir}/core.prometheus.json"
|
|
chmod 0444 "${temp_dir}"/*.prometheus.json
|
|
run_promtool "$temp_dir" check rules \
|
|
/work/storage-quota.prometheus.json \
|
|
/work/certificate.prometheus.json \
|
|
/work/verified-service.prometheus.json \
|
|
/work/core.prometheus.json
|
|
pass "promtool check rules"
|
|
|
|
printf '%s\n' \
|
|
'rule_files:' \
|
|
' - /work/storage-quota.prometheus.json' \
|
|
' - /work/certificate.prometheus.json' \
|
|
'evaluation_interval: 1m' \
|
|
'tests:' \
|
|
' - name: quota warning starts at 70 and excludes 85' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \
|
|
' values: '\''70+0x20'\''' \
|
|
' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \
|
|
' values: '\''100+0x20'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 16m' \
|
|
' alertname: PlatformAIStorBucketQuotaUsage' \
|
|
' exp_alerts:' \
|
|
' - exp_labels:' \
|
|
' bucket: loki' \
|
|
' severity: warning' \
|
|
' exp_annotations:' \
|
|
' summary: AIStor bucket quota usage is at warning level' \
|
|
' description: Bucket loki has remained between 70 and 85 percent used for 15 minutes.' \
|
|
' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
|
|
' - name: quota critical starts at 85 and excludes 95' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="tempo"}'\''' \
|
|
' values: '\''85+0x20'\''' \
|
|
' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="tempo"}'\''' \
|
|
' values: '\''100+0x20'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 11m' \
|
|
' alertname: PlatformAIStorBucketQuotaUsage' \
|
|
' exp_alerts:' \
|
|
' - exp_labels:' \
|
|
' bucket: tempo' \
|
|
' severity: critical' \
|
|
' exp_annotations:' \
|
|
' summary: AIStor bucket quota usage is at critical level' \
|
|
' description: Bucket tempo has remained between 85 and 95 percent used for 10 minutes.' \
|
|
' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
|
|
' - name: quota emergency starts at 95' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \
|
|
' values: '\''95+0x20'\''' \
|
|
' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \
|
|
' values: '\''100+0x20'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 6m' \
|
|
' alertname: PlatformAIStorBucketQuotaUsage' \
|
|
' exp_alerts:' \
|
|
' - exp_labels:' \
|
|
' bucket: loki' \
|
|
' severity: emergency' \
|
|
' exp_annotations:' \
|
|
' summary: AIStor bucket quota usage is at emergency level' \
|
|
' description: Stop the loki ingest path according to the runbook; do not delete objects automatically.' \
|
|
' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
|
|
' - name: zero quota cannot produce a percentage alert' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''minio_cluster_usage_buckets_total_bytes{bucket="loki"}'\''' \
|
|
' values: '\''100+0x20'\''' \
|
|
' - series: '\''minio_cluster_usage_buckets_quota_total_bytes{bucket="loki"}'\''' \
|
|
' values: '\''0+0x20'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 20m' \
|
|
' alertname: PlatformAIStorBucketQuotaUsage' \
|
|
' exp_alerts: []' \
|
|
' - name: successful expected 403 private probe remains healthy' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''probe_success{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \
|
|
' values: '\''1+0x10'\''' \
|
|
' - series: '\''probe_http_status_code{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \
|
|
' values: '\''403+0x10'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 10m' \
|
|
' alertname: PlatformPrivateEdgeProbeFailed' \
|
|
' exp_alerts: []' \
|
|
' - name: failed private probe fires after five minutes' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''probe_success{namespace="observability",job="blackbox-private-edge",instance="https://db-admin.learn.hyeonworks.com/","observability.hyeonworks.com/probe-group"="private-edge"}'\''' \
|
|
' values: '\''0+0x10'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 6m' \
|
|
' alertname: PlatformPrivateEdgeProbeFailed' \
|
|
' exp_alerts:' \
|
|
' - exp_labels:' \
|
|
' namespace: observability' \
|
|
' job: blackbox-private-edge' \
|
|
' instance: https://db-admin.learn.hyeonworks.com/' \
|
|
' "observability.hyeonworks.com/probe-group": private-edge' \
|
|
' severity: critical' \
|
|
' exp_annotations:' \
|
|
' summary: Private edge boundary probe failed' \
|
|
' description: The expected private edge response for https://db-admin.learn.hyeonworks.com/ has failed for 5 minutes; an expected HTTP 403 with probe_success=1 is healthy.' \
|
|
' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
|
|
' - name: TLS warning includes exactly fourteen days' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \
|
|
' values: '\''1209660+0x2'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 1m' \
|
|
' alertname: PlatformCertificateExpiry' \
|
|
' exp_alerts:' \
|
|
' - exp_labels:' \
|
|
' namespace: observability' \
|
|
' instance: https://git.learn.hyeonworks.com/api/healthz' \
|
|
' "observability.hyeonworks.com/probe-group": public-edge' \
|
|
' severity: warning' \
|
|
' exp_annotations:' \
|
|
' summary: TLS certificate expires within 30 days' \
|
|
' description: The earliest certificate for https://git.learn.hyeonworks.com/api/healthz expires in fewer than 30 days but not fewer than 14 days.' \
|
|
' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
|
|
' - name: TLS warning excludes exactly thirty days' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \
|
|
' values: '\''2592060+0x2'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 1m' \
|
|
' alertname: PlatformCertificateExpiry' \
|
|
' exp_alerts: []' \
|
|
' - name: TLS critical covers fewer than fourteen days' \
|
|
' interval: 1m' \
|
|
' input_series:' \
|
|
' - series: '\''probe_ssl_earliest_cert_expiry{namespace="observability",instance="https://git.learn.hyeonworks.com/api/healthz","observability.hyeonworks.com/probe-group"="public-edge"}'\''' \
|
|
' values: '\''864060+0x2'\''' \
|
|
' alert_rule_test:' \
|
|
' - eval_time: 1m' \
|
|
' alertname: PlatformCertificateExpiry' \
|
|
' exp_alerts:' \
|
|
' - exp_labels:' \
|
|
' namespace: observability' \
|
|
' instance: https://git.learn.hyeonworks.com/api/healthz' \
|
|
' "observability.hyeonworks.com/probe-group": public-edge' \
|
|
' severity: critical' \
|
|
' exp_annotations:' \
|
|
' summary: TLS certificate expires within 14 days' \
|
|
' description: The earliest certificate for https://git.learn.hyeonworks.com/api/healthz expires in fewer than 14 days.' \
|
|
' runbook_url: https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md' \
|
|
>"${temp_dir}/rules.test.yaml"
|
|
chmod 0444 "${temp_dir}/rules.test.yaml"
|
|
|
|
run_promtool "$temp_dir" test rules /work/rules.test.yaml
|
|
pass "promtool quota, probe, private-403, and TLS boundaries"
|
|
|
|
if (( live_prometheus == 1 )); then
|
|
run_live_promql "$quota_json" "$certificate_json" "$services_json" "$core_json"
|
|
assert_live_private_403
|
|
pass "live Prometheus instant and range queries"
|
|
fi
|
|
|
|
(( runbook_url_blocked == 0 )) ||
|
|
fail "operator-reachable HTTPS runbook_url contract is not defined"
|
|
}
|
|
|
|
main "$@"
|