Files
platform-core/scripts/validate/test-render-observability-core.sh
T
DongHyeonka 7e8a6aa1a4 fix(observability): pin the heartbeat route in the core renderer
render-observability-core.sh keeps its own exact copy of the
AlertmanagerConfig, and test-render-observability-core.sh feeds it the
same copy as a fixture. 11c13bb added the platform-heartbeat route and
receiver to the source and to test-observability-alerting.sh but not
to these two, so the pinned renderer rejected the source with
"platform AlertmanagerConfig is not the exact global Slack contract".

Add the route and receiver to both copies and name the heartbeat in
the reject message.

Verification: render-observability-core.sh (Helm v3.19.4, kustomize
v5.8.1) prints OBSERVABILITY CORE STATIC RENDER PASS, also with
--verified-output-dir; test-render-observability-core.sh passes.

Fixes: 11c13bb87b ("feat(observability): alert when Prometheus itself stops working")
2026-11-12 07:25:25 +09:00

1094 lines
59 KiB
Bash
Executable File

#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly RENDERER="$ROOT/scripts/validate/render-observability-core.sh"
readonly EXPECTED_REJECTION_STATUS=23
if [[ ! -f "$RENDERER" ]]; then
printf 'EXPECTED RED: renderer is missing: %s\n' "$RENDERER" >&2
exit 1
fi
# Sourcing must define assertions without running the renderer or changing shell options.
source "$RENDERER"
fixture_dir="$(mktemp -d /tmp/platform-observability-render-test.XXXXXX)"
handoff_dirs=()
cleanup() {
local rc=$?
local directory
for directory in "${handoff_dirs[@]}"; do
case "$directory" in
/tmp/platform-observability-core-apply.*) rm -rf -- "$directory" ;;
esac
done
case "$fixture_dir" in
/tmp/platform-observability-render-test.*) rm -rf -- "$fixture_dir" ;;
esac
exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
pass_count=0
expect_pass() {
local name="$1"
local output
shift
if ! output="$({ "$@"; } 2>&1)"; then
printf 'FAIL (expected pass): %s\n%s\n' "$name" "$output" >&2
exit 1
fi
pass_count=$((pass_count + 1))
}
expect_stateful_pass() {
local name="$1"
local output_file="$fixture_dir/stateful-output"
local rc
shift
set +e
"$@" >"$output_file" 2>&1
rc=$?
set -e
if [[ "$rc" != 0 ]]; then
printf 'FAIL (expected stateful pass): %s\n' "$name" >&2
sed -n '1,20p' "$output_file" >&2
exit 1
fi
pass_count=$((pass_count + 1))
}
expect_fail() {
local name="$1"
local output rc
shift
set +e
output="$({ "$@"; } 2>&1)"
rc=$?
set -e
if [[ "$rc" != "$EXPECTED_REJECTION_STATUS" ]]; then
printf 'FAIL (expected rejection status %s, got %s): %s\n%s\n' \
"$EXPECTED_REJECTION_STATUS" "$rc" "$name" "$output" >&2
exit 1
fi
if [[ "$output" != REJECT:* ]]; then
printf 'FAIL (expected pinned rejection message): %s\n%s\n' "$name" "$output" >&2
exit 1
fi
if [[ "$output" == *'DO-NOT-LEAK-FIXTURE'* ]]; then
printf 'FAIL (credential leaked): %s\n' "$name" >&2
exit 1
fi
pass_count=$((pass_count + 1))
}
write_file() {
local path="$1"
shift
printf '%s\n' "$@" >"$path"
}
assert_chart_pin_row() {
local expected="$*"
local label="$1"
local actual
actual="$(chart_pin_rows | awk -F '\t' -v label="$label" '$1 == label { print $1, $2, $3, $4, $5, $6, $7 }')"
[[ "$actual" == "$expected" ]]
}
assert_image_pin_row() {
local expected="$*"
local image="$1"
local actual
actual="$(image_pin_rows | awk -F '\t' -v image="$image" '$1 == image { print $1, $2 }')"
[[ "$actual" == "$expected" ]]
}
expect_pass 'Helm version pin' test "$(observability_expected_helm_version)" = v3.19.4
expect_pass 'Kustomize version pin' test "$(observability_expected_kustomize_version)" = v5.8.1
expect_pass 'Kubernetes render target pin' test "$(observability_helm_kube_version)" = 1.36.2
expected_render_roots=$'infrastructure/namespaces/observability\n'
expected_render_roots+=$'infrastructure/controllers/prometheus-operator-crds\n'
expected_render_roots+=$'infrastructure/storage/observability-local-pv\n'
expected_render_roots+=$'services/observability/kube-prometheus-stack\n'
expected_render_roots+=$'services/observability/loki\n'
expected_render_roots+=$'services/observability/tempo\n'
expected_render_roots+=$'services/observability/alloy\n'
expected_render_roots+=$'services/observability/node-exporter\n'
expected_render_roots+=$'services/observability/grafana\n'
expected_render_roots+=$'services/observability/blackbox-exporter\n'
expected_render_roots+=$'services/observability/network-policies\n'
expected_render_roots+=$'services/observability/targets\n'
expected_render_roots+=$'services/observability/rules/platform\n'
expected_render_roots+=$'services/observability/alerting\n'
expected_render_roots+=$'services/observability/dashboards/platform\n'
expected_render_roots+='services/observability'
expect_pass 'complete observability render root table' test "$(render_root_rows)" = "$expected_render_roots"
expected_chart_table=$'prometheus-operator-crds\thttps://prometheus-community.github.io/helm-charts\tprometheus-operator-crds\t31.0.0\tv0.93.0\t5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd\tinfrastructure/controllers/prometheus-operator-crds\n'
expected_chart_table+=$'kube-prometheus-stack\thttps://prometheus-community.github.io/helm-charts\tkube-prometheus-stack\t88.0.1\tv0.93.0\t0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117\tservices/observability/kube-prometheus-stack\n'
expected_chart_table+=$'loki\thttps://grafana-community.github.io/helm-charts\tloki\t18.7.1\t3.7.4\tde9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d\tservices/observability/loki\n'
expected_chart_table+=$'tempo\thttps://grafana-community.github.io/helm-charts\ttempo\t2.2.3\t2.10.7\t14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe\tservices/observability/tempo\n'
expected_chart_table+=$'alloy\thttps://grafana.github.io/helm-charts\talloy\t1.11.0\tv1.18.0\t11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd\tservices/observability/alloy\n'
expected_chart_table+=$'prometheus-node-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-node-exporter\t4.56.1\t1.12.1\t70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a\tservices/observability/node-exporter'
expected_chart_table+=$'\ngrafana\thttps://grafana-community.github.io/helm-charts\tgrafana\t12.10.1\t13.1.1\tff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8\tservices/observability/grafana'
expected_chart_table+=$'\nprometheus-blackbox-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-blackbox-exporter\t11.16.0\tv0.28.0\t932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b\tservices/observability/blackbox-exporter'
expect_pass 'exact eight-row chart table with no extras' test "$(chart_pin_rows)" = "$expected_chart_table"
expected_image_table=$'quay.io/prometheus-operator/prometheus-operator:v0.93.0\tsha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2\n'
expected_image_table+=$'quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0\tsha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905\n'
expected_image_table+=$'quay.io/prometheus/prometheus:v3.13.2-distroless\tsha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b\n'
expected_image_table+=$'quay.io/prometheus/alertmanager:v0.33.1\tsha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b\n'
expected_image_table+=$'registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1\tsha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4\n'
expected_image_table+=$'quay.io/prometheus/node-exporter:v1.12.1\tsha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798\n'
expected_image_table+=$'docker.io/grafana/loki:3.7.4\tsha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e\n'
expected_image_table+=$'docker.io/grafana/tempo:2.10.7\tsha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c\n'
expected_image_table+=$'docker.io/grafana/alloy:v1.18.0\tsha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17\n'
expected_image_table+=$'ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0\tsha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522\n'
expected_image_table+=$'docker.io/library/busybox:1.37.0\tsha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0\n'
expected_image_table+=$'quay.io/minio/aistor/mc\tsha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b\n'
expected_image_table+=$'docker.io/grafana/grafana:13.1.1\tsha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b\n'
expected_image_table+=$'quay.io/kiwigrid/k8s-sidecar:2.10.0\tsha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3\n'
expected_image_table+=$'quay.io/prometheus/blackbox-exporter:v0.28.0\tsha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68'
expect_pass 'exact fifteen-row image table with no extras' test "$(image_pin_rows)" = "$expected_image_table"
expect_pass 'Prometheus CRD chart pin' assert_chart_pin_row prometheus-operator-crds https://prometheus-community.github.io/helm-charts prometheus-operator-crds 31.0.0 v0.93.0 5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd infrastructure/controllers/prometheus-operator-crds
expect_pass 'KPS chart pin' assert_chart_pin_row kube-prometheus-stack https://prometheus-community.github.io/helm-charts kube-prometheus-stack 88.0.1 v0.93.0 0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117 services/observability/kube-prometheus-stack
expect_pass 'Loki chart pin' assert_chart_pin_row loki https://grafana-community.github.io/helm-charts loki 18.7.1 3.7.4 de9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d services/observability/loki
expect_pass 'Tempo chart pin' assert_chart_pin_row tempo https://grafana-community.github.io/helm-charts tempo 2.2.3 2.10.7 14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe services/observability/tempo
expect_pass 'Alloy chart pin' assert_chart_pin_row alloy https://grafana.github.io/helm-charts alloy 1.11.0 v1.18.0 11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd services/observability/alloy
expect_pass 'node-exporter chart pin' assert_chart_pin_row prometheus-node-exporter https://prometheus-community.github.io/helm-charts prometheus-node-exporter 4.56.1 1.12.1 70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a services/observability/node-exporter
expect_pass 'Grafana chart pin' assert_chart_pin_row grafana https://grafana-community.github.io/helm-charts grafana 12.10.1 13.1.1 ff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8 services/observability/grafana
expect_pass 'Blackbox chart pin' assert_chart_pin_row prometheus-blackbox-exporter https://prometheus-community.github.io/helm-charts prometheus-blackbox-exporter 11.16.0 v0.28.0 932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b services/observability/blackbox-exporter
expect_pass 'operator image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-operator:v0.93.0 sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
expect_pass 'reloader image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0 sha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
expect_pass 'Prometheus image pin' assert_image_pin_row quay.io/prometheus/prometheus:v3.13.2-distroless sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
expect_pass 'Alertmanager image pin' assert_image_pin_row quay.io/prometheus/alertmanager:v0.33.1 sha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
expect_pass 'KSM image pin' assert_image_pin_row registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
expect_pass 'node-exporter image pin' assert_image_pin_row quay.io/prometheus/node-exporter:v1.12.1 sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798
expect_pass 'Loki image pin' assert_image_pin_row docker.io/grafana/loki:3.7.4 sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e
expect_pass 'Tempo image pin' assert_image_pin_row docker.io/grafana/tempo:2.10.7 sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c
expect_pass 'Alloy image pin' assert_image_pin_row docker.io/grafana/alloy:v1.18.0 sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
expect_pass 'telemetrygen image pin' assert_image_pin_row ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0 sha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522
expect_pass 'BusyBox image pin' assert_image_pin_row docker.io/library/busybox:1.37.0 sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0
expect_pass 'AIStor mc image pin' assert_image_pin_row quay.io/minio/aistor/mc sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b
expect_pass 'Grafana image pin' assert_image_pin_row docker.io/grafana/grafana:13.1.1 sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b
expect_pass 'Grafana sidecar image pin' assert_image_pin_row quay.io/kiwigrid/k8s-sidecar:2.10.0 sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3
expect_pass 'Blackbox image pin' assert_image_pin_row quay.io/prometheus/blackbox-exporter:v0.28.0 sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68
expect_pass 'tagged Grafana digest is in the rendered-image allowlist' grep -Fqx \
docker.io/grafana/grafana:13.1.1@sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b \
<(_observability_image_allowlist)
expect_pass 'tagged Blackbox digest is in the rendered-image allowlist' grep -Fqx \
quay.io/prometheus/blackbox-exporter:v0.28.0@sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 \
<(_observability_image_allowlist)
alertmanager_fixture="$fixture_dir/alertmanager.yaml"
write_file "$alertmanager_fixture" \
'apiVersion: monitoring.coreos.com/v1' \
'kind: Alertmanager' \
'metadata:' \
' name: observability-core-kube-pr-alertmanager' \
' namespace: observability' \
'spec:' \
' alertmanagerConfiguration:' \
' name: platform-alertmanager'
alerting_fixture="$fixture_dir/alerting.yaml"
write_file "$alerting_fixture" '{"apiVersion":"monitoring.coreos.com/v1alpha1","kind":"AlertmanagerConfig","metadata":{"name":"platform-alertmanager","namespace":"observability","labels":{"observability.hyeonworks.com/instance":"home"}},"spec":{"route":{"receiver":"platform-slack","groupBy":["cluster","namespace","alertname","severity"],"groupWait":"30s","groupInterval":"5m","repeatInterval":"4h","routes":[{"receiver":"platform-null","matchers":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}]},{"receiver":"platform-heartbeat","matchers":[{"name":"alertname","matchType":"=","value":"PlatformHeartbeat"}],"groupWait":"0s","groupInterval":"1m","repeatInterval":"5m"}]},"inhibitRules":[{"sourceMatch":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}],"targetMatch":[{"name":"severity","matchType":"=","value":"info"}],"equal":["namespace"]}],"receivers":[{"name":"platform-null"},{"name":"platform-heartbeat","webhookConfigs":[{"urlSecret":{"name":"alertmanager-heartbeat","key":"url"},"sendResolved":false,"maxAlerts":1}]},{"name":"platform-slack","slackConfigs":[{"apiURL":{"name":"alertmanager-slack-webhook","key":"url"},"sendResolved":true,"linkNames":false,"mrkdwnIn":["text","fields"],"fallback":"{{ if eq .Status \"firing\" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","title":"{{ if eq .Status \"firing\" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","titleLink":"https://grafana.learn.hyeonworks.com/","color":"{{ if eq .Status \"resolved\" }}good{{ else if or (eq .CommonLabels.severity \"emergency\") (eq .CommonLabels.severity \"critical\") }}danger{{ else if eq .CommonLabels.severity \"warning\" }}warning{{ else }}#439FE0{{ end }}","fields":[{"title":"Status","value":"{{ .Status | toUpper }}","short":true},{"title":"Severity","value":"{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}","short":true},{"title":"Location","value":"{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}","short":true},{"title":"Alert count","value":"{{ len .Alerts }}","short":true}],"text":"{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>","footer":"hyeonworks observability · Alertmanager"}]}]}}' \
'---' \
'apiVersion: networking.k8s.io/v1' \
'kind: NetworkPolicy' \
'metadata:' \
' name: observability-allow-alertmanager-public-https' \
' namespace: observability' \
'spec:' \
' podSelector:' \
' matchLabels:' \
' app.kubernetes.io/name: alertmanager' \
' app.kubernetes.io/instance: observability-core-kube-pr-alertmanager' \
' policyTypes: [Egress]' \
' egress:' \
' - to:' \
' - ipBlock:' \
' cidr: 0.0.0.0/0' \
' except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]' \
' ports:' \
' - protocol: TCP' \
' port: 443'
expect_pass 'exact structured Alertmanager routing contract' \
assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_fixture"
alertmanager_selector_fixture="$fixture_dir/alertmanager-selector.yaml"
sed '/^spec:$/a\ alertmanagerConfigSelector: {}' \
"$alertmanager_fixture" >"$alertmanager_selector_fixture"
expect_fail 'Alertmanager discovery selector bypasses the global configuration' \
assert_alertmanager_routing_contract "$alertmanager_selector_fixture" "$alerting_fixture"
alerting_route_fixture="$fixture_dir/alerting-route-drift.yaml"
sed 's/"repeatInterval":"4h"/"repeatInterval":"12h"/' \
"$alerting_fixture" >"$alerting_route_fixture"
expect_fail 'Alertmanager Slack route drift' \
assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_route_fixture"
write_pod() {
local path="$1"
local namespace="$2"
local name="$3"
local pod_fields="$4"
local container_fields="${5:-}"
cat >"$path" <<EOF
apiVersion: v1
kind: Pod
metadata:
name: $name
namespace: $namespace
spec:
automountServiceAccountToken: false
$pod_fields
containers:
- name: main
image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
$container_fields
EOF
}
valid_image="$fixture_dir/valid-image.yaml"
write_file "$valid_image" \
'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: pinned' \
' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \
' containers:' ' - name: prometheus' \
' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
expect_pass 'pinned image accepted' assert_pinned_images "$valid_image"
expect_pass 'credential assertion positive control' assert_no_credentials "$valid_image"
expect_pass 'observability namespace security positive control' assert_namespace_pod_security "$valid_image"
tag_image="$fixture_dir/tag-image.yaml"
sed 's|@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b|:v3.13.2-distroless|' \
"$valid_image" >"$tag_image"
expect_fail 'tag-only image' assert_pinned_images "$tag_image"
wrong_digest="$fixture_dir/wrong-digest.yaml"
sed 's/ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' \
"$valid_image" >"$wrong_digest"
expect_fail 'unpinned digest' assert_pinned_images "$wrong_digest"
secret="$fixture_dir/secret.yaml"
write_file "$secret" \
'apiVersion: v1' 'kind: Secret' 'metadata:' ' name: forbidden' \
' namespace: observability' 'stringData:' ' password: DO-NOT-LEAK-FIXTURE'
expect_fail 'credential Secret' assert_no_credentials "$secret"
literal_credential="$fixture_dir/literal-credential.yaml"
write_file "$literal_credential" \
'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: forbidden' \
' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \
' containers:' ' - name: app' ' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' \
' env:' ' - name: ACCESS_KEY' ' value: DO-NOT-LEAK-FIXTURE'
expect_fail 'literal credential payload' assert_no_credentials "$literal_credential"
for field in \
' privileged: true' \
' hostNetwork: true' \
' hostPID: true' \
' hostIPC: true'; do
slug="$(printf '%s' "$field" | tr -cd '[:alpha:]' | tr '[:upper:]' '[:lower:]')"
pod="$fixture_dir/observability-$slug.yaml"
if [[ "$field" == *privileged* ]]; then
container_security="$(printf ' securityContext:\n privileged: true')"
write_pod "$pod" observability core '' "$container_security"
else
write_pod "$pod" observability core "$field" ''
fi
expect_fail "observability forbids $slug" assert_namespace_pod_security "$pod"
done
obs_hostpath="$fixture_dir/observability-hostpath.yaml"
write_pod "$obs_hostpath" observability core \
$' volumes:\n - name: host\n hostPath:\n path: /var/log/pods' \
$' volumeMounts:\n - name: host\n mountPath: /logs\n readOnly: true'
expect_fail 'observability hostPath' assert_namespace_pod_security "$obs_hostpath"
controller_namespace="$fixture_dir/controller-namespace.yaml"
write_file "$controller_namespace" 'apiVersion: apps/v1' 'kind: Deployment' \
'metadata:' ' name: controller-security' ' namespace: observability' 'spec:' \
' selector:' ' matchLabels:' ' app: controller-security' ' template:' \
' metadata:' ' labels:' ' app: controller-security' ' spec:' \
' automountServiceAccountToken: false' ' hostPID: true' ' containers:' \
' - name: app' \
' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
expect_fail 'controller template inherits namespace security' assert_namespace_pod_security "$controller_namespace"
controller_valid="$fixture_dir/controller-valid.yaml"
sed '/hostPID: true/d' "$controller_namespace" >"$controller_valid"
expect_pass 'controller observability namespace positive control' \
assert_namespace_pod_security "$controller_valid"
controller_missing_namespace="$fixture_dir/controller-missing-namespace.yaml"
sed '/namespace: observability/d' "$controller_valid" >"$controller_missing_namespace"
expect_fail 'controller namespace missing' \
assert_namespace_pod_security "$controller_missing_namespace"
controller_foreign_namespace="$fixture_dir/controller-foreign-namespace.yaml"
sed 's/namespace: observability/namespace: default/' \
"$controller_valid" >"$controller_foreign_namespace"
expect_fail 'controller namespace foreign' \
assert_namespace_pod_security "$controller_foreign_namespace"
agent_allowed="$fixture_dir/agent-allowed.yaml"
write_pod "$agent_allowed" observability-agent alloy \
$' volumes:\n - name: pod-logs\n hostPath:\n path: /var/log/pods' \
$' volumeMounts:\n - name: pod-logs\n mountPath: /var/log/pods\n readOnly: true'
expect_pass 'Alloy read-only allowlisted hostPath' assert_namespace_pod_security "$agent_allowed"
missing_namespace="$fixture_dir/missing-namespace.yaml"
sed '/namespace: observability-agent/d' "$agent_allowed" >"$missing_namespace"
expect_fail 'Pod namespace missing' assert_namespace_pod_security "$missing_namespace"
foreign_namespace="$fixture_dir/foreign-namespace.yaml"
sed 's/namespace: observability-agent/namespace: default/' "$agent_allowed" >"$foreign_namespace"
expect_fail 'Pod namespace foreign' assert_namespace_pod_security "$foreign_namespace"
agent_bad_path="$fixture_dir/agent-bad-path.yaml"
sed 's|/var/log/pods|/etc|g' "$agent_allowed" >"$agent_bad_path"
expect_fail 'agent hostPath allowlist' assert_namespace_pod_security "$agent_bad_path"
agent_rw="$fixture_dir/agent-rw.yaml"
sed 's/readOnly: true/readOnly: false/' "$agent_allowed" >"$agent_rw"
expect_fail 'agent hostPath readOnly' assert_namespace_pod_security "$agent_rw"
automount_missing="$fixture_dir/automount-missing.yaml"
sed '/automountServiceAccountToken:/d' "$valid_image" >"$automount_missing"
expect_fail 'automount missing' assert_namespace_pod_security "$automount_missing"
automount_true="$fixture_dir/automount-true.yaml"
sed 's/automountServiceAccountToken: false/automountServiceAccountToken: true/' \
"$valid_image" >"$automount_true"
expect_fail 'automount true' assert_namespace_pod_security "$automount_true"
for exposure in ingress NodePort LoadBalancer; do
public="$fixture_dir/public-$exposure.yaml"
if [[ "$exposure" == ingress ]]; then
write_file "$public" 'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
'metadata:' ' name: public' ' namespace: observability' 'spec: {}'
else
write_file "$public" 'apiVersion: v1' 'kind: Service' 'metadata:' \
' name: public' ' namespace: observability' 'spec:' " type: $exposure" \
' ports:' ' - port: 80'
fi
expect_fail "public $exposure" assert_no_public_exposure "$public"
done
grafana_ingress="$fixture_dir/grafana-private-ingress.yaml"
write_file "$grafana_ingress" \
'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
'metadata:' ' name: grafana' ' namespace: observability' \
'spec:' ' ingressClassName: traefik' ' rules:' \
' - host: grafana.learn.hyeonworks.com' ' http:' ' paths:' \
' - path: /' ' pathType: Prefix' ' backend:' \
' service:' ' name: grafana' ' port:' \
' number: 80'
expect_pass 'exact Host-Nginx-fronted Grafana ingress' \
assert_no_public_exposure "$grafana_ingress"
grafana_public_ingress="$fixture_dir/grafana-public-ingress.yaml"
sed 's/grafana.learn.hyeonworks.com/public.example.com/' \
"$grafana_ingress" >"$grafana_public_ingress"
expect_fail 'Grafana ingress host drift' \
assert_no_public_exposure "$grafana_public_ingress"
cluster_ip="$fixture_dir/cluster-ip.yaml"
write_file "$cluster_ip" 'apiVersion: v1' 'kind: Service' 'metadata:' \
' name: internal' ' namespace: observability' 'spec:' ' type: ClusterIP' \
' ports:' ' - port: 9090'
expect_pass 'public exposure assertion positive control' assert_no_public_exposure "$cluster_ip"
for auxiliary in gateway canary chunks-cache results-cache test minio; do
loki_aux="$fixture_dir/loki-$auxiliary.yaml"
write_file "$loki_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
" name: loki-$auxiliary" ' namespace: observability' 'spec: {}'
expect_fail "Loki auxiliary $auxiliary" assert_no_loki_auxiliary "$loki_aux"
done
loki_main="$fixture_dir/loki-main.yaml"
write_file "$loki_main" 'apiVersion: apps/v1' 'kind: StatefulSet' 'metadata:' \
' name: loki' ' namespace: observability' 'spec: {}'
expect_pass 'Loki auxiliary assertion positive control' assert_no_loki_auxiliary "$loki_main"
for auxiliary in metrics-generator tempo-query; do
tempo_aux="$fixture_dir/tempo-$auxiliary.yaml"
write_file "$tempo_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
" name: $auxiliary" ' namespace: observability' 'spec: {}'
expect_fail "Tempo auxiliary $auxiliary" assert_tempo_topology "$tempo_aux"
done
tempo_http="$fixture_dir/tempo-http.yaml"
write_file "$tempo_http" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \
' receivers:' ' otlp:' ' protocols:' ' grpc:' \
' endpoint: 0.0.0.0:4317' ' http:' \
' endpoint: 0.0.0.0:4318'
expect_fail 'Tempo OTLP HTTP receiver' assert_tempo_topology "$tempo_http"
tempo_grpc="$fixture_dir/tempo-grpc.yaml"
write_file "$tempo_grpc" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \
' receivers:' ' otlp:' ' protocols:' ' grpc:' \
' endpoint: 0.0.0.0:4317'
expect_pass 'Tempo topology assertion positive control' assert_tempo_topology "$tempo_grpc"
selector_values="$fixture_dir/selector-values.yaml"
write_file "$selector_values" 'prometheus:' ' prometheusSpec:' \
' serviceMonitorSelectorNilUsesHelmValues: false' \
' podMonitorSelectorNilUsesHelmValues: false' \
' probeSelectorNilUsesHelmValues: false' \
' ruleSelectorNilUsesHelmValues: false'
expect_pass 'selector values false' assert_selector_values "$selector_values"
selector_true="$fixture_dir/selector-true.yaml"
sed '0,/false/s//true/' "$selector_values" >"$selector_true"
expect_fail 'nilUsesHelmValues true' assert_selector_values "$selector_true"
prometheus_missing="$fixture_dir/prometheus-missing-selector.yaml"
write_file "$prometheus_missing" 'apiVersion: monitoring.coreos.com/v1' \
'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \
'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \
' probeSelector: {}'
expect_fail 'Prometheus selector missing' assert_prometheus_selectors "$prometheus_missing"
prometheus_valid="$fixture_dir/prometheus-valid.yaml"
write_file "$prometheus_valid" 'apiVersion: monitoring.coreos.com/v1' \
'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \
'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \
' probeSelector: {}' ' ruleSelector: {}'
expect_pass 'Prometheus selector assertion positive control' assert_prometheus_selectors "$prometheus_valid"
crd="$fixture_dir/kps-crd.yaml"
write_file "$crd" 'apiVersion: apiextensions.k8s.io/v1' \
'kind: CustomResourceDefinition' 'metadata:' ' name: prometheuses.monitoring.coreos.com'
expect_fail 'CRD in workload render' assert_no_crds "$crd"
expect_pass 'CRD assertion positive control' assert_no_crds "$valid_image"
shared_pv="$fixture_dir/shared-pv.yaml"
write_file "$shared_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
' name: prometheus' 'spec:' ' storageClassName: shared-local' ' local:' \
' path: /srv/prometheus' '---' 'apiVersion: v1' 'kind: PersistentVolume' \
'metadata:' ' name: alertmanager' 'spec:' ' storageClassName: shared-local' \
' local:' ' path: /srv/alertmanager'
expect_fail 'shared Local PV StorageClass' assert_unique_local_pv_storage_classes "$shared_pv"
single_pv="$fixture_dir/single-pv.yaml"
write_file "$single_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
' name: prometheus' 'spec:' ' storageClassName: prometheus-local' ' local:' \
' path: /srv/prometheus'
expect_pass 'Local PV assertion positive control' assert_unique_local_pv_storage_classes "$single_pv"
for fake in spring kafka batch backup; do
fake_monitor="$fixture_dir/fake-$fake.yaml"
write_file "$fake_monitor" 'apiVersion: monitoring.coreos.com/v1' \
'kind: ServiceMonitor' 'metadata:' " name: $fake-monitor" \
' namespace: observability' 'spec: {}'
expect_fail "fake $fake monitor or rule" assert_no_unscoped_monitors_rules "$fake_monitor"
done
scoped_monitor="$fixture_dir/scoped-monitor.yaml"
write_file "$scoped_monitor" 'apiVersion: monitoring.coreos.com/v1' \
'kind: ServiceMonitor' 'metadata:' ' name: prometheus' \
' namespace: observability' 'spec: {}'
expect_pass 'monitor/rule assertion positive control' assert_no_unscoped_monitors_rules "$scoped_monitor"
chart="$fixture_dir/Chart.yaml"
write_file "$chart" 'apiVersion: v2' 'name: loki' 'version: 18.7.1' 'appVersion: 3.7.4'
expect_pass 'chart metadata pin accepted' validate_chart_metadata "$chart" loki 18.7.1 3.7.4
expect_fail 'chart metadata name mismatch' validate_chart_metadata "$chart" tempo 18.7.1 3.7.4
expect_fail 'chart metadata version mismatch' validate_chart_metadata "$chart" loki 18.7.2 3.7.4
expect_fail 'chart metadata appVersion mismatch' validate_chart_metadata "$chart" loki 18.7.1 3.7.5
handoff="$fixture_dir/platform-observability-core-apply.fixture"
mkdir -m 0700 "$handoff"
expect_fail 'handoff prefix restricted to /tmp' validate_handoff_directory "$handoff"
safe_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$safe_handoff")
chmod 0700 "$safe_handoff"
expect_pass 'safe empty handoff' validate_handoff_directory "$safe_handoff"
touch "$safe_handoff/unexpected"
expect_fail 'handoff must be empty' validate_handoff_directory "$safe_handoff"
rm -f -- "$safe_handoff/unexpected"
rmdir -- "$safe_handoff"
empty_suffix='/tmp/platform-observability-core-apply.'
expect_fail 'handoff suffix must be nonempty' validate_handoff_directory "$empty_suffix"
root_fixture="$fixture_dir/render-roots"
mkdir -p "$root_fixture/existing/child"
expect_pass 'physical render roots accepted' validate_render_roots "$root_fixture" existing/child
expect_fail 'missing render root rejected' validate_render_roots "$root_fixture" missing/child
expect_pass 'missing root preflight creates nothing' test ! -e "$root_fixture/missing"
mkdir -p "$root_fixture/physical/child"
ln -s physical "$root_fixture/symlinked"
expect_fail 'symlinked render root lineage rejected' validate_render_roots "$root_fixture" symlinked/child
cache_root="$fixture_dir/cache-root"
mkdir -p "$cache_root/services/component/.helm/charts"
_OBS_ROOT="$cache_root"
_OBS_CREATED_CACHE_DIRS=()
expect_stateful_pass 'exclusive cache directory creation' _observability_create_cache_directory \
services/component chart 1.0.0
expect_fail 'existing exact cache is no-clobber' _observability_create_cache_directory \
services/component chart 1.0.0
expect_stateful_pass 'tracked cache cleanup' _observability_cleanup_cache_dirs
expect_pass 'pre-existing .helm preserved' test -d "$cache_root/services/component/.helm"
expect_pass 'pre-existing charts preserved' test -d "$cache_root/services/component/.helm/charts"
expect_pass 'exact created cache removed' test ! -e "$cache_root/services/component/.helm/charts/chart-1.0.0"
mkdir -p "$cache_root/services/missing-parent"
expect_fail 'missing component root rejected before cache mkdir' _observability_create_cache_directory \
services/missing-parent/component chart 1.0.0
expect_pass 'missing component root leaves no .helm' test ! -e "$cache_root/services/missing-parent/component"
mkdir -p "$cache_root/services/new-component"
_OBS_CREATED_CACHE_DIRS=()
_OBS_CREATED_CACHE_IDENTITIES=()
_OBS_CREATED_CACHE_TYPES=()
expect_stateful_pass 'new cache parents tracked' _observability_create_cache_directory \
services/new-component chart 1.0.0
expect_stateful_pass 'new cache parents cleaned' _observability_cleanup_cache_dirs
expect_pass 'created .helm parent removed' test ! -e "$cache_root/services/new-component/.helm"
mkdir -p "$cache_root/services/race-component/.helm/charts"
_OBS_CREATED_CACHE_DIRS=()
_OBS_CREATED_CACHE_IDENTITIES=()
_OBS_CREATED_CACHE_TYPES=()
expect_stateful_pass 'race cache created' _observability_create_cache_directory \
services/race-component chart 1.0.0
race_cache="${_OBS_CREATED_CACHE_DIRS[0]}"
race_identity="${_OBS_CREATED_CACHE_IDENTITIES[0]}"
mkdir "$race_cache/owned-subdirectory"
write_file "$race_cache/owned-subdirectory/owned-marker" 'OWNED-CACHE-CONTENT'
cleanup_race_hook="$fixture_dir/cache-cleanup-race-hook.sh"
write_file "$cleanup_race_hook" '#!/usr/bin/env bash' 'set -Eeuo pipefail' \
'replacement="$1"' 'mkdir -m 0700 -- "$replacement"' \
'printf "%s\n" REPLACEMENT-MUST-SURVIVE >"$replacement/replacement-marker"'
chmod 0700 "$cleanup_race_hook"
expect_stateful_pass 'owned cache cleanup is identity-bound across replacement race' \
_observability_cleanup_owned_cache_directory \
"$race_cache" "$race_identity" tree "$cleanup_race_hook"
expect_pass 'replacement directory survives cleanup race' test -d "$race_cache"
expect_pass 'replacement content survives cleanup race' test \
"$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
expect_pass 'owned quarantine is completely cleaned' test -z \
"$(find "$cache_root/services/race-component/.helm/charts" \
-mindepth 1 -maxdepth 1 ! -name chart-1.0.0 -print -quit)"
run_cleanup_exit_gate() (
_OBS_WORK=''
trap _observability_cleanup EXIT
:
)
expect_fail 'cleanup identity mismatch fails the successful exit gate' run_cleanup_exit_gate
expect_pass 'exit-gate mismatch still preserves replacement' test \
"$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
_OBS_CREATED_CACHE_DIRS=()
_OBS_CREATED_CACHE_IDENTITIES=()
_OBS_CREATED_CACHE_TYPES=()
handoff_work="$fixture_dir/handoff-work"
mkdir -m 0700 "$handoff_work"
for handoff_name in namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate; do
write_file "$handoff_work/$handoff_name.yaml" "fixture-$handoff_name"
chmod 0600 "$handoff_work/$handoff_name.yaml"
done
write_file "$handoff_work/resource-index.tsv" 'fixture-index'
chmod 0600 "$handoff_work/resource-index.tsv"
published_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$published_handoff")
chmod 0700 "$published_handoff"
published_identity="$(pin_handoff_directory "$published_handoff")"
expect_pass 'atomic no-clobber handoff publication' publish_verified_handoff \
"$published_handoff" "$published_identity" "$handoff_work"
expect_pass 'exact handoff entry set accepted' validate_handoff_entry_set \
"$published_handoff" "$published_identity"
ln -s /dev/null "$published_handoff/extra-symlink"
expect_fail 'extra handoff symlink rejected' validate_handoff_entry_set \
"$published_handoff" "$published_identity"
rm -f -- "$published_handoff/extra-symlink"
mkdir "$published_handoff/extra-directory"
expect_fail 'extra handoff directory rejected' validate_handoff_entry_set \
"$published_handoff" "$published_identity"
rmdir "$published_handoff/extra-directory"
mv "$published_handoff/aggregate.yaml" "$fixture_dir/aggregate-handoff-saved"
ln -s /dev/null "$published_handoff/aggregate.yaml"
expect_fail 'exact handoff entry symlink rejected' validate_handoff_entry_set \
"$published_handoff" "$published_identity"
rm -f -- "$published_handoff/aggregate.yaml"
mv "$fixture_dir/aggregate-handoff-saved" "$published_handoff/aggregate.yaml"
mv "$published_handoff/core-rules.yaml" "$fixture_dir/core-rules-handoff-saved"
mkdir "$published_handoff/core-rules.yaml"
expect_fail 'exact handoff entry directory rejected' validate_handoff_entry_set \
"$published_handoff" "$published_identity"
rmdir "$published_handoff/core-rules.yaml"
mv "$fixture_dir/core-rules-handoff-saved" "$published_handoff/core-rules.yaml"
replaced_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$replaced_handoff")
chmod 0700 "$replaced_handoff"
replaced_identity="$(pin_handoff_directory "$replaced_handoff")"
replaced_original="$replaced_handoff.original"
mv "$replaced_handoff" "$replaced_original"
handoff_dirs+=("$replaced_original")
mkdir -m 0700 "$replaced_handoff"
expect_fail 'replaced handoff identity rejected before writes' publish_verified_handoff \
"$replaced_handoff" "$replaced_identity" "$handoff_work"
expect_pass 'replacement remains empty' test -z "$(find "$replaced_handoff" -mindepth 1 -maxdepth 1 -print -quit)"
occupied_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
handoff_dirs+=("$occupied_handoff")
chmod 0700 "$occupied_handoff"
occupied_identity="$(pin_handoff_directory "$occupied_handoff")"
write_file "$occupied_handoff/namespaces.yaml" 'DO-NOT-CLOBBER'
expect_fail 'nonempty handoff rejected immediately before writes' publish_verified_handoff \
"$occupied_handoff" "$occupied_identity" "$handoff_work"
expect_pass 'existing handoff file is not clobbered' test \
"$(cat "$occupied_handoff/namespaces.yaml")" = DO-NOT-CLOBBER
child_a="$fixture_dir/child-a.yaml"
child_b="$fixture_dir/child-b.yaml"
aggregate="$fixture_dir/aggregate.yaml"
write_file "$child_a" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
' name: a' ' namespace: observability' 'data:' ' value: one'
write_file "$child_b" 'apiVersion: v1' 'kind: Service' 'metadata:' \
' name: b' ' namespace: observability' 'spec:' ' type: ClusterIP' \
' ports:' ' - port: 80'
{ cat "$child_a"; printf '%s\n' '---'; cat "$child_b"; } >"$aggregate"
index="$fixture_dir/resource-index.tsv"
expect_pass 'ordered resource index' build_resource_index "$index" \
"kps:$child_a" "loki:$child_b"
expected_index=$'1\tkps\tv1\tConfigMap\tobservability\ta\t'
if [[ "$(sed -n '2p' "$index")" != "$expected_index"* ]]; then
printf 'FAIL: resource index order or schema differs\n' >&2
exit 1
fi
pass_count=$((pass_count + 1))
expect_pass 'child aggregate equivalence' assert_child_aggregate_equivalence \
"$aggregate" "kps:$child_a" "loki:$child_b"
aggregate_missing="$fixture_dir/aggregate-missing.yaml"
cp "$child_a" "$aggregate_missing"
expect_fail 'aggregate missing resource' assert_child_aggregate_equivalence \
"$aggregate_missing" "kps:$child_a" "loki:$child_b"
aggregate_drift="$fixture_dir/aggregate-drift.yaml"
sed 's/value: one/value: changed/' "$aggregate" >"$aggregate_drift"
expect_fail 'aggregate canonical hash drift' assert_child_aggregate_equivalence \
"$aggregate_drift" "kps:$child_a" "loki:$child_b"
duplicate_child="$fixture_dir/duplicate-child.yaml"
cp "$child_a" "$duplicate_child"
expect_fail 'duplicate child tuple' assert_child_aggregate_equivalence \
"$aggregate" "kps:$child_a" "loki:$child_b" "tempo:$duplicate_child"
assert_task2_manifests() {
local manifest_root="${1:-$ROOT}"
python3 - "$manifest_root" <<'PY'
import pathlib
import sys
import yaml
root = pathlib.Path(sys.argv[1])
namespace_root = root / "infrastructure/namespaces/observability"
storage_root = root / "infrastructure/storage/observability-local-pv"
def load_documents(path):
with path.open(encoding="utf-8") as stream:
return [value for value in yaml.safe_load_all(stream) if value is not None]
namespace_kustomization = yaml.safe_load(
(namespace_root / "kustomization.yaml").read_text(encoding="utf-8")
)
assert namespace_kustomization["resources"] == [
"observability.yaml",
"observability-agent.yaml",
]
namespaces = []
for resource in namespace_kustomization["resources"]:
namespaces.extend(load_documents(namespace_root / resource))
assert len(namespaces) == 2
assert [item["kind"] for item in namespaces].count("Namespace") == 2
assert len({item["metadata"]["name"] for item in namespaces}) == 2
by_name = {item["metadata"]["name"]: item for item in namespaces}
assert set(by_name) == {"observability", "observability-agent"}
for item in by_name.values():
assert item["metadata"]["labels"]["observability.hyeonworks.com/enabled"] == "true"
for level in ("enforce", "audit", "warn"):
labels = by_name["observability"]["metadata"]["labels"]
assert labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
assert labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
agent_labels = by_name["observability-agent"]["metadata"]["labels"]
assert agent_labels["pod-security.kubernetes.io/enforce"] == "privileged"
assert agent_labels["pod-security.kubernetes.io/enforce-version"] == "latest"
for level in ("audit", "warn"):
assert agent_labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
assert agent_labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
assert (namespace_root / "discovery-namespaces.txt").read_text(encoding="utf-8") == (
"kube-system\ngitea\nkeycloak\nplatform-data\nobject-storage\n"
)
storage_kustomization = yaml.safe_load(
(storage_root / "kustomization.yaml").read_text(encoding="utf-8")
)
assert storage_kustomization["resources"] == [
"storage-classes.yaml",
"persistent-volumes.yaml",
]
storage_items = []
for resource in storage_kustomization["resources"]:
storage_items.extend(load_documents(storage_root / resource))
assert len(storage_items) == 12
assert [item["kind"] for item in storage_items].count("StorageClass") == 6
assert [item["kind"] for item in storage_items].count("PersistentVolume") == 6
assert {item["kind"] for item in storage_items} == {"StorageClass", "PersistentVolume"}
class_items = [item for item in storage_items if item["kind"] == "StorageClass"]
volume_items = [item for item in storage_items if item["kind"] == "PersistentVolume"]
assert len({item["metadata"]["name"] for item in class_items}) == 6
assert len({item["metadata"]["name"] for item in volume_items}) == 6
assert len({item["metadata"]["labels"]["app.kubernetes.io/component"] for item in volume_items}) == 6
classes = {
item["metadata"]["name"]: item
for item in class_items
}
volumes = {
item["metadata"]["labels"]["app.kubernetes.io/component"]: item
for item in volume_items
}
expected = {
"prometheus": ("/srv/k3s/ssd/observability/prometheus", "20Gi", "ssd-local-observability-prometheus-retain"),
"grafana": ("/srv/k3s/ssd/observability/grafana", "2Gi", "ssd-local-observability-grafana-retain"),
"alertmanager": ("/srv/k3s/ssd/observability/alertmanager", "1Gi", "ssd-local-observability-alertmanager-retain"),
"alloy": ("/srv/k3s/ssd/observability/alloy", "2Gi", "ssd-local-observability-alloy-retain"),
"loki": ("/srv/k3s/ssd/observability/loki", "5Gi", "ssd-local-observability-loki-retain"),
"tempo": ("/srv/k3s/ssd/observability/tempo", "5Gi", "ssd-local-observability-tempo-retain"),
}
assert set(classes) == {row[2] for row in expected.values()}
assert set(volumes) == set(expected)
for owner, (path, capacity, class_name) in expected.items():
storage_class = classes[class_name]
assert storage_class["provisioner"] == "kubernetes.io/no-provisioner"
assert storage_class["reclaimPolicy"] == "Retain"
assert storage_class["volumeBindingMode"] == "WaitForFirstConsumer"
assert storage_class["allowVolumeExpansion"] is False
volume = volumes[owner]
spec = volume["spec"]
assert spec["capacity"]["storage"] == capacity
assert spec["accessModes"] == ["ReadWriteOnce"]
assert spec["persistentVolumeReclaimPolicy"] == "Retain"
assert spec["storageClassName"] == class_name
assert spec["local"]["path"] == path
expression = spec["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"][0]
assert expression == {
"key": "kubernetes.io/hostname",
"operator": "In",
"values": ["donghyeon-system-product-name"],
}
actual_capacity = 0
for volume in volume_items:
value = volume["spec"]["capacity"]["storage"]
assert isinstance(value, str) and value.endswith("Gi") and value[:-2].isdigit()
actual_capacity += int(value[:-2])
assert actual_capacity == 35
PY
}
expect_pass 'Task 2 namespace and 35Gi Local PV matrix' assert_task2_manifests
expect_any_failure() {
local name="$1"
shift
if "$@" >"$fixture_dir/expected-any-failure.out" 2>&1; then
printf 'FAIL (expected any failure): %s\n' "$name" >&2
exit 1
fi
pass_count=$((pass_count + 1))
}
task2_manifest_fixture() {
local destination="$1"
mkdir -p "$destination/infrastructure/namespaces" "$destination/infrastructure/storage"
cp -R "$ROOT/infrastructure/namespaces/observability" \
"$destination/infrastructure/namespaces/observability"
cp -R "$ROOT/infrastructure/storage/observability-local-pv" \
"$destination/infrastructure/storage/observability-local-pv"
}
duplicate_manifest_root="$fixture_dir/task2-manifest-duplicate"
task2_manifest_fixture "$duplicate_manifest_root"
{
printf '%s\n' '---'
sed -n '1,/^---$/p' \
"$ROOT/infrastructure/storage/observability-local-pv/storage-classes.yaml" | sed '$d'
} >>"$duplicate_manifest_root/infrastructure/storage/observability-local-pv/storage-classes.yaml"
expect_any_failure 'duplicate StorageClass object is rejected' \
assert_task2_manifests "$duplicate_manifest_root"
extra_kind_root="$fixture_dir/task2-manifest-extra-kind"
task2_manifest_fixture "$extra_kind_root"
write_file "$extra_kind_root/infrastructure/storage/observability-local-pv/extra.yaml" \
'apiVersion: v1' 'kind: ConfigMap' 'metadata:' ' name: unexpected'
printf '%s\n' ' - extra.yaml' >> \
"$extra_kind_root/infrastructure/storage/observability-local-pv/kustomization.yaml"
expect_any_failure 'extra manifest kind is rejected' \
assert_task2_manifests "$extra_kind_root"
capacity_drift_root="$fixture_dir/task2-manifest-capacity-drift"
task2_manifest_fixture "$capacity_drift_root"
sed -i '0,/storage: 20Gi/s//storage: 19Gi/' \
"$capacity_drift_root/infrastructure/storage/observability-local-pv/persistent-volumes.yaml"
expect_any_failure 'actual rendered capacity drift is rejected' \
assert_task2_manifests "$capacity_drift_root"
readonly PREPARE_OBSERVABILITY_PATHS="$ROOT/scripts/bootstrap/prepare-observability-local-paths.sh"
source_options_before="$-"
# shellcheck source=/dev/null
source "$PREPARE_OBSERVABILITY_PATHS"
expect_pass 'path preparation script is source-safe' test "$-" = "$source_options_before"
olp_fixture="$fixture_dir/observability-local-paths"
olp_root="$olp_fixture/root"
olp_ssd="$olp_fixture/srv/k3s/ssd"
olp_aistor="$olp_fixture/srv/k3s/aistor"
olp_external="$olp_fixture/external"
mkdir -p "$olp_root" "$olp_ssd/observability" "$olp_aistor" "$olp_external"
olp_available='64424509440'
olp_uses=('0' '0' '0' '0' '0' '0')
olp_context='default'
olp_node='donghyeon-system-product-name'
olp_confirmation='APPLY default'
olp_helper_audit="$olp_fixture/helper-audit.log"
olp_helper_race=''
olp_helper_signal=''
olp_helper_signal_target=''
_olp_root_facts() { printf '%s\t%s\n' '/dev/root' 'ext4'; }
_olp_available_bytes() { printf '%s\n' "$olp_available"; }
_olp_current_context() { printf '%s\n' "$olp_context"; }
_olp_node_names() { printf '%s\n' "$olp_node"; }
_olp_read_confirmation() { printf '%s\n' "$olp_confirmation"; }
_olp_fixture_privileged_helper() {
if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]] || \
[[ "$2" != "$olp_root" || "$3" != "$olp_ssd" || "$4" != "$olp_aistor" ]]; then
return 97
fi
local program
program="$(_olp_helper_program)" || return
OLP_HELPER_FIXTURE_ROOT="$olp_fixture" \
OLP_HELPER_FIXTURE_VIRTUAL_AISTOR=1 \
OLP_HELPER_FIXTURE_OWNERS=1 \
OLP_HELPER_FIXTURE_AUDIT="$olp_helper_audit" \
OLP_HELPER_FIXTURE_RACE="$olp_helper_race" \
OLP_HELPER_FIXTURE_EXTERNAL="$olp_external" \
OLP_HELPER_FIXTURE_SIGNAL="$olp_helper_signal" \
OLP_HELPER_FIXTURE_SIGNAL_TARGET="$olp_helper_signal_target" \
/usr/bin/python3 - "$@" <<<"$program"
}
_olp_privileged_helper() { _olp_fixture_privileged_helper "$@"; }
dry_output="$fixture_dir/observability-dry-run.out"
expect_stateful_pass 'default dry-run emits facts without mutation' \
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor"
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor" >"$dry_output"
expect_pass 'dry-run leaves every planned path absent' test -z \
"$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)"
expect_pass 'dry-run prints sanitized root filesystem facts' test "$(sed -n '1,3p' "$dry_output")" = \
$'Root SSD device: /dev/root\nRoot SSD filesystem: ext4\nRoot SSD available bytes: 64424509440'
expect_pass 'dry-run prints all six exact planned fixture paths' test \
"$(sed -n '4,9p' "$dry_output")" = \
"$(for owner in prometheus grafana alertmanager alloy loki tempo; do
printf 'Planned path: %s/observability/%s\n' "$olp_ssd" "$owner"
done)"
assert_olp_execute_fails() {
local output rc
set +e
output="$(observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" 2>&1)"
rc=$?
set -e
[[ "$rc" != 0 && "$output" == ERROR:* ]]
}
assert_olp_execute_rejected() {
assert_olp_execute_fails
[[ -z "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)" ]]
}
exact_numeric_reply=$'PREFLIGHT\t53687091200\t1073741824\t0\t0\t0\t0\t0'
expect_pass '50GiB available and 1GiB aggregate are inclusive' \
_olp_validate_preflight_reply "$exact_numeric_reply"
expect_any_failure 'one byte below 50GiB is rejected' \
_olp_validate_preflight_reply $'PREFLIGHT\t53687091199\t0\t0\t0\t0\t0\t0'
expect_any_failure 'one target byte above 1GiB is rejected before summing' \
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t1073741825\t0\t0\t0\t0\t0'
expect_any_failure 'aggregate one byte above 1GiB is rejected' \
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t536870913\t536870912\t0\t0\t0\t0'
expect_any_failure 'signed decimal is rejected' \
_olp_validate_preflight_reply $'PREFLIGHT\t+53687091200\t0\t0\t0\t0\t0\t0'
expect_any_failure 'leading-zero decimal is rejected' \
_olp_validate_preflight_reply $'PREFLIGHT\t053687091200\t0\t0\t0\t0\t0\t0'
expect_any_failure 'multiline numeric protocol is rejected' \
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t0\t0\t0\t0\t0\t0\n0'
expect_any_failure '2^63 available value is rejected' \
_olp_validate_preflight_reply $'PREFLIGHT\t9223372036854775808\t0\t0\t0\t0\t0\t0'
expect_any_failure 'repeated 2^63 target values fail before mutation' \
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808'
write_file "$olp_external/command-marker" DO-NOT-EXECUTE
for unexpected in sudo install chown rm curl; do
set +e
_olp_fixture_privileged_helper "$unexpected" "$olp_root" "$olp_ssd" "$olp_aistor" \
>"$fixture_dir/unexpected-helper-$unexpected.out" 2>&1
unexpected_rc=$?
set -e
expect_pass "unexpected helper action $unexpected returns 97" test "$unexpected_rc" = 97
expect_pass "unexpected helper action $unexpected executes nothing" test \
"$(cat "$olp_external/command-marker")" = DO-NOT-EXECUTE
done
set +e
_olp_fixture_privileged_helper apply "$olp_root" "$olp_ssd" \
>"$fixture_dir/unexpected-helper-argc.out" 2>&1
unexpected_rc=$?
set -e
expect_pass 'unexpected helper argc returns 97' test "$unexpected_rc" = 97
olp_context='other'
expect_pass 'execute rejects non-default context' assert_olp_execute_rejected
olp_context='default'
olp_node='other-node'
expect_pass 'execute rejects any other node set' assert_olp_execute_rejected
olp_node='donghyeon-system-product-name'
olp_confirmation='apply default'
expect_pass 'execute rejects inexact TTY confirmation' assert_olp_execute_rejected
olp_confirmation='APPLY default'
olp_context_calls="$fixture_dir/observability-context-calls"
write_file "$olp_context_calls" 0
_olp_current_context() {
local calls
calls="$(cat "$olp_context_calls")"
calls=$((calls + 1))
printf '%s\n' "$calls" >"$olp_context_calls"
if (( calls == 1 )); then
printf '%s\n' default
else
printf '%s\n' changed-during-confirmation
fi
}
expect_pass 'execute rechecks context after TTY confirmation' assert_olp_execute_rejected
olp_context='default'
_olp_current_context() { printf '%s\n' "$olp_context"; }
mkdir "$olp_ssd/observability/prometheus"
write_file "$olp_ssd/observability/prometheus/existing" DO-NOT-CLOBBER
expect_pass 'execute rejects nonempty existing target' assert_olp_execute_fails
expect_pass 'nonempty rejection preserves content' test \
"$(cat "$olp_ssd/observability/prometheus/existing")" = DO-NOT-CLOBBER
rm -f -- "$olp_ssd/observability/prometheus/existing"
rmdir "$olp_ssd/observability/prometheus"
ln -s "$olp_aistor" "$olp_ssd/observability/prometheus"
expect_pass 'execute rejects symlink target' assert_olp_execute_fails
expect_pass 'symlink rejection preserves link' test -L "$olp_ssd/observability/prometheus"
unlink "$olp_ssd/observability/prometheus"
write_file "$olp_helper_audit" START
expect_stateful_pass 'execute prepares all six paths' \
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor"
expected_owner_audit="$(printf '%s\n' \
'set-owner-mode prometheus 1000 2000 0770' \
'set-owner-mode grafana 472 472 0770' \
'set-owner-mode alertmanager 1000 2000 0770' \
'set-owner-mode alloy 473 473 0770' \
'set-owner-mode loki 10001 10001 0770' \
'set-owner-mode tempo 10001 10001 0770')"
expect_pass 'descriptor helper uses exact owner group mode matrix' test \
"$(sed -n '/^set-owner-mode /p' "$olp_helper_audit")" = "$expected_owner_audit"
for owner in prometheus grafana alertmanager alloy loki tempo; do
expect_pass "execute creates $owner path" test -d "$olp_ssd/observability/$owner"
expect_pass "execute sets $owner mode" test "$(stat -c %a "$olp_ssd/observability/$owner")" = 770
rmdir "$olp_ssd/observability/$owner"
done
chmod 0751 "$olp_external"
write_file "$olp_external/race-marker" EXTERNAL-MUST-SURVIVE
mkdir -m 0755 "$olp_ssd/observability/prometheus"
olp_helper_race='ancestor'
set +e
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
>"$fixture_dir/observability-ancestor-race.out" 2>&1
race_rc=$?
set -e
expect_pass 'ancestor replacement race is rejected' test "$race_rc" != 0
expect_pass 'ancestor replacement leaves external marker untouched' test \
"$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
expect_pass 'ancestor replacement leaves external metadata untouched' test \
"$(stat -c %a "$olp_external")" = 751
expect_pass 'ancestor replacement restores pinned pre-existing metadata' test \
"$(stat -c %a "$olp_ssd/observability.pinned/prometheus")" = 755
unlink "$olp_ssd/observability"
mv "$olp_ssd/observability.pinned" "$olp_ssd/observability"
olp_helper_race='target:prometheus'
set +e
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
>"$fixture_dir/observability-target-race.out" 2>&1
race_rc=$?
set -e
expect_pass 'target replacement race is rejected' test "$race_rc" != 0
expect_pass 'target replacement leaves external marker untouched' test \
"$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
expect_pass 'target replacement leaves external metadata untouched' test \
"$(stat -c %a "$olp_external")" = 751
expect_pass 'target replacement restores pinned target metadata' test \
"$(stat -c %a "$olp_ssd/observability/prometheus.pinned")" = 755
unlink "$olp_ssd/observability/prometheus"
mv "$olp_ssd/observability/prometheus.pinned" "$olp_ssd/observability/prometheus"
rmdir "$olp_ssd/observability/prometheus"
for signal_case in HUP:prometheus INT:alloy TERM:alloy; do
olp_helper_race=''
olp_helper_signal="${signal_case%%:*}"
olp_helper_signal_target="${signal_case#*:}"
write_file "$olp_helper_audit" START
if [[ "$olp_helper_signal_target" == prometheus ]]; then
rmdir "$olp_ssd/observability"
else
mkdir -m 0755 "$olp_ssd/observability/prometheus"
fi
set +e
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
>"$fixture_dir/observability-signal-${olp_helper_signal}.out" 2>&1
signal_rc=$?
set -e
expect_pass "$olp_helper_signal post-create signal is rejected" test "$signal_rc" != 0
for owner in grafana alertmanager alloy loki tempo; do
expect_pass "$olp_helper_signal leaves no created $owner residue" test \
! -e "$olp_ssd/observability/$owner"
done
if [[ "$olp_helper_signal_target" == prometheus ]]; then
expect_pass "$olp_helper_signal leaves no first-target residue" test \
! -e "$olp_ssd/observability/prometheus"
expect_pass "$olp_helper_signal leaves no created parent residue" test \
! -e "$olp_ssd/observability"
mkdir "$olp_ssd/observability"
else
expect_pass "$olp_helper_signal restores pre-existing mode" test \
"$(stat -c %a "$olp_ssd/observability/prometheus")" = 755
expected_restore="$(printf 'restore-chown prometheus %s %s\nrestore-chmod prometheus 0755' \
"$(id -u)" "$(id -g)")"
expect_pass "$olp_helper_signal restores chown then chmod in exact order" test \
"$(tail -n 2 "$olp_helper_audit")" = "$expected_restore"
rmdir "$olp_ssd/observability/prometheus"
fi
done
olp_helper_signal=''
olp_helper_signal_target=''
printf 'Assertions: %s\n' "$pass_count"
printf 'OBSERVABILITY CORE RENDER ASSERTION TEST PASS\n'