render-observability-core.sh keeps its own exact copy of the AlertmanagerConfig, and test-render-observability-core.sh feeds it the same copy as a fixture.11c13bbadded the platform-heartbeat route and receiver to the source and to test-observability-alerting.sh but not to these two, so the pinned renderer rejected the source with "platform AlertmanagerConfig is not the exact global Slack contract". Add the route and receiver to both copies and name the heartbeat in the reject message. Verification: render-observability-core.sh (Helm v3.19.4, kustomize v5.8.1) prints OBSERVABILITY CORE STATIC RENDER PASS, also with --verified-output-dir; test-render-observability-core.sh passes. Fixes:11c13bb87b("feat(observability): alert when Prometheus itself stops working")
1094 lines
59 KiB
Bash
Executable File
1094 lines
59 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly RENDERER="$ROOT/scripts/validate/render-observability-core.sh"
|
|
readonly EXPECTED_REJECTION_STATUS=23
|
|
|
|
if [[ ! -f "$RENDERER" ]]; then
|
|
printf 'EXPECTED RED: renderer is missing: %s\n' "$RENDERER" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Sourcing must define assertions without running the renderer or changing shell options.
|
|
source "$RENDERER"
|
|
|
|
fixture_dir="$(mktemp -d /tmp/platform-observability-render-test.XXXXXX)"
|
|
handoff_dirs=()
|
|
cleanup() {
|
|
local rc=$?
|
|
local directory
|
|
for directory in "${handoff_dirs[@]}"; do
|
|
case "$directory" in
|
|
/tmp/platform-observability-core-apply.*) rm -rf -- "$directory" ;;
|
|
esac
|
|
done
|
|
case "$fixture_dir" in
|
|
/tmp/platform-observability-render-test.*) rm -rf -- "$fixture_dir" ;;
|
|
esac
|
|
exit "$rc"
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
pass_count=0
|
|
|
|
expect_pass() {
|
|
local name="$1"
|
|
local output
|
|
shift
|
|
if ! output="$({ "$@"; } 2>&1)"; then
|
|
printf 'FAIL (expected pass): %s\n%s\n' "$name" "$output" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
expect_stateful_pass() {
|
|
local name="$1"
|
|
local output_file="$fixture_dir/stateful-output"
|
|
local rc
|
|
shift
|
|
set +e
|
|
"$@" >"$output_file" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ "$rc" != 0 ]]; then
|
|
printf 'FAIL (expected stateful pass): %s\n' "$name" >&2
|
|
sed -n '1,20p' "$output_file" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
expect_fail() {
|
|
local name="$1"
|
|
local output rc
|
|
shift
|
|
set +e
|
|
output="$({ "$@"; } 2>&1)"
|
|
rc=$?
|
|
set -e
|
|
if [[ "$rc" != "$EXPECTED_REJECTION_STATUS" ]]; then
|
|
printf 'FAIL (expected rejection status %s, got %s): %s\n%s\n' \
|
|
"$EXPECTED_REJECTION_STATUS" "$rc" "$name" "$output" >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$output" != REJECT:* ]]; then
|
|
printf 'FAIL (expected pinned rejection message): %s\n%s\n' "$name" "$output" >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$output" == *'DO-NOT-LEAK-FIXTURE'* ]]; then
|
|
printf 'FAIL (credential leaked): %s\n' "$name" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
write_file() {
|
|
local path="$1"
|
|
shift
|
|
printf '%s\n' "$@" >"$path"
|
|
}
|
|
|
|
assert_chart_pin_row() {
|
|
local expected="$*"
|
|
local label="$1"
|
|
local actual
|
|
actual="$(chart_pin_rows | awk -F '\t' -v label="$label" '$1 == label { print $1, $2, $3, $4, $5, $6, $7 }')"
|
|
[[ "$actual" == "$expected" ]]
|
|
}
|
|
|
|
assert_image_pin_row() {
|
|
local expected="$*"
|
|
local image="$1"
|
|
local actual
|
|
actual="$(image_pin_rows | awk -F '\t' -v image="$image" '$1 == image { print $1, $2 }')"
|
|
[[ "$actual" == "$expected" ]]
|
|
}
|
|
|
|
expect_pass 'Helm version pin' test "$(observability_expected_helm_version)" = v3.19.4
|
|
expect_pass 'Kustomize version pin' test "$(observability_expected_kustomize_version)" = v5.8.1
|
|
expect_pass 'Kubernetes render target pin' test "$(observability_helm_kube_version)" = 1.36.2
|
|
expected_render_roots=$'infrastructure/namespaces/observability\n'
|
|
expected_render_roots+=$'infrastructure/controllers/prometheus-operator-crds\n'
|
|
expected_render_roots+=$'infrastructure/storage/observability-local-pv\n'
|
|
expected_render_roots+=$'services/observability/kube-prometheus-stack\n'
|
|
expected_render_roots+=$'services/observability/loki\n'
|
|
expected_render_roots+=$'services/observability/tempo\n'
|
|
expected_render_roots+=$'services/observability/alloy\n'
|
|
expected_render_roots+=$'services/observability/node-exporter\n'
|
|
expected_render_roots+=$'services/observability/grafana\n'
|
|
expected_render_roots+=$'services/observability/blackbox-exporter\n'
|
|
expected_render_roots+=$'services/observability/network-policies\n'
|
|
expected_render_roots+=$'services/observability/targets\n'
|
|
expected_render_roots+=$'services/observability/rules/platform\n'
|
|
expected_render_roots+=$'services/observability/alerting\n'
|
|
expected_render_roots+=$'services/observability/dashboards/platform\n'
|
|
expected_render_roots+='services/observability'
|
|
expect_pass 'complete observability render root table' test "$(render_root_rows)" = "$expected_render_roots"
|
|
expected_chart_table=$'prometheus-operator-crds\thttps://prometheus-community.github.io/helm-charts\tprometheus-operator-crds\t31.0.0\tv0.93.0\t5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd\tinfrastructure/controllers/prometheus-operator-crds\n'
|
|
expected_chart_table+=$'kube-prometheus-stack\thttps://prometheus-community.github.io/helm-charts\tkube-prometheus-stack\t88.0.1\tv0.93.0\t0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117\tservices/observability/kube-prometheus-stack\n'
|
|
expected_chart_table+=$'loki\thttps://grafana-community.github.io/helm-charts\tloki\t18.7.1\t3.7.4\tde9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d\tservices/observability/loki\n'
|
|
expected_chart_table+=$'tempo\thttps://grafana-community.github.io/helm-charts\ttempo\t2.2.3\t2.10.7\t14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe\tservices/observability/tempo\n'
|
|
expected_chart_table+=$'alloy\thttps://grafana.github.io/helm-charts\talloy\t1.11.0\tv1.18.0\t11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd\tservices/observability/alloy\n'
|
|
expected_chart_table+=$'prometheus-node-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-node-exporter\t4.56.1\t1.12.1\t70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a\tservices/observability/node-exporter'
|
|
expected_chart_table+=$'\ngrafana\thttps://grafana-community.github.io/helm-charts\tgrafana\t12.10.1\t13.1.1\tff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8\tservices/observability/grafana'
|
|
expected_chart_table+=$'\nprometheus-blackbox-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-blackbox-exporter\t11.16.0\tv0.28.0\t932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b\tservices/observability/blackbox-exporter'
|
|
expect_pass 'exact eight-row chart table with no extras' test "$(chart_pin_rows)" = "$expected_chart_table"
|
|
|
|
expected_image_table=$'quay.io/prometheus-operator/prometheus-operator:v0.93.0\tsha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2\n'
|
|
expected_image_table+=$'quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0\tsha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905\n'
|
|
expected_image_table+=$'quay.io/prometheus/prometheus:v3.13.2-distroless\tsha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b\n'
|
|
expected_image_table+=$'quay.io/prometheus/alertmanager:v0.33.1\tsha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b\n'
|
|
expected_image_table+=$'registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1\tsha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4\n'
|
|
expected_image_table+=$'quay.io/prometheus/node-exporter:v1.12.1\tsha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798\n'
|
|
expected_image_table+=$'docker.io/grafana/loki:3.7.4\tsha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e\n'
|
|
expected_image_table+=$'docker.io/grafana/tempo:2.10.7\tsha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c\n'
|
|
expected_image_table+=$'docker.io/grafana/alloy:v1.18.0\tsha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17\n'
|
|
expected_image_table+=$'ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0\tsha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522\n'
|
|
expected_image_table+=$'docker.io/library/busybox:1.37.0\tsha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0\n'
|
|
expected_image_table+=$'quay.io/minio/aistor/mc\tsha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b\n'
|
|
expected_image_table+=$'docker.io/grafana/grafana:13.1.1\tsha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b\n'
|
|
expected_image_table+=$'quay.io/kiwigrid/k8s-sidecar:2.10.0\tsha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3\n'
|
|
expected_image_table+=$'quay.io/prometheus/blackbox-exporter:v0.28.0\tsha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68'
|
|
expect_pass 'exact fifteen-row image table with no extras' test "$(image_pin_rows)" = "$expected_image_table"
|
|
expect_pass 'Prometheus CRD chart pin' assert_chart_pin_row prometheus-operator-crds https://prometheus-community.github.io/helm-charts prometheus-operator-crds 31.0.0 v0.93.0 5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd infrastructure/controllers/prometheus-operator-crds
|
|
expect_pass 'KPS chart pin' assert_chart_pin_row kube-prometheus-stack https://prometheus-community.github.io/helm-charts kube-prometheus-stack 88.0.1 v0.93.0 0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117 services/observability/kube-prometheus-stack
|
|
expect_pass 'Loki chart pin' assert_chart_pin_row loki https://grafana-community.github.io/helm-charts loki 18.7.1 3.7.4 de9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d services/observability/loki
|
|
expect_pass 'Tempo chart pin' assert_chart_pin_row tempo https://grafana-community.github.io/helm-charts tempo 2.2.3 2.10.7 14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe services/observability/tempo
|
|
expect_pass 'Alloy chart pin' assert_chart_pin_row alloy https://grafana.github.io/helm-charts alloy 1.11.0 v1.18.0 11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd services/observability/alloy
|
|
expect_pass 'node-exporter chart pin' assert_chart_pin_row prometheus-node-exporter https://prometheus-community.github.io/helm-charts prometheus-node-exporter 4.56.1 1.12.1 70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a services/observability/node-exporter
|
|
expect_pass 'Grafana chart pin' assert_chart_pin_row grafana https://grafana-community.github.io/helm-charts grafana 12.10.1 13.1.1 ff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8 services/observability/grafana
|
|
expect_pass 'Blackbox chart pin' assert_chart_pin_row prometheus-blackbox-exporter https://prometheus-community.github.io/helm-charts prometheus-blackbox-exporter 11.16.0 v0.28.0 932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b services/observability/blackbox-exporter
|
|
|
|
expect_pass 'operator image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-operator:v0.93.0 sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
|
|
expect_pass 'reloader image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0 sha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
|
|
expect_pass 'Prometheus image pin' assert_image_pin_row quay.io/prometheus/prometheus:v3.13.2-distroless sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
|
|
expect_pass 'Alertmanager image pin' assert_image_pin_row quay.io/prometheus/alertmanager:v0.33.1 sha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
|
|
expect_pass 'KSM image pin' assert_image_pin_row registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
|
|
expect_pass 'node-exporter image pin' assert_image_pin_row quay.io/prometheus/node-exporter:v1.12.1 sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798
|
|
expect_pass 'Loki image pin' assert_image_pin_row docker.io/grafana/loki:3.7.4 sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e
|
|
expect_pass 'Tempo image pin' assert_image_pin_row docker.io/grafana/tempo:2.10.7 sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c
|
|
expect_pass 'Alloy image pin' assert_image_pin_row docker.io/grafana/alloy:v1.18.0 sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
|
|
expect_pass 'telemetrygen image pin' assert_image_pin_row ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0 sha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522
|
|
expect_pass 'BusyBox image pin' assert_image_pin_row docker.io/library/busybox:1.37.0 sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0
|
|
expect_pass 'AIStor mc image pin' assert_image_pin_row quay.io/minio/aistor/mc sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b
|
|
expect_pass 'Grafana image pin' assert_image_pin_row docker.io/grafana/grafana:13.1.1 sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b
|
|
expect_pass 'Grafana sidecar image pin' assert_image_pin_row quay.io/kiwigrid/k8s-sidecar:2.10.0 sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3
|
|
expect_pass 'Blackbox image pin' assert_image_pin_row quay.io/prometheus/blackbox-exporter:v0.28.0 sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68
|
|
expect_pass 'tagged Grafana digest is in the rendered-image allowlist' grep -Fqx \
|
|
docker.io/grafana/grafana:13.1.1@sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b \
|
|
<(_observability_image_allowlist)
|
|
expect_pass 'tagged Blackbox digest is in the rendered-image allowlist' grep -Fqx \
|
|
quay.io/prometheus/blackbox-exporter:v0.28.0@sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 \
|
|
<(_observability_image_allowlist)
|
|
|
|
alertmanager_fixture="$fixture_dir/alertmanager.yaml"
|
|
write_file "$alertmanager_fixture" \
|
|
'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: Alertmanager' \
|
|
'metadata:' \
|
|
' name: observability-core-kube-pr-alertmanager' \
|
|
' namespace: observability' \
|
|
'spec:' \
|
|
' alertmanagerConfiguration:' \
|
|
' name: platform-alertmanager'
|
|
alerting_fixture="$fixture_dir/alerting.yaml"
|
|
write_file "$alerting_fixture" '{"apiVersion":"monitoring.coreos.com/v1alpha1","kind":"AlertmanagerConfig","metadata":{"name":"platform-alertmanager","namespace":"observability","labels":{"observability.hyeonworks.com/instance":"home"}},"spec":{"route":{"receiver":"platform-slack","groupBy":["cluster","namespace","alertname","severity"],"groupWait":"30s","groupInterval":"5m","repeatInterval":"4h","routes":[{"receiver":"platform-null","matchers":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}]},{"receiver":"platform-heartbeat","matchers":[{"name":"alertname","matchType":"=","value":"PlatformHeartbeat"}],"groupWait":"0s","groupInterval":"1m","repeatInterval":"5m"}]},"inhibitRules":[{"sourceMatch":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}],"targetMatch":[{"name":"severity","matchType":"=","value":"info"}],"equal":["namespace"]}],"receivers":[{"name":"platform-null"},{"name":"platform-heartbeat","webhookConfigs":[{"urlSecret":{"name":"alertmanager-heartbeat","key":"url"},"sendResolved":false,"maxAlerts":1}]},{"name":"platform-slack","slackConfigs":[{"apiURL":{"name":"alertmanager-slack-webhook","key":"url"},"sendResolved":true,"linkNames":false,"mrkdwnIn":["text","fields"],"fallback":"{{ if eq .Status \"firing\" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","title":"{{ if eq .Status \"firing\" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","titleLink":"https://grafana.learn.hyeonworks.com/","color":"{{ if eq .Status \"resolved\" }}good{{ else if or (eq .CommonLabels.severity \"emergency\") (eq .CommonLabels.severity \"critical\") }}danger{{ else if eq .CommonLabels.severity \"warning\" }}warning{{ else }}#439FE0{{ end }}","fields":[{"title":"Status","value":"{{ .Status | toUpper }}","short":true},{"title":"Severity","value":"{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}","short":true},{"title":"Location","value":"{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}","short":true},{"title":"Alert count","value":"{{ len .Alerts }}","short":true}],"text":"{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>","footer":"hyeonworks observability · Alertmanager"}]}]}}' \
|
|
'---' \
|
|
'apiVersion: networking.k8s.io/v1' \
|
|
'kind: NetworkPolicy' \
|
|
'metadata:' \
|
|
' name: observability-allow-alertmanager-public-https' \
|
|
' namespace: observability' \
|
|
'spec:' \
|
|
' podSelector:' \
|
|
' matchLabels:' \
|
|
' app.kubernetes.io/name: alertmanager' \
|
|
' app.kubernetes.io/instance: observability-core-kube-pr-alertmanager' \
|
|
' policyTypes: [Egress]' \
|
|
' egress:' \
|
|
' - to:' \
|
|
' - ipBlock:' \
|
|
' cidr: 0.0.0.0/0' \
|
|
' except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]' \
|
|
' ports:' \
|
|
' - protocol: TCP' \
|
|
' port: 443'
|
|
expect_pass 'exact structured Alertmanager routing contract' \
|
|
assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_fixture"
|
|
|
|
alertmanager_selector_fixture="$fixture_dir/alertmanager-selector.yaml"
|
|
sed '/^spec:$/a\ alertmanagerConfigSelector: {}' \
|
|
"$alertmanager_fixture" >"$alertmanager_selector_fixture"
|
|
expect_fail 'Alertmanager discovery selector bypasses the global configuration' \
|
|
assert_alertmanager_routing_contract "$alertmanager_selector_fixture" "$alerting_fixture"
|
|
|
|
alerting_route_fixture="$fixture_dir/alerting-route-drift.yaml"
|
|
sed 's/"repeatInterval":"4h"/"repeatInterval":"12h"/' \
|
|
"$alerting_fixture" >"$alerting_route_fixture"
|
|
expect_fail 'Alertmanager Slack route drift' \
|
|
assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_route_fixture"
|
|
|
|
write_pod() {
|
|
local path="$1"
|
|
local namespace="$2"
|
|
local name="$3"
|
|
local pod_fields="$4"
|
|
local container_fields="${5:-}"
|
|
cat >"$path" <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $name
|
|
namespace: $namespace
|
|
spec:
|
|
automountServiceAccountToken: false
|
|
$pod_fields
|
|
containers:
|
|
- name: main
|
|
image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
|
|
$container_fields
|
|
EOF
|
|
}
|
|
|
|
valid_image="$fixture_dir/valid-image.yaml"
|
|
write_file "$valid_image" \
|
|
'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: pinned' \
|
|
' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \
|
|
' containers:' ' - name: prometheus' \
|
|
' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
|
|
expect_pass 'pinned image accepted' assert_pinned_images "$valid_image"
|
|
expect_pass 'credential assertion positive control' assert_no_credentials "$valid_image"
|
|
expect_pass 'observability namespace security positive control' assert_namespace_pod_security "$valid_image"
|
|
|
|
tag_image="$fixture_dir/tag-image.yaml"
|
|
sed 's|@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b|:v3.13.2-distroless|' \
|
|
"$valid_image" >"$tag_image"
|
|
expect_fail 'tag-only image' assert_pinned_images "$tag_image"
|
|
|
|
wrong_digest="$fixture_dir/wrong-digest.yaml"
|
|
sed 's/ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' \
|
|
"$valid_image" >"$wrong_digest"
|
|
expect_fail 'unpinned digest' assert_pinned_images "$wrong_digest"
|
|
|
|
secret="$fixture_dir/secret.yaml"
|
|
write_file "$secret" \
|
|
'apiVersion: v1' 'kind: Secret' 'metadata:' ' name: forbidden' \
|
|
' namespace: observability' 'stringData:' ' password: DO-NOT-LEAK-FIXTURE'
|
|
expect_fail 'credential Secret' assert_no_credentials "$secret"
|
|
|
|
literal_credential="$fixture_dir/literal-credential.yaml"
|
|
write_file "$literal_credential" \
|
|
'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: forbidden' \
|
|
' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \
|
|
' containers:' ' - name: app' ' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' \
|
|
' env:' ' - name: ACCESS_KEY' ' value: DO-NOT-LEAK-FIXTURE'
|
|
expect_fail 'literal credential payload' assert_no_credentials "$literal_credential"
|
|
|
|
for field in \
|
|
' privileged: true' \
|
|
' hostNetwork: true' \
|
|
' hostPID: true' \
|
|
' hostIPC: true'; do
|
|
slug="$(printf '%s' "$field" | tr -cd '[:alpha:]' | tr '[:upper:]' '[:lower:]')"
|
|
pod="$fixture_dir/observability-$slug.yaml"
|
|
if [[ "$field" == *privileged* ]]; then
|
|
container_security="$(printf ' securityContext:\n privileged: true')"
|
|
write_pod "$pod" observability core '' "$container_security"
|
|
else
|
|
write_pod "$pod" observability core "$field" ''
|
|
fi
|
|
expect_fail "observability forbids $slug" assert_namespace_pod_security "$pod"
|
|
done
|
|
|
|
obs_hostpath="$fixture_dir/observability-hostpath.yaml"
|
|
write_pod "$obs_hostpath" observability core \
|
|
$' volumes:\n - name: host\n hostPath:\n path: /var/log/pods' \
|
|
$' volumeMounts:\n - name: host\n mountPath: /logs\n readOnly: true'
|
|
expect_fail 'observability hostPath' assert_namespace_pod_security "$obs_hostpath"
|
|
|
|
controller_namespace="$fixture_dir/controller-namespace.yaml"
|
|
write_file "$controller_namespace" 'apiVersion: apps/v1' 'kind: Deployment' \
|
|
'metadata:' ' name: controller-security' ' namespace: observability' 'spec:' \
|
|
' selector:' ' matchLabels:' ' app: controller-security' ' template:' \
|
|
' metadata:' ' labels:' ' app: controller-security' ' spec:' \
|
|
' automountServiceAccountToken: false' ' hostPID: true' ' containers:' \
|
|
' - name: app' \
|
|
' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
|
|
expect_fail 'controller template inherits namespace security' assert_namespace_pod_security "$controller_namespace"
|
|
controller_valid="$fixture_dir/controller-valid.yaml"
|
|
sed '/hostPID: true/d' "$controller_namespace" >"$controller_valid"
|
|
expect_pass 'controller observability namespace positive control' \
|
|
assert_namespace_pod_security "$controller_valid"
|
|
controller_missing_namespace="$fixture_dir/controller-missing-namespace.yaml"
|
|
sed '/namespace: observability/d' "$controller_valid" >"$controller_missing_namespace"
|
|
expect_fail 'controller namespace missing' \
|
|
assert_namespace_pod_security "$controller_missing_namespace"
|
|
controller_foreign_namespace="$fixture_dir/controller-foreign-namespace.yaml"
|
|
sed 's/namespace: observability/namespace: default/' \
|
|
"$controller_valid" >"$controller_foreign_namespace"
|
|
expect_fail 'controller namespace foreign' \
|
|
assert_namespace_pod_security "$controller_foreign_namespace"
|
|
|
|
agent_allowed="$fixture_dir/agent-allowed.yaml"
|
|
write_pod "$agent_allowed" observability-agent alloy \
|
|
$' volumes:\n - name: pod-logs\n hostPath:\n path: /var/log/pods' \
|
|
$' volumeMounts:\n - name: pod-logs\n mountPath: /var/log/pods\n readOnly: true'
|
|
expect_pass 'Alloy read-only allowlisted hostPath' assert_namespace_pod_security "$agent_allowed"
|
|
|
|
missing_namespace="$fixture_dir/missing-namespace.yaml"
|
|
sed '/namespace: observability-agent/d' "$agent_allowed" >"$missing_namespace"
|
|
expect_fail 'Pod namespace missing' assert_namespace_pod_security "$missing_namespace"
|
|
|
|
foreign_namespace="$fixture_dir/foreign-namespace.yaml"
|
|
sed 's/namespace: observability-agent/namespace: default/' "$agent_allowed" >"$foreign_namespace"
|
|
expect_fail 'Pod namespace foreign' assert_namespace_pod_security "$foreign_namespace"
|
|
|
|
agent_bad_path="$fixture_dir/agent-bad-path.yaml"
|
|
sed 's|/var/log/pods|/etc|g' "$agent_allowed" >"$agent_bad_path"
|
|
expect_fail 'agent hostPath allowlist' assert_namespace_pod_security "$agent_bad_path"
|
|
|
|
agent_rw="$fixture_dir/agent-rw.yaml"
|
|
sed 's/readOnly: true/readOnly: false/' "$agent_allowed" >"$agent_rw"
|
|
expect_fail 'agent hostPath readOnly' assert_namespace_pod_security "$agent_rw"
|
|
|
|
automount_missing="$fixture_dir/automount-missing.yaml"
|
|
sed '/automountServiceAccountToken:/d' "$valid_image" >"$automount_missing"
|
|
expect_fail 'automount missing' assert_namespace_pod_security "$automount_missing"
|
|
|
|
automount_true="$fixture_dir/automount-true.yaml"
|
|
sed 's/automountServiceAccountToken: false/automountServiceAccountToken: true/' \
|
|
"$valid_image" >"$automount_true"
|
|
expect_fail 'automount true' assert_namespace_pod_security "$automount_true"
|
|
|
|
for exposure in ingress NodePort LoadBalancer; do
|
|
public="$fixture_dir/public-$exposure.yaml"
|
|
if [[ "$exposure" == ingress ]]; then
|
|
write_file "$public" 'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
|
|
'metadata:' ' name: public' ' namespace: observability' 'spec: {}'
|
|
else
|
|
write_file "$public" 'apiVersion: v1' 'kind: Service' 'metadata:' \
|
|
' name: public' ' namespace: observability' 'spec:' " type: $exposure" \
|
|
' ports:' ' - port: 80'
|
|
fi
|
|
expect_fail "public $exposure" assert_no_public_exposure "$public"
|
|
done
|
|
grafana_ingress="$fixture_dir/grafana-private-ingress.yaml"
|
|
write_file "$grafana_ingress" \
|
|
'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
|
|
'metadata:' ' name: grafana' ' namespace: observability' \
|
|
'spec:' ' ingressClassName: traefik' ' rules:' \
|
|
' - host: grafana.learn.hyeonworks.com' ' http:' ' paths:' \
|
|
' - path: /' ' pathType: Prefix' ' backend:' \
|
|
' service:' ' name: grafana' ' port:' \
|
|
' number: 80'
|
|
expect_pass 'exact Host-Nginx-fronted Grafana ingress' \
|
|
assert_no_public_exposure "$grafana_ingress"
|
|
grafana_public_ingress="$fixture_dir/grafana-public-ingress.yaml"
|
|
sed 's/grafana.learn.hyeonworks.com/public.example.com/' \
|
|
"$grafana_ingress" >"$grafana_public_ingress"
|
|
expect_fail 'Grafana ingress host drift' \
|
|
assert_no_public_exposure "$grafana_public_ingress"
|
|
cluster_ip="$fixture_dir/cluster-ip.yaml"
|
|
write_file "$cluster_ip" 'apiVersion: v1' 'kind: Service' 'metadata:' \
|
|
' name: internal' ' namespace: observability' 'spec:' ' type: ClusterIP' \
|
|
' ports:' ' - port: 9090'
|
|
expect_pass 'public exposure assertion positive control' assert_no_public_exposure "$cluster_ip"
|
|
|
|
for auxiliary in gateway canary chunks-cache results-cache test minio; do
|
|
loki_aux="$fixture_dir/loki-$auxiliary.yaml"
|
|
write_file "$loki_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
|
|
" name: loki-$auxiliary" ' namespace: observability' 'spec: {}'
|
|
expect_fail "Loki auxiliary $auxiliary" assert_no_loki_auxiliary "$loki_aux"
|
|
done
|
|
loki_main="$fixture_dir/loki-main.yaml"
|
|
write_file "$loki_main" 'apiVersion: apps/v1' 'kind: StatefulSet' 'metadata:' \
|
|
' name: loki' ' namespace: observability' 'spec: {}'
|
|
expect_pass 'Loki auxiliary assertion positive control' assert_no_loki_auxiliary "$loki_main"
|
|
|
|
for auxiliary in metrics-generator tempo-query; do
|
|
tempo_aux="$fixture_dir/tempo-$auxiliary.yaml"
|
|
write_file "$tempo_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
|
|
" name: $auxiliary" ' namespace: observability' 'spec: {}'
|
|
expect_fail "Tempo auxiliary $auxiliary" assert_tempo_topology "$tempo_aux"
|
|
done
|
|
|
|
tempo_http="$fixture_dir/tempo-http.yaml"
|
|
write_file "$tempo_http" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
|
|
' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \
|
|
' receivers:' ' otlp:' ' protocols:' ' grpc:' \
|
|
' endpoint: 0.0.0.0:4317' ' http:' \
|
|
' endpoint: 0.0.0.0:4318'
|
|
expect_fail 'Tempo OTLP HTTP receiver' assert_tempo_topology "$tempo_http"
|
|
tempo_grpc="$fixture_dir/tempo-grpc.yaml"
|
|
write_file "$tempo_grpc" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
|
|
' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \
|
|
' receivers:' ' otlp:' ' protocols:' ' grpc:' \
|
|
' endpoint: 0.0.0.0:4317'
|
|
expect_pass 'Tempo topology assertion positive control' assert_tempo_topology "$tempo_grpc"
|
|
|
|
selector_values="$fixture_dir/selector-values.yaml"
|
|
write_file "$selector_values" 'prometheus:' ' prometheusSpec:' \
|
|
' serviceMonitorSelectorNilUsesHelmValues: false' \
|
|
' podMonitorSelectorNilUsesHelmValues: false' \
|
|
' probeSelectorNilUsesHelmValues: false' \
|
|
' ruleSelectorNilUsesHelmValues: false'
|
|
expect_pass 'selector values false' assert_selector_values "$selector_values"
|
|
|
|
selector_true="$fixture_dir/selector-true.yaml"
|
|
sed '0,/false/s//true/' "$selector_values" >"$selector_true"
|
|
expect_fail 'nilUsesHelmValues true' assert_selector_values "$selector_true"
|
|
|
|
prometheus_missing="$fixture_dir/prometheus-missing-selector.yaml"
|
|
write_file "$prometheus_missing" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \
|
|
'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \
|
|
' probeSelector: {}'
|
|
expect_fail 'Prometheus selector missing' assert_prometheus_selectors "$prometheus_missing"
|
|
prometheus_valid="$fixture_dir/prometheus-valid.yaml"
|
|
write_file "$prometheus_valid" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \
|
|
'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \
|
|
' probeSelector: {}' ' ruleSelector: {}'
|
|
expect_pass 'Prometheus selector assertion positive control' assert_prometheus_selectors "$prometheus_valid"
|
|
|
|
crd="$fixture_dir/kps-crd.yaml"
|
|
write_file "$crd" 'apiVersion: apiextensions.k8s.io/v1' \
|
|
'kind: CustomResourceDefinition' 'metadata:' ' name: prometheuses.monitoring.coreos.com'
|
|
expect_fail 'CRD in workload render' assert_no_crds "$crd"
|
|
expect_pass 'CRD assertion positive control' assert_no_crds "$valid_image"
|
|
|
|
shared_pv="$fixture_dir/shared-pv.yaml"
|
|
write_file "$shared_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
|
|
' name: prometheus' 'spec:' ' storageClassName: shared-local' ' local:' \
|
|
' path: /srv/prometheus' '---' 'apiVersion: v1' 'kind: PersistentVolume' \
|
|
'metadata:' ' name: alertmanager' 'spec:' ' storageClassName: shared-local' \
|
|
' local:' ' path: /srv/alertmanager'
|
|
expect_fail 'shared Local PV StorageClass' assert_unique_local_pv_storage_classes "$shared_pv"
|
|
single_pv="$fixture_dir/single-pv.yaml"
|
|
write_file "$single_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
|
|
' name: prometheus' 'spec:' ' storageClassName: prometheus-local' ' local:' \
|
|
' path: /srv/prometheus'
|
|
expect_pass 'Local PV assertion positive control' assert_unique_local_pv_storage_classes "$single_pv"
|
|
|
|
for fake in spring kafka batch backup; do
|
|
fake_monitor="$fixture_dir/fake-$fake.yaml"
|
|
write_file "$fake_monitor" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: ServiceMonitor' 'metadata:' " name: $fake-monitor" \
|
|
' namespace: observability' 'spec: {}'
|
|
expect_fail "fake $fake monitor or rule" assert_no_unscoped_monitors_rules "$fake_monitor"
|
|
done
|
|
scoped_monitor="$fixture_dir/scoped-monitor.yaml"
|
|
write_file "$scoped_monitor" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: ServiceMonitor' 'metadata:' ' name: prometheus' \
|
|
' namespace: observability' 'spec: {}'
|
|
expect_pass 'monitor/rule assertion positive control' assert_no_unscoped_monitors_rules "$scoped_monitor"
|
|
|
|
chart="$fixture_dir/Chart.yaml"
|
|
write_file "$chart" 'apiVersion: v2' 'name: loki' 'version: 18.7.1' 'appVersion: 3.7.4'
|
|
expect_pass 'chart metadata pin accepted' validate_chart_metadata "$chart" loki 18.7.1 3.7.4
|
|
expect_fail 'chart metadata name mismatch' validate_chart_metadata "$chart" tempo 18.7.1 3.7.4
|
|
expect_fail 'chart metadata version mismatch' validate_chart_metadata "$chart" loki 18.7.2 3.7.4
|
|
expect_fail 'chart metadata appVersion mismatch' validate_chart_metadata "$chart" loki 18.7.1 3.7.5
|
|
|
|
handoff="$fixture_dir/platform-observability-core-apply.fixture"
|
|
mkdir -m 0700 "$handoff"
|
|
expect_fail 'handoff prefix restricted to /tmp' validate_handoff_directory "$handoff"
|
|
safe_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$safe_handoff")
|
|
chmod 0700 "$safe_handoff"
|
|
expect_pass 'safe empty handoff' validate_handoff_directory "$safe_handoff"
|
|
touch "$safe_handoff/unexpected"
|
|
expect_fail 'handoff must be empty' validate_handoff_directory "$safe_handoff"
|
|
rm -f -- "$safe_handoff/unexpected"
|
|
rmdir -- "$safe_handoff"
|
|
|
|
empty_suffix='/tmp/platform-observability-core-apply.'
|
|
expect_fail 'handoff suffix must be nonempty' validate_handoff_directory "$empty_suffix"
|
|
|
|
root_fixture="$fixture_dir/render-roots"
|
|
mkdir -p "$root_fixture/existing/child"
|
|
expect_pass 'physical render roots accepted' validate_render_roots "$root_fixture" existing/child
|
|
expect_fail 'missing render root rejected' validate_render_roots "$root_fixture" missing/child
|
|
expect_pass 'missing root preflight creates nothing' test ! -e "$root_fixture/missing"
|
|
mkdir -p "$root_fixture/physical/child"
|
|
ln -s physical "$root_fixture/symlinked"
|
|
expect_fail 'symlinked render root lineage rejected' validate_render_roots "$root_fixture" symlinked/child
|
|
|
|
cache_root="$fixture_dir/cache-root"
|
|
mkdir -p "$cache_root/services/component/.helm/charts"
|
|
_OBS_ROOT="$cache_root"
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
expect_stateful_pass 'exclusive cache directory creation' _observability_create_cache_directory \
|
|
services/component chart 1.0.0
|
|
expect_fail 'existing exact cache is no-clobber' _observability_create_cache_directory \
|
|
services/component chart 1.0.0
|
|
expect_stateful_pass 'tracked cache cleanup' _observability_cleanup_cache_dirs
|
|
expect_pass 'pre-existing .helm preserved' test -d "$cache_root/services/component/.helm"
|
|
expect_pass 'pre-existing charts preserved' test -d "$cache_root/services/component/.helm/charts"
|
|
expect_pass 'exact created cache removed' test ! -e "$cache_root/services/component/.helm/charts/chart-1.0.0"
|
|
mkdir -p "$cache_root/services/missing-parent"
|
|
expect_fail 'missing component root rejected before cache mkdir' _observability_create_cache_directory \
|
|
services/missing-parent/component chart 1.0.0
|
|
expect_pass 'missing component root leaves no .helm' test ! -e "$cache_root/services/missing-parent/component"
|
|
mkdir -p "$cache_root/services/new-component"
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
_OBS_CREATED_CACHE_IDENTITIES=()
|
|
_OBS_CREATED_CACHE_TYPES=()
|
|
expect_stateful_pass 'new cache parents tracked' _observability_create_cache_directory \
|
|
services/new-component chart 1.0.0
|
|
expect_stateful_pass 'new cache parents cleaned' _observability_cleanup_cache_dirs
|
|
expect_pass 'created .helm parent removed' test ! -e "$cache_root/services/new-component/.helm"
|
|
|
|
mkdir -p "$cache_root/services/race-component/.helm/charts"
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
_OBS_CREATED_CACHE_IDENTITIES=()
|
|
_OBS_CREATED_CACHE_TYPES=()
|
|
expect_stateful_pass 'race cache created' _observability_create_cache_directory \
|
|
services/race-component chart 1.0.0
|
|
race_cache="${_OBS_CREATED_CACHE_DIRS[0]}"
|
|
race_identity="${_OBS_CREATED_CACHE_IDENTITIES[0]}"
|
|
mkdir "$race_cache/owned-subdirectory"
|
|
write_file "$race_cache/owned-subdirectory/owned-marker" 'OWNED-CACHE-CONTENT'
|
|
cleanup_race_hook="$fixture_dir/cache-cleanup-race-hook.sh"
|
|
write_file "$cleanup_race_hook" '#!/usr/bin/env bash' 'set -Eeuo pipefail' \
|
|
'replacement="$1"' 'mkdir -m 0700 -- "$replacement"' \
|
|
'printf "%s\n" REPLACEMENT-MUST-SURVIVE >"$replacement/replacement-marker"'
|
|
chmod 0700 "$cleanup_race_hook"
|
|
expect_stateful_pass 'owned cache cleanup is identity-bound across replacement race' \
|
|
_observability_cleanup_owned_cache_directory \
|
|
"$race_cache" "$race_identity" tree "$cleanup_race_hook"
|
|
expect_pass 'replacement directory survives cleanup race' test -d "$race_cache"
|
|
expect_pass 'replacement content survives cleanup race' test \
|
|
"$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
|
|
expect_pass 'owned quarantine is completely cleaned' test -z \
|
|
"$(find "$cache_root/services/race-component/.helm/charts" \
|
|
-mindepth 1 -maxdepth 1 ! -name chart-1.0.0 -print -quit)"
|
|
|
|
run_cleanup_exit_gate() (
|
|
_OBS_WORK=''
|
|
trap _observability_cleanup EXIT
|
|
:
|
|
)
|
|
expect_fail 'cleanup identity mismatch fails the successful exit gate' run_cleanup_exit_gate
|
|
expect_pass 'exit-gate mismatch still preserves replacement' test \
|
|
"$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
_OBS_CREATED_CACHE_IDENTITIES=()
|
|
_OBS_CREATED_CACHE_TYPES=()
|
|
|
|
handoff_work="$fixture_dir/handoff-work"
|
|
mkdir -m 0700 "$handoff_work"
|
|
for handoff_name in namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate; do
|
|
write_file "$handoff_work/$handoff_name.yaml" "fixture-$handoff_name"
|
|
chmod 0600 "$handoff_work/$handoff_name.yaml"
|
|
done
|
|
write_file "$handoff_work/resource-index.tsv" 'fixture-index'
|
|
chmod 0600 "$handoff_work/resource-index.tsv"
|
|
|
|
published_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$published_handoff")
|
|
chmod 0700 "$published_handoff"
|
|
published_identity="$(pin_handoff_directory "$published_handoff")"
|
|
expect_pass 'atomic no-clobber handoff publication' publish_verified_handoff \
|
|
"$published_handoff" "$published_identity" "$handoff_work"
|
|
expect_pass 'exact handoff entry set accepted' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
|
|
ln -s /dev/null "$published_handoff/extra-symlink"
|
|
expect_fail 'extra handoff symlink rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rm -f -- "$published_handoff/extra-symlink"
|
|
mkdir "$published_handoff/extra-directory"
|
|
expect_fail 'extra handoff directory rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rmdir "$published_handoff/extra-directory"
|
|
mv "$published_handoff/aggregate.yaml" "$fixture_dir/aggregate-handoff-saved"
|
|
ln -s /dev/null "$published_handoff/aggregate.yaml"
|
|
expect_fail 'exact handoff entry symlink rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rm -f -- "$published_handoff/aggregate.yaml"
|
|
mv "$fixture_dir/aggregate-handoff-saved" "$published_handoff/aggregate.yaml"
|
|
mv "$published_handoff/core-rules.yaml" "$fixture_dir/core-rules-handoff-saved"
|
|
mkdir "$published_handoff/core-rules.yaml"
|
|
expect_fail 'exact handoff entry directory rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rmdir "$published_handoff/core-rules.yaml"
|
|
mv "$fixture_dir/core-rules-handoff-saved" "$published_handoff/core-rules.yaml"
|
|
|
|
replaced_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$replaced_handoff")
|
|
chmod 0700 "$replaced_handoff"
|
|
replaced_identity="$(pin_handoff_directory "$replaced_handoff")"
|
|
replaced_original="$replaced_handoff.original"
|
|
mv "$replaced_handoff" "$replaced_original"
|
|
handoff_dirs+=("$replaced_original")
|
|
mkdir -m 0700 "$replaced_handoff"
|
|
expect_fail 'replaced handoff identity rejected before writes' publish_verified_handoff \
|
|
"$replaced_handoff" "$replaced_identity" "$handoff_work"
|
|
expect_pass 'replacement remains empty' test -z "$(find "$replaced_handoff" -mindepth 1 -maxdepth 1 -print -quit)"
|
|
|
|
occupied_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$occupied_handoff")
|
|
chmod 0700 "$occupied_handoff"
|
|
occupied_identity="$(pin_handoff_directory "$occupied_handoff")"
|
|
write_file "$occupied_handoff/namespaces.yaml" 'DO-NOT-CLOBBER'
|
|
expect_fail 'nonempty handoff rejected immediately before writes' publish_verified_handoff \
|
|
"$occupied_handoff" "$occupied_identity" "$handoff_work"
|
|
expect_pass 'existing handoff file is not clobbered' test \
|
|
"$(cat "$occupied_handoff/namespaces.yaml")" = DO-NOT-CLOBBER
|
|
|
|
child_a="$fixture_dir/child-a.yaml"
|
|
child_b="$fixture_dir/child-b.yaml"
|
|
aggregate="$fixture_dir/aggregate.yaml"
|
|
write_file "$child_a" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
|
|
' name: a' ' namespace: observability' 'data:' ' value: one'
|
|
write_file "$child_b" 'apiVersion: v1' 'kind: Service' 'metadata:' \
|
|
' name: b' ' namespace: observability' 'spec:' ' type: ClusterIP' \
|
|
' ports:' ' - port: 80'
|
|
{ cat "$child_a"; printf '%s\n' '---'; cat "$child_b"; } >"$aggregate"
|
|
index="$fixture_dir/resource-index.tsv"
|
|
expect_pass 'ordered resource index' build_resource_index "$index" \
|
|
"kps:$child_a" "loki:$child_b"
|
|
expected_index=$'1\tkps\tv1\tConfigMap\tobservability\ta\t'
|
|
if [[ "$(sed -n '2p' "$index")" != "$expected_index"* ]]; then
|
|
printf 'FAIL: resource index order or schema differs\n' >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
expect_pass 'child aggregate equivalence' assert_child_aggregate_equivalence \
|
|
"$aggregate" "kps:$child_a" "loki:$child_b"
|
|
|
|
aggregate_missing="$fixture_dir/aggregate-missing.yaml"
|
|
cp "$child_a" "$aggregate_missing"
|
|
expect_fail 'aggregate missing resource' assert_child_aggregate_equivalence \
|
|
"$aggregate_missing" "kps:$child_a" "loki:$child_b"
|
|
|
|
aggregate_drift="$fixture_dir/aggregate-drift.yaml"
|
|
sed 's/value: one/value: changed/' "$aggregate" >"$aggregate_drift"
|
|
expect_fail 'aggregate canonical hash drift' assert_child_aggregate_equivalence \
|
|
"$aggregate_drift" "kps:$child_a" "loki:$child_b"
|
|
|
|
duplicate_child="$fixture_dir/duplicate-child.yaml"
|
|
cp "$child_a" "$duplicate_child"
|
|
expect_fail 'duplicate child tuple' assert_child_aggregate_equivalence \
|
|
"$aggregate" "kps:$child_a" "loki:$child_b" "tempo:$duplicate_child"
|
|
|
|
assert_task2_manifests() {
|
|
local manifest_root="${1:-$ROOT}"
|
|
python3 - "$manifest_root" <<'PY'
|
|
import pathlib
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
|
|
root = pathlib.Path(sys.argv[1])
|
|
namespace_root = root / "infrastructure/namespaces/observability"
|
|
storage_root = root / "infrastructure/storage/observability-local-pv"
|
|
|
|
|
|
def load_documents(path):
|
|
with path.open(encoding="utf-8") as stream:
|
|
return [value for value in yaml.safe_load_all(stream) if value is not None]
|
|
|
|
|
|
namespace_kustomization = yaml.safe_load(
|
|
(namespace_root / "kustomization.yaml").read_text(encoding="utf-8")
|
|
)
|
|
assert namespace_kustomization["resources"] == [
|
|
"observability.yaml",
|
|
"observability-agent.yaml",
|
|
]
|
|
namespaces = []
|
|
for resource in namespace_kustomization["resources"]:
|
|
namespaces.extend(load_documents(namespace_root / resource))
|
|
assert len(namespaces) == 2
|
|
assert [item["kind"] for item in namespaces].count("Namespace") == 2
|
|
assert len({item["metadata"]["name"] for item in namespaces}) == 2
|
|
by_name = {item["metadata"]["name"]: item for item in namespaces}
|
|
assert set(by_name) == {"observability", "observability-agent"}
|
|
for item in by_name.values():
|
|
assert item["metadata"]["labels"]["observability.hyeonworks.com/enabled"] == "true"
|
|
|
|
for level in ("enforce", "audit", "warn"):
|
|
labels = by_name["observability"]["metadata"]["labels"]
|
|
assert labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
|
|
assert labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
|
|
agent_labels = by_name["observability-agent"]["metadata"]["labels"]
|
|
assert agent_labels["pod-security.kubernetes.io/enforce"] == "privileged"
|
|
assert agent_labels["pod-security.kubernetes.io/enforce-version"] == "latest"
|
|
for level in ("audit", "warn"):
|
|
assert agent_labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
|
|
assert agent_labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
|
|
|
|
assert (namespace_root / "discovery-namespaces.txt").read_text(encoding="utf-8") == (
|
|
"kube-system\ngitea\nkeycloak\nplatform-data\nobject-storage\n"
|
|
)
|
|
|
|
storage_kustomization = yaml.safe_load(
|
|
(storage_root / "kustomization.yaml").read_text(encoding="utf-8")
|
|
)
|
|
assert storage_kustomization["resources"] == [
|
|
"storage-classes.yaml",
|
|
"persistent-volumes.yaml",
|
|
]
|
|
storage_items = []
|
|
for resource in storage_kustomization["resources"]:
|
|
storage_items.extend(load_documents(storage_root / resource))
|
|
assert len(storage_items) == 12
|
|
assert [item["kind"] for item in storage_items].count("StorageClass") == 6
|
|
assert [item["kind"] for item in storage_items].count("PersistentVolume") == 6
|
|
assert {item["kind"] for item in storage_items} == {"StorageClass", "PersistentVolume"}
|
|
class_items = [item for item in storage_items if item["kind"] == "StorageClass"]
|
|
volume_items = [item for item in storage_items if item["kind"] == "PersistentVolume"]
|
|
assert len({item["metadata"]["name"] for item in class_items}) == 6
|
|
assert len({item["metadata"]["name"] for item in volume_items}) == 6
|
|
assert len({item["metadata"]["labels"]["app.kubernetes.io/component"] for item in volume_items}) == 6
|
|
classes = {
|
|
item["metadata"]["name"]: item
|
|
for item in class_items
|
|
}
|
|
volumes = {
|
|
item["metadata"]["labels"]["app.kubernetes.io/component"]: item
|
|
for item in volume_items
|
|
}
|
|
expected = {
|
|
"prometheus": ("/srv/k3s/ssd/observability/prometheus", "20Gi", "ssd-local-observability-prometheus-retain"),
|
|
"grafana": ("/srv/k3s/ssd/observability/grafana", "2Gi", "ssd-local-observability-grafana-retain"),
|
|
"alertmanager": ("/srv/k3s/ssd/observability/alertmanager", "1Gi", "ssd-local-observability-alertmanager-retain"),
|
|
"alloy": ("/srv/k3s/ssd/observability/alloy", "2Gi", "ssd-local-observability-alloy-retain"),
|
|
"loki": ("/srv/k3s/ssd/observability/loki", "5Gi", "ssd-local-observability-loki-retain"),
|
|
"tempo": ("/srv/k3s/ssd/observability/tempo", "5Gi", "ssd-local-observability-tempo-retain"),
|
|
}
|
|
assert set(classes) == {row[2] for row in expected.values()}
|
|
assert set(volumes) == set(expected)
|
|
for owner, (path, capacity, class_name) in expected.items():
|
|
storage_class = classes[class_name]
|
|
assert storage_class["provisioner"] == "kubernetes.io/no-provisioner"
|
|
assert storage_class["reclaimPolicy"] == "Retain"
|
|
assert storage_class["volumeBindingMode"] == "WaitForFirstConsumer"
|
|
assert storage_class["allowVolumeExpansion"] is False
|
|
volume = volumes[owner]
|
|
spec = volume["spec"]
|
|
assert spec["capacity"]["storage"] == capacity
|
|
assert spec["accessModes"] == ["ReadWriteOnce"]
|
|
assert spec["persistentVolumeReclaimPolicy"] == "Retain"
|
|
assert spec["storageClassName"] == class_name
|
|
assert spec["local"]["path"] == path
|
|
expression = spec["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"][0]
|
|
assert expression == {
|
|
"key": "kubernetes.io/hostname",
|
|
"operator": "In",
|
|
"values": ["donghyeon-system-product-name"],
|
|
}
|
|
actual_capacity = 0
|
|
for volume in volume_items:
|
|
value = volume["spec"]["capacity"]["storage"]
|
|
assert isinstance(value, str) and value.endswith("Gi") and value[:-2].isdigit()
|
|
actual_capacity += int(value[:-2])
|
|
assert actual_capacity == 35
|
|
PY
|
|
}
|
|
|
|
expect_pass 'Task 2 namespace and 35Gi Local PV matrix' assert_task2_manifests
|
|
|
|
expect_any_failure() {
|
|
local name="$1"
|
|
shift
|
|
if "$@" >"$fixture_dir/expected-any-failure.out" 2>&1; then
|
|
printf 'FAIL (expected any failure): %s\n' "$name" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
task2_manifest_fixture() {
|
|
local destination="$1"
|
|
mkdir -p "$destination/infrastructure/namespaces" "$destination/infrastructure/storage"
|
|
cp -R "$ROOT/infrastructure/namespaces/observability" \
|
|
"$destination/infrastructure/namespaces/observability"
|
|
cp -R "$ROOT/infrastructure/storage/observability-local-pv" \
|
|
"$destination/infrastructure/storage/observability-local-pv"
|
|
}
|
|
|
|
duplicate_manifest_root="$fixture_dir/task2-manifest-duplicate"
|
|
task2_manifest_fixture "$duplicate_manifest_root"
|
|
{
|
|
printf '%s\n' '---'
|
|
sed -n '1,/^---$/p' \
|
|
"$ROOT/infrastructure/storage/observability-local-pv/storage-classes.yaml" | sed '$d'
|
|
} >>"$duplicate_manifest_root/infrastructure/storage/observability-local-pv/storage-classes.yaml"
|
|
expect_any_failure 'duplicate StorageClass object is rejected' \
|
|
assert_task2_manifests "$duplicate_manifest_root"
|
|
|
|
extra_kind_root="$fixture_dir/task2-manifest-extra-kind"
|
|
task2_manifest_fixture "$extra_kind_root"
|
|
write_file "$extra_kind_root/infrastructure/storage/observability-local-pv/extra.yaml" \
|
|
'apiVersion: v1' 'kind: ConfigMap' 'metadata:' ' name: unexpected'
|
|
printf '%s\n' ' - extra.yaml' >> \
|
|
"$extra_kind_root/infrastructure/storage/observability-local-pv/kustomization.yaml"
|
|
expect_any_failure 'extra manifest kind is rejected' \
|
|
assert_task2_manifests "$extra_kind_root"
|
|
|
|
capacity_drift_root="$fixture_dir/task2-manifest-capacity-drift"
|
|
task2_manifest_fixture "$capacity_drift_root"
|
|
sed -i '0,/storage: 20Gi/s//storage: 19Gi/' \
|
|
"$capacity_drift_root/infrastructure/storage/observability-local-pv/persistent-volumes.yaml"
|
|
expect_any_failure 'actual rendered capacity drift is rejected' \
|
|
assert_task2_manifests "$capacity_drift_root"
|
|
|
|
readonly PREPARE_OBSERVABILITY_PATHS="$ROOT/scripts/bootstrap/prepare-observability-local-paths.sh"
|
|
source_options_before="$-"
|
|
# shellcheck source=/dev/null
|
|
source "$PREPARE_OBSERVABILITY_PATHS"
|
|
expect_pass 'path preparation script is source-safe' test "$-" = "$source_options_before"
|
|
|
|
olp_fixture="$fixture_dir/observability-local-paths"
|
|
olp_root="$olp_fixture/root"
|
|
olp_ssd="$olp_fixture/srv/k3s/ssd"
|
|
olp_aistor="$olp_fixture/srv/k3s/aistor"
|
|
olp_external="$olp_fixture/external"
|
|
mkdir -p "$olp_root" "$olp_ssd/observability" "$olp_aistor" "$olp_external"
|
|
|
|
olp_available='64424509440'
|
|
olp_uses=('0' '0' '0' '0' '0' '0')
|
|
olp_context='default'
|
|
olp_node='donghyeon-system-product-name'
|
|
olp_confirmation='APPLY default'
|
|
olp_helper_audit="$olp_fixture/helper-audit.log"
|
|
olp_helper_race=''
|
|
olp_helper_signal=''
|
|
olp_helper_signal_target=''
|
|
|
|
_olp_root_facts() { printf '%s\t%s\n' '/dev/root' 'ext4'; }
|
|
_olp_available_bytes() { printf '%s\n' "$olp_available"; }
|
|
_olp_current_context() { printf '%s\n' "$olp_context"; }
|
|
_olp_node_names() { printf '%s\n' "$olp_node"; }
|
|
_olp_read_confirmation() { printf '%s\n' "$olp_confirmation"; }
|
|
|
|
_olp_fixture_privileged_helper() {
|
|
if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]] || \
|
|
[[ "$2" != "$olp_root" || "$3" != "$olp_ssd" || "$4" != "$olp_aistor" ]]; then
|
|
return 97
|
|
fi
|
|
local program
|
|
program="$(_olp_helper_program)" || return
|
|
OLP_HELPER_FIXTURE_ROOT="$olp_fixture" \
|
|
OLP_HELPER_FIXTURE_VIRTUAL_AISTOR=1 \
|
|
OLP_HELPER_FIXTURE_OWNERS=1 \
|
|
OLP_HELPER_FIXTURE_AUDIT="$olp_helper_audit" \
|
|
OLP_HELPER_FIXTURE_RACE="$olp_helper_race" \
|
|
OLP_HELPER_FIXTURE_EXTERNAL="$olp_external" \
|
|
OLP_HELPER_FIXTURE_SIGNAL="$olp_helper_signal" \
|
|
OLP_HELPER_FIXTURE_SIGNAL_TARGET="$olp_helper_signal_target" \
|
|
/usr/bin/python3 - "$@" <<<"$program"
|
|
}
|
|
_olp_privileged_helper() { _olp_fixture_privileged_helper "$@"; }
|
|
|
|
dry_output="$fixture_dir/observability-dry-run.out"
|
|
expect_stateful_pass 'default dry-run emits facts without mutation' \
|
|
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor"
|
|
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor" >"$dry_output"
|
|
expect_pass 'dry-run leaves every planned path absent' test -z \
|
|
"$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)"
|
|
expect_pass 'dry-run prints sanitized root filesystem facts' test "$(sed -n '1,3p' "$dry_output")" = \
|
|
$'Root SSD device: /dev/root\nRoot SSD filesystem: ext4\nRoot SSD available bytes: 64424509440'
|
|
expect_pass 'dry-run prints all six exact planned fixture paths' test \
|
|
"$(sed -n '4,9p' "$dry_output")" = \
|
|
"$(for owner in prometheus grafana alertmanager alloy loki tempo; do
|
|
printf 'Planned path: %s/observability/%s\n' "$olp_ssd" "$owner"
|
|
done)"
|
|
|
|
assert_olp_execute_fails() {
|
|
local output rc
|
|
set +e
|
|
output="$(observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" 2>&1)"
|
|
rc=$?
|
|
set -e
|
|
[[ "$rc" != 0 && "$output" == ERROR:* ]]
|
|
}
|
|
|
|
assert_olp_execute_rejected() {
|
|
assert_olp_execute_fails
|
|
[[ -z "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)" ]]
|
|
}
|
|
|
|
exact_numeric_reply=$'PREFLIGHT\t53687091200\t1073741824\t0\t0\t0\t0\t0'
|
|
expect_pass '50GiB available and 1GiB aggregate are inclusive' \
|
|
_olp_validate_preflight_reply "$exact_numeric_reply"
|
|
expect_any_failure 'one byte below 50GiB is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091199\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'one target byte above 1GiB is rejected before summing' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t1073741825\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'aggregate one byte above 1GiB is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t536870913\t536870912\t0\t0\t0\t0'
|
|
expect_any_failure 'signed decimal is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t+53687091200\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'leading-zero decimal is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t053687091200\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'multiline numeric protocol is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t0\t0\t0\t0\t0\t0\n0'
|
|
expect_any_failure '2^63 available value is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t9223372036854775808\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'repeated 2^63 target values fail before mutation' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808'
|
|
|
|
write_file "$olp_external/command-marker" DO-NOT-EXECUTE
|
|
for unexpected in sudo install chown rm curl; do
|
|
set +e
|
|
_olp_fixture_privileged_helper "$unexpected" "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/unexpected-helper-$unexpected.out" 2>&1
|
|
unexpected_rc=$?
|
|
set -e
|
|
expect_pass "unexpected helper action $unexpected returns 97" test "$unexpected_rc" = 97
|
|
expect_pass "unexpected helper action $unexpected executes nothing" test \
|
|
"$(cat "$olp_external/command-marker")" = DO-NOT-EXECUTE
|
|
done
|
|
set +e
|
|
_olp_fixture_privileged_helper apply "$olp_root" "$olp_ssd" \
|
|
>"$fixture_dir/unexpected-helper-argc.out" 2>&1
|
|
unexpected_rc=$?
|
|
set -e
|
|
expect_pass 'unexpected helper argc returns 97' test "$unexpected_rc" = 97
|
|
|
|
olp_context='other'
|
|
expect_pass 'execute rejects non-default context' assert_olp_execute_rejected
|
|
olp_context='default'
|
|
olp_node='other-node'
|
|
expect_pass 'execute rejects any other node set' assert_olp_execute_rejected
|
|
olp_node='donghyeon-system-product-name'
|
|
olp_confirmation='apply default'
|
|
expect_pass 'execute rejects inexact TTY confirmation' assert_olp_execute_rejected
|
|
olp_confirmation='APPLY default'
|
|
|
|
olp_context_calls="$fixture_dir/observability-context-calls"
|
|
write_file "$olp_context_calls" 0
|
|
_olp_current_context() {
|
|
local calls
|
|
calls="$(cat "$olp_context_calls")"
|
|
calls=$((calls + 1))
|
|
printf '%s\n' "$calls" >"$olp_context_calls"
|
|
if (( calls == 1 )); then
|
|
printf '%s\n' default
|
|
else
|
|
printf '%s\n' changed-during-confirmation
|
|
fi
|
|
}
|
|
expect_pass 'execute rechecks context after TTY confirmation' assert_olp_execute_rejected
|
|
olp_context='default'
|
|
_olp_current_context() { printf '%s\n' "$olp_context"; }
|
|
|
|
mkdir "$olp_ssd/observability/prometheus"
|
|
write_file "$olp_ssd/observability/prometheus/existing" DO-NOT-CLOBBER
|
|
expect_pass 'execute rejects nonempty existing target' assert_olp_execute_fails
|
|
expect_pass 'nonempty rejection preserves content' test \
|
|
"$(cat "$olp_ssd/observability/prometheus/existing")" = DO-NOT-CLOBBER
|
|
rm -f -- "$olp_ssd/observability/prometheus/existing"
|
|
rmdir "$olp_ssd/observability/prometheus"
|
|
ln -s "$olp_aistor" "$olp_ssd/observability/prometheus"
|
|
expect_pass 'execute rejects symlink target' assert_olp_execute_fails
|
|
expect_pass 'symlink rejection preserves link' test -L "$olp_ssd/observability/prometheus"
|
|
unlink "$olp_ssd/observability/prometheus"
|
|
|
|
write_file "$olp_helper_audit" START
|
|
expect_stateful_pass 'execute prepares all six paths' \
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor"
|
|
expected_owner_audit="$(printf '%s\n' \
|
|
'set-owner-mode prometheus 1000 2000 0770' \
|
|
'set-owner-mode grafana 472 472 0770' \
|
|
'set-owner-mode alertmanager 1000 2000 0770' \
|
|
'set-owner-mode alloy 473 473 0770' \
|
|
'set-owner-mode loki 10001 10001 0770' \
|
|
'set-owner-mode tempo 10001 10001 0770')"
|
|
expect_pass 'descriptor helper uses exact owner group mode matrix' test \
|
|
"$(sed -n '/^set-owner-mode /p' "$olp_helper_audit")" = "$expected_owner_audit"
|
|
for owner in prometheus grafana alertmanager alloy loki tempo; do
|
|
expect_pass "execute creates $owner path" test -d "$olp_ssd/observability/$owner"
|
|
expect_pass "execute sets $owner mode" test "$(stat -c %a "$olp_ssd/observability/$owner")" = 770
|
|
rmdir "$olp_ssd/observability/$owner"
|
|
done
|
|
|
|
chmod 0751 "$olp_external"
|
|
write_file "$olp_external/race-marker" EXTERNAL-MUST-SURVIVE
|
|
mkdir -m 0755 "$olp_ssd/observability/prometheus"
|
|
olp_helper_race='ancestor'
|
|
set +e
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/observability-ancestor-race.out" 2>&1
|
|
race_rc=$?
|
|
set -e
|
|
expect_pass 'ancestor replacement race is rejected' test "$race_rc" != 0
|
|
expect_pass 'ancestor replacement leaves external marker untouched' test \
|
|
"$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
|
|
expect_pass 'ancestor replacement leaves external metadata untouched' test \
|
|
"$(stat -c %a "$olp_external")" = 751
|
|
expect_pass 'ancestor replacement restores pinned pre-existing metadata' test \
|
|
"$(stat -c %a "$olp_ssd/observability.pinned/prometheus")" = 755
|
|
unlink "$olp_ssd/observability"
|
|
mv "$olp_ssd/observability.pinned" "$olp_ssd/observability"
|
|
|
|
olp_helper_race='target:prometheus'
|
|
set +e
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/observability-target-race.out" 2>&1
|
|
race_rc=$?
|
|
set -e
|
|
expect_pass 'target replacement race is rejected' test "$race_rc" != 0
|
|
expect_pass 'target replacement leaves external marker untouched' test \
|
|
"$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
|
|
expect_pass 'target replacement leaves external metadata untouched' test \
|
|
"$(stat -c %a "$olp_external")" = 751
|
|
expect_pass 'target replacement restores pinned target metadata' test \
|
|
"$(stat -c %a "$olp_ssd/observability/prometheus.pinned")" = 755
|
|
unlink "$olp_ssd/observability/prometheus"
|
|
mv "$olp_ssd/observability/prometheus.pinned" "$olp_ssd/observability/prometheus"
|
|
rmdir "$olp_ssd/observability/prometheus"
|
|
|
|
for signal_case in HUP:prometheus INT:alloy TERM:alloy; do
|
|
olp_helper_race=''
|
|
olp_helper_signal="${signal_case%%:*}"
|
|
olp_helper_signal_target="${signal_case#*:}"
|
|
write_file "$olp_helper_audit" START
|
|
if [[ "$olp_helper_signal_target" == prometheus ]]; then
|
|
rmdir "$olp_ssd/observability"
|
|
else
|
|
mkdir -m 0755 "$olp_ssd/observability/prometheus"
|
|
fi
|
|
set +e
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/observability-signal-${olp_helper_signal}.out" 2>&1
|
|
signal_rc=$?
|
|
set -e
|
|
expect_pass "$olp_helper_signal post-create signal is rejected" test "$signal_rc" != 0
|
|
for owner in grafana alertmanager alloy loki tempo; do
|
|
expect_pass "$olp_helper_signal leaves no created $owner residue" test \
|
|
! -e "$olp_ssd/observability/$owner"
|
|
done
|
|
if [[ "$olp_helper_signal_target" == prometheus ]]; then
|
|
expect_pass "$olp_helper_signal leaves no first-target residue" test \
|
|
! -e "$olp_ssd/observability/prometheus"
|
|
expect_pass "$olp_helper_signal leaves no created parent residue" test \
|
|
! -e "$olp_ssd/observability"
|
|
mkdir "$olp_ssd/observability"
|
|
else
|
|
expect_pass "$olp_helper_signal restores pre-existing mode" test \
|
|
"$(stat -c %a "$olp_ssd/observability/prometheus")" = 755
|
|
expected_restore="$(printf 'restore-chown prometheus %s %s\nrestore-chmod prometheus 0755' \
|
|
"$(id -u)" "$(id -g)")"
|
|
expect_pass "$olp_helper_signal restores chown then chmod in exact order" test \
|
|
"$(tail -n 2 "$olp_helper_audit")" = "$expected_restore"
|
|
rmdir "$olp_ssd/observability/prometheus"
|
|
fi
|
|
done
|
|
olp_helper_signal=''
|
|
olp_helper_signal_target=''
|
|
|
|
printf 'Assertions: %s\n' "$pass_count"
|
|
printf 'OBSERVABILITY CORE RENDER ASSERTION TEST PASS\n'
|
|
|