1094 lines
59 KiB
Bash
Executable File
1094 lines
59 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly RENDERER="$ROOT/scripts/validate/render-observability-core.sh"
|
|
readonly EXPECTED_REJECTION_STATUS=23
|
|
|
|
if [[ ! -f "$RENDERER" ]]; then
|
|
printf 'EXPECTED RED: renderer is missing: %s\n' "$RENDERER" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Sourcing must define assertions without running the renderer or changing shell options.
|
|
source "$RENDERER"
|
|
|
|
fixture_dir="$(mktemp -d /tmp/platform-observability-render-test.XXXXXX)"
|
|
handoff_dirs=()
|
|
cleanup() {
|
|
local rc=$?
|
|
local directory
|
|
for directory in "${handoff_dirs[@]}"; do
|
|
case "$directory" in
|
|
/tmp/platform-observability-core-apply.*) rm -rf -- "$directory" ;;
|
|
esac
|
|
done
|
|
case "$fixture_dir" in
|
|
/tmp/platform-observability-render-test.*) rm -rf -- "$fixture_dir" ;;
|
|
esac
|
|
exit "$rc"
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
pass_count=0
|
|
|
|
expect_pass() {
|
|
local name="$1"
|
|
local output
|
|
shift
|
|
if ! output="$({ "$@"; } 2>&1)"; then
|
|
printf 'FAIL (expected pass): %s\n%s\n' "$name" "$output" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
expect_stateful_pass() {
|
|
local name="$1"
|
|
local output_file="$fixture_dir/stateful-output"
|
|
local rc
|
|
shift
|
|
set +e
|
|
"$@" >"$output_file" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ "$rc" != 0 ]]; then
|
|
printf 'FAIL (expected stateful pass): %s\n' "$name" >&2
|
|
sed -n '1,20p' "$output_file" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
expect_fail() {
|
|
local name="$1"
|
|
local output rc
|
|
shift
|
|
set +e
|
|
output="$({ "$@"; } 2>&1)"
|
|
rc=$?
|
|
set -e
|
|
if [[ "$rc" != "$EXPECTED_REJECTION_STATUS" ]]; then
|
|
printf 'FAIL (expected rejection status %s, got %s): %s\n%s\n' \
|
|
"$EXPECTED_REJECTION_STATUS" "$rc" "$name" "$output" >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$output" != REJECT:* ]]; then
|
|
printf 'FAIL (expected pinned rejection message): %s\n%s\n' "$name" "$output" >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$output" == *'DO-NOT-LEAK-FIXTURE'* ]]; then
|
|
printf 'FAIL (credential leaked): %s\n' "$name" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
write_file() {
|
|
local path="$1"
|
|
shift
|
|
printf '%s\n' "$@" >"$path"
|
|
}
|
|
|
|
assert_chart_pin_row() {
|
|
local expected="$*"
|
|
local label="$1"
|
|
local actual
|
|
actual="$(chart_pin_rows | awk -F '\t' -v label="$label" '$1 == label { print $1, $2, $3, $4, $5, $6, $7 }')"
|
|
[[ "$actual" == "$expected" ]]
|
|
}
|
|
|
|
assert_image_pin_row() {
|
|
local expected="$*"
|
|
local image="$1"
|
|
local actual
|
|
actual="$(image_pin_rows | awk -F '\t' -v image="$image" '$1 == image { print $1, $2 }')"
|
|
[[ "$actual" == "$expected" ]]
|
|
}
|
|
|
|
expect_pass 'Helm version pin' test "$(observability_expected_helm_version)" = v3.19.4
|
|
expect_pass 'Kustomize version pin' test "$(observability_expected_kustomize_version)" = v5.8.1
|
|
expect_pass 'Kubernetes render target pin' test "$(observability_helm_kube_version)" = 1.36.2
|
|
expected_render_roots=$'infrastructure/namespaces/observability\n'
|
|
expected_render_roots+=$'infrastructure/controllers/prometheus-operator-crds\n'
|
|
expected_render_roots+=$'infrastructure/storage/observability-local-pv\n'
|
|
expected_render_roots+=$'services/observability/kube-prometheus-stack\n'
|
|
expected_render_roots+=$'services/observability/loki\n'
|
|
expected_render_roots+=$'services/observability/tempo\n'
|
|
expected_render_roots+=$'services/observability/alloy\n'
|
|
expected_render_roots+=$'services/observability/node-exporter\n'
|
|
expected_render_roots+=$'services/observability/grafana\n'
|
|
expected_render_roots+=$'services/observability/blackbox-exporter\n'
|
|
expected_render_roots+=$'services/observability/network-policies\n'
|
|
expected_render_roots+=$'services/observability/targets\n'
|
|
expected_render_roots+=$'services/observability/rules/platform\n'
|
|
expected_render_roots+=$'services/observability/alerting\n'
|
|
expected_render_roots+=$'services/observability/dashboards/platform\n'
|
|
expected_render_roots+='services/observability'
|
|
expect_pass 'complete observability render root table' test "$(render_root_rows)" = "$expected_render_roots"
|
|
expected_chart_table=$'prometheus-operator-crds\thttps://prometheus-community.github.io/helm-charts\tprometheus-operator-crds\t31.0.0\tv0.93.0\t5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd\tinfrastructure/controllers/prometheus-operator-crds\n'
|
|
expected_chart_table+=$'kube-prometheus-stack\thttps://prometheus-community.github.io/helm-charts\tkube-prometheus-stack\t88.0.1\tv0.93.0\t0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117\tservices/observability/kube-prometheus-stack\n'
|
|
expected_chart_table+=$'loki\thttps://grafana-community.github.io/helm-charts\tloki\t18.7.1\t3.7.4\tde9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d\tservices/observability/loki\n'
|
|
expected_chart_table+=$'tempo\thttps://grafana-community.github.io/helm-charts\ttempo\t2.2.3\t2.10.7\t14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe\tservices/observability/tempo\n'
|
|
expected_chart_table+=$'alloy\thttps://grafana.github.io/helm-charts\talloy\t1.11.0\tv1.18.0\t11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd\tservices/observability/alloy\n'
|
|
expected_chart_table+=$'prometheus-node-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-node-exporter\t4.56.1\t1.12.1\t70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a\tservices/observability/node-exporter'
|
|
expected_chart_table+=$'\ngrafana\thttps://grafana-community.github.io/helm-charts\tgrafana\t12.10.1\t13.1.1\tff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8\tservices/observability/grafana'
|
|
expected_chart_table+=$'\nprometheus-blackbox-exporter\thttps://prometheus-community.github.io/helm-charts\tprometheus-blackbox-exporter\t11.16.0\tv0.28.0\t932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b\tservices/observability/blackbox-exporter'
|
|
expect_pass 'exact eight-row chart table with no extras' test "$(chart_pin_rows)" = "$expected_chart_table"
|
|
|
|
expected_image_table=$'quay.io/prometheus-operator/prometheus-operator:v0.93.0\tsha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2\n'
|
|
expected_image_table+=$'quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0\tsha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905\n'
|
|
expected_image_table+=$'quay.io/prometheus/prometheus:v3.13.2-distroless\tsha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b\n'
|
|
expected_image_table+=$'quay.io/prometheus/alertmanager:v0.33.1\tsha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b\n'
|
|
expected_image_table+=$'registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1\tsha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4\n'
|
|
expected_image_table+=$'quay.io/prometheus/node-exporter:v1.12.1\tsha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798\n'
|
|
expected_image_table+=$'docker.io/grafana/loki:3.7.4\tsha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e\n'
|
|
expected_image_table+=$'docker.io/grafana/tempo:2.10.7\tsha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c\n'
|
|
expected_image_table+=$'docker.io/grafana/alloy:v1.18.0\tsha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17\n'
|
|
expected_image_table+=$'ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0\tsha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522\n'
|
|
expected_image_table+=$'docker.io/library/busybox:1.37.0\tsha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0\n'
|
|
expected_image_table+=$'quay.io/minio/aistor/mc\tsha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b\n'
|
|
expected_image_table+=$'docker.io/grafana/grafana:13.1.1\tsha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b\n'
|
|
expected_image_table+=$'quay.io/kiwigrid/k8s-sidecar:2.10.0\tsha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3\n'
|
|
expected_image_table+=$'quay.io/prometheus/blackbox-exporter:v0.28.0\tsha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68'
|
|
expect_pass 'exact fifteen-row image table with no extras' test "$(image_pin_rows)" = "$expected_image_table"
|
|
expect_pass 'Prometheus CRD chart pin' assert_chart_pin_row prometheus-operator-crds https://prometheus-community.github.io/helm-charts prometheus-operator-crds 31.0.0 v0.93.0 5d11cac50321bc7ae56b35d2701e56f21fe079ca3aea872abb28a3e2119d5cfd infrastructure/controllers/prometheus-operator-crds
|
|
expect_pass 'KPS chart pin' assert_chart_pin_row kube-prometheus-stack https://prometheus-community.github.io/helm-charts kube-prometheus-stack 88.0.1 v0.93.0 0998b3fae8660cd9ea224caa1c7c4b5d2040d5a02c64e3cdfd3f48d184b50117 services/observability/kube-prometheus-stack
|
|
expect_pass 'Loki chart pin' assert_chart_pin_row loki https://grafana-community.github.io/helm-charts loki 18.7.1 3.7.4 de9ab59827f2ed24634e94db120ea2467f5d08b02e8d7c250e6bf88c17bfb48d services/observability/loki
|
|
expect_pass 'Tempo chart pin' assert_chart_pin_row tempo https://grafana-community.github.io/helm-charts tempo 2.2.3 2.10.7 14c52efe5d0cad5456ffa5a8be1e5107be47d184be3e209f748be51a7b0316fe services/observability/tempo
|
|
expect_pass 'Alloy chart pin' assert_chart_pin_row alloy https://grafana.github.io/helm-charts alloy 1.11.0 v1.18.0 11d253b62e47beeacd89eb4283fc056962ecbf143984863c1998be13da0772dd services/observability/alloy
|
|
expect_pass 'node-exporter chart pin' assert_chart_pin_row prometheus-node-exporter https://prometheus-community.github.io/helm-charts prometheus-node-exporter 4.56.1 1.12.1 70fab291e00a34c58fd943bb075fecc2e2e73fa95db1bcbbe7cdfe008795354a services/observability/node-exporter
|
|
expect_pass 'Grafana chart pin' assert_chart_pin_row grafana https://grafana-community.github.io/helm-charts grafana 12.10.1 13.1.1 ff865a8b9d1d7563901b8e6aa653b4f345bd4d88344c149065bd23958e2beeb8 services/observability/grafana
|
|
expect_pass 'Blackbox chart pin' assert_chart_pin_row prometheus-blackbox-exporter https://prometheus-community.github.io/helm-charts prometheus-blackbox-exporter 11.16.0 v0.28.0 932aa65df0538d9dc003c46bb663ebe44a06078af7a6a54577af5339668ac65b services/observability/blackbox-exporter
|
|
|
|
expect_pass 'operator image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-operator:v0.93.0 sha256:64eb7914e4705dbb64438e3b3193da1226ad2ea4db2924983693999888cda9b2
|
|
expect_pass 'reloader image pin' assert_image_pin_row quay.io/prometheus-operator/prometheus-config-reloader:v0.93.0 sha256:65b90f44d5535b106015ac60bafb57803f65dc928c187874de6cd7a9ec6c8905
|
|
expect_pass 'Prometheus image pin' assert_image_pin_row quay.io/prometheus/prometheus:v3.13.2-distroless sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
|
|
expect_pass 'Alertmanager image pin' assert_image_pin_row quay.io/prometheus/alertmanager:v0.33.1 sha256:a89f8d4520954079275441eecdb71444328bd90633dd4eddfc33b9ed657f349b
|
|
expect_pass 'KSM image pin' assert_image_pin_row registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 sha256:7661da8c99b733d43117e4cba12bd9865d335e5777191d0af3d789807aded9f4
|
|
expect_pass 'node-exporter image pin' assert_image_pin_row quay.io/prometheus/node-exporter:v1.12.1 sha256:da83fae85603c4e47e6c68369a7d746e2dda683dc35ea2e234b4f171e0d92798
|
|
expect_pass 'Loki image pin' assert_image_pin_row docker.io/grafana/loki:3.7.4 sha256:d80be589cf74b6d92747a1b155bec6f07bdd65e261f797fe93a78fb4e833032e
|
|
expect_pass 'Tempo image pin' assert_image_pin_row docker.io/grafana/tempo:2.10.7 sha256:6616b00287a4d7001951b5de117828ad5c6f93744935c1b7a5e044736373352c
|
|
expect_pass 'Alloy image pin' assert_image_pin_row docker.io/grafana/alloy:v1.18.0 sha256:eb21f4c0858edffcdd1b385910ddeef26f692fc2c282f61baa724fc09d274a17
|
|
expect_pass 'telemetrygen image pin' assert_image_pin_row ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.153.0 sha256:9708a1d33398c8725549104d1f6009be62569e47e094430543da0f66758cc522
|
|
expect_pass 'BusyBox image pin' assert_image_pin_row docker.io/library/busybox:1.37.0 sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0
|
|
expect_pass 'AIStor mc image pin' assert_image_pin_row quay.io/minio/aistor/mc sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b
|
|
expect_pass 'Grafana image pin' assert_image_pin_row docker.io/grafana/grafana:13.1.1 sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b
|
|
expect_pass 'Grafana sidecar image pin' assert_image_pin_row quay.io/kiwigrid/k8s-sidecar:2.10.0 sha256:129877c81acf2bc8c3fa000e89a62e020eb89d41ceb94767c657aef5bb0cc0d3
|
|
expect_pass 'Blackbox image pin' assert_image_pin_row quay.io/prometheus/blackbox-exporter:v0.28.0 sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68
|
|
expect_pass 'tagged Grafana digest is in the rendered-image allowlist' grep -Fqx \
|
|
docker.io/grafana/grafana:13.1.1@sha256:f33c692ba1a5ee15724cf6b22db65e9de39dde14d80f7d73a9546e3fc917270b \
|
|
<(_observability_image_allowlist)
|
|
expect_pass 'tagged Blackbox digest is in the rendered-image allowlist' grep -Fqx \
|
|
quay.io/prometheus/blackbox-exporter:v0.28.0@sha256:43027b43fb785b7c5adc53bd3b5dbc1a258270a2e8aff24f477b45c4e38dac68 \
|
|
<(_observability_image_allowlist)
|
|
|
|
alertmanager_fixture="$fixture_dir/alertmanager.yaml"
|
|
write_file "$alertmanager_fixture" \
|
|
'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: Alertmanager' \
|
|
'metadata:' \
|
|
' name: observability-core-kube-pr-alertmanager' \
|
|
' namespace: observability' \
|
|
'spec:' \
|
|
' alertmanagerConfiguration:' \
|
|
' name: platform-alertmanager'
|
|
alerting_fixture="$fixture_dir/alerting.yaml"
|
|
write_file "$alerting_fixture" '{"apiVersion":"monitoring.coreos.com/v1alpha1","kind":"AlertmanagerConfig","metadata":{"name":"platform-alertmanager","namespace":"observability","labels":{"observability.hyeonworks.com/instance":"home"}},"spec":{"route":{"receiver":"platform-slack","groupBy":["cluster","namespace","alertname","severity"],"groupWait":"30s","groupInterval":"5m","repeatInterval":"4h","routes":[{"receiver":"platform-null","matchers":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}]}]},"inhibitRules":[{"sourceMatch":[{"name":"alertname","matchType":"=","value":"InfoInhibitor"}],"targetMatch":[{"name":"severity","matchType":"=","value":"info"}],"equal":["namespace"]}],"receivers":[{"name":"platform-null"},{"name":"platform-slack","slackConfigs":[{"apiURL":{"name":"alertmanager-slack-webhook","key":"url"},"sendResolved":true,"linkNames":false,"mrkdwnIn":["text","fields"],"fallback":"{{ if eq .Status \"firing\" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","title":"{{ if eq .Status \"firing\" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}","titleLink":"https://grafana.learn.hyeonworks.com/","color":"{{ if eq .Status \"resolved\" }}good{{ else if or (eq .CommonLabels.severity \"emergency\") (eq .CommonLabels.severity \"critical\") }}danger{{ else if eq .CommonLabels.severity \"warning\" }}warning{{ else }}#439FE0{{ end }}","fields":[{"title":"Status","value":"{{ .Status | toUpper }}","short":true},{"title":"Severity","value":"{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}","short":true},{"title":"Location","value":"{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}","short":true},{"title":"Alert count","value":"{{ len .Alerts }}","short":true}],"text":"{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>","footer":"hyeonworks observability · Alertmanager"}]}]}}' \
|
|
'---' \
|
|
'apiVersion: networking.k8s.io/v1' \
|
|
'kind: NetworkPolicy' \
|
|
'metadata:' \
|
|
' name: observability-allow-alertmanager-public-https' \
|
|
' namespace: observability' \
|
|
'spec:' \
|
|
' podSelector:' \
|
|
' matchLabels:' \
|
|
' app.kubernetes.io/name: alertmanager' \
|
|
' app.kubernetes.io/instance: observability-core-kube-pr-alertmanager' \
|
|
' policyTypes: [Egress]' \
|
|
' egress:' \
|
|
' - to:' \
|
|
' - ipBlock:' \
|
|
' cidr: 0.0.0.0/0' \
|
|
' except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]' \
|
|
' ports:' \
|
|
' - protocol: TCP' \
|
|
' port: 443'
|
|
expect_pass 'exact structured Alertmanager routing contract' \
|
|
assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_fixture"
|
|
|
|
alertmanager_selector_fixture="$fixture_dir/alertmanager-selector.yaml"
|
|
sed '/^spec:$/a\ alertmanagerConfigSelector: {}' \
|
|
"$alertmanager_fixture" >"$alertmanager_selector_fixture"
|
|
expect_fail 'Alertmanager discovery selector bypasses the global configuration' \
|
|
assert_alertmanager_routing_contract "$alertmanager_selector_fixture" "$alerting_fixture"
|
|
|
|
alerting_route_fixture="$fixture_dir/alerting-route-drift.yaml"
|
|
sed 's/"repeatInterval":"4h"/"repeatInterval":"12h"/' \
|
|
"$alerting_fixture" >"$alerting_route_fixture"
|
|
expect_fail 'Alertmanager Slack route drift' \
|
|
assert_alertmanager_routing_contract "$alertmanager_fixture" "$alerting_route_fixture"
|
|
|
|
write_pod() {
|
|
local path="$1"
|
|
local namespace="$2"
|
|
local name="$3"
|
|
local pod_fields="$4"
|
|
local container_fields="${5:-}"
|
|
cat >"$path" <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $name
|
|
namespace: $namespace
|
|
spec:
|
|
automountServiceAccountToken: false
|
|
$pod_fields
|
|
containers:
|
|
- name: main
|
|
image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b
|
|
$container_fields
|
|
EOF
|
|
}
|
|
|
|
valid_image="$fixture_dir/valid-image.yaml"
|
|
write_file "$valid_image" \
|
|
'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: pinned' \
|
|
' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \
|
|
' containers:' ' - name: prometheus' \
|
|
' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
|
|
expect_pass 'pinned image accepted' assert_pinned_images "$valid_image"
|
|
expect_pass 'credential assertion positive control' assert_no_credentials "$valid_image"
|
|
expect_pass 'observability namespace security positive control' assert_namespace_pod_security "$valid_image"
|
|
|
|
tag_image="$fixture_dir/tag-image.yaml"
|
|
sed 's|@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b|:v3.13.2-distroless|' \
|
|
"$valid_image" >"$tag_image"
|
|
expect_fail 'tag-only image' assert_pinned_images "$tag_image"
|
|
|
|
wrong_digest="$fixture_dir/wrong-digest.yaml"
|
|
sed 's/ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' \
|
|
"$valid_image" >"$wrong_digest"
|
|
expect_fail 'unpinned digest' assert_pinned_images "$wrong_digest"
|
|
|
|
secret="$fixture_dir/secret.yaml"
|
|
write_file "$secret" \
|
|
'apiVersion: v1' 'kind: Secret' 'metadata:' ' name: forbidden' \
|
|
' namespace: observability' 'stringData:' ' password: DO-NOT-LEAK-FIXTURE'
|
|
expect_fail 'credential Secret' assert_no_credentials "$secret"
|
|
|
|
literal_credential="$fixture_dir/literal-credential.yaml"
|
|
write_file "$literal_credential" \
|
|
'apiVersion: v1' 'kind: Pod' 'metadata:' ' name: forbidden' \
|
|
' namespace: observability' 'spec:' ' automountServiceAccountToken: false' \
|
|
' containers:' ' - name: app' ' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b' \
|
|
' env:' ' - name: ACCESS_KEY' ' value: DO-NOT-LEAK-FIXTURE'
|
|
expect_fail 'literal credential payload' assert_no_credentials "$literal_credential"
|
|
|
|
for field in \
|
|
' privileged: true' \
|
|
' hostNetwork: true' \
|
|
' hostPID: true' \
|
|
' hostIPC: true'; do
|
|
slug="$(printf '%s' "$field" | tr -cd '[:alpha:]' | tr '[:upper:]' '[:lower:]')"
|
|
pod="$fixture_dir/observability-$slug.yaml"
|
|
if [[ "$field" == *privileged* ]]; then
|
|
container_security="$(printf ' securityContext:\n privileged: true')"
|
|
write_pod "$pod" observability core '' "$container_security"
|
|
else
|
|
write_pod "$pod" observability core "$field" ''
|
|
fi
|
|
expect_fail "observability forbids $slug" assert_namespace_pod_security "$pod"
|
|
done
|
|
|
|
obs_hostpath="$fixture_dir/observability-hostpath.yaml"
|
|
write_pod "$obs_hostpath" observability core \
|
|
$' volumes:\n - name: host\n hostPath:\n path: /var/log/pods' \
|
|
$' volumeMounts:\n - name: host\n mountPath: /logs\n readOnly: true'
|
|
expect_fail 'observability hostPath' assert_namespace_pod_security "$obs_hostpath"
|
|
|
|
controller_namespace="$fixture_dir/controller-namespace.yaml"
|
|
write_file "$controller_namespace" 'apiVersion: apps/v1' 'kind: Deployment' \
|
|
'metadata:' ' name: controller-security' ' namespace: observability' 'spec:' \
|
|
' selector:' ' matchLabels:' ' app: controller-security' ' template:' \
|
|
' metadata:' ' labels:' ' app: controller-security' ' spec:' \
|
|
' automountServiceAccountToken: false' ' hostPID: true' ' containers:' \
|
|
' - name: app' \
|
|
' image: quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
|
|
expect_fail 'controller template inherits namespace security' assert_namespace_pod_security "$controller_namespace"
|
|
controller_valid="$fixture_dir/controller-valid.yaml"
|
|
sed '/hostPID: true/d' "$controller_namespace" >"$controller_valid"
|
|
expect_pass 'controller observability namespace positive control' \
|
|
assert_namespace_pod_security "$controller_valid"
|
|
controller_missing_namespace="$fixture_dir/controller-missing-namespace.yaml"
|
|
sed '/namespace: observability/d' "$controller_valid" >"$controller_missing_namespace"
|
|
expect_fail 'controller namespace missing' \
|
|
assert_namespace_pod_security "$controller_missing_namespace"
|
|
controller_foreign_namespace="$fixture_dir/controller-foreign-namespace.yaml"
|
|
sed 's/namespace: observability/namespace: default/' \
|
|
"$controller_valid" >"$controller_foreign_namespace"
|
|
expect_fail 'controller namespace foreign' \
|
|
assert_namespace_pod_security "$controller_foreign_namespace"
|
|
|
|
agent_allowed="$fixture_dir/agent-allowed.yaml"
|
|
write_pod "$agent_allowed" observability-agent alloy \
|
|
$' volumes:\n - name: pod-logs\n hostPath:\n path: /var/log/pods' \
|
|
$' volumeMounts:\n - name: pod-logs\n mountPath: /var/log/pods\n readOnly: true'
|
|
expect_pass 'Alloy read-only allowlisted hostPath' assert_namespace_pod_security "$agent_allowed"
|
|
|
|
missing_namespace="$fixture_dir/missing-namespace.yaml"
|
|
sed '/namespace: observability-agent/d' "$agent_allowed" >"$missing_namespace"
|
|
expect_fail 'Pod namespace missing' assert_namespace_pod_security "$missing_namespace"
|
|
|
|
foreign_namespace="$fixture_dir/foreign-namespace.yaml"
|
|
sed 's/namespace: observability-agent/namespace: default/' "$agent_allowed" >"$foreign_namespace"
|
|
expect_fail 'Pod namespace foreign' assert_namespace_pod_security "$foreign_namespace"
|
|
|
|
agent_bad_path="$fixture_dir/agent-bad-path.yaml"
|
|
sed 's|/var/log/pods|/etc|g' "$agent_allowed" >"$agent_bad_path"
|
|
expect_fail 'agent hostPath allowlist' assert_namespace_pod_security "$agent_bad_path"
|
|
|
|
agent_rw="$fixture_dir/agent-rw.yaml"
|
|
sed 's/readOnly: true/readOnly: false/' "$agent_allowed" >"$agent_rw"
|
|
expect_fail 'agent hostPath readOnly' assert_namespace_pod_security "$agent_rw"
|
|
|
|
automount_missing="$fixture_dir/automount-missing.yaml"
|
|
sed '/automountServiceAccountToken:/d' "$valid_image" >"$automount_missing"
|
|
expect_fail 'automount missing' assert_namespace_pod_security "$automount_missing"
|
|
|
|
automount_true="$fixture_dir/automount-true.yaml"
|
|
sed 's/automountServiceAccountToken: false/automountServiceAccountToken: true/' \
|
|
"$valid_image" >"$automount_true"
|
|
expect_fail 'automount true' assert_namespace_pod_security "$automount_true"
|
|
|
|
for exposure in ingress NodePort LoadBalancer; do
|
|
public="$fixture_dir/public-$exposure.yaml"
|
|
if [[ "$exposure" == ingress ]]; then
|
|
write_file "$public" 'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
|
|
'metadata:' ' name: public' ' namespace: observability' 'spec: {}'
|
|
else
|
|
write_file "$public" 'apiVersion: v1' 'kind: Service' 'metadata:' \
|
|
' name: public' ' namespace: observability' 'spec:' " type: $exposure" \
|
|
' ports:' ' - port: 80'
|
|
fi
|
|
expect_fail "public $exposure" assert_no_public_exposure "$public"
|
|
done
|
|
grafana_ingress="$fixture_dir/grafana-private-ingress.yaml"
|
|
write_file "$grafana_ingress" \
|
|
'apiVersion: networking.k8s.io/v1' 'kind: Ingress' \
|
|
'metadata:' ' name: grafana' ' namespace: observability' \
|
|
'spec:' ' ingressClassName: traefik' ' rules:' \
|
|
' - host: grafana.learn.hyeonworks.com' ' http:' ' paths:' \
|
|
' - path: /' ' pathType: Prefix' ' backend:' \
|
|
' service:' ' name: grafana' ' port:' \
|
|
' number: 80'
|
|
expect_pass 'exact Host-Nginx-fronted Grafana ingress' \
|
|
assert_no_public_exposure "$grafana_ingress"
|
|
grafana_public_ingress="$fixture_dir/grafana-public-ingress.yaml"
|
|
sed 's/grafana.learn.hyeonworks.com/public.example.com/' \
|
|
"$grafana_ingress" >"$grafana_public_ingress"
|
|
expect_fail 'Grafana ingress host drift' \
|
|
assert_no_public_exposure "$grafana_public_ingress"
|
|
cluster_ip="$fixture_dir/cluster-ip.yaml"
|
|
write_file "$cluster_ip" 'apiVersion: v1' 'kind: Service' 'metadata:' \
|
|
' name: internal' ' namespace: observability' 'spec:' ' type: ClusterIP' \
|
|
' ports:' ' - port: 9090'
|
|
expect_pass 'public exposure assertion positive control' assert_no_public_exposure "$cluster_ip"
|
|
|
|
for auxiliary in gateway canary chunks-cache results-cache test minio; do
|
|
loki_aux="$fixture_dir/loki-$auxiliary.yaml"
|
|
write_file "$loki_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
|
|
" name: loki-$auxiliary" ' namespace: observability' 'spec: {}'
|
|
expect_fail "Loki auxiliary $auxiliary" assert_no_loki_auxiliary "$loki_aux"
|
|
done
|
|
loki_main="$fixture_dir/loki-main.yaml"
|
|
write_file "$loki_main" 'apiVersion: apps/v1' 'kind: StatefulSet' 'metadata:' \
|
|
' name: loki' ' namespace: observability' 'spec: {}'
|
|
expect_pass 'Loki auxiliary assertion positive control' assert_no_loki_auxiliary "$loki_main"
|
|
|
|
for auxiliary in metrics-generator tempo-query; do
|
|
tempo_aux="$fixture_dir/tempo-$auxiliary.yaml"
|
|
write_file "$tempo_aux" 'apiVersion: apps/v1' 'kind: Deployment' 'metadata:' \
|
|
" name: $auxiliary" ' namespace: observability' 'spec: {}'
|
|
expect_fail "Tempo auxiliary $auxiliary" assert_tempo_topology "$tempo_aux"
|
|
done
|
|
|
|
tempo_http="$fixture_dir/tempo-http.yaml"
|
|
write_file "$tempo_http" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
|
|
' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \
|
|
' receivers:' ' otlp:' ' protocols:' ' grpc:' \
|
|
' endpoint: 0.0.0.0:4317' ' http:' \
|
|
' endpoint: 0.0.0.0:4318'
|
|
expect_fail 'Tempo OTLP HTTP receiver' assert_tempo_topology "$tempo_http"
|
|
tempo_grpc="$fixture_dir/tempo-grpc.yaml"
|
|
write_file "$tempo_grpc" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
|
|
' name: tempo' ' namespace: observability' 'data:' ' tempo.yaml: |' \
|
|
' receivers:' ' otlp:' ' protocols:' ' grpc:' \
|
|
' endpoint: 0.0.0.0:4317'
|
|
expect_pass 'Tempo topology assertion positive control' assert_tempo_topology "$tempo_grpc"
|
|
|
|
selector_values="$fixture_dir/selector-values.yaml"
|
|
write_file "$selector_values" 'prometheus:' ' prometheusSpec:' \
|
|
' serviceMonitorSelectorNilUsesHelmValues: false' \
|
|
' podMonitorSelectorNilUsesHelmValues: false' \
|
|
' probeSelectorNilUsesHelmValues: false' \
|
|
' ruleSelectorNilUsesHelmValues: false'
|
|
expect_pass 'selector values false' assert_selector_values "$selector_values"
|
|
|
|
selector_true="$fixture_dir/selector-true.yaml"
|
|
sed '0,/false/s//true/' "$selector_values" >"$selector_true"
|
|
expect_fail 'nilUsesHelmValues true' assert_selector_values "$selector_true"
|
|
|
|
prometheus_missing="$fixture_dir/prometheus-missing-selector.yaml"
|
|
write_file "$prometheus_missing" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \
|
|
'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \
|
|
' probeSelector: {}'
|
|
expect_fail 'Prometheus selector missing' assert_prometheus_selectors "$prometheus_missing"
|
|
prometheus_valid="$fixture_dir/prometheus-valid.yaml"
|
|
write_file "$prometheus_valid" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: Prometheus' 'metadata:' ' name: home' ' namespace: observability' \
|
|
'spec:' ' serviceMonitorSelector: {}' ' podMonitorSelector: {}' \
|
|
' probeSelector: {}' ' ruleSelector: {}'
|
|
expect_pass 'Prometheus selector assertion positive control' assert_prometheus_selectors "$prometheus_valid"
|
|
|
|
crd="$fixture_dir/kps-crd.yaml"
|
|
write_file "$crd" 'apiVersion: apiextensions.k8s.io/v1' \
|
|
'kind: CustomResourceDefinition' 'metadata:' ' name: prometheuses.monitoring.coreos.com'
|
|
expect_fail 'CRD in workload render' assert_no_crds "$crd"
|
|
expect_pass 'CRD assertion positive control' assert_no_crds "$valid_image"
|
|
|
|
shared_pv="$fixture_dir/shared-pv.yaml"
|
|
write_file "$shared_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
|
|
' name: prometheus' 'spec:' ' storageClassName: shared-local' ' local:' \
|
|
' path: /srv/prometheus' '---' 'apiVersion: v1' 'kind: PersistentVolume' \
|
|
'metadata:' ' name: alertmanager' 'spec:' ' storageClassName: shared-local' \
|
|
' local:' ' path: /srv/alertmanager'
|
|
expect_fail 'shared Local PV StorageClass' assert_unique_local_pv_storage_classes "$shared_pv"
|
|
single_pv="$fixture_dir/single-pv.yaml"
|
|
write_file "$single_pv" 'apiVersion: v1' 'kind: PersistentVolume' 'metadata:' \
|
|
' name: prometheus' 'spec:' ' storageClassName: prometheus-local' ' local:' \
|
|
' path: /srv/prometheus'
|
|
expect_pass 'Local PV assertion positive control' assert_unique_local_pv_storage_classes "$single_pv"
|
|
|
|
for fake in spring kafka batch backup; do
|
|
fake_monitor="$fixture_dir/fake-$fake.yaml"
|
|
write_file "$fake_monitor" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: ServiceMonitor' 'metadata:' " name: $fake-monitor" \
|
|
' namespace: observability' 'spec: {}'
|
|
expect_fail "fake $fake monitor or rule" assert_no_unscoped_monitors_rules "$fake_monitor"
|
|
done
|
|
scoped_monitor="$fixture_dir/scoped-monitor.yaml"
|
|
write_file "$scoped_monitor" 'apiVersion: monitoring.coreos.com/v1' \
|
|
'kind: ServiceMonitor' 'metadata:' ' name: prometheus' \
|
|
' namespace: observability' 'spec: {}'
|
|
expect_pass 'monitor/rule assertion positive control' assert_no_unscoped_monitors_rules "$scoped_monitor"
|
|
|
|
chart="$fixture_dir/Chart.yaml"
|
|
write_file "$chart" 'apiVersion: v2' 'name: loki' 'version: 18.7.1' 'appVersion: 3.7.4'
|
|
expect_pass 'chart metadata pin accepted' validate_chart_metadata "$chart" loki 18.7.1 3.7.4
|
|
expect_fail 'chart metadata name mismatch' validate_chart_metadata "$chart" tempo 18.7.1 3.7.4
|
|
expect_fail 'chart metadata version mismatch' validate_chart_metadata "$chart" loki 18.7.2 3.7.4
|
|
expect_fail 'chart metadata appVersion mismatch' validate_chart_metadata "$chart" loki 18.7.1 3.7.5
|
|
|
|
handoff="$fixture_dir/platform-observability-core-apply.fixture"
|
|
mkdir -m 0700 "$handoff"
|
|
expect_fail 'handoff prefix restricted to /tmp' validate_handoff_directory "$handoff"
|
|
safe_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$safe_handoff")
|
|
chmod 0700 "$safe_handoff"
|
|
expect_pass 'safe empty handoff' validate_handoff_directory "$safe_handoff"
|
|
touch "$safe_handoff/unexpected"
|
|
expect_fail 'handoff must be empty' validate_handoff_directory "$safe_handoff"
|
|
rm -f -- "$safe_handoff/unexpected"
|
|
rmdir -- "$safe_handoff"
|
|
|
|
empty_suffix='/tmp/platform-observability-core-apply.'
|
|
expect_fail 'handoff suffix must be nonempty' validate_handoff_directory "$empty_suffix"
|
|
|
|
root_fixture="$fixture_dir/render-roots"
|
|
mkdir -p "$root_fixture/existing/child"
|
|
expect_pass 'physical render roots accepted' validate_render_roots "$root_fixture" existing/child
|
|
expect_fail 'missing render root rejected' validate_render_roots "$root_fixture" missing/child
|
|
expect_pass 'missing root preflight creates nothing' test ! -e "$root_fixture/missing"
|
|
mkdir -p "$root_fixture/physical/child"
|
|
ln -s physical "$root_fixture/symlinked"
|
|
expect_fail 'symlinked render root lineage rejected' validate_render_roots "$root_fixture" symlinked/child
|
|
|
|
cache_root="$fixture_dir/cache-root"
|
|
mkdir -p "$cache_root/services/component/.helm/charts"
|
|
_OBS_ROOT="$cache_root"
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
expect_stateful_pass 'exclusive cache directory creation' _observability_create_cache_directory \
|
|
services/component chart 1.0.0
|
|
expect_fail 'existing exact cache is no-clobber' _observability_create_cache_directory \
|
|
services/component chart 1.0.0
|
|
expect_stateful_pass 'tracked cache cleanup' _observability_cleanup_cache_dirs
|
|
expect_pass 'pre-existing .helm preserved' test -d "$cache_root/services/component/.helm"
|
|
expect_pass 'pre-existing charts preserved' test -d "$cache_root/services/component/.helm/charts"
|
|
expect_pass 'exact created cache removed' test ! -e "$cache_root/services/component/.helm/charts/chart-1.0.0"
|
|
mkdir -p "$cache_root/services/missing-parent"
|
|
expect_fail 'missing component root rejected before cache mkdir' _observability_create_cache_directory \
|
|
services/missing-parent/component chart 1.0.0
|
|
expect_pass 'missing component root leaves no .helm' test ! -e "$cache_root/services/missing-parent/component"
|
|
mkdir -p "$cache_root/services/new-component"
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
_OBS_CREATED_CACHE_IDENTITIES=()
|
|
_OBS_CREATED_CACHE_TYPES=()
|
|
expect_stateful_pass 'new cache parents tracked' _observability_create_cache_directory \
|
|
services/new-component chart 1.0.0
|
|
expect_stateful_pass 'new cache parents cleaned' _observability_cleanup_cache_dirs
|
|
expect_pass 'created .helm parent removed' test ! -e "$cache_root/services/new-component/.helm"
|
|
|
|
mkdir -p "$cache_root/services/race-component/.helm/charts"
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
_OBS_CREATED_CACHE_IDENTITIES=()
|
|
_OBS_CREATED_CACHE_TYPES=()
|
|
expect_stateful_pass 'race cache created' _observability_create_cache_directory \
|
|
services/race-component chart 1.0.0
|
|
race_cache="${_OBS_CREATED_CACHE_DIRS[0]}"
|
|
race_identity="${_OBS_CREATED_CACHE_IDENTITIES[0]}"
|
|
mkdir "$race_cache/owned-subdirectory"
|
|
write_file "$race_cache/owned-subdirectory/owned-marker" 'OWNED-CACHE-CONTENT'
|
|
cleanup_race_hook="$fixture_dir/cache-cleanup-race-hook.sh"
|
|
write_file "$cleanup_race_hook" '#!/usr/bin/env bash' 'set -Eeuo pipefail' \
|
|
'replacement="$1"' 'mkdir -m 0700 -- "$replacement"' \
|
|
'printf "%s\n" REPLACEMENT-MUST-SURVIVE >"$replacement/replacement-marker"'
|
|
chmod 0700 "$cleanup_race_hook"
|
|
expect_stateful_pass 'owned cache cleanup is identity-bound across replacement race' \
|
|
_observability_cleanup_owned_cache_directory \
|
|
"$race_cache" "$race_identity" tree "$cleanup_race_hook"
|
|
expect_pass 'replacement directory survives cleanup race' test -d "$race_cache"
|
|
expect_pass 'replacement content survives cleanup race' test \
|
|
"$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
|
|
expect_pass 'owned quarantine is completely cleaned' test -z \
|
|
"$(find "$cache_root/services/race-component/.helm/charts" \
|
|
-mindepth 1 -maxdepth 1 ! -name chart-1.0.0 -print -quit)"
|
|
|
|
run_cleanup_exit_gate() (
|
|
_OBS_WORK=''
|
|
trap _observability_cleanup EXIT
|
|
:
|
|
)
|
|
expect_fail 'cleanup identity mismatch fails the successful exit gate' run_cleanup_exit_gate
|
|
expect_pass 'exit-gate mismatch still preserves replacement' test \
|
|
"$(cat "$race_cache/replacement-marker")" = REPLACEMENT-MUST-SURVIVE
|
|
_OBS_CREATED_CACHE_DIRS=()
|
|
_OBS_CREATED_CACHE_IDENTITIES=()
|
|
_OBS_CREATED_CACHE_TYPES=()
|
|
|
|
handoff_work="$fixture_dir/handoff-work"
|
|
mkdir -m 0700 "$handoff_work"
|
|
for handoff_name in namespaces crds storage kps loki tempo alloy node-exporter grafana blackbox core-policies targets core-rules alerting dashboards aggregate; do
|
|
write_file "$handoff_work/$handoff_name.yaml" "fixture-$handoff_name"
|
|
chmod 0600 "$handoff_work/$handoff_name.yaml"
|
|
done
|
|
write_file "$handoff_work/resource-index.tsv" 'fixture-index'
|
|
chmod 0600 "$handoff_work/resource-index.tsv"
|
|
|
|
published_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$published_handoff")
|
|
chmod 0700 "$published_handoff"
|
|
published_identity="$(pin_handoff_directory "$published_handoff")"
|
|
expect_pass 'atomic no-clobber handoff publication' publish_verified_handoff \
|
|
"$published_handoff" "$published_identity" "$handoff_work"
|
|
expect_pass 'exact handoff entry set accepted' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
|
|
ln -s /dev/null "$published_handoff/extra-symlink"
|
|
expect_fail 'extra handoff symlink rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rm -f -- "$published_handoff/extra-symlink"
|
|
mkdir "$published_handoff/extra-directory"
|
|
expect_fail 'extra handoff directory rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rmdir "$published_handoff/extra-directory"
|
|
mv "$published_handoff/aggregate.yaml" "$fixture_dir/aggregate-handoff-saved"
|
|
ln -s /dev/null "$published_handoff/aggregate.yaml"
|
|
expect_fail 'exact handoff entry symlink rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rm -f -- "$published_handoff/aggregate.yaml"
|
|
mv "$fixture_dir/aggregate-handoff-saved" "$published_handoff/aggregate.yaml"
|
|
mv "$published_handoff/core-rules.yaml" "$fixture_dir/core-rules-handoff-saved"
|
|
mkdir "$published_handoff/core-rules.yaml"
|
|
expect_fail 'exact handoff entry directory rejected' validate_handoff_entry_set \
|
|
"$published_handoff" "$published_identity"
|
|
rmdir "$published_handoff/core-rules.yaml"
|
|
mv "$fixture_dir/core-rules-handoff-saved" "$published_handoff/core-rules.yaml"
|
|
|
|
replaced_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$replaced_handoff")
|
|
chmod 0700 "$replaced_handoff"
|
|
replaced_identity="$(pin_handoff_directory "$replaced_handoff")"
|
|
replaced_original="$replaced_handoff.original"
|
|
mv "$replaced_handoff" "$replaced_original"
|
|
handoff_dirs+=("$replaced_original")
|
|
mkdir -m 0700 "$replaced_handoff"
|
|
expect_fail 'replaced handoff identity rejected before writes' publish_verified_handoff \
|
|
"$replaced_handoff" "$replaced_identity" "$handoff_work"
|
|
expect_pass 'replacement remains empty' test -z "$(find "$replaced_handoff" -mindepth 1 -maxdepth 1 -print -quit)"
|
|
|
|
occupied_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
handoff_dirs+=("$occupied_handoff")
|
|
chmod 0700 "$occupied_handoff"
|
|
occupied_identity="$(pin_handoff_directory "$occupied_handoff")"
|
|
write_file "$occupied_handoff/namespaces.yaml" 'DO-NOT-CLOBBER'
|
|
expect_fail 'nonempty handoff rejected immediately before writes' publish_verified_handoff \
|
|
"$occupied_handoff" "$occupied_identity" "$handoff_work"
|
|
expect_pass 'existing handoff file is not clobbered' test \
|
|
"$(cat "$occupied_handoff/namespaces.yaml")" = DO-NOT-CLOBBER
|
|
|
|
child_a="$fixture_dir/child-a.yaml"
|
|
child_b="$fixture_dir/child-b.yaml"
|
|
aggregate="$fixture_dir/aggregate.yaml"
|
|
write_file "$child_a" 'apiVersion: v1' 'kind: ConfigMap' 'metadata:' \
|
|
' name: a' ' namespace: observability' 'data:' ' value: one'
|
|
write_file "$child_b" 'apiVersion: v1' 'kind: Service' 'metadata:' \
|
|
' name: b' ' namespace: observability' 'spec:' ' type: ClusterIP' \
|
|
' ports:' ' - port: 80'
|
|
{ cat "$child_a"; printf '%s\n' '---'; cat "$child_b"; } >"$aggregate"
|
|
index="$fixture_dir/resource-index.tsv"
|
|
expect_pass 'ordered resource index' build_resource_index "$index" \
|
|
"kps:$child_a" "loki:$child_b"
|
|
expected_index=$'1\tkps\tv1\tConfigMap\tobservability\ta\t'
|
|
if [[ "$(sed -n '2p' "$index")" != "$expected_index"* ]]; then
|
|
printf 'FAIL: resource index order or schema differs\n' >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
expect_pass 'child aggregate equivalence' assert_child_aggregate_equivalence \
|
|
"$aggregate" "kps:$child_a" "loki:$child_b"
|
|
|
|
aggregate_missing="$fixture_dir/aggregate-missing.yaml"
|
|
cp "$child_a" "$aggregate_missing"
|
|
expect_fail 'aggregate missing resource' assert_child_aggregate_equivalence \
|
|
"$aggregate_missing" "kps:$child_a" "loki:$child_b"
|
|
|
|
aggregate_drift="$fixture_dir/aggregate-drift.yaml"
|
|
sed 's/value: one/value: changed/' "$aggregate" >"$aggregate_drift"
|
|
expect_fail 'aggregate canonical hash drift' assert_child_aggregate_equivalence \
|
|
"$aggregate_drift" "kps:$child_a" "loki:$child_b"
|
|
|
|
duplicate_child="$fixture_dir/duplicate-child.yaml"
|
|
cp "$child_a" "$duplicate_child"
|
|
expect_fail 'duplicate child tuple' assert_child_aggregate_equivalence \
|
|
"$aggregate" "kps:$child_a" "loki:$child_b" "tempo:$duplicate_child"
|
|
|
|
assert_task2_manifests() {
|
|
local manifest_root="${1:-$ROOT}"
|
|
python3 - "$manifest_root" <<'PY'
|
|
import pathlib
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
|
|
root = pathlib.Path(sys.argv[1])
|
|
namespace_root = root / "infrastructure/namespaces/observability"
|
|
storage_root = root / "infrastructure/storage/observability-local-pv"
|
|
|
|
|
|
def load_documents(path):
|
|
with path.open(encoding="utf-8") as stream:
|
|
return [value for value in yaml.safe_load_all(stream) if value is not None]
|
|
|
|
|
|
namespace_kustomization = yaml.safe_load(
|
|
(namespace_root / "kustomization.yaml").read_text(encoding="utf-8")
|
|
)
|
|
assert namespace_kustomization["resources"] == [
|
|
"observability.yaml",
|
|
"observability-agent.yaml",
|
|
]
|
|
namespaces = []
|
|
for resource in namespace_kustomization["resources"]:
|
|
namespaces.extend(load_documents(namespace_root / resource))
|
|
assert len(namespaces) == 2
|
|
assert [item["kind"] for item in namespaces].count("Namespace") == 2
|
|
assert len({item["metadata"]["name"] for item in namespaces}) == 2
|
|
by_name = {item["metadata"]["name"]: item for item in namespaces}
|
|
assert set(by_name) == {"observability", "observability-agent"}
|
|
for item in by_name.values():
|
|
assert item["metadata"]["labels"]["observability.hyeonworks.com/enabled"] == "true"
|
|
|
|
for level in ("enforce", "audit", "warn"):
|
|
labels = by_name["observability"]["metadata"]["labels"]
|
|
assert labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
|
|
assert labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
|
|
agent_labels = by_name["observability-agent"]["metadata"]["labels"]
|
|
assert agent_labels["pod-security.kubernetes.io/enforce"] == "privileged"
|
|
assert agent_labels["pod-security.kubernetes.io/enforce-version"] == "latest"
|
|
for level in ("audit", "warn"):
|
|
assert agent_labels[f"pod-security.kubernetes.io/{level}"] == "restricted"
|
|
assert agent_labels[f"pod-security.kubernetes.io/{level}-version"] == "latest"
|
|
|
|
assert (namespace_root / "discovery-namespaces.txt").read_text(encoding="utf-8") == (
|
|
"kube-system\ngitea\nkeycloak\nplatform-data\nobject-storage\n"
|
|
)
|
|
|
|
storage_kustomization = yaml.safe_load(
|
|
(storage_root / "kustomization.yaml").read_text(encoding="utf-8")
|
|
)
|
|
assert storage_kustomization["resources"] == [
|
|
"storage-classes.yaml",
|
|
"persistent-volumes.yaml",
|
|
]
|
|
storage_items = []
|
|
for resource in storage_kustomization["resources"]:
|
|
storage_items.extend(load_documents(storage_root / resource))
|
|
assert len(storage_items) == 12
|
|
assert [item["kind"] for item in storage_items].count("StorageClass") == 6
|
|
assert [item["kind"] for item in storage_items].count("PersistentVolume") == 6
|
|
assert {item["kind"] for item in storage_items} == {"StorageClass", "PersistentVolume"}
|
|
class_items = [item for item in storage_items if item["kind"] == "StorageClass"]
|
|
volume_items = [item for item in storage_items if item["kind"] == "PersistentVolume"]
|
|
assert len({item["metadata"]["name"] for item in class_items}) == 6
|
|
assert len({item["metadata"]["name"] for item in volume_items}) == 6
|
|
assert len({item["metadata"]["labels"]["app.kubernetes.io/component"] for item in volume_items}) == 6
|
|
classes = {
|
|
item["metadata"]["name"]: item
|
|
for item in class_items
|
|
}
|
|
volumes = {
|
|
item["metadata"]["labels"]["app.kubernetes.io/component"]: item
|
|
for item in volume_items
|
|
}
|
|
expected = {
|
|
"prometheus": ("/srv/k3s/ssd/observability/prometheus", "20Gi", "ssd-local-observability-prometheus-retain"),
|
|
"grafana": ("/srv/k3s/ssd/observability/grafana", "2Gi", "ssd-local-observability-grafana-retain"),
|
|
"alertmanager": ("/srv/k3s/ssd/observability/alertmanager", "1Gi", "ssd-local-observability-alertmanager-retain"),
|
|
"alloy": ("/srv/k3s/ssd/observability/alloy", "2Gi", "ssd-local-observability-alloy-retain"),
|
|
"loki": ("/srv/k3s/ssd/observability/loki", "5Gi", "ssd-local-observability-loki-retain"),
|
|
"tempo": ("/srv/k3s/ssd/observability/tempo", "5Gi", "ssd-local-observability-tempo-retain"),
|
|
}
|
|
assert set(classes) == {row[2] for row in expected.values()}
|
|
assert set(volumes) == set(expected)
|
|
for owner, (path, capacity, class_name) in expected.items():
|
|
storage_class = classes[class_name]
|
|
assert storage_class["provisioner"] == "kubernetes.io/no-provisioner"
|
|
assert storage_class["reclaimPolicy"] == "Retain"
|
|
assert storage_class["volumeBindingMode"] == "WaitForFirstConsumer"
|
|
assert storage_class["allowVolumeExpansion"] is False
|
|
volume = volumes[owner]
|
|
spec = volume["spec"]
|
|
assert spec["capacity"]["storage"] == capacity
|
|
assert spec["accessModes"] == ["ReadWriteOnce"]
|
|
assert spec["persistentVolumeReclaimPolicy"] == "Retain"
|
|
assert spec["storageClassName"] == class_name
|
|
assert spec["local"]["path"] == path
|
|
expression = spec["nodeAffinity"]["required"]["nodeSelectorTerms"][0]["matchExpressions"][0]
|
|
assert expression == {
|
|
"key": "kubernetes.io/hostname",
|
|
"operator": "In",
|
|
"values": ["donghyeon-system-product-name"],
|
|
}
|
|
actual_capacity = 0
|
|
for volume in volume_items:
|
|
value = volume["spec"]["capacity"]["storage"]
|
|
assert isinstance(value, str) and value.endswith("Gi") and value[:-2].isdigit()
|
|
actual_capacity += int(value[:-2])
|
|
assert actual_capacity == 35
|
|
PY
|
|
}
|
|
|
|
expect_pass 'Task 2 namespace and 35Gi Local PV matrix' assert_task2_manifests
|
|
|
|
expect_any_failure() {
|
|
local name="$1"
|
|
shift
|
|
if "$@" >"$fixture_dir/expected-any-failure.out" 2>&1; then
|
|
printf 'FAIL (expected any failure): %s\n' "$name" >&2
|
|
exit 1
|
|
fi
|
|
pass_count=$((pass_count + 1))
|
|
}
|
|
|
|
task2_manifest_fixture() {
|
|
local destination="$1"
|
|
mkdir -p "$destination/infrastructure/namespaces" "$destination/infrastructure/storage"
|
|
cp -R "$ROOT/infrastructure/namespaces/observability" \
|
|
"$destination/infrastructure/namespaces/observability"
|
|
cp -R "$ROOT/infrastructure/storage/observability-local-pv" \
|
|
"$destination/infrastructure/storage/observability-local-pv"
|
|
}
|
|
|
|
duplicate_manifest_root="$fixture_dir/task2-manifest-duplicate"
|
|
task2_manifest_fixture "$duplicate_manifest_root"
|
|
{
|
|
printf '%s\n' '---'
|
|
sed -n '1,/^---$/p' \
|
|
"$ROOT/infrastructure/storage/observability-local-pv/storage-classes.yaml" | sed '$d'
|
|
} >>"$duplicate_manifest_root/infrastructure/storage/observability-local-pv/storage-classes.yaml"
|
|
expect_any_failure 'duplicate StorageClass object is rejected' \
|
|
assert_task2_manifests "$duplicate_manifest_root"
|
|
|
|
extra_kind_root="$fixture_dir/task2-manifest-extra-kind"
|
|
task2_manifest_fixture "$extra_kind_root"
|
|
write_file "$extra_kind_root/infrastructure/storage/observability-local-pv/extra.yaml" \
|
|
'apiVersion: v1' 'kind: ConfigMap' 'metadata:' ' name: unexpected'
|
|
printf '%s\n' ' - extra.yaml' >> \
|
|
"$extra_kind_root/infrastructure/storage/observability-local-pv/kustomization.yaml"
|
|
expect_any_failure 'extra manifest kind is rejected' \
|
|
assert_task2_manifests "$extra_kind_root"
|
|
|
|
capacity_drift_root="$fixture_dir/task2-manifest-capacity-drift"
|
|
task2_manifest_fixture "$capacity_drift_root"
|
|
sed -i '0,/storage: 20Gi/s//storage: 19Gi/' \
|
|
"$capacity_drift_root/infrastructure/storage/observability-local-pv/persistent-volumes.yaml"
|
|
expect_any_failure 'actual rendered capacity drift is rejected' \
|
|
assert_task2_manifests "$capacity_drift_root"
|
|
|
|
readonly PREPARE_OBSERVABILITY_PATHS="$ROOT/scripts/bootstrap/prepare-observability-local-paths.sh"
|
|
source_options_before="$-"
|
|
# shellcheck source=/dev/null
|
|
source "$PREPARE_OBSERVABILITY_PATHS"
|
|
expect_pass 'path preparation script is source-safe' test "$-" = "$source_options_before"
|
|
|
|
olp_fixture="$fixture_dir/observability-local-paths"
|
|
olp_root="$olp_fixture/root"
|
|
olp_ssd="$olp_fixture/srv/k3s/ssd"
|
|
olp_aistor="$olp_fixture/srv/k3s/aistor"
|
|
olp_external="$olp_fixture/external"
|
|
mkdir -p "$olp_root" "$olp_ssd/observability" "$olp_aistor" "$olp_external"
|
|
|
|
olp_available='64424509440'
|
|
olp_uses=('0' '0' '0' '0' '0' '0')
|
|
olp_context='default'
|
|
olp_node='donghyeon-system-product-name'
|
|
olp_confirmation='APPLY default'
|
|
olp_helper_audit="$olp_fixture/helper-audit.log"
|
|
olp_helper_race=''
|
|
olp_helper_signal=''
|
|
olp_helper_signal_target=''
|
|
|
|
_olp_root_facts() { printf '%s\t%s\n' '/dev/root' 'ext4'; }
|
|
_olp_available_bytes() { printf '%s\n' "$olp_available"; }
|
|
_olp_current_context() { printf '%s\n' "$olp_context"; }
|
|
_olp_node_names() { printf '%s\n' "$olp_node"; }
|
|
_olp_read_confirmation() { printf '%s\n' "$olp_confirmation"; }
|
|
|
|
_olp_fixture_privileged_helper() {
|
|
if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]] || \
|
|
[[ "$2" != "$olp_root" || "$3" != "$olp_ssd" || "$4" != "$olp_aistor" ]]; then
|
|
return 97
|
|
fi
|
|
local program
|
|
program="$(_olp_helper_program)" || return
|
|
OLP_HELPER_FIXTURE_ROOT="$olp_fixture" \
|
|
OLP_HELPER_FIXTURE_VIRTUAL_AISTOR=1 \
|
|
OLP_HELPER_FIXTURE_OWNERS=1 \
|
|
OLP_HELPER_FIXTURE_AUDIT="$olp_helper_audit" \
|
|
OLP_HELPER_FIXTURE_RACE="$olp_helper_race" \
|
|
OLP_HELPER_FIXTURE_EXTERNAL="$olp_external" \
|
|
OLP_HELPER_FIXTURE_SIGNAL="$olp_helper_signal" \
|
|
OLP_HELPER_FIXTURE_SIGNAL_TARGET="$olp_helper_signal_target" \
|
|
/usr/bin/python3 - "$@" <<<"$program"
|
|
}
|
|
_olp_privileged_helper() { _olp_fixture_privileged_helper "$@"; }
|
|
|
|
dry_output="$fixture_dir/observability-dry-run.out"
|
|
expect_stateful_pass 'default dry-run emits facts without mutation' \
|
|
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor"
|
|
observability_local_paths_dry_run "$olp_root" "$olp_ssd" "$olp_aistor" >"$dry_output"
|
|
expect_pass 'dry-run leaves every planned path absent' test -z \
|
|
"$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)"
|
|
expect_pass 'dry-run prints sanitized root filesystem facts' test "$(sed -n '1,3p' "$dry_output")" = \
|
|
$'Root SSD device: /dev/root\nRoot SSD filesystem: ext4\nRoot SSD available bytes: 64424509440'
|
|
expect_pass 'dry-run prints all six exact planned fixture paths' test \
|
|
"$(sed -n '4,9p' "$dry_output")" = \
|
|
"$(for owner in prometheus grafana alertmanager alloy loki tempo; do
|
|
printf 'Planned path: %s/observability/%s\n' "$olp_ssd" "$owner"
|
|
done)"
|
|
|
|
assert_olp_execute_fails() {
|
|
local output rc
|
|
set +e
|
|
output="$(observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" 2>&1)"
|
|
rc=$?
|
|
set -e
|
|
[[ "$rc" != 0 && "$output" == ERROR:* ]]
|
|
}
|
|
|
|
assert_olp_execute_rejected() {
|
|
assert_olp_execute_fails
|
|
[[ -z "$(find "$olp_ssd/observability" -mindepth 1 -maxdepth 1 -print -quit)" ]]
|
|
}
|
|
|
|
exact_numeric_reply=$'PREFLIGHT\t53687091200\t1073741824\t0\t0\t0\t0\t0'
|
|
expect_pass '50GiB available and 1GiB aggregate are inclusive' \
|
|
_olp_validate_preflight_reply "$exact_numeric_reply"
|
|
expect_any_failure 'one byte below 50GiB is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091199\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'one target byte above 1GiB is rejected before summing' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t1073741825\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'aggregate one byte above 1GiB is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t536870913\t536870912\t0\t0\t0\t0'
|
|
expect_any_failure 'signed decimal is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t+53687091200\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'leading-zero decimal is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t053687091200\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'multiline numeric protocol is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t0\t0\t0\t0\t0\t0\n0'
|
|
expect_any_failure '2^63 available value is rejected' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t9223372036854775808\t0\t0\t0\t0\t0\t0'
|
|
expect_any_failure 'repeated 2^63 target values fail before mutation' \
|
|
_olp_validate_preflight_reply $'PREFLIGHT\t53687091200\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808\t9223372036854775808'
|
|
|
|
write_file "$olp_external/command-marker" DO-NOT-EXECUTE
|
|
for unexpected in sudo install chown rm curl; do
|
|
set +e
|
|
_olp_fixture_privileged_helper "$unexpected" "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/unexpected-helper-$unexpected.out" 2>&1
|
|
unexpected_rc=$?
|
|
set -e
|
|
expect_pass "unexpected helper action $unexpected returns 97" test "$unexpected_rc" = 97
|
|
expect_pass "unexpected helper action $unexpected executes nothing" test \
|
|
"$(cat "$olp_external/command-marker")" = DO-NOT-EXECUTE
|
|
done
|
|
set +e
|
|
_olp_fixture_privileged_helper apply "$olp_root" "$olp_ssd" \
|
|
>"$fixture_dir/unexpected-helper-argc.out" 2>&1
|
|
unexpected_rc=$?
|
|
set -e
|
|
expect_pass 'unexpected helper argc returns 97' test "$unexpected_rc" = 97
|
|
|
|
olp_context='other'
|
|
expect_pass 'execute rejects non-default context' assert_olp_execute_rejected
|
|
olp_context='default'
|
|
olp_node='other-node'
|
|
expect_pass 'execute rejects any other node set' assert_olp_execute_rejected
|
|
olp_node='donghyeon-system-product-name'
|
|
olp_confirmation='apply default'
|
|
expect_pass 'execute rejects inexact TTY confirmation' assert_olp_execute_rejected
|
|
olp_confirmation='APPLY default'
|
|
|
|
olp_context_calls="$fixture_dir/observability-context-calls"
|
|
write_file "$olp_context_calls" 0
|
|
_olp_current_context() {
|
|
local calls
|
|
calls="$(cat "$olp_context_calls")"
|
|
calls=$((calls + 1))
|
|
printf '%s\n' "$calls" >"$olp_context_calls"
|
|
if (( calls == 1 )); then
|
|
printf '%s\n' default
|
|
else
|
|
printf '%s\n' changed-during-confirmation
|
|
fi
|
|
}
|
|
expect_pass 'execute rechecks context after TTY confirmation' assert_olp_execute_rejected
|
|
olp_context='default'
|
|
_olp_current_context() { printf '%s\n' "$olp_context"; }
|
|
|
|
mkdir "$olp_ssd/observability/prometheus"
|
|
write_file "$olp_ssd/observability/prometheus/existing" DO-NOT-CLOBBER
|
|
expect_pass 'execute rejects nonempty existing target' assert_olp_execute_fails
|
|
expect_pass 'nonempty rejection preserves content' test \
|
|
"$(cat "$olp_ssd/observability/prometheus/existing")" = DO-NOT-CLOBBER
|
|
rm -f -- "$olp_ssd/observability/prometheus/existing"
|
|
rmdir "$olp_ssd/observability/prometheus"
|
|
ln -s "$olp_aistor" "$olp_ssd/observability/prometheus"
|
|
expect_pass 'execute rejects symlink target' assert_olp_execute_fails
|
|
expect_pass 'symlink rejection preserves link' test -L "$olp_ssd/observability/prometheus"
|
|
unlink "$olp_ssd/observability/prometheus"
|
|
|
|
write_file "$olp_helper_audit" START
|
|
expect_stateful_pass 'execute prepares all six paths' \
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor"
|
|
expected_owner_audit="$(printf '%s\n' \
|
|
'set-owner-mode prometheus 1000 2000 0770' \
|
|
'set-owner-mode grafana 472 472 0770' \
|
|
'set-owner-mode alertmanager 1000 2000 0770' \
|
|
'set-owner-mode alloy 473 473 0770' \
|
|
'set-owner-mode loki 10001 10001 0770' \
|
|
'set-owner-mode tempo 10001 10001 0770')"
|
|
expect_pass 'descriptor helper uses exact owner group mode matrix' test \
|
|
"$(sed -n '/^set-owner-mode /p' "$olp_helper_audit")" = "$expected_owner_audit"
|
|
for owner in prometheus grafana alertmanager alloy loki tempo; do
|
|
expect_pass "execute creates $owner path" test -d "$olp_ssd/observability/$owner"
|
|
expect_pass "execute sets $owner mode" test "$(stat -c %a "$olp_ssd/observability/$owner")" = 770
|
|
rmdir "$olp_ssd/observability/$owner"
|
|
done
|
|
|
|
chmod 0751 "$olp_external"
|
|
write_file "$olp_external/race-marker" EXTERNAL-MUST-SURVIVE
|
|
mkdir -m 0755 "$olp_ssd/observability/prometheus"
|
|
olp_helper_race='ancestor'
|
|
set +e
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/observability-ancestor-race.out" 2>&1
|
|
race_rc=$?
|
|
set -e
|
|
expect_pass 'ancestor replacement race is rejected' test "$race_rc" != 0
|
|
expect_pass 'ancestor replacement leaves external marker untouched' test \
|
|
"$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
|
|
expect_pass 'ancestor replacement leaves external metadata untouched' test \
|
|
"$(stat -c %a "$olp_external")" = 751
|
|
expect_pass 'ancestor replacement restores pinned pre-existing metadata' test \
|
|
"$(stat -c %a "$olp_ssd/observability.pinned/prometheus")" = 755
|
|
unlink "$olp_ssd/observability"
|
|
mv "$olp_ssd/observability.pinned" "$olp_ssd/observability"
|
|
|
|
olp_helper_race='target:prometheus'
|
|
set +e
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/observability-target-race.out" 2>&1
|
|
race_rc=$?
|
|
set -e
|
|
expect_pass 'target replacement race is rejected' test "$race_rc" != 0
|
|
expect_pass 'target replacement leaves external marker untouched' test \
|
|
"$(cat "$olp_external/race-marker")" = EXTERNAL-MUST-SURVIVE
|
|
expect_pass 'target replacement leaves external metadata untouched' test \
|
|
"$(stat -c %a "$olp_external")" = 751
|
|
expect_pass 'target replacement restores pinned target metadata' test \
|
|
"$(stat -c %a "$olp_ssd/observability/prometheus.pinned")" = 755
|
|
unlink "$olp_ssd/observability/prometheus"
|
|
mv "$olp_ssd/observability/prometheus.pinned" "$olp_ssd/observability/prometheus"
|
|
rmdir "$olp_ssd/observability/prometheus"
|
|
|
|
for signal_case in HUP:prometheus INT:alloy TERM:alloy; do
|
|
olp_helper_race=''
|
|
olp_helper_signal="${signal_case%%:*}"
|
|
olp_helper_signal_target="${signal_case#*:}"
|
|
write_file "$olp_helper_audit" START
|
|
if [[ "$olp_helper_signal_target" == prometheus ]]; then
|
|
rmdir "$olp_ssd/observability"
|
|
else
|
|
mkdir -m 0755 "$olp_ssd/observability/prometheus"
|
|
fi
|
|
set +e
|
|
observability_local_paths_execute "$olp_root" "$olp_ssd" "$olp_aistor" \
|
|
>"$fixture_dir/observability-signal-${olp_helper_signal}.out" 2>&1
|
|
signal_rc=$?
|
|
set -e
|
|
expect_pass "$olp_helper_signal post-create signal is rejected" test "$signal_rc" != 0
|
|
for owner in grafana alertmanager alloy loki tempo; do
|
|
expect_pass "$olp_helper_signal leaves no created $owner residue" test \
|
|
! -e "$olp_ssd/observability/$owner"
|
|
done
|
|
if [[ "$olp_helper_signal_target" == prometheus ]]; then
|
|
expect_pass "$olp_helper_signal leaves no first-target residue" test \
|
|
! -e "$olp_ssd/observability/prometheus"
|
|
expect_pass "$olp_helper_signal leaves no created parent residue" test \
|
|
! -e "$olp_ssd/observability"
|
|
mkdir "$olp_ssd/observability"
|
|
else
|
|
expect_pass "$olp_helper_signal restores pre-existing mode" test \
|
|
"$(stat -c %a "$olp_ssd/observability/prometheus")" = 755
|
|
expected_restore="$(printf 'restore-chown prometheus %s %s\nrestore-chmod prometheus 0755' \
|
|
"$(id -u)" "$(id -g)")"
|
|
expect_pass "$olp_helper_signal restores chown then chmod in exact order" test \
|
|
"$(tail -n 2 "$olp_helper_audit")" = "$expected_restore"
|
|
rmdir "$olp_ssd/observability/prometheus"
|
|
fi
|
|
done
|
|
olp_helper_signal=''
|
|
olp_helper_signal_target=''
|
|
|
|
printf 'Assertions: %s\n' "$pass_count"
|
|
printf 'OBSERVABILITY CORE RENDER ASSERTION TEST PASS\n'
|
|
|