#!/usr/bin/env bash set -Eeuo pipefail readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly ALERTING_ROOT="$ROOT/services/observability/alerting" readonly ALERTMANAGER_CONFIG="$ALERTING_ROOT/platform-alertmanager.yaml" readonly PUBLIC_HTTPS_POLICY="$ALERTING_ROOT/alertmanager-public-https-egress.yaml" readonly ALERTING_KUSTOMIZATION="$ALERTING_ROOT/kustomization.yaml" readonly OBSERVABILITY_KUSTOMIZATION="$ROOT/services/observability/kustomization.yaml" readonly KPS_ROOT="$ROOT/services/observability/kube-prometheus-stack" readonly KPS_VALUES="$ROOT/services/observability/kube-prometheus-stack/values/home.yaml" usage() { printf 'usage: %s [--server-dry-run]\n' "${0##*/}" >&2 exit 2 } server_dry_run=false case $# in 0) ;; 1) [[ "$1" == --server-dry-run ]] || usage server_dry_run=true ;; *) usage ;; esac python3 - \ "$ALERTMANAGER_CONFIG" \ "$PUBLIC_HTTPS_POLICY" \ "$ALERTING_KUSTOMIZATION" \ "$OBSERVABILITY_KUSTOMIZATION" \ "$KPS_VALUES" <<'PY' import pathlib import sys import yaml def fail(reason): print(f"FAIL: observability alerting contract: {reason}", file=sys.stderr) raise SystemExit(1) def load_one(path): source = pathlib.Path(path) try: with source.open(encoding="utf-8") as stream: documents = [item for item in yaml.safe_load_all(stream) if item is not None] except (OSError, yaml.YAMLError): fail(f"{source.name} is missing or invalid YAML") if len(documents) != 1 or not isinstance(documents[0], dict): fail(f"{source.name} must contain exactly one object") return documents[0] alert_path, policy_path, child_kustomization_path, root_kustomization_path, values_path = sys.argv[1:] alert = load_one(alert_path) policy = load_one(policy_path) child = load_one(child_kustomization_path) root = load_one(root_kustomization_path) values = load_one(values_path) if alert != { "apiVersion": "monitoring.coreos.com/v1alpha1", "kind": "AlertmanagerConfig", "metadata": { "name": "platform-alertmanager", "namespace": "observability", "labels": {"observability.hyeonworks.com/instance": "home"}, }, "spec": { "route": { "receiver": "platform-slack", "groupBy": ["cluster", "namespace", "alertname", "severity"], "groupWait": "30s", "groupInterval": "5m", "repeatInterval": "4h", "routes": [ { "receiver": "platform-null", "matchers": [ { "name": "alertname", "matchType": "=", "value": "InfoInhibitor", } ], } ], }, "inhibitRules": [ { "sourceMatch": [ { "name": "alertname", "matchType": "=", "value": "InfoInhibitor", } ], "targetMatch": [ { "name": "severity", "matchType": "=", "value": "info", } ], "equal": ["namespace"], } ], "receivers": [ {"name": "platform-null"}, { "name": "platform-slack", "slackConfigs": [ { "apiURL": { "name": "alertmanager-slack-webhook", "key": "url", }, "sendResolved": True, "linkNames": False, "mrkdwnIn": ["text", "fields"], "fallback": '{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}', "title": '{{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}', "titleLink": "https://grafana.learn.hyeonworks.com/", "color": '{{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}', "fields": [ { "title": "Status", "value": "{{ .Status | toUpper }}", "short": True, }, { "title": "Severity", "value": "{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}", "short": True, }, { "title": "Location", "value": "{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}", "short": True, }, { "title": "Alert count", "value": "{{ len .Alerts }}", "short": True, }, ], "text": "{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n · ", "footer": "hyeonworks observability · Alertmanager", } ], } ], }, }: fail("platform-alertmanager must match the exact global Slack contract") if policy != { "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": { "name": "observability-allow-alertmanager-public-https", "namespace": "observability", }, "spec": { "podSelector": { "matchLabels": { "app.kubernetes.io/name": "alertmanager", "app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager", } }, "policyTypes": ["Egress"], "egress": [ { "to": [ { "ipBlock": { "cidr": "0.0.0.0/0", "except": [ "10.0.0.0/8", "100.64.0.0/10", "172.16.0.0/12", "192.168.0.0/16", ], } } ], "ports": [{"protocol": "TCP", "port": 443}], } ], }, }: fail("Alertmanager egress must allow only public IPv4 TCP/443") if child.get("apiVersion") != "kustomize.config.k8s.io/v1beta1" or child.get("kind") != "Kustomization": fail("alerting kustomization identity is invalid") if child.get("namespace") != "observability": fail("alerting kustomization namespace must be observability") if child.get("resources") != [ "platform-alertmanager.yaml", "alertmanager-public-https-egress.yaml", ]: fail("alerting kustomization resources are not exact") if set(child) != {"apiVersion", "kind", "namespace", "resources"}: fail("alerting kustomization contains unapproved behavior") root_resources = root.get("resources") if not isinstance(root_resources, list) or len(root_resources) != len(set(root_resources)): fail("observability root resources must be a unique list") for required in ("targets", "rules/platform", "alerting", "dashboards/platform"): if root_resources.count(required) != 1: fail(f"observability root must include {required} exactly once") try: alertmanager_spec = values["alertmanager"]["alertmanagerSpec"] except (KeyError, TypeError): fail("kube-prometheus-stack alertmanagerSpec is missing") if alertmanager_spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}: fail("global alertmanagerConfiguration reference is not exact") for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"): if forbidden in alertmanager_spec: fail(f"{forbidden} must remain absent when using the global reference") for document, label in ((alert, "AlertmanagerConfig"), (policy, "NetworkPolicy")): if document.get("kind") == "Secret": fail(f"{label} source must not contain a Secret") print("OBSERVABILITY ALERTING STATIC CONTRACT PASS") PY render_dir="$(mktemp -d /tmp/platform-observability-alerting-test.XXXXXX)" cleanup() { case "$render_dir" in /tmp/platform-observability-alerting-test.*) rm -rf -- "$render_dir" ;; esac } trap cleanup EXIT chmod 0700 "$render_dir" kubectl kustomize --enable-helm "$KPS_ROOT" >"$render_dir/kps.yaml" chmod 0600 "$render_dir/kps.yaml" python3 - "$render_dir/kps.yaml" <<'PY' import sys import yaml with open(sys.argv[1], encoding="utf-8") as stream: items = [item for item in yaml.safe_load_all(stream) if item is not None] alertmanagers = [item for item in items if item.get("kind") == "Alertmanager"] if len(alertmanagers) != 1: raise SystemExit("FAIL: rendered KPS must contain exactly one Alertmanager") spec = alertmanagers[0].get("spec") or {} if spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}: raise SystemExit("FAIL: rendered Alertmanager global reference is not exact") for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"): if forbidden in spec: raise SystemExit(f"FAIL: rendered Alertmanager retains forbidden {forbidden}") print("OBSERVABILITY ALERTMANAGER GLOBAL RENDER CONTRACT PASS") PY if [[ "$server_dry_run" == true ]]; then readonly KUBECTL_BIN="${PLATFORM_KUBECTL_BIN:-/usr/local/bin/kubectl}" [[ "$KUBECTL_BIN" == /* && -f "$KUBECTL_BIN" && -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || { printf 'FAIL: PLATFORM_KUBECTL_BIN must be an executable absolute regular file\n' >&2 exit 1 } "$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$ALERTMANAGER_CONFIG" >/dev/null "$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$PUBLIC_HTTPS_POLICY" >/dev/null printf 'OBSERVABILITY ALERTING SERVER DRY RUN PASS\n' fi