274 lines
11 KiB
Bash
Executable File
274 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly ALERTING_ROOT="$ROOT/services/observability/alerting"
|
|
readonly ALERTMANAGER_CONFIG="$ALERTING_ROOT/platform-alertmanager.yaml"
|
|
readonly PUBLIC_HTTPS_POLICY="$ALERTING_ROOT/alertmanager-public-https-egress.yaml"
|
|
readonly ALERTING_KUSTOMIZATION="$ALERTING_ROOT/kustomization.yaml"
|
|
readonly OBSERVABILITY_KUSTOMIZATION="$ROOT/services/observability/kustomization.yaml"
|
|
readonly KPS_ROOT="$ROOT/services/observability/kube-prometheus-stack"
|
|
readonly KPS_VALUES="$ROOT/services/observability/kube-prometheus-stack/values/home.yaml"
|
|
|
|
usage() {
|
|
printf 'usage: %s [--server-dry-run]\n' "${0##*/}" >&2
|
|
exit 2
|
|
}
|
|
|
|
server_dry_run=false
|
|
case $# in
|
|
0) ;;
|
|
1)
|
|
[[ "$1" == --server-dry-run ]] || usage
|
|
server_dry_run=true
|
|
;;
|
|
*) usage ;;
|
|
esac
|
|
|
|
python3 - \
|
|
"$ALERTMANAGER_CONFIG" \
|
|
"$PUBLIC_HTTPS_POLICY" \
|
|
"$ALERTING_KUSTOMIZATION" \
|
|
"$OBSERVABILITY_KUSTOMIZATION" \
|
|
"$KPS_VALUES" <<'PY'
|
|
import pathlib
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
|
|
def fail(reason):
|
|
print(f"FAIL: observability alerting contract: {reason}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def load_one(path):
|
|
source = pathlib.Path(path)
|
|
try:
|
|
with source.open(encoding="utf-8") as stream:
|
|
documents = [item for item in yaml.safe_load_all(stream) if item is not None]
|
|
except (OSError, yaml.YAMLError):
|
|
fail(f"{source.name} is missing or invalid YAML")
|
|
if len(documents) != 1 or not isinstance(documents[0], dict):
|
|
fail(f"{source.name} must contain exactly one object")
|
|
return documents[0]
|
|
|
|
|
|
alert_path, policy_path, child_kustomization_path, root_kustomization_path, values_path = sys.argv[1:]
|
|
alert = load_one(alert_path)
|
|
policy = load_one(policy_path)
|
|
child = load_one(child_kustomization_path)
|
|
root = load_one(root_kustomization_path)
|
|
values = load_one(values_path)
|
|
|
|
if alert != {
|
|
"apiVersion": "monitoring.coreos.com/v1alpha1",
|
|
"kind": "AlertmanagerConfig",
|
|
"metadata": {
|
|
"name": "platform-alertmanager",
|
|
"namespace": "observability",
|
|
"labels": {"observability.hyeonworks.com/instance": "home"},
|
|
},
|
|
"spec": {
|
|
"route": {
|
|
"receiver": "platform-slack",
|
|
"groupBy": ["cluster", "namespace", "alertname", "severity"],
|
|
"groupWait": "30s",
|
|
"groupInterval": "5m",
|
|
"repeatInterval": "4h",
|
|
"routes": [
|
|
{
|
|
"receiver": "platform-null",
|
|
"matchers": [
|
|
{
|
|
"name": "alertname",
|
|
"matchType": "=",
|
|
"value": "InfoInhibitor",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
},
|
|
"inhibitRules": [
|
|
{
|
|
"sourceMatch": [
|
|
{
|
|
"name": "alertname",
|
|
"matchType": "=",
|
|
"value": "InfoInhibitor",
|
|
}
|
|
],
|
|
"targetMatch": [
|
|
{
|
|
"name": "severity",
|
|
"matchType": "=",
|
|
"value": "info",
|
|
}
|
|
],
|
|
"equal": ["namespace"],
|
|
}
|
|
],
|
|
"receivers": [
|
|
{"name": "platform-null"},
|
|
{
|
|
"name": "platform-slack",
|
|
"slackConfigs": [
|
|
{
|
|
"apiURL": {
|
|
"name": "alertmanager-slack-webhook",
|
|
"key": "url",
|
|
},
|
|
"sendResolved": True,
|
|
"linkNames": False,
|
|
"mrkdwnIn": ["text", "fields"],
|
|
"fallback": '{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}',
|
|
"title": '{{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}',
|
|
"titleLink": "https://grafana.learn.hyeonworks.com/",
|
|
"color": '{{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}',
|
|
"fields": [
|
|
{
|
|
"title": "Status",
|
|
"value": "{{ .Status | toUpper }}",
|
|
"short": True,
|
|
},
|
|
{
|
|
"title": "Severity",
|
|
"value": "{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}",
|
|
"short": True,
|
|
},
|
|
{
|
|
"title": "Location",
|
|
"value": "{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}",
|
|
"short": True,
|
|
},
|
|
{
|
|
"title": "Alert count",
|
|
"value": "{{ len .Alerts }}",
|
|
"short": True,
|
|
},
|
|
],
|
|
"text": "{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>",
|
|
"footer": "hyeonworks observability · Alertmanager",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
},
|
|
}:
|
|
fail("platform-alertmanager must match the exact global Slack contract")
|
|
|
|
if policy != {
|
|
"apiVersion": "networking.k8s.io/v1",
|
|
"kind": "NetworkPolicy",
|
|
"metadata": {
|
|
"name": "observability-allow-alertmanager-public-https",
|
|
"namespace": "observability",
|
|
},
|
|
"spec": {
|
|
"podSelector": {
|
|
"matchLabels": {
|
|
"app.kubernetes.io/name": "alertmanager",
|
|
"app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager",
|
|
}
|
|
},
|
|
"policyTypes": ["Egress"],
|
|
"egress": [
|
|
{
|
|
"to": [
|
|
{
|
|
"ipBlock": {
|
|
"cidr": "0.0.0.0/0",
|
|
"except": [
|
|
"10.0.0.0/8",
|
|
"100.64.0.0/10",
|
|
"172.16.0.0/12",
|
|
"192.168.0.0/16",
|
|
],
|
|
}
|
|
}
|
|
],
|
|
"ports": [{"protocol": "TCP", "port": 443}],
|
|
}
|
|
],
|
|
},
|
|
}:
|
|
fail("Alertmanager egress must allow only public IPv4 TCP/443")
|
|
|
|
if child.get("apiVersion") != "kustomize.config.k8s.io/v1beta1" or child.get("kind") != "Kustomization":
|
|
fail("alerting kustomization identity is invalid")
|
|
if child.get("namespace") != "observability":
|
|
fail("alerting kustomization namespace must be observability")
|
|
if child.get("resources") != [
|
|
"platform-alertmanager.yaml",
|
|
"alertmanager-public-https-egress.yaml",
|
|
]:
|
|
fail("alerting kustomization resources are not exact")
|
|
if set(child) != {"apiVersion", "kind", "namespace", "resources"}:
|
|
fail("alerting kustomization contains unapproved behavior")
|
|
|
|
root_resources = root.get("resources")
|
|
if not isinstance(root_resources, list) or len(root_resources) != len(set(root_resources)):
|
|
fail("observability root resources must be a unique list")
|
|
for required in ("targets", "rules/platform", "alerting", "dashboards/platform"):
|
|
if root_resources.count(required) != 1:
|
|
fail(f"observability root must include {required} exactly once")
|
|
|
|
try:
|
|
alertmanager_spec = values["alertmanager"]["alertmanagerSpec"]
|
|
except (KeyError, TypeError):
|
|
fail("kube-prometheus-stack alertmanagerSpec is missing")
|
|
if alertmanager_spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}:
|
|
fail("global alertmanagerConfiguration reference is not exact")
|
|
for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"):
|
|
if forbidden in alertmanager_spec:
|
|
fail(f"{forbidden} must remain absent when using the global reference")
|
|
|
|
for document, label in ((alert, "AlertmanagerConfig"), (policy, "NetworkPolicy")):
|
|
if document.get("kind") == "Secret":
|
|
fail(f"{label} source must not contain a Secret")
|
|
|
|
print("OBSERVABILITY ALERTING STATIC CONTRACT PASS")
|
|
PY
|
|
|
|
render_dir="$(mktemp -d /tmp/platform-observability-alerting-test.XXXXXX)"
|
|
cleanup() {
|
|
case "$render_dir" in
|
|
/tmp/platform-observability-alerting-test.*) rm -rf -- "$render_dir" ;;
|
|
esac
|
|
}
|
|
trap cleanup EXIT
|
|
chmod 0700 "$render_dir"
|
|
kubectl kustomize --enable-helm "$KPS_ROOT" >"$render_dir/kps.yaml"
|
|
chmod 0600 "$render_dir/kps.yaml"
|
|
python3 - "$render_dir/kps.yaml" <<'PY'
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as stream:
|
|
items = [item for item in yaml.safe_load_all(stream) if item is not None]
|
|
alertmanagers = [item for item in items if item.get("kind") == "Alertmanager"]
|
|
if len(alertmanagers) != 1:
|
|
raise SystemExit("FAIL: rendered KPS must contain exactly one Alertmanager")
|
|
spec = alertmanagers[0].get("spec") or {}
|
|
if spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}:
|
|
raise SystemExit("FAIL: rendered Alertmanager global reference is not exact")
|
|
for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"):
|
|
if forbidden in spec:
|
|
raise SystemExit(f"FAIL: rendered Alertmanager retains forbidden {forbidden}")
|
|
print("OBSERVABILITY ALERTMANAGER GLOBAL RENDER CONTRACT PASS")
|
|
PY
|
|
|
|
if [[ "$server_dry_run" == true ]]; then
|
|
readonly KUBECTL_BIN="${PLATFORM_KUBECTL_BIN:-/usr/local/bin/kubectl}"
|
|
[[ "$KUBECTL_BIN" == /* && -f "$KUBECTL_BIN" && -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || {
|
|
printf 'FAIL: PLATFORM_KUBECTL_BIN must be an executable absolute regular file\n' >&2
|
|
exit 1
|
|
}
|
|
"$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$ALERTMANAGER_CONFIG" >/dev/null
|
|
"$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$PUBLIC_HTTPS_POLICY" >/dev/null
|
|
printf 'OBSERVABILITY ALERTING SERVER DRY RUN PASS\n'
|
|
fi
|