826 lines
38 KiB
Bash
826 lines
38 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Static renderer and sourceable assertion library for the observability access
|
|
# layer. It intentionally reuses the pinned core renderer for Helm/cache and
|
|
# image/credential boundaries instead of maintaining a second downloader.
|
|
|
|
set -o pipefail
|
|
|
|
readonly ACCESS_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
# shellcheck source=render-observability-core.sh
|
|
source "$ACCESS_ROOT/scripts/validate/render-observability-core.sh"
|
|
|
|
access_output_names() {
|
|
case "$1" in
|
|
grafana) printf '%s\n' grafana.yaml ;;
|
|
blackbox) printf '%s\n' blackbox.yaml ;;
|
|
targets) printf '%s\n' targets.yaml ;;
|
|
rules-alerts) printf '%s\n' dashboards.yaml rules.yaml alertmanager.yaml ;;
|
|
complete) printf '%s\n' grafana.yaml blackbox.yaml targets.yaml dashboards.yaml rules.yaml alertmanager.yaml private-dns.yaml ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
access_component_requires_inventory() {
|
|
[[ "$1" == rules-alerts || "$1" == complete ]]
|
|
}
|
|
|
|
access_secure_render_context() {
|
|
umask 077
|
|
}
|
|
|
|
_access_assert() {
|
|
local mode=$1
|
|
shift
|
|
python3 - "$mode" "$@" <<'PY'
|
|
import ipaddress
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import yaml
|
|
|
|
class AccessSafeLoader(yaml.SafeLoader):
|
|
pass
|
|
|
|
|
|
AccessSafeLoader.add_constructor(
|
|
"tag:yaml.org,2002:value",
|
|
lambda loader, node: loader.construct_scalar(node),
|
|
)
|
|
|
|
mode = sys.argv[1]
|
|
paths = [pathlib.Path(value) for value in sys.argv[2:]]
|
|
|
|
def reject(message):
|
|
print(f"REJECT: {message}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
def load(path):
|
|
with path.open(encoding="utf-8") as stream:
|
|
return [item for item in yaml.load_all(stream, Loader=AccessSafeLoader) if item is not None]
|
|
|
|
def one(items, kind, name, namespace=None):
|
|
matches = [
|
|
item for item in items
|
|
if item.get("kind") == kind
|
|
and (item.get("metadata") or {}).get("name") == name
|
|
and (namespace is None or (item.get("metadata") or {}).get("namespace") == namespace)
|
|
]
|
|
if len(matches) != 1:
|
|
reject(f"expected one {kind}/{name}, found {len(matches)}")
|
|
return matches[0]
|
|
|
|
def labels(item):
|
|
return (item.get("metadata") or {}).get("labels") or {}
|
|
|
|
def exact_policy_specs(items, expected):
|
|
policies = {
|
|
(item.get("metadata") or {}).get("name"): item.get("spec") or {}
|
|
for item in items if item.get("kind") == "NetworkPolicy"
|
|
}
|
|
for name, spec in expected.items():
|
|
if policies.get(name) != spec:
|
|
reject(f"NetworkPolicy spec is not exact: {name}")
|
|
|
|
def selector_matches(selector, pod_labels):
|
|
selector = selector or {}
|
|
for key, value in (selector.get("matchLabels") or {}).items():
|
|
if pod_labels.get(key) != value:
|
|
return False
|
|
for expression in selector.get("matchExpressions") or []:
|
|
key = expression.get("key")
|
|
operator = expression.get("operator")
|
|
values = expression.get("values") or []
|
|
value = pod_labels.get(key)
|
|
if operator == "In" and value not in values:
|
|
return False
|
|
if operator == "NotIn" and value in values:
|
|
return False
|
|
if operator == "Exists" and value is None:
|
|
return False
|
|
if operator == "DoesNotExist" and value is not None:
|
|
return False
|
|
if operator not in {"In", "NotIn", "Exists", "DoesNotExist"}:
|
|
reject("NetworkPolicy selector operator is unsupported")
|
|
return True
|
|
|
|
def exact_selecting_policy_names(items, namespace, pod_labels, expected):
|
|
selected = {
|
|
(item.get("metadata") or {}).get("name")
|
|
for item in items
|
|
if item.get("kind") == "NetworkPolicy"
|
|
and (item.get("metadata") or {}).get("namespace") == namespace
|
|
and selector_matches((item.get("spec") or {}).get("podSelector") or {}, pod_labels)
|
|
}
|
|
if selected != set(expected):
|
|
reject("NetworkPolicy selecting set is not exact")
|
|
|
|
if mode == "grafana":
|
|
items, policies = load(paths[0]), load(paths[1])
|
|
service = one(items, "Service", "grafana", "observability")
|
|
if (service.get("spec") or {}).get("type") != "ClusterIP":
|
|
reject("Grafana Service is not ClusterIP")
|
|
ingress = one(items, "Ingress", "grafana", "observability")
|
|
spec = ingress.get("spec") or {}
|
|
rules = spec.get("rules") or []
|
|
if spec.get("ingressClassName") != "traefik" or spec.get("tls") not in (None, []):
|
|
reject("Grafana Ingress boundary is not exact")
|
|
if len(rules) != 1 or rules[0].get("host") != "grafana.learn.hyeonworks.com":
|
|
reject("Grafana host is not exact")
|
|
deployment = one(items, "Deployment", "grafana", "observability")
|
|
pod = (((deployment.get("spec") or {}).get("template") or {}).get("spec") or {})
|
|
if pod.get("automountServiceAccountToken") is not False:
|
|
reject("Grafana Pod token automount is not disabled")
|
|
containers = {container.get("name"): container for container in pod.get("containers") or []}
|
|
if set(containers) != {"grafana", "grafana-sc-dashboard"}:
|
|
reject("Grafana container set is not exact")
|
|
token_volume = "dashboard-sidecar-api-access"
|
|
main_mounts = {mount.get("name") for mount in containers["grafana"].get("volumeMounts") or []}
|
|
side_mounts = {mount.get("name") for mount in containers["grafana-sc-dashboard"].get("volumeMounts") or []}
|
|
if token_volume in main_mounts or token_volume not in side_mounts:
|
|
reject("dashboard API token mount is not sidecar-only")
|
|
volume = next((volume for volume in pod.get("volumes") or [] if volume.get("name") == token_volume), None)
|
|
sources = (((volume or {}).get("projected") or {}).get("sources") or [])
|
|
if len(sources) != 3 or not any("serviceAccountToken" in source for source in sources) \
|
|
or not any("configMap" in source for source in sources) \
|
|
or not any("downwardAPI" in source for source in sources):
|
|
reject("sidecar projected API volume is not exact")
|
|
one(items, "Role", "grafana-dashboard-sidecar", "observability")
|
|
if any(item.get("kind") == "ClusterRole" for item in items):
|
|
reject("cluster-wide Grafana RBAC is forbidden")
|
|
config = one(items, "ConfigMap", "grafana", "observability")
|
|
ini = ((config.get("data") or {}).get("grafana.ini") or "")
|
|
required = (
|
|
"allowed_groups = /platform-observability-admins /platform-observability-viewers",
|
|
"role_attribute_strict = true",
|
|
"allow_assign_grafana_admin = false",
|
|
"&& 'Admin' || contains(groups[*], '/platform-observability-viewers') && 'Viewer' || null",
|
|
)
|
|
if not all(token in ini for token in required) or "&& 'Editor'" in ini or "|| 'Viewer'" in ini:
|
|
reject("Grafana OIDC role mapping is not fail-closed")
|
|
exact_policy_specs(policies, {
|
|
"observability-default-deny": {
|
|
"podSelector": {}, "policyTypes": ["Ingress", "Egress"],
|
|
},
|
|
"observability-allow-grafana-ingress": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
|
|
}},
|
|
"policyTypes": ["Ingress"],
|
|
"ingress": [
|
|
{"from": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}},
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "traefik-kube-system",
|
|
"app.kubernetes.io/name": "traefik",
|
|
}},
|
|
}], "ports": [{"protocol": "TCP", "port": 3000}]},
|
|
{"from": [{"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/name": "prometheus",
|
|
}}}], "ports": [{"protocol": "TCP", "port": 3000}]},
|
|
{"from": [{"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "blackbox-exporter",
|
|
"app.kubernetes.io/name": "prometheus-blackbox-exporter",
|
|
}}}], "ports": [{"protocol": "TCP", "port": 3000}]},
|
|
],
|
|
},
|
|
"observability-allow-grafana-datasources": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
|
|
}},
|
|
"policyTypes": ["Egress"],
|
|
"egress": [
|
|
{"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "prometheus"}}}],
|
|
"ports": [{"protocol": "TCP", "port": 9090}]},
|
|
{"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "loki"}}}],
|
|
"ports": [{"protocol": "TCP", "port": 3100}]},
|
|
{"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "tempo"}}}],
|
|
"ports": [{"protocol": "TCP", "port": 3200}]},
|
|
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}],
|
|
"ports": [{"protocol": "TCP", "port": 443}]},
|
|
],
|
|
},
|
|
"observability-allow-grafana-dashboard-api": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
|
|
}},
|
|
"policyTypes": ["Egress"],
|
|
"egress": [
|
|
{"to": [{"ipBlock": {"cidr": "10.43.0.1/32"}}],
|
|
"ports": [{"protocol": "TCP", "port": 443}]},
|
|
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}],
|
|
"ports": [{"protocol": "TCP", "port": 6443}]},
|
|
],
|
|
},
|
|
})
|
|
exact_selecting_policy_names(policies, "observability", {
|
|
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
|
|
}, {
|
|
"observability-default-deny", "observability-allow-dns",
|
|
"observability-allow-grafana-ingress", "observability-allow-grafana-datasources",
|
|
"observability-allow-grafana-dashboard-api",
|
|
})
|
|
|
|
elif mode == "blackbox":
|
|
items = load(paths[0])
|
|
service = one(items, "Service", "blackbox-exporter", "observability")
|
|
if (service.get("spec") or {}).get("type") != "ClusterIP":
|
|
reject("Blackbox Service is public")
|
|
deployment = one(items, "Deployment", "blackbox-exporter", "observability")
|
|
pod = (((deployment.get("spec") or {}).get("template") or {}).get("spec") or {})
|
|
if pod.get("automountServiceAccountToken") is not False or len(pod.get("containers") or []) != 1:
|
|
reject("Blackbox Pod attack surface is not exact")
|
|
expected = {
|
|
"platform-public-edge": {
|
|
"https://git.learn.hyeonworks.com/api/healthz",
|
|
"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration",
|
|
},
|
|
"platform-private-edge": {
|
|
"https://grafana.learn.hyeonworks.com/", "https://storage-admin.learn.hyeonworks.com/",
|
|
"https://db-admin.learn.hyeonworks.com/",
|
|
},
|
|
"platform-private-internal": {
|
|
"http://grafana.observability.svc.cluster.local/api/health",
|
|
"http://pgadmin.platform-admin.svc.cluster.local/misc/ping",
|
|
"http://minio-aistor-console.object-storage.svc.cluster.local:9090/",
|
|
},
|
|
}
|
|
for name, targets in expected.items():
|
|
probe = one(items, "Probe", name, "observability")
|
|
actual = set((((probe.get("spec") or {}).get("targets") or {}).get("staticConfig") or {}).get("static") or [])
|
|
if actual != targets:
|
|
reject(f"Blackbox target set drifted: {name}")
|
|
exact_policy_specs(items, {
|
|
"observability-allow-prometheus-to-blackbox": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "blackbox-exporter",
|
|
"app.kubernetes.io/name": "prometheus-blackbox-exporter",
|
|
}},
|
|
"policyTypes": ["Ingress"],
|
|
"ingress": [{
|
|
"from": [{"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "observability-core-kube-pr-prometheus",
|
|
"app.kubernetes.io/name": "prometheus",
|
|
}}}],
|
|
"ports": [{"protocol": "TCP", "port": 9115}],
|
|
}],
|
|
},
|
|
"observability-allow-blackbox-egress": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "blackbox-exporter",
|
|
"app.kubernetes.io/name": "prometheus-blackbox-exporter",
|
|
}},
|
|
"policyTypes": ["Egress"],
|
|
"egress": [
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}},
|
|
"podSelector": {"matchLabels": {"k8s-app": "kube-dns"}},
|
|
}], "ports": [{"protocol": "UDP", "port": 53}, {"protocol": "TCP", "port": 53}]},
|
|
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}],
|
|
"ports": [{"protocol": "TCP", "port": 443}]},
|
|
{"to": [{"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
|
|
}}}], "ports": [{"protocol": "TCP", "port": 3000}]},
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "platform-admin"}},
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "pgadmin", "app.kubernetes.io/name": "pgadmin4",
|
|
}},
|
|
}], "ports": [{"protocol": "TCP", "port": 5050}]},
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "object-storage"}},
|
|
"podSelector": {"matchLabels": {"aistor.min.io/objectStore": "minio-aistor"}},
|
|
}], "ports": [{"protocol": "TCP", "port": 9090}]},
|
|
],
|
|
},
|
|
})
|
|
if {
|
|
(item.get("metadata") or {}).get("name")
|
|
for item in items if item.get("kind") == "NetworkPolicy"
|
|
} != {"observability-allow-prometheus-to-blackbox", "observability-allow-blackbox-egress"}:
|
|
reject("Blackbox NetworkPolicy name set is not exact")
|
|
|
|
elif mode == "targets":
|
|
items = load(paths[0])
|
|
monitors = [(item.get("kind"), (item.get("metadata") or {}).get("namespace"), (item.get("metadata") or {}).get("name"))
|
|
for item in items if item.get("kind") in {"ServiceMonitor", "PodMonitor"}]
|
|
if sorted(monitors) != sorted([
|
|
("PodMonitor", "platform-data", "platform-postgres"),
|
|
("ServiceMonitor", "object-storage", "aistor-bucket-usage"),
|
|
]):
|
|
reject("manual target monitor set is not exact")
|
|
exact_policy_specs(items, {
|
|
"traefik-preserve-ingress-and-allow-prometheus-metrics": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik",
|
|
}},
|
|
"policyTypes": ["Ingress"],
|
|
"ingress": [
|
|
{"ports": [
|
|
{"protocol": "TCP", "port": 8000}, {"protocol": "TCP", "port": 8443},
|
|
{"protocol": "TCP", "port": 8080},
|
|
]},
|
|
{"from": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "observability"}},
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "observability-core-kube-pr-prometheus",
|
|
"app.kubernetes.io/name": "prometheus",
|
|
}},
|
|
}], "ports": [{"protocol": "TCP", "port": 9100}]},
|
|
],
|
|
},
|
|
"observability-allow-prometheus-platform-targets": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "observability-core-kube-pr-prometheus",
|
|
"app.kubernetes.io/name": "prometheus",
|
|
}},
|
|
"policyTypes": ["Egress"],
|
|
"egress": [
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "gitea"}},
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "gitea", "app.kubernetes.io/name": "gitea",
|
|
}},
|
|
}], "ports": [{"protocol": "TCP", "port": 3000}]},
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "keycloak"}},
|
|
"podSelector": {"matchLabels": {
|
|
"app": "keycloak", "app.kubernetes.io/instance": "keycloak",
|
|
"app.kubernetes.io/managed-by": "keycloak-operator",
|
|
}},
|
|
}], "ports": [{"protocol": "TCP", "port": 9000}]},
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "platform-data"}},
|
|
"podSelector": {"matchLabels": {"cnpg.io/cluster": "platform-postgres"}},
|
|
}], "ports": [{"protocol": "TCP", "port": 9187}]},
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "object-storage"}},
|
|
"podSelector": {"matchLabels": {"aistor.min.io/objectStore": "minio-aistor"}},
|
|
}], "ports": [{"protocol": "TCP", "port": 9000}]},
|
|
{"to": [{
|
|
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}},
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik",
|
|
}},
|
|
}], "ports": [{"protocol": "TCP", "port": 9100}]},
|
|
],
|
|
},
|
|
})
|
|
if {
|
|
(item.get("metadata") or {}).get("name")
|
|
for item in items if item.get("kind") == "NetworkPolicy"
|
|
} != {
|
|
"traefik-preserve-ingress-and-allow-prometheus-metrics",
|
|
"observability-allow-prometheus-platform-targets",
|
|
}:
|
|
reject("target NetworkPolicy name set is not exact")
|
|
|
|
elif mode == "rules-alerts":
|
|
dashboards, rules, alertmanager = map(load, paths)
|
|
dashboard_names = sorted((item.get("metadata") or {}).get("name") for item in dashboards)
|
|
expected_dashboards = sorted([
|
|
"grafana-dashboard-kubernetes-node", "grafana-dashboard-workload-health",
|
|
"grafana-dashboard-platform-services", "grafana-dashboard-observability-backends",
|
|
"grafana-dashboard-https-endpoints",
|
|
])
|
|
if dashboard_names != expected_dashboards or any(labels(item).get("grafana_dashboard") != "1" for item in dashboards):
|
|
reject("dashboard ConfigMap set is not exact")
|
|
rule_names = sorted((item.get("metadata") or {}).get("name") for item in rules)
|
|
if rule_names != sorted(["platform-aistor-storage-quota", "platform-certificate-probes", "platform-observability-core", "platform-verified-services"]):
|
|
reject("PrometheusRule set is not exact")
|
|
am = one(alertmanager, "Alertmanager", "observability-core-kube-pr-alertmanager", "observability")
|
|
if (((am.get("spec") or {}).get("alertmanagerConfiguration") or {}).get("name")) != "platform-alertmanager":
|
|
reject("global AlertmanagerConfig reference is absent")
|
|
config = one(alertmanager, "AlertmanagerConfig", "platform-alertmanager", "observability")
|
|
receivers = (config.get("spec") or {}).get("receivers") or []
|
|
slack_receiver = next((receiver for receiver in receivers if receiver.get("name") == "platform-slack"), {})
|
|
slack = ((slack_receiver.get("slackConfigs") or [{}])[0])
|
|
api_url = slack.get("apiURL") or {}
|
|
if api_url != {"name": "alertmanager-slack-webhook", "key": "url"} or slack.get("sendResolved") is not True:
|
|
reject("Slack Secret selector or resolved route is not exact")
|
|
one(alertmanager, "NetworkPolicy", "observability-allow-alertmanager-public-https", "observability")
|
|
exact_policy_specs(alertmanager, {
|
|
"observability-allow-alertmanager-public-https": {
|
|
"podSelector": {"matchLabels": {
|
|
"app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager",
|
|
"app.kubernetes.io/name": "alertmanager",
|
|
}},
|
|
"policyTypes": ["Egress"],
|
|
"egress": [{
|
|
"to": [{"ipBlock": {
|
|
"cidr": "0.0.0.0/0",
|
|
"except": ["10.0.0.0/8", "100.64.0.0/10", "172.16.0.0/12", "192.168.0.0/16"],
|
|
}}],
|
|
"ports": [{"protocol": "TCP", "port": 443}],
|
|
}],
|
|
},
|
|
})
|
|
if {
|
|
(item.get("metadata") or {}).get("name")
|
|
for item in alertmanager if item.get("kind") == "NetworkPolicy"
|
|
} != {"observability-allow-alertmanager-public-https"}:
|
|
reject("Alertmanager NetworkPolicy name set is not exact")
|
|
|
|
elif mode == "scope":
|
|
items = []
|
|
for path in paths:
|
|
items.extend(load(path))
|
|
forbidden = re.compile(r"spring|jvm|kafka|batch|backup", re.I)
|
|
for item in items:
|
|
kind = item.get("kind")
|
|
name = (item.get("metadata") or {}).get("name") or ""
|
|
if kind in {"Probe", "ServiceMonitor", "PodMonitor", "PrometheusRule"}:
|
|
if labels(item).get("observability.hyeonworks.com/instance") != "home":
|
|
reject(f"missing monitor/rule instance label: {kind}/{name}")
|
|
semantic_names = [name]
|
|
if kind == "PrometheusRule":
|
|
for group in (item.get("spec") or {}).get("groups") or []:
|
|
semantic_names.append(group.get("name") or "")
|
|
for rule in group.get("rules") or []:
|
|
semantic_names.append(rule.get("alert") or rule.get("record") or "")
|
|
if any(forbidden.search(value) for value in semantic_names):
|
|
reject(f"forbidden product scope: {kind}/{name}")
|
|
if kind in {"ServiceMonitor", "PodMonitor"}:
|
|
namespace = (item.get("metadata") or {}).get("namespace")
|
|
allowed = {
|
|
("observability", "blackbox-exporter"),
|
|
("platform-data", "platform-postgres"),
|
|
("object-storage", "aistor-bucket-usage"),
|
|
}
|
|
if (namespace, name) not in allowed:
|
|
reject(f"out-of-scope access target: {namespace}/{name}")
|
|
|
|
elif mode == "private-dns":
|
|
items = load(paths[0])
|
|
config = one(items, "ConfigMap", "coredns-custom", "kube-system")
|
|
content = "\n".join((config.get("data") or {}).values())
|
|
lan = paths[1].read_text()
|
|
tail = paths[2].read_text()
|
|
hosts = ("git", "id", "storage-admin", "db-admin", "grafana")
|
|
for short in hosts:
|
|
host = f"{short}.learn.hyeonworks.com"
|
|
if f"192.168.0.107 {host}" not in content \
|
|
or f"address=/{host}/192.168.0.107" not in lan \
|
|
or f"address=/{host}/100.92.240.34" not in tail:
|
|
reject(f"private DNS mapping is not exact: {host}")
|
|
if re.search(r"(?:^|\s)(?!192\.168\.0\.107\b)\d+\.\d+\.\d+\.\d+\s+grafana\.learn", content):
|
|
reject("Grafana DNS assumes a non-private address")
|
|
|
|
elif mode == "traefik":
|
|
items = load(paths[0])
|
|
hcc = one(items, "HelmChartConfig", "traefik", "kube-system")
|
|
values = yaml.safe_load((hcc.get("spec") or {}).get("valuesContent") or "") or {}
|
|
web = ((((values.get("ports") or {}).get("web") or {}).get("forwardedHeaders") or {}))
|
|
if web.get("trustedIPs") != ["10.42.0.1/32"] or web.get("insecure") is True:
|
|
reject("Traefik trust overlay is not exact")
|
|
|
|
else:
|
|
reject(f"unknown assertion mode: {mode}")
|
|
PY
|
|
}
|
|
|
|
assert_access_grafana_contract() { _access_assert grafana "$@"; }
|
|
assert_access_blackbox_contract() { _access_assert blackbox "$@"; }
|
|
assert_access_targets_contract() { _access_assert targets "$@"; }
|
|
assert_access_rules_alerts_contract() {
|
|
(( $# == 3 )) || return 1
|
|
_access_assert rules-alerts "$@" || return 1
|
|
assert_alertmanager_routing_contract "$3"
|
|
}
|
|
assert_access_scope_contract() { _access_assert scope "$@"; }
|
|
assert_access_private_dns_contract() { _access_assert private-dns "$@"; }
|
|
assert_access_traefik_trust_contract() { _access_assert traefik "$@"; }
|
|
|
|
validate_access_inventory_root() {
|
|
local root=$1 phase directory expected actual filename extra entries current_uid
|
|
current_uid="$(id -u)" || return 1
|
|
[[ "$root" =~ ^/tmp/platform-observability-metrics\.[A-Za-z0-9]{6}$ ]] || return 1
|
|
[[ -d /tmp && ! -L /tmp && "$(readlink -f -- /tmp)" == /tmp &&
|
|
"$(stat -c %u:%a -- /tmp)" == 0:1777 ]] || return 1
|
|
[[ -d "$root" && ! -L "$root" && "$(readlink -f -- "$root")" == "$root" &&
|
|
"$(stat -c %u:%a -- "$root")" == "$current_uid:700" ]] || return 1
|
|
entries="$(find "$root" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1
|
|
[[ "$entries" == $'post-substrate\ntarget-initial' ]] || return 1
|
|
for phase in target-initial post-substrate; do
|
|
directory="$root/$phase"
|
|
[[ -d "$directory" && ! -L "$directory" && -f "$directory/inventory.json" &&
|
|
-f "$directory/inventory.sha256" && ! -L "$directory/inventory.json" &&
|
|
! -L "$directory/inventory.sha256" ]] || return 1
|
|
[[ "$(readlink -f -- "$directory")" == "$directory" &&
|
|
"$(stat -c %u:%a -- "$directory")" == "$current_uid:700" ]] || return 1
|
|
entries="$(find "$directory" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1
|
|
[[ "$entries" == $'inventory.json\ninventory.sha256' ]] || return 1
|
|
for filename in inventory.json inventory.sha256; do
|
|
[[ -f "$directory/$filename" && ! -L "$directory/$filename" &&
|
|
"$(stat -c %u:%a:%h -- "$directory/$filename")" == "$current_uid:600:1" ]] || return 1
|
|
done
|
|
read -r expected filename extra <"$directory/inventory.sha256"
|
|
[[ "$expected" =~ ^[0-9a-f]{64}$ && "$filename" == inventory.json && -z "${extra:-}" ]] || return 1
|
|
actual="$(sha256sum "$directory/inventory.json" | awk '{print $1}')"
|
|
[[ "$actual" == "$expected" ]] || return 1
|
|
jq -e --arg phase "$phase" '
|
|
.schema == "platform-observability-metric-inventory/v1" and
|
|
.phase == $phase and (.targets | type == "array" and length > 0) and
|
|
all(.targets[]; .health == "up" and .last_error == "" and (.metrics | type == "array" and length > 0))
|
|
' "$directory/inventory.json" >/dev/null || return 1
|
|
done
|
|
}
|
|
|
|
publish_access_outputs() {
|
|
local root=$1 work=$2 component=$3
|
|
local -a names=()
|
|
mapfile -t names < <(access_output_names "$component") || return 2
|
|
(( ${#names[@]} > 0 )) || return 2
|
|
python3 - "$root" "$work" "${names[@]}" <<'PY'
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import stat
|
|
import sys
|
|
|
|
|
|
class Rejected(Exception):
|
|
pass
|
|
|
|
|
|
def reject(reason):
|
|
raise Rejected(reason)
|
|
|
|
|
|
def same_identity(metadata, expected):
|
|
return (metadata.st_dev, metadata.st_ino) == expected
|
|
|
|
|
|
def sha256_descriptor(descriptor):
|
|
digest = hashlib.sha256()
|
|
os.lseek(descriptor, 0, os.SEEK_SET)
|
|
while True:
|
|
chunk = os.read(descriptor, 1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
os.lseek(descriptor, 0, os.SEEK_SET)
|
|
return digest.hexdigest()
|
|
|
|
|
|
root, work, *names = sys.argv[1:]
|
|
root_descriptor = None
|
|
work_descriptor = None
|
|
source_descriptors = {}
|
|
temporary_names = []
|
|
created = {}
|
|
try:
|
|
if os.path.dirname(root) != "/tmp" or not re.fullmatch(
|
|
r"platform-observability-metrics\.[A-Za-z0-9]{6}", os.path.basename(root)
|
|
):
|
|
reject("output root is not the expected direct /tmp handoff")
|
|
tmp_metadata = os.lstat("/tmp")
|
|
root_metadata = os.lstat(root)
|
|
if not stat.S_ISDIR(tmp_metadata.st_mode) or stat.S_ISLNK(tmp_metadata.st_mode):
|
|
reject("/tmp is not a physical directory")
|
|
if tmp_metadata.st_uid != 0 or stat.S_IMODE(tmp_metadata.st_mode) != 0o1777:
|
|
reject("/tmp owner or mode differs")
|
|
if os.path.realpath("/tmp") != "/tmp" or os.path.realpath(root) != root:
|
|
reject("output root lineage is not physical")
|
|
if not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode):
|
|
reject("output root is not a physical directory")
|
|
if root_metadata.st_uid != os.getuid() or stat.S_IMODE(root_metadata.st_mode) != 0o700:
|
|
reject("output root owner or mode differs")
|
|
root_identity = (root_metadata.st_dev, root_metadata.st_ino)
|
|
root_descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
if not same_identity(os.fstat(root_descriptor), root_identity):
|
|
reject("opened output root identity differs")
|
|
|
|
base_entries = {"target-initial", "post-substrate"}
|
|
if set(os.listdir(root_descriptor)) != base_entries:
|
|
reject("output root is not pristine before publication")
|
|
for phase in base_entries:
|
|
phase_metadata = os.stat(phase, dir_fd=root_descriptor, follow_symlinks=False)
|
|
if not stat.S_ISDIR(phase_metadata.st_mode) or phase_metadata.st_uid != os.getuid():
|
|
reject("inventory phase type or owner differs")
|
|
if stat.S_IMODE(phase_metadata.st_mode) != 0o700:
|
|
reject("inventory phase mode differs")
|
|
phase_descriptor = os.open(
|
|
phase, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_descriptor
|
|
)
|
|
try:
|
|
if set(os.listdir(phase_descriptor)) != {"inventory.json", "inventory.sha256"}:
|
|
reject("inventory phase entry set differs")
|
|
for filename in ("inventory.json", "inventory.sha256"):
|
|
metadata = os.stat(filename, dir_fd=phase_descriptor, follow_symlinks=False)
|
|
if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid():
|
|
reject("inventory file type or owner differs")
|
|
if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1:
|
|
reject("inventory file mode or link count differs")
|
|
finally:
|
|
os.close(phase_descriptor)
|
|
|
|
if len(set(names)) != len(names) or any("/" in name or name in {".", ".."} for name in names):
|
|
reject("publication names are malformed")
|
|
if any(name in base_entries for name in names):
|
|
reject("publication name collides with inventory")
|
|
for name in names:
|
|
try:
|
|
os.stat(name, dir_fd=root_descriptor, follow_symlinks=False)
|
|
except FileNotFoundError:
|
|
continue
|
|
reject("output destination already exists")
|
|
|
|
work_metadata = os.lstat(work)
|
|
if not stat.S_ISDIR(work_metadata.st_mode) or stat.S_ISLNK(work_metadata.st_mode):
|
|
reject("verified render directory is not physical")
|
|
if work_metadata.st_uid != os.getuid() or stat.S_IMODE(work_metadata.st_mode) != 0o700:
|
|
reject("verified render directory owner or mode differs")
|
|
work_identity = (work_metadata.st_dev, work_metadata.st_ino)
|
|
work_descriptor = os.open(work, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
if not same_identity(os.fstat(work_descriptor), work_identity):
|
|
reject("opened render directory identity differs")
|
|
source_hashes = {}
|
|
for name in names:
|
|
descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=work_descriptor)
|
|
source_descriptors[name] = descriptor
|
|
metadata = os.fstat(descriptor)
|
|
if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid():
|
|
reject("verified output type or owner differs")
|
|
if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1:
|
|
reject("verified output mode or link count differs")
|
|
source_hashes[name] = sha256_descriptor(descriptor)
|
|
|
|
for position, name in enumerate(names):
|
|
temporary = f".access-handoff-{os.getpid()}-{position}"
|
|
target_descriptor = os.open(
|
|
temporary,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
0o600,
|
|
dir_fd=root_descriptor,
|
|
)
|
|
temporary_names.append(temporary)
|
|
try:
|
|
source_descriptor = source_descriptors[name]
|
|
os.lseek(source_descriptor, 0, os.SEEK_SET)
|
|
while True:
|
|
chunk = os.read(source_descriptor, 1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
view = memoryview(chunk)
|
|
while view:
|
|
view = view[os.write(target_descriptor, view):]
|
|
os.fchmod(target_descriptor, 0o600)
|
|
os.fsync(target_descriptor)
|
|
staged_identity = (os.fstat(target_descriptor).st_dev, os.fstat(target_descriptor).st_ino)
|
|
finally:
|
|
os.close(target_descriptor)
|
|
os.link(
|
|
temporary,
|
|
name,
|
|
src_dir_fd=root_descriptor,
|
|
dst_dir_fd=root_descriptor,
|
|
follow_symlinks=False,
|
|
)
|
|
created[name] = staged_identity
|
|
os.unlink(temporary, dir_fd=root_descriptor)
|
|
temporary_names.remove(temporary)
|
|
|
|
os.fsync(root_descriptor)
|
|
if set(os.listdir(root_descriptor)) != base_entries | set(names):
|
|
reject("published output entry set differs")
|
|
for name in names:
|
|
metadata = os.stat(name, dir_fd=root_descriptor, follow_symlinks=False)
|
|
if not same_identity(metadata, created[name]) or not stat.S_ISREG(metadata.st_mode):
|
|
reject("published output identity or type differs")
|
|
if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600:
|
|
reject("published output owner or mode differs")
|
|
descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root_descriptor)
|
|
try:
|
|
if sha256_descriptor(descriptor) != source_hashes[name]:
|
|
reject("published output content differs")
|
|
finally:
|
|
os.close(descriptor)
|
|
if not same_identity(os.lstat(root), root_identity):
|
|
reject("output root identity changed during publication")
|
|
except BaseException as error:
|
|
if root_descriptor is not None:
|
|
for name, identity in reversed(tuple(created.items())):
|
|
try:
|
|
metadata = os.stat(name, dir_fd=root_descriptor, follow_symlinks=False)
|
|
if same_identity(metadata, identity):
|
|
os.unlink(name, dir_fd=root_descriptor)
|
|
except OSError:
|
|
pass
|
|
for name in reversed(temporary_names):
|
|
try:
|
|
os.unlink(name, dir_fd=root_descriptor)
|
|
except OSError:
|
|
pass
|
|
try:
|
|
os.fsync(root_descriptor)
|
|
except OSError:
|
|
pass
|
|
if isinstance(error, Rejected):
|
|
print(f"REJECT: access output publication failed: {error}", file=sys.stderr)
|
|
else:
|
|
print("REJECT: access output publication failed: filesystem operation failed", file=sys.stderr)
|
|
raise SystemExit(23)
|
|
finally:
|
|
for descriptor in source_descriptors.values():
|
|
try:
|
|
os.close(descriptor)
|
|
except OSError:
|
|
pass
|
|
if work_descriptor is not None:
|
|
os.close(work_descriptor)
|
|
if root_descriptor is not None:
|
|
os.close(root_descriptor)
|
|
PY
|
|
}
|
|
|
|
_access_usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
|
|
bash scripts/validate/render-observability-access.sh \
|
|
--component grafana|blackbox|targets|rules-alerts|complete \
|
|
[--verified-output-dir /tmp/platform-observability-metrics.XXXXXX]
|
|
|
|
Static rendering only; no Kubernetes resource is applied.
|
|
USAGE
|
|
}
|
|
|
|
access_render_main() (
|
|
local component=complete output_root='' argument core_handoff work
|
|
access_secure_render_context
|
|
while (( $# > 0 )); do
|
|
argument=$1
|
|
case "$argument" in
|
|
--component) (( $# >= 2 )) || return 2; component=$2; shift 2 ;;
|
|
--verified-output-dir) (( $# >= 2 )) || return 2; output_root=$2; shift 2 ;;
|
|
-h|--help) _access_usage; return 0 ;;
|
|
*) _access_usage >&2; return 2 ;;
|
|
esac
|
|
done
|
|
access_output_names "$component" >/dev/null || return 2
|
|
if access_component_requires_inventory "$component"; then
|
|
[[ -n "$output_root" ]] || { printf 'REJECT: rules-alerts/complete requires --verified-output-dir\n' >&2; return 1; }
|
|
validate_access_inventory_root "$output_root" || { printf 'REJECT: verified inventory root is invalid\n' >&2; return 1; }
|
|
fi
|
|
work="$(mktemp -d /tmp/platform-observability-access-render.XXXXXX)"
|
|
chmod 0700 "$work"
|
|
core_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
|
|
chmod 0700 "$core_handoff"
|
|
trap '[[ -n ${work:-} && $work == /tmp/platform-observability-access-render.* ]] && rm -rf -- "$work"; [[ -n ${core_handoff:-} && $core_handoff == /tmp/platform-observability-core-apply.* ]] && rm -rf -- "$core_handoff"' EXIT INT TERM
|
|
/usr/bin/bash "$ACCESS_ROOT/scripts/validate/render-observability-core.sh" \
|
|
--verified-output-dir "$core_handoff"
|
|
|
|
cp -- "$core_handoff/grafana.yaml" "$work/grafana.yaml"
|
|
cp -- "$core_handoff/blackbox.yaml" "$work/blackbox.yaml"
|
|
cp -- "$core_handoff/targets.yaml" "$work/targets.yaml"
|
|
cp -- "$core_handoff/dashboards.yaml" "$work/dashboards.yaml"
|
|
cp -- "$core_handoff/core-rules.yaml" "$work/rules.yaml"
|
|
kubectl kustomize "$ACCESS_ROOT/infrastructure/networking/private-dns/kubernetes" >"$work/private-dns.yaml"
|
|
python3 - "$core_handoff/kps.yaml" "$core_handoff/alerting.yaml" "$work/alertmanager.yaml" <<'PY'
|
|
import sys, yaml
|
|
|
|
class AccessExtractionSafeLoader(yaml.SafeLoader):
|
|
pass
|
|
|
|
AccessExtractionSafeLoader.add_constructor(
|
|
"tag:yaml.org,2002:value",
|
|
lambda loader, node: loader.construct_scalar(node),
|
|
)
|
|
|
|
items=[]
|
|
for path in sys.argv[1:3]:
|
|
with open(path, encoding="utf-8") as stream:
|
|
items.extend(
|
|
item
|
|
for item in yaml.load_all(stream, Loader=AccessExtractionSafeLoader)
|
|
if item is not None
|
|
)
|
|
selected=[item for item in items if item.get("kind") in {"Alertmanager", "AlertmanagerConfig", "NetworkPolicy"}]
|
|
with open(sys.argv[3], "w", encoding="utf-8") as stream:
|
|
yaml.safe_dump_all(selected, stream, explicit_start=True, sort_keys=False)
|
|
PY
|
|
assert_access_grafana_contract "$work/grafana.yaml" "$core_handoff/core-policies.yaml"
|
|
assert_pinned_images "$work/grafana.yaml"
|
|
assert_access_blackbox_contract "$work/blackbox.yaml"
|
|
assert_pinned_images "$work/blackbox.yaml"
|
|
assert_access_targets_contract "$work/targets.yaml"
|
|
assert_access_rules_alerts_contract "$work/dashboards.yaml" "$work/rules.yaml" "$work/alertmanager.yaml"
|
|
assert_access_scope_contract "$work/blackbox.yaml" "$work/targets.yaml" "$work/rules.yaml"
|
|
assert_access_private_dns_contract "$work/private-dns.yaml" \
|
|
"$ACCESS_ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf" \
|
|
"$ACCESS_ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf"
|
|
assert_no_credentials "$work"/*.yaml
|
|
|
|
if [[ -n "$output_root" ]]; then
|
|
publish_access_outputs "$output_root" "$work" "$component"
|
|
fi
|
|
printf 'OBSERVABILITY ACCESS STATIC RENDER PASS\n'
|
|
)
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
set -Eeuo pipefail
|
|
access_render_main "$@"
|
|
fi
|