1809 lines
95 KiB
Bash
1809 lines
95 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# End-to-end acceptance for the private observability access and alerting
|
|
# cutover. No-argument mode is intentionally a no-contact plan printer.
|
|
set -Eeuo pipefail
|
|
set +x
|
|
umask 077
|
|
|
|
readonly EXPECTED_CONTEXT='default'
|
|
readonly EXPECTED_API_SERVER='https://127.0.0.1:6443'
|
|
readonly EXPECTED_NODE='donghyeon-system-product-name'
|
|
readonly NAMESPACE='observability'
|
|
readonly GRAFANA_HOST='grafana.learn.hyeonworks.com'
|
|
readonly GITEA_HOST='git.learn.hyeonworks.com'
|
|
readonly KEYCLOAK_HOST='id.learn.hyeonworks.com'
|
|
readonly LAN_IP='192.168.0.107'
|
|
readonly TAILSCALE_IP='100.92.240.34'
|
|
readonly PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus'
|
|
readonly ALERTMANAGER_SERVICE='observability-core-kube-pr-alertmanager'
|
|
readonly RUNBOOK_URL='https://git.learn.hyeonworks.com/donghyeon.kang/project-infra/src/branch/main/docs/runbooks/2026-07-31-observability-access-cutover.md'
|
|
readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0'
|
|
readonly KUBECTL_DEFAULT='/usr/local/bin/kubectl'
|
|
readonly CURL_DEFAULT='/usr/bin/curl'
|
|
readonly DIG_DEFAULT='/usr/bin/dig'
|
|
readonly OPENSSL_DEFAULT='/usr/bin/openssl'
|
|
readonly NC_DEFAULT='/usr/bin/nc.openbsd'
|
|
readonly SUDO_DEFAULT='/usr/bin/sudo'
|
|
readonly DATE_DEFAULT='/usr/bin/date'
|
|
readonly CERTBOT_BIN='/snap/bin/certbot'
|
|
readonly CERTBOT_REAL='/usr/bin/snap'
|
|
readonly SYSTEMCTL_BIN='/usr/bin/systemctl'
|
|
readonly TIMEOUT_BIN='/usr/bin/timeout'
|
|
readonly JQ_BIN='/usr/bin/jq'
|
|
readonly PYTHON_BIN='/usr/bin/python3.12'
|
|
readonly CAT_BIN='/usr/bin/cat'
|
|
readonly CHMOD_BIN='/usr/bin/chmod'
|
|
readonly CMP_BIN='/usr/bin/cmp'
|
|
readonly MKTEMP_BIN='/usr/bin/mktemp'
|
|
readonly MV_BIN='/usr/bin/mv'
|
|
readonly READLINK_BIN='/usr/bin/readlink'
|
|
readonly RM_BIN='/usr/bin/rm'
|
|
readonly STAT_BIN='/usr/bin/stat'
|
|
readonly REQUEST_TIMEOUT='10s'
|
|
readonly PROCESS_TIMEOUT='20s'
|
|
readonly HUMAN_TIMEOUT_SECONDS=300
|
|
|
|
KUBECTL_BIN="$KUBECTL_DEFAULT"
|
|
CURL_BIN="$CURL_DEFAULT"
|
|
DIG_BIN="$DIG_DEFAULT"
|
|
OPENSSL_BIN="$OPENSSL_DEFAULT"
|
|
NC_BIN="$NC_DEFAULT"
|
|
SUDO_BIN="$SUDO_DEFAULT"
|
|
DATE_BIN="$DATE_DEFAULT"
|
|
TEST_MODE=false
|
|
TEST_CONFIRMATIONS=''
|
|
CONFIRMATION_INDEX=0
|
|
declare -a CONFIRMATION_QUEUE=()
|
|
|
|
execute_requested=false
|
|
WORK=''
|
|
API_PROXY_SOCKET=''
|
|
API_PROXY_PID=''
|
|
API_PROXY_START=''
|
|
EXIT_HANDLER_RUNNING=false
|
|
MANUAL_RECOVERY=false
|
|
SILENCE_ID=''
|
|
SILENCE_RUN=''
|
|
SILENCE_POLL_ATTEMPTS=30
|
|
SILENCE_POLL_DELAY=1
|
|
MUTATION_CRITICAL_DEPTH=0
|
|
DEFERRED_SIGNAL_RC=0
|
|
EXPECTED_NODE_UID=''
|
|
PUBLIC_EDGE_IP=''
|
|
KEYCLOAK_TEST_ADDED=false
|
|
KEYCLOAK_MEMBERSHIP_PENDING=false
|
|
KEYCLOAK_MEMBERSHIP_REMOVED=false
|
|
KEYCLOAK_USER_ID=''
|
|
KEYCLOAK_VIEWER_GROUP_ID=''
|
|
KEYCLOAK_MEMBERSHIP_BEFORE=''
|
|
KEYCLOAK_AUTH_CONFIG=''
|
|
declare -a OWNED_KIND=()
|
|
declare -a OWNED_NAMESPACE=()
|
|
declare -a OWNED_NAME=()
|
|
declare -a OWNED_UID=()
|
|
declare -a OWNED_RUN=()
|
|
|
|
fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
|
|
emit_manual_recovery() {
|
|
[[ "$MANUAL_RECOVERY" == true ]] && return 0
|
|
MANUAL_RECOVERY=true
|
|
printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2
|
|
}
|
|
|
|
usage() {
|
|
"$CAT_BIN" <<'USAGE'
|
|
Usage:
|
|
bash scripts/validate/observability-smoke.sh
|
|
bash scripts/validate/observability-smoke.sh --execute
|
|
|
|
No arguments prints the exact acceptance plan without external reads or
|
|
mutations. Execute requires the visible `APPLY default` gate, creates only
|
|
short-lived acceptance objects, restarts only Grafana/Alertmanager Pods, and
|
|
requires exact human OIDC, Slack, and external-client evidence tokens.
|
|
USAGE
|
|
}
|
|
|
|
print_dry_run() {
|
|
printf '%s\n' \
|
|
'OBSERVABILITY_SMOKE_DRY_RUN=PASS' \
|
|
'CHECK_01=Grafana public/private DNS exactness' \
|
|
'CHECK_02=Grafana certificate chain,hostname,one-SAN,expiry' \
|
|
'CHECK_03=LAN,Tailscale,denied-source,public-edge HTTPS status' \
|
|
'CHECK_04=Gitea,Grafana,Keycloak public metrics denial' \
|
|
'CHECK_05=unknown-SNI and Traefik NodePort boundary' \
|
|
'CHECK_06=exact Prometheus target allowlist and health' \
|
|
'CHECK_07=Gitea,Keycloak,CNPG,AIStor,Traefik metric contracts' \
|
|
'CHECK_08=untrusted-Pod AIStor and blackbox denial' \
|
|
'CHECK_09=public,private,internal,TLS blackbox series' \
|
|
'CHECK_10=Grafana datasource health and Loki-to-Tempo trace link' \
|
|
'CHECK_11=AIStor quota and recording-rule series' \
|
|
'CHECK_12=Grafana,Alertmanager restart persistence' \
|
|
'CHECK_13=Certbot dry-run deploy-hook and Nginx health' \
|
|
'HUMAN_OIDC=admin,viewer,denied,break-glass,session-revoke,relogin-denied' \
|
|
'HUMAN_SLACK=firing,resolved' \
|
|
'HUMAN_EXTERNAL_CLIENT=required' \
|
|
'TEMPORARY_OBJECTS=NetworkPolicy,Pod,AlertmanagerSilence,PrometheusRule' \
|
|
'PRESERVE=Secret,PVC,preexisting-Keycloak-membership' \
|
|
'MUTATION=NOT_REQUESTED'
|
|
}
|
|
|
|
parse_arguments() {
|
|
if (( $# == 0 )); then return 0; fi
|
|
if (( $# == 1 )) && [[ "$1" == --execute ]]; then execute_requested=true; return 0; fi
|
|
usage >&2
|
|
fail 'supported arguments are empty or exactly --execute'
|
|
}
|
|
|
|
reject_production_overrides() {
|
|
local variable
|
|
for variable in ${!PLATFORM_OBSERVABILITY_SMOKE_@}; do
|
|
fail "production boundary override is forbidden: $variable"
|
|
return 1
|
|
done
|
|
}
|
|
|
|
validate_dns_contract() {
|
|
(( $# == 8 )) || return 1
|
|
[[ "$1" == '' && "$2" == '' && "$3" == "$LAN_IP" && "$4" == '' &&
|
|
"$5" == "$TAILSCALE_IP" && "$6" == '' && "$7" == "$LAN_IP" && "$8" == '' ]]
|
|
}
|
|
|
|
validate_certificate_contract() {
|
|
local sans=$1 verify_rc=$2 expiry_rc=$3
|
|
[[ "$verify_rc" == 0 && "$expiry_rc" == 0 &&
|
|
"$sans" == "DNS:$GRAFANA_HOST" ]]
|
|
}
|
|
|
|
validate_certificate_extension_contract() {
|
|
local extension=$1 verify_rc=$2 expiry_rc=$3
|
|
[[ "$verify_rc" == 0 && "$expiry_rc" == 0 ]] || return 1
|
|
python_bounded -I -S - "$extension" "$GRAFANA_HOST" <<'PY'
|
|
import sys
|
|
extension,host=sys.argv[1:]
|
|
lines=[line.strip() for line in extension.splitlines() if line.strip()]
|
|
if len(lines)<2 or lines[0]!='X509v3 Subject Alternative Name:':
|
|
raise SystemExit(1)
|
|
names=[name.strip() for name in ' '.join(lines[1:]).split(',')]
|
|
if names != ['DNS:'+host]:
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_target_contract() {
|
|
local file=$1
|
|
python_bounded -I -S - "$file" <<'PY'
|
|
import json,pathlib,re,sys
|
|
from urllib.parse import urlsplit
|
|
expected={
|
|
"podMonitor/platform-data/platform-postgres/0":1,
|
|
"probe/observability/platform-private-edge":3,
|
|
"probe/observability/platform-private-internal":3,
|
|
"probe/observability/platform-public-edge":2,
|
|
"serviceMonitor/gitea/gitea/0":1,
|
|
"serviceMonitor/keycloak/keycloak/0":1,
|
|
"serviceMonitor/kube-system/traefik/0":1,
|
|
"serviceMonitor/object-storage/aistor-bucket-usage/0":1,
|
|
"serviceMonitor/observability-agent/alloy/0":1,
|
|
"serviceMonitor/observability-agent/node-exporter/0":1,
|
|
"serviceMonitor/observability/blackbox-exporter/0":1,
|
|
"serviceMonitor/observability/grafana/0":1,
|
|
"serviceMonitor/observability/loki/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-coredns/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-operator/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":1,
|
|
"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":1,
|
|
"serviceMonitor/observability/observability-core-kube-state-metrics/0":1,
|
|
"serviceMonitor/observability/tempo/0":1,
|
|
}
|
|
expected_probe_targets={
|
|
"probe/observability/platform-private-edge":{
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","https://db-admin.learn.hyeonworks.com/","blackbox-private-edge","private-edge"),
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","https://grafana.learn.hyeonworks.com/","blackbox-private-edge","private-edge"),
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","https://storage-admin.learn.hyeonworks.com/","blackbox-private-edge","private-edge"),
|
|
},
|
|
"probe/observability/platform-private-internal":{
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","http://pgadmin.platform-admin.svc.cluster.local/misc/ping","blackbox-private-internal","private-internal"),
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","http://grafana.observability.svc.cluster.local/api/health","blackbox-private-internal","private-internal"),
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","http://minio-aistor-console.object-storage.svc.cluster.local:9090/","blackbox-private-internal","private-internal"),
|
|
},
|
|
"probe/observability/platform-public-edge":{
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","https://git.learn.hyeonworks.com/api/healthz","blackbox-public-edge","public-edge"),
|
|
("http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","blackbox-public-edge","public-edge"),
|
|
},
|
|
}
|
|
endpoint_patterns={
|
|
"podMonitor/platform-data/platform-postgres/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9187/metrics$",
|
|
"serviceMonitor/gitea/gitea/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3000/metrics$",
|
|
"serviceMonitor/keycloak/keycloak/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9000/metrics$",
|
|
"serviceMonitor/kube-system/traefik/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9100/metrics$",
|
|
"serviceMonitor/object-storage/aistor-bucket-usage/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9000/minio/metrics/v3/cluster/usage/buckets$",
|
|
"serviceMonitor/observability-agent/alloy/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:12345/metrics$",
|
|
"serviceMonitor/observability-agent/node-exporter/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9100/metrics$",
|
|
"serviceMonitor/observability/blackbox-exporter/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9115/metrics$",
|
|
"serviceMonitor/observability/grafana/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3000/metrics$",
|
|
"serviceMonitor/observability/loki/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3100/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9093/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:6443/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-coredns/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9153/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics/cadvisor$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":r"^https://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:10250/metrics/probes$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-operator/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:9090/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
|
|
"serviceMonitor/observability/observability-core-kube-state-metrics/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080/metrics$",
|
|
"serviceMonitor/observability/tempo/0":r"^http://(?:[0-9]{1,3}\.){3}[0-9]{1,3}:3200/metrics$",
|
|
}
|
|
target_identities={
|
|
"podMonitor/platform-data/platform-postgres/0":("platform-postgres","platform-data"),
|
|
"serviceMonitor/gitea/gitea/0":("gitea","gitea"),
|
|
"serviceMonitor/keycloak/keycloak/0":("keycloak-service","keycloak"),
|
|
"serviceMonitor/kube-system/traefik/0":("traefik","kube-system"),
|
|
"serviceMonitor/object-storage/aistor-bucket-usage/0":("minio-aistor","object-storage"),
|
|
"serviceMonitor/observability-agent/alloy/0":("alloy","observability-agent"),
|
|
"serviceMonitor/observability-agent/node-exporter/0":("prometheus-node-exporter","observability-agent"),
|
|
"serviceMonitor/observability/blackbox-exporter/0":("blackbox-exporter","observability"),
|
|
"serviceMonitor/observability/grafana/0":("grafana","observability"),
|
|
"serviceMonitor/observability/loki/0":("observability/loki","observability"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/0":("observability-core-kube-pr-alertmanager","observability"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-alertmanager/1":("observability-core-kube-pr-alertmanager","observability"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-apiserver/0":("apiserver","default"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-coredns/0":("coredns","kube-system"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/0":("kubelet","kube-system"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/1":("kubelet","kube-system"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-kubelet/2":("kubelet","kube-system"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-operator/0":("observability-core-kube-pr-operator","observability"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-prometheus/0":("observability-core-kube-pr-prometheus","observability"),
|
|
"serviceMonitor/observability/observability-core-kube-pr-prometheus/1":("observability-core-kube-pr-prometheus","observability"),
|
|
"serviceMonitor/observability/observability-core-kube-state-metrics/0":("kube-state-metrics","observability"),
|
|
"serviceMonitor/observability/tempo/0":("tempo","observability"),
|
|
}
|
|
try: item=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
targets=((item.get("data") or {}).get("activeTargets"))
|
|
if item.get("status")!="success" or not isinstance(targets,list) or len(targets)!=30: raise SystemExit(1)
|
|
actual={}
|
|
probe_targets={pool:set() for pool in expected_probe_targets}
|
|
for target in targets:
|
|
if target.get("health")!="up" or target.get("lastError","") not in ("",None): raise SystemExit(1)
|
|
pool=target.get("scrapePool")
|
|
url=target.get("scrapeUrl")
|
|
if not isinstance(pool,str) or not isinstance(url,str): raise SystemExit(1)
|
|
if pool in endpoint_patterns and re.fullmatch(endpoint_patterns[pool],url) is None: raise SystemExit(1)
|
|
if pool in target_identities:
|
|
labels=target.get("labels")
|
|
discovered=target.get("discoveredLabels")
|
|
parsed=urlsplit(url)
|
|
expected_job,expected_namespace=target_identities[pool]
|
|
if (not isinstance(labels,dict) or labels.get("job")!=expected_job or
|
|
labels.get("namespace")!=expected_namespace or labels.get("instance")!=parsed.netloc):
|
|
raise SystemExit(1)
|
|
if (target.get("globalUrl")!=url or not isinstance(discovered,dict) or
|
|
discovered.get("__address__")!=parsed.netloc or
|
|
discovered.get("__scheme__")!=parsed.scheme or
|
|
discovered.get("__metrics_path__")!=parsed.path):
|
|
raise SystemExit(1)
|
|
if pool in expected_probe_targets:
|
|
labels=target.get("labels")
|
|
if not isinstance(labels,dict) or set(labels)!={"instance","job","namespace","observability.hyeonworks.com/probe-group"}: raise SystemExit(1)
|
|
identity=(url,labels.get("instance"),labels.get("job"),labels.get("observability.hyeonworks.com/probe-group"))
|
|
if labels.get("namespace")!="observability" or identity in probe_targets[pool]: raise SystemExit(1)
|
|
probe_targets[pool].add(identity)
|
|
actual[pool]=actual.get(pool,0)+1
|
|
if actual!=expected or probe_targets!=expected_probe_targets: raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_series_contract() {
|
|
local file=$1 expected_count=$2 required_csv=$3 expected_values_csv=$4 value_label=$5
|
|
python_bounded -I -S - "$file" "$expected_count" "$required_csv" "$expected_values_csv" "$value_label" <<'PY'
|
|
import json,pathlib,sys
|
|
path,count,required,values,label=sys.argv[1:]
|
|
try: item=json.loads(pathlib.Path(path).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
result=((item.get("data") or {}).get("result"))
|
|
if item.get("status")!="success" or not isinstance(result,list) or len(result)!=int(count): raise SystemExit(1)
|
|
required=set(filter(None,required.split(',')))
|
|
expected=sorted(filter(None,values.split(',')))
|
|
seen=[]
|
|
for series in result:
|
|
metric=series.get("metric")
|
|
if not isinstance(metric,dict) or not required.issubset(metric): raise SystemExit(1)
|
|
value=series.get("value")
|
|
if not isinstance(value,list) or len(value)!=2:
|
|
raise SystemExit(1)
|
|
try: float(value[1])
|
|
except (TypeError,ValueError): raise SystemExit(1)
|
|
if label: seen.append(metric.get(label))
|
|
if label and (any(not isinstance(x,str) for x in seen) or sorted(seen)!=expected): raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_series_identity_contract() {
|
|
local file=$1 expected_json=$2 instance_pattern=${3:-}
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$PYTHON_BIN" -I -S - \
|
|
"$file" "$expected_json" "$instance_pattern" <<'PY'
|
|
import json,pathlib,re,sys
|
|
path,expected_raw,instance_pattern=sys.argv[1:]
|
|
try:
|
|
item=json.loads(pathlib.Path(path).read_text())
|
|
expected=json.loads(expected_raw)
|
|
except Exception: raise SystemExit(1)
|
|
result=((item.get('data') or {}).get('result'))
|
|
if item.get('status')!='success' or not isinstance(result,list) or not isinstance(expected,list) or len(result)!=len(expected) or not expected:
|
|
raise SystemExit(1)
|
|
if any(not isinstance(value,dict) or not value for value in expected): raise SystemExit(1)
|
|
keys=set(expected[0])
|
|
if any(set(value)!=keys or any(not isinstance(x,str) for x in value.values()) for value in expected): raise SystemExit(1)
|
|
def canonical(value): return json.dumps(value,sort_keys=True,separators=(',',':'))
|
|
expected_rows=sorted(canonical(value) for value in expected)
|
|
if len(set(expected_rows))!=len(expected_rows): raise SystemExit(1)
|
|
actual=[]
|
|
for series in result:
|
|
metric=series.get('metric')
|
|
if not isinstance(metric,dict) or any(not isinstance(metric.get(key),str) for key in keys): raise SystemExit(1)
|
|
if instance_pattern:
|
|
instance=metric.get('instance')
|
|
if not isinstance(instance,str) or re.fullmatch(instance_pattern,instance) is None: raise SystemExit(1)
|
|
actual.append(canonical({key:metric[key] for key in keys}))
|
|
if sorted(actual)!=expected_rows: raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_datasource_contract() {
|
|
local file=$1
|
|
python_bounded -I -S - "$file" <<'PY'
|
|
import base64,binascii,json,pathlib,re,sys
|
|
try: item=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
if any((item.get(x) or {}).get("status")!="OK" for x in ("prometheus","loki","tempo")): raise SystemExit(1)
|
|
loki=item.get("loki_config") or {}; tempo=item.get("tempo_config") or {}
|
|
fields=((loki.get("jsonData") or {}).get("derivedFields")) or []
|
|
if not any(x.get("name")=="trace_id" and x.get("datasourceUid")=="tempo" and x.get("url")=="${__value.raw}" for x in fields): raise SystemExit(1)
|
|
link=((tempo.get("jsonData") or {}).get("tracesToLogsV2")) or {}
|
|
if link.get("datasourceUid")!="loki" or link.get("filterByTraceID") is not True: raise SystemExit(1)
|
|
lines=[]
|
|
for stream in (((item.get("loki_query") or {}).get("data") or {}).get("result") or []):
|
|
for value in stream.get("values") or []:
|
|
if isinstance(value,list) and len(value)==2 and isinstance(value[1],str): lines.append(value[1])
|
|
selected=item.get('selected_trace')
|
|
if not isinstance(selected,str) or re.fullmatch(r'[0-9a-f]{32}',selected) is None: raise SystemExit(1)
|
|
search=(item.get('tempo_search') or {}).get('traces')
|
|
if not isinstance(search,list): raise SystemExit(1)
|
|
candidates=[x for x in search if isinstance(x,dict) and x.get('traceID')==selected and x.get('rootServiceName')=='platform-smoke']
|
|
if len(candidates)!=1: raise SystemExit(1)
|
|
matches=[]
|
|
for line in lines: matches.extend(re.findall(r'"trace_id"\s*:\s*"([0-9a-f]{32})"',line))
|
|
if not matches or set(matches)!={selected}: raise SystemExit(1)
|
|
batches=(item.get("tempo_query") or {}).get("batches")
|
|
if not isinstance(batches,list) or not batches: raise SystemExit(1)
|
|
seen_span=False
|
|
for batch in batches:
|
|
if not isinstance(batch,dict): raise SystemExit(1)
|
|
attrs=((batch.get('resource') or {}).get('attributes'))
|
|
if not isinstance(attrs,list): raise SystemExit(1)
|
|
service=[x for x in attrs if isinstance(x,dict) and x.get('key')=='service.name']
|
|
if len(service)!=1 or ((service[0].get('value') or {}).get('stringValue'))!='platform-smoke': raise SystemExit(1)
|
|
for scope in batch.get('scopeSpans') or []:
|
|
for span in scope.get('spans') or []:
|
|
encoded=span.get('traceId')
|
|
if not isinstance(encoded,str): raise SystemExit(1)
|
|
try: decoded=base64.b64decode(encoded,validate=True).hex()
|
|
except (binascii.Error,ValueError): raise SystemExit(1)
|
|
if decoded!=selected: raise SystemExit(1)
|
|
seen_span=True
|
|
if not seen_span: raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
synthetic_loki_query() {
|
|
local trace=$1
|
|
[[ "$trace" =~ ^[0-9a-f]{32}$ ]] || return 1
|
|
printf '{cluster="home",namespace="observability"} | json | trace_id="%s"' "$trace"
|
|
}
|
|
|
|
validate_human_tokens() {
|
|
local -a expected=(OIDC_ADMIN_CONFIRMED OIDC_VIEWER_CONFIRMED OIDC_DENIED_CONFIRMED
|
|
BREAK_GLASS_CONFIRMED REMOVAL_VIEWER_CONFIRMED SESSION_REVOKED RELOGIN_DENIED
|
|
FIRING RESOLVED EXTERNAL_CLIENT_VERIFIED)
|
|
local -a actual=()
|
|
mapfile -t actual
|
|
(( ${#actual[@]} == ${#expected[@]} )) || return 1
|
|
local index
|
|
for ((index=0; index<${#expected[@]}; index++)); do
|
|
[[ "${actual[$index]}" == "${expected[$index]}" ]] || return 1
|
|
done
|
|
}
|
|
|
|
validate_ipv4() {
|
|
local value=$1 octet
|
|
local -a parts=()
|
|
[[ "$value" =~ ^[0-9]{1,3}([.][0-9]{1,3}){3}$ ]] || return 1
|
|
IFS=. read -r -a parts <<<"$value"
|
|
(( ${#parts[@]} == 4 )) || return 1
|
|
for octet in "${parts[@]}"; do
|
|
(( 10#$octet <= 255 )) || return 1
|
|
done
|
|
}
|
|
|
|
external_attestation_token() {
|
|
local public_ip=$1
|
|
validate_ipv4 "$public_ip" || return 1
|
|
printf 'EXTERNAL_CLIENT_VERIFIED public_ip=%s status=403 source=outside-lan-tailscale proxy=disabled' "$public_ip"
|
|
}
|
|
|
|
render_external_attestation() {
|
|
local public_ip=$1 token
|
|
token="$(external_attestation_token "$public_ip")" || return 1
|
|
printf '%s\n' \
|
|
'Run this from a client outside both LAN and Tailscale, with no HTTP(S) proxy:' \
|
|
"/usr/bin/curl --disable --silent --show-error --output /dev/null --write-out '%{http_code}\\n' --connect-timeout 5 --max-time 15 --noproxy '*' --resolve $GRAFANA_HOST:443:$public_ip https://$GRAFANA_HOST/" \
|
|
'Expected exact result: HTTP 403' \
|
|
"After verifying the client source and result, type: $token"
|
|
}
|
|
|
|
new_smoke_run_id() {
|
|
local prefix=$1 stamp nonce
|
|
[[ "$prefix" =~ ^[a-z][a-z0-9-]{0,14}$ ]] || return 1
|
|
stamp="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1
|
|
nonce="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$PYTHON_BIN" -I -S - <<'PY'
|
|
import os
|
|
print(os.urandom(16).hex(),end='')
|
|
PY
|
|
)" || return 1
|
|
[[ "$stamp" =~ ^[0-9]{8}T[0-9]{6}Z$ && "$nonce" =~ ^[0-9a-f]{32}$ ]] || return 1
|
|
printf '%s-%s-%s' "$prefix" "${stamp,,}" "$nonce"
|
|
}
|
|
|
|
kubernetes_smoke_name() {
|
|
local run=$1 nonce
|
|
[[ "$run" =~ ^[a-z][a-z0-9-]{0,14}-[0-9]{8}t[0-9]{6}z-([0-9a-f]{32})$ ]] || return 1
|
|
nonce=${BASH_REMATCH[1]}
|
|
printf 'observability-smoke-%s' "$nonce"
|
|
}
|
|
|
|
handle_termination_signal() {
|
|
local rc=$1
|
|
[[ "$rc" =~ ^(129|130|143)$ ]] || rc=143
|
|
if (( MUTATION_CRITICAL_DEPTH > 0 )); then
|
|
(( DEFERRED_SIGNAL_RC == 0 )) && DEFERRED_SIGNAL_RC=$rc
|
|
return 0
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
|
|
begin_mutation_critical() {
|
|
MUTATION_CRITICAL_DEPTH=$((MUTATION_CRITICAL_DEPTH + 1))
|
|
}
|
|
|
|
end_mutation_critical() {
|
|
(( MUTATION_CRITICAL_DEPTH > 0 )) || return 1
|
|
MUTATION_CRITICAL_DEPTH=$((MUTATION_CRITICAL_DEPTH - 1))
|
|
if (( MUTATION_CRITICAL_DEPTH == 0 && DEFERRED_SIGNAL_RC != 0 )); then
|
|
local rc=$DEFERRED_SIGNAL_RC
|
|
DEFERRED_SIGNAL_RC=0
|
|
exit "$rc"
|
|
fi
|
|
}
|
|
|
|
require_confirmation() {
|
|
local expected=$1 answer=''
|
|
if [[ "$TEST_MODE" == true ]]; then
|
|
(( CONFIRMATION_INDEX < ${#CONFIRMATION_QUEUE[@]} )) || return 1
|
|
answer=${CONFIRMATION_QUEUE[$CONFIRMATION_INDEX]}
|
|
CONFIRMATION_INDEX=$((CONFIRMATION_INDEX + 1))
|
|
else
|
|
[[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; }
|
|
printf 'Type %s: ' "$expected" >&2
|
|
IFS= read -r -t "$HUMAN_TIMEOUT_SECONDS" answer || return 1
|
|
fi
|
|
[[ "$answer" == "$expected" ]]
|
|
}
|
|
|
|
confirmation_remaining_seconds() {
|
|
local deadline=$1 now=$2 remaining
|
|
[[ "$deadline" =~ ^[0-9]+$ && "$now" =~ ^[0-9]+$ ]] || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining >= 1 && remaining <= HUMAN_TIMEOUT_SECONDS )) || return 1
|
|
printf '%s' "$remaining"
|
|
}
|
|
|
|
require_confirmation_before() {
|
|
local expected=$1 deadline=$2 now remaining answer=''
|
|
if [[ "$TEST_MODE" == true ]]; then require_confirmation "$expected"; return; fi
|
|
now="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%s)" || return 1
|
|
remaining="$(confirmation_remaining_seconds "$deadline" "$now")" || return 1
|
|
[[ -t 0 ]] || return 1
|
|
printf 'Type %s within the shared deadline: ' "$expected" >&2
|
|
IFS= read -r -t "$remaining" answer || return 1
|
|
[[ "$answer" == "$expected" ]]
|
|
}
|
|
|
|
read_identity() {
|
|
local purpose=$1 answer=''
|
|
[[ "$TEST_MODE" == false ]] || return 1
|
|
[[ -t 0 ]] || return 1
|
|
printf 'Enter the existing %s username (not recorded): ' "$purpose" >&2
|
|
IFS= read -r -t "$HUMAN_TIMEOUT_SECONDS" answer || return 1
|
|
[[ "$answer" =~ ^[A-Za-z0-9][A-Za-z0-9._@+-]{0,127}$ ]] || return 1
|
|
printf '%s' "$answer"
|
|
}
|
|
|
|
process_start_time() {
|
|
local pid=$1
|
|
python_bounded -I -S - "$pid" <<'PY'
|
|
import pathlib,sys
|
|
raw=pathlib.Path('/proc',sys.argv[1],'stat').read_text()
|
|
end=raw.rfind(') ')
|
|
if end<0: raise SystemExit(1)
|
|
fields=raw[end+2:].split()
|
|
if len(fields)<20 or not fields[19].isdigit(): raise SystemExit(1)
|
|
print(fields[19],end='')
|
|
PY
|
|
}
|
|
|
|
stop_owned_process() {
|
|
local pid=$1 expected_start=$2 current attempt
|
|
[[ "$pid" =~ ^[0-9]+$ && "$expected_start" =~ ^[0-9]+$ ]] || return 0
|
|
current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; }
|
|
[[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; }
|
|
/bin/kill -TERM "$pid" 2>/dev/null || true
|
|
for ((attempt=0; attempt<40; attempt++)); do
|
|
/bin/kill -0 "$pid" 2>/dev/null || { wait "$pid" 2>/dev/null || true; return 0; }
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
current="$(process_start_time "$pid" 2>/dev/null)" || { wait "$pid" 2>/dev/null || true; return 0; }
|
|
[[ "$current" == "$expected_start" ]] || { emit_manual_recovery; return 1; }
|
|
/bin/kill -KILL "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
}
|
|
|
|
kubectl_bounded() {
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=2s "$PROCESS_TIMEOUT" \
|
|
"$KUBECTL_BIN" --request-timeout="$REQUEST_TIMEOUT" "$@"
|
|
}
|
|
|
|
kubectl_wait_90_bounded() {
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=2s 100s \
|
|
"$KUBECTL_BIN" --request-timeout=95s "$@"
|
|
}
|
|
|
|
kubectl_wait_180_bounded() {
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=2s 190s \
|
|
"$KUBECTL_BIN" --request-timeout=185s "$@"
|
|
}
|
|
|
|
jq_bounded() {
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$JQ_BIN" "$@"
|
|
}
|
|
|
|
python_bounded() {
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$PYTHON_BIN" "$@"
|
|
}
|
|
|
|
safe_work_file() {
|
|
[[ -n "$WORK" && "$WORK" == /tmp/platform-observability-smoke.???????? &&
|
|
"$1" == "$WORK"/* && -f "$1" && ! -L "$1" && -O "$1" &&
|
|
"$("$STAT_BIN" -c '%a:%h' -- "$1")" == 600:1 ]]
|
|
}
|
|
|
|
make_work() {
|
|
WORK="$("$MKTEMP_BIN" -d /tmp/platform-observability-smoke.XXXXXXXX)" || return 1
|
|
[[ "$WORK" == /tmp/platform-observability-smoke.???????? && -d "$WORK" && ! -L "$WORK" ]] || return 1
|
|
"$CHMOD_BIN" 0700 -- "$WORK"
|
|
}
|
|
|
|
start_api_proxy() {
|
|
[[ -n "$WORK" ]] || return 1
|
|
API_PROXY_SOCKET="$WORK/kube-api.sock"
|
|
"$KUBECTL_BIN" --request-timeout="$REQUEST_TIMEOUT" proxy \
|
|
--unix-socket="$API_PROXY_SOCKET" --api-prefix=/ \
|
|
--accept-paths='^/(api|apis)/.*$' >"$WORK/kube-api-proxy.log" 2>&1 &
|
|
API_PROXY_PID=$!
|
|
API_PROXY_START="$(process_start_time "$API_PROXY_PID")" || return 1
|
|
local attempt
|
|
for ((attempt=0; attempt<100; attempt++)); do
|
|
[[ -S "$API_PROXY_SOCKET" ]] && { "$CHMOD_BIN" 0600 -- "$API_PROXY_SOCKET"; return 0; }
|
|
/bin/kill -0 "$API_PROXY_PID" 2>/dev/null || return 1
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
return 1
|
|
}
|
|
|
|
proxy_request() {
|
|
local method=$1 path=$2 output=$3 body=${4:-} extra=${5:-}
|
|
local config status
|
|
[[ "$method" =~ ^(GET|POST|PUT|DELETE)$ && "$path" == /api* &&
|
|
"$path" != *$'\n'* && "$path" != *'"'* && "$path" != *'\\'* ]] || return 1
|
|
config="$("$MKTEMP_BIN" "$WORK/curl.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$config"
|
|
: >"$output"; "$CHMOD_BIN" 0600 -- "$output"
|
|
{
|
|
printf 'silent\nshow-error\nconnect-timeout = 2\nmax-time = 12\n'
|
|
printf 'unix-socket = "%s"\nrequest = "%s"\n' "$API_PROXY_SOCKET" "$method"
|
|
printf 'url = "http://localhost%s"\noutput = "%s"\nwrite-out = "%%{http_code}"\n' "$path" "$output"
|
|
[[ -z "$body" ]] || printf 'header = "Content-Type: application/json"\ndata-binary = "@%s"\n' "$body"
|
|
[[ -z "$extra" ]] || /bin/cat -- "$extra"
|
|
} >"$config"
|
|
safe_work_file "$config" && safe_work_file "$output" || return 1
|
|
status="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$CURL_BIN" --disable --config "$config")" || return 1
|
|
[[ "$status" =~ ^[0-9]{3}$ ]] || return 1
|
|
printf '%s' "$status"
|
|
}
|
|
|
|
prometheus_path() {
|
|
printf '/api/v1/namespaces/observability/services/http:%s:9090/proxy%s' "$PROMETHEUS_SERVICE" "$1"
|
|
}
|
|
|
|
alertmanager_path() {
|
|
printf '/api/v1/namespaces/observability/services/http:%s:9093/proxy%s' "$ALERTMANAGER_SERVICE" "$1"
|
|
}
|
|
|
|
grafana_path() { printf '/api/v1/namespaces/observability/services/http:grafana:80/proxy%s' "$1"; }
|
|
keycloak_path() { printf '/api/v1/namespaces/keycloak/services/http:keycloak-service:8080/proxy%s' "$1"; }
|
|
|
|
proxy_get_200() {
|
|
local path=$1 output=$2 extra=${3:-} status
|
|
status="$(proxy_request GET "$path" "$output" '' "$extra")" || return 1
|
|
[[ "$status" == 200 && -s "$output" ]]
|
|
}
|
|
|
|
prom_query() {
|
|
local expression=$1 output=$2 encoded
|
|
encoded="$(jq_bounded -nr --arg value "$expression" '$value|@uri')" || return 1
|
|
proxy_get_200 "$(prometheus_path "/api/v1/query?query=$encoded")" "$output"
|
|
}
|
|
|
|
api_path_for_owned() {
|
|
local kind=$1 namespace=$2 name=$3
|
|
[[ "$namespace" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ &&
|
|
"$name" =~ ^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$ ]] || return 1
|
|
case "$kind" in
|
|
Pod) printf '/api/v1/namespaces/%s/pods/%s' "$namespace" "$name" ;;
|
|
NetworkPolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s' "$namespace" "$name" ;;
|
|
PrometheusRule) printf '/apis/monitoring.coreos.com/v1/namespaces/%s/prometheusrules/%s' "$namespace" "$name" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
kind_resource() {
|
|
case "$1" in
|
|
Pod) printf pod ;;
|
|
NetworkPolicy) printf networkpolicy ;;
|
|
PrometheusRule) printf prometheusrule ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
owned_object_state() {
|
|
local kind=$1 namespace=$2 name=$3 resource file uid run
|
|
resource="$(kind_resource "$kind")" || return 4
|
|
file="$("$MKTEMP_BIN" "$WORK/state.XXXXXXXX")" || return 4
|
|
"$CHMOD_BIN" 0600 -- "$file"
|
|
kubectl_bounded -n "$namespace" get "$resource" "$name" --ignore-not-found -o json >"$file" || return 4
|
|
[[ -s "$file" ]] || return 1
|
|
uid="$(jq_bounded -er '.metadata.uid | select(type=="string" and length>0)' "$file")" || return 4
|
|
run="$(jq_bounded -er '.metadata.labels["platform.hyeonworks.com/smoke-run"] | select(type=="string" and length>0)' "$file")" || return 4
|
|
printf '%s|%s\n' "$uid" "$run"
|
|
}
|
|
|
|
delete_owned_uid() {
|
|
local kind=$1 namespace=$2 name=$3 uid=$4 path body response status state attempt
|
|
[[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
|
|
path="$(api_path_for_owned "$kind" "$namespace" "$name")" || return 1
|
|
body="$("$MKTEMP_BIN" "$WORK/delete.XXXXXXXX")" || return 1
|
|
response="$("$MKTEMP_BIN" "$WORK/delete-response.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$body" "$response"
|
|
jq_bounded -nc --arg uid "$uid" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid}}' >"$body"
|
|
status="$(proxy_request DELETE "$path" "$response" "$body")" || status=000
|
|
for ((attempt=0; attempt<30; attempt++)); do
|
|
state="$(owned_object_state "$kind" "$namespace" "$name" 2>/dev/null)" && {
|
|
[[ "${state%%|*}" == "$uid" ]] || return 1
|
|
} || {
|
|
[[ $? == 1 ]] && return 0
|
|
return 1
|
|
}
|
|
/usr/bin/sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
register_owned() {
|
|
OWNED_KIND+=("$1"); OWNED_NAMESPACE+=("$2"); OWNED_NAME+=("$3");
|
|
OWNED_UID+=("$4"); OWNED_RUN+=("$5")
|
|
}
|
|
|
|
cleanup_owned_objects() {
|
|
local index state uid run rc=0
|
|
for ((index=${#OWNED_KIND[@]}-1; index>=0; index--)); do
|
|
state="$(owned_object_state "${OWNED_KIND[$index]}" "${OWNED_NAMESPACE[$index]}" "${OWNED_NAME[$index]}" 2>/dev/null)" || {
|
|
[[ $? == 1 ]] && continue
|
|
emit_manual_recovery; rc=1; continue
|
|
}
|
|
IFS='|' read -r uid run <<<"$state"
|
|
if [[ "$uid" != "${OWNED_UID[$index]}" || "$run" != "${OWNED_RUN[$index]}" ]]; then
|
|
emit_manual_recovery; rc=1; continue
|
|
fi
|
|
delete_owned_uid "${OWNED_KIND[$index]}" "${OWNED_NAMESPACE[$index]}" "${OWNED_NAME[$index]}" "$uid" || {
|
|
emit_manual_recovery; rc=1;
|
|
}
|
|
done
|
|
OWNED_KIND=(); OWNED_NAMESPACE=(); OWNED_NAME=(); OWNED_UID=(); OWNED_RUN=()
|
|
return "$rc"
|
|
}
|
|
|
|
validate_commit_environment() {
|
|
local permission result context api node_uid
|
|
validate_command_boundaries || return 1
|
|
context="$(kubectl_bounded config current-context)" || return 1
|
|
[[ "$context" == "$EXPECTED_CONTEXT" ]] || { fail 'Kubernetes context is not default'; return 1; }
|
|
api="$(kubectl_bounded config view --minify -o jsonpath='{.clusters[0].cluster.server}')" || return 1
|
|
[[ "$api" == "$EXPECTED_API_SERVER" ]] || { fail 'Kubernetes API server is not the exact local endpoint'; return 1; }
|
|
kubectl_bounded get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; }
|
|
node_uid="$(kubectl_bounded get node "$EXPECTED_NODE" -o json | jq_bounded -er --arg node "$EXPECTED_NODE" '
|
|
select(.metadata.name==$node and (.metadata.uid|type=="string" and length>0) and
|
|
any(.status.conditions[]?; .type=="Ready" and .status=="True")) | .metadata.uid
|
|
')" || { fail 'target node identity/readiness changed'; return 1; }
|
|
if [[ -z "$EXPECTED_NODE_UID" ]]; then EXPECTED_NODE_UID=$node_uid; fi
|
|
[[ "$node_uid" == "$EXPECTED_NODE_UID" ]] || { fail 'target node UID changed'; return 1; }
|
|
for permission in \
|
|
'get pods observability' 'list pods observability' 'watch pods observability' \
|
|
'create pods observability' 'delete pods observability' 'get pods/log observability' \
|
|
'get services observability' 'get services object-storage' 'get services kube-system' \
|
|
'get services/proxy observability' 'create services/proxy observability' \
|
|
'delete services/proxy observability' \
|
|
'get services/proxy keycloak' 'create services/proxy keycloak' \
|
|
'update services/proxy keycloak' 'delete services/proxy keycloak' \
|
|
'get persistentvolumeclaims observability' 'list persistentvolumeclaims observability' \
|
|
'get configmaps observability' 'list configmaps observability' \
|
|
'get deployments.apps observability' 'watch deployments.apps observability' \
|
|
'get statefulsets.apps observability' 'watch statefulsets.apps observability' \
|
|
'get networkpolicies.networking.k8s.io observability' \
|
|
'create networkpolicies.networking.k8s.io observability' \
|
|
'delete networkpolicies.networking.k8s.io observability' \
|
|
'get prometheusrules.monitoring.coreos.com observability' \
|
|
'create prometheusrules.monitoring.coreos.com observability' \
|
|
'delete prometheusrules.monitoring.coreos.com observability' \
|
|
'get secrets observability' 'get secrets keycloak'; do
|
|
read -r -a _permission_parts <<<"$permission"
|
|
result="$(kubectl_bounded auth can-i "${_permission_parts[0]}" "${_permission_parts[1]}" -n "${_permission_parts[2]}")" || return 1
|
|
[[ "$result" == yes ]] || { fail "missing Kubernetes authority: $permission"; return 1; }
|
|
done
|
|
}
|
|
|
|
require_execute_environment() {
|
|
validate_commit_environment || return 1
|
|
make_work || { fail 'cannot create private work directory'; return 1; }
|
|
start_api_proxy || { fail 'cannot start bounded Kubernetes API proxy'; return 1; }
|
|
}
|
|
|
|
validate_command_boundaries() {
|
|
local path certbot_real metadata
|
|
for path in "$KUBECTL_BIN" "$CURL_BIN" "$DIG_BIN" "$OPENSSL_BIN" "$NC_BIN" \
|
|
"$SUDO_BIN" "$DATE_BIN" "$TIMEOUT_BIN" "$JQ_BIN" "$PYTHON_BIN" \
|
|
"$CAT_BIN" "$CHMOD_BIN" "$CMP_BIN" "$MKTEMP_BIN" "$MV_BIN" \
|
|
"$READLINK_BIN" "$RM_BIN" "$STAT_BIN"; do
|
|
[[ -x "$path" && ! -L "$path" ]] || { fail "required command boundary is unavailable: $path"; return 1; }
|
|
done
|
|
[[ -L "$CERTBOT_BIN" && -x "$CERTBOT_BIN" ]] || {
|
|
fail "Certbot command boundary is unavailable: $CERTBOT_BIN"; return 1;
|
|
}
|
|
certbot_real="$("$READLINK_BIN" -f -- "$CERTBOT_BIN")" || return 1
|
|
[[ "$certbot_real" == "$CERTBOT_REAL" && -x "$CERTBOT_REAL" && ! -L "$CERTBOT_REAL" ]] || {
|
|
fail 'Certbot command boundary does not resolve to the fixed snap executable'; return 1;
|
|
}
|
|
metadata="$("$STAT_BIN" -c '%u:%g:%a:%F' -- "$CERTBOT_REAL")" || return 1
|
|
[[ "$metadata" == '0:0:755:regular file' ]] || {
|
|
fail 'Certbot real executable metadata is unsafe'; return 1;
|
|
}
|
|
[[ -x "$SYSTEMCTL_BIN" && ! -L "$SYSTEMCTL_BIN" &&
|
|
"$("$STAT_BIN" -c '%u:%g:%a:%F' -- "$SYSTEMCTL_BIN")" == '0:0:755:regular file' ]] || {
|
|
fail 'systemctl command boundary is unsafe'; return 1;
|
|
}
|
|
}
|
|
|
|
dig_values() {
|
|
local resolver=$1 record=$2 host=$3
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 8s "$DIG_BIN" +time=2 +tries=1 +short \
|
|
"@$resolver" "$record" "$host" | /usr/bin/sed '/^[[:space:]]*$/d' | LC_ALL=C /usr/bin/sort -u
|
|
}
|
|
|
|
http_status() {
|
|
local -a args=(--disable --silent --show-error --output /dev/null --write-out '%{http_code}'
|
|
--connect-timeout 3 --max-time 12 --noproxy '*')
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$CURL_BIN" "${args[@]}" "$@"
|
|
}
|
|
|
|
check_dns_host() {
|
|
local public_a public_aaaa lan_a lan_aaaa tail_a tail_aaaa
|
|
public_a="$(dig_values 1.1.1.1 A "$GRAFANA_HOST")" || return 1
|
|
public_aaaa="$(dig_values 1.1.1.1 AAAA "$GRAFANA_HOST")" || return 1
|
|
lan_a="$(dig_values "$LAN_IP" A "$GRAFANA_HOST")" || return 1
|
|
lan_aaaa="$(dig_values "$LAN_IP" AAAA "$GRAFANA_HOST")" || return 1
|
|
tail_a="$(dig_values "$TAILSCALE_IP" A "$GRAFANA_HOST")" || return 1
|
|
tail_aaaa="$(dig_values "$TAILSCALE_IP" AAAA "$GRAFANA_HOST")" || return 1
|
|
validate_dns_contract "$public_a" "$public_aaaa" "$lan_a" "$lan_aaaa" \
|
|
"$tail_a" "$tail_aaaa" "$LAN_IP" '' || return 1
|
|
printf 'OBSERVABILITY_MACHINE_DNS_HOST=PASS\n'
|
|
}
|
|
|
|
check_certificate() {
|
|
local raw cert extension verify_rc=0 expiry_rc=0
|
|
raw="$WORK/grafana-sclient.txt"; cert="$WORK/grafana-leaf.pem"
|
|
: >"$raw"; : >"$cert"; "$CHMOD_BIN" 0600 -- "$raw" "$cert"
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$OPENSSL_BIN" s_client \
|
|
-connect "$LAN_IP:443" -servername "$GRAFANA_HOST" -showcerts </dev/null >"$raw" 2>/dev/null || return 1
|
|
/usr/bin/awk '/-----BEGIN CERTIFICATE-----/{copy=1} copy{print} /-----END CERTIFICATE-----/{exit}' "$raw" >"$cert"
|
|
[[ -s "$cert" ]] || return 1
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 15s "$OPENSSL_BIN" s_client -connect "$LAN_IP:443" -servername "$GRAFANA_HOST" \
|
|
-verify_hostname "$GRAFANA_HOST" -verify_return_error </dev/null >/dev/null 2>&1 || verify_rc=$?
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$OPENSSL_BIN" x509 -in "$cert" -noout -checkend 604800 >/dev/null 2>&1 || expiry_rc=$?
|
|
extension="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$OPENSSL_BIN" x509 \
|
|
-in "$cert" -noout -ext subjectAltName 2>/dev/null)" || return 1
|
|
validate_certificate_extension_contract "$extension" "$verify_rc" "$expiry_rc" || return 1
|
|
printf 'OBSERVABILITY_MACHINE_CERTIFICATE=PASS\n'
|
|
}
|
|
|
|
check_nodeport_identity() {
|
|
local service="$WORK/traefik-nodeport-service.json"
|
|
kubectl_bounded -n kube-system get service traefik -o json >"$service" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$service"
|
|
jq_bounded -e '
|
|
.apiVersion=="v1" and .kind=="Service" and
|
|
.metadata.name=="traefik" and .metadata.namespace=="kube-system" and
|
|
(.metadata.uid|type=="string" and length>0) and
|
|
.spec.type=="NodePort" and .spec.externalTrafficPolicy=="Cluster" and
|
|
.spec.selector=={
|
|
"app.kubernetes.io/name":"traefik",
|
|
"app.kubernetes.io/instance":"traefik-kube-system"} and
|
|
([.spec.ports[]|{name,port,protocol,targetPort,nodePort}]|sort_by(.name))==([
|
|
{name:"web",port:80,protocol:"TCP",targetPort:"web",nodePort:30080},
|
|
{name:"websecure",port:443,protocol:"TCP",targetPort:"websecure",nodePort:30443}
|
|
]|sort_by(.name))
|
|
' "$service" >/dev/null || return 1
|
|
[[ "$(http_status --header 'Host: observability-smoke.invalid' http://127.0.0.1:30080/)" == 404 ]]
|
|
}
|
|
|
|
check_https_boundaries() {
|
|
local public_ip
|
|
[[ "$(http_status --interface "$LAN_IP" --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/")" =~ ^(200|302)$ ]] || return 1
|
|
[[ "$(http_status --interface "$TAILSCALE_IP" --resolve "$GRAFANA_HOST:443:$TAILSCALE_IP" "https://$GRAFANA_HOST/")" =~ ^(200|302)$ ]] || return 1
|
|
[[ "$(http_status --resolve "$GRAFANA_HOST:443:127.0.0.1" "https://$GRAFANA_HOST/")" == 403 ]] || return 1
|
|
public_ip="$(dig_values 1.1.1.1 A "$GITEA_HOST" | /usr/bin/head -n1)" || return 1
|
|
validate_ipv4 "$public_ip" || return 1
|
|
if [[ -n "$PUBLIC_EDGE_IP" && "$PUBLIC_EDGE_IP" != "$public_ip" ]]; then return 1; fi
|
|
PUBLIC_EDGE_IP=$public_ip
|
|
[[ "$(http_status --resolve "$GRAFANA_HOST:443:$public_ip" "https://$GRAFANA_HOST/")" == 403 ]] || return 1
|
|
[[ "$(http_status --resolve "$GRAFANA_HOST:443:$LAN_IP" "https://$GRAFANA_HOST/metrics")" == 404 ]] || return 1
|
|
[[ "$(http_status "https://$GITEA_HOST/metrics")" == 404 ]] || return 1
|
|
[[ "$(http_status "https://$KEYCLOAK_HOST/metrics")" == 404 ]] || return 1
|
|
[[ "$(http_status "$RUNBOOK_URL")" == 200 ]] || return 1
|
|
if "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 8s "$OPENSSL_BIN" s_client -connect "$LAN_IP:443" \
|
|
-servername unknown-sni.invalid </dev/null >/dev/null 2>&1; then return 1; fi
|
|
! "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$NC_BIN" -z -w 2 "$LAN_IP" 30080 || return 1
|
|
! "$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$NC_BIN" -z -w 2 "$LAN_IP" 30443 || return 1
|
|
check_nodeport_identity || return 1
|
|
printf 'OBSERVABILITY_MACHINE_EDGE=PASS\n'
|
|
}
|
|
|
|
check_targets() {
|
|
local file="$WORK/prometheus-targets.json"
|
|
proxy_get_200 "$(prometheus_path '/api/v1/targets?state=active')" "$file" || return 1
|
|
validate_target_contract "$file" || return 1
|
|
printf 'OBSERVABILITY_MACHINE_TARGETS=PASS\n'
|
|
}
|
|
|
|
check_one_query() {
|
|
local name=$1 expression=$2 count=$3 labels=$4 values=${5:-} value_label=${6:-}
|
|
local identity_json=${7:-} instance_pattern=${8:-}
|
|
local file="$WORK/query-$name.json"
|
|
prom_query "$expression" "$file" || return 1
|
|
validate_series_contract "$file" "$count" "$labels" "$values" "$value_label" || return 1
|
|
[[ -z "$identity_json" ]] || validate_series_identity_contract "$file" "$identity_json" "$instance_pattern"
|
|
}
|
|
|
|
check_metric_contracts() {
|
|
check_one_query gitea 'gitea_build_info{job="gitea",namespace="gitea"}' 1 'instance,job,namespace' '' '' \
|
|
'[{"job":"gitea","namespace":"gitea"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:3000$' || return 1
|
|
check_one_query keycloak 'process_uptime_seconds{job="keycloak-service",namespace="keycloak"}' 1 'instance,job,namespace' '' '' \
|
|
'[{"job":"keycloak-service","namespace":"keycloak"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1
|
|
check_one_query cnpg 'cnpg_collector_up{job="platform-postgres",namespace="platform-data",cluster="platform-postgres"}' 1 'cluster,instance,job,namespace' '' '' \
|
|
'[{"cluster":"platform-postgres","job":"platform-postgres","namespace":"platform-data"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9187$' || return 1
|
|
check_one_query traefik 'traefik_config_last_reload_success{job="traefik",namespace="kube-system"}' 1 'instance,job,namespace' '' '' \
|
|
'[{"job":"traefik","namespace":"kube-system"}]' '^([0-9]{1,3}[.]){3}[0-9]{1,3}:9100$' || return 1
|
|
check_one_query aistor-usage 'minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}' 2 \
|
|
'bucket,instance,job,namespace' 'loki,tempo' bucket \
|
|
'[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \
|
|
'^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1
|
|
check_one_query aistor-quota 'minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"} > 0' 2 \
|
|
'bucket,instance,job,namespace' 'loki,tempo' bucket \
|
|
'[{"bucket":"loki","job":"minio-aistor","namespace":"object-storage"},{"bucket":"tempo","job":"minio-aistor","namespace":"object-storage"}]' \
|
|
'^([0-9]{1,3}[.]){3}[0-9]{1,3}:9000$' || return 1
|
|
check_one_query quota-rule 'platform:aistor_bucket_quota_usage_percent' 2 'bucket' 'loki,tempo' bucket \
|
|
'[{"bucket":"loki"},{"bucket":"tempo"}]' || return 1
|
|
printf 'OBSERVABILITY_MACHINE_METRICS_QUOTA=PASS\n'
|
|
}
|
|
|
|
check_blackbox_series() {
|
|
local public='[{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"},{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]'
|
|
local private='[{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"},{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"},{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}]'
|
|
local internal='[{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"},{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"},{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}]'
|
|
check_one_query probe-public 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="public-edge"} == 1' 2 \
|
|
'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$public" || return 1
|
|
check_one_query probe-private 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"} == 1' 3 \
|
|
'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$private" || return 1
|
|
check_one_query probe-private-status 'probe_http_status_code{namespace="observability","observability.hyeonworks.com/probe-group"="private-edge"} == 403' 3 \
|
|
'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$private" || return 1
|
|
check_one_query probe-internal 'probe_success{namespace="observability","observability.hyeonworks.com/probe-group"="private-internal"} == 1' 3 \
|
|
'instance,job,namespace,observability.hyeonworks.com/probe-group' '' '' "$internal" || return 1
|
|
local tls="$WORK/query-probe-tls.json"
|
|
prom_query 'probe_ssl_earliest_cert_expiry{job=~"blackbox-public-edge|blackbox-private-edge"} > time()' "$tls" || return 1
|
|
validate_series_identity_contract "$tls" "$(jq_bounded -cn --argjson public "$public" --argjson private "$private" '$public+$private')" || return 1
|
|
printf 'OBSERVABILITY_MACHINE_BLACKBOX=PASS\n'
|
|
}
|
|
|
|
decode_exact_secret_to_netrc() {
|
|
local secret_json=$1 output=$2
|
|
python_bounded -I -S - "$secret_json" "$output" <<'PY'
|
|
import base64,binascii,json,os,pathlib,sys
|
|
source,destination=sys.argv[1:]
|
|
try: item=json.loads(pathlib.Path(source).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
meta=item.get('metadata') or {}; data=item.get('data')
|
|
if item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='observability' or meta.get('name')!='grafana-admin' or sorted(data or {})!=['admin-password','admin-user']:
|
|
raise SystemExit(1)
|
|
values=[]
|
|
for key in ('admin-user','admin-password'):
|
|
try: value=base64.b64decode(data[key],validate=True).decode('utf-8')
|
|
except (binascii.Error,UnicodeError,ValueError): raise SystemExit(1)
|
|
if not value or any(ch.isspace() for ch in value) or '\x00' in value: raise SystemExit(1)
|
|
values.append(value)
|
|
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
|
with os.fdopen(fd,'w') as stream: stream.write('machine localhost\nlogin '+values[0]+'\npassword '+values[1]+'\n')
|
|
PY
|
|
}
|
|
|
|
prepare_grafana_auth() {
|
|
local secret="$WORK/grafana-admin-secret.json" netrc="$WORK/grafana.netrc" extra="$WORK/grafana-curl.conf"
|
|
if safe_work_file "$netrc" && safe_work_file "$extra"; then
|
|
printf '%s' "$extra"
|
|
return 0
|
|
fi
|
|
kubectl_bounded -n observability get secret grafana-admin -o json >"$secret" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$secret"; safe_work_file "$secret" || return 1
|
|
decode_exact_secret_to_netrc "$secret" "$netrc" || return 1
|
|
printf 'netrc\nnetrc-file = "%s"\n' "$netrc" >"$extra"; "$CHMOD_BIN" 0600 -- "$extra"
|
|
safe_work_file "$netrc" && safe_work_file "$extra" || return 1
|
|
printf '%s' "$extra"
|
|
}
|
|
|
|
grafana_get() {
|
|
local api_path=$1 output=$2 auth=$3
|
|
proxy_get_200 "$(grafana_path "$api_path")" "$output" "$auth"
|
|
}
|
|
|
|
check_datasources() {
|
|
local auth prometheus loki tempo loki_config tempo_config tempo_search loki_query tempo_query combined trace encoded query
|
|
auth="$(prepare_grafana_auth)" || return 1
|
|
prometheus="$WORK/grafana-prometheus-health.json"; loki="$WORK/grafana-loki-health.json"; tempo="$WORK/grafana-tempo-health.json"
|
|
loki_config="$WORK/grafana-loki-config.json"; tempo_config="$WORK/grafana-tempo-config.json"
|
|
tempo_search="$WORK/grafana-tempo-search.json"; loki_query="$WORK/grafana-loki-query.json"
|
|
tempo_query="$WORK/grafana-tempo-query.json"; combined="$WORK/grafana-datasource-contract.json"
|
|
grafana_get '/api/datasources/uid/prometheus/health' "$prometheus" "$auth" || return 1
|
|
grafana_get '/api/datasources/uid/loki/health' "$loki" "$auth" || return 1
|
|
grafana_get '/api/datasources/uid/tempo/health' "$tempo" "$auth" || return 1
|
|
grafana_get '/api/datasources/uid/loki' "$loki_config" "$auth" || return 1
|
|
grafana_get '/api/datasources/uid/tempo' "$tempo_config" "$auth" || return 1
|
|
encoded="$(jq_bounded -nr --arg value 'service.name=platform-smoke' '$value|@uri')" || return 1
|
|
grafana_get "/api/datasources/proxy/uid/tempo/api/search?tags=$encoded&limit=20" "$tempo_search" "$auth" || return 1
|
|
trace="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$PYTHON_BIN" -I -S - "$tempo_search" <<'PY'
|
|
import json,pathlib,re,sys
|
|
try: item=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
traces=item.get('traces')
|
|
if not isinstance(traces,list): raise SystemExit(1)
|
|
candidates=[]
|
|
for trace in traces:
|
|
if not isinstance(trace,dict) or trace.get('rootServiceName')!='platform-smoke': continue
|
|
identifier=trace.get('traceID')
|
|
if isinstance(identifier,str) and re.fullmatch(r'[0-9a-f]{32}',identifier): candidates.append(identifier)
|
|
if not candidates: raise SystemExit(1)
|
|
print(candidates[0],end='')
|
|
PY
|
|
)" || return 1
|
|
[[ "$trace" =~ ^[0-9a-f]{32}$ ]] || return 1
|
|
query="$(synthetic_loki_query "$trace")" || return 1
|
|
encoded="$(jq_bounded -nr --arg value "$query" '$value|@uri')" || return 1
|
|
grafana_get "/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=$encoded&limit=20&direction=backward&since=720h" "$loki_query" "$auth" || return 1
|
|
grafana_get "/api/datasources/proxy/uid/tempo/api/traces/$trace" "$tempo_query" "$auth" || return 1
|
|
jq_bounded -n --slurpfile prometheus "$prometheus" --slurpfile loki "$loki" --slurpfile tempo "$tempo" \
|
|
--slurpfile loki_config "$loki_config" --slurpfile tempo_config "$tempo_config" \
|
|
--slurpfile tempo_search "$tempo_search" --slurpfile loki_query "$loki_query" --slurpfile tempo_query "$tempo_query" \
|
|
--arg selected_trace "$trace" \
|
|
'{prometheus:$prometheus[0],loki:$loki[0],tempo:$tempo[0],loki_config:$loki_config[0],tempo_config:$tempo_config[0],tempo_search:$tempo_search[0],selected_trace:$selected_trace,loki_query:$loki_query[0],tempo_query:$tempo_query[0]}' >"$combined" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$combined"
|
|
validate_datasource_contract "$combined" || return 1
|
|
printf 'OBSERVABILITY_MACHINE_DATASOURCES=PASS\n'
|
|
}
|
|
|
|
run_read_only_machine_checks() {
|
|
check_dns_host || { fail 'DNS host acceptance failed'; return 1; }
|
|
check_certificate || { fail 'certificate acceptance failed'; return 1; }
|
|
check_https_boundaries || { fail 'HTTPS/NodePort boundary acceptance failed'; return 1; }
|
|
check_targets || { fail 'Prometheus target acceptance failed'; return 1; }
|
|
check_metric_contracts || { fail 'metric/quota acceptance failed'; return 1; }
|
|
check_blackbox_series || { fail 'blackbox acceptance failed'; return 1; }
|
|
check_datasources || { fail 'Grafana datasource/link acceptance failed'; return 1; }
|
|
}
|
|
|
|
repeat_commit_gate() {
|
|
validate_commit_environment
|
|
}
|
|
|
|
validate_owned_object_json() {
|
|
local file=$1 kind=$2 namespace=$3 name=$4 run=$5 require_uid=${6:-false}
|
|
jq_bounded -e --arg kind "$kind" --arg namespace "$namespace" --arg name "$name" --arg run "$run" --argjson require_uid "$require_uid" '
|
|
.kind==$kind and .metadata.namespace==$namespace and .metadata.name==$name and
|
|
.metadata.labels["app.kubernetes.io/managed-by"]=="platform-observability-smoke" and
|
|
.metadata.labels["platform.hyeonworks.com/smoke-run"]==$run and
|
|
(($require_uid|not) or (.metadata.uid|type=="string" and length>0))
|
|
' "$file" >/dev/null
|
|
}
|
|
|
|
validate_live_matches_desired() {
|
|
local desired=$1 live=$2
|
|
python_bounded -I -S - "$desired" "$live" <<'PY'
|
|
import json,pathlib,sys
|
|
try:
|
|
desired=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
live=json.loads(pathlib.Path(sys.argv[2]).read_text())
|
|
except Exception:
|
|
raise SystemExit(1)
|
|
|
|
def subset(want,got):
|
|
if isinstance(want,dict):
|
|
return isinstance(got,dict) and all(key in got and subset(value,got[key]) for key,value in want.items())
|
|
if isinstance(want,list):
|
|
return isinstance(got,list) and len(want)==len(got) and all(subset(a,b) for a,b in zip(want,got))
|
|
return want==got
|
|
|
|
if desired.get('apiVersion')!=live.get('apiVersion') or desired.get('kind')!=live.get('kind'):
|
|
raise SystemExit(1)
|
|
dmeta=desired.get('metadata') or {}; lmeta=live.get('metadata') or {}
|
|
for field in ('name','namespace'):
|
|
if dmeta.get(field)!=lmeta.get(field): raise SystemExit(1)
|
|
if dmeta.get('labels')!=lmeta.get('labels'): raise SystemExit(1)
|
|
if not isinstance(lmeta.get('uid'),str) or not lmeta['uid'] or not isinstance(lmeta.get('resourceVersion'),str) or not lmeta['resourceVersion']:
|
|
raise SystemExit(1)
|
|
if lmeta.get('deletionTimestamp') or lmeta.get('ownerReferences') or lmeta.get('finalizers'):
|
|
raise SystemExit(1)
|
|
if not subset(desired.get('spec'),live.get('spec')): raise SystemExit(1)
|
|
if desired.get('kind')=='Pod':
|
|
dspec=desired.get('spec') or {}; lspec=live.get('spec') or {}
|
|
if len(lspec.get('containers') or [])!=len(dspec.get('containers') or []): raise SystemExit(1)
|
|
if lspec.get('initContainers') or lspec.get('ephemeralContainers'): raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
create_owned_from_file() {
|
|
local file=$1 kind=$2 namespace=$3 name=$4 run=$5 resource live rc=0 uid state outcome=0
|
|
resource="$(kind_resource "$kind")" || return 1
|
|
if state="$(owned_object_state "$kind" "$namespace" "$name" 2>/dev/null)"; then
|
|
fail "temporary object already exists: $kind/$namespace/$name"; return 1
|
|
else [[ $? == 1 ]] || return 1; fi
|
|
validate_owned_object_json "$file" "$kind" "$namespace" "$name" "$run" false || return 1
|
|
kubectl_bounded create --dry-run=server -f "$file" -o json >/dev/null || return 1
|
|
live="$WORK/created-${kind}-${name}.json"
|
|
begin_mutation_critical
|
|
if kubectl_bounded create -f "$file" -o json >"$live"; then rc=0; else rc=$?; fi
|
|
"$CHMOD_BIN" 0600 -- "$live"
|
|
if (( rc != 0 )); then
|
|
if ! kubectl_bounded -n "$namespace" get "$resource" "$name" --ignore-not-found -o json >"$live"; then
|
|
emit_manual_recovery; outcome=1
|
|
fi
|
|
fi
|
|
if (( outcome == 0 )) && [[ ! -s "$live" ]]; then emit_manual_recovery; outcome=1; fi
|
|
if (( outcome == 0 )) && ! validate_owned_object_json "$live" "$kind" "$namespace" "$name" "$run" true; then
|
|
emit_manual_recovery; outcome=1
|
|
fi
|
|
if (( outcome == 0 )); then
|
|
uid="$(jq_bounded -r '.metadata.uid' "$live")" || { emit_manual_recovery; outcome=1; }
|
|
fi
|
|
if (( outcome == 0 && rc == 0 )); then
|
|
# A successful create response is authoritative ownership even if an
|
|
# admission/defaulting drift makes the object unsafe; register its UID so
|
|
# the exit transaction can remove only that exact object.
|
|
register_owned "$kind" "$namespace" "$name" "$uid" "$run"
|
|
validate_live_matches_desired "$file" "$live" || outcome=1
|
|
elif (( outcome == 0 )); then
|
|
# A nonzero create is response-ambiguous. Copied labels are insufficient:
|
|
# only an exact desired-spec match can be classified as ours and cleaned.
|
|
if validate_live_matches_desired "$file" "$live"; then
|
|
register_owned "$kind" "$namespace" "$name" "$uid" "$run"
|
|
else
|
|
emit_manual_recovery
|
|
fi
|
|
outcome=1
|
|
fi
|
|
end_mutation_critical
|
|
return "$outcome"
|
|
}
|
|
|
|
validate_deployed_ingress_policy_contracts() {
|
|
local aistor=$1 blackbox=$2
|
|
jq_bounded -e '
|
|
.apiVersion=="networking.k8s.io/v1" and .kind=="NetworkPolicy" and
|
|
.metadata.name=="object-storage-allow-prometheus-metrics" and
|
|
.metadata.namespace=="object-storage" and
|
|
(.metadata.uid|type=="string" and length>0) and
|
|
(.metadata.resourceVersion|type=="string" and length>0) and
|
|
(.metadata.deletionTimestamp==null) and
|
|
.spec=={
|
|
podSelector:{matchLabels:{"aistor.min.io/objectStore":"minio-aistor"}},
|
|
policyTypes:["Ingress"],
|
|
ingress:[{from:[{
|
|
namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"observability"}},
|
|
podSelector:{matchLabels:{
|
|
"app.kubernetes.io/name":"prometheus",
|
|
"app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}
|
|
}],ports:[{protocol:"TCP",port:9000}]}]
|
|
}
|
|
' "$aistor" >/dev/null || return 1
|
|
jq_bounded -e '
|
|
.apiVersion=="networking.k8s.io/v1" and .kind=="NetworkPolicy" and
|
|
.metadata.name=="observability-allow-prometheus-to-blackbox" and
|
|
.metadata.namespace=="observability" and
|
|
(.metadata.uid|type=="string" and length>0) and
|
|
(.metadata.resourceVersion|type=="string" and length>0) and
|
|
(.metadata.deletionTimestamp==null) and
|
|
.spec=={
|
|
podSelector:{matchLabels:{
|
|
"app.kubernetes.io/name":"prometheus-blackbox-exporter",
|
|
"app.kubernetes.io/instance":"blackbox-exporter"}},
|
|
policyTypes:["Ingress"],
|
|
ingress:[{from:[{podSelector:{matchLabels:{
|
|
"app.kubernetes.io/name":"prometheus",
|
|
"app.kubernetes.io/instance":"observability-core-kube-pr-prometheus"}}
|
|
}],ports:[{protocol:"TCP",port:9115}]}]
|
|
}
|
|
' "$blackbox" >/dev/null
|
|
}
|
|
|
|
run_network_boundary_transaction() {
|
|
local run name policy pod minio_ip blackbox_ip pod_result aistor_ingress blackbox_ingress
|
|
run="$(new_smoke_run_id network)" || return 1
|
|
name="$(kubernetes_smoke_name "$run")" || return 1
|
|
policy="$WORK/network-policy.json"; pod="$WORK/network-pod.json"
|
|
aistor_ingress="$WORK/aistor-ingress-policy.json"
|
|
blackbox_ingress="$WORK/blackbox-ingress-policy.json"
|
|
kubectl_bounded -n object-storage get networkpolicy object-storage-allow-prometheus-metrics -o json >"$aistor_ingress" || return 1
|
|
kubectl_bounded -n observability get networkpolicy observability-allow-prometheus-to-blackbox -o json >"$blackbox_ingress" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$aistor_ingress" "$blackbox_ingress"
|
|
validate_deployed_ingress_policy_contracts "$aistor_ingress" "$blackbox_ingress" || return 1
|
|
minio_ip="$(kubectl_bounded -n object-storage get service minio -o jsonpath='{.spec.clusterIP}')" || return 1
|
|
blackbox_ip="$(kubectl_bounded -n observability get service blackbox-exporter -o jsonpath='{.spec.clusterIP}')" || return 1
|
|
[[ "$minio_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$blackbox_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || return 1
|
|
jq_bounded -n --arg name "$name" --arg run "$run" '{
|
|
apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",
|
|
metadata:{name:$name,namespace:"observability",labels:{
|
|
"app.kubernetes.io/managed-by":"platform-observability-smoke",
|
|
"platform.hyeonworks.com/smoke-run":$run}},
|
|
spec:{podSelector:{matchLabels:{"platform.hyeonworks.com/smoke-run":$run}},
|
|
policyTypes:["Egress"],egress:[{to:[{
|
|
namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"kube-system"}},
|
|
podSelector:{matchLabels:{"k8s-app":"kube-dns"}}}],
|
|
ports:[{protocol:"UDP",port:53},{protocol:"TCP",port:53}]},{to:[{
|
|
namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"object-storage"}},
|
|
podSelector:{matchLabels:{"aistor.min.io/objectStore":"minio-aistor"}}}],
|
|
ports:[{protocol:"TCP",port:9000}]},{to:[{
|
|
namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":"observability"}},
|
|
podSelector:{matchLabels:{
|
|
"app.kubernetes.io/name":"prometheus-blackbox-exporter",
|
|
"app.kubernetes.io/instance":"blackbox-exporter"}}}],
|
|
ports:[{protocol:"TCP",port:9115}]}]}}
|
|
' >"$policy" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$policy"
|
|
local command
|
|
command="set -eu
|
|
resolved=\$(nslookup $GRAFANA_HOST | awk '/^Address: /{print \$2}' | tail -n1)
|
|
[ \"\$resolved\" = $LAN_IP ]
|
|
if nc -z -w 4 $minio_ip 9000; then exit 41; fi
|
|
if nc -z -w 4 $blackbox_ip 9115; then exit 42; fi
|
|
printf '%s\\n' OBSERVABILITY_NETWORK_BOUNDARY_PASS"
|
|
jq_bounded -n --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg command "$command" '{
|
|
apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:"observability",labels:{
|
|
"app.kubernetes.io/managed-by":"platform-observability-smoke",
|
|
"platform.hyeonworks.com/smoke-run":$run}},
|
|
spec:{automountServiceAccountToken:false,restartPolicy:"Never",terminationGracePeriodSeconds:1,
|
|
securityContext:{runAsNonRoot:true,runAsUser:65534,runAsGroup:65534,seccompProfile:{type:"RuntimeDefault"}},
|
|
containers:[{name:"network-smoke",image:$image,imagePullPolicy:"IfNotPresent",
|
|
command:["sh","-ec",$command],securityContext:{allowPrivilegeEscalation:false,
|
|
readOnlyRootFilesystem:true,capabilities:{drop:["ALL"]}}}]}}
|
|
' >"$pod" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$pod"
|
|
create_owned_from_file "$policy" NetworkPolicy observability "$name" "$run" || return 1
|
|
create_owned_from_file "$pod" Pod observability "$name" "$run" || return 1
|
|
kubectl_wait_90_bounded -n observability wait --for=jsonpath='{.status.phase}'=Succeeded \
|
|
--timeout=90s "pod/$name" >/dev/null || return 1
|
|
pod_result="$(kubectl_bounded -n observability logs "$name" -c network-smoke)" || return 1
|
|
[[ "$pod_result" == OBSERVABILITY_NETWORK_BOUNDARY_PASS ]] || return 1
|
|
cleanup_owned_objects || return 1
|
|
printf 'OBSERVABILITY_MACHINE_NETWORK_POLICY=PASS\n'
|
|
}
|
|
|
|
canonical_object_snapshot() {
|
|
local file=$1
|
|
jq_bounded -S '{metadata:{name:.metadata.name,namespace:.metadata.namespace,uid:.metadata.uid},data:(.data//{}),status:(.status//{})}' "$file"
|
|
}
|
|
|
|
list_one_ready_pod() {
|
|
local selector=$1 output=$2
|
|
kubectl_bounded -n observability get pods -l "$selector" -o json >"$output" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$output"
|
|
jq_bounded -e '.items|length==1 and .[0].metadata.uid and any(.[0].status.conditions[]?; .type=="Ready" and .status=="True")' "$output" >/dev/null
|
|
}
|
|
|
|
delete_pod_uid_for_restart() {
|
|
local name=$1 uid=$2 body response path status attempt current
|
|
body="$("$MKTEMP_BIN" "$WORK/restart-delete.XXXXXXXX")" || return 1
|
|
response="$("$MKTEMP_BIN" "$WORK/restart-response.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$body" "$response"
|
|
jq_bounded -nc --arg uid "$uid" '{apiVersion:"meta.k8s.io/v1",kind:"DeleteOptions",propagationPolicy:"Background",preconditions:{uid:$uid}}' >"$body"
|
|
path="/api/v1/namespaces/observability/pods/$name"
|
|
status="$(proxy_request DELETE "$path" "$response" "$body")" || status=000
|
|
for ((attempt=0; attempt<60; attempt++)); do
|
|
current="$(kubectl_bounded -n observability get pod "$name" --ignore-not-found -o json 2>/dev/null | jq_bounded -r '.metadata.uid//empty')" || return 1
|
|
[[ -z "$current" || "$current" != "$uid" ]] && return 0
|
|
/usr/bin/sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
fetch_silences() {
|
|
local output=$1 status
|
|
status="$(proxy_request GET "$(alertmanager_path '/api/v2/silences')" "$output")" || return 1
|
|
[[ "$status" == 200 ]] || return 1
|
|
jq_bounded -e 'type=="array"' "$output" >/dev/null
|
|
}
|
|
|
|
silence_inventory_state() {
|
|
local source=$1 run=$2
|
|
python_bounded -I -S - "$source" "$run" <<'PY'
|
|
import json,pathlib,re,sys
|
|
try: items=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
run=sys.argv[2]
|
|
if not isinstance(items,list) or re.fullmatch(r'persistence-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}',run) is None: raise SystemExit(1)
|
|
related=[]; active=[]; expired=[]; structural=[]
|
|
expected={
|
|
('alertname','PlatformObservabilityPersistenceSmoke',False,True),
|
|
('instance',run,False,True),
|
|
}
|
|
for item in items:
|
|
if not isinstance(item,dict): raise SystemExit(1)
|
|
matchers=item.get('matchers')
|
|
if not isinstance(matchers,list): continue
|
|
tuples=[]
|
|
malformed=False
|
|
for matcher in matchers:
|
|
if not isinstance(matcher,dict): malformed=True; break
|
|
value=(matcher.get('name'),matcher.get('value'),matcher.get('isRegex'),matcher.get('isEqual'))
|
|
tuples.append(value)
|
|
if malformed: continue
|
|
if any(value[0]=='instance' and value[1]==run for value in tuples): related.append(item)
|
|
state=((item.get('status') or {}).get('state'))
|
|
if set(tuples)==expected and len(tuples)==2 and item.get('createdBy')=='platform-observability-smoke' and item.get('comment')=='Temporary restart-persistence acceptance '+run:
|
|
identifier=item.get('id')
|
|
if not isinstance(identifier,str) or re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*',identifier) is None: raise SystemExit(1)
|
|
structural.append(identifier)
|
|
if state in ('active','pending'): active.append(identifier)
|
|
elif state=='expired': expired.append(identifier)
|
|
print(f'{len(related)}|{len(active)}|{len(expired)}|{structural[0] if len(structural)==1 else ""}')
|
|
PY
|
|
}
|
|
|
|
read_silence_state() {
|
|
local run=$1 output
|
|
output="$("$MKTEMP_BIN" "$WORK/silence-list.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$output"
|
|
fetch_silences "$output" || return 1
|
|
silence_inventory_state "$output" "$run"
|
|
}
|
|
|
|
create_persistence_silence() {
|
|
local body="$WORK/silence-create.json" response="$WORK/silence-create-response.json"
|
|
local status='' start end state related active expired observed_id response_id='' request_rc=0 attempt outcome=1 classified=false
|
|
SILENCE_RUN="$(new_smoke_run_id persistence)" || return 1
|
|
state="$(read_silence_state "$SILENCE_RUN")" || return 1
|
|
IFS='|' read -r related active expired observed_id <<<"$state"
|
|
if [[ "$related" != 0 || "$active" != 0 || "$expired" != 0 || -n "$observed_id" ]]; then
|
|
emit_manual_recovery; return 1
|
|
fi
|
|
start="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1
|
|
end="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u -d '+20 minutes' +%Y-%m-%dT%H:%M:%SZ)" || return 1
|
|
jq_bounded -n --arg start "$start" --arg end "$end" --arg run "$SILENCE_RUN" '{
|
|
matchers:[
|
|
{name:"alertname",value:"PlatformObservabilityPersistenceSmoke",isRegex:false,isEqual:true},
|
|
{name:"instance",value:$run,isRegex:false,isEqual:true}
|
|
],startsAt:$start,endsAt:$end,createdBy:"platform-observability-smoke",
|
|
comment:("Temporary restart-persistence acceptance "+$run)
|
|
}' >"$body" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$body"
|
|
begin_mutation_critical
|
|
status="$(proxy_request POST "$(alertmanager_path '/api/v2/silences')" "$response" "$body")" || request_rc=$?
|
|
if (( request_rc == 0 )) && [[ "$status" == 200 ]]; then
|
|
response_id="$(jq_bounded -er '.silenceID|select(type=="string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))' "$response" 2>/dev/null)" || response_id=''
|
|
fi
|
|
for ((attempt=0; attempt<SILENCE_POLL_ATTEMPTS; attempt++)); do
|
|
state="$(read_silence_state "$SILENCE_RUN")" || { (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"; continue; }
|
|
IFS='|' read -r related active expired observed_id <<<"$state"
|
|
if [[ "$related" == 1 && "$active" == 1 && "$expired" == 0 && "$observed_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
|
|
if [[ -n "$response_id" && "$response_id" != "$observed_id" ]]; then
|
|
emit_manual_recovery
|
|
else
|
|
SILENCE_ID=$observed_id
|
|
outcome=0
|
|
fi
|
|
classified=true
|
|
break
|
|
fi
|
|
if [[ "$related" != 0 || "$active" != 0 || "$expired" != 0 ]]; then emit_manual_recovery; classified=true; break; fi
|
|
(( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"
|
|
done
|
|
[[ "$classified" == true ]] || emit_manual_recovery
|
|
end_mutation_critical
|
|
return "$outcome"
|
|
}
|
|
|
|
delete_persistence_silence() {
|
|
local response status='' request_rc=0 state related active expired observed_id attempt terminal=0
|
|
if [[ -z "$SILENCE_ID" && -z "$SILENCE_RUN" ]]; then return 0; fi
|
|
if [[ ! "$SILENCE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ || ! "$SILENCE_RUN" =~ ^persistence-[0-9]{8}t[0-9]{6}z-[0-9a-f]{32}$ ]]; then
|
|
emit_manual_recovery; return 1
|
|
fi
|
|
response="$WORK/silence-delete-response.json"
|
|
status="$(proxy_request DELETE "$(alertmanager_path "/api/v2/silence/$SILENCE_ID")" "$response")" || request_rc=$?
|
|
for ((attempt=0; attempt<SILENCE_POLL_ATTEMPTS; attempt++)); do
|
|
state="$(read_silence_state "$SILENCE_RUN")" || { terminal=0; (( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"; continue; }
|
|
IFS='|' read -r related active expired observed_id <<<"$state"
|
|
if [[ ( "$related" == 0 && "$active" == 0 && "$expired" == 0 && -z "$observed_id" ) ||
|
|
( "$related" == 1 && "$active" == 0 && "$expired" == 1 && "$observed_id" == "$SILENCE_ID" ) ]]; then
|
|
terminal=$((terminal + 1))
|
|
if (( terminal >= 2 )); then SILENCE_ID=''; SILENCE_RUN=''; return 0; fi
|
|
else
|
|
terminal=0
|
|
if [[ "$related" != 1 || "$active" != 1 || "$expired" != 0 || "$observed_id" != "$SILENCE_ID" ]]; then
|
|
emit_manual_recovery; return 1
|
|
fi
|
|
fi
|
|
(( SILENCE_POLL_DELAY == 0 )) || /usr/bin/sleep "$SILENCE_POLL_DELAY"
|
|
done
|
|
emit_manual_recovery
|
|
return 1
|
|
}
|
|
|
|
run_restart_persistence_transaction() {
|
|
local grafana_before="$WORK/grafana-pod-before.json" grafana_after="$WORK/grafana-pod-after.json"
|
|
local alert_before="$WORK/alertmanager-pod-before.json" alert_after="$WORK/alertmanager-pod-after.json"
|
|
local pvc_before="$WORK/pvc-before.json" pvc_after="$WORK/pvc-after.json"
|
|
local dashboards_before="$WORK/dashboards-before.json" dashboards_after="$WORK/dashboards-after.json"
|
|
local api_before="$WORK/dashboard-api-before.json" api_after="$WORK/dashboard-api-after.json"
|
|
local silence="$WORK/silence-after-restart.json" auth name uid state related active expired observed_id
|
|
auth="$WORK/grafana-curl.conf"; safe_work_file "$auth" || return 1
|
|
list_one_ready_pod 'app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana' "$grafana_before" || return 1
|
|
list_one_ready_pod 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' "$alert_before" || return 1
|
|
{
|
|
kubectl_bounded -n observability get pvc grafana -o json
|
|
kubectl_bounded -n observability get pvc -l 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' -o json
|
|
} | jq_bounded -s '.[0] as $grafana | .[1] as $am | {grafana:{name:$grafana.metadata.name,uid:$grafana.metadata.uid,phase:$grafana.status.phase},alertmanager:[$am.items[]|{name:.metadata.name,uid:.metadata.uid,phase:.status.phase}]}' >"$pvc_before" || return 1
|
|
kubectl_bounded -n observability get configmaps -l grafana_dashboard=1 -o json | jq_bounded -S \
|
|
'[.items[]|{name:.metadata.name,uid:.metadata.uid,data:.data}]|sort_by(.name)' >"$dashboards_before" || return 1
|
|
grafana_get '/api/search?type=dash-db' "$api_before" "$auth" || return 1
|
|
jq_bounded -S '[.[]|{uid,title,type}]|sort_by(.uid)' "$api_before" >"$api_before.canonical" || return 1
|
|
"$MV_BIN" -- "$api_before.canonical" "$api_before"
|
|
"$CHMOD_BIN" 0600 -- "$pvc_before" "$dashboards_before" "$api_before"
|
|
jq_bounded -e '.grafana.phase=="Bound" and (.alertmanager|length==1) and all(.alertmanager[];.phase=="Bound")' "$pvc_before" >/dev/null || return 1
|
|
jq_bounded -e 'length==5' "$dashboards_before" >/dev/null || return 1
|
|
jq_bounded -e 'length==5 and all(.[];.uid|type=="string" and length>0)' "$api_before" >/dev/null || return 1
|
|
create_persistence_silence || return 1
|
|
|
|
name="$(jq_bounded -r '.items[0].metadata.name' "$grafana_before")"; uid="$(jq_bounded -r '.items[0].metadata.uid' "$grafana_before")"
|
|
delete_pod_uid_for_restart "$name" "$uid" || return 1
|
|
kubectl_wait_180_bounded -n observability rollout status deployment/grafana --timeout=180s >/dev/null || return 1
|
|
name="$(jq_bounded -r '.items[0].metadata.name' "$alert_before")"; uid="$(jq_bounded -r '.items[0].metadata.uid' "$alert_before")"
|
|
delete_pod_uid_for_restart "$name" "$uid" || return 1
|
|
kubectl_wait_180_bounded -n observability rollout status statefulset/alertmanager-observability-core-kube-pr-alertmanager --timeout=180s >/dev/null || return 1
|
|
|
|
list_one_ready_pod 'app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana' "$grafana_after" || return 1
|
|
list_one_ready_pod 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' "$alert_after" || return 1
|
|
[[ "$(jq_bounded -r '.items[0].metadata.uid' "$grafana_before")" != "$(jq_bounded -r '.items[0].metadata.uid' "$grafana_after")" ]] || return 1
|
|
[[ "$(jq_bounded -r '.items[0].metadata.uid' "$alert_before")" != "$(jq_bounded -r '.items[0].metadata.uid' "$alert_after")" ]] || return 1
|
|
{
|
|
kubectl_bounded -n observability get pvc grafana -o json
|
|
kubectl_bounded -n observability get pvc -l 'app.kubernetes.io/name=alertmanager,app.kubernetes.io/instance=observability-core-kube-pr-alertmanager' -o json
|
|
} | jq_bounded -s '.[0] as $grafana | .[1] as $am | {grafana:{name:$grafana.metadata.name,uid:$grafana.metadata.uid,phase:$grafana.status.phase},alertmanager:[$am.items[]|{name:.metadata.name,uid:.metadata.uid,phase:.status.phase}]}' >"$pvc_after" || return 1
|
|
kubectl_bounded -n observability get configmaps -l grafana_dashboard=1 -o json | jq_bounded -S \
|
|
'[.items[]|{name:.metadata.name,uid:.metadata.uid,data:.data}]|sort_by(.name)' >"$dashboards_after" || return 1
|
|
grafana_get '/api/search?type=dash-db' "$api_after" "$auth" || return 1
|
|
jq_bounded -S '[.[]|{uid,title,type}]|sort_by(.uid)' "$api_after" >"$api_after.canonical" || return 1
|
|
"$MV_BIN" -- "$api_after.canonical" "$api_after"
|
|
"$CMP_BIN" --silent -- "$pvc_before" "$pvc_after" || return 1
|
|
"$CMP_BIN" --silent -- "$dashboards_before" "$dashboards_after" || return 1
|
|
"$CMP_BIN" --silent -- "$api_before" "$api_after" || return 1
|
|
fetch_silences "$silence" || return 1
|
|
state="$(silence_inventory_state "$silence" "$SILENCE_RUN")" || return 1
|
|
IFS='|' read -r related active expired observed_id <<<"$state"
|
|
[[ "$related" == 1 && "$active" == 1 && "$expired" == 0 && "$observed_id" == "$SILENCE_ID" ]] || return 1
|
|
delete_persistence_silence || return 1
|
|
printf 'OBSERVABILITY_MACHINE_RESTART_PERSISTENCE=PASS\n'
|
|
}
|
|
|
|
run_certbot_acceptance() {
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=2s 30s "$SUDO_BIN" -v || return 1
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$SUDO_BIN" -n /usr/bin/true || return 1
|
|
"$TIMEOUT_BIN" --signal=TERM --kill-after=5s 610s "$SUDO_BIN" -n "$TIMEOUT_BIN" --signal=TERM --kill-after=5s 600s \
|
|
/snap/bin/certbot renew --dry-run --run-deploy-hooks >/dev/null || return 1
|
|
[[ "$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 10s "$SUDO_BIN" -n /usr/bin/systemctl is-active nginx)" == active ]] || return 1
|
|
printf 'OBSERVABILITY_MACHINE_CERTBOT=PASS\n'
|
|
}
|
|
|
|
decode_keycloak_admin_secret() {
|
|
local secret_json=$1 username_file=$2 password_file=$3
|
|
python_bounded -I -S - "$secret_json" "$username_file" "$password_file" <<'PY'
|
|
import base64,binascii,json,os,pathlib,sys
|
|
source,user_path,password_path=sys.argv[1:]
|
|
try: item=json.loads(pathlib.Path(source).read_text())
|
|
except Exception: raise SystemExit(1)
|
|
meta=item.get('metadata') or {}; data=item.get('data')
|
|
if item.get('kind')!='Secret' or item.get('type')!='Opaque' or meta.get('namespace')!='keycloak' or meta.get('name')!='keycloak-initial-admin' or sorted(data or {})!=['password','username']:
|
|
raise SystemExit(1)
|
|
for key,destination in [('username',user_path),('password',password_path)]:
|
|
try: value=base64.b64decode(data[key],validate=True)
|
|
except (binascii.Error,ValueError): raise SystemExit(1)
|
|
if not value or b'\x00' in value or b'\r' in value or b'\n' in value: raise SystemExit(1)
|
|
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
|
with os.fdopen(fd,'wb') as stream: stream.write(value)
|
|
PY
|
|
}
|
|
|
|
prepare_keycloak_auth() {
|
|
local secret="$WORK/keycloak-admin-secret.json" username="$WORK/keycloak-admin-user" password="$WORK/keycloak-admin-password"
|
|
local form="$WORK/keycloak-token-form" config="$WORK/keycloak-token-curl.conf" response="$WORK/keycloak-token.json" status token
|
|
kubectl_bounded -n keycloak get secret keycloak-initial-admin -o json >"$secret" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$secret"; decode_keycloak_admin_secret "$secret" "$username" "$password" || return 1
|
|
python_bounded -I -S - "$username" "$password" "$form" <<'PY'
|
|
import os,pathlib,sys,urllib.parse
|
|
user,password,destination=sys.argv[1:]
|
|
values={'grant_type':'password','client_id':'admin-cli','username':pathlib.Path(user).read_text(),'password':pathlib.Path(password).read_text()}
|
|
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
|
with os.fdopen(fd,'w') as stream: stream.write(urllib.parse.urlencode(values))
|
|
PY
|
|
printf 'header = "Content-Type: application/x-www-form-urlencoded"\ndata-binary = "@%s"\n' "$form" >"$config"; "$CHMOD_BIN" 0600 -- "$config"
|
|
status="$(proxy_request POST "$(keycloak_path '/realms/master/protocol/openid-connect/token')" "$response" '' "$config")" || return 1
|
|
[[ "$status" == 200 ]] || return 1
|
|
token="$(jq_bounded -er '.access_token|select(type=="string" and test("^[A-Za-z0-9._~-]+$"))' "$response")" || return 1
|
|
KEYCLOAK_AUTH_CONFIG="$WORK/keycloak-auth.conf"
|
|
printf 'header = "Authorization: Bearer %s"\n' "$token" >"$KEYCLOAK_AUTH_CONFIG"; "$CHMOD_BIN" 0600 -- "$KEYCLOAK_AUTH_CONFIG"
|
|
safe_work_file "$KEYCLOAK_AUTH_CONFIG"
|
|
}
|
|
|
|
keycloak_request() {
|
|
local method=$1 api_path=$2 output=$3 body=${4:-} status
|
|
safe_work_file "$KEYCLOAK_AUTH_CONFIG" || return 1
|
|
status="$(proxy_request "$method" "$(keycloak_path "$api_path")" "$output" "$body" "$KEYCLOAK_AUTH_CONFIG")" || return 1
|
|
printf '%s' "$status"
|
|
}
|
|
|
|
keycloak_lookup_group() {
|
|
local name=$1 expected_path=$2 output=$3 encoded status
|
|
encoded="$(jq_bounded -nr --arg value "$name" '$value|@uri')" || return 1
|
|
status="$(keycloak_request GET "/admin/realms/hyeonworks/groups?search=$encoded&exact=true&max=2" "$output")" || return 1
|
|
[[ "$status" == 200 ]] || return 1
|
|
jq_bounded -er --arg name "$name" --arg path "$expected_path" '
|
|
[ .[] | select(.name==$name and .path==$path) ] as $items |
|
|
select(($items|length)==1) | $items[0].id | select(type=="string" and test("^[A-Za-z0-9._-]+$"))
|
|
' "$output"
|
|
}
|
|
|
|
keycloak_lookup_user() {
|
|
local username=$1 output=$2 encoded status
|
|
encoded="$(jq_bounded -nr --arg value "$username" '$value|@uri')" || return 1
|
|
status="$(keycloak_request GET "/admin/realms/hyeonworks/users?username=$encoded&exact=true&max=2" "$output")" || return 1
|
|
[[ "$status" == 200 ]] || return 1
|
|
jq_bounded -er --arg username "$username" '
|
|
[ .[] | select(.username==$username and .enabled==true) ] as $items |
|
|
select(($items|length)==1) | $items[0].id | select(type=="string" and test("^[A-Za-z0-9._-]+$"))
|
|
' "$output"
|
|
}
|
|
|
|
keycloak_memberships() {
|
|
local output=$1 status page aggregate next count first=0 pages=0
|
|
aggregate="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-all.XXXXXXXX")" || return 1
|
|
printf '[]\n' >"$aggregate"; "$CHMOD_BIN" 0600 -- "$aggregate"
|
|
while (( pages < 100 )); do
|
|
page="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-page.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$page"
|
|
status="$(keycloak_request GET "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups?briefRepresentation=true&first=$first&max=100" "$page")" || return 1
|
|
[[ "$status" == 200 ]] || return 1
|
|
count="$(jq_bounded -er 'select(type=="array" and length<=100)|length' "$page")" || return 1
|
|
next="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-next.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$next"
|
|
jq_bounded -s '.[0]+.[1]' "$aggregate" "$page" >"$next" || return 1
|
|
"$MV_BIN" -- "$next" "$aggregate"
|
|
pages=$((pages + 1))
|
|
(( count == 100 )) || break
|
|
first=$((first + 100))
|
|
done
|
|
(( pages < 100 || count < 100 )) || return 1
|
|
jq_bounded -S '
|
|
select(type=="array" and all(.[];
|
|
(.id|type=="string" and test("^[A-Za-z0-9._-]+$")) and
|
|
(.name|type=="string" and length>0) and (.path|type=="string" and startswith("/")))) |
|
|
select(([.[].id]|length)==([.[].id]|unique|length)) |
|
|
sort_by(.id)
|
|
' "$aggregate" >"$output" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$output"
|
|
}
|
|
|
|
keycloak_membership_state() {
|
|
local memberships=$1 group_id=$2
|
|
jq_bounded -er --arg id "$group_id" '
|
|
[ .[] | select(.id==$id) ] as $items |
|
|
if ($items|length)==0 then "absent"
|
|
elif ($items|length)==1 and $items[0].name=="platform-observability-viewers" and
|
|
$items[0].path=="/platform-observability-viewers" then "present"
|
|
else error("ambiguous viewer membership") end
|
|
' "$memberships"
|
|
}
|
|
|
|
capture_keycloak_absent_baseline() {
|
|
local admin_group_id=$1 first second state
|
|
[[ "$admin_group_id" =~ ^[A-Za-z0-9._-]+$ && "$KEYCLOAK_VIEWER_GROUP_ID" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
|
|
first="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-before-a.XXXXXXXX")" || return 1
|
|
second="$("$MKTEMP_BIN" "$WORK/keycloak-memberships-before-b.XXXXXXXX")" || return 1
|
|
keycloak_memberships "$first" || return 1
|
|
keycloak_memberships "$second" || return 1
|
|
"$CMP_BIN" --silent -- "$first" "$second" || return 1
|
|
state="$(keycloak_membership_state "$first" "$KEYCLOAK_VIEWER_GROUP_ID")" || return 1
|
|
[[ "$state" == absent ]] || return 1
|
|
jq_bounded -e --arg admin "$admin_group_id" 'all(.[]?; .id!=$admin)' "$first" >/dev/null || return 1
|
|
[[ -n "$KEYCLOAK_MEMBERSHIP_BEFORE" && "$KEYCLOAK_MEMBERSHIP_BEFORE" == "$WORK"/* ]] || return 1
|
|
"$MV_BIN" -- "$first" "$KEYCLOAK_MEMBERSHIP_BEFORE"
|
|
"$CHMOD_BIN" 0600 -- "$KEYCLOAK_MEMBERSHIP_BEFORE"
|
|
}
|
|
|
|
add_keycloak_viewer_membership() {
|
|
local response memberships status=000 state outcome=1
|
|
[[ -f "$KEYCLOAK_MEMBERSHIP_BEFORE" && ! -L "$KEYCLOAK_MEMBERSHIP_BEFORE" ]] || return 1
|
|
[[ "$(keycloak_membership_state "$KEYCLOAK_MEMBERSHIP_BEFORE" "$KEYCLOAK_VIEWER_GROUP_ID")" == absent ]] || return 1
|
|
response="$WORK/keycloak-membership-add-response.json"
|
|
memberships="$WORK/keycloak-memberships-added.json"
|
|
begin_mutation_critical
|
|
KEYCLOAK_MEMBERSHIP_PENDING=true
|
|
status="$(keycloak_request PUT "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups/$KEYCLOAK_VIEWER_GROUP_ID" "$response")" || status=000
|
|
if keycloak_memberships "$memberships"; then
|
|
state="$(keycloak_membership_state "$memberships" "$KEYCLOAK_VIEWER_GROUP_ID")" || state=ambiguous
|
|
if [[ "$state" == present ]]; then
|
|
KEYCLOAK_TEST_ADDED=true
|
|
KEYCLOAK_MEMBERSHIP_PENDING=false
|
|
outcome=0
|
|
elif [[ "$state" == absent ]]; then
|
|
KEYCLOAK_MEMBERSHIP_PENDING=false
|
|
fi
|
|
fi
|
|
end_mutation_critical
|
|
(( outcome == 0 )) || return 1
|
|
[[ "$status" == 204 || "$status" == 000 ]]
|
|
}
|
|
|
|
cleanup_keycloak_membership() {
|
|
local response status memberships verify state
|
|
[[ ( "$KEYCLOAK_TEST_ADDED" == true || "$KEYCLOAK_MEMBERSHIP_PENDING" == true ) &&
|
|
"$KEYCLOAK_MEMBERSHIP_REMOVED" == false ]] || return 0
|
|
[[ "$KEYCLOAK_USER_ID" =~ ^[A-Za-z0-9._-]+$ && "$KEYCLOAK_VIEWER_GROUP_ID" =~ ^[A-Za-z0-9._-]+$ ]] || {
|
|
emit_manual_recovery; return 1;
|
|
}
|
|
[[ -f "$KEYCLOAK_MEMBERSHIP_BEFORE" && ! -L "$KEYCLOAK_MEMBERSHIP_BEFORE" &&
|
|
"$(keycloak_membership_state "$KEYCLOAK_MEMBERSHIP_BEFORE" "$KEYCLOAK_VIEWER_GROUP_ID" 2>/dev/null)" == absent ]] || {
|
|
emit_manual_recovery; return 1;
|
|
}
|
|
memberships="$WORK/keycloak-cleanup-memberships-before.json"
|
|
keycloak_memberships "$memberships" || { emit_manual_recovery; return 1; }
|
|
state="$(keycloak_membership_state "$memberships" "$KEYCLOAK_VIEWER_GROUP_ID")" || {
|
|
emit_manual_recovery; return 1;
|
|
}
|
|
if [[ "$state" == absent ]]; then
|
|
KEYCLOAK_MEMBERSHIP_PENDING=false; KEYCLOAK_MEMBERSHIP_REMOVED=true
|
|
return 0
|
|
fi
|
|
response="$WORK/keycloak-cleanup-response.json"
|
|
status="$(keycloak_request DELETE "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/groups/$KEYCLOAK_VIEWER_GROUP_ID" "$response")" || status=000
|
|
verify="$WORK/keycloak-cleanup-memberships-after.json"
|
|
keycloak_memberships "$verify" || { emit_manual_recovery; return 1; }
|
|
[[ "$(keycloak_membership_state "$verify" "$KEYCLOAK_VIEWER_GROUP_ID")" == absent ]] || {
|
|
emit_manual_recovery; return 1;
|
|
}
|
|
KEYCLOAK_MEMBERSHIP_PENDING=false
|
|
KEYCLOAK_MEMBERSHIP_REMOVED=true
|
|
}
|
|
|
|
run_oidc_acceptance() {
|
|
local identity admin_identity viewer_identity denied_identity removal_identity
|
|
local admin_group viewer_group user memberships response status encoded grafana_user grafana_auth deadline now
|
|
printf '%s\n' 'Browser check: organization Admin and server-admin=false.' >&2
|
|
admin_identity="$(read_identity 'OIDC organization Admin')" || return 1
|
|
require_confirmation OIDC_ADMIN_CONFIRMED || return 1
|
|
printf '%s\n' 'Browser check: Viewer and edit/admin actions denied.' >&2
|
|
viewer_identity="$(read_identity 'OIDC Viewer')" || return 1
|
|
require_confirmation OIDC_VIEWER_CONFIRMED || return 1
|
|
printf '%s\n' 'Browser check: no observability groups and login denied.' >&2
|
|
denied_identity="$(read_identity 'OIDC denied-user')" || return 1
|
|
require_confirmation OIDC_DENIED_CONFIRMED || return 1
|
|
printf '%s\n' 'Browser check: local break-glass login and server-admin=true.' >&2
|
|
require_confirmation BREAK_GLASS_CONFIRMED || return 1
|
|
[[ "$admin_identity" != "$viewer_identity" && "$admin_identity" != "$denied_identity" && "$viewer_identity" != "$denied_identity" ]] || return 1
|
|
|
|
removal_identity="$(read_identity 'group-removal test')" || return 1
|
|
[[ "$removal_identity" != "$admin_identity" && "$removal_identity" != "$viewer_identity" && "$removal_identity" != "$denied_identity" ]] || return 1
|
|
prepare_keycloak_auth || return 1
|
|
admin_group="$WORK/keycloak-admin-group.json"; viewer_group="$WORK/keycloak-viewer-group.json"; user="$WORK/keycloak-removal-user.json"
|
|
local admin_group_id
|
|
admin_group_id="$(keycloak_lookup_group platform-observability-admins /platform-observability-admins "$admin_group")" || return 1
|
|
KEYCLOAK_VIEWER_GROUP_ID="$(keycloak_lookup_group platform-observability-viewers /platform-observability-viewers "$viewer_group")" || return 1
|
|
KEYCLOAK_USER_ID="$(keycloak_lookup_user "$removal_identity" "$user")" || return 1
|
|
KEYCLOAK_MEMBERSHIP_BEFORE="$WORK/keycloak-memberships-before.json"
|
|
capture_keycloak_absent_baseline "$admin_group_id" || return 1
|
|
add_keycloak_viewer_membership || return 1
|
|
printf '%s\n' 'Browser check: removal-test user is Viewer before membership removal.' >&2
|
|
require_confirmation REMOVAL_VIEWER_CONFIRMED || return 1
|
|
cleanup_keycloak_membership || return 1
|
|
response="$WORK/keycloak-session-revoke.json"
|
|
status="$(keycloak_request POST "/admin/realms/hyeonworks/users/$KEYCLOAK_USER_ID/logout" "$response")" || return 1
|
|
[[ "$status" == 204 ]] || return 1
|
|
|
|
grafana_auth="$WORK/grafana-curl.conf"; safe_work_file "$grafana_auth" || return 1
|
|
encoded="$(jq_bounded -nr --arg value "$removal_identity" '$value|@uri')" || return 1
|
|
grafana_user="$WORK/grafana-removal-user.json"
|
|
grafana_get "/api/users/lookup?loginOrEmail=$encoded" "$grafana_user" "$grafana_auth" || return 1
|
|
local grafana_id
|
|
grafana_id="$(jq_bounded -er '.id|select(type=="number" and .>0 and floor==.)' "$grafana_user")" || return 1
|
|
response="$WORK/grafana-session-revoke.json"
|
|
status="$(proxy_request POST "$(grafana_path "/api/admin/users/$grafana_id/logout")" "$response" '' "$grafana_auth")" || return 1
|
|
[[ "$status" == 200 || "$status" == 204 ]] || return 1
|
|
now="$("$TIMEOUT_BIN" --signal=TERM --kill-after=1s 5s "$DATE_BIN" -u +%s)" || return 1
|
|
[[ "$now" =~ ^[0-9]+$ ]] || return 1
|
|
deadline=$((now + HUMAN_TIMEOUT_SECONDS))
|
|
printf '%s\n' 'Within five minutes, verify old session rejection and OIDC relogin denial.' >&2
|
|
require_confirmation_before SESSION_REVOKED "$deadline" || return 1
|
|
require_confirmation_before RELOGIN_DENIED "$deadline" || return 1
|
|
printf 'OBSERVABILITY_OIDC_ACCEPTANCE=PASS\n'
|
|
}
|
|
|
|
wait_for_smoke_alert() {
|
|
local run=$1 expected=$2 attempt query alerts
|
|
for ((attempt=0; attempt<60; attempt++)); do
|
|
query="$WORK/slack-query-$attempt.json"; alerts="$WORK/slack-alerts-$attempt.json"
|
|
prom_query "ALERTS{alertname=\"PlatformObservabilitySmoke\",instance=\"$run\",alertstate=\"firing\"}" "$query" || return 1
|
|
proxy_get_200 "$(alertmanager_path '/api/v2/alerts')" "$alerts" || return 1
|
|
if [[ "$expected" == present ]]; then
|
|
if jq_bounded -e --arg run "$run" --arg runbook "$RUNBOOK_URL" '
|
|
(.status=="success" and (.data.result|length)==1 and
|
|
.data.result[0].metric.alertname=="PlatformObservabilitySmoke" and
|
|
.data.result[0].metric.instance==$run)
|
|
' "$query" >/dev/null &&
|
|
jq_bounded -e --arg run "$run" --arg runbook "$RUNBOOK_URL" '
|
|
[.[]|select(.labels.alertname=="PlatformObservabilitySmoke" and .labels.instance==$run)] as $items |
|
|
($items|length)==1 and $items[0].labels.cluster=="home" and
|
|
$items[0].labels.namespace=="observability" and $items[0].labels.severity=="info" and
|
|
$items[0].annotations.runbook_url==$runbook
|
|
' "$alerts" >/dev/null; then return 0; fi
|
|
else
|
|
if jq_bounded -e '.status=="success" and (.data.result|length)==0' "$query" >/dev/null &&
|
|
jq_bounded -e --arg run "$run" '[.[]|select(.labels.alertname=="PlatformObservabilitySmoke" and .labels.instance==$run)]|length==0' "$alerts" >/dev/null; then return 0; fi
|
|
fi
|
|
/usr/bin/sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
run_slack_acceptance() {
|
|
local run name rule
|
|
run="$(new_smoke_run_id alert)" || return 1
|
|
name="$(kubernetes_smoke_name "$run")" || return 1
|
|
rule="$WORK/slack-smoke-rule.json"
|
|
jq_bounded -n --arg name "$name" --arg run "$run" --arg runbook "$RUNBOOK_URL" '{
|
|
apiVersion:"monitoring.coreos.com/v1",kind:"PrometheusRule",
|
|
metadata:{name:$name,namespace:"observability",labels:{
|
|
"app.kubernetes.io/managed-by":"platform-observability-smoke",
|
|
"platform.hyeonworks.com/smoke-run":$run,
|
|
"observability.hyeonworks.com/instance":"home"}},
|
|
spec:{groups:[{name:("platform.observability-smoke."+$run),rules:[{
|
|
alert:"PlatformObservabilitySmoke",expr:"vector(1)",for:"0m",
|
|
labels:{cluster:"home",namespace:"observability",instance:$run,severity:"info"},
|
|
annotations:{summary:"Observability acceptance smoke",runbook_url:$runbook}
|
|
}]}]}}
|
|
' >"$rule" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$rule"
|
|
create_owned_from_file "$rule" PrometheusRule observability "$name" "$run" || return 1
|
|
wait_for_smoke_alert "$run" present || return 1
|
|
printf '%s\n' 'Confirm the Slack message groups cluster, namespace, alertname and includes severity/runbook.' >&2
|
|
require_confirmation FIRING || return 1
|
|
cleanup_owned_objects || return 1
|
|
printf '%s\n' 'Confirm the matching Slack resolved message.' >&2
|
|
require_confirmation RESOLVED || return 1
|
|
wait_for_smoke_alert "$run" absent || return 1
|
|
printf 'OBSERVABILITY_SLACK_ACCEPTANCE=PASS\n'
|
|
}
|
|
|
|
cleanup_all() {
|
|
local rc=0
|
|
cleanup_owned_objects || rc=1
|
|
cleanup_keycloak_membership || rc=1
|
|
delete_persistence_silence || rc=1
|
|
if [[ -n "$API_PROXY_PID" ]]; then
|
|
stop_owned_process "$API_PROXY_PID" "$API_PROXY_START" || rc=1
|
|
API_PROXY_PID=''; API_PROXY_START=''
|
|
fi
|
|
if [[ -n "$WORK" ]]; then
|
|
if [[ "$WORK" == /tmp/platform-observability-smoke.???????? && -d "$WORK" && ! -L "$WORK" ]]; then
|
|
"$RM_BIN" -rf -- "$WORK"
|
|
WORK=''
|
|
else
|
|
emit_manual_recovery; rc=1
|
|
fi
|
|
fi
|
|
return "$rc"
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$?
|
|
[[ "$EXIT_HANDLER_RUNNING" == false ]] || exit "$rc"
|
|
EXIT_HANDLER_RUNNING=true
|
|
trap - EXIT HUP INT TERM
|
|
cleanup_all || { (( rc != 0 )) || rc=1; }
|
|
exit "$rc"
|
|
}
|
|
|
|
main() {
|
|
parse_arguments "$@" || return 1
|
|
if [[ "$TEST_MODE" == false ]]; then reject_production_overrides || return 1; fi
|
|
if [[ "$execute_requested" == false ]]; then print_dry_run; return 0; fi
|
|
|
|
trap on_exit EXIT
|
|
trap 'handle_termination_signal 129' HUP
|
|
trap 'handle_termination_signal 130' INT
|
|
trap 'handle_termination_signal 143' TERM
|
|
require_execute_environment || { fail 'execute preflight failed'; return 1; }
|
|
run_read_only_machine_checks || return 1
|
|
require_confirmation 'APPLY default' || { fail 'confirmation did not match APPLY default'; return 1; }
|
|
repeat_commit_gate || { fail 'post-confirmation commit gate changed'; return 1; }
|
|
run_network_boundary_transaction || { fail 'network boundary transaction failed'; return 1; }
|
|
run_restart_persistence_transaction || { fail 'restart persistence transaction failed'; return 1; }
|
|
run_certbot_acceptance || { fail 'Certbot/Nginx acceptance failed'; return 1; }
|
|
repeat_commit_gate || { fail 'post-mutation context/API gate changed'; return 1; }
|
|
run_read_only_machine_checks || { fail 'post-mutation machine acceptance failed'; return 1; }
|
|
printf 'OBSERVABILITY_MACHINE_ACCEPTANCE=PASS\n'
|
|
run_oidc_acceptance || { fail 'OIDC human/session acceptance failed'; return 1; }
|
|
run_slack_acceptance || { fail 'Slack firing/resolved acceptance failed'; return 1; }
|
|
local external_token
|
|
external_token="$(external_attestation_token "$PUBLIC_EDGE_IP")" || return 1
|
|
render_external_attestation "$PUBLIC_EDGE_IP" >&2 || return 1
|
|
if ! require_confirmation "$external_token"; then
|
|
printf 'OBSERVABILITY_EXTERNAL_BOUNDARY=PARTIAL\n'
|
|
return 2
|
|
fi
|
|
printf '%s\n' 'OBSERVABILITY_EXTERNAL_BOUNDARY=PASS' 'OBSERVABILITY_SMOKE=PASS'
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
main "$@"
|
|
fi
|