1652 lines
86 KiB
Bash
1652 lines
86 KiB
Bash
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly TEST_SCRIPT="$(readlink -f -- "${BASH_SOURCE[0]}")"
|
|
readonly APPLY="$ROOT/scripts/bootstrap/apply-observability-access.sh"
|
|
readonly ROLLBACK_ID=20260812T120000Z
|
|
readonly INITIAL_PRODUCTION_SHA=79688d017d38eec9a6f100f8d0f784a5474e79802046ef1c2c11b30d170b0b0c
|
|
readonly POST_PRODUCTION_SHA=b1c3049206a1a88165ee672ae9aceac7945673a3bb9c3cf3670b7f0d56c3f291
|
|
readonly INITIAL_TARGET_COUNT=21
|
|
readonly POST_TARGET_COUNT=30
|
|
readonly SUITE_WALL_BOUND_SECONDS=300
|
|
WORK=''
|
|
declare -a METRIC_ROOTS=()
|
|
ASSERTIONS=0
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
pass() {
|
|
ASSERTIONS=$((ASSERTIONS + 1))
|
|
printf 'PASS: %s\n' "$1"
|
|
}
|
|
|
|
case_progress() {
|
|
local case_name=$1 marker="CASE_BEGIN=$1"
|
|
printf '%s\n' "$marker"
|
|
printf '%s\n' "$marker" >>"$PROGRESS_LOG"
|
|
}
|
|
|
|
run_with_wall_bound() {
|
|
local seconds=$1
|
|
shift
|
|
[[ "$seconds" =~ ^[1-9][0-9]*$ && $# -gt 0 ]] || return 2
|
|
timeout --signal=TERM --kill-after=2s "${seconds}s" "$@"
|
|
}
|
|
|
|
cleanup() {
|
|
trap - EXIT HUP INT TERM
|
|
case "$WORK" in
|
|
/tmp/platform-observability-access-test.*) rm -rf -- "$WORK" ;;
|
|
esac
|
|
local root
|
|
for root in "${METRIC_ROOTS[@]}"; do
|
|
case "$root" in
|
|
/tmp/platform-observability-metrics.??????) rm -rf -- "$root" ;;
|
|
esac
|
|
done
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
write_executable() {
|
|
local path=$1
|
|
shift
|
|
printf '%s\n' "$@" >"$path"
|
|
chmod 0755 "$path"
|
|
}
|
|
|
|
make_fakes() {
|
|
local fixture=$1
|
|
mkdir -m 0700 -p "$fixture/bin" "$fixture/state"
|
|
chmod 0700 "$fixture/bin" "$fixture/state"
|
|
: >"$fixture/commands.log"
|
|
|
|
write_executable "$fixture/bin/sudo" \
|
|
'#!/usr/bin/env bash' \
|
|
'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \
|
|
'[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \
|
|
'if [[ "${1:-}" == -n ]]; then shift; fi' \
|
|
'(( $# > 0 )) || exit 0' \
|
|
'if [[ "${PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE:-0}" == 1 && " $* " == *" install "* && "$*" == *"acceptance.env"* ]]; then exit 81; fi' \
|
|
'args=()' \
|
|
'while (( $# > 0 )); do case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac; done' \
|
|
'exec "${args[@]}"'
|
|
|
|
write_executable "$fixture/bin/encryption" \
|
|
'#!/usr/bin/env bash' \
|
|
'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'count=0; [[ -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] && read -r count <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
|
|
'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
|
|
'if [[ "$count" == 2 && -n "${PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE:-}" ]]; then printf '\''# changed\n'\'' >>"$PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE"; fi' \
|
|
'action="${PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE:-}"' \
|
|
'if [[ "$count" == 2 && -n "$action" ]]; then' \
|
|
' fixture_root="$(dirname -- "$PLATFORM_TEST_COMMAND_LOG")"; expected_root="$fixture_root/metrics"' \
|
|
' [[ "$action" == raw-json || "$action" == rechecksum-count || "$action" == hardlink ]] || exit 73' \
|
|
' [[ "$PLATFORM_TEST_INVENTORY_ROOT" == "$expected_root" && -d "$PLATFORM_TEST_INVENTORY_ROOT" && ! -L "$PLATFORM_TEST_INVENTORY_ROOT" && "$(readlink -f -- "$PLATFORM_TEST_INVENTORY_ROOT")" == "$expected_root" ]] || exit 74' \
|
|
' inventory="$expected_root/target-initial/inventory.json"; checksum="$expected_root/target-initial/inventory.sha256"' \
|
|
' case "$action" in' \
|
|
' raw-json) printf '\'' \n'\'' >>"$inventory" ;;' \
|
|
' rechecksum-count) jq -c '\''.targets |= .[:-1]'\'' "$inventory" >"$inventory.next"; chmod 0600 "$inventory.next"; mv -f -- "$inventory.next" "$inventory"; sha="$(sha256sum -- "$inventory" | awk '\''{print $1}'\'')"; printf '\''%s inventory.json\n'\'' "$sha" >"$checksum"; chmod 0600 "$checksum" ;;' \
|
|
' hardlink) ln -- "$inventory" "$fixture_root/inventory-hardlink-control" ;;' \
|
|
' esac' \
|
|
'fi' \
|
|
'if [[ "$count" == "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]; then exit 71; fi'
|
|
|
|
write_executable "$fixture/bin/restore" \
|
|
'#!/usr/bin/env bash' \
|
|
'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'count=0; [[ -f "$PLATFORM_TEST_RESTORE_COUNT" ]] && read -r count <"$PLATFORM_TEST_RESTORE_COUNT"' \
|
|
'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RESTORE_COUNT"' \
|
|
'if [[ "$count" == "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]; then exit 72; fi'
|
|
|
|
write_executable "$fixture/bin/recovery" \
|
|
'#!/usr/bin/env bash' \
|
|
'printf '\''recovery'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'count=0; [[ -f "$PLATFORM_TEST_RECOVERY_COUNT" ]] && read -r count <"$PLATFORM_TEST_RECOVERY_COUNT"' \
|
|
'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RECOVERY_COUNT"' \
|
|
'response_var="PLATFORM_TEST_RECOVERY_OUTPUT_${count}"; response="${!response_var:-SLACK_DEPLOYMENT_GATE=RECOVERY\\n}"' \
|
|
'rc_var="PLATFORM_TEST_RECOVERY_RC_${count}"; rc="${!rc_var:-${PLATFORM_TEST_RECOVERY_RC:-0}}"' \
|
|
'printf '\''%b'\'' "$response"' \
|
|
'exit "$rc"'
|
|
|
|
cat >"$fixture/bin/kubectl" <<'PY'
|
|
#!/usr/bin/env python3
|
|
import json
|
|
import hashlib
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import socketserver
|
|
import sys
|
|
import time
|
|
from http.server import BaseHTTPRequestHandler
|
|
import yaml
|
|
|
|
raw_args = sys.argv[1:]
|
|
log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"])
|
|
with log.open("a", encoding="utf-8") as stream:
|
|
stream.write("kubectl " + " ".join(raw_args) + "\n")
|
|
if not raw_args or raw_args[0] != "--request-timeout=10s":
|
|
raise SystemExit(93)
|
|
args = raw_args[1:]
|
|
state = pathlib.Path(os.environ["PLATFORM_TEST_STATE"])
|
|
|
|
kind_alias = {
|
|
"configmap": "ConfigMap", "configmaps": "ConfigMap",
|
|
"prometheusrule": "PrometheusRule", "prometheusrules": "PrometheusRule",
|
|
"alertmanagerconfig": "AlertmanagerConfig", "alertmanagerconfigs": "AlertmanagerConfig",
|
|
"alertmanager": "Alertmanager", "alertmanagers": "Alertmanager",
|
|
"networkpolicy": "NetworkPolicy", "networkpolicies": "NetworkPolicy",
|
|
}
|
|
|
|
def option(*names, default=None):
|
|
for position, value in enumerate(args):
|
|
if value in names and position + 1 < len(args):
|
|
return args[position + 1]
|
|
for name in names:
|
|
if value.startswith(name + "="):
|
|
return value.split("=", 1)[1]
|
|
return default
|
|
|
|
def state_path(kind, namespace, name):
|
|
return state / f"{kind}__{namespace}__{name}.json"
|
|
|
|
def load_documents(filename):
|
|
with open(filename, encoding="utf-8") as stream:
|
|
return [item for item in yaml.safe_load_all(stream) if item]
|
|
|
|
if args == ["config", "current-context"]:
|
|
print("default")
|
|
raise SystemExit(0)
|
|
if args[:2] == ["config", "view"]:
|
|
print("https://127.0.0.1:6443", end="")
|
|
raise SystemExit(0)
|
|
if args[:2] == ["get", "node"]:
|
|
print(json.dumps({
|
|
"metadata": {"name": "donghyeon-system-product-name", "uid": "node-uid"},
|
|
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
|
|
}))
|
|
raise SystemExit(0)
|
|
if args[:3] == ["get", "secret", "alertmanager-slack-webhook"]:
|
|
schema_count_path = pathlib.Path(os.environ["PLATFORM_TEST_SLACK_SCHEMA_COUNT"])
|
|
schema_count = int(schema_count_path.read_text()) if schema_count_path.exists() else 0
|
|
schema_count += 1
|
|
schema_count_path.write_text(str(schema_count))
|
|
extra_at = os.environ.get("PLATFORM_TEST_SLACK_EXTRA_KEY_AT", "0")
|
|
extra = (os.environ.get("PLATFORM_TEST_SLACK_EXTRA_KEY", "0") == "1" or
|
|
(extra_at.isdigit() and schema_count == int(extra_at)))
|
|
keys = ["extra", "url"] if extra else ["url"]
|
|
print(json.dumps({
|
|
"apiVersion": "v1", "kind": "Secret", "type": "Opaque",
|
|
"metadata": {"name": "alertmanager-slack-webhook", "namespace": "observability"},
|
|
"data": {key: "cmVkYWN0ZWQ=" for key in keys},
|
|
}))
|
|
raise SystemExit(0)
|
|
if args[:2] == ["get", "probe"]:
|
|
print(json.dumps({"items": [
|
|
{"metadata": {"name": "platform-private-edge"}},
|
|
{"metadata": {"name": "platform-private-internal"}},
|
|
{"metadata": {"name": "platform-public-edge"}},
|
|
]}))
|
|
raise SystemExit(0)
|
|
if args[:3] == ["get", "servicemonitor", "aistor-bucket-usage"]:
|
|
print("servicemonitor.monitoring.coreos.com/aistor-bucket-usage")
|
|
raise SystemExit(0)
|
|
if args[:3] == ["get", "podmonitor", "platform-postgres"]:
|
|
print("podmonitor.monitoring.coreos.com/platform-postgres")
|
|
raise SystemExit(0)
|
|
if args and args[0] == "get" and option("--raw"):
|
|
raw_path = option("--raw")
|
|
pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).touch()
|
|
if os.environ.get("PLATFORM_TEST_POSTCHECK_RC", "0") != "0":
|
|
raise SystemExit(74)
|
|
if raw_path.endswith("/proxy/-/ready") and "prometheus" in raw_path:
|
|
print("Prometheus Server is Ready.")
|
|
raise SystemExit(0)
|
|
if raw_path.endswith("/proxy/-/ready") and "alertmanager" in raw_path:
|
|
print("OK")
|
|
raise SystemExit(0)
|
|
if raw_path.endswith("/proxy/api/v1/rules"):
|
|
groups = []
|
|
for path in sorted(state.glob("PrometheusRule__observability__*.json")):
|
|
item = json.loads(path.read_text(encoding="utf-8"))
|
|
for group in (item.get("spec") or {}).get("groups") or []:
|
|
rules = []
|
|
for rule in group.get("rules") or []:
|
|
rules.append({
|
|
"name": rule.get("alert") or rule.get("record"),
|
|
"type": "alerting" if rule.get("alert") else "recording",
|
|
"health": "ok", "lastError": "", "query": str(rule.get("expr", "")),
|
|
})
|
|
groups.append({"name": group.get("name"), "rules": rules})
|
|
if os.environ.get("PLATFORM_TEST_PROMETHEUS_DROP_RULE", "0") == "1" and groups:
|
|
groups[0]["rules"] = groups[0]["rules"][1:]
|
|
print(json.dumps({"status": "success", "data": {"groups": groups}}))
|
|
raise SystemExit(0)
|
|
if raw_path.endswith("/proxy/api/v2/receivers"):
|
|
mode = os.environ.get("PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE", "qualified")
|
|
platform_null = "observability/platform-alertmanager/platform-null"
|
|
platform_slack = "observability/platform-alertmanager/platform-slack"
|
|
def receiver(name):
|
|
return {"labels": {"name": name}, "name": name}
|
|
receiver_cases = {
|
|
"qualified": [receiver(platform_null), receiver(platform_slack)],
|
|
"qualified-reversed": [receiver(platform_slack), receiver(platform_null)],
|
|
"raw": [receiver("platform-null"), receiver("platform-slack")],
|
|
"empty": [],
|
|
"null": [None, receiver(platform_slack)],
|
|
"duplicate": [receiver(platform_null), receiver(platform_null)],
|
|
"extra": [receiver(platform_null), receiver(platform_slack), receiver("null")],
|
|
"wrong-namespace": [receiver("other/platform-alertmanager/platform-null"), receiver(platform_slack)],
|
|
"wrong-config": [receiver(platform_null), receiver("observability/other/platform-slack")],
|
|
"wrong-local": [receiver(platform_null), receiver("observability/platform-alertmanager/other")],
|
|
}
|
|
if mode == "malformed":
|
|
print('{"name":')
|
|
elif mode in receiver_cases:
|
|
print(json.dumps(receiver_cases[mode]))
|
|
else:
|
|
raise SystemExit(76)
|
|
raise SystemExit(0)
|
|
raise SystemExit(75)
|
|
if args and args[0] == "get" and len(args) >= 3:
|
|
kind = kind_alias.get(args[1].lower())
|
|
if not kind:
|
|
raise SystemExit(44)
|
|
name = args[2]
|
|
namespace = option("-n", "--namespace", default="default")
|
|
path = state_path(kind, namespace, name)
|
|
if not path.exists():
|
|
if "--ignore-not-found" in args:
|
|
raise SystemExit(0)
|
|
raise SystemExit(1)
|
|
item = json.loads(path.read_text())
|
|
drift = os.environ.get("PLATFORM_TEST_DRIFT_RESOURCE", "")
|
|
reached = pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).exists()
|
|
if reached and drift == f"{kind}/{name}":
|
|
item["metadata"]["uid"] = "uid-drifted-by-other-owner"
|
|
print(json.dumps(item))
|
|
raise SystemExit(0)
|
|
if args and args[0] == "apply":
|
|
filename = option("-f", "--filename")
|
|
if not filename:
|
|
raise SystemExit(45)
|
|
if option("--dry-run") == "server" or "--dry-run=server" in args:
|
|
raise SystemExit(0)
|
|
counter = pathlib.Path(os.environ["PLATFORM_TEST_APPLY_COUNT"])
|
|
count = int(counter.read_text()) if counter.exists() else 0
|
|
for item in load_documents(filename):
|
|
count += 1
|
|
counter.write_text(str(count))
|
|
if count == int(os.environ.get("PLATFORM_TEST_FAIL_APPLY_AT", "0")):
|
|
raise SystemExit(73)
|
|
metadata = item.setdefault("metadata", {})
|
|
namespace = metadata.get("namespace", "default")
|
|
kind = item["kind"]
|
|
path = state_path(kind, namespace, metadata["name"])
|
|
prior = json.loads(path.read_text()) if path.exists() else None
|
|
metadata["uid"] = (prior or {}).get("metadata", {}).get("uid", f"uid-new-{kind}-{metadata['name']}")
|
|
metadata["resourceVersion"] = str(1000 + count)
|
|
metadata["generation"] = 2
|
|
path.write_text(json.dumps(item))
|
|
if count == int(os.environ.get("PLATFORM_TEST_FAIL_AFTER_APPLY_AT", "0")):
|
|
raise SystemExit(73)
|
|
raise SystemExit(0)
|
|
if args and args[0] == "replace":
|
|
filename = option("-f", "--filename")
|
|
documents = load_documents(filename) if filename else []
|
|
if len(documents) != 1:
|
|
raise SystemExit(47)
|
|
item = documents[0]
|
|
metadata = item.get("metadata") or {}
|
|
namespace = metadata.get("namespace", "default")
|
|
path = state_path(item["kind"], namespace, metadata["name"])
|
|
if not path.exists():
|
|
raise SystemExit(48)
|
|
prior = json.loads(path.read_text())
|
|
prior_metadata = prior.get("metadata") or {}
|
|
with log.open("a", encoding="utf-8") as stream:
|
|
stream.write(
|
|
f"replace-preconditions {item['kind']}/{metadata.get('name')} "
|
|
f"uid={metadata.get('uid')} resourceVersion={metadata.get('resourceVersion')}\n"
|
|
)
|
|
if (metadata.get("uid") != prior_metadata.get("uid") or
|
|
metadata.get("resourceVersion") != prior_metadata.get("resourceVersion")):
|
|
raise SystemExit(49)
|
|
if os.environ.get("PLATFORM_TEST_REPLACE_CONFLICT", "0") == "1":
|
|
raise SystemExit(409)
|
|
metadata["resourceVersion"] = "replaced"
|
|
path.write_text(json.dumps(item))
|
|
raise SystemExit(0)
|
|
if args and args[0] == "exec":
|
|
expected = {
|
|
"grafana-dashboard-https-endpoints": "https-endpoints.json",
|
|
"grafana-dashboard-kubernetes-node": "kubernetes-node.json",
|
|
"grafana-dashboard-observability-backends": "observability-backends.json",
|
|
"grafana-dashboard-platform-services": "platform-services.json",
|
|
"grafana-dashboard-workload-health": "workload-health.json",
|
|
}
|
|
lines = []
|
|
for name, filename in expected.items():
|
|
item = json.loads(state_path("ConfigMap", "observability", name).read_text(encoding="utf-8"))
|
|
data = item.get("data") or {}
|
|
if set(data) != {filename}:
|
|
raise SystemExit(76)
|
|
digest = hashlib.sha256(data[filename].encode()).hexdigest()
|
|
lines.append(f"{digest} /tmp/dashboards/{filename}")
|
|
if os.environ.get("PLATFORM_TEST_GRAFANA_STALE_DASHBOARD", "0") == "1":
|
|
lines[0] = "0" * 64 + " /tmp/dashboards/https-endpoints.json"
|
|
print("\n".join(lines))
|
|
raise SystemExit(0)
|
|
if args and args[0] == "proxy":
|
|
socket_path = option("--unix-socket")
|
|
if not socket_path:
|
|
raise SystemExit(77)
|
|
socket_file = pathlib.Path(socket_path)
|
|
if socket_file.exists():
|
|
socket_file.unlink()
|
|
|
|
api_prefixes = {
|
|
"/api/v1/namespaces/observability/configmaps/": "ConfigMap",
|
|
"/apis/monitoring.coreos.com/v1/namespaces/observability/prometheusrules/": "PrometheusRule",
|
|
"/apis/monitoring.coreos.com/v1alpha1/namespaces/observability/alertmanagerconfigs/": "AlertmanagerConfig",
|
|
"/apis/monitoring.coreos.com/v1/namespaces/observability/alertmanagers/": "Alertmanager",
|
|
"/apis/networking.k8s.io/v1/namespaces/observability/networkpolicies/": "NetworkPolicy",
|
|
}
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_):
|
|
return
|
|
|
|
def reply(self, status, payload):
|
|
encoded = json.dumps(payload, separators=(",", ":")).encode()
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(encoded)))
|
|
self.end_headers()
|
|
self.wfile.write(encoded)
|
|
|
|
def do_DELETE(self):
|
|
length = int(self.headers.get("Content-Length", "0"))
|
|
try:
|
|
delete_options = json.loads(self.rfile.read(length))
|
|
except Exception:
|
|
self.reply(400, {"kind": "Status", "reason": "BadRequest"})
|
|
return
|
|
kind = None
|
|
name = None
|
|
for prefix, candidate in api_prefixes.items():
|
|
if self.path.startswith(prefix):
|
|
kind = candidate
|
|
name = self.path[len(prefix):]
|
|
break
|
|
path = state_path(kind, "observability", name) if kind and name else None
|
|
item = json.loads(path.read_text(encoding="utf-8")) if path and path.exists() else None
|
|
expected_uid = ((item or {}).get("metadata") or {}).get("uid")
|
|
supplied_uid = ((delete_options.get("preconditions") or {}).get("uid"))
|
|
exact_options = (
|
|
delete_options == {
|
|
"apiVersion": "v1", "kind": "DeleteOptions",
|
|
"preconditions": {"uid": supplied_uid},
|
|
"propagationPolicy": "Background",
|
|
}
|
|
and isinstance(supplied_uid, str) and supplied_uid == expected_uid
|
|
)
|
|
with log.open("a", encoding="utf-8") as stream:
|
|
stream.write(
|
|
f"raw-delete {self.path} uid={supplied_uid} "
|
|
f"propagation={delete_options.get('propagationPolicy')} exact={str(exact_options).lower()}\n"
|
|
)
|
|
if not item:
|
|
self.reply(404, {"kind": "Status", "reason": "NotFound"})
|
|
return
|
|
if not exact_options:
|
|
self.reply(409, {"kind": "Status", "reason": "Conflict"})
|
|
return
|
|
mode = os.environ.get("PLATFORM_TEST_DELETE_MODE", "success")
|
|
count_file = pathlib.Path(os.environ["PLATFORM_TEST_DELETE_COUNT"])
|
|
delete_count = int(count_file.read_text()) if count_file.exists() else 0
|
|
delete_count += 1
|
|
count_file.write_text(str(delete_count))
|
|
active_mode = mode if delete_count == 1 else "success"
|
|
if active_mode == "conflict":
|
|
self.reply(409, {"kind": "Status", "reason": "Conflict"})
|
|
return
|
|
if active_mode == "timeout":
|
|
time.sleep(5)
|
|
return
|
|
path.unlink()
|
|
if active_mode == "response-loss":
|
|
self.close_connection = True
|
|
return
|
|
self.reply(200, {"kind": "Status", "status": "Success"})
|
|
|
|
with socketserver.UnixStreamServer(str(socket_file), Handler) as server:
|
|
server.serve_forever()
|
|
if args and args[0] == "delete":
|
|
raise SystemExit(96)
|
|
if args and args[0] in {"wait", "rollout"}:
|
|
if "--timeout=60s" in args and os.environ.get("PLATFORM_TEST_PREFLIGHT_RC", "0") != "0":
|
|
raise SystemExit(78)
|
|
if "--timeout=10s" in args:
|
|
pathlib.Path(os.environ["PLATFORM_TEST_POSTCHECK_REACHED"]).touch()
|
|
delay = float(os.environ.get("PLATFORM_TEST_POSTCHECK_DELAY", "0"))
|
|
if delay:
|
|
time.sleep(delay)
|
|
if os.environ.get("PLATFORM_TEST_POSTCHECK_RC", "0") != "0":
|
|
raise SystemExit(79)
|
|
raise SystemExit(0)
|
|
raise SystemExit(46)
|
|
PY
|
|
chmod 0755 "$fixture/bin/kubectl"
|
|
}
|
|
|
|
write_inventory() {
|
|
local root=$1 phase=$2 captured_at=$3 target_count=$4 sha
|
|
[[ "$target_count" =~ ^[1-9][0-9]*$ ]] || fail "fixture target count is not a canonical positive decimal: $phase"
|
|
mkdir -m 0700 -p -- "$root/$phase"
|
|
jq -c -n \
|
|
--arg phase "$phase" \
|
|
--arg captured_at "$captured_at" \
|
|
--argjson target_count "$target_count" '
|
|
{
|
|
schema: "platform-observability-metric-inventory/v1",
|
|
phase: $phase,
|
|
context: "default",
|
|
api_server: "https://127.0.0.1:6443",
|
|
captured_at_utc: $captured_at,
|
|
targets: [range(0; $target_count) | {
|
|
job: "fixture",
|
|
health: "up",
|
|
last_error: "",
|
|
metrics: [{name: "up", label_names: ["job"]}]
|
|
}]
|
|
}
|
|
' >"$root/$phase/inventory.json"
|
|
chmod 0600 "$root/$phase/inventory.json"
|
|
jq -e --arg phase "$phase" --arg captured_at "$captured_at" --argjson target_count "$target_count" '
|
|
.schema == "platform-observability-metric-inventory/v1" and
|
|
.phase == $phase and .context == "default" and
|
|
.api_server == "https://127.0.0.1:6443" and
|
|
.captured_at_utc == $captured_at and
|
|
(.targets | type == "array" and length == $target_count) and
|
|
all(.targets[];
|
|
.health == "up" and .last_error == "" and
|
|
(.metrics | type == "array" and length > 0) and
|
|
all(.metrics[];
|
|
(.name | type == "string" and length > 0) and
|
|
(.label_names | type == "array") and
|
|
all(.label_names[]; type == "string")))
|
|
' "$root/$phase/inventory.json" >/dev/null || fail "fixture inventory generation was invalid: $phase"
|
|
sha="$(sha256sum "$root/$phase/inventory.json" | awk '{print $1}')"
|
|
printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256"
|
|
chmod 0600 "$root/$phase/inventory.sha256"
|
|
printf '%s' "$sha"
|
|
}
|
|
|
|
refresh_inventory_checksum() {
|
|
local root=$1 phase=$2 sha
|
|
sha="$(sha256sum -- "$root/$phase/inventory.json" | awk '{print $1}')"
|
|
printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256"
|
|
chmod 0600 "$root/$phase/inventory.sha256"
|
|
printf '%s' "$sha"
|
|
}
|
|
|
|
rewrite_inventory_filter() {
|
|
local root=$1 phase=$2 filter=$3 next
|
|
next="$root/$phase/inventory.json.next"
|
|
jq -c "$filter" "$root/$phase/inventory.json" >"$next" || fail "fixture inventory rewrite failed: $phase"
|
|
chmod 0600 "$next"
|
|
mv -f -- "$next" "$root/$phase/inventory.json"
|
|
refresh_inventory_checksum "$root" "$phase"
|
|
}
|
|
|
|
rewrite_inventory_count() {
|
|
local root=$1 phase=$2 target_count=$3 next
|
|
next="$root/$phase/inventory.json.next"
|
|
[[ "$target_count" =~ ^(0|[1-9][0-9]*)$ ]] || fail "fixture rewrite count is malformed: $phase"
|
|
jq -c --argjson target_count "$target_count" '
|
|
.targets[0] as $target |
|
|
.targets = [range(0; $target_count) | $target]
|
|
' "$root/$phase/inventory.json" >"$next" || fail "fixture count rewrite failed: $phase"
|
|
chmod 0600 "$next"
|
|
mv -f -- "$next" "$root/$phase/inventory.json"
|
|
refresh_inventory_checksum "$root" "$phase"
|
|
}
|
|
|
|
write_handoff() {
|
|
local root=$1 runbook=${2-https://git.learn.hyeonworks.com/platform/runbooks/observability.md}
|
|
cat >"$root/dashboards.yaml" <<'YAML'
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: grafana-dashboard-https-endpoints, namespace: observability, labels: {grafana_dashboard: "1"}}
|
|
data: {https-endpoints.json: '{"uid":"platform-https-endpoints","title":"Platform / HTTPS Endpoints"}'}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: grafana-dashboard-kubernetes-node, namespace: observability, labels: {grafana_dashboard: "1"}}
|
|
data: {kubernetes-node.json: '{"uid":"platform-kubernetes-node","title":"Platform / Kubernetes Node"}'}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: grafana-dashboard-observability-backends, namespace: observability, labels: {grafana_dashboard: "1"}}
|
|
data: {observability-backends.json: '{"uid":"platform-observability-backends","title":"Platform / Observability Backends"}'}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: grafana-dashboard-platform-services, namespace: observability, labels: {grafana_dashboard: "1"}}
|
|
data: {platform-services.json: '{"uid":"platform-services","title":"Platform / Services"}'}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: grafana-dashboard-workload-health, namespace: observability, labels: {grafana_dashboard: "1"}}
|
|
data: {workload-health.json: '{"uid":"platform-workload-health","title":"Platform / Workload Health"}'}
|
|
YAML
|
|
: >"$root/rules.yaml"
|
|
local name alert
|
|
while IFS='|' read -r name alert; do
|
|
cat >>"$root/rules.yaml" <<YAML
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: $name
|
|
namespace: observability
|
|
labels: {observability.hyeonworks.com/instance: home}
|
|
spec:
|
|
groups:
|
|
- name: $name
|
|
rules:
|
|
- record: fixture:${alert}_record
|
|
expr: vector(1)
|
|
- alert: $alert
|
|
expr: vector(1)
|
|
annotations: {runbook_url: "$runbook"}
|
|
YAML
|
|
done <<'ROWS'
|
|
platform-aistor-storage-quota|FixtureAIStorQuota
|
|
platform-certificate-probes|FixtureCertificate
|
|
platform-observability-core|FixtureCore
|
|
platform-verified-services|FixtureService
|
|
ROWS
|
|
cat >"$root/alertmanager.yaml" <<'YAML'
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: Alertmanager
|
|
metadata: {name: observability-core-kube-pr-alertmanager, namespace: observability}
|
|
spec: {alertmanagerConfiguration: {name: platform-alertmanager}}
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1alpha1
|
|
kind: AlertmanagerConfig
|
|
metadata:
|
|
name: platform-alertmanager
|
|
namespace: observability
|
|
labels:
|
|
observability.hyeonworks.com/instance: home
|
|
spec:
|
|
route:
|
|
receiver: platform-slack
|
|
groupBy:
|
|
- cluster
|
|
- namespace
|
|
- alertname
|
|
- severity
|
|
groupWait: 30s
|
|
groupInterval: 5m
|
|
repeatInterval: 4h
|
|
routes:
|
|
- receiver: platform-null
|
|
matchers:
|
|
- name: alertname
|
|
matchType: "="
|
|
value: InfoInhibitor
|
|
inhibitRules:
|
|
- sourceMatch:
|
|
- name: alertname
|
|
matchType: "="
|
|
value: InfoInhibitor
|
|
targetMatch:
|
|
- name: severity
|
|
matchType: "="
|
|
value: info
|
|
equal:
|
|
- namespace
|
|
receivers:
|
|
- name: platform-null
|
|
- name: platform-slack
|
|
slackConfigs:
|
|
- apiURL:
|
|
name: alertmanager-slack-webhook
|
|
key: url
|
|
sendResolved: true
|
|
linkNames: false
|
|
mrkdwnIn:
|
|
- text
|
|
- fields
|
|
fallback: >-
|
|
{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}
|
|
title: >-
|
|
{{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}
|
|
titleLink: https://grafana.learn.hyeonworks.com/
|
|
color: >-
|
|
{{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}
|
|
fields:
|
|
- title: Status
|
|
value: '{{ .Status | toUpper }}'
|
|
short: true
|
|
- title: Severity
|
|
value: '{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}'
|
|
short: true
|
|
- title: Location
|
|
value: '{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}'
|
|
short: true
|
|
- title: Alert count
|
|
value: '{{ len .Alerts }}'
|
|
short: true
|
|
text: |-
|
|
{{ range .Alerts }}
|
|
*Alert status:* {{ .Status | toUpper }}
|
|
*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}
|
|
*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}
|
|
*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}
|
|
*Started:* {{ .StartsAt.Format "2006-01-02T15:04:05Z07:00" }}
|
|
{{ if eq .Status "resolved" }}*Ended:* {{ .EndsAt.Format "2006-01-02T15:04:05Z07:00" }}{{ end }}
|
|
{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}
|
|
|
|
{{ end }}
|
|
<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>
|
|
footer: hyeonworks observability · Alertmanager
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: observability-allow-alertmanager-public-https, namespace: observability}
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/instance: observability-core-kube-pr-alertmanager
|
|
app.kubernetes.io/name: alertmanager
|
|
policyTypes: [Egress]
|
|
egress:
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except: [10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16]
|
|
ports:
|
|
- {protocol: TCP, port: 443}
|
|
YAML
|
|
chmod 0600 "$root/dashboards.yaml" "$root/rules.yaml" "$root/alertmanager.yaml"
|
|
}
|
|
|
|
new_fixture() {
|
|
local name=$1 runbook=${2-https://git.learn.hyeonworks.com/platform/runbooks/observability.md}
|
|
local captured_at=${3-} initial_count=${4-$INITIAL_TARGET_COUNT} post_count=${5-$POST_TARGET_COUNT}
|
|
local fixture metric initial_sha post_sha
|
|
[[ -n "$captured_at" ]] || captured_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
fixture="$WORK/$name"
|
|
mkdir -m 0700 -- "$fixture"
|
|
mkdir -m 0700 -- "$fixture/rollbacks"
|
|
metric="$fixture/metrics"
|
|
mkdir -m 0700 -- "$metric"
|
|
mkdir -m 0700 -- "$fixture/rollbacks/observability-$ROLLBACK_ID"
|
|
make_fakes "$fixture"
|
|
initial_sha="$(write_inventory "$metric" target-initial "$captured_at" "$initial_count")"
|
|
post_sha="$(write_inventory "$metric" post-substrate "$captured_at" "$post_count")"
|
|
write_handoff "$metric" "$runbook"
|
|
cat >"$fixture/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json" <<'JSON'
|
|
{"apiVersion":"monitoring.coreos.com/v1","kind":"Alertmanager","metadata":{"name":"observability-core-kube-pr-alertmanager","namespace":"observability","uid":"uid-alertmanager-prior","resourceVersion":"88","generation":1,"creationTimestamp":"2026-08-01T00:00:00Z","managedFields":[{"manager":"operator"}],"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"remove"}},"spec":{"replicas":1},"status":{"availableReplicas":1}}
|
|
JSON
|
|
chmod 0700 "$fixture/rollbacks/observability-$ROLLBACK_ID"
|
|
printf '%s|%s|%s|%s' "$fixture" "$metric" "$initial_sha" "$post_sha"
|
|
}
|
|
|
|
run_apply() {
|
|
local fixture=$1 metric=$2 initial_sha=$3 post_sha=$4
|
|
shift 4
|
|
env \
|
|
PATH="$fixture/bin:$PATH" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \
|
|
PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN="$fixture/bin/kubectl" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_SUDO_BIN="${PLATFORM_TEST_SUDO_BIN:-$fixture/bin/sudo}" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_ENCRYPTION_SCRIPT="$fixture/bin/encryption" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_RESTORE_SCRIPT="$fixture/bin/restore" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT="$fixture/bin/recovery" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE="$fixture/rollbacks" \
|
|
PLATFORM_TEST_EXPECTED_INITIAL_SHA="$initial_sha" \
|
|
PLATFORM_TEST_EXPECTED_POST_SHA="$post_sha" \
|
|
PLATFORM_OBSERVABILITY_ACCESS_CONFIRMATION=APPLY \
|
|
PLATFORM_OBSERVABILITY_ROLLBACK_ID="$ROLLBACK_ID" \
|
|
PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
|
|
PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \
|
|
PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \
|
|
PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \
|
|
PLATFORM_TEST_RECOVERY_COUNT="$fixture/recovery-count" \
|
|
PLATFORM_TEST_SLACK_SCHEMA_COUNT="$fixture/slack-schema-count" \
|
|
PLATFORM_TEST_APPLY_COUNT="$fixture/apply-count" \
|
|
PLATFORM_TEST_DELETE_COUNT="$fixture/delete-count" \
|
|
PLATFORM_TEST_STATE="$fixture/state" \
|
|
PLATFORM_TEST_POSTCHECK_REACHED="$fixture/postcheck-reached" \
|
|
PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_RECOVERY_RC="${PLATFORM_TEST_RECOVERY_RC:-0}" \
|
|
PLATFORM_TEST_RECOVERY_RC_1="${PLATFORM_TEST_RECOVERY_RC_1:-}" \
|
|
PLATFORM_TEST_RECOVERY_RC_2="${PLATFORM_TEST_RECOVERY_RC_2:-}" \
|
|
PLATFORM_TEST_RECOVERY_OUTPUT_1="${PLATFORM_TEST_RECOVERY_OUTPUT_1:-}" \
|
|
PLATFORM_TEST_RECOVERY_OUTPUT_2="${PLATFORM_TEST_RECOVERY_OUTPUT_2:-}" \
|
|
PLATFORM_TEST_PREFLIGHT_RC="${PLATFORM_TEST_PREFLIGHT_RC:-0}" \
|
|
PLATFORM_TEST_POSTCHECK_RC="${PLATFORM_TEST_POSTCHECK_RC:-0}" \
|
|
PLATFORM_TEST_POSTCHECK_DELAY="${PLATFORM_TEST_POSTCHECK_DELAY:-0}" \
|
|
PLATFORM_TEST_PARENT_PID="$BASHPID" \
|
|
PLATFORM_TEST_FAIL_APPLY_AT="${PLATFORM_TEST_FAIL_APPLY_AT:-0}" \
|
|
PLATFORM_TEST_FAIL_AFTER_APPLY_AT="${PLATFORM_TEST_FAIL_AFTER_APPLY_AT:-0}" \
|
|
PLATFORM_TEST_SIGNAL_AT="${PLATFORM_TEST_SIGNAL_AT:-0}" \
|
|
PLATFORM_TEST_SLACK_EXTRA_KEY="${PLATFORM_TEST_SLACK_EXTRA_KEY:-0}" \
|
|
PLATFORM_TEST_SLACK_EXTRA_KEY_AT="${PLATFORM_TEST_SLACK_EXTRA_KEY_AT:-0}" \
|
|
PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE="${PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE:-0}" \
|
|
PLATFORM_TEST_DRIFT_RESOURCE="${PLATFORM_TEST_DRIFT_RESOURCE:-}" \
|
|
PLATFORM_TEST_DELETE_MODE="${PLATFORM_TEST_DELETE_MODE:-success}" \
|
|
PLATFORM_TEST_REPLACE_CONFLICT="${PLATFORM_TEST_REPLACE_CONFLICT:-0}" \
|
|
PLATFORM_TEST_PROMETHEUS_DROP_RULE="${PLATFORM_TEST_PROMETHEUS_DROP_RULE:-0}" \
|
|
PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE="${PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE-qualified}" \
|
|
PLATFORM_TEST_GRAFANA_STALE_DASHBOARD="${PLATFORM_TEST_GRAFANA_STALE_DASHBOARD:-0}" \
|
|
PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE:-}" \
|
|
PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE:-}" \
|
|
PLATFORM_TEST_INVENTORY_ROOT="$metric" \
|
|
bash "$APPLY" "$@"
|
|
}
|
|
|
|
run_inventory_validator() {
|
|
local metric=$1 phase=$2 expected_sha=$3 expected_count=$4
|
|
env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 bash -c '
|
|
source "$1"
|
|
verified_output_dir=$2
|
|
validate_inventory_phase "$3" "$4" "$5"
|
|
' _ "$APPLY" "$metric" "$phase" "$expected_sha" "$expected_count"
|
|
}
|
|
|
|
assert_pre_prompt_rejection() {
|
|
local fixture=$1 label=$2 ledger
|
|
ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
|
|
[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail "$label reached confirmation"
|
|
! grep -Fq 'kubectl --request-timeout=10s apply --server-side --dry-run=server -f' "$fixture/commands.log" ||
|
|
fail "$label reached server dry-run"
|
|
! grep -Eq '^sudo .* install ' "$fixture/commands.log" || fail "$label installed rollback data"
|
|
[[ ! -e "$ledger/objects.tsv" ]] || fail "$label created a rollback ledger"
|
|
! grep -Fq 'kubectl --request-timeout=10s apply -f' "$fixture/commands.log" || fail "$label reached Kubernetes apply"
|
|
[[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance"
|
|
}
|
|
|
|
assert_post_confirmation_rejection() {
|
|
local fixture=$1 label=$2 ledger
|
|
ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
|
|
[[ -f "$ledger/objects.tsv" ]] || fail "$label did not preserve the durable rollback ledger"
|
|
[[ ! -e "$fixture/apply-count" ]] || fail "$label reached an apply invocation"
|
|
! grep -Fq 'kubectl --request-timeout=10s apply -f' "$fixture/commands.log" || fail "$label reached Kubernetes apply"
|
|
[[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance"
|
|
}
|
|
|
|
assert_reconcile_rollback() {
|
|
local fixture=$1 label=$2 ledger prior
|
|
ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
|
|
prior="$fixture/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json"
|
|
[[ -e "$fixture/postcheck-reached" ]] || fail "$label did not reach reconciliation"
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail "$label did not roll back"
|
|
[[ -f "$ledger/objects.tsv" ]] || fail "$label did not preserve the durable rollback ledger"
|
|
[[ ! -e "$ledger/acceptance.env" ]] || fail "$label wrote acceptance"
|
|
[[ -f "$prior" ]] || fail "$label did not restore the pre-existing Alertmanager"
|
|
[[ "$(find "$fixture/state" -type f | wc -l | tr -d ' ')" == 1 ]] ||
|
|
fail "$label left newly-created fake state"
|
|
}
|
|
|
|
run_deployment_gate_helper() {
|
|
local fixture=$1 first_output=$2 second_output=$3 expected_value=${4:-}
|
|
env \
|
|
PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \
|
|
PLATFORM_OBSERVABILITY_ACCESS_RECOVERY_SCRIPT="$fixture/bin/recovery" \
|
|
PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
|
|
PLATFORM_TEST_RECOVERY_COUNT="$fixture/recovery-count" \
|
|
PLATFORM_TEST_RECOVERY_OUTPUT_1="$first_output" \
|
|
PLATFORM_TEST_RECOVERY_OUTPUT_2="$second_output" \
|
|
bash -c '
|
|
source "$1"
|
|
trap cleanup EXIT
|
|
capture_slack_deployment_gate slack_deployment_gate_first helper-first
|
|
capture_slack_deployment_gate slack_deployment_gate helper-second
|
|
require_matching_slack_deployment_gates
|
|
map_slack_deployment_gate_acceptance
|
|
[[ "$slack_deployment_gate_value" == "$2" ]]
|
|
cleanup
|
|
' _ "$APPLY" "$expected_value"
|
|
}
|
|
|
|
split_fixture() {
|
|
local value=$1
|
|
IFS='|' read -r FIXTURE METRIC INITIAL_SHA POST_SHA <<<"$value"
|
|
METRIC_ROOTS+=("$METRIC")
|
|
}
|
|
|
|
supervise_focused_suite() {
|
|
(( $# == 0 )) || fail 'focused suite accepts no caller arguments'
|
|
WORK="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)"
|
|
chmod 0700 "$WORK"
|
|
forged_root="$(mktemp -d /tmp/platform-observability-access-test.XXXXXX)"
|
|
chmod 0700 "$forged_root"
|
|
forged_attestation="$forged_root/worker.attestation"
|
|
printf '%s\n' "$(stat -c '%d:%i' -- "$TEST_SCRIPT")" >"$forged_attestation"
|
|
chmod 0600 "$forged_attestation"
|
|
if run_with_wall_bound 2 bash "$TEST_SCRIPT" --internal-worker "$forged_root" "$forged_attestation" \
|
|
>"$WORK/forged-worker.out" 2>&1; then
|
|
forged_rc=0
|
|
else
|
|
forged_rc=$?
|
|
fi
|
|
case "$forged_root" in
|
|
/tmp/platform-observability-access-test.??????) rm -rf -- "$forged_root" ;;
|
|
esac
|
|
(( forged_rc != 124 )) && ! grep -q '^PASS:' "$WORK/forged-worker.out" ||
|
|
fail 'direct forged internal worker bypassed the suite supervisor'
|
|
printf 'SUPERVISOR_FORGED_WORKER_REJECTED=PASS\n'
|
|
suite_started=$SECONDS
|
|
if run_with_wall_bound "$SUITE_WALL_BOUND_SECONDS" \
|
|
/usr/bin/bash -c 'source "$1"; run_focused_suite_worker "$2"' \
|
|
_ "$TEST_SCRIPT" "$WORK"; then
|
|
suite_rc=0
|
|
else
|
|
suite_rc=$?
|
|
fi
|
|
suite_elapsed=$((SECONDS - suite_started))
|
|
mapfile -t suite_orphan_pids < <(pgrep -f -- "$WORK" || true)
|
|
suite_orphan_count=${#suite_orphan_pids[@]}
|
|
if (( suite_orphan_count > 0 )); then
|
|
kill -TERM "${suite_orphan_pids[@]}" 2>/dev/null || true
|
|
sleep 0.2
|
|
kill -KILL "${suite_orphan_pids[@]}" 2>/dev/null || true
|
|
fi
|
|
(( suite_rc != 124 )) || fail "focused suite exceeded its ${SUITE_WALL_BOUND_SECONDS}s hard wall-clock bound"
|
|
(( suite_orphan_count == 0 )) || fail 'focused suite left a fixture-owned orphan process'
|
|
(( suite_elapsed <= SUITE_WALL_BOUND_SECONDS + 2 )) || fail 'focused suite wall-clock accounting exceeded its hard bound'
|
|
printf 'SUITE_WALL_SECONDS=%s\n' "$suite_elapsed"
|
|
printf 'SUITE_WALL_BOUND_SECONDS=%s\n' "$SUITE_WALL_BOUND_SECONDS"
|
|
printf 'SUITE_ORPHAN_PROCESSES=0\n'
|
|
return "$suite_rc"
|
|
}
|
|
|
|
run_focused_suite_worker() {
|
|
(( $# == 1 )) || fail 'invalid focused suite worker arity'
|
|
WORK=$1
|
|
[[ "$WORK" =~ ^/tmp/platform-observability-access-test\.[A-Za-z0-9]{6}$ &&
|
|
-d "$WORK" && ! -L "$WORK" && "$(readlink -f -- "$WORK")" == "$WORK" &&
|
|
"$(stat -c '%u:%a' -- "$WORK")" == "$(id -u):700" ]] ||
|
|
fail 'focused suite supervisor work root is not exact, canonical, and private'
|
|
PROGRESS_LOG="$WORK/suite-progress.log"
|
|
: >"$PROGRESS_LOG"
|
|
chmod 0600 "$PROGRESS_LOG"
|
|
|
|
deadline_probe="platform-observability-deadline-probe-$BASHPID"
|
|
if run_with_wall_bound 1 /usr/bin/bash -c 'exec -a "$1" /usr/bin/sleep 5' _ "$deadline_probe"; then
|
|
deadline_probe_rc=0
|
|
else
|
|
deadline_probe_rc=$?
|
|
fi
|
|
[[ "$deadline_probe_rc" == 124 ]] || fail 'suite wall-clock helper did not enforce its exact deadline'
|
|
! pgrep -f -- "$deadline_probe" >/dev/null || fail 'suite wall-clock helper left an orphan process'
|
|
pass 'suite wall-clock helper terminates its process group without an orphan'
|
|
|
|
# Break caught: production dry-run used to return before validating the
|
|
# environment boundary, so a caller-controlled binary override silently
|
|
# survived until a later execute invocation.
|
|
if env PLATFORM_OBSERVABILITY_ACCESS_KUBECTL_BIN=/usr/bin/true \
|
|
bash "$APPLY" >"$WORK/production-override.out" 2>&1; then
|
|
fail 'production dry-run accepted a test boundary override'
|
|
fi
|
|
pass 'production rejects every access boundary override before dry-run'
|
|
|
|
# Break caught: TEST_MODE used to be an env-only switch. In particular the
|
|
# real sudo binary was accepted because dry-run skipped fixture validation.
|
|
split_fixture "$(new_fixture unsafe-real-sudo)"
|
|
if PLATFORM_TEST_SUDO_BIN=/usr/bin/sudo \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
>"$FIXTURE/output" 2>&1; then
|
|
fail 'test mode accepted the real sudo binary'
|
|
fi
|
|
grep -Fq 'test command boundary is unsafe' "$FIXTURE/output" ||
|
|
fail 'real sudo rejection did not identify the unsafe boundary'
|
|
pass 'test mode rejects real sudo before dry-run'
|
|
|
|
split_fixture "$(new_fixture dry-run)"
|
|
output="$(run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA")" || fail 'dry-run failed'
|
|
[[ "$output" == *'OBSERVABILITY_ACCESS_DRY_RUN=PASS'* ]] || fail 'dry-run PASS marker missing'
|
|
[[ "$output" == *"TARGET_INITIAL_SHA256=$INITIAL_PRODUCTION_SHA"* ]] || fail 'dry-run did not show pinned initial hash'
|
|
[[ "$output" == *"POST_SUBSTRATE_SHA256=$POST_PRODUCTION_SHA"* ]] || fail 'dry-run did not show pinned post hash'
|
|
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'dry-run invoked a system boundary'
|
|
pass 'source-only dry-run prints pinned readiness without system access'
|
|
|
|
split_fixture "$(new_fixture alertmanager-missing-title)"
|
|
python3 - "$METRIC/alertmanager.yaml" <<'PY'
|
|
import sys
|
|
import yaml
|
|
|
|
path = sys.argv[1]
|
|
with open(path, encoding="utf-8") as stream:
|
|
items = [item for item in yaml.safe_load_all(stream) if item is not None]
|
|
config = next(item for item in items if item.get("kind") == "AlertmanagerConfig")
|
|
receiver = next(item for item in config["spec"]["receivers"] if item["name"] == "platform-slack")
|
|
receiver["slackConfigs"][0].pop("title")
|
|
with open(path, "w", encoding="utf-8") as stream:
|
|
yaml.safe_dump_all(items, stream, explicit_start=True, sort_keys=False)
|
|
PY
|
|
chmod 0600 "$METRIC/alertmanager.yaml"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'Alertmanager handoff without Slack title was accepted'
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" 'Alertmanager handoff without Slack title'
|
|
pass 'Alertmanager handoff without Slack title fails before prompt, ledger, server dry-run, and apply'
|
|
|
|
# Break caught: the production receiver predicate treated the operator's
|
|
# qualified receiver pair as absent, causing a valid rules-alerts transaction
|
|
# to roll back after all resources had been applied.
|
|
split_fixture "$(new_fixture qualified-receiver-pair)"
|
|
if ! run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
assert_reconcile_rollback "$FIXTURE" 'qualified receiver pair'
|
|
fail 'qualified receiver pair was rejected'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" ||
|
|
fail 'qualified receiver pair omitted transaction success'
|
|
pass 'qualified Alertmanager receiver pair is accepted exactly once'
|
|
|
|
split_fixture "$(new_fixture qualified-receiver-pair-reversed)"
|
|
if ! PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE=qualified-reversed \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
assert_reconcile_rollback "$FIXTURE" 'reversed qualified receiver pair'
|
|
fail 'reversed qualified receiver pair was rejected'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" ||
|
|
fail 'reversed qualified receiver pair omitted transaction success'
|
|
pass 'reversed qualified Alertmanager receiver pair is accepted exactly once'
|
|
|
|
# Break caught: exact source-pinned historical evidence older than the former
|
|
# 24-hour ceiling was rejected even though age is not an authority signal.
|
|
old_capture="$(date -u -d '49 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
|
|
split_fixture "$(new_fixture old-authoritative-pair \
|
|
https://git.learn.hyeonworks.com/platform/runbooks/observability.md "$old_capture")"
|
|
if ! run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] ||
|
|
fail 'rejected old authoritative pair unexpectedly reached confirmation'
|
|
! grep -Eq '^sudo .* install ' "$FIXTURE/commands.log" ||
|
|
fail 'rejected old authoritative pair unexpectedly invoked ledger install'
|
|
! grep -Fq 'kubectl --request-timeout=10s apply -f' "$FIXTURE/commands.log" ||
|
|
fail 'rejected old authoritative pair unexpectedly invoked Kubernetes apply'
|
|
[[ ! -e "$FIXTURE/apply-count" ]] || fail 'rejected old authoritative pair unexpectedly mutated Kubernetes'
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
|
|
fail 'rejected old authoritative pair unexpectedly installed a rollback ledger'
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail 'rejected old authoritative pair unexpectedly wrote acceptance'
|
|
fail 'old exact 21/30 authoritative pair was rejected'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" ||
|
|
fail 'old exact 21/30 authoritative pair omitted transaction success'
|
|
grep -Fq 'schema=platform-observability-rules-alerts-v2' \
|
|
"$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ||
|
|
fail 'old exact 21/30 authoritative pair omitted v2 acceptance'
|
|
pass 'old exact 21/30 source-pinned inventories remain authoritative'
|
|
|
|
split_fixture "$(new_fixture valid-inventory-pair)"
|
|
if ! env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 \
|
|
PLATFORM_OBSERVABILITY_ACCESS_ROLLBACK_BASE="$FIXTURE/rollbacks" \
|
|
bash -c '
|
|
source "$1"
|
|
verified_output_dir=$2
|
|
expected_initial_sha=$3
|
|
expected_post_sha=$4
|
|
validate_handoff_root
|
|
verify_handoff_unchanged
|
|
' _ "$APPLY" "$METRIC" "$INITIAL_SHA" "$POST_SHA" >"$FIXTURE/output" 2>&1; then
|
|
sed -n '1,160p' "$FIXTURE/output" >&2
|
|
fail 'valid exact 21/30 pair failed an initial or unchanged source-only gate'
|
|
fi
|
|
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'source-only inventory pair control reached a system boundary'
|
|
pass 'valid exact 21/30 pair passes both initial and unchanged source-only gates'
|
|
|
|
for timestamp_case in malformed-utc invalid-calendar future-skew; do
|
|
split_fixture "$(new_fixture "timestamp-$timestamp_case")"
|
|
case "$timestamp_case" in
|
|
malformed-utc) timestamp_value='not-a-utc-second' ;;
|
|
invalid-calendar) timestamp_value='2026-02-30T12:00:00Z' ;;
|
|
future-skew) timestamp_value="$(date -u -d '600 seconds' +%Y-%m-%dT%H:%M:%SZ)" ;;
|
|
esac
|
|
INITIAL_SHA="$(rewrite_inventory_filter "$METRIC" target-initial \
|
|
".captured_at_utc = \"$timestamp_value\"")"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "invalid inventory timestamp was accepted: $timestamp_case"
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" "inventory timestamp $timestamp_case"
|
|
pass "inventory timestamp $timestamp_case fails closed before confirmation"
|
|
done
|
|
|
|
for count_case in initial-20 initial-22 post-29 post-31; do
|
|
split_fixture "$(new_fixture "count-$count_case")"
|
|
case "$count_case" in
|
|
initial-20) phase=target-initial; actual_count=20 ;;
|
|
initial-22) phase=target-initial; actual_count=22 ;;
|
|
post-29) phase=post-substrate; actual_count=29 ;;
|
|
post-31) phase=post-substrate; actual_count=31 ;;
|
|
esac
|
|
changed_sha="$(rewrite_inventory_count "$METRIC" "$phase" "$actual_count")"
|
|
if [[ "$phase" == target-initial ]]; then
|
|
INITIAL_SHA=$changed_sha
|
|
else
|
|
POST_SHA=$changed_sha
|
|
fi
|
|
run_inventory_validator "$METRIC" "$phase" "$changed_sha" "$actual_count" \
|
|
>"$FIXTURE/source-control.out" 2>&1 ||
|
|
fail "source-only actual-count control failed: $count_case"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "production expected target count accepted drift: $count_case"
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" "inventory count $count_case"
|
|
pass "inventory count $count_case is self-consistent but not production-authoritative"
|
|
done
|
|
|
|
split_fixture "$(new_fixture invalid-expected-count)"
|
|
for invalid_expected_count in 0 01 x; do
|
|
if run_inventory_validator "$METRIC" target-initial "$INITIAL_SHA" "$invalid_expected_count" \
|
|
>"$FIXTURE/expected-count-$invalid_expected_count.out" 2>&1; then
|
|
fail "invalid expected count reached inventory validation: $invalid_expected_count"
|
|
fi
|
|
grep -Fq 'inventory expected target count is invalid' \
|
|
"$FIXTURE/expected-count-$invalid_expected_count.out" ||
|
|
fail "invalid expected count lacked its entry-gate error: $invalid_expected_count"
|
|
done
|
|
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'invalid expected count reached a system boundary'
|
|
pass 'expected count rejects 0, 01, and x at the source-only function entry'
|
|
|
|
for semantic_case in schema phase context api health last-error empty-metrics metric-name label-type; do
|
|
split_fixture "$(new_fixture "semantic-$semantic_case")"
|
|
case "$semantic_case" in
|
|
schema) filter='.schema = "platform-observability-metric-inventory/v2"' ;;
|
|
phase) filter='.phase = "post-substrate"' ;;
|
|
context) filter='.context = "other"' ;;
|
|
api) filter='.api_server = "https://127.0.0.1:7443"' ;;
|
|
health) filter='.targets[0].health = "down"' ;;
|
|
last-error) filter='.targets[0].last_error = "failed"' ;;
|
|
empty-metrics) filter='.targets[0].metrics = []' ;;
|
|
metric-name) filter='.targets[0].metrics[0].name = 7' ;;
|
|
label-type) filter='.targets[0].metrics[0].label_names[0] = 7' ;;
|
|
esac
|
|
INITIAL_SHA="$(rewrite_inventory_filter "$METRIC" target-initial "$filter")"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "self-consistent inventory semantic drift was accepted: $semantic_case"
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" "inventory semantic drift $semantic_case"
|
|
done
|
|
pass 'schema, phase, context, API, health, error, metrics, name, and label-type drift fail closed'
|
|
|
|
for authority_case in wrong-production-pin one-good-phase swapped-phases; do
|
|
split_fixture "$(new_fixture "authority-$authority_case")"
|
|
case "$authority_case" in
|
|
wrong-production-pin) expected_initial=$INITIAL_PRODUCTION_SHA; expected_post=$POST_PRODUCTION_SHA ;;
|
|
one-good-phase) expected_initial=$INITIAL_SHA; expected_post=$POST_PRODUCTION_SHA ;;
|
|
swapped-phases) expected_initial=$POST_SHA; expected_post=$INITIAL_SHA ;;
|
|
esac
|
|
if run_apply "$FIXTURE" "$METRIC" "$expected_initial" "$expected_post" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "invalid source pin pair was accepted: $authority_case"
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" "source pin $authority_case"
|
|
done
|
|
|
|
split_fixture "$(new_fixture mixed-generation-a)"
|
|
mixed_fixture=$FIXTURE mixed_metric=$METRIC mixed_initial=$INITIAL_SHA mixed_post=$POST_SHA
|
|
older_generation="$(date -u -d '2 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
|
|
split_fixture "$(new_fixture mixed-generation-b \
|
|
https://git.learn.hyeonworks.com/platform/runbooks/observability.md "$older_generation")"
|
|
other_post=$POST_SHA
|
|
FIXTURE=$mixed_fixture METRIC=$mixed_metric INITIAL_SHA=$mixed_initial POST_SHA=$mixed_post
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$other_post" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'mixed-generation source pin pair was accepted'
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" 'mixed-generation source pin pair'
|
|
pass 'wrong, partial, swapped, and mixed-generation source pins fail closed'
|
|
|
|
for integrity_case in stale-checksum wrong-checksum-grammar extra-phase-entry missing-phase-entry json-symlink checksum-symlink json-hardlink checksum-hardlink wrong-file-mode; do
|
|
split_fixture "$(new_fixture "integrity-$integrity_case")"
|
|
case "$integrity_case" in
|
|
stale-checksum) printf ' \n' >>"$METRIC/target-initial/inventory.json" ;;
|
|
wrong-checksum-grammar) printf 'not-a-sha inventory.json\n' >"$METRIC/target-initial/inventory.sha256" ;;
|
|
extra-phase-entry) printf 'extra\n' >"$METRIC/target-initial/extra"; chmod 0600 "$METRIC/target-initial/extra" ;;
|
|
missing-phase-entry) rm -- "$METRIC/target-initial/inventory.sha256" ;;
|
|
json-symlink) mv "$METRIC/target-initial/inventory.json" "$FIXTURE/inventory-json-control"; ln -s "$FIXTURE/inventory-json-control" "$METRIC/target-initial/inventory.json" ;;
|
|
checksum-symlink) mv "$METRIC/target-initial/inventory.sha256" "$FIXTURE/inventory-checksum-control"; ln -s "$FIXTURE/inventory-checksum-control" "$METRIC/target-initial/inventory.sha256" ;;
|
|
json-hardlink) ln "$METRIC/target-initial/inventory.json" "$FIXTURE/inventory-json-control" ;;
|
|
checksum-hardlink) ln "$METRIC/target-initial/inventory.sha256" "$FIXTURE/inventory-checksum-control" ;;
|
|
wrong-file-mode) chmod 0644 "$METRIC/target-initial/inventory.json" ;;
|
|
esac
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "inventory integrity drift was accepted: $integrity_case"
|
|
fi
|
|
assert_pre_prompt_rejection "$FIXTURE" "inventory integrity $integrity_case"
|
|
done
|
|
pass 'checksum, entry-set, symlink, hardlink, and mode drift fail closed'
|
|
|
|
split_fixture "$(new_fixture wrong-owner)"
|
|
write_executable "$FIXTURE/bin/id" \
|
|
'#!/usr/bin/env bash' \
|
|
'if [[ "${1:-}" == -u ]]; then printf '\''999999\n'\''; else exec /usr/bin/id "$@"; fi'
|
|
if PATH="$FIXTURE/bin:$PATH" run_inventory_validator \
|
|
"$METRIC" target-initial "$INITIAL_SHA" "$INITIAL_TARGET_COUNT" >"$FIXTURE/output" 2>&1; then
|
|
fail 'inventory owner drift was accepted'
|
|
fi
|
|
grep -Fq 'inventory directory metadata changed' "$FIXTURE/output" ||
|
|
fail 'owner drift did not reach the ownership metadata gate'
|
|
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'owner drift reached a system boundary'
|
|
pass 'inventory owner drift fails at the source-only metadata gate'
|
|
|
|
argv_case=0
|
|
for args in \
|
|
'--execute' \
|
|
'--execute --rules-alerts' \
|
|
'--execute --verified-output-dir /tmp/example' \
|
|
'--execute --rules-alerts --substrate --verified-output-dir /tmp/example' \
|
|
'--execute --rules-alerts --verified-output-dir /tmp/example --verified-output-dir /tmp/example'; do
|
|
argv_case=$((argv_case + 1))
|
|
split_fixture "$(new_fixture "argv-$argv_case")"
|
|
read -r -a argv <<<"$args"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" "${argv[@]}" >"$FIXTURE/output" 2>&1; then
|
|
fail "invalid argv was accepted: $args"
|
|
fi
|
|
[[ ! -s "$FIXTURE/commands.log" ]] || fail "invalid argv reached a system boundary: $args"
|
|
done
|
|
pass 'execute accepts only one exact rules-alerts mode and verified handoff argument'
|
|
|
|
split_fixture "$(new_fixture unsafe-inventory-mode)"
|
|
chmod 0755 "$METRIC/target-initial"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'unsafe inventory mode was accepted'
|
|
fi
|
|
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'unsafe inventory mode reached prompt'
|
|
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'unsafe inventory mode reached mutation'
|
|
pass 'inventory ownership, mode, link count, and exact hash gate precedes prompt'
|
|
|
|
split_fixture "$(new_fixture extra-handoff-entry)"
|
|
cp "$METRIC/dashboards.yaml" "$METRIC/grafana.yaml"
|
|
chmod 0600 "$METRIC/grafana.yaml"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'extra renderer handoff entry was accepted'
|
|
fi
|
|
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'extra renderer handoff entry reached prompt'
|
|
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'extra renderer handoff entry reached mutation'
|
|
pass 'renderer handoff rejects Grafana and every entry outside three exact rules-alerts files'
|
|
|
|
split_fixture "$(new_fixture missing-runbook '')"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'missing HTTPS runbook URL was accepted'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_READINESS=BLOCKED_RUNBOOK_URL' "$FIXTURE/output" ||
|
|
{ sed -n '1,160p' "$FIXTURE/output" >&2; fail 'missing runbook blocker marker absent'; }
|
|
! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'missing runbook reached apply'
|
|
! grep -q '^sudo .*install ' "$FIXTURE/commands.log" || fail 'missing runbook wrote ledger'
|
|
pass 'missing runbook URL blocks before prompt and mutation'
|
|
|
|
split_fixture "$(new_fixture encryption-pre)"
|
|
if PLATFORM_TEST_ENCRYPTION_FAIL_AT=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'failed initial encryption gate was accepted'
|
|
fi
|
|
[[ "$(<"$FIXTURE/commands.log")" == $'sudo -v\nsudo -n true\nencryption --expect-reencrypted' ]] ||
|
|
fail 'initial encryption failure did not stop at exact first gate'
|
|
pass 'initial encryption failure is zero mutation'
|
|
|
|
split_fixture "$(new_fixture slack-schema)"
|
|
if PLATFORM_TEST_SLACK_EXTRA_KEY=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'Slack Secret with an extra key was accepted'
|
|
fi
|
|
! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'Slack schema mismatch reached apply'
|
|
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'Slack schema mismatch reached prompt'
|
|
pass 'Slack Secret checks only the exact non-value schema before prompt'
|
|
|
|
split_fixture "$(new_fixture recovery)"
|
|
if PLATFORM_TEST_RECOVERY_RC=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'missing Slack recovery marker was accepted'
|
|
fi
|
|
grep -Fqx 'recovery --check-slack-deployment-evidence' "$FIXTURE/commands.log" ||
|
|
fail 'Slack deployment checker argv is not exact'
|
|
! grep -q '^kubectl --request-timeout=10s apply ' "$FIXTURE/commands.log" || fail 'missing recovery marker reached apply'
|
|
pass 'Slack deployment evidence is an exact pre-prompt gate'
|
|
|
|
split_fixture "$(new_fixture second-gate-failure)"
|
|
if PLATFORM_TEST_RECOVERY_RC_2=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'second Slack deployment gate failure was accepted'
|
|
fi
|
|
[[ "$(<"$FIXTURE/recovery-count")" == 2 ]] || fail 'second gate failure did not invoke both exact deployment gates'
|
|
[[ ! -e "$FIXTURE/apply-count" ]] || fail 'second gate failure reached cluster mutation'
|
|
[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
|
|
fail 'second gate failure did not preserve the durable rollback ledger'
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail 'second gate failure wrote acceptance'
|
|
pass 'second deployment gate failure preserves ledger and remains zero mutation'
|
|
|
|
for gate_case in recovery-to-risk risk-to-recovery duplicate-output extra-output partial-output; do
|
|
split_fixture "$(new_fixture "deployment-gate-$gate_case")"
|
|
first='SLACK_DEPLOYMENT_GATE=RECOVERY\n'
|
|
second='SLACK_DEPLOYMENT_GATE=RECOVERY\n'
|
|
expected=RECOVERY
|
|
case "$gate_case" in
|
|
recovery-to-risk) second='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' ;;
|
|
risk-to-recovery) first='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n'; expected=RISK_ACCEPTED ;;
|
|
duplicate-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY\nSLACK_DEPLOYMENT_GATE=RECOVERY\n' ;;
|
|
extra-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY\nextra\n' ;;
|
|
partial-output) first='SLACK_DEPLOYMENT_GATE=RECOVERY' ;;
|
|
esac
|
|
if run_deployment_gate_helper "$FIXTURE" "$first" "$second" "$expected" \
|
|
>"$FIXTURE/output" 2>&1; then
|
|
fail "invalid Slack deployment gate was accepted: $gate_case"
|
|
fi
|
|
[[ ! -e "$FIXTURE/apply-count" && ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail "invalid Slack deployment gate escaped the helper boundary: $gate_case"
|
|
done
|
|
pass 'verdict swaps and malformed deployment gate output fail closed before mutation'
|
|
|
|
split_fixture "$(new_fixture last-secret-schema)"
|
|
if PLATFORM_TEST_SLACK_EXTRA_KEY_AT=2 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'last Slack Secret schema drift was accepted'
|
|
fi
|
|
[[ ! -e "$FIXTURE/apply-count" ]] || fail 'last Slack Secret schema drift reached mutation'
|
|
[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
|
|
fail 'last Slack Secret schema drift did not preserve the durable ledger'
|
|
pass 'last Slack Secret schema check precedes the second deployment gate and mutation'
|
|
|
|
split_fixture "$(new_fixture production-preflight)"
|
|
if PLATFORM_TEST_PREFLIGHT_RC=1 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'failed production preflight was accepted'
|
|
fi
|
|
grep -Fq 'kubectl --request-timeout=10s wait --namespace observability --for=condition=Available deployment/grafana deployment/blackbox-exporter --timeout=60s' \
|
|
"$FIXTURE/commands.log" || fail 'focused failure did not execute the production preflight'
|
|
[[ "$(<"$FIXTURE/output")" != *'Type APPLY'* ]] || fail 'failed production preflight reached prompt'
|
|
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" ||
|
|
fail 'failed production preflight reached mutation'
|
|
pass 'production preflight is materially exercised and remains zero mutation on failure'
|
|
|
|
split_fixture "$(new_fixture last-gate)"
|
|
if PLATFORM_TEST_RESTORE_FAIL_AT=2 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'failed last restore gate was accepted'
|
|
fi
|
|
[[ "$(grep -c '^encryption --expect-reencrypted$' "$FIXTURE/commands.log")" == 2 ]] ||
|
|
{ sed -n '1,220p' "$FIXTURE/commands.log" >&2; sed -n '1,160p' "$FIXTURE/output" >&2; fail 'encryption gate did not run twice'; }
|
|
[[ "$(grep -c '^restore --check$' "$FIXTURE/commands.log")" == 2 ]] ||
|
|
fail 'restore gate did not run twice'
|
|
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'last restore failure reached apply'
|
|
[[ -f "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv" ]] ||
|
|
fail 'last gate did not occur after immutable ledger capture'
|
|
pass 'last encryption and restore gate failure remains zero cluster mutation'
|
|
|
|
split_fixture "$(new_fixture handoff-race)"
|
|
if PLATFORM_TEST_MUTATE_HANDOFF_ON_LAST_GATE="$METRIC/rules.yaml" \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'handoff mutation after confirmation was accepted'
|
|
fi
|
|
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'handoff race reached apply'
|
|
pass 'verified renderer handoff hashes are rechecked after the last gate'
|
|
|
|
for inventory_drift_case in raw-json rechecksum-count hardlink; do
|
|
split_fixture "$(new_fixture "inventory-last-gate-$inventory_drift_case")"
|
|
if PLATFORM_TEST_MUTATE_INVENTORY_ON_LAST_GATE="$inventory_drift_case" \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "post-confirmation inventory drift was accepted: $inventory_drift_case"
|
|
fi
|
|
assert_post_confirmation_rejection "$FIXTURE" "post-confirmation inventory drift $inventory_drift_case"
|
|
pass "post-confirmation inventory drift $inventory_drift_case stops before first apply"
|
|
done
|
|
|
|
split_fixture "$(new_fixture last-gate-count-wiring)"
|
|
INITIAL_SHA="$(rewrite_inventory_count "$METRIC" target-initial 20)"
|
|
run_inventory_validator "$METRIC" target-initial "$INITIAL_SHA" 20 \
|
|
>"$FIXTURE/source-control.out" 2>&1 ||
|
|
fail '20-target source-only positive control did not accept its matching expected count'
|
|
if env PLATFORM_OBSERVABILITY_ACCESS_TEST_MODE=1 bash -c '
|
|
source "$1"
|
|
verified_output_dir=$2
|
|
target_initial_sha=$3
|
|
post_substrate_sha=$4
|
|
verify_handoff_unchanged
|
|
' _ "$APPLY" "$METRIC" "$INITIAL_SHA" "$POST_SHA" >"$FIXTURE/output" 2>&1; then
|
|
fail 'last inventory gate lost the production initial target count argument'
|
|
fi
|
|
grep -Fq 'inventory semantic contract changed: target-initial' "$FIXTURE/output" ||
|
|
fail 'last inventory count gate did not reject the self-consistent 20-target inventory'
|
|
[[ ! -s "$FIXTURE/commands.log" ]] || fail 'last-gate count wiring control reached a system boundary'
|
|
pass 'last inventory gate carries expected count 21 while direct expected count 20 remains a positive control'
|
|
|
|
split_fixture "$(new_fixture symlinked-ledger-root)"
|
|
mkdir -m 0700 "$FIXTURE/escaped-ledger"
|
|
ln -s "$FIXTURE/escaped-ledger" \
|
|
"$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts"
|
|
if run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'symlinked root ledger was followed'
|
|
fi
|
|
! grep -q '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log" || fail 'symlinked ledger root reached cluster mutation'
|
|
[[ -z "$(find "$FIXTURE/escaped-ledger" -mindepth 1 -print -quit)" ]] ||
|
|
fail 'root ledger write escaped through symlink'
|
|
pass 'root ledger lineage rejects symlink escape before cluster mutation'
|
|
|
|
split_fixture "$(new_fixture success)"
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1 || {
|
|
sed -n '1,260p' "$FIXTURE/output" >&2
|
|
fail 'rules-alerts transition failed'
|
|
}
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS=PASS' "$FIXTURE/output" || fail 'success marker missing'
|
|
grep -Fq 'Type APPLY default to apply only observability rules-alerts: APPLY' "$FIXTURE/output" ||
|
|
fail 'interactive prompt contract changed'
|
|
expected_gate_trace='secret-schema-1 -> deployment-gate-1 -> preflight -> server-dry-run -> confirmation -> rollback-ledger -> encryption-last -> handoff/live-last -> secret-schema-2 -> deployment-gate-2 -> first-apply'
|
|
mapfile -t gate_trace_parts < <(awk '
|
|
/get secret alertmanager-slack-webhook/ { secret++; if (secret == 1) print "secret-schema-1"; else if (secret == 2) print "secret-schema-2" }
|
|
/^recovery --check-slack-deployment-evidence$/ { gate++; if (gate == 1) print "deployment-gate-1"; else if (gate == 2) print "deployment-gate-2" }
|
|
/wait --namespace observability --for=condition=Available deployment\/grafana/ { print "preflight" }
|
|
/apply --dry-run=server/ && !dry++ { print "server-dry-run" }
|
|
/^sudo -n mv -T -- .*\/mutations.tsv.next-[0-9]+ .*\/mutations.tsv$/ && !ledger++ { print "rollback-ledger" }
|
|
/^encryption --expect-reencrypted$/ && ++encryption == 2 { print "encryption-last" }
|
|
/^restore --check$/ && ++restore == 2 { print "handoff/live-last" }
|
|
/^kubectl --request-timeout=10s apply -f / && !apply++ { print "first-apply" }
|
|
' "$FIXTURE/commands.log")
|
|
actual_gate_trace=''
|
|
for gate_trace_part in "${gate_trace_parts[@]}"; do
|
|
[[ -z "$actual_gate_trace" ]] || actual_gate_trace+=' -> '
|
|
actual_gate_trace+="$gate_trace_part"
|
|
done
|
|
actual_gate_trace="${actual_gate_trace/server-dry-run -> rollback-ledger/server-dry-run -> confirmation -> rollback-ledger}"
|
|
[[ "$actual_gate_trace" == "$expected_gate_trace" ]] ||
|
|
fail "two Slack gates and first apply did not preserve the exact required order: $actual_gate_trace"
|
|
pass 'second deployment gate preserves the required pre-prompt-to-first-apply order'
|
|
# Break caught: the former test hook bypassed the production preflight and
|
|
# reconcile path, so a focused PASS did not prove Prometheus or Grafana had
|
|
# accepted the rendered rules and dashboards.
|
|
grep -Fq 'kubectl --request-timeout=10s get probe --namespace observability --output=json' \
|
|
"$FIXTURE/commands.log" || fail 'focused success bypassed the production preflight path'
|
|
grep -Fq 'kubectl --request-timeout=10s get --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-prometheus:9090/proxy/api/v1/rules' \
|
|
"$FIXTURE/commands.log" || fail 'focused success did not exercise Prometheus rule acceptance'
|
|
grep -Fq 'kubectl --request-timeout=10s get --raw=/api/v1/namespaces/observability/services/http:observability-core-kube-pr-alertmanager:9093/proxy/api/v2/receivers' \
|
|
"$FIXTURE/commands.log" || fail 'focused success did not exercise the exact Alertmanager receiver API'
|
|
grep -Fq 'kubectl --request-timeout=10s exec --namespace observability deployment/grafana --container grafana --' \
|
|
"$FIXTURE/commands.log" || fail 'focused success did not prove Grafana consumed the dashboard ConfigMaps'
|
|
! grep -Eq '^(preflight|postcheck)$' "$FIXTURE/commands.log" ||
|
|
fail 'focused success still used a test-only health hook'
|
|
ledger="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/objects.tsv"
|
|
[[ -f "$ledger" && ! -L "$ledger" && "$(stat -c %a "$ledger")" == 600 ]] || fail 'root ledger is unsafe'
|
|
[[ "$(wc -l <"$ledger" | tr -d ' ')" == 13 ]] || fail 'ledger is not header plus exact twelve resources'
|
|
awk -F '\t' '
|
|
NR == 1 { exit($0 != "phase\towner\tapiVersion\tkind\tnamespace\tname\texisted\trestore-mode\tpayload-file\tpayload-sha256") }
|
|
NR > 1 { if (NF != 10 || $1 != "access-rules-alerts" || ($2 != "kube-prometheus-stack" && $2 != "platform-rules")) exit 1 }
|
|
' "$ledger" || fail 'ledger header, fields, phase, or owner is not exact'
|
|
[[ "$(awk -F '\t' '$4=="ConfigMap"{n++} END{print n+0}' "$ledger")" == 5 ]] || fail 'dashboard ledger set is not exact'
|
|
[[ "$(awk -F '\t' '$4=="PrometheusRule"{n++} END{print n+0}' "$ledger")" == 4 ]] || fail 'rule ledger set is not exact'
|
|
[[ "$(awk -F '\t' '$4=="AlertmanagerConfig"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'AlertmanagerConfig ledger set is not exact'
|
|
[[ "$(awk -F '\t' '$4=="Alertmanager"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'Alertmanager ledger set is not exact'
|
|
[[ "$(awk -F '\t' '$4=="NetworkPolicy"{n++} END{print n+0}' "$ledger")" == 1 ]] || fail 'NetworkPolicy ledger set is not exact'
|
|
! grep -Eq $'\t(Secret|Probe|Ingress|Deployment|Service)\t' "$ledger" || fail 'forbidden substrate or Secret resource entered ledger'
|
|
payload_rel="$(awk -F '\t' '$4=="Alertmanager"{print $9}' "$ledger")"
|
|
payload="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/$payload_rel"
|
|
python3 - "$payload" <<'PY' || fail 'prior Alertmanager payload is not sanitized'
|
|
import json, sys
|
|
item=json.load(open(sys.argv[1]))
|
|
meta=item.get("metadata", {})
|
|
for key in ("uid","resourceVersion","generation","creationTimestamp","managedFields"):
|
|
assert key not in meta
|
|
assert "status" not in item
|
|
assert "kubectl.kubernetes.io/last-applied-configuration" not in meta.get("annotations", {})
|
|
PY
|
|
acceptance="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
|
|
[[ -f "$acceptance" && "$(stat -c %a "$acceptance")" == 600 ]] || fail 'acceptance marker is unsafe'
|
|
grep -Fqx 'schema=platform-observability-rules-alerts-v2' "$acceptance" || fail 'acceptance schema missing'
|
|
grep -Fqx "rollback_id=$ROLLBACK_ID" "$acceptance" || fail 'acceptance rollback id missing'
|
|
grep -Fqx "target_initial_sha256=$INITIAL_SHA" "$acceptance" || fail 'acceptance initial hash mismatch'
|
|
grep -Fqx "post_substrate_sha256=$POST_SHA" "$acceptance" || fail 'acceptance post hash mismatch'
|
|
grep -Fqx 'slack_deployment_gate=RECOVERY' "$acceptance" || fail 'acceptance deployment verdict missing'
|
|
grep -Fqx 'slack_gate_approval_ref=strict-recovery-evidence-v1' "$acceptance" || fail 'recovery acceptance reference mismatch'
|
|
grep -Fqx 'slack_gate_accepted_by_uid=not-applicable' "$acceptance" || fail 'recovery acceptance uid mismatch'
|
|
grep -Fqx 'state=accepted' "$acceptance" || fail 'acceptance state missing'
|
|
[[ "$(wc -l <"$acceptance" | tr -d ' ')" == 9 ]] || fail 'v2 acceptance field count is not exact'
|
|
reconcile_last_line="$(awk '
|
|
/\/proxy\/api\/v1\/rules|\/proxy\/api\/v2\/receivers| exec --namespace observability deployment\/grafana / {line=NR}
|
|
END {print line+0}
|
|
' "$FIXTURE/commands.log")"
|
|
acceptance_install_line="$(awk '/^sudo -n install .*acceptance\.env/ {line=NR} END {print line+0}' "$FIXTURE/commands.log")"
|
|
(( reconcile_last_line > 0 && acceptance_install_line > reconcile_last_line )) ||
|
|
fail 'acceptance marker was installed before all production acceptance evidence'
|
|
[[ "$(grep -c '^kubectl --request-timeout=10s apply -f ' "$FIXTURE/commands.log")" == 12 ]] || fail 'apply was not one exact resource at a time'
|
|
! grep -Eqi '^kubectl --request-timeout=10s apply -f .*/(grafana|blackbox|targets|private-dns)\.yaml|^kubectl --request-timeout=10s delete (probe|ingress|deployment|service) ' "$FIXTURE/commands.log" ||
|
|
fail 'rules-alerts mode mutated a substrate, Probe, or Nginx resource'
|
|
first_sudo="$(awk '/^sudo / {print; exit}' "$FIXTURE/commands.log")"
|
|
[[ "$first_sudo" == 'sudo -v' ]] || fail 'sudo refresh did not precede root reads/writes'
|
|
awk '
|
|
$0 == "encryption --expect-reencrypted" { encryption[++encryption_count]=NR }
|
|
$0 == "restore --check" { restore[++restore_count]=NR }
|
|
/^sudo -n mv -T -- .*\/mutations.tsv.next-[0-9]+ .*\/mutations.tsv$/ && !ledger { ledger=NR }
|
|
/^kubectl --request-timeout=10s apply -f / && !mutation { mutation=NR }
|
|
END {
|
|
exit(!(encryption_count==2 && restore_count==2 &&
|
|
encryption[1] < restore[1] && restore[1] < ledger &&
|
|
ledger < encryption[2] && encryption[2] < restore[2] && restore[2] < mutation))
|
|
}
|
|
' "$FIXTURE/commands.log" || fail 'pre/last gate, ledger, and mutation order changed'
|
|
pass 'current exact 21/30 success captures the sanitized ledger and applies only twelve rules-alerts resources'
|
|
|
|
split_fixture "$(new_fixture risk-accepted-success)"
|
|
PLATFORM_TEST_RECOVERY_OUTPUT_1='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' \
|
|
PLATFORM_TEST_RECOVERY_OUTPUT_2='SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1 ||
|
|
fail 'risk-accepted deployment gate success fixture failed'
|
|
acceptance="$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
|
|
grep -Fqx 'slack_deployment_gate=RISK_ACCEPTED' "$acceptance" || fail 'risk acceptance verdict mismatch'
|
|
grep -Fqx 'slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' "$acceptance" || fail 'risk acceptance reference mismatch'
|
|
grep -Fqx 'slack_gate_accepted_by_uid=1000' "$acceptance" || fail 'risk acceptance uid mismatch'
|
|
pass 'risk-accepted deployment verdict records the exact v2 acceptance mapping'
|
|
|
|
split_fixture "$(new_fixture acceptance-write-failure)"
|
|
if PLATFORM_TEST_SUDO_FAIL_ACCEPTANCE=1 \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'v2 acceptance write failure returned success'
|
|
fi
|
|
grep -Fq 'APPLIED_ACCEPTANCE_UNRECORDED' "$FIXTURE/output" || fail 'acceptance write failure did not identify missing acceptance'
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail 'acceptance write failure left an acceptance marker'
|
|
pass 'v2 acceptance write failure remains unaccepted'
|
|
|
|
for reconcile_case in prometheus-rule grafana-sidecar; do
|
|
case_progress "reconcile-$reconcile_case"
|
|
split_fixture "$(new_fixture "reconcile-$reconcile_case")"
|
|
prometheus_drop=0
|
|
grafana_stale=0
|
|
case "$reconcile_case" in
|
|
prometheus-rule)
|
|
prometheus_drop=1
|
|
;;
|
|
grafana-sidecar)
|
|
grafana_stale=1
|
|
;;
|
|
esac
|
|
if PLATFORM_TEST_PROMETHEUS_DROP_RULE="$prometheus_drop" \
|
|
PLATFORM_TEST_GRAFANA_STALE_DASHBOARD="$grafana_stale" \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "incomplete production acceptance was accepted: $reconcile_case"
|
|
fi
|
|
assert_reconcile_rollback "$FIXTURE" "incomplete production acceptance $reconcile_case"
|
|
done
|
|
for receiver_case in raw empty null duplicate extra wrong-namespace wrong-config wrong-local malformed; do
|
|
case_progress "reconcile-alertmanager-receiver-$receiver_case"
|
|
split_fixture "$(new_fixture "reconcile-alertmanager-receiver-$receiver_case")"
|
|
if PLATFORM_TEST_ALERTMANAGER_RECEIVER_MODE="$receiver_case" \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "invalid Alertmanager receiver was accepted: $receiver_case"
|
|
fi
|
|
assert_reconcile_rollback "$FIXTURE" "invalid Alertmanager receiver $receiver_case"
|
|
done
|
|
expected_reconcile_progress=$'CASE_BEGIN=reconcile-prometheus-rule\nCASE_BEGIN=reconcile-grafana-sidecar\nCASE_BEGIN=reconcile-alertmanager-receiver-raw\nCASE_BEGIN=reconcile-alertmanager-receiver-empty\nCASE_BEGIN=reconcile-alertmanager-receiver-null\nCASE_BEGIN=reconcile-alertmanager-receiver-duplicate\nCASE_BEGIN=reconcile-alertmanager-receiver-extra\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-namespace\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-config\nCASE_BEGIN=reconcile-alertmanager-receiver-wrong-local\nCASE_BEGIN=reconcile-alertmanager-receiver-malformed'
|
|
[[ "$(<"$PROGRESS_LOG")" == "$expected_reconcile_progress" ]] ||
|
|
fail 'long reconciliation failure matrix did not emit exact per-case progress'
|
|
pass 'Prometheus rules, exact Alertmanager receiver matrix, and Grafana sidecar evidence each fail closed before acceptance'
|
|
|
|
split_fixture "$(new_fixture apply-outcome-unknown)"
|
|
if PLATFORM_TEST_FAIL_APPLY_AT=5 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'failed apply returned success'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
|
|
fail 'non-zero apply result was incorrectly reported as a certain no-op'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
|
|
fail 'unknown apply outcome did not require manual recovery'
|
|
[[ "$(find "$FIXTURE/state" -type f | wc -l | tr -d ' ')" == 1 ]] ||
|
|
fail 'recorded prefix was not rolled back after an ambiguous apply result'
|
|
pass 'non-zero apply response is treated as an unknown outcome while owned prefix rolls back'
|
|
|
|
split_fixture "$(new_fixture apply-response-loss)"
|
|
if PLATFORM_TEST_FAIL_AFTER_APPLY_AT=5 run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'lost apply response returned success'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
|
|
fail 'lost apply response was reported as a certain rollback'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
|
|
fail 'lost apply response omitted manual recovery'
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail 'lost apply response wrote acceptance'
|
|
pass 'apply response loss preserves truthful ambiguity and no acceptance marker'
|
|
|
|
split_fixture "$(new_fixture rollback)"
|
|
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'failed reconcile returned success'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$FIXTURE/output" || fail 'rollback PASS marker missing'
|
|
prior="$FIXTURE/state/Alertmanager__observability__observability-core-kube-pr-alertmanager.json"
|
|
jq -e '.metadata.uid == "uid-alertmanager-prior" and .spec.replicas == 1' "$prior" >/dev/null ||
|
|
fail 'rollback did not restore the prior Alertmanager owner payload'
|
|
[[ "$(find "$FIXTURE/state" -type f | wc -l | tr -d ' ')" == 1 ]] || fail 'rollback left newly-created objects'
|
|
! grep -Eq '^kubectl --request-timeout=10s (delete|apply).*(Secret|PersistentVolumeClaim|Probe)' "$FIXTURE/commands.log" ||
|
|
fail 'rollback touched a preserved or non-rules resource'
|
|
delete_order="$(awk '$1=="raw-delete" {path=$2; sub(".*/","",path); print path}' "$FIXTURE/commands.log")"
|
|
expected_delete_order=$'observability-allow-alertmanager-public-https\nplatform-alertmanager\nplatform-verified-services\nplatform-observability-core\nplatform-certificate-probes\nplatform-aistor-storage-quota\ngrafana-dashboard-workload-health\ngrafana-dashboard-platform-services\ngrafana-dashboard-observability-backends\ngrafana-dashboard-kubernetes-node\ngrafana-dashboard-https-endpoints'
|
|
[[ "$delete_order" == "$expected_delete_order" ]] || fail 'rollback did not delete owned creations in exact reverse dependency order'
|
|
[[ "$(grep -c '^raw-delete .* exact=true$' "$FIXTURE/commands.log")" == 11 ]] ||
|
|
fail 'rollback did not send exact UID-preconditioned DeleteOptions through the raw API'
|
|
grep -Fq 'replace-preconditions Alertmanager/observability-core-kube-pr-alertmanager uid=uid-alertmanager-prior resourceVersion=' \
|
|
"$FIXTURE/commands.log" || fail 'rollback restore did not bind both UID and live resourceVersion'
|
|
! grep -Fq 'kubectl --request-timeout=10s delete ' "$FIXTURE/commands.log" ||
|
|
fail 'rollback used a plain kubectl delete branch'
|
|
pass 'failed reconcile reverses only ledger-owned resources and restores exact prior owner'
|
|
|
|
for delete_mode in response-loss conflict timeout; do
|
|
split_fixture "$(new_fixture "delete-$delete_mode")"
|
|
started_at="$(date +%s)"
|
|
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 PLATFORM_TEST_DELETE_MODE="$delete_mode" \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail "raw DELETE $delete_mode returned success"
|
|
fi
|
|
elapsed=$(( $(date +%s) - started_at ))
|
|
(( elapsed <= 30 )) || fail "raw DELETE $delete_mode exceeded its bounded timeout"
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
|
|
fail "raw DELETE $delete_mode was not reported truthfully"
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
|
|
fail "raw DELETE $delete_mode omitted manual recovery"
|
|
grep -Eq '^raw-delete .* exact=true$' "$FIXTURE/commands.log" ||
|
|
fail "raw DELETE $delete_mode did not carry exact UID preconditions"
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail "raw DELETE $delete_mode wrote acceptance"
|
|
done
|
|
pass 'raw DELETE response loss, conflict, and timeout stay bounded and truthfully ambiguous'
|
|
|
|
split_fixture "$(new_fixture replace-conflict)"
|
|
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=0 PLATFORM_TEST_REPLACE_CONFLICT=1 \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'UID/resourceVersion restore conflict returned success'
|
|
fi
|
|
grep -Fq 'replace-preconditions Alertmanager/observability-core-kube-pr-alertmanager uid=uid-alertmanager-prior resourceVersion=' \
|
|
"$FIXTURE/commands.log" || fail 'conflicted restore did not carry UID and resourceVersion'
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
|
|
fail 'restore conflict did not remain ambiguous'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" ||
|
|
fail 'restore conflict omitted manual recovery'
|
|
pass 'UID and resourceVersion restore conflict fails closed with manual recovery'
|
|
|
|
split_fixture "$(new_fixture signal)"
|
|
(
|
|
trap 'exit 143' TERM
|
|
PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_POSTCHECK_DELAY=5 \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC"
|
|
) >"$FIXTURE/output" 2>&1 &
|
|
signal_pid=$!
|
|
for ((signal_attempt=1; signal_attempt<=300; signal_attempt++)); do
|
|
[[ -e "$FIXTURE/postcheck-reached" ]] && break
|
|
kill -0 "$signal_pid" 2>/dev/null || break
|
|
sleep 0.05
|
|
done
|
|
[[ -e "$FIXTURE/postcheck-reached" ]] || fail 'signal fixture never reached a mutation-owned reconciliation boundary'
|
|
apply_pid="$(pgrep -P "$signal_pid" -f "$APPLY" | head -n1)"
|
|
[[ "$apply_pid" =~ ^[0-9]+$ ]] || fail 'signal fixture could not bind the apply process'
|
|
kill -TERM "$apply_pid"
|
|
if wait "$signal_pid"; then
|
|
fail 'signal during mutation returned success'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=PASS' "$FIXTURE/output" ||
|
|
{ sed -n '1,200p' "$FIXTURE/output" >&2; fail 'signal during reconciliation did not complete owned rollback'; }
|
|
[[ ! -e "$FIXTURE/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" ]] ||
|
|
fail 'signal during mutation wrote acceptance'
|
|
pass 'signal during reconciliation invokes owned rollback and leaves no false acceptance'
|
|
|
|
split_fixture "$(new_fixture uid-drift)"
|
|
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_DRIFT_RESOURCE='ConfigMap/grafana-dashboard-workload-health' \
|
|
run_apply "$FIXTURE" "$METRIC" "$INITIAL_SHA" "$POST_SHA" \
|
|
--execute --rules-alerts --verified-output-dir "$METRIC" >"$FIXTURE/output" 2>&1; then
|
|
fail 'UID-drifted rollback returned success'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_ACCESS_RULES_ALERTS_ROLLBACK=AMBIGUOUS' "$FIXTURE/output" ||
|
|
fail 'UID drift did not produce truthful ambiguity'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$FIXTURE/output" || fail 'manual recovery marker missing'
|
|
[[ -f "$FIXTURE/state/ConfigMap__observability__grafana-dashboard-workload-health.json" ]] ||
|
|
fail 'UID-drifted object was deleted without ownership'
|
|
pass 'rollback refuses UID drift and reports manual recovery truthfully'
|
|
|
|
printf 'Assertions: %d\n' "$ASSERTIONS"
|
|
printf 'APPLY OBSERVABILITY ACCESS TEST PASS\n'
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
supervise_focused_suite "$@"
|
|
fi
|