1729 lines
88 KiB
Bash
Executable File
1729 lines
88 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly SCRIPT="$ROOT/scripts/bootstrap/create-observability-secrets.sh"
|
|
readonly NOW_UTC='2026-08-12T07:00:00Z'
|
|
readonly ADMIN_USER='breakglass-admin'
|
|
readonly ADMIN_PASSWORD='GrafanaPassword-123456789'
|
|
readonly SLACK_URL='https://hooks.slack.com/services/T123/B456/S789'
|
|
|
|
WORK=''
|
|
ASSERTIONS=0
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
pass() {
|
|
ASSERTIONS=$((ASSERTIONS + 1))
|
|
printf 'PASS: %s\n' "$1"
|
|
}
|
|
|
|
cleanup() {
|
|
trap - EXIT HUP INT TERM
|
|
case "$WORK" in
|
|
/tmp/platform-observability-secrets-test.??????)
|
|
rm -rf -- "$WORK"
|
|
;;
|
|
esac
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
write_executable() {
|
|
local path=$1
|
|
shift
|
|
printf '%s\n' "$@" >"$path"
|
|
chmod 0755 "$path"
|
|
}
|
|
|
|
make_fakes() {
|
|
local fixture=$1
|
|
mkdir -p "$fixture/bin" "$fixture/state" "$fixture/evidence-parent"
|
|
chmod 0700 "$fixture" "$fixture/state" "$fixture/evidence-parent"
|
|
chmod 0755 "$fixture/bin"
|
|
: >"$fixture/commands.log"
|
|
|
|
write_executable "$fixture/bin/encryption" \
|
|
'#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''%s\n'\'' '\''--'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \
|
|
'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'count=0; [[ ! -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] || read -r count <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
|
|
'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
|
|
'if [[ "$count" == 2 && -n "${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}" ]]; then printf '\''%s'\'' '\''MutatedPassword-987654321'\'' >"$PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE"; fi' \
|
|
'[[ "$count" != "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]'
|
|
|
|
write_executable "$fixture/bin/restore" \
|
|
'#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''%s\n'\'' '\''--'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \
|
|
'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'count=0; [[ ! -f "$PLATFORM_TEST_RESTORE_COUNT" ]] || read -r count <"$PLATFORM_TEST_RESTORE_COUNT"' \
|
|
'count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_RESTORE_COUNT"' \
|
|
'[[ "$count" != "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]'
|
|
|
|
write_executable "$fixture/bin/sudo" \
|
|
'#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
|
|
'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \
|
|
'[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \
|
|
'[[ "${1:-}" != -n && "${1:-}" != --non-interactive ]] || shift' \
|
|
'command=${1:-}; shift || :' \
|
|
'case "$command" in' \
|
|
' /usr/bin/python3)' \
|
|
' if [[ "${4:-}" == platform-observability-marker-helper && ( "${5:-}" == check-dir || "${5:-}" == ensure-dir ) && ( -n "${PLATFORM_TEST_HIERARCHY_FAULT:-}" || -n "${PLATFORM_TEST_HIERARCHY_TRACE:-}" ) ]]; then' \
|
|
' exec "$PLATFORM_TEST_HIERARCHY_SHIM" "$@"' \
|
|
' fi' \
|
|
' if [[ "${4:-}" == platform-observability-marker-helper && "${5:-}" == install ]]; then' \
|
|
' count=0; [[ ! -f "$PLATFORM_TEST_MARKER_MOVE_COUNT" ]] || read -r count <"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \
|
|
' count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \
|
|
' [[ "$count" != "${PLATFORM_TEST_MARKER_MOVE_FAIL_AT:-0}" ]] || exit 75' \
|
|
' if /usr/bin/python3 "$@"; then marker_rc=0; else marker_rc=$?; fi' \
|
|
' (( marker_rc == 0 )) || exit "$marker_rc"' \
|
|
' [[ "$count" != "${PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT:-0}" ]] || exit 76' \
|
|
' if [[ "$count" == "${PLATFORM_TEST_MARKER_SIGNAL_AT:-0}" ]]; then /bin/kill -"${PLATFORM_TEST_MARKER_SIGNAL:-HUP}" "$PPID"; fi' \
|
|
' exit 0' \
|
|
' fi' \
|
|
' exec /usr/bin/python3 "$@" ;;' \
|
|
' /usr/bin/install|/bin/install|install)' \
|
|
' args=(); while (( $# > 0 )); do case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac; done' \
|
|
' exec /usr/bin/install "${args[@]}" ;;' \
|
|
' /usr/bin/stat|/bin/stat|stat)' \
|
|
' format=; path=; while (( $# > 0 )); do case "$1" in --format=*) format=${1#*=}; shift ;; --format) format=$2; shift 2 ;; --) shift ;; *) path=$1; shift ;; esac; done' \
|
|
' [[ "$format" == "%u:%g:%a:%F" ]] || exit 93' \
|
|
' metadata=$(/usr/bin/stat --format="%a:%F" -- "$path")' \
|
|
' printf '\''0:0:%s\n'\'' "$metadata" ;;' \
|
|
' /bin/mv|/usr/bin/mv)' \
|
|
' count=0; [[ ! -f "$PLATFORM_TEST_MARKER_MOVE_COUNT" ]] || read -r count <"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \
|
|
' count=$((count + 1)); printf '\''%s\n'\'' "$count" >"$PLATFORM_TEST_MARKER_MOVE_COUNT"' \
|
|
' [[ "$count" != "${PLATFORM_TEST_MARKER_MOVE_FAIL_AT:-0}" ]] || exit 75' \
|
|
' exec /bin/mv "$@" ;;' \
|
|
' *) exec "$command" "$@" ;;' \
|
|
'esac'
|
|
|
|
cat >"$fixture/bin/hierarchy-python" <<'PY'
|
|
#!/usr/bin/env python3
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
|
|
arguments = sys.argv[1:]
|
|
if arguments[:3] != ["-I", "-S", "-"]:
|
|
raise SystemExit(90)
|
|
sys.argv = ["-"] + arguments[3:]
|
|
|
|
fixture = pathlib.Path(os.environ["PLATFORM_TEST_STATE"]).parent
|
|
fault = os.environ.get("PLATFORM_TEST_HIERARCHY_FAULT", "")
|
|
trace_raw = os.environ.get("PLATFORM_TEST_HIERARCHY_TRACE", "")
|
|
phase = None
|
|
component = None
|
|
target = None
|
|
if fault:
|
|
try:
|
|
phase, component = fault.split(":", 1)
|
|
except ValueError:
|
|
raise SystemExit(91)
|
|
if phase not in {
|
|
"wrong-uid",
|
|
"wrong-gid",
|
|
"wrong-mode",
|
|
"replace-after-open",
|
|
"replace-after-create",
|
|
"replace-after-reopen",
|
|
"replace-before-attestation",
|
|
"replace-before-safe-absence",
|
|
"concurrent-create-exact",
|
|
"concurrent-create-unsafe",
|
|
}:
|
|
raise SystemExit(91)
|
|
if component not in {"evidence-parent", "platform", "recovery-evidence"}:
|
|
raise SystemExit(91)
|
|
if component == "evidence-parent":
|
|
target = fixture / component
|
|
elif component == "platform":
|
|
target = fixture / "evidence-parent" / component
|
|
else:
|
|
target = fixture / "evidence-parent" / "platform" / component
|
|
if not fault and not trace_raw:
|
|
raise SystemExit(91)
|
|
trace_path = pathlib.Path(trace_raw) if trace_raw else None
|
|
managed_paths = {
|
|
fixture / "evidence-parent": "evidence-parent",
|
|
fixture / "evidence-parent" / "platform": "platform",
|
|
fixture / "evidence-parent" / "platform" / "recovery-evidence": "recovery-evidence",
|
|
}
|
|
|
|
real_open = os.open
|
|
real_mkdir = os.mkdir
|
|
real_fstat = os.fstat
|
|
real_fsync = os.fsync
|
|
real_rename = os.rename
|
|
real_chmod = os.chmod
|
|
created = set()
|
|
reopened = set()
|
|
attested = set()
|
|
fsynced = set()
|
|
injected = False
|
|
|
|
|
|
def fd_path(descriptor):
|
|
try:
|
|
return pathlib.Path(os.readlink(f"/proc/self/fd/{descriptor}"))
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def call_path(path, directory_fd):
|
|
decoded = pathlib.Path(os.fsdecode(path))
|
|
if decoded.is_absolute():
|
|
return decoded
|
|
parent = fd_path(directory_fd)
|
|
if parent is None:
|
|
return decoded
|
|
return parent / decoded
|
|
|
|
|
|
def replace_target():
|
|
backup = target.with_name(f".{target.name}.{phase}.original")
|
|
real_rename(target, backup)
|
|
real_mkdir(target, 0o700)
|
|
real_chmod(target, 0o755 if phase == "replace-after-create" else 0o700)
|
|
pathlib.Path(os.environ["PLATFORM_TEST_HIERARCHY_FAULT_LOG"]).write_text(
|
|
f"{phase}:{component}\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def create_concurrent_target(created_path):
|
|
global injected
|
|
injected = True
|
|
real_mkdir(created_path, 0o700)
|
|
real_chmod(created_path, 0o755 if phase == "concurrent-create-unsafe" else 0o700)
|
|
created.add(created_path)
|
|
trace_event("concurrent-create", created_path)
|
|
pathlib.Path(os.environ["PLATFORM_TEST_HIERARCHY_FAULT_LOG"]).write_text(
|
|
f"{phase}:{component}\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def trace_event(event, path):
|
|
if trace_path is None or path not in managed_paths:
|
|
return
|
|
with trace_path.open("a", encoding="utf-8") as stream:
|
|
stream.write(f"{event}:{managed_paths[path]}\n")
|
|
|
|
|
|
def shim_open(path, flags, mode=0o777, *, dir_fd=None):
|
|
global injected
|
|
opened_path = call_path(path, dir_fd)
|
|
try:
|
|
if dir_fd is None:
|
|
descriptor = real_open(path, flags, mode)
|
|
else:
|
|
descriptor = real_open(path, flags, mode, dir_fd=dir_fd)
|
|
except FileNotFoundError:
|
|
safe_absence_child = None
|
|
if component == "evidence-parent":
|
|
safe_absence_child = target / "platform"
|
|
elif component == "platform":
|
|
safe_absence_child = target / "recovery-evidence"
|
|
if phase == "replace-before-safe-absence" and opened_path == safe_absence_child and not injected:
|
|
injected = True
|
|
replace_target()
|
|
raise
|
|
should_replace = (
|
|
opened_path == target
|
|
and not injected
|
|
and (
|
|
phase == "replace-after-open"
|
|
or (phase == "replace-after-reopen" and target in created)
|
|
)
|
|
)
|
|
if should_replace:
|
|
injected = True
|
|
replace_target()
|
|
if opened_path in created and opened_path not in reopened:
|
|
reopened.add(opened_path)
|
|
trace_event("reopen", opened_path)
|
|
return descriptor
|
|
|
|
|
|
def shim_mkdir(path, mode=0o777, *, dir_fd=None):
|
|
global injected
|
|
created_path = call_path(path, dir_fd)
|
|
if (
|
|
phase in {"concurrent-create-exact", "concurrent-create-unsafe"}
|
|
and created_path == target
|
|
and not injected
|
|
):
|
|
create_concurrent_target(created_path)
|
|
if dir_fd is None:
|
|
result = real_mkdir(path, mode)
|
|
else:
|
|
result = real_mkdir(path, mode, dir_fd=dir_fd)
|
|
created.add(created_path)
|
|
trace_event("mkdir", created_path)
|
|
if phase == "replace-after-create" and created_path == target and not injected:
|
|
injected = True
|
|
replace_target()
|
|
return result
|
|
|
|
|
|
def shim_fsync(descriptor):
|
|
parent = fd_path(descriptor)
|
|
for created_path in reversed(tuple(created)):
|
|
if created_path.parent == parent and created_path not in fsynced:
|
|
fsynced.add(created_path)
|
|
trace_event("fsync-parent", created_path)
|
|
break
|
|
return real_fsync(descriptor)
|
|
|
|
|
|
def shim_fstat(descriptor):
|
|
global injected
|
|
opened_path = fd_path(descriptor)
|
|
if (
|
|
phase == "replace-before-attestation"
|
|
and opened_path == target
|
|
and target in created
|
|
and not injected
|
|
):
|
|
injected = True
|
|
replace_target()
|
|
metadata = real_fstat(descriptor)
|
|
if opened_path in created and opened_path not in attested:
|
|
attested.add(opened_path)
|
|
trace_event("attest", opened_path)
|
|
if opened_path != target or injected or phase not in {"wrong-uid", "wrong-gid", "wrong-mode"}:
|
|
return metadata
|
|
injected = True
|
|
values = list(metadata)
|
|
if phase == "wrong-uid":
|
|
values[4] = metadata.st_uid + 1
|
|
elif phase == "wrong-gid":
|
|
values[5] = metadata.st_gid + 1
|
|
else:
|
|
values[0] = (metadata.st_mode & ~0o7777) | 0o755
|
|
return os.stat_result(values)
|
|
|
|
|
|
os.open = shim_open
|
|
os.mkdir = shim_mkdir
|
|
os.fstat = shim_fstat
|
|
os.fsync = shim_fsync
|
|
source = sys.stdin.read()
|
|
exec(compile(source, "<hierarchy-helper>", "exec"), {"__name__": "__main__"})
|
|
PY
|
|
chmod 0755 "$fixture/bin/hierarchy-python"
|
|
|
|
cat >"$fixture/bin/kubectl" <<'PY'
|
|
#!/usr/bin/env python3
|
|
import base64
|
|
import http.server
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import socketserver
|
|
import sys
|
|
import time
|
|
|
|
args = sys.argv[1:]
|
|
state = pathlib.Path(os.environ["PLATFORM_TEST_STATE"])
|
|
log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"])
|
|
with log.open("a", encoding="utf-8") as stream:
|
|
stream.write("kubectl " + " ".join(args) + "\n")
|
|
|
|
required_timeout = os.environ.get("PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT", "0") == "1"
|
|
if required_timeout and args and "--request-timeout=5s" not in args:
|
|
raise SystemExit(84)
|
|
required_process_timeout = os.environ.get("PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT", "0") == "1"
|
|
if required_process_timeout:
|
|
parent_cmdline = pathlib.Path(f"/proc/{os.getppid()}/cmdline").read_bytes().split(b"\0")
|
|
if not parent_cmdline or parent_cmdline[0] != b"/usr/bin/timeout":
|
|
raise SystemExit(85)
|
|
|
|
def option(name, default=None):
|
|
for i, value in enumerate(args):
|
|
if value == name and i + 1 < len(args):
|
|
return args[i + 1]
|
|
if value.startswith(name + "="):
|
|
return value.split("=", 1)[1]
|
|
return default
|
|
|
|
def normalized():
|
|
output = []
|
|
skip = False
|
|
for i, value in enumerate(args):
|
|
if skip:
|
|
skip = False
|
|
continue
|
|
if value in {"--namespace", "-n"}:
|
|
skip = True
|
|
continue
|
|
if value.startswith("--namespace="):
|
|
continue
|
|
if value.startswith("--request-timeout="):
|
|
continue
|
|
output.append(value)
|
|
return output
|
|
|
|
def secret_path(name):
|
|
return state / f"secret__observability__{name}.json"
|
|
|
|
def load_secret(name):
|
|
path = secret_path(name)
|
|
return json.loads(path.read_text()) if path.exists() else None
|
|
|
|
def save_secret(name, item):
|
|
secret_path(name).write_text(json.dumps(item), encoding="utf-8")
|
|
|
|
nargs = normalized()
|
|
if os.environ.get("PLATFORM_TEST_HANG_KUBECTL", "0") == "1" and nargs and nargs[0] != "proxy":
|
|
import time
|
|
time.sleep(30)
|
|
if nargs and nargs[0] == "proxy":
|
|
sock = option("--unix-socket")
|
|
if not sock:
|
|
raise SystemExit(81)
|
|
(state / "proxy-pid").write_text(str(os.getpid()))
|
|
|
|
class Server(socketserver.UnixStreamServer):
|
|
allow_reuse_address = False
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def log_message(self, *_):
|
|
return
|
|
|
|
def do_DELETE(self):
|
|
name = self.path.rsplit("/", 1)[-1]
|
|
length = int(self.headers.get("Content-Length", "0"))
|
|
try:
|
|
body = json.loads(self.rfile.read(length))
|
|
expected_uid = body["preconditions"]["uid"]
|
|
except Exception:
|
|
self.send_response(400); self.end_headers(); return
|
|
item = load_secret(name)
|
|
if item is None:
|
|
self.send_response(404); self.end_headers(); return
|
|
if item["metadata"]["uid"] != expected_uid:
|
|
self.send_response(409); self.end_headers(); return
|
|
secret_path(name).unlink()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.end_headers()
|
|
self.wfile.write(b'{"kind":"Status","status":"Success"}')
|
|
|
|
with Server(sock, Handler) as server:
|
|
server.serve_forever()
|
|
raise SystemExit(0)
|
|
|
|
if nargs == ["config", "current-context"]:
|
|
print(os.environ.get("PLATFORM_TEST_CONTEXT", "default"))
|
|
raise SystemExit(0)
|
|
if nargs[:2] == ["get", "namespace"] and nargs[2:] == ["observability"]:
|
|
raise SystemExit(0)
|
|
if len(nargs) >= 3 and nargs[:2] == ["get", "secret"]:
|
|
name = nargs[2]
|
|
item = load_secret(name)
|
|
ignore = "--ignore-not-found" in args
|
|
if item is None:
|
|
if ignore:
|
|
raise SystemExit(0)
|
|
raise SystemExit(1)
|
|
output = option("--output", option("-o", ""))
|
|
if output == "json" and os.environ.get("PLATFORM_TEST_POSTCREATE_EXTRA_KEY", "0") == "1" and item["metadata"]["uid"].startswith("uid-owned-"):
|
|
item["data"]["extra"] = base64.b64encode(b"drift").decode()
|
|
if output in {"json", "jsonpath={.metadata.uid}"}:
|
|
if output == "json":
|
|
print(json.dumps(item))
|
|
else:
|
|
print(item["metadata"]["uid"], end="")
|
|
elif output == "name":
|
|
print("secret/" + name)
|
|
else:
|
|
raise SystemExit(82)
|
|
raise SystemExit(0)
|
|
if nargs and nargs[0] == "create" and option("--filename"):
|
|
item = json.loads(pathlib.Path(option("--filename")).read_text())
|
|
name = item["metadata"]["name"]
|
|
count_file = pathlib.Path(os.environ["PLATFORM_TEST_CREATE_COUNT"])
|
|
count = int(count_file.read_text()) if count_file.exists() else 0
|
|
count += 1
|
|
count_file.write_text(str(count))
|
|
if secret_path(name).exists():
|
|
raise SystemExit(1)
|
|
item["metadata"]["uid"] = f"uid-owned-{count}"
|
|
ambiguous = count == int(os.environ.get("PLATFORM_TEST_CREATE_AMBIGUOUS_AT", "0"))
|
|
failing = count == int(os.environ.get("PLATFORM_TEST_CREATE_FAIL_AT", "0"))
|
|
if ambiguous or not failing:
|
|
save_secret(name, item)
|
|
signal_at = int(os.environ.get("PLATFORM_TEST_CREATE_SIGNAL_AT", "0"))
|
|
if count == signal_at:
|
|
signal_name = os.environ.get("PLATFORM_TEST_CREATE_SIGNAL", "HUP")
|
|
signal_number = getattr(signal, "SIG" + signal_name)
|
|
timeout_parent = os.getppid()
|
|
timeout_status = pathlib.Path(f"/proc/{timeout_parent}/stat").read_text().split()
|
|
shell_parent = int(timeout_status[3])
|
|
os.kill(shell_parent, signal_number)
|
|
time.sleep(0.2)
|
|
raise SystemExit(74)
|
|
if failing or ambiguous:
|
|
if os.environ.get("PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE", "0") == "1":
|
|
for path in state.glob("secret__observability__*.json"):
|
|
current = json.loads(path.read_text())
|
|
current["metadata"]["uid"] = "uid-external-replacement"
|
|
path.write_text(json.dumps(current))
|
|
raise SystemExit(73)
|
|
if option("--output", option("-o", "")) == "json":
|
|
print(json.dumps(item))
|
|
sys.stdout.flush()
|
|
else:
|
|
print(f"secret/{name} created")
|
|
sys.stdout.flush()
|
|
signal_after_response_at = int(os.environ.get("PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT", "0"))
|
|
if count == signal_after_response_at:
|
|
signal_name = os.environ.get("PLATFORM_TEST_CREATE_SIGNAL", "HUP")
|
|
signal_number = getattr(signal, "SIG" + signal_name)
|
|
timeout_parent = os.getppid()
|
|
timeout_status = pathlib.Path(f"/proc/{timeout_parent}/stat").read_text().split()
|
|
shell_parent = int(timeout_status[3])
|
|
os.kill(shell_parent, signal_number)
|
|
time.sleep(0.2)
|
|
raise SystemExit(74)
|
|
raise SystemExit(0)
|
|
raise SystemExit(83)
|
|
PY
|
|
chmod 0755 "$fixture/bin/kubectl"
|
|
}
|
|
|
|
new_fixture() {
|
|
local name=$1 fixture
|
|
fixture="$WORK/$name"
|
|
mkdir -p "$fixture"
|
|
make_fakes "$fixture"
|
|
printf '%s' "$ADMIN_USER" >"$fixture/admin-user"
|
|
printf '%s' "$ADMIN_PASSWORD" >"$fixture/admin-password"
|
|
printf '%s' "$SLACK_URL" >"$fixture/slack-url"
|
|
chmod 0600 "$fixture/admin-user" "$fixture/admin-password" "$fixture/slack-url"
|
|
printf '%s\n' "$fixture"
|
|
}
|
|
|
|
create_exact_evidence_hierarchy() {
|
|
local fixture=$1
|
|
mkdir -m 0700 -- "$fixture/evidence-parent/platform"
|
|
mkdir -m 0700 -- "$fixture/evidence-parent/platform/recovery-evidence"
|
|
}
|
|
|
|
prepare_managed_component_parent() {
|
|
local fixture=$1 component=$2
|
|
case "$component" in
|
|
evidence-parent|platform) ;;
|
|
recovery-evidence) mkdir -m 0700 -- "$fixture/evidence-parent/platform" ;;
|
|
*) fail "unknown managed component: $component" ;;
|
|
esac
|
|
}
|
|
|
|
managed_component_path() {
|
|
local fixture=$1 component=$2
|
|
case "$component" in
|
|
evidence-parent) printf '%s\n' "$fixture/evidence-parent" ;;
|
|
platform) printf '%s\n' "$fixture/evidence-parent/platform" ;;
|
|
recovery-evidence) printf '%s\n' "$fixture/evidence-parent/platform/recovery-evidence" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
seed_secret() {
|
|
local fixture=$1 name=$2 type=$3 keys=$4
|
|
python3 -I -S - "$fixture" "$name" "$type" "$keys" <<'PY'
|
|
import base64, json, pathlib, sys
|
|
fixture, name, secret_type, keys = sys.argv[1:]
|
|
values = {
|
|
"admin-user": "breakglass-admin",
|
|
"admin-password": "GrafanaPassword-123456789",
|
|
"url": "https://hooks.slack.com/services/T123/B456/S789",
|
|
"extra": "not-secret-fixture",
|
|
}
|
|
item = {
|
|
"apiVersion": "v1", "kind": "Secret", "type": secret_type,
|
|
"metadata": {"namespace": "observability", "name": name, "uid": "uid-existing-" + name},
|
|
"data": {key: base64.b64encode(values[key].encode()).decode() for key in keys.split(",") if key},
|
|
}
|
|
path = pathlib.Path(fixture) / "state" / f"secret__observability__{name}.json"
|
|
path.write_text(json.dumps(item))
|
|
PY
|
|
}
|
|
|
|
run_script() {
|
|
local fixture=$1
|
|
shift
|
|
env \
|
|
PATH="$fixture/bin:$PATH" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS="${PLATFORM_TEST_CONFIRMATIONS:-}" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE="${PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE:-}" \
|
|
PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
|
|
PLATFORM_TEST_STATE="$fixture/state" \
|
|
PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \
|
|
PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \
|
|
PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \
|
|
PLATFORM_TEST_CREATE_COUNT="$fixture/create-count" \
|
|
PLATFORM_TEST_MARKER_MOVE_COUNT="$fixture/marker-move-count" \
|
|
PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_CREATE_FAIL_AT="${PLATFORM_TEST_CREATE_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_CREATE_AMBIGUOUS_AT="${PLATFORM_TEST_CREATE_AMBIGUOUS_AT:-0}" \
|
|
PLATFORM_TEST_CREATE_SIGNAL_AT="${PLATFORM_TEST_CREATE_SIGNAL_AT:-0}" \
|
|
PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT="${PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT:-0}" \
|
|
PLATFORM_TEST_CREATE_SIGNAL="${PLATFORM_TEST_CREATE_SIGNAL:-HUP}" \
|
|
PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE="${PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE:-0}" \
|
|
PLATFORM_TEST_POSTCREATE_EXTRA_KEY="${PLATFORM_TEST_POSTCREATE_EXTRA_KEY:-0}" \
|
|
PLATFORM_TEST_MARKER_MOVE_FAIL_AT="${PLATFORM_TEST_MARKER_MOVE_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT="${PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT:-0}" \
|
|
PLATFORM_TEST_MARKER_SIGNAL_AT="${PLATFORM_TEST_MARKER_SIGNAL_AT:-0}" \
|
|
PLATFORM_TEST_MARKER_SIGNAL="${PLATFORM_TEST_MARKER_SIGNAL:-HUP}" \
|
|
PLATFORM_TEST_HIERARCHY_FAULT="${PLATFORM_TEST_HIERARCHY_FAULT:-}" \
|
|
PLATFORM_TEST_HIERARCHY_TRACE="${PLATFORM_TEST_HIERARCHY_TRACE:-}" \
|
|
PLATFORM_TEST_HIERARCHY_SHIM="$fixture/bin/hierarchy-python" \
|
|
PLATFORM_TEST_HIERARCHY_FAULT_LOG="$fixture/hierarchy-fault.log" \
|
|
PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE="${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}" \
|
|
PLATFORM_TEST_CONTEXT="${PLATFORM_TEST_CONTEXT:-default}" \
|
|
PLATFORM_TEST_VALIDATOR_ENV_LOG="${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" \
|
|
PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT="${PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT:-0}" \
|
|
PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT="${PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT:-0}" \
|
|
PLATFORM_TEST_HANG_KUBECTL="${PLATFORM_TEST_HANG_KUBECTL:-0}" \
|
|
bash -c 'source "$1"; shift; platform_observability_secrets_fixture_main "$@"' \
|
|
platform-observability-secrets-test "$SCRIPT" "$fixture" "$@"
|
|
}
|
|
|
|
run_internal_root_helper() {
|
|
local fixture=$1 action=$2 directory=$3 basename=$4
|
|
local source=${5:-} expected_kind=${6:-} expected_source=${7:-}
|
|
env \
|
|
PATH="$fixture/bin:$PATH" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \
|
|
PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
|
|
PLATFORM_TEST_STATE="$fixture/state" \
|
|
PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \
|
|
PLATFORM_TEST_MARKER_MOVE_COUNT="$fixture/marker-move-count" \
|
|
PLATFORM_TEST_MARKER_MOVE_FAIL_AT=0 \
|
|
PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=0 \
|
|
PLATFORM_TEST_MARKER_SIGNAL_AT=0 \
|
|
PLATFORM_TEST_HIERARCHY_FAULT="${PLATFORM_TEST_HIERARCHY_FAULT:-}" \
|
|
PLATFORM_TEST_HIERARCHY_TRACE="${PLATFORM_TEST_HIERARCHY_TRACE:-}" \
|
|
PLATFORM_TEST_HIERARCHY_SHIM="$fixture/bin/hierarchy-python" \
|
|
PLATFORM_TEST_HIERARCHY_FAULT_LOG="$fixture/hierarchy-fault.log" \
|
|
bash -c '
|
|
source "$1"
|
|
TEST_MODE=true
|
|
configure_test_boundaries "$2"
|
|
sudo_refresh
|
|
directory=$4
|
|
basename=$5
|
|
[[ "$directory" != @configured-anchor ]] || directory=$EVIDENCE_HIERARCHY_ANCHOR
|
|
[[ "$basename" != @configured-profile ]] || basename=$EVIDENCE_HIERARCHY_PROFILE
|
|
root_marker_helper /dev/null "$3" "$directory" "$basename" \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$6" "$7" "$8" "$EUID"
|
|
' platform-observability-hierarchy-helper-test "$SCRIPT" "$fixture" "$action" "$directory" "$basename" \
|
|
"$source" "$expected_kind" "$expected_source"
|
|
}
|
|
|
|
run_hierarchy_action() {
|
|
local fixture=$1 action=$2
|
|
run_internal_root_helper "$fixture" "$action" @configured-anchor @configured-profile
|
|
}
|
|
|
|
assert_slack_risk_hierarchy_failure() {
|
|
local fixture=$1 label=$2 fault=${3:-}
|
|
if PLATFORM_TEST_HIERARCHY_FAULT="$fault" \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail "$label was accepted"
|
|
fi
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail "$label published a Slack risk marker"
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] ||
|
|
fail "$label left a Slack Secret"
|
|
! grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" ||
|
|
fail "$label emitted a transaction success marker"
|
|
pass "$label fails closed before marker publication"
|
|
}
|
|
|
|
WORK="$(mktemp -d /tmp/platform-observability-secrets-test.XXXXXX)"
|
|
chmod 0700 "$WORK"
|
|
|
|
[[ -f "$SCRIPT" ]] || fail 'production script is absent'
|
|
|
|
fixture="$(new_fixture dry-run)"
|
|
output="$(run_script "$fixture")" || fail 'dry-run failed'
|
|
[[ "$output" == *'OBSERVABILITY_SECRETS_DRY_RUN=PASS'* ]] || fail 'dry-run marker absent'
|
|
[[ ! -s "$fixture/commands.log" ]] || fail 'dry-run crossed a system boundary'
|
|
[[ "$output" != *"$ADMIN_PASSWORD"* && "$output" != *"$SLACK_URL"* ]] || fail 'dry-run leaked payload'
|
|
pass 'dry-run is fixed, payload-free, and read-only'
|
|
|
|
fixture="$(new_fixture direct-test-mode)"
|
|
if env \
|
|
PLATFORM_OBSERVABILITY_SECRETS_TEST_MODE=1 \
|
|
PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN="$fixture/bin/kubectl" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_SUDO_BIN="$fixture/bin/sudo" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_ENCRYPTION_SCRIPT="$fixture/bin/encryption" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_RESTORE_SCRIPT="$fixture/bin/restore" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_EVIDENCE_DIR="$fixture/evidence-parent/platform/recovery-evidence" \
|
|
PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \
|
|
bash "$SCRIPT" >"$fixture/direct-output" 2>&1; then
|
|
fail 'direct production execution accepted environment-only test mode'
|
|
fi
|
|
if PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN="$fixture/bin/kubectl" \
|
|
bash "$SCRIPT" >"$fixture/override-output" 2>&1; then
|
|
fail 'production execution silently ignored a boundary override'
|
|
fi
|
|
if PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE=first-write \
|
|
bash "$SCRIPT" >"$fixture/marker-fault-override-output" 2>&1; then
|
|
fail 'production execution accepted a marker fault injection override'
|
|
fi
|
|
pass 'production entrypoint rejects test mode and all boundary overrides'
|
|
|
|
for argv in \
|
|
'--execute' \
|
|
'--grafana-admin' \
|
|
'--execute --grafana-admin --grafana-admin-user-file /tmp/a' \
|
|
'--execute --slack-webhook --slack-webhook-file relative' \
|
|
'--check-grafana-recovery-evidence --execute --grafana-admin' \
|
|
'--accept-no-off-host-slack-recovery' \
|
|
'--check-slack-deployment-evidence --accept-no-off-host-slack-recovery' \
|
|
'--execute --grafana-admin --grafana-admin-user-file /tmp/a --grafana-admin-password-file /tmp/b --accept-no-off-host-slack-recovery'; do
|
|
fixture="$(new_fixture argv)"
|
|
read -r -a args <<<"$argv"
|
|
if run_script "$fixture" "${args[@]}" >"$fixture/output" 2>&1; then
|
|
fail "invalid argv accepted: $argv"
|
|
fi
|
|
[[ "$(PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC="$NOW_UTC" \
|
|
bash -c 'source "$1"; shift; platform_observability_secrets_fixture_main "$@"' \
|
|
platform-observability-secrets-test "$SCRIPT" "$fixture" "${args[@]}" >/dev/null 2>&1; printf %s "$?")" == 2 ]] ||
|
|
fail "invalid argv did not return usage RC 2: $argv"
|
|
[[ ! -s "$fixture/commands.log" ]] || fail "invalid argv crossed a system boundary: $argv"
|
|
done
|
|
pass 'closed CLI rejects partial, relative, and mixed modes before boundaries'
|
|
|
|
fixture="$(new_fixture insecure-file)"
|
|
chmod 0644 "$fixture/admin-password"
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'insecure input mode accepted'
|
|
fi
|
|
[[ ! -s "$fixture/commands.log" ]] || fail 'insecure input reached a system boundary'
|
|
pass 'input metadata is gated before external or payload access'
|
|
|
|
fixture="$(new_fixture input-symlink)"
|
|
mv -- "$fixture/admin-user" "$fixture/admin-user-real"
|
|
ln -s -- admin-user-real "$fixture/admin-user"
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'symlink input accepted'
|
|
fi
|
|
[[ ! -s "$fixture/commands.log" ]] || fail 'symlink input reached a system boundary'
|
|
pass 'symlink payload input is rejected before external access'
|
|
|
|
fixture="$(new_fixture input-hardlink)"
|
|
ln -- "$fixture/admin-password" "$fixture/admin-password-second-name"
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'multiply-linked payload input accepted'
|
|
fi
|
|
[[ ! -s "$fixture/commands.log" ]] || fail 'multiply-linked input reached a system boundary'
|
|
pass 'payload input must have exactly one hard link'
|
|
|
|
fixture="$(new_fixture short-password)"
|
|
printf '%s' '1234567890123456789' >"$fixture/admin-password"
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'short Grafana password accepted'
|
|
fi
|
|
! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'short password reached create'
|
|
[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'short password reached confirmation'
|
|
pass 'Grafana password shorter than 20 bytes is rejected before prompt'
|
|
|
|
fixture="$(new_fixture first-gate)"
|
|
if PLATFORM_TEST_ENCRYPTION_FAIL_AT=1 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'failed initial encryption validator accepted'
|
|
fi
|
|
[[ "$(<"$fixture/commands.log")" == 'encryption --expect-reencrypted' ]] ||
|
|
fail 'initial encryption failure did not stop at the fresh validator process'
|
|
[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'initial validator failure reached confirmation'
|
|
pass 'initial encryption gate is zero-read and zero-mutation'
|
|
|
|
fixture="$(new_fixture validator-env)"
|
|
: >"$fixture/validator-env.log"
|
|
printf '%s\n' 'if [[ "${1:-}" == --expect-reencrypted || "${1:-}" == --check ]]; then printf INJECTED >"$PLATFORM_TEST_STATE/injected"; fi' >"$fixture/bash-env"
|
|
if ! BASH_ENV=/dev/fd/9 \
|
|
PLATFORM_TEST_VALIDATOR_ENV_LOG="$fixture/validator-env.log" \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 \
|
|
9< <(printf '%s\n' 'if [[ "${1:-}" == --expect-reencrypted || "${1:-}" == --check ]]; then printf INJECTED >"$PLATFORM_TEST_STATE/injected"; fi'); then
|
|
sed -n '1,180p' "$fixture/output" >&2
|
|
sed -n '1,220p' "$fixture/commands.log" >&2
|
|
fail 'safe validator environment transaction failed'
|
|
fi
|
|
[[ ! -e "$fixture/state/injected" ]] || fail 'validator child sourced inherited BASH_ENV'
|
|
! grep -Eq '^(BASH_ENV|ENV|CDPATH|SHELLOPTS)=' "$fixture/validator-env.log" ||
|
|
fail 'validator child inherited a shell injection variable'
|
|
pass 'fresh validator processes receive only an exact safe environment'
|
|
|
|
fixture="$(new_fixture kubectl-timeout)"
|
|
if ! PLATFORM_TEST_REQUIRE_REQUEST_TIMEOUT=1 \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'kubectl calls omitted the exact bounded request timeout'
|
|
fi
|
|
pass 'every non-proxy kubectl API call carries the bounded request timeout'
|
|
|
|
fixture="$(new_fixture kubectl-process-timeout)"
|
|
if ! PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT=1 \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'kubectl processes were not enclosed by an exact process deadline'
|
|
fi
|
|
fixture="$(new_fixture kubectl-proxy-process-timeout)"
|
|
if PLATFORM_TEST_REQUIRE_PROCESS_TIMEOUT=1 PLATFORM_TEST_CREATE_FAIL_AT=2 \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
fail 'process-timeout proxy rollback fault was unexpectedly accepted'
|
|
fi
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail 'rollback proxy was not enclosed by a process deadline'
|
|
grep -Eq '^kubectl .* proxy ' "$fixture/commands.log" || fail 'proxy deadline regression did not reach proxy'
|
|
proxy_pid="$(<"$fixture/state/proxy-pid")"
|
|
if /bin/kill -0 "$proxy_pid" 2>/dev/null; then
|
|
fail 'rollback returned while the kubectl proxy process was still alive'
|
|
fi
|
|
pass 'every kubectl process, including proxy, is enclosed by a process deadline'
|
|
|
|
fixture="$(new_fixture kubectl-hang)"
|
|
hang_started="$(date +%s)"
|
|
if PLATFORM_TEST_HANG_KUBECTL=1 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'hanging kubectl fake was accepted'
|
|
fi
|
|
hang_elapsed=$(( $(date +%s) - hang_started ))
|
|
(( hang_elapsed <= 6 )) || fail 'hanging kubectl process exceeded its fixed deadline'
|
|
[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'hanging kubectl process reached confirmation'
|
|
pass 'a kubectl process that ignores request timeout is forcibly bounded'
|
|
|
|
fixture="$(new_fixture create-response)"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 ||
|
|
fail 'create response transaction failed'
|
|
grep -Eq '^kubectl .*create --filename=.*create-grafana.json --output=json' "$fixture/commands.log" ||
|
|
fail 'Secret creation did not consume one JSON create response'
|
|
create_line="$(grep -n '^kubectl .*create --filename=.*create-grafana.json ' "$fixture/commands.log" | cut -d: -f1)"
|
|
uid_get_line="$(grep -nF 'jsonpath={.metadata.uid}' "$fixture/commands.log" | head -1 | cut -d: -f1 || true)"
|
|
[[ -z "$uid_get_line" || "$uid_get_line" -lt "$create_line" ]] ||
|
|
fail 'transaction ownership was adopted from a separate post-create UID GET'
|
|
pass 'Secret UID and payload ownership come from the exact create response'
|
|
|
|
fixture="$(new_fixture malformed-slack)"
|
|
printf '%s' 'https://hooks.slack.com/services/T1/B2' >"$fixture/slack-url"
|
|
if run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
fail 'malformed Slack URL accepted'
|
|
fi
|
|
! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'bad Slack URL reached create'
|
|
[[ "$(<"$fixture/output")" != *'Type APPLY'* ]] || fail 'bad Slack URL reached confirmation'
|
|
pass 'Slack input requires the exact prefix and three path components'
|
|
|
|
fixture="$(new_fixture schema-drift)"
|
|
seed_secret "$fixture" grafana-admin kubernetes.io/basic-auth 'admin-user,admin-password'
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'wrong existing Secret type accepted'
|
|
fi
|
|
! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'schema drift reached create'
|
|
pass 'existing Secret requires exact Opaque type and key set'
|
|
|
|
fixture="$(new_fixture payload-drift)"
|
|
seed_secret "$fixture" grafana-admin Opaque 'admin-user,admin-password'
|
|
printf '%s' 'DifferentPassword-123456789' >"$fixture/admin-password"
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'existing payload drift accepted'
|
|
fi
|
|
! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'payload drift reached create'
|
|
pass 'existing payload mismatch refuses implicit rotation'
|
|
|
|
fixture="$(new_fixture last-gate)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_RESTORE_FAIL_AT=2 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'failed last restore gate accepted'
|
|
fi
|
|
[[ "$(grep -c '^encryption --expect-reencrypted$' "$fixture/commands.log")" == 2 ]] ||
|
|
fail 'encryption validator was not a fresh two-call boundary'
|
|
[[ "$(grep -c '^restore --check$' "$fixture/commands.log")" == 2 ]] ||
|
|
fail 'restore validator was not a fresh two-call boundary'
|
|
! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'last gate failure reached create'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence" ]] || fail 'last gate failure mutated the root evidence tree'
|
|
pass 'immediate pre-create validators fail with zero Secret mutation'
|
|
|
|
fixture="$(new_fixture both-success)"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1 || {
|
|
sed -n '1,240p' "$fixture/output" >&2
|
|
fail 'combined create transaction failed'
|
|
}
|
|
[[ -f "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'Grafana Secret absent'
|
|
[[ -f "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'Slack Secret absent'
|
|
[[ "$(grep -c '^encryption --expect-reencrypted$' "$fixture/commands.log")" == 2 ]] || fail 'combined transaction encryption gate count changed'
|
|
[[ "$(grep -c '^restore --check$' "$fixture/commands.log")" == 2 ]] || fail 'combined transaction restore gate count changed'
|
|
[[ "$(grep -c '^kubectl .*create --filename=' "$fixture/commands.log")" == 2 ]] || fail 'combined transaction did not create exactly two Secrets'
|
|
grep -Fq 'OBSERVABILITY_GRAFANA_ADMIN=CREATE_CONFIRMED' "$fixture/output" || fail 'Grafana success marker absent'
|
|
grep -Fq 'OBSERVABILITY_SLACK_WEBHOOK=CREATE_CONFIRMED' "$fixture/output" || fail 'Slack success marker absent'
|
|
[[ "$(<"$fixture/output")" != *"$ADMIN_PASSWORD"* && "$(<"$fixture/output")" != *"$SLACK_URL"* ]] || fail 'success output leaked payload'
|
|
! grep -Fq "$ADMIN_PASSWORD" "$fixture/commands.log" || fail 'password leaked to argv log'
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'webhook leaked to argv log'
|
|
for marker in grafana-local slack; do
|
|
path="$fixture/evidence-parent/platform/recovery-evidence/$marker.env"
|
|
[[ -f "$path" && ! -L "$path" ]] || fail "$marker recovery marker absent"
|
|
[[ "$(stat -c '%a' "$path")" == 600 ]] || fail "$marker recovery marker mode changed"
|
|
[[ "$(wc -l <"$path" | tr -d '[:space:]')" == 4 ]] || fail "$marker recovery marker key count changed"
|
|
done
|
|
grep -Fqx 'schema=platform-observability-recovery-evidence-v1' "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'marker schema changed'
|
|
grep -Fqx 'context=default' "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'marker context changed'
|
|
grep -Fqx 'resource=observability/grafana-admin' "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'Grafana marker resource changed'
|
|
grep -Fqx "checked_at_utc=$NOW_UTC" "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" || fail 'marker timestamp changed'
|
|
grep -Fqx 'resource=observability/alertmanager-slack-webhook' "$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'Slack marker resource changed'
|
|
pass 'combined create is one payload-safe transaction with exact evidence markers'
|
|
|
|
: >"$fixture/commands.log"
|
|
run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/check-output" 2>&1 || {
|
|
sed -n '1,160p' "$fixture/check-output" >&2
|
|
sed -n '1,240p' "$fixture/commands.log" >&2
|
|
fail 'fresh Grafana recovery marker rejected'
|
|
}
|
|
run_script "$fixture" --check-slack-recovery-evidence >>"$fixture/check-output" 2>&1 || {
|
|
sed -n '1,200p' "$fixture/check-output" >&2
|
|
sed -n '1,280p' "$fixture/commands.log" >&2
|
|
fail 'fresh Slack recovery marker rejected'
|
|
}
|
|
grep -Fq 'GRAFANA_RECOVERY_EVIDENCE=PASS' "$fixture/check-output" || fail 'Grafana check marker absent'
|
|
grep -Fq 'SLACK_RECOVERY_EVIDENCE=PASS' "$fixture/check-output" || fail 'Slack check marker absent'
|
|
! grep -q '^encryption\|^restore' "$fixture/commands.log" || fail 'evidence-only check crossed encryption validators'
|
|
[[ "$(grep -c '^kubectl --request-timeout=5s config current-context$' "$fixture/commands.log")" == 2 ]] || fail 'evidence check did not bind current-user context'
|
|
grep -q '^sudo ' "$fixture/commands.log" || fail 'evidence checker did not use narrow sudo'
|
|
pass 'standalone recovery checks use only narrow sudo and accept exact fresh evidence'
|
|
|
|
if PLATFORM_TEST_CONTEXT=wrong-context run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/context-output" 2>&1; then
|
|
fail 'recovery evidence accepted under the wrong current-user context'
|
|
fi
|
|
pass 'recovery evidence is bound to the live current-user context'
|
|
|
|
mv -- "$fixture/evidence-parent/platform/recovery-evidence" "$fixture/evidence-parent/platform/recovery-evidence-real"
|
|
ln -s -- recovery-evidence-real "$fixture/evidence-parent/platform/recovery-evidence"
|
|
if run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/symlink-output" 2>&1; then
|
|
fail 'recovery evidence under a symlink directory was accepted'
|
|
fi
|
|
unlink -- "$fixture/evidence-parent/platform/recovery-evidence"
|
|
mv -- "$fixture/evidence-parent/platform/recovery-evidence-real" "$fixture/evidence-parent/platform/recovery-evidence"
|
|
pass 'recovery checker rejects a symlink evidence directory'
|
|
|
|
ln -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" \
|
|
"$fixture/evidence-parent/platform/recovery-evidence/grafana-local-second-name.env"
|
|
if run_script "$fixture" --check-grafana-recovery-evidence >"$fixture/hardlink-output" 2>&1; then
|
|
fail 'multiply-linked recovery evidence was accepted'
|
|
fi
|
|
unlink -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local-second-name.env"
|
|
pass 'recovery evidence must have exactly one hard link'
|
|
|
|
printf 'extra=forbidden\n' >>"$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
if run_script "$fixture" --check-slack-recovery-evidence >"$fixture/extra-output" 2>&1; then
|
|
fail 'recovery evidence with an extra key was accepted'
|
|
fi
|
|
sed -i '$d' "$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
pass 'recovery checker rejects an extra marker field'
|
|
|
|
sed -i "s/checked_at_utc=.*/checked_at_utc=2026-07-01T00:00:00Z/" "$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
if run_script "$fixture" --check-slack-recovery-evidence >"$fixture/stale-output" 2>&1; then
|
|
fail 'stale Slack evidence accepted'
|
|
fi
|
|
pass 'recovery evidence older than 30 days is rejected'
|
|
|
|
fixture="$(new_fixture reuse)"
|
|
seed_secret "$fixture" grafana-admin Opaque 'admin-user,admin-password'
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 || fail 'exact existing Grafana Secret not reusable'
|
|
! grep -q '^kubectl .*create secret' "$fixture/commands.log" || fail 'exact existing Secret was rotated'
|
|
grep -Fq 'OBSERVABILITY_GRAFANA_ADMIN=REUSED_UNCHANGED' "$fixture/output" || fail 'reuse marker absent'
|
|
pass 'exact existing contract is reused without rotation'
|
|
|
|
fixture="$(new_fixture input-race)"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE="$fixture/admin-password" \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1 || fail 'immutable input transaction failed'
|
|
python3 -I -S - "$fixture/state/secret__observability__grafana-admin.json" "$ADMIN_PASSWORD" <<'PY' ||
|
|
import base64, json, pathlib, sys
|
|
item = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
if base64.b64decode(item["data"]["admin-password"]).decode() != sys.argv[2]:
|
|
raise SystemExit(1)
|
|
PY
|
|
fail 'last-gate input replacement changed the created Secret payload'
|
|
pass 'create consumes the private pre-confirmation snapshot, not a raced input path'
|
|
|
|
fixture="$(new_fixture rollback)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \
|
|
PLATFORM_TEST_CREATE_FAIL_AT=2 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
fail 'second create failure accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'owned Grafana Secret was not UID-precondition rolled back'
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'failed Slack create unexpectedly exists'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || fail 'rollback success marker absent'
|
|
pass 'failed combined create rolls back only captured transaction UID'
|
|
|
|
fixture="$(new_fixture marker-rollback)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \
|
|
PLATFORM_TEST_MARKER_MOVE_FAIL_AT=2 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
fail 'second marker install failure accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'marker failure left Grafana Secret'
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'marker failure left Slack Secret'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] || fail 'marker failure left first transaction marker'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || fail 'marker failure left failed transaction marker'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || fail 'marker failure rollback marker absent'
|
|
pass 'combined transaction rolls Secrets and prior marker writes back together'
|
|
|
|
fixture="$(new_fixture marker-post-rename)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=1 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'post-rename marker failure accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail 'post-rename marker failure left the transaction Secret'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] ||
|
|
fail 'post-rename marker failure left an untracked committed marker'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail 'post-rename marker failure did not complete truthful rollback'
|
|
pass 'post-rename failure is classified and rolls the committed marker back'
|
|
|
|
fixture="$(new_fixture marker-post-rename-prior)"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
printf '%s\n' \
|
|
'schema=platform-observability-recovery-evidence-v1' \
|
|
'context=default' \
|
|
'resource=observability/grafana-admin' \
|
|
'checked_at_utc=2026-08-01T00:00:00Z' \
|
|
>"$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env"
|
|
chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env"
|
|
cp -- "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" "$fixture/prior-marker-copy"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=1 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'post-rename failure over a prior marker was accepted'
|
|
fi
|
|
cmp --silent -- "$fixture/prior-marker-copy" \
|
|
"$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ||
|
|
fail 'post-rename classification did not restore the exact prior marker'
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail 'prior-marker rollback left the transaction Secret'
|
|
pass 'post-rename rollback restores an exact prior marker, not just absence'
|
|
|
|
fixture="$(new_fixture marker-hup)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL=HUP \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'HUP during the post-rename window was accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail 'HUP left the transaction Secret untracked'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] ||
|
|
fail 'HUP left the committed marker untracked'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail 'HUP did not run the transaction rollback path'
|
|
pass 'HUP closes the post-rename marker ownership window'
|
|
|
|
for marker_signal in INT TERM; do
|
|
fixture="$(new_fixture "marker-${marker_signal,,}")"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL="$marker_signal" \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail "$marker_signal during the post-rename window was accepted"
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail "$marker_signal left the transaction Secret untracked"
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] ||
|
|
fail "$marker_signal left the committed marker untracked"
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail "$marker_signal did not run the transaction rollback path"
|
|
done
|
|
pass 'INT and TERM close the post-rename marker ownership window'
|
|
|
|
fixture="$(new_fixture marker-exit-fallback)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL=USR1 \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'unexpected process exit during the post-rename window was accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail 'EXIT fallback left the transaction Secret untracked'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] ||
|
|
fail 'EXIT fallback left the committed marker untracked'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail 'EXIT fallback did not run the transaction rollback path'
|
|
pass 'EXIT fallback classifies and rolls back an unhandled fatal signal'
|
|
|
|
fixture="$(new_fixture postcreate-drift)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_POSTCREATE_EXTRA_KEY=1 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'post-create schema drift accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'post-create validation failure bypassed rollback'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" || fail 'post-create rollback marker absent'
|
|
pass 'post-create contract failure enters the UID-precondition rollback path'
|
|
|
|
fixture="$(new_fixture ambiguous)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \
|
|
PLATFORM_TEST_CREATE_AMBIGUOUS_AT=2 run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
fail 'ambiguous create outcome accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'known-owned earlier Secret was not rolled back'
|
|
[[ -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] || fail 'ambiguous resource was destructively deleted'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || fail 'ambiguous create manual marker absent'
|
|
pass 'ambiguous create preserves unknown ownership and requests manual recovery'
|
|
|
|
for create_signal in HUP INT TERM; do
|
|
fixture="$(new_fixture "create-signal-${create_signal,,}")"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_CREATE_SIGNAL_AT=1 PLATFORM_TEST_CREATE_SIGNAL="$create_signal" \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail "$create_signal before the JSON create response was accepted"
|
|
fi
|
|
[[ -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail "$create_signal destructively removed an ambiguously-owned Secret"
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/grafana-local.env" ]] ||
|
|
fail "$create_signal unexpectedly committed recovery evidence"
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" ||
|
|
fail "$create_signal ambiguous create omitted the manual recovery marker"
|
|
done
|
|
pass 'HUP, INT, and TERM never adopt a Secret without its exact JSON create response'
|
|
|
|
fixture="$(new_fixture create-signal-after-response)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default' \
|
|
PLATFORM_TEST_CREATE_SIGNAL_AFTER_RESPONSE_AT=1 PLATFORM_TEST_CREATE_SIGNAL=HUP \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" >"$fixture/output" 2>&1; then
|
|
fail 'HUP after the complete JSON create response was accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__grafana-admin.json" ]] ||
|
|
fail 'complete create-response ownership was not UID-precondition rolled back'
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail 'complete create-response signal did not complete rollback'
|
|
! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" ||
|
|
fail 'complete create-response ownership was incorrectly classified ambiguous'
|
|
pass 'a complete JSON create response closes the signal ownership window'
|
|
|
|
fixture="$(new_fixture uid-drift)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY GRAFANA-LOCAL default\nRECOVERY SLACK default' \
|
|
PLATFORM_TEST_CREATE_FAIL_AT=2 PLATFORM_TEST_DRIFT_UID_ON_CREATE_FAILURE=1 \
|
|
run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
fail 'UID drift rollback accepted'
|
|
fi
|
|
[[ -e "$fixture/state/secret__observability__grafana-admin.json" ]] || fail 'replacement UID was deleted'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || fail 'UID precondition conflict manual marker absent'
|
|
pass 'UID precondition prevents deletion of a concurrent replacement'
|
|
|
|
fixture="$(new_fixture slack-risk-fully-absent-hierarchy)"
|
|
rmdir -- "$fixture/evidence-parent"
|
|
if PLATFORM_TEST_HIERARCHY_TRACE="$fixture/hierarchy.trace" \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
hierarchy_rc=0
|
|
else
|
|
hierarchy_rc=$?
|
|
fi
|
|
if (( hierarchy_rc != 0 )); then
|
|
[[ ! -e "$fixture/evidence-parent" ]] ||
|
|
fail 'failed fully absent hierarchy transaction created evidence hierarchy'
|
|
! grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" ||
|
|
fail 'failed fully absent hierarchy transaction emitted a success marker'
|
|
fail 'fully absent managed hierarchy blocked the Slack risk transaction'
|
|
fi
|
|
[[ -d "$fixture/evidence-parent/platform/recovery-evidence" ]] ||
|
|
fail 'fully absent managed hierarchy was not created'
|
|
[[ -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail 'fully absent managed hierarchy omitted the Slack risk marker'
|
|
expected_managed_metadata="$(printf '%s\n%s\n%s' \
|
|
"$EUID:$(id -g):700" "$EUID:$(id -g):700" "$EUID:$(id -g):700")"
|
|
[[ "$(stat -c '%u:%g:%a' -- \
|
|
"$fixture/evidence-parent" \
|
|
"$fixture/evidence-parent/platform" \
|
|
"$fixture/evidence-parent/platform/recovery-evidence")" == \
|
|
"$expected_managed_metadata" ]] ||
|
|
fail 'fully absent managed hierarchy metadata is not exact'
|
|
[[ "$(<"$fixture/hierarchy.trace")" == $'mkdir:evidence-parent\nreopen:evidence-parent\nattest:evidence-parent\nfsync-parent:evidence-parent\nmkdir:platform\nreopen:platform\nattest:platform\nfsync-parent:platform\nmkdir:recovery-evidence\nreopen:recovery-evidence\nattest:recovery-evidence\nfsync-parent:recovery-evidence' ]] ||
|
|
fail 'fully absent hierarchy mkdir, reopen, attestation, and parent-fsync ordering changed'
|
|
grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" ||
|
|
fail 'fully absent managed hierarchy transaction success marker absent'
|
|
pass 'Slack risk transaction safely creates a fully absent managed hierarchy'
|
|
|
|
fixture="$(new_fixture slack-risk-fully-present-hierarchy)"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
present_parent_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent")"
|
|
present_platform_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")"
|
|
present_leaf_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform/recovery-evidence")"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 ||
|
|
fail 'fully present exact hierarchy blocked the Slack risk transaction'
|
|
[[ "$(stat -c '%d:%i' -- "$fixture/evidence-parent")" == "$present_parent_identity" &&
|
|
"$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")" == "$present_platform_identity" &&
|
|
"$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform/recovery-evidence")" == "$present_leaf_identity" ]] ||
|
|
fail 'fully present exact hierarchy was replaced'
|
|
[[ -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail 'fully present exact hierarchy omitted the Slack risk marker'
|
|
grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" ||
|
|
fail 'fully present exact hierarchy transaction success marker absent'
|
|
pass 'Slack risk transaction preserves a fully present exact hierarchy'
|
|
|
|
for hierarchy_action in check-dir ensure-dir; do
|
|
fixture="$(new_fixture "hierarchy-helper-arbitrary-anchor-${hierarchy_action}")"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory/evidence-parent"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory/evidence-parent/platform"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory/evidence-parent/platform/recovery-evidence"
|
|
if run_internal_root_helper "$fixture" "$hierarchy_action" \
|
|
"$fixture/arbitrary-directory" fixture-hierarchy >/dev/null 2>&1; then
|
|
fail "$hierarchy_action accepted an arbitrary hierarchy anchor"
|
|
fi
|
|
[[ -d "$fixture/arbitrary-directory/evidence-parent/platform/recovery-evidence" ]] ||
|
|
fail "$hierarchy_action mutated the arbitrary hierarchy anchor"
|
|
|
|
fixture="$(new_fixture "hierarchy-helper-arbitrary-profile-${hierarchy_action}")"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
if run_internal_root_helper "$fixture" "$hierarchy_action" \
|
|
@configured-anchor arbitrary-hierarchy >/dev/null 2>&1; then
|
|
fail "$hierarchy_action accepted an arbitrary hierarchy profile"
|
|
fi
|
|
done
|
|
|
|
fixture="$(new_fixture hierarchy-helper-arbitrary-read)"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory"
|
|
printf '%s\n' 'non-secret-marker-fixture' >"$fixture/arbitrary-directory/slack.env"
|
|
chmod 0600 -- "$fixture/arbitrary-directory/slack.env"
|
|
if run_internal_root_helper "$fixture" read \
|
|
"$fixture/arbitrary-directory" slack.env >/dev/null 2>&1; then
|
|
fail 'read accepted an arbitrary marker directory'
|
|
fi
|
|
|
|
fixture="$(new_fixture hierarchy-helper-arbitrary-install)"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory"
|
|
printf '%s\n' 'non-secret-marker-fixture' >"$fixture/marker-source"
|
|
chmod 0600 -- "$fixture/marker-source"
|
|
if run_internal_root_helper "$fixture" install \
|
|
"$fixture/arbitrary-directory" slack.env "$fixture/marker-source" absent '' >/dev/null 2>&1; then
|
|
fail 'install accepted an arbitrary marker directory'
|
|
fi
|
|
[[ ! -e "$fixture/arbitrary-directory/slack.env" ]] ||
|
|
fail 'rejected arbitrary install mutated its marker directory'
|
|
|
|
fixture="$(new_fixture hierarchy-helper-arbitrary-unlink)"
|
|
mkdir -m 0700 -- "$fixture/arbitrary-directory"
|
|
printf '%s\n' 'non-secret-marker-fixture' >"$fixture/marker-source"
|
|
cp -- "$fixture/marker-source" "$fixture/arbitrary-directory/slack.env"
|
|
chmod 0600 -- "$fixture/marker-source" "$fixture/arbitrary-directory/slack.env"
|
|
if run_internal_root_helper "$fixture" unlink-exact \
|
|
"$fixture/arbitrary-directory" slack.env "$fixture/marker-source" file \
|
|
"$fixture/marker-source" >/dev/null 2>&1; then
|
|
fail 'unlink-exact accepted an arbitrary marker directory'
|
|
fi
|
|
cmp --silent -- "$fixture/marker-source" "$fixture/arbitrary-directory/slack.env" ||
|
|
fail 'rejected arbitrary unlink changed its exact marker'
|
|
pass 'every helper action rejects arbitrary hierarchy profiles and directories'
|
|
|
|
for managed_component in evidence-parent platform recovery-evidence; do
|
|
fixture="$(new_fixture "hierarchy-symlink-${managed_component}")"
|
|
prepare_managed_component_parent "$fixture" "$managed_component"
|
|
managed_path="$(managed_component_path "$fixture" "$managed_component")"
|
|
if [[ "$managed_component" == evidence-parent ]]; then
|
|
rmdir -- "$managed_path"
|
|
fi
|
|
mkdir -m 0700 -- "${managed_path}-target"
|
|
ln -s -- "$(basename -- "${managed_path}-target")" "$managed_path"
|
|
assert_slack_risk_hierarchy_failure "$fixture" "symlink at managed component $managed_component"
|
|
|
|
fixture="$(new_fixture "hierarchy-nondirectory-${managed_component}")"
|
|
prepare_managed_component_parent "$fixture" "$managed_component"
|
|
managed_path="$(managed_component_path "$fixture" "$managed_component")"
|
|
if [[ "$managed_component" == evidence-parent ]]; then
|
|
rmdir -- "$managed_path"
|
|
fi
|
|
: >"$managed_path"
|
|
chmod 0700 -- "$managed_path"
|
|
assert_slack_risk_hierarchy_failure "$fixture" "non-directory at managed component $managed_component"
|
|
|
|
fixture="$(new_fixture "hierarchy-wrong-mode-${managed_component}")"
|
|
prepare_managed_component_parent "$fixture" "$managed_component"
|
|
managed_path="$(managed_component_path "$fixture" "$managed_component")"
|
|
[[ -e "$managed_path" ]] || mkdir -m 0700 -- "$managed_path"
|
|
chmod 0755 -- "$managed_path"
|
|
assert_slack_risk_hierarchy_failure "$fixture" "wrong mode at managed component $managed_component"
|
|
|
|
for metadata_fault in wrong-uid wrong-gid; do
|
|
fixture="$(new_fixture "hierarchy-${metadata_fault}-${managed_component}")"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
assert_slack_risk_hierarchy_failure "$fixture" \
|
|
"$metadata_fault at managed component $managed_component" \
|
|
"$metadata_fault:$managed_component"
|
|
done
|
|
done
|
|
|
|
for managed_component in evidence-parent platform recovery-evidence; do
|
|
fixture="$(new_fixture "hierarchy-replace-after-open-${managed_component}")"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
if PLATFORM_TEST_HIERARCHY_FAULT="replace-after-open:$managed_component" \
|
|
run_hierarchy_action "$fixture" check-dir >/dev/null 2>&1; then
|
|
fail "check-dir accepted replacement after open at managed component $managed_component"
|
|
fi
|
|
[[ "$(<"$fixture/hierarchy-fault.log")" == "replace-after-open:$managed_component" ]] ||
|
|
fail "check-dir did not reach replacement after open at managed component $managed_component"
|
|
pass "check-dir binds the opened $managed_component descriptor to its current parent entry"
|
|
|
|
for replacement_phase in replace-after-create replace-after-reopen replace-before-attestation; do
|
|
fixture="$(new_fixture "hierarchy-${replacement_phase}-${managed_component}")"
|
|
prepare_managed_component_parent "$fixture" "$managed_component"
|
|
if [[ "$managed_component" == evidence-parent ]]; then
|
|
rmdir -- "$fixture/evidence-parent"
|
|
fi
|
|
if PLATFORM_TEST_HIERARCHY_FAULT="$replacement_phase:$managed_component" \
|
|
run_hierarchy_action "$fixture" ensure-dir >/dev/null 2>&1; then
|
|
fail "ensure-dir accepted $replacement_phase at managed component $managed_component"
|
|
fi
|
|
[[ "$(<"$fixture/hierarchy-fault.log")" == "$replacement_phase:$managed_component" ]] ||
|
|
fail "ensure-dir did not reach $replacement_phase at managed component $managed_component"
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail "ensure-dir $replacement_phase published a marker"
|
|
pass "ensure-dir rejects $replacement_phase at managed component $managed_component"
|
|
done
|
|
done
|
|
|
|
for prefix_component in evidence-parent platform; do
|
|
fixture="$(new_fixture "hierarchy-safe-absence-race-${prefix_component}")"
|
|
if [[ "$prefix_component" == platform ]]; then
|
|
mkdir -m 0700 -- "$fixture/evidence-parent/platform"
|
|
fi
|
|
if PLATFORM_TEST_HIERARCHY_FAULT="replace-before-safe-absence:$prefix_component" \
|
|
run_hierarchy_action "$fixture" check-dir >/dev/null 2>&1; then
|
|
safe_absence_rc=0
|
|
else
|
|
safe_absence_rc=$?
|
|
fi
|
|
(( safe_absence_rc != 0 )) ||
|
|
fail "check-dir accepted prefix replacement before safe absence at $prefix_component"
|
|
(( safe_absence_rc != 4 )) ||
|
|
fail "check-dir misclassified replaced prefix $prefix_component as safe absence"
|
|
[[ "$(<"$fixture/hierarchy-fault.log")" == "replace-before-safe-absence:$prefix_component" ]] ||
|
|
fail "check-dir did not reach the safe-absence race at $prefix_component"
|
|
pass "check-dir revalidates $prefix_component before reporting a missing child"
|
|
done
|
|
|
|
for managed_component in evidence-parent platform recovery-evidence; do
|
|
fixture="$(new_fixture "hierarchy-concurrent-exact-${managed_component}")"
|
|
prepare_managed_component_parent "$fixture" "$managed_component"
|
|
if [[ "$managed_component" == evidence-parent ]]; then
|
|
rmdir -- "$fixture/evidence-parent"
|
|
fi
|
|
PLATFORM_TEST_HIERARCHY_FAULT="concurrent-create-exact:$managed_component" \
|
|
PLATFORM_TEST_HIERARCHY_TRACE="$fixture/hierarchy.trace" \
|
|
run_hierarchy_action "$fixture" ensure-dir >/dev/null 2>&1 ||
|
|
fail "ensure-dir rejected exact concurrent creation at $managed_component"
|
|
[[ "$(<"$fixture/hierarchy-fault.log")" == "concurrent-create-exact:$managed_component" ]] ||
|
|
fail "ensure-dir did not reach exact concurrent creation at $managed_component"
|
|
grep -Fqx "fsync-parent:$managed_component" "$fixture/hierarchy.trace" ||
|
|
fail "ensure-dir omitted parent fsync after exact concurrent creation at $managed_component"
|
|
[[ "$(stat -c '%u:%g:%a' -- "$(managed_component_path "$fixture" "$managed_component")")" == \
|
|
"$EUID:$(id -g):700" ]] ||
|
|
fail "ensure-dir accepted wrong exact-concurrent metadata at $managed_component"
|
|
pass "ensure-dir accepts and parent-fsyncs exact concurrent creation at $managed_component"
|
|
|
|
fixture="$(new_fixture "hierarchy-concurrent-unsafe-${managed_component}")"
|
|
prepare_managed_component_parent "$fixture" "$managed_component"
|
|
if [[ "$managed_component" == evidence-parent ]]; then
|
|
rmdir -- "$fixture/evidence-parent"
|
|
fi
|
|
if PLATFORM_TEST_HIERARCHY_FAULT="concurrent-create-unsafe:$managed_component" \
|
|
run_hierarchy_action "$fixture" ensure-dir >/dev/null 2>&1; then
|
|
fail "ensure-dir accepted unsafe concurrent creation at $managed_component"
|
|
fi
|
|
[[ "$(<"$fixture/hierarchy-fault.log")" == "concurrent-create-unsafe:$managed_component" ]] ||
|
|
fail "ensure-dir did not reach unsafe concurrent creation at $managed_component"
|
|
pass "ensure-dir rejects unsafe concurrent creation at $managed_component"
|
|
done
|
|
|
|
fixture="$(new_fixture slack-risk-create-failure-fully-absent-hierarchy)"
|
|
rmdir -- "$fixture/evidence-parent"
|
|
if PLATFORM_TEST_CREATE_FAIL_AT=1 \
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail 'Slack create failure was accepted with a fully absent hierarchy'
|
|
fi
|
|
[[ ! -e "$fixture/evidence-parent" ]] ||
|
|
fail 'Slack create failure mutated the fully absent hierarchy'
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] ||
|
|
fail 'Slack create failure left a Secret with a fully absent hierarchy'
|
|
[[ -f "$fixture/create-count" && "$(<"$fixture/create-count")" == 1 ]] ||
|
|
fail 'Slack create failure did not reach the pre-ensure Secret create boundary'
|
|
! grep -Fq 'OBSERVABILITY_SECRETS_TRANSACTION=PASS' "$fixture/output" ||
|
|
fail 'Slack create failure emitted a transaction success marker'
|
|
pass 'Secret create failure leaves the fully absent hierarchy absent'
|
|
|
|
fixture="$(new_fixture slack-risk-final-component-absent-hierarchy)"
|
|
mkdir -m 0700 -- "$fixture/evidence-parent/platform"
|
|
partial_parent_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent")"
|
|
partial_platform_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 ||
|
|
fail 'final-component-absent hierarchy blocked the Slack risk transaction'
|
|
[[ "$(stat -c '%d:%i' -- "$fixture/evidence-parent")" == "$partial_parent_identity" &&
|
|
"$(stat -c '%d:%i' -- "$fixture/evidence-parent/platform")" == "$partial_platform_identity" &&
|
|
-f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail 'final-component-absent hierarchy changed an existing prefix or omitted its marker'
|
|
pass 'Slack risk transaction creates only the missing final managed component'
|
|
|
|
fixture="$(new_fixture slack-risk-partially-absent-hierarchy)"
|
|
partial_parent_identity="$(stat -c '%d:%i' -- "$fixture/evidence-parent")"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 ||
|
|
fail 'partially absent hierarchy blocked the Slack risk transaction'
|
|
[[ "$(stat -c '%d:%i' -- "$fixture/evidence-parent")" == "$partial_parent_identity" ]] ||
|
|
fail 'partially absent hierarchy replaced its existing parent'
|
|
expected_managed_metadata="$(printf '%s\n%s\n%s' \
|
|
"$EUID:$(id -g):700" "$EUID:$(id -g):700" "$EUID:$(id -g):700")"
|
|
[[ "$(stat -c '%u:%g:%a' -- \
|
|
"$fixture/evidence-parent" \
|
|
"$fixture/evidence-parent/platform" \
|
|
"$fixture/evidence-parent/platform/recovery-evidence")" == \
|
|
"$expected_managed_metadata" ]] ||
|
|
fail 'partially absent hierarchy metadata changed'
|
|
[[ -f "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] || fail 'risk marker absent'
|
|
[[ "$(wc -l <"$fixture/evidence-parent/platform/recovery-evidence/slack.env" | tr -d '[:space:]')" == 8 ]] ||
|
|
fail 'risk marker schema line count changed'
|
|
grep -Fqx 'schema=platform-observability-slack-risk-acceptance-v1' \
|
|
"$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'risk marker schema absent'
|
|
pass 'Slack risk acceptance creates an exact marker in a partially absent hierarchy'
|
|
|
|
if run_script "$fixture" --check-slack-recovery-evidence >"$fixture/strict-risk-output" 2>&1; then
|
|
fail 'strict recovery checker accepted risk evidence'
|
|
fi
|
|
run_script "$fixture" --check-slack-deployment-evidence >"$fixture/deployment-risk-output" 2>&1 ||
|
|
fail 'deployment checker rejected risk evidence'
|
|
[[ "$(<"$fixture/deployment-risk-output")" == 'SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED' ]] ||
|
|
fail 'deployment checker risk verdict changed'
|
|
pass 'strict checker rejects and deployment checker accepts exact risk evidence'
|
|
|
|
: >"$fixture/marker-move-count"
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/risk-noop-output" 2>&1 ||
|
|
fail 'exact risk marker did not permit a verified Slack no-op'
|
|
[[ ! -s "$fixture/marker-move-count" ]] || fail 'exact risk marker was rewritten'
|
|
pass 'exact risk marker is a verified no-rewrite Slack no-op'
|
|
|
|
fixture="$(new_fixture slack-risk-existing)"
|
|
seed_secret "$fixture" alertmanager-slack-webhook Opaque url
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1 ||
|
|
fail 'existing exact Slack Secret did not accept risk marker installation'
|
|
[[ ! -e "$fixture/create-count" ]] || fail 'existing exact Slack Secret was created during risk marker installation'
|
|
[[ "$(cat "$fixture/marker-move-count")" == 1 ]] || fail 'absent risk marker was not installed exactly once'
|
|
pass 'existing exact Slack Secret installs only an absent risk marker'
|
|
|
|
fixture="$(new_fixture slack-risk-marker-failure)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
PLATFORM_TEST_MARKER_MOVE_FAIL_AT=1 run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail 'risk marker installation failure was accepted'
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] ||
|
|
fail 'risk marker failure left a newly-created Slack Secret'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail 'risk marker failure left a marker'
|
|
pass 'risk marker write failure rolls back only the newly-created Slack Secret'
|
|
|
|
fixture="$(new_fixture slack-risk-first-write-red)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE=first-write \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
printf 'TEST FAILURE: helper first-write-loop failure boundary is unavailable\n' >&2
|
|
exit 1
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" &&
|
|
! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" &&
|
|
-z "$(find "$fixture/evidence-parent" -name '.slack.env.new.*' -print -quit)" ]] ||
|
|
fail 'first-write-loop failure changed Slack Secret or marker state'
|
|
[[ -f "$fixture/marker-move-count" && "$(cat "$fixture/marker-move-count")" == 1 ]] || {
|
|
printf 'TEST FAILURE: helper first-write-loop failure did not reach marker install\n' >&2
|
|
exit 1
|
|
}
|
|
! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" ||
|
|
fail 'first-write-loop failure incorrectly requested manual recovery'
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'first-write-loop failure leaked payload to output'
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'first-write-loop failure leaked payload to command log'
|
|
pass 'first-write-loop helper failure preserves absent marker state and rolls back owned Slack Secret'
|
|
|
|
fixture="$(new_fixture slack-strict-pre-rename-red)"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
printf '%s\n' \
|
|
'schema=platform-observability-slack-risk-acceptance-v1' \
|
|
'context=default' \
|
|
'resource=observability/alertmanager-slack-webhook' \
|
|
'reason=off-host-slack-recovery-deferred' \
|
|
'accepted_by_uid=1000' \
|
|
'approval_method=interactive-exact-confirmation' \
|
|
'approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' \
|
|
"checked_at_utc=$NOW_UTC" >"$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
cp -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env" "$fixture/prior-slack-marker"
|
|
seed_secret "$fixture" alertmanager-slack-webhook Opaque url
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY SLACK default' \
|
|
PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE=pre-rename \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1; then
|
|
printf 'TEST FAILURE: helper pre-rename failure boundary is unavailable\n' >&2
|
|
exit 1
|
|
fi
|
|
[[ -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ]] ||
|
|
fail 'pre-rename failure removed an existing Slack Secret'
|
|
cmp --silent -- "$fixture/prior-slack-marker" "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ||
|
|
fail 'pre-rename failure changed the prior exact marker'
|
|
[[ -z "$(find "$fixture/evidence-parent" -name '.slack.env.new.*' -print -quit)" ]] ||
|
|
fail 'pre-rename failure left helper temporary residue'
|
|
! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" ||
|
|
fail 'pre-rename failure incorrectly requested manual recovery'
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'pre-rename failure leaked payload to output'
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'pre-rename failure leaked payload to command log'
|
|
pass 'post-fsync pre-rename helper failure preserves prior marker and existing Slack Secret'
|
|
|
|
fixture="$(new_fixture slack-risk-post-rename-loss-red)"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
PLATFORM_TEST_MARKER_POST_RENAME_FAIL_AT=1 run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail 'risk post-rename response loss was accepted'
|
|
fi
|
|
if [[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" ||
|
|
! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output"; then
|
|
fail 'risk post-rename response loss did not preserve the marker and Secret'
|
|
fi
|
|
[[ "$(cat "$fixture/create-count")" == 1 && "$(cat "$fixture/marker-move-count")" == 1 ]] ||
|
|
fail 'risk post-rename response loss mutation count changed'
|
|
[[ ! -e "$fixture/evidence-parent/platform/recovery-evidence/.slack.env.new."* ]] ||
|
|
fail 'risk post-rename response loss left temporary marker residue'
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'risk post-rename response loss leaked payload to output'
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail 'risk post-rename response loss leaked payload to command log'
|
|
pass 'risk post-rename response loss preserves marker and Secret for manual recovery'
|
|
|
|
fixture="$(new_fixture slack-risk-transition)"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
printf '%s\n' \
|
|
'schema=platform-observability-slack-risk-acceptance-v1' \
|
|
'context=default' \
|
|
'resource=observability/alertmanager-slack-webhook' \
|
|
'reason=off-host-slack-recovery-deferred' \
|
|
'accepted_by_uid=1000' \
|
|
'approval_method=interactive-exact-confirmation' \
|
|
'approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design' \
|
|
"checked_at_utc=$NOW_UTC" >"$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
seed_secret "$fixture" alertmanager-slack-webhook Opaque url
|
|
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY SLACK default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" >"$fixture/output" 2>&1 ||
|
|
fail 'strict Slack recovery did not promote risk evidence'
|
|
grep -Fqx 'schema=platform-observability-recovery-evidence-v1' \
|
|
"$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'risk marker was not promoted to recovery'
|
|
pass 'strict Slack recovery atomically promotes risk evidence'
|
|
|
|
fixture="$(new_fixture slack-risk-noop-and-downgrade)"
|
|
create_exact_evidence_hierarchy "$fixture"
|
|
printf '%s\n' \
|
|
'schema=platform-observability-recovery-evidence-v1' \
|
|
'context=default' \
|
|
'resource=observability/alertmanager-slack-webhook' \
|
|
"checked_at_utc=$NOW_UTC" >"$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
chmod 0600 -- "$fixture/evidence-parent/platform/recovery-evidence/slack.env"
|
|
seed_secret "$fixture" alertmanager-slack-webhook Opaque url
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail 'risk path downgraded recovery evidence'
|
|
fi
|
|
grep -Fqx 'schema=platform-observability-recovery-evidence-v1' \
|
|
"$fixture/evidence-parent/platform/recovery-evidence/slack.env" || fail 'risk downgrade changed recovery marker'
|
|
pass 'risk path rejects recovery-marker downgrade before mutation'
|
|
|
|
for confirmation_case in wrong partial eof; do
|
|
fixture="$(new_fixture "slack-risk-confirmation-${confirmation_case}")"
|
|
rmdir -- "$fixture/evidence-parent"
|
|
case "$confirmation_case" in
|
|
wrong) confirmations=$'WRONG\nACCEPT NO OFF-HOST SLACK RECOVERY default' ;;
|
|
partial) confirmations=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY' ;;
|
|
eof) confirmations='APPLY default' ;;
|
|
esac
|
|
if PLATFORM_TEST_CONFIRMATIONS="$confirmations" run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail "risk $confirmation_case confirmation was accepted"
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" &&
|
|
! -e "$fixture/evidence-parent" && ! -e "$fixture/create-count" ]] ||
|
|
fail "risk $confirmation_case confirmation mutated Secret or fully absent hierarchy state"
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail "risk $confirmation_case confirmation leaked payload to output"
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail "risk $confirmation_case confirmation leaked payload to command log"
|
|
pass "risk $confirmation_case confirmation has zero mutation and no payload residue"
|
|
done
|
|
|
|
fixture="$(new_fixture slack-risk-combined-reject)"
|
|
if run_script "$fixture" --execute --grafana-admin \
|
|
--grafana-admin-user-file "$fixture/admin-user" \
|
|
--grafana-admin-password-file "$fixture/admin-password" \
|
|
--slack-webhook --slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail 'combined Grafana and Slack risk interface was accepted'
|
|
fi
|
|
[[ ! -s "$fixture/commands.log" && ! -e "$fixture/evidence-parent/platform/recovery-evidence" ]] ||
|
|
fail 'combined Grafana and Slack risk rejection crossed a mutation boundary'
|
|
pass 'combined Grafana and Slack risk interface is rejected before contact'
|
|
|
|
fixture="$(new_fixture slack-risk-missing-file)"
|
|
if run_script "$fixture" --execute --slack-webhook \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail 'risk interface without a Slack file was accepted'
|
|
fi
|
|
[[ ! -s "$fixture/commands.log" && ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" &&
|
|
! -e "$fixture/evidence-parent/platform/recovery-evidence" ]] ||
|
|
fail 'risk interface without a Slack file crossed a mutation boundary'
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail 'risk missing file leaked payload to output'
|
|
pass 'risk interface without a Slack file is rejected before contact'
|
|
|
|
for risk_signal in HUP INT TERM; do
|
|
fixture="$(new_fixture "slack-risk-create-signal-${risk_signal,,}")"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
PLATFORM_TEST_CREATE_SIGNAL_AT=1 PLATFORM_TEST_CREATE_SIGNAL="$risk_signal" \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail "risk $risk_signal during transaction activation was accepted"
|
|
fi
|
|
[[ -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" &&
|
|
! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail "risk $risk_signal transaction-activation state was not preserved"
|
|
[[ "$(cat "$fixture/create-count")" == 1 && ! -e "$fixture/marker-move-count" ]] ||
|
|
fail "risk $risk_signal transaction-activation mutation count changed"
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" ||
|
|
fail "risk $risk_signal transaction-activation omitted manual recovery"
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail "risk $risk_signal signal leaked payload to output"
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail "risk $risk_signal signal leaked payload to command log"
|
|
pass "risk $risk_signal during transaction activation preserves ambiguous Secret only"
|
|
done
|
|
|
|
for risk_signal in HUP INT TERM; do
|
|
fixture="$(new_fixture "slack-risk-marker-signal-${risk_signal,,}")"
|
|
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nACCEPT NO OFF-HOST SLACK RECOVERY default' \
|
|
PLATFORM_TEST_MARKER_SIGNAL_AT=1 PLATFORM_TEST_MARKER_SIGNAL="$risk_signal" \
|
|
run_script "$fixture" --execute --slack-webhook \
|
|
--slack-webhook-file "$fixture/slack-url" \
|
|
--accept-no-off-host-slack-recovery >"$fixture/output" 2>&1; then
|
|
fail "risk $risk_signal after marker activation was accepted"
|
|
fi
|
|
[[ ! -e "$fixture/state/secret__observability__alertmanager-slack-webhook.json" &&
|
|
! -e "$fixture/evidence-parent/platform/recovery-evidence/slack.env" ]] ||
|
|
fail "risk $risk_signal post-marker signal left a committed resource"
|
|
[[ "$(cat "$fixture/create-count")" == 1 && "$(cat "$fixture/marker-move-count")" == 1 ]] ||
|
|
fail "risk $risk_signal post-marker signal mutation count changed"
|
|
grep -Fq 'OBSERVABILITY_SECRET_ROLLBACK=PASS' "$fixture/output" ||
|
|
fail "risk $risk_signal post-marker signal omitted rollback"
|
|
! grep -Fq "$SLACK_URL" "$fixture/output" || fail "risk $risk_signal signal leaked payload to output"
|
|
! grep -Fq "$SLACK_URL" "$fixture/commands.log" || fail "risk $risk_signal signal leaked payload to command log"
|
|
pass "risk $risk_signal after marker activation rolls back known transaction state"
|
|
done
|
|
|
|
printf 'OBSERVABILITY SECRETS TEST PASS (%s assertions)\n' "$ASSERTIONS"
|