#!/usr/bin/env bash set -Eeuo pipefail # Secret payloads must never be exposed by a caller's xtrace setting. set +x readonly SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly EXPECTED_CONTEXT='default' readonly NAMESPACE='observability' readonly EVIDENCE_SCHEMA='platform-observability-recovery-evidence-v1' readonly RISK_EVIDENCE_SCHEMA='platform-observability-slack-risk-acceptance-v1' readonly RISK_APPROVAL_REF='2026-08-14-observability-slack-recovery-risk-acceptance-design' readonly RISK_ACCEPTED_BY_UID='1000' readonly EVIDENCE_MAX_AGE_SECONDS=2592000 readonly PRODUCTION_EVIDENCE_DIR='/etc/hyeonworks/platform/recovery-evidence' readonly PRODUCTION_EVIDENCE_ANCHOR='/etc' readonly PRODUCTION_ENCRYPTION_SCRIPT="$SCRIPT_ROOT/scripts/validate/k3s-secret-encryption.sh" readonly PRODUCTION_RESTORE_SCRIPT="$SCRIPT_ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh" readonly VALIDATOR_PATH='/usr/sbin:/usr/bin:/sbin:/bin' readonly VALIDATOR_HOME='/home/donghyeon' readonly KUBECTL_REQUEST_TIMEOUT='5s' KUBECTL_PROCESS_TIMEOUT='10s' KUBECTL_PROXY_PROCESS_TIMEOUT='15s' TEST_MODE=false KUBECTL_BIN='kubectl' SUDO_BIN='sudo' ENCRYPTION_SCRIPT="$PRODUCTION_ENCRYPTION_SCRIPT" RESTORE_SCRIPT="$PRODUCTION_RESTORE_SCRIPT" EVIDENCE_DIR="$PRODUCTION_EVIDENCE_DIR" EVIDENCE_HIERARCHY_PROFILE='production-hierarchy' EVIDENCE_HIERARCHY_ANCHOR="$PRODUCTION_EVIDENCE_ANCHOR" TEST_CONFIRMATIONS='' TEST_NOW_UTC='' MARKER_TEST_FAILURE_PHASE='' execute_requested=false grafana_requested=false slack_requested=false check_grafana_requested=false check_slack_requested=false accept_slack_risk_requested=false check_slack_deployment_requested=false grafana_user_file='' grafana_password_file='' slack_webhook_file='' grafana_user_snapshot='' grafana_password_snapshot='' slack_webhook_snapshot='' temporary_dir='' transaction_active=false manual_recovery_required=false rollback_in_progress=false exit_handler_running=false declare -a OWNED_ASSETS=() declare -a OWNED_NAMES=() declare -a OWNED_UIDS=() pending_asset='' pending_name='' pending_token='' pending_manifest='' pending_response='' pending_marker_asset='' declare -a WRITTEN_MARKERS=() declare -A MARKER_HAD_PRIOR=() declare -A MARKER_PRIOR_FILE=() declare -A MARKER_DESIRED_FILE=() declare -A MARKER_KIND=() EVIDENCE_EXPECTED_UID=0 EVIDENCE_EXPECTED_GID=0 confirmation_index=0 declare -a confirmation_answers=() usage() { cat <<'USAGE' Usage: bash scripts/bootstrap/create-observability-secrets.sh bash scripts/bootstrap/create-observability-secrets.sh \ --execute --grafana-admin \ --grafana-admin-user-file /absolute/0600/file \ --grafana-admin-password-file /absolute/0600/file bash scripts/bootstrap/create-observability-secrets.sh \ --execute --slack-webhook \ --slack-webhook-file /absolute/0600/file bash scripts/bootstrap/create-observability-secrets.sh \ --check-grafana-recovery-evidence bash scripts/bootstrap/create-observability-secrets.sh \ --check-slack-recovery-evidence bash scripts/bootstrap/create-observability-secrets.sh \ --check-slack-deployment-evidence With no arguments, prints a source-only plan and performs no system access. Execute mode creates only absent selected Secrets, reuses exact existing contracts, and never rotates or prints payloads. Both asset flags form one create-only transaction. USAGE } usage_fail() { printf 'ERROR: %s\n' "$*" >&2 usage >&2 exit 2 } emit_manual_recovery() { [[ "$manual_recovery_required" == false ]] || return 0 manual_recovery_required=true printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2 } plain_fail() { if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then transaction_fail "$*" fi printf 'ERROR: %s\n' "$*" >&2 exit 1 } safe_temporary_path() { local path=$1 [[ -n "$temporary_dir" && "$temporary_dir" == /tmp/platform-observability-secrets.?????? ]] || return 1 [[ "$path" == "$temporary_dir" || "$path" == "$temporary_dir"/* ]] } cleanup() { trap - EXIT if [[ -n "$temporary_dir" ]] && safe_temporary_path "$temporary_dir"; then rm -rf -- "$temporary_dir" fi } on_process_exit() { local rc=$1 ambiguous=false classification [[ "$exit_handler_running" == false ]] || exit "$rc" exit_handler_running=true trap - EXIT ERR HUP INT TERM set +e if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then if [[ -n "$pending_marker_asset" ]]; then classify_pending_marker classification=$? [[ "$classification" != 2 ]] || ambiguous=true fi if [[ -n "$pending_asset" ]]; then classify_pending_create classification=$? [[ "$classification" == 0 || "$classification" == 1 ]] || ambiguous=true fi [[ "$ambiguous" != true ]] || emit_manual_recovery rollback_transaction || true transaction_active=false printf 'ERROR: transaction exited before a confirmed terminal state\n' >&2 (( rc != 0 )) || rc=1 fi cleanup exit "$rc" } trap 'on_process_exit "$?"' EXIT reject_production_overrides() { local variable local -a forbidden=( PLATFORM_OBSERVABILITY_SECRETS_TEST_MODE PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN PLATFORM_OBSERVABILITY_SECRETS_SUDO_BIN PLATFORM_OBSERVABILITY_SECRETS_ENCRYPTION_SCRIPT PLATFORM_OBSERVABILITY_SECRETS_RESTORE_SCRIPT PLATFORM_OBSERVABILITY_SECRETS_EVIDENCE_DIR PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE ) for variable in "${forbidden[@]}"; do [[ ! -v "$variable" ]] || plain_fail "production boundary override is forbidden: $variable" done } attest_test_executable() { local path=$1 metadata [[ -f "$path" && ! -L "$path" && -O "$path" && -x "$path" ]] || return 1 metadata="$(stat --format='%u:%a:%h:%F' -- "$path")" || return 1 [[ "$metadata" == "${EUID}:755:1:regular file" ]] } configure_test_boundaries() { local fixture_root=$1 fixture_metadata physical_root [[ "$fixture_root" =~ ^/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+$ ]] || plain_fail 'test fixture root has an invalid shape' [[ -d "$fixture_root" && ! -L "$fixture_root" && -O "$fixture_root" ]] || plain_fail 'test fixture root is not an owned non-symlink directory' physical_root="$(cd -- "$fixture_root" && pwd -P)" || plain_fail 'test fixture root cannot be resolved' [[ "$physical_root" == "$fixture_root" ]] || plain_fail 'test fixture root contains a symlink boundary' fixture_metadata="$(stat --format='%u:%a:%F' -- "$fixture_root")" || plain_fail 'test fixture root cannot be attested' [[ "$fixture_metadata" == "${EUID}:700:directory" ]] || plain_fail 'test fixture root metadata is invalid' [[ "$(cd -- "$fixture_root/bin" && pwd -P)" == "$fixture_root/bin" ]] || plain_fail 'test binary directory contains a symlink boundary' [[ "$(stat --format='%u:%a:%F' -- "$fixture_root/bin")" == "${EUID}:755:directory" ]] || plain_fail 'test binary directory metadata is invalid' KUBECTL_BIN="$fixture_root/bin/kubectl" SUDO_BIN="$fixture_root/bin/sudo" ENCRYPTION_SCRIPT="$fixture_root/bin/encryption" RESTORE_SCRIPT="$fixture_root/bin/restore" EVIDENCE_DIR="$fixture_root/evidence-parent/platform/recovery-evidence" EVIDENCE_HIERARCHY_PROFILE='fixture-hierarchy' EVIDENCE_HIERARCHY_ANCHOR="$fixture_root" TEST_CONFIRMATIONS="${PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS:-}" TEST_NOW_UTC="${PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC:-}" MARKER_TEST_FAILURE_PHASE="${PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE:-}" KUBECTL_PROCESS_TIMEOUT='2s' KUBECTL_PROXY_PROCESS_TIMEOUT='5s' EVIDENCE_EXPECTED_UID=$EUID EVIDENCE_EXPECTED_GID="$(/usr/bin/id -g)" [[ "$SUDO_BIN" != /usr/bin/sudo && "$SUDO_BIN" != /bin/sudo ]] || plain_fail 'real sudo is forbidden in the isolated test entrypoint' attest_test_executable "$KUBECTL_BIN" || plain_fail 'test kubectl boundary is not attested' attest_test_executable "$SUDO_BIN" || plain_fail 'test sudo boundary is not attested' attest_test_executable "$ENCRYPTION_SCRIPT" || plain_fail 'test encryption boundary is not attested' attest_test_executable "$RESTORE_SCRIPT" || plain_fail 'test restore boundary is not attested' [[ "$TEST_NOW_UTC" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || plain_fail 'test clock is invalid' [[ "$MARKER_TEST_FAILURE_PHASE" == '' || "$MARKER_TEST_FAILURE_PHASE" == first-write || "$MARKER_TEST_FAILURE_PHASE" == pre-rename ]] || plain_fail 'test marker failure phase is invalid' mapfile -t confirmation_answers <<<"$TEST_CONFIRMATIONS" } parse_arguments() { local seen_execute=false seen_grafana=false seen_slack=false local seen_user_file=false seen_password_file=false seen_slack_file=false local seen_check_grafana=false seen_check_slack=false seen_accept_slack_risk=false local seen_check_slack_deployment=false while (( $# > 0 )); do case "$1" in --execute) [[ "$seen_execute" == false ]] || usage_fail 'duplicate --execute' seen_execute=true; execute_requested=true; shift ;; --grafana-admin) [[ "$seen_grafana" == false ]] || usage_fail 'duplicate --grafana-admin' seen_grafana=true; grafana_requested=true; shift ;; --slack-webhook) [[ "$seen_slack" == false ]] || usage_fail 'duplicate --slack-webhook' seen_slack=true; slack_requested=true; shift ;; --grafana-admin-user-file) [[ "$seen_user_file" == false && $# -ge 2 ]] || usage_fail 'invalid --grafana-admin-user-file' seen_user_file=true; grafana_user_file=$2; shift 2 ;; --grafana-admin-password-file) [[ "$seen_password_file" == false && $# -ge 2 ]] || usage_fail 'invalid --grafana-admin-password-file' seen_password_file=true; grafana_password_file=$2; shift 2 ;; --slack-webhook-file) [[ "$seen_slack_file" == false && $# -ge 2 ]] || usage_fail 'invalid --slack-webhook-file' seen_slack_file=true; slack_webhook_file=$2; shift 2 ;; --check-grafana-recovery-evidence) [[ "$seen_check_grafana" == false ]] || usage_fail 'duplicate Grafana recovery check' seen_check_grafana=true; check_grafana_requested=true; shift ;; --check-slack-recovery-evidence) [[ "$seen_check_slack" == false ]] || usage_fail 'duplicate Slack recovery check' seen_check_slack=true; check_slack_requested=true; shift ;; --accept-no-off-host-slack-recovery) [[ "$seen_accept_slack_risk" == false ]] || usage_fail 'duplicate Slack risk acceptance' seen_accept_slack_risk=true; accept_slack_risk_requested=true; shift ;; --check-slack-deployment-evidence) [[ "$seen_check_slack_deployment" == false ]] || usage_fail 'duplicate Slack deployment check' seen_check_slack_deployment=true; check_slack_deployment_requested=true; shift ;; -h|--help) usage exit 0 ;; *) usage_fail "unsupported argument: $1" ;; esac done if [[ "$check_grafana_requested" == true || "$check_slack_requested" == true || "$check_slack_deployment_requested" == true ]]; then if [[ "$check_grafana_requested" == true ]]; then [[ "$check_slack_requested" == false && "$check_slack_deployment_requested" == false ]] || usage_fail 'select exactly one recovery evidence check' elif [[ "$check_slack_requested" == true ]]; then [[ "$check_slack_deployment_requested" == false ]] || usage_fail 'select exactly one recovery evidence check' fi [[ "$execute_requested" == false && "$grafana_requested" == false && "$slack_requested" == false ]] || usage_fail 'recovery evidence checks cannot be mixed with execute mode' [[ "$accept_slack_risk_requested" == false ]] || usage_fail 'Slack risk acceptance cannot be mixed with evidence checks' [[ "$seen_user_file" == false && "$seen_password_file" == false && "$seen_slack_file" == false ]] || usage_fail 'recovery evidence checks do not accept payload files' return fi if [[ "$execute_requested" == false ]]; then [[ "$grafana_requested" == false && "$slack_requested" == false ]] || usage_fail 'asset selection requires --execute' [[ "$seen_user_file" == false && "$seen_password_file" == false && "$seen_slack_file" == false ]] || usage_fail 'payload files require --execute and an asset selection' [[ "$accept_slack_risk_requested" == false ]] || usage_fail 'Slack risk acceptance requires the Slack execute interface' return fi [[ "$grafana_requested" == true || "$slack_requested" == true ]] || usage_fail '--execute requires at least one asset selection' if [[ "$grafana_requested" == true ]]; then [[ "$seen_user_file" == true && "$seen_password_file" == true ]] || usage_fail '--grafana-admin requires both Grafana input files' [[ "$grafana_user_file" == /* && "$grafana_password_file" == /* ]] || usage_fail 'Grafana input files must be absolute paths' else [[ "$seen_user_file" == false && "$seen_password_file" == false ]] || usage_fail 'Grafana input files require --grafana-admin' fi if [[ "$slack_requested" == true ]]; then [[ "$seen_slack_file" == true ]] || usage_fail '--slack-webhook requires its input file' [[ "$slack_webhook_file" == /* ]] || usage_fail 'Slack webhook file must be an absolute path' else [[ "$seen_slack_file" == false ]] || usage_fail 'Slack webhook file requires --slack-webhook' fi if [[ "$accept_slack_risk_requested" == true ]]; then [[ "$slack_requested" == true && "$grafana_requested" == false ]] || usage_fail 'Slack risk acceptance is only available for the Slack-only execute interface' fi } require_commands() { local command_name for command_name in bash base64 cmp curl date install kubectl mktemp python3 sort stat sudo timeout wc; do if [[ "$command_name" == kubectl ]]; then [[ -x "$KUBECTL_BIN" ]] || command -v "$KUBECTL_BIN" >/dev/null 2>&1 || plain_fail 'kubectl is required' elif [[ "$command_name" == sudo ]]; then [[ -x "$SUDO_BIN" ]] || command -v "$SUDO_BIN" >/dev/null 2>&1 || plain_fail 'sudo is required' else command -v "$command_name" >/dev/null 2>&1 || plain_fail "$command_name is required" fi done [[ -f "$ENCRYPTION_SCRIPT" && ! -L "$ENCRYPTION_SCRIPT" && -r "$ENCRYPTION_SCRIPT" ]] || plain_fail 'k3s encryption validator is unavailable' [[ -f "$RESTORE_SCRIPT" && ! -L "$RESTORE_SCRIPT" && -r "$RESTORE_SCRIPT" ]] || plain_fail 'k3s restore evidence validator is unavailable' } make_temporary_dir() { umask 077 temporary_dir="$(mktemp -d /tmp/platform-observability-secrets.XXXXXX)" || plain_fail 'could not create private temporary directory' [[ "$temporary_dir" == /tmp/platform-observability-secrets.?????? && ! -L "$temporary_dir" ]] || plain_fail 'private temporary directory boundary is invalid' chmod 0700 -- "$temporary_dir" } validate_private_input_metadata() { local file=$1 description=$2 metadata [[ "$file" == /* && -f "$file" && ! -L "$file" && -O "$file" && -r "$file" && -s "$file" ]] || plain_fail "$description must be a readable non-empty current-user-owned non-symlink regular file" metadata="$(stat --format='%u:%a:%h:%F' -- "$file")" || plain_fail "cannot stat $description" [[ "$metadata" == "${EUID}:600:1:regular file" ]] || plain_fail "$description must be current-user-owned mode 0600 with one hard link" } validate_selected_input_metadata() { if [[ "$grafana_requested" == true ]]; then validate_private_input_metadata "$grafana_user_file" 'Grafana admin user file' validate_private_input_metadata "$grafana_password_file" 'Grafana admin password file' fi if [[ "$slack_requested" == true ]]; then validate_private_input_metadata "$slack_webhook_file" 'Slack webhook file' fi } snapshot_private_input() { local source=$1 destination=$2 description=$3 python3 -I -S - "$source" "$destination" <<'PY' || import os, stat, sys source, destination = sys.argv[1:] required = ("O_NOFOLLOW", "O_CLOEXEC") if any(not hasattr(os, name) for name in required): raise SystemExit(1) source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) try: before = os.fstat(source_fd) if ( not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode) != 0o600 or before.st_uid != os.geteuid() or before.st_size <= 0 or before.st_nlink != 1 ): raise SystemExit(1) chunks = [] while True: chunk = os.read(source_fd, 65536) if not chunk: break chunks.append(chunk) after = os.fstat(source_fd) identity_before = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) identity_after = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) if identity_before != identity_after: raise SystemExit(1) payload = b"".join(chunks) if len(payload) != before.st_size: raise SystemExit(1) finally: os.close(source_fd) destination_fd = os.open( destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, 0o600, ) try: view = memoryview(payload) while view: written = os.write(destination_fd, view) if written <= 0: raise SystemExit(1) view = view[written:] finally: os.close(destination_fd) PY plain_fail "could not take a stable private snapshot of $description" } capture_selected_inputs() { if [[ "$grafana_requested" == true ]]; then grafana_user_snapshot="$temporary_dir/input-grafana-user" grafana_password_snapshot="$temporary_dir/input-grafana-password" snapshot_private_input "$grafana_user_file" "$grafana_user_snapshot" 'Grafana admin user file' snapshot_private_input "$grafana_password_file" "$grafana_password_snapshot" 'Grafana admin password file' fi if [[ "$slack_requested" == true ]]; then slack_webhook_snapshot="$temporary_dir/input-slack-webhook" snapshot_private_input "$slack_webhook_file" "$slack_webhook_snapshot" 'Slack webhook file' fi } recheck_selected_inputs() { local second if [[ "$grafana_requested" == true ]]; then second="$temporary_dir/recheck-grafana-user" snapshot_private_input "$grafana_user_file" "$second" 'Grafana admin user file' cmp --silent -- "$grafana_user_snapshot" "$second" || plain_fail 'Grafana admin user input changed during confirmation' second="$temporary_dir/recheck-grafana-password" snapshot_private_input "$grafana_password_file" "$second" 'Grafana admin password file' cmp --silent -- "$grafana_password_snapshot" "$second" || plain_fail 'Grafana admin password input changed during confirmation' fi if [[ "$slack_requested" == true ]]; then second="$temporary_dir/recheck-slack-webhook" snapshot_private_input "$slack_webhook_file" "$second" 'Slack webhook file' cmp --silent -- "$slack_webhook_snapshot" "$second" || plain_fail 'Slack webhook input changed during confirmation' fi } validate_admin_user_content() { python3 -I -S - "$grafana_user_snapshot" <<'PY' import pathlib, sys data = pathlib.Path(sys.argv[1]).read_bytes() if not data or any(byte < 0x20 or byte > 0x7e for byte in data): raise SystemExit(1) PY } validate_admin_password_content() { python3 -I -S - "$grafana_password_snapshot" <<'PY' import pathlib, sys data = pathlib.Path(sys.argv[1]).read_bytes() if len(data) < 20 or b"\x00" in data or b"\r" in data or b"\n" in data: raise SystemExit(1) PY } validate_slack_webhook_content() { python3 -I -S - "$slack_webhook_snapshot" <<'PY' import pathlib, re, sys data = pathlib.Path(sys.argv[1]).read_bytes() try: value = data.decode("ascii") except UnicodeDecodeError: raise SystemExit(1) pattern = r"https://hooks[.]slack[.]com/services/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+" if re.fullmatch(pattern, value) is None: raise SystemExit(1) PY } validate_selected_input_content() { if [[ "$grafana_requested" == true ]]; then validate_admin_user_content || plain_fail 'Grafana admin user must be one non-empty printable ASCII line' validate_admin_password_content || plain_fail 'Grafana admin password must be at least 20 bytes with no NUL or line break' fi if [[ "$slack_requested" == true ]]; then validate_slack_webhook_content || plain_fail 'Slack webhook must have the exact hooks.slack.com/services prefix and three path components' fi } run_common_gates() { local -a safe_environment=( /usr/bin/env -i "PATH=$VALIDATOR_PATH" "HOME=$VALIDATOR_HOME" ) if [[ "$TEST_MODE" == true ]]; then safe_environment+=( "PLATFORM_TEST_COMMAND_LOG=${PLATFORM_TEST_COMMAND_LOG:-}" "PLATFORM_TEST_ENCRYPTION_COUNT=${PLATFORM_TEST_ENCRYPTION_COUNT:-}" "PLATFORM_TEST_RESTORE_COUNT=${PLATFORM_TEST_RESTORE_COUNT:-}" "PLATFORM_TEST_ENCRYPTION_FAIL_AT=${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" "PLATFORM_TEST_RESTORE_FAIL_AT=${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" "PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE=${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}" "PLATFORM_TEST_VALIDATOR_ENV_LOG=${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" "PLATFORM_TEST_STATE=${PLATFORM_TEST_STATE:-}" ) fi "${safe_environment[@]}" /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted >/dev/null || plain_fail 'k3s Secret encryption is not fully re-encrypted' "${safe_environment[@]}" /usr/bin/bash "$RESTORE_SCRIPT" --check >/dev/null || plain_fail 'current k3s restore evidence is unavailable' } require_context_and_namespace() { local context context="$(kubectl_bounded config current-context)" || plain_fail 'cannot read current Kubernetes context' [[ "$context" == "$EXPECTED_CONTEXT" ]] || plain_fail 'current Kubernetes context is not default' kubectl_bounded get namespace "$NAMESPACE" >/dev/null || plain_fail 'observability namespace is absent' } require_current_context() { local context context="$(kubectl_bounded config current-context)" || plain_fail 'cannot read current Kubernetes context' [[ "$context" == "$EXPECTED_CONTEXT" ]] || plain_fail 'current Kubernetes context is not default' } kubectl_bounded() { local process_timeout=$KUBECTL_PROCESS_TIMEOUT [[ "${1:-}" != proxy ]] || process_timeout=$KUBECTL_PROXY_PROCESS_TIMEOUT /usr/bin/timeout --signal=TERM --kill-after=1s "$process_timeout" \ "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@" } secret_exists() { local name=$1 result result="$(kubectl_bounded --namespace "$NAMESPACE" get secret "$name" \ --ignore-not-found --output=name)" || return 2 if [[ -z "$result" ]]; then return 1; fi [[ "$result" == "secret/$name" ]] || return 2 return 0 } validate_secret_json() { local json_file=$1 name=$2 expected_keys=$3 decoded_prefix=$4 expected_token=${5:-} python3 -I -S - "$json_file" "$name" "$expected_keys" "$decoded_prefix" "$expected_token" <<'PY' import base64, binascii, json, os, pathlib, re, sys json_file, expected_name, expected_keys_raw, decoded_prefix, expected_token = sys.argv[1:] try: item = json.loads(pathlib.Path(json_file).read_text(encoding="utf-8")) except Exception: raise SystemExit(1) expected_keys = expected_keys_raw.split(",") metadata = item.get("metadata") or {} data = item.get("data") uid = metadata.get("uid") annotations = metadata.get("annotations") or {} if ( item.get("apiVersion") != "v1" or item.get("kind") != "Secret" or item.get("type") != "Opaque" or metadata.get("namespace") != "observability" or metadata.get("name") != expected_name or not isinstance(data, dict) or sorted(data) != sorted(expected_keys) or not isinstance(uid, str) or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", uid) is None ): raise SystemExit(1) if expected_token and annotations.get("observability.hyeonworks.com/create-transaction") != expected_token: raise SystemExit(1) for key in expected_keys: encoded = data.get(key) if not isinstance(encoded, str): raise SystemExit(1) try: decoded = base64.b64decode(encoded, validate=True) except (binascii.Error, ValueError): raise SystemExit(1) destination = pathlib.Path(decoded_prefix + "." + key) flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL descriptor = os.open(destination, flags, 0o600) try: os.write(descriptor, decoded) finally: os.close(descriptor) print(uid, end="") PY } new_transaction_token() { python3 -I -S -c 'import secrets; print(secrets.token_hex(16))' } build_secret_manifest() { local asset=$1 token=$2 destination=$3 name case "$asset" in grafana) name='grafana-admin' ;; slack) name='alertmanager-slack-webhook' ;; *) return 1 ;; esac python3 -I -S - "$asset" "$name" "$token" "$destination" \ "$grafana_user_snapshot" "$grafana_password_snapshot" "$slack_webhook_snapshot" <<'PY' import base64, json, os, pathlib, re, sys asset, name, token, destination, user_file, password_file, slack_file = sys.argv[1:] if re.fullmatch(r"[0-9a-f]{32}", token) is None: raise SystemExit(1) if asset == "grafana": sources = {"admin-user": user_file, "admin-password": password_file} elif asset == "slack": sources = {"url": slack_file} else: raise SystemExit(1) data = { key: base64.b64encode(pathlib.Path(path).read_bytes()).decode("ascii") for key, path in sources.items() } item = { "apiVersion": "v1", "kind": "Secret", "type": "Opaque", "metadata": { "namespace": "observability", "name": name, "annotations": { "observability.hyeonworks.com/create-transaction": token, }, }, "data": data, } descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, 0o600) try: encoded = json.dumps(item, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" view = memoryview(encoded) while view: written = os.write(descriptor, view) if written <= 0: raise SystemExit(1) view = view[written:] finally: os.close(descriptor) PY } inspect_contract() { local asset=$1 phase=$2 state_var=$3 uid_var=$4 local name expected_keys json_file decoded_prefix uid case "$asset" in grafana) name='grafana-admin'; expected_keys='admin-user,admin-password' ;; slack) name='alertmanager-slack-webhook'; expected_keys='url' ;; *) return 1 ;; esac if secret_exists "$name"; then json_file="$temporary_dir/${phase}-${asset}.json" decoded_prefix="$temporary_dir/${phase}-${asset}" kubectl_bounded --namespace "$NAMESPACE" get secret "$name" --output=json >"$json_file" || plain_fail "cannot read existing $asset Secret contract" chmod 0600 -- "$json_file" uid="$(validate_secret_json "$json_file" "$name" "$expected_keys" "$decoded_prefix")" || plain_fail "existing $asset Secret has wrong type, keys, metadata, or encoding" case "$asset" in grafana) cmp --silent -- "$grafana_user_snapshot" "$decoded_prefix.admin-user" || plain_fail 'existing Grafana admin user differs; rotation was refused' cmp --silent -- "$grafana_password_snapshot" "$decoded_prefix.admin-password" || plain_fail 'existing Grafana admin password differs; rotation was refused' ;; slack) cmp --silent -- "$slack_webhook_snapshot" "$decoded_prefix.url" || plain_fail 'existing Slack webhook differs; rotation was refused' ;; esac printf -v "$state_var" '%s' existing printf -v "$uid_var" '%s' "$uid" else case $? in 1) printf -v "$state_var" '%s' absent printf -v "$uid_var" '%s' '' ;; *) plain_fail "cannot determine $asset Secret presence" ;; esac fi } read_confirmation() { local prompt=$1 expected=$2 answer='' printf '%s' "$prompt" >&2 if [[ "$TEST_MODE" == true ]]; then (( confirmation_index < ${#confirmation_answers[@]} )) || plain_fail 'confirmation input is missing' answer="${confirmation_answers[$confirmation_index]}" confirmation_index=$((confirmation_index + 1)) else [[ -t 0 ]] || plain_fail 'an interactive terminal is required for execute confirmation' IFS= read -r answer fi [[ "$answer" == "$expected" ]] || plain_fail 'confirmation was not exact' } now_utc() { if [[ "$TEST_MODE" == true ]]; then printf '%s\n' "$TEST_NOW_UTC"; else date -u +%Y-%m-%dT%H:%M:%SZ; fi } timestamp_epoch() { local timestamp=$1 normalized normalized="$(date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1 [[ "$normalized" == "$timestamp" ]] || return 1 date -u -d "$timestamp" +%s } now_epoch() { timestamp_epoch "$(now_utc)" } evidence_asset_fields() { local asset=$1 path_var=$2 resource_var=$3 case "$asset" in grafana) printf -v "$path_var" '%s' "$EVIDENCE_DIR/grafana-local.env" printf -v "$resource_var" '%s' 'observability/grafana-admin' ;; slack) printf -v "$path_var" '%s' "$EVIDENCE_DIR/slack.env" printf -v "$resource_var" '%s' 'observability/alertmanager-slack-webhook' ;; *) return 1 ;; esac } sudo_refresh() { "$SUDO_BIN" -v || plain_fail 'sudo credential refresh failed' "$SUDO_BIN" -n /usr/bin/true || plain_fail 'narrow non-interactive sudo is unavailable' } validate_recovery_evidence_stream() { local file=$1 expected_resource=$2 timestamp_var=$3 local -a lines=() local parsed_timestamp marker_epoch current_epoch mapfile -t lines <"$file" [[ "$(wc -l <"$file" | tr -d '[:space:]')" == 4 && ${#lines[@]} -eq 4 ]] || return 1 [[ "${lines[0]}" == "schema=$EVIDENCE_SCHEMA" ]] || return 1 [[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1 [[ "${lines[2]}" == "resource=$expected_resource" ]] || return 1 [[ "${lines[3]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1 parsed_timestamp="${BASH_REMATCH[1]}" timestamp_epoch "$parsed_timestamp" >/dev/null || return 1 printf -v "$timestamp_var" '%s' "$parsed_timestamp" } validate_slack_risk_evidence_stream() { local file=$1 timestamp_var=$2 local -a lines=() local parsed_timestamp marker_epoch current_epoch mapfile -t lines <"$file" [[ "$(wc -l <"$file" | tr -d '[:space:]')" == 8 && ${#lines[@]} -eq 8 ]] || return 1 [[ "${lines[0]}" == "schema=$RISK_EVIDENCE_SCHEMA" ]] || return 1 [[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1 [[ "${lines[2]}" == 'resource=observability/alertmanager-slack-webhook' ]] || return 1 [[ "${lines[3]}" == 'reason=off-host-slack-recovery-deferred' ]] || return 1 [[ "${lines[4]}" == "accepted_by_uid=$RISK_ACCEPTED_BY_UID" ]] || return 1 [[ "${lines[5]}" == 'approval_method=interactive-exact-confirmation' ]] || return 1 [[ "${lines[6]}" == "approval_ref=$RISK_APPROVAL_REF" ]] || return 1 [[ "${lines[7]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1 parsed_timestamp="${BASH_REMATCH[1]}" timestamp_epoch "$parsed_timestamp" >/dev/null || return 1 marker_epoch="$(timestamp_epoch "$parsed_timestamp")" || return 1 current_epoch="$(now_epoch)" || return 1 (( marker_epoch <= current_epoch )) || return 1 printf -v "$timestamp_var" '%s' "$parsed_timestamp" } classify_slack_evidence_stream() { local file=$1 kind_var=$2 timestamp_var=$3 timestamp if validate_recovery_evidence_stream "$file" 'observability/alertmanager-slack-webhook' timestamp; then printf -v "$kind_var" '%s' RECOVERY elif validate_slack_risk_evidence_stream "$file" timestamp; then printf -v "$kind_var" '%s' RISK_ACCEPTED else return 1 fi printf -v "$timestamp_var" '%s' "$timestamp" } validate_evidence_stream() { validate_recovery_evidence_stream "$@" } root_marker_helper() { local output=$1 failure_phase='' shift if [[ "${1:-}" == install && "$TEST_MODE" == true ]]; then failure_phase=$MARKER_TEST_FAILURE_PHASE fi "$SUDO_BIN" -n /usr/bin/python3 -I -S - platform-observability-marker-helper "$@" \ "$failure_phase" >"$output" 2>/dev/null <<'PY' import errno import os import pathlib import re import stat import sys ( tag, action, directory, basename, expected_uid_raw, expected_gid_raw, source, expected_kind, expected_source, source_uid_raw, failure_phase, ) = sys.argv[1:] if tag != "platform-observability-marker-helper": raise SystemExit(2) if action not in {"check-dir", "ensure-dir", "read", "install", "unlink-exact"}: raise SystemExit(2) if failure_phase not in {"", "first-write", "pre-rename"}: raise SystemExit(2) if failure_phase and action != "install": raise SystemExit(2) if not directory.startswith("/") or "\x00" in directory: raise SystemExit(2) expected_uid = int(expected_uid_raw) expected_gid = int(expected_gid_raw) source_uid = int(source_uid_raw) if min(expected_uid, expected_gid, source_uid) < 0: raise SystemExit(2) if action in {"check-dir", "ensure-dir"}: if basename not in {"production-hierarchy", "fixture-hierarchy"}: raise SystemExit(2) if source or expected_kind or expected_source or failure_phase: raise SystemExit(2) else: if basename not in {"grafana-local.env", "slack.env"}: raise SystemExit(2) fixture_marker_directory = re.fullmatch( r"/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+/evidence-parent/platform/recovery-evidence", directory, ) if directory == "/etc/hyeonworks/platform/recovery-evidence": if expected_uid != 0 or expected_gid != 0: raise SystemExit(2) elif fixture_marker_directory is not None: if expected_uid != source_uid: raise SystemExit(2) else: raise SystemExit(2) directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC def open_directory(path): if path == "/etc/hyeonworks/platform/recovery-evidence": profile = "production-hierarchy" anchor = "/etc" components = ("hyeonworks", "platform", "recovery-evidence") else: fixture_suffix = "/evidence-parent/platform/recovery-evidence" if not path.endswith(fixture_suffix): raise OSError(errno.EINVAL, "invalid marker directory") profile = "fixture-hierarchy" anchor = path[: -len(fixture_suffix)] components = ("evidence-parent", "platform", "recovery-evidence") descriptors = [open_hierarchy_anchor(anchor, profile)] identities = [] try: for component in components: parent_descriptor = descriptors[-1] following = os.open(component, directory_flags, dir_fd=parent_descriptor) try: identity = attest_managed_directory(following) attest_managed_entry(parent_descriptor, component, identity) except BaseException: os.close(following) raise descriptors.append(following) identities.append(identity) for index, component in enumerate(components): attest_managed_entry(descriptors[index], component, identities[index]) result = descriptors.pop() for descriptor in reversed(descriptors): os.close(descriptor) return result except BaseException: for descriptor in reversed(descriptors): os.close(descriptor) raise def require_managed_metadata(metadata): if ( not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700 or metadata.st_uid != expected_uid or metadata.st_gid != expected_gid ): raise OSError(errno.EPERM, "managed directory metadata mismatch") def attest_managed_directory(descriptor): metadata = os.fstat(descriptor) require_managed_metadata(metadata) return metadata.st_dev, metadata.st_ino def attest_managed_entry(parent_descriptor, component, expected_identity): metadata = os.stat(component, dir_fd=parent_descriptor, follow_symlinks=False) require_managed_metadata(metadata) if (metadata.st_dev, metadata.st_ino) != expected_identity: raise OSError(errno.ESTALE, "managed directory identity changed") def open_hierarchy_anchor(path, profile): descriptor = os.open(path, directory_flags) try: metadata = os.fstat(descriptor) if not stat.S_ISDIR(metadata.st_mode): raise OSError(errno.ENOTDIR, "hierarchy anchor is not a directory") if profile == "production-hierarchy": if metadata.st_uid != 0 or metadata.st_gid != 0: raise OSError(errno.EPERM, "production hierarchy anchor metadata mismatch") elif ( stat.S_IMODE(metadata.st_mode) != 0o700 or metadata.st_uid != expected_uid or metadata.st_gid != expected_gid ): raise OSError(errno.EPERM, "fixture hierarchy anchor metadata mismatch") return descriptor except BaseException: os.close(descriptor) raise def manage_evidence_hierarchy(profile, anchor, create): if profile == "production-hierarchy": if anchor != "/etc" or expected_uid != 0 or expected_gid != 0: raise OSError(errno.EINVAL, "invalid production hierarchy profile") components = ("hyeonworks", "platform", "recovery-evidence") elif profile == "fixture-hierarchy": if ( expected_uid != source_uid or re.fullmatch( r"/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+", anchor, ) is None ): raise OSError(errno.EINVAL, "invalid fixture hierarchy profile") components = ("evidence-parent", "platform", "recovery-evidence") else: raise OSError(errno.EINVAL, "unknown hierarchy profile") descriptors = [open_hierarchy_anchor(anchor, profile)] identities = [] def revalidate_opened_prefix(): for index, opened_component in enumerate(components[: len(identities)]): attest_managed_entry(descriptors[index], opened_component, identities[index]) try: for component in components: parent_descriptor = descriptors[-1] needs_parent_fsync = False try: following = os.open(component, directory_flags, dir_fd=parent_descriptor) except FileNotFoundError: if not create: revalidate_opened_prefix() return 4 try: previous_umask = os.umask(0) try: os.mkdir(component, 0o700, dir_fd=parent_descriptor) finally: os.umask(previous_umask) except FileExistsError: needs_parent_fsync = True else: needs_parent_fsync = True following = os.open(component, directory_flags, dir_fd=parent_descriptor) try: identity = attest_managed_directory(following) attest_managed_entry(parent_descriptor, component, identity) if needs_parent_fsync: os.fsync(parent_descriptor) attest_managed_entry(parent_descriptor, component, identity) except BaseException: os.close(following) raise descriptors.append(following) identities.append(identity) revalidate_opened_prefix() return 0 finally: for descriptor in reversed(descriptors): os.close(descriptor) def read_open_file(directory_fd, name): descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=directory_fd) try: before = os.fstat(descriptor) if ( not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode) != 0o600 or before.st_uid != expected_uid or before.st_gid != expected_gid or before.st_nlink != 1 ): raise OSError(errno.EPERM, "file metadata mismatch") chunks = [] while True: chunk = os.read(descriptor, 65536) if not chunk: break chunks.append(chunk) after = os.fstat(descriptor) identity_before = ( before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, before.st_nlink, ) identity_after = ( after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, after.st_nlink, ) if identity_before != identity_after: raise OSError(errno.ESTALE, "file changed while open") payload = b"".join(chunks) if len(payload) != before.st_size: raise OSError(errno.EIO, "short read") return payload finally: os.close(descriptor) def read_source(path): descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) try: before = os.fstat(descriptor) if ( not stat.S_ISREG(before.st_mode) or stat.S_IMODE(before.st_mode) != 0o600 or before.st_uid != source_uid or before.st_nlink != 1 ): raise OSError(errno.EPERM, "source metadata mismatch") chunks = [] while True: chunk = os.read(descriptor, 65536) if not chunk: break chunks.append(chunk) after = os.fstat(descriptor) before_identity = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, before.st_nlink) after_identity = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, after.st_nlink) if before_identity != after_identity: raise OSError(errno.ESTALE, "source changed") payload = b"".join(chunks) if len(payload) != before.st_size: raise OSError(errno.EIO, "short source read") return payload finally: os.close(descriptor) if action in {"check-dir", "ensure-dir"}: raise SystemExit( manage_evidence_hierarchy( basename, directory, action == "ensure-dir", ) ) try: directory_fd = open_directory(directory) except FileNotFoundError: if action == "read": raise SystemExit(4) raise try: if action == "read": try: payload = read_open_file(directory_fd, basename) except FileNotFoundError: raise SystemExit(4) os.write(1, payload) raise SystemExit(0) desired = read_source(source) if expected_kind == "absent": try: read_open_file(directory_fd, basename) except FileNotFoundError: pass else: raise OSError(errno.EEXIST, "expected absent marker") elif expected_kind == "file": if read_open_file(directory_fd, basename) != read_source(expected_source): raise OSError(errno.ESTALE, "marker precondition mismatch") else: raise OSError(errno.EINVAL, "invalid marker precondition") if action == "unlink-exact": if read_open_file(directory_fd, basename) != desired: raise OSError(errno.ESTALE, "unlink precondition mismatch") os.unlink(basename, dir_fd=directory_fd) os.fsync(directory_fd) try: os.stat(basename, dir_fd=directory_fd, follow_symlinks=False) except FileNotFoundError: raise SystemExit(0) raise OSError(errno.EEXIST, "marker remained after unlink") temporary_name = f".{basename}.new.{os.getpid():x}" temporary_fd = os.open( temporary_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=directory_fd, ) temporary_exists = True try: try: os.fchown(temporary_fd, expected_uid, expected_gid) os.fchmod(temporary_fd, 0o600) view = memoryview(desired) while view: if failure_phase == "first-write": written = 0 failure_phase = "" else: written = os.write(temporary_fd, view) if written <= 0: raise OSError(errno.EIO, "short marker write") view = view[written:] os.fsync(temporary_fd) metadata = os.fstat(temporary_fd) if metadata.st_nlink != 1: raise OSError(errno.EPERM, "temporary marker link count mismatch") if failure_phase == "pre-rename": raise OSError(errno.EIO, "injected marker pre-rename failure") finally: os.close(temporary_fd) os.replace(temporary_name, basename, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) temporary_exists = False os.fsync(directory_fd) if read_open_file(directory_fd, basename) != desired: raise OSError(errno.EIO, "installed marker verification failed") finally: if temporary_exists: try: os.unlink(temporary_name, dir_fd=directory_fd) except FileNotFoundError: pass finally: os.close(directory_fd) PY } root_marker_read() { local asset=$1 destination=$2 path resource rc evidence_asset_fields "$asset" path resource || return 1 if root_marker_helper "$destination" read "$EVIDENCE_DIR" "${path##*/}" \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID"; then chmod 0600 -- "$destination" return 0 else rc=$? rm -f -- "$destination" return "$rc" fi } read_root_evidence() { local asset=$1 destination=$2 resource path timestamp evidence_asset_fields "$asset" path resource || return 1 root_marker_read "$asset" "$destination" || return 1 validate_evidence_stream "$destination" "$resource" timestamp } check_recovery_evidence() { local asset=$1 marker timestamp marker_epoch current_epoch age require_current_context sudo_refresh marker="$temporary_dir/check-${asset}.env" read_root_evidence "$asset" "$marker" || plain_fail "$asset recovery evidence is missing or malformed" case "$asset" in grafana) validate_evidence_stream "$marker" 'observability/grafana-admin' timestamp ;; slack) validate_evidence_stream "$marker" 'observability/alertmanager-slack-webhook' timestamp ;; esac || plain_fail "$asset recovery evidence is malformed" marker_epoch="$(timestamp_epoch "$timestamp")" || plain_fail 'recovery timestamp is invalid' current_epoch="$(now_epoch)" || plain_fail 'current UTC clock is invalid' age=$((current_epoch - marker_epoch)) (( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || plain_fail "$asset recovery evidence is outside the 30-day window" case "$asset" in grafana) printf 'GRAFANA_RECOVERY_EVIDENCE=PASS\n' ;; slack) printf 'SLACK_RECOVERY_EVIDENCE=PASS\n' ;; esac } check_slack_deployment_evidence() { local marker kind timestamp marker_epoch current_epoch age require_current_context sudo_refresh marker="$temporary_dir/check-slack-deployment.env" root_marker_read slack "$marker" || plain_fail 'Slack deployment evidence is missing or malformed' classify_slack_evidence_stream "$marker" kind timestamp || plain_fail 'Slack deployment evidence is malformed' if [[ "$kind" == RECOVERY ]]; then marker_epoch="$(timestamp_epoch "$timestamp")" || plain_fail 'recovery timestamp is invalid' current_epoch="$(now_epoch)" || plain_fail 'current UTC clock is invalid' age=$((current_epoch - marker_epoch)) (( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) || plain_fail 'Slack recovery evidence is outside the 30-day window' printf 'SLACK_DEPLOYMENT_GATE=RECOVERY\n' else printf 'SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n' fi } ensure_evidence_directory() { root_marker_helper /dev/null ensure-dir "$EVIDENCE_HIERARCHY_ANCHOR" \ "$EVIDENCE_HIERARCHY_PROFILE" \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID" || plain_fail 'cannot safely create or verify the recovery evidence directory' } preflight_evidence_directory() { local rc if root_marker_helper /dev/null check-dir "$EVIDENCE_HIERARCHY_ANCHOR" \ "$EVIDENCE_HIERARCHY_PROFILE" \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID"; then return 0 else rc=$? [[ "$rc" == 4 ]] || plain_fail 'recovery evidence directory boundary is unsafe or drifted' fi } prepare_marker() { local asset=$1 kind=${2:-RECOVERY} path resource snapshot timestamp desired read_rc prior_kind evidence_asset_fields "$asset" path resource || plain_fail 'unknown recovery evidence asset' snapshot="$temporary_dir/prior-${asset}.env" desired="$temporary_dir/desired-${asset}.env" MARKER_PRIOR_FILE["$asset"]="$snapshot" MARKER_DESIRED_FILE["$asset"]="$desired" MARKER_KIND["$asset"]="$kind" MARKER_HAD_PRIOR["$asset"]=false if root_marker_read "$asset" "$snapshot"; then if [[ "$asset" == slack ]]; then classify_slack_evidence_stream "$snapshot" prior_kind timestamp || plain_fail 'Slack recovery marker schema drifted' if [[ "$kind" == RISK && "$prior_kind" == RECOVERY ]]; then plain_fail 'Slack recovery evidence cannot be downgraded to risk acceptance' fi if [[ "$kind" == RISK && "$prior_kind" == RISK_ACCEPTED ]]; then cp -- "$snapshot" "$desired" chmod 0600 -- "$desired" MARKER_HAD_PRIOR["$asset"]=true return fi else validate_recovery_evidence_stream "$snapshot" "$resource" timestamp || plain_fail "$asset recovery marker schema drifted" fi MARKER_HAD_PRIOR["$asset"]=true else read_rc=$? [[ "$read_rc" == 4 ]] || plain_fail "cannot safely read $asset recovery marker" fi if [[ "$kind" == RISK ]]; then printf '%s\n' \ "schema=$RISK_EVIDENCE_SCHEMA" \ "context=$EXPECTED_CONTEXT" \ "resource=$resource" \ 'reason=off-host-slack-recovery-deferred' \ "accepted_by_uid=$RISK_ACCEPTED_BY_UID" \ 'approval_method=interactive-exact-confirmation' \ "approval_ref=$RISK_APPROVAL_REF" \ "checked_at_utc=$(now_utc)" >"$desired" else printf '%s\n' \ "schema=$EVIDENCE_SCHEMA" \ "context=$EXPECTED_CONTEXT" \ "resource=$resource" \ "checked_at_utc=$(now_utc)" >"$desired" fi chmod 0600 -- "$desired" if [[ "$kind" == RISK ]]; then validate_slack_risk_evidence_stream "$desired" timestamp || plain_fail 'generated Slack risk acceptance evidence is invalid' else validate_recovery_evidence_stream "$desired" "$resource" timestamp || plain_fail 'generated recovery evidence is invalid' fi } root_marker_install() { local asset=$1 source=$2 expected_kind=$3 expected_source=${4:-} path resource evidence_asset_fields "$asset" path resource || return 1 root_marker_helper /dev/null install "$EVIDENCE_DIR" "${path##*/}" \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$source" "$expected_kind" \ "$expected_source" "$EUID" } marker_prestate_unchanged() { local asset=$1 current="$temporary_dir/prestate-${asset}-$RANDOM" read_rc if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then root_marker_read "$asset" "$current" || return 1 cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "$current" else if root_marker_read "$asset" "$current"; then return 1 else read_rc=$? [[ "$read_rc" == 4 ]] fi fi } marker_record_written() { local asset=$1 existing for existing in "${WRITTEN_MARKERS[@]}"; do [[ "$existing" != "$asset" ]] || return 0 done WRITTEN_MARKERS+=("$asset") } clear_pending_marker() { pending_marker_asset='' } classify_pending_marker() { local asset=$pending_marker_asset current read_rc [[ -n "$asset" ]] || return 0 current="$temporary_dir/classify-marker-${asset}-$RANDOM" if root_marker_read "$asset" "$current"; then if cmp --silent -- "${MARKER_DESIRED_FILE[$asset]}" "$current"; then marker_record_written "$asset" clear_pending_marker return 0 fi if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]] && cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "$current"; then clear_pending_marker return 1 fi return 2 else read_rc=$? if [[ "$read_rc" == 4 && "${MARKER_HAD_PRIOR[$asset]}" == false ]]; then clear_pending_marker return 1 fi return 2 fi } install_marker() { local asset=$1 install_rc classification if [[ "$asset" == slack && "${MARKER_HAD_PRIOR[$asset]}" == true ]] && cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "${MARKER_DESIRED_FILE[$asset]}"; then return 0 fi marker_prestate_unchanged "$asset" || return 1 pending_marker_asset=$asset if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then root_marker_install "$asset" "${MARKER_DESIRED_FILE[$asset]}" file \ "${MARKER_PRIOR_FILE[$asset]}" || install_rc=$? else root_marker_install "$asset" "${MARKER_DESIRED_FILE[$asset]}" absent '' || install_rc=$? fi if [[ -n "${install_rc:-}" ]]; then classify_pending_marker classification=$? if [[ "$asset" == slack && "${MARKER_KIND[$asset]}" == RISK && "$classification" == 0 ]]; then transaction_active=false emit_manual_recovery printf 'ERROR: Slack risk marker installation outcome is uncertain\n' >&2 exit "$install_rc" fi [[ "$classification" != 2 ]] || emit_manual_recovery return "$install_rc" fi classify_pending_marker || return 1 } restore_written_markers() { local index asset current desired prior read_rc rc=0 for ((index=${#WRITTEN_MARKERS[@]} - 1; index >= 0; index--)); do asset="${WRITTEN_MARKERS[$index]}" desired="${MARKER_DESIRED_FILE[$asset]}" current="$temporary_dir/rollback-marker-${asset}-$RANDOM" if ! root_marker_read "$asset" "$current" || ! cmp --silent -- "$desired" "$current"; then rc=1 continue fi if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then prior="${MARKER_PRIOR_FILE[$asset]}" root_marker_install "$asset" "$prior" file "$desired" || rc=1 else root_marker_helper /dev/null unlink-exact "$EVIDENCE_DIR" \ "$(case "$asset" in grafana) printf grafana-local.env ;; slack) printf slack.env ;; esac)" \ "$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$desired" file "$desired" "$EUID" || rc=1 fi done return "$rc" } stop_proxy() { local pid=$1 attempt /bin/kill -TERM "$pid" 2>/dev/null || true for ((attempt=0; attempt<20; attempt++)); do if ! /bin/kill -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi /usr/bin/sleep 0.05 done /bin/kill -KILL "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true } delete_secret_with_uid_precondition() { local name=$1 uid=$2 socket options response proxy_log proxy_pid attempt http_code curl_rc=0 [[ "$name" == grafana-admin || "$name" == alertmanager-slack-webhook ]] || return 1 [[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 socket="$temporary_dir/proxy-${name}.sock" options="$temporary_dir/delete-${name}.json" response="$temporary_dir/delete-${name}.response" proxy_log="$temporary_dir/proxy-${name}.log" printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' \ "$uid" >"$options" chmod 0600 -- "$options" "$response" 2>/dev/null || true /usr/bin/timeout --signal=TERM --kill-after=1s "$KUBECTL_PROXY_PROCESS_TIMEOUT" \ "$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" proxy \ --unix-socket="$socket" \ --api-prefix=/ \ --accept-paths="^/api/v1/namespaces/${NAMESPACE}/secrets/${name}$" \ --reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 & proxy_pid=$! for ((attempt=0; attempt<50; attempt++)); do [[ -S "$socket" ]] && break /bin/kill -0 "$proxy_pid" 2>/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; } /usr/bin/sleep 0.05 done [[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; } http_code="$(curl --silent --show-error --max-time 5 \ --unix-socket "$socket" \ --output "$response" \ --write-out '%{http_code}' \ --request DELETE \ --header 'Content-Type: application/json' \ --data-binary "@$options" \ "http://localhost/api/v1/namespaces/${NAMESPACE}/secrets/${name}")" || curl_rc=$? stop_proxy "$proxy_pid" (( curl_rc == 0 )) || return 1 [[ "$http_code" == 200 || "$http_code" == 202 ]] || return 1 if secret_exists "$name"; then return 1; else [[ $? == 1 ]]; fi } rollback_owned_secrets() { local index rc=0 for ((index=${#OWNED_NAMES[@]} - 1; index >= 0; index--)); do if ! delete_secret_with_uid_precondition "${OWNED_NAMES[$index]}" "${OWNED_UIDS[$index]}"; then rc=1 fi done return "$rc" } rollback_transaction() { local rc=0 [[ "$rollback_in_progress" == false ]] || return 1 rollback_in_progress=true restore_written_markers || rc=1 rollback_owned_secrets || rc=1 if (( rc == 0 )); then printf 'OBSERVABILITY_SECRET_ROLLBACK=PASS\n' >&2 else printf 'OBSERVABILITY_SECRET_ROLLBACK=FAIL\n' >&2 emit_manual_recovery fi rollback_in_progress=false return "$rc" } clear_pending_create() { pending_asset='' pending_name='' pending_token='' pending_manifest='' pending_response='' } classify_pending_create() { local live_json live_prefix response_prefix response_uid live_uid presence_rc expected_keys [[ -n "$pending_asset" && -n "$pending_name" && -n "$pending_token" && -n "$pending_manifest" ]] || return 0 case "$pending_asset" in grafana) expected_keys='admin-user,admin-password' ;; slack) expected_keys='url' ;; *) return 2 ;; esac if [[ -n "$pending_response" && -f "$pending_response" && ! -L "$pending_response" && -s "$pending_response" ]]; then response_prefix="$temporary_dir/pending-${pending_asset}-response" response_uid="$(validate_secret_json "$pending_response" "$pending_name" "$expected_keys" \ "$response_prefix" "$pending_token")" || response_uid='' if [[ -n "$response_uid" ]]; then case "$pending_asset" in grafana) cmp --silent -- "$grafana_user_snapshot" "$response_prefix.admin-user" || response_uid='' cmp --silent -- "$grafana_password_snapshot" "$response_prefix.admin-password" || response_uid='' ;; slack) cmp --silent -- "$slack_webhook_snapshot" "$response_prefix.url" || response_uid='' ;; esac fi else response_uid='' fi if secret_exists "$pending_name"; then live_json="$temporary_dir/pending-${pending_asset}-live.json" live_prefix="$temporary_dir/pending-${pending_asset}-live" kubectl_bounded --namespace "$NAMESPACE" get secret "$pending_name" --output=json >"$live_json" || return 2 chmod 0600 -- "$live_json" live_uid="$(validate_secret_json "$live_json" "$pending_name" "$expected_keys" \ "$live_prefix" "$pending_token")" || return 2 [[ -n "$response_uid" && "$live_uid" == "$response_uid" ]] || return 2 case "$pending_asset" in grafana) cmp --silent -- "$grafana_user_snapshot" "$live_prefix.admin-user" || return 2 cmp --silent -- "$grafana_password_snapshot" "$live_prefix.admin-password" || return 2 ;; slack) cmp --silent -- "$slack_webhook_snapshot" "$live_prefix.url" || return 2 ;; esac OWNED_ASSETS+=("$pending_asset") OWNED_NAMES+=("$pending_name") OWNED_UIDS+=("$response_uid") clear_pending_create return 0 else presence_rc=$? [[ "$presence_rc" == 1 ]] || return 2 clear_pending_create return 1 fi } transaction_fail() { local message=$1 ambiguous=${2:-false} exit_status=${3:-1} classification trap - ERR HUP INT TERM set +e if [[ -n "$pending_marker_asset" ]]; then classify_pending_marker classification=$? [[ "$classification" != 2 ]] || ambiguous=true fi if [[ -n "$pending_asset" ]]; then classify_pending_create case $? in 0|1) ;; *) ambiguous=true ;; esac fi [[ "$ambiguous" != true ]] || emit_manual_recovery if [[ "$transaction_active" == true ]]; then rollback_transaction || true; fi transaction_active=false printf 'ERROR: %s\n' "$message" >&2 exit "$exit_status" } on_error() { local rc=$1 line=$2 transaction_fail "unexpected failure at line $line (exit $rc)" } on_signal() { local rc=$1 transaction_fail "interrupted (exit $rc)" false "$rc" } create_asset_secret() { local asset=$1 name uid ambiguous=false response decoded_prefix create_rc=0 case "$asset" in grafana) name='grafana-admin' ;; slack) name='alertmanager-slack-webhook' ;; *) transaction_fail 'unknown Secret asset' ;; esac pending_asset="$asset" pending_name="$name" pending_token="$(new_transaction_token)" || transaction_fail 'could not create transaction ownership token' [[ "$pending_token" =~ ^[0-9a-f]{32}$ ]] || transaction_fail 'transaction ownership token is invalid' pending_manifest="$temporary_dir/create-${asset}.json" response="$temporary_dir/create-${asset}-response.json" pending_response="$response" decoded_prefix="$temporary_dir/create-${asset}-response" build_secret_manifest "$asset" "$pending_token" "$pending_manifest" || transaction_fail "could not build $asset Secret create request" if kubectl_bounded create --filename="$pending_manifest" --output=json >"$response"; then : else create_rc=$? transaction_fail "create outcome for $NAMESPACE/$name was not confirmed (exit $create_rc)" fi chmod 0600 -- "$response" case "$asset" in grafana) uid="$(validate_secret_json "$response" "$name" 'admin-user,admin-password' \ "$decoded_prefix" "$pending_token")" || transaction_fail 'Grafana create response contract is invalid' cmp --silent -- "$grafana_user_snapshot" "$decoded_prefix.admin-user" || transaction_fail 'Grafana create response user differs' cmp --silent -- "$grafana_password_snapshot" "$decoded_prefix.admin-password" || transaction_fail 'Grafana create response password differs' ;; slack) uid="$(validate_secret_json "$response" "$name" url "$decoded_prefix" "$pending_token")" || transaction_fail 'Slack create response contract is invalid' cmp --silent -- "$slack_webhook_snapshot" "$decoded_prefix.url" || transaction_fail 'Slack create response webhook differs' ;; esac OWNED_ASSETS+=("$asset") OWNED_NAMES+=("$name") OWNED_UIDS+=("$uid") clear_pending_create } verify_reinspection() { local asset=$1 expected_state=$2 expected_uid=$3 actual_state actual_uid inspect_contract "$asset" postprompt actual_state actual_uid [[ "$actual_state" == "$expected_state" && "$actual_uid" == "$expected_uid" ]] || plain_fail "$asset Secret state changed before the last gate" } execute_transaction() { local grafana_state='' grafana_uid='' slack_state='' slack_uid='' local post_state post_uid index asset validate_selected_input_metadata run_common_gates validate_selected_input_metadata capture_selected_inputs validate_selected_input_content require_context_and_namespace if [[ "$grafana_requested" == true ]]; then inspect_contract grafana preprompt grafana_state grafana_uid; fi if [[ "$slack_requested" == true ]]; then inspect_contract slack preprompt slack_state slack_uid; fi sudo_refresh read_confirmation 'Type APPLY default: ' 'APPLY default' if [[ "$grafana_requested" == true ]]; then read_confirmation 'Type RECOVERY GRAFANA-LOCAL default: ' 'RECOVERY GRAFANA-LOCAL default' fi if [[ "$slack_requested" == true ]]; then if [[ "$accept_slack_risk_requested" == true ]]; then read_confirmation 'Type ACCEPT NO OFF-HOST SLACK RECOVERY default: ' \ 'ACCEPT NO OFF-HOST SLACK RECOVERY default' else read_confirmation 'Type RECOVERY SLACK default: ' 'RECOVERY SLACK default' fi fi preflight_evidence_directory if [[ "$grafana_requested" == true ]]; then prepare_marker grafana; fi if [[ "$slack_requested" == true ]]; then if [[ "$accept_slack_risk_requested" == true ]]; then prepare_marker slack RISK else prepare_marker slack RECOVERY fi fi recheck_selected_inputs require_context_and_namespace if [[ "$grafana_requested" == true ]]; then verify_reinspection grafana "$grafana_state" "$grafana_uid"; fi if [[ "$slack_requested" == true ]]; then verify_reinspection slack "$slack_state" "$slack_uid"; fi run_common_gates transaction_active=true trap 'on_error "$?" "$LINENO"' ERR trap 'on_signal 129' HUP trap 'on_signal 130' INT trap 'on_signal 143' TERM if [[ "$grafana_requested" == true && "$grafana_state" == absent ]]; then create_asset_secret grafana; fi if [[ "$slack_requested" == true && "$slack_state" == absent ]]; then create_asset_secret slack; fi if [[ "$grafana_requested" == true ]]; then inspect_contract grafana postcreate post_state post_uid if [[ "$grafana_state" == absent ]]; then [[ "$post_uid" == "${OWNED_UIDS[0]}" ]] || transaction_fail 'Grafana Secret UID changed after creation' true else [[ "$post_uid" == "$grafana_uid" ]] || transaction_fail 'existing Grafana Secret UID changed' true fi fi if [[ "$slack_requested" == true ]]; then inspect_contract slack postcreate post_state post_uid if [[ "$slack_state" == absent ]]; then for ((index=0; index<${#OWNED_ASSETS[@]}; index++)); do if [[ "${OWNED_ASSETS[$index]}" == slack ]]; then [[ "$post_uid" == "${OWNED_UIDS[$index]}" ]] || transaction_fail 'Slack Secret UID changed after creation' true fi done else [[ "$post_uid" == "$slack_uid" ]] || transaction_fail 'existing Slack Secret UID changed' true fi fi ensure_evidence_directory if [[ "$grafana_requested" == true ]]; then install_marker grafana; fi if [[ "$slack_requested" == true ]]; then install_marker slack; fi trap - ERR HUP INT TERM transaction_active=false if [[ "$grafana_requested" == true ]]; then if [[ "$grafana_state" == absent ]]; then printf 'OBSERVABILITY_GRAFANA_ADMIN=CREATE_CONFIRMED\n' else printf 'OBSERVABILITY_GRAFANA_ADMIN=REUSED_UNCHANGED\n' fi fi if [[ "$slack_requested" == true ]]; then if [[ "$slack_state" == absent ]]; then printf 'OBSERVABILITY_SLACK_WEBHOOK=CREATE_CONFIRMED\n' else printf 'OBSERVABILITY_SLACK_WEBHOOK=REUSED_UNCHANGED\n' fi fi printf 'OBSERVABILITY_SECRETS_TRANSACTION=PASS\n' } observability_secrets_main() { parse_arguments "$@" if [[ "$execute_requested" == false && "$check_grafana_requested" == false && "$check_slack_requested" == false && "$check_slack_deployment_requested" == false ]]; then printf '%s\n' \ 'OBSERVABILITY_SECRETS_DRY_RUN=PASS' \ 'CONTEXT=default' \ 'GRAFANA_ADMIN_CONTRACT=observability/grafana-admin:Opaque:admin-user,admin-password' \ 'SLACK_WEBHOOK_CONTRACT=observability/alertmanager-slack-webhook:Opaque:url' \ 'MUTATION=NOT_REQUESTED' return 0 fi (( EUID != 0 )) || plain_fail 'run this script as the current user, not through sudo' require_commands make_temporary_dir if [[ "$check_grafana_requested" == true ]]; then check_recovery_evidence grafana elif [[ "$check_slack_requested" == true ]]; then check_recovery_evidence slack elif [[ "$check_slack_deployment_requested" == true ]]; then check_slack_deployment_evidence else execute_transaction fi } platform_observability_secrets_fixture_main() { local fixture_root=${1:-} [[ "${BASH_SOURCE[0]}" != "$0" ]] || plain_fail 'test fixture entrypoint must be sourced' [[ -n "$fixture_root" ]] || plain_fail 'test fixture root is required' shift TEST_MODE=true configure_test_boundaries "$fixture_root" observability_secrets_main "$@" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then reject_production_overrides observability_secrets_main "$@" fi