1827 lines
67 KiB
Bash
Executable File
1827 lines
67 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
# Secret payloads must never be exposed by a caller's xtrace setting.
|
|
set +x
|
|
|
|
readonly SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly EXPECTED_CONTEXT='default'
|
|
readonly NAMESPACE='observability'
|
|
readonly EVIDENCE_SCHEMA='platform-observability-recovery-evidence-v1'
|
|
readonly RISK_EVIDENCE_SCHEMA='platform-observability-slack-risk-acceptance-v1'
|
|
readonly RISK_APPROVAL_REF='2026-08-14-observability-slack-recovery-risk-acceptance-design'
|
|
readonly RISK_ACCEPTED_BY_UID='1000'
|
|
readonly EVIDENCE_MAX_AGE_SECONDS=2592000
|
|
readonly PRODUCTION_EVIDENCE_DIR='/etc/hyeonworks/platform/recovery-evidence'
|
|
readonly PRODUCTION_EVIDENCE_ANCHOR='/etc'
|
|
readonly PRODUCTION_ENCRYPTION_SCRIPT="$SCRIPT_ROOT/scripts/validate/k3s-secret-encryption.sh"
|
|
readonly PRODUCTION_RESTORE_SCRIPT="$SCRIPT_ROOT/scripts/validate/k3s-secret-encryption-restore-evidence.sh"
|
|
readonly VALIDATOR_PATH='/usr/sbin:/usr/bin:/sbin:/bin'
|
|
readonly VALIDATOR_HOME='/home/donghyeon'
|
|
readonly KUBECTL_REQUEST_TIMEOUT='5s'
|
|
KUBECTL_PROCESS_TIMEOUT='10s'
|
|
KUBECTL_PROXY_PROCESS_TIMEOUT='15s'
|
|
|
|
TEST_MODE=false
|
|
KUBECTL_BIN='kubectl'
|
|
SUDO_BIN='sudo'
|
|
ENCRYPTION_SCRIPT="$PRODUCTION_ENCRYPTION_SCRIPT"
|
|
RESTORE_SCRIPT="$PRODUCTION_RESTORE_SCRIPT"
|
|
EVIDENCE_DIR="$PRODUCTION_EVIDENCE_DIR"
|
|
EVIDENCE_HIERARCHY_PROFILE='production-hierarchy'
|
|
EVIDENCE_HIERARCHY_ANCHOR="$PRODUCTION_EVIDENCE_ANCHOR"
|
|
TEST_CONFIRMATIONS=''
|
|
TEST_NOW_UTC=''
|
|
MARKER_TEST_FAILURE_PHASE=''
|
|
|
|
execute_requested=false
|
|
grafana_requested=false
|
|
slack_requested=false
|
|
check_grafana_requested=false
|
|
check_slack_requested=false
|
|
accept_slack_risk_requested=false
|
|
check_slack_deployment_requested=false
|
|
grafana_user_file=''
|
|
grafana_password_file=''
|
|
slack_webhook_file=''
|
|
grafana_user_snapshot=''
|
|
grafana_password_snapshot=''
|
|
slack_webhook_snapshot=''
|
|
|
|
temporary_dir=''
|
|
transaction_active=false
|
|
manual_recovery_required=false
|
|
rollback_in_progress=false
|
|
exit_handler_running=false
|
|
declare -a OWNED_ASSETS=()
|
|
declare -a OWNED_NAMES=()
|
|
declare -a OWNED_UIDS=()
|
|
pending_asset=''
|
|
pending_name=''
|
|
pending_token=''
|
|
pending_manifest=''
|
|
pending_response=''
|
|
pending_marker_asset=''
|
|
declare -a WRITTEN_MARKERS=()
|
|
declare -A MARKER_HAD_PRIOR=()
|
|
declare -A MARKER_PRIOR_FILE=()
|
|
declare -A MARKER_DESIRED_FILE=()
|
|
declare -A MARKER_KIND=()
|
|
EVIDENCE_EXPECTED_UID=0
|
|
EVIDENCE_EXPECTED_GID=0
|
|
confirmation_index=0
|
|
declare -a confirmation_answers=()
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
bash scripts/bootstrap/create-observability-secrets.sh
|
|
bash scripts/bootstrap/create-observability-secrets.sh \
|
|
--execute --grafana-admin \
|
|
--grafana-admin-user-file /absolute/0600/file \
|
|
--grafana-admin-password-file /absolute/0600/file
|
|
bash scripts/bootstrap/create-observability-secrets.sh \
|
|
--execute --slack-webhook \
|
|
--slack-webhook-file /absolute/0600/file
|
|
bash scripts/bootstrap/create-observability-secrets.sh \
|
|
--check-grafana-recovery-evidence
|
|
bash scripts/bootstrap/create-observability-secrets.sh \
|
|
--check-slack-recovery-evidence
|
|
bash scripts/bootstrap/create-observability-secrets.sh \
|
|
--check-slack-deployment-evidence
|
|
|
|
With no arguments, prints a source-only plan and performs no system access.
|
|
Execute mode creates only absent selected Secrets, reuses exact existing
|
|
contracts, and never rotates or prints payloads. Both asset flags form one
|
|
create-only transaction.
|
|
USAGE
|
|
}
|
|
|
|
usage_fail() {
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
usage >&2
|
|
exit 2
|
|
}
|
|
|
|
emit_manual_recovery() {
|
|
[[ "$manual_recovery_required" == false ]] || return 0
|
|
manual_recovery_required=true
|
|
printf 'MANUAL_RECOVERY_REQUIRED=YES\n' >&2
|
|
}
|
|
|
|
plain_fail() {
|
|
if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then
|
|
transaction_fail "$*"
|
|
fi
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
safe_temporary_path() {
|
|
local path=$1
|
|
[[ -n "$temporary_dir" && "$temporary_dir" == /tmp/platform-observability-secrets.?????? ]] || return 1
|
|
[[ "$path" == "$temporary_dir" || "$path" == "$temporary_dir"/* ]]
|
|
}
|
|
|
|
cleanup() {
|
|
trap - EXIT
|
|
if [[ -n "$temporary_dir" ]] && safe_temporary_path "$temporary_dir"; then
|
|
rm -rf -- "$temporary_dir"
|
|
fi
|
|
}
|
|
|
|
on_process_exit() {
|
|
local rc=$1 ambiguous=false classification
|
|
[[ "$exit_handler_running" == false ]] || exit "$rc"
|
|
exit_handler_running=true
|
|
trap - EXIT ERR HUP INT TERM
|
|
set +e
|
|
if [[ "$transaction_active" == true && "$rollback_in_progress" == false ]]; then
|
|
if [[ -n "$pending_marker_asset" ]]; then
|
|
classify_pending_marker
|
|
classification=$?
|
|
[[ "$classification" != 2 ]] || ambiguous=true
|
|
fi
|
|
if [[ -n "$pending_asset" ]]; then
|
|
classify_pending_create
|
|
classification=$?
|
|
[[ "$classification" == 0 || "$classification" == 1 ]] || ambiguous=true
|
|
fi
|
|
[[ "$ambiguous" != true ]] || emit_manual_recovery
|
|
rollback_transaction || true
|
|
transaction_active=false
|
|
printf 'ERROR: transaction exited before a confirmed terminal state\n' >&2
|
|
(( rc != 0 )) || rc=1
|
|
fi
|
|
cleanup
|
|
exit "$rc"
|
|
}
|
|
trap 'on_process_exit "$?"' EXIT
|
|
|
|
reject_production_overrides() {
|
|
local variable
|
|
local -a forbidden=(
|
|
PLATFORM_OBSERVABILITY_SECRETS_TEST_MODE
|
|
PLATFORM_OBSERVABILITY_SECRETS_KUBECTL_BIN
|
|
PLATFORM_OBSERVABILITY_SECRETS_SUDO_BIN
|
|
PLATFORM_OBSERVABILITY_SECRETS_ENCRYPTION_SCRIPT
|
|
PLATFORM_OBSERVABILITY_SECRETS_RESTORE_SCRIPT
|
|
PLATFORM_OBSERVABILITY_SECRETS_EVIDENCE_DIR
|
|
PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS
|
|
PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC
|
|
PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE
|
|
)
|
|
for variable in "${forbidden[@]}"; do
|
|
[[ ! -v "$variable" ]] || plain_fail "production boundary override is forbidden: $variable"
|
|
done
|
|
}
|
|
|
|
attest_test_executable() {
|
|
local path=$1 metadata
|
|
[[ -f "$path" && ! -L "$path" && -O "$path" && -x "$path" ]] || return 1
|
|
metadata="$(stat --format='%u:%a:%h:%F' -- "$path")" || return 1
|
|
[[ "$metadata" == "${EUID}:755:1:regular file" ]]
|
|
}
|
|
|
|
configure_test_boundaries() {
|
|
local fixture_root=$1 fixture_metadata physical_root
|
|
[[ "$fixture_root" =~ ^/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+$ ]] ||
|
|
plain_fail 'test fixture root has an invalid shape'
|
|
[[ -d "$fixture_root" && ! -L "$fixture_root" && -O "$fixture_root" ]] ||
|
|
plain_fail 'test fixture root is not an owned non-symlink directory'
|
|
physical_root="$(cd -- "$fixture_root" && pwd -P)" || plain_fail 'test fixture root cannot be resolved'
|
|
[[ "$physical_root" == "$fixture_root" ]] || plain_fail 'test fixture root contains a symlink boundary'
|
|
fixture_metadata="$(stat --format='%u:%a:%F' -- "$fixture_root")" ||
|
|
plain_fail 'test fixture root cannot be attested'
|
|
[[ "$fixture_metadata" == "${EUID}:700:directory" ]] ||
|
|
plain_fail 'test fixture root metadata is invalid'
|
|
[[ "$(cd -- "$fixture_root/bin" && pwd -P)" == "$fixture_root/bin" ]] ||
|
|
plain_fail 'test binary directory contains a symlink boundary'
|
|
[[ "$(stat --format='%u:%a:%F' -- "$fixture_root/bin")" == "${EUID}:755:directory" ]] ||
|
|
plain_fail 'test binary directory metadata is invalid'
|
|
|
|
KUBECTL_BIN="$fixture_root/bin/kubectl"
|
|
SUDO_BIN="$fixture_root/bin/sudo"
|
|
ENCRYPTION_SCRIPT="$fixture_root/bin/encryption"
|
|
RESTORE_SCRIPT="$fixture_root/bin/restore"
|
|
EVIDENCE_DIR="$fixture_root/evidence-parent/platform/recovery-evidence"
|
|
EVIDENCE_HIERARCHY_PROFILE='fixture-hierarchy'
|
|
EVIDENCE_HIERARCHY_ANCHOR="$fixture_root"
|
|
TEST_CONFIRMATIONS="${PLATFORM_OBSERVABILITY_SECRETS_CONFIRMATIONS:-}"
|
|
TEST_NOW_UTC="${PLATFORM_OBSERVABILITY_SECRETS_NOW_UTC:-}"
|
|
MARKER_TEST_FAILURE_PHASE="${PLATFORM_OBSERVABILITY_SECRETS_MARKER_FAILURE_PHASE:-}"
|
|
KUBECTL_PROCESS_TIMEOUT='2s'
|
|
KUBECTL_PROXY_PROCESS_TIMEOUT='5s'
|
|
EVIDENCE_EXPECTED_UID=$EUID
|
|
EVIDENCE_EXPECTED_GID="$(/usr/bin/id -g)"
|
|
|
|
[[ "$SUDO_BIN" != /usr/bin/sudo && "$SUDO_BIN" != /bin/sudo ]] ||
|
|
plain_fail 'real sudo is forbidden in the isolated test entrypoint'
|
|
attest_test_executable "$KUBECTL_BIN" || plain_fail 'test kubectl boundary is not attested'
|
|
attest_test_executable "$SUDO_BIN" || plain_fail 'test sudo boundary is not attested'
|
|
attest_test_executable "$ENCRYPTION_SCRIPT" || plain_fail 'test encryption boundary is not attested'
|
|
attest_test_executable "$RESTORE_SCRIPT" || plain_fail 'test restore boundary is not attested'
|
|
[[ "$TEST_NOW_UTC" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] ||
|
|
plain_fail 'test clock is invalid'
|
|
[[ "$MARKER_TEST_FAILURE_PHASE" == '' || "$MARKER_TEST_FAILURE_PHASE" == first-write ||
|
|
"$MARKER_TEST_FAILURE_PHASE" == pre-rename ]] ||
|
|
plain_fail 'test marker failure phase is invalid'
|
|
mapfile -t confirmation_answers <<<"$TEST_CONFIRMATIONS"
|
|
}
|
|
|
|
parse_arguments() {
|
|
local seen_execute=false seen_grafana=false seen_slack=false
|
|
local seen_user_file=false seen_password_file=false seen_slack_file=false
|
|
local seen_check_grafana=false seen_check_slack=false seen_accept_slack_risk=false
|
|
local seen_check_slack_deployment=false
|
|
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--execute)
|
|
[[ "$seen_execute" == false ]] || usage_fail 'duplicate --execute'
|
|
seen_execute=true; execute_requested=true; shift
|
|
;;
|
|
--grafana-admin)
|
|
[[ "$seen_grafana" == false ]] || usage_fail 'duplicate --grafana-admin'
|
|
seen_grafana=true; grafana_requested=true; shift
|
|
;;
|
|
--slack-webhook)
|
|
[[ "$seen_slack" == false ]] || usage_fail 'duplicate --slack-webhook'
|
|
seen_slack=true; slack_requested=true; shift
|
|
;;
|
|
--grafana-admin-user-file)
|
|
[[ "$seen_user_file" == false && $# -ge 2 ]] || usage_fail 'invalid --grafana-admin-user-file'
|
|
seen_user_file=true; grafana_user_file=$2; shift 2
|
|
;;
|
|
--grafana-admin-password-file)
|
|
[[ "$seen_password_file" == false && $# -ge 2 ]] || usage_fail 'invalid --grafana-admin-password-file'
|
|
seen_password_file=true; grafana_password_file=$2; shift 2
|
|
;;
|
|
--slack-webhook-file)
|
|
[[ "$seen_slack_file" == false && $# -ge 2 ]] || usage_fail 'invalid --slack-webhook-file'
|
|
seen_slack_file=true; slack_webhook_file=$2; shift 2
|
|
;;
|
|
--check-grafana-recovery-evidence)
|
|
[[ "$seen_check_grafana" == false ]] || usage_fail 'duplicate Grafana recovery check'
|
|
seen_check_grafana=true; check_grafana_requested=true; shift
|
|
;;
|
|
--check-slack-recovery-evidence)
|
|
[[ "$seen_check_slack" == false ]] || usage_fail 'duplicate Slack recovery check'
|
|
seen_check_slack=true; check_slack_requested=true; shift
|
|
;;
|
|
--accept-no-off-host-slack-recovery)
|
|
[[ "$seen_accept_slack_risk" == false ]] || usage_fail 'duplicate Slack risk acceptance'
|
|
seen_accept_slack_risk=true; accept_slack_risk_requested=true; shift
|
|
;;
|
|
--check-slack-deployment-evidence)
|
|
[[ "$seen_check_slack_deployment" == false ]] || usage_fail 'duplicate Slack deployment check'
|
|
seen_check_slack_deployment=true; check_slack_deployment_requested=true; shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
usage_fail "unsupported argument: $1"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$check_grafana_requested" == true || "$check_slack_requested" == true ||
|
|
"$check_slack_deployment_requested" == true ]]; then
|
|
if [[ "$check_grafana_requested" == true ]]; then
|
|
[[ "$check_slack_requested" == false && "$check_slack_deployment_requested" == false ]] ||
|
|
usage_fail 'select exactly one recovery evidence check'
|
|
elif [[ "$check_slack_requested" == true ]]; then
|
|
[[ "$check_slack_deployment_requested" == false ]] ||
|
|
usage_fail 'select exactly one recovery evidence check'
|
|
fi
|
|
[[ "$execute_requested" == false && "$grafana_requested" == false && "$slack_requested" == false ]] ||
|
|
usage_fail 'recovery evidence checks cannot be mixed with execute mode'
|
|
[[ "$accept_slack_risk_requested" == false ]] ||
|
|
usage_fail 'Slack risk acceptance cannot be mixed with evidence checks'
|
|
[[ "$seen_user_file" == false && "$seen_password_file" == false && "$seen_slack_file" == false ]] ||
|
|
usage_fail 'recovery evidence checks do not accept payload files'
|
|
return
|
|
fi
|
|
|
|
if [[ "$execute_requested" == false ]]; then
|
|
[[ "$grafana_requested" == false && "$slack_requested" == false ]] ||
|
|
usage_fail 'asset selection requires --execute'
|
|
[[ "$seen_user_file" == false && "$seen_password_file" == false && "$seen_slack_file" == false ]] ||
|
|
usage_fail 'payload files require --execute and an asset selection'
|
|
[[ "$accept_slack_risk_requested" == false ]] ||
|
|
usage_fail 'Slack risk acceptance requires the Slack execute interface'
|
|
return
|
|
fi
|
|
|
|
[[ "$grafana_requested" == true || "$slack_requested" == true ]] ||
|
|
usage_fail '--execute requires at least one asset selection'
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
[[ "$seen_user_file" == true && "$seen_password_file" == true ]] ||
|
|
usage_fail '--grafana-admin requires both Grafana input files'
|
|
[[ "$grafana_user_file" == /* && "$grafana_password_file" == /* ]] ||
|
|
usage_fail 'Grafana input files must be absolute paths'
|
|
else
|
|
[[ "$seen_user_file" == false && "$seen_password_file" == false ]] ||
|
|
usage_fail 'Grafana input files require --grafana-admin'
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
[[ "$seen_slack_file" == true ]] || usage_fail '--slack-webhook requires its input file'
|
|
[[ "$slack_webhook_file" == /* ]] || usage_fail 'Slack webhook file must be an absolute path'
|
|
else
|
|
[[ "$seen_slack_file" == false ]] || usage_fail 'Slack webhook file requires --slack-webhook'
|
|
fi
|
|
if [[ "$accept_slack_risk_requested" == true ]]; then
|
|
[[ "$slack_requested" == true && "$grafana_requested" == false ]] ||
|
|
usage_fail 'Slack risk acceptance is only available for the Slack-only execute interface'
|
|
fi
|
|
}
|
|
|
|
require_commands() {
|
|
local command_name
|
|
for command_name in bash base64 cmp curl date install kubectl mktemp python3 sort stat sudo timeout wc; do
|
|
if [[ "$command_name" == kubectl ]]; then
|
|
[[ -x "$KUBECTL_BIN" ]] || command -v "$KUBECTL_BIN" >/dev/null 2>&1 ||
|
|
plain_fail 'kubectl is required'
|
|
elif [[ "$command_name" == sudo ]]; then
|
|
[[ -x "$SUDO_BIN" ]] || command -v "$SUDO_BIN" >/dev/null 2>&1 ||
|
|
plain_fail 'sudo is required'
|
|
else
|
|
command -v "$command_name" >/dev/null 2>&1 || plain_fail "$command_name is required"
|
|
fi
|
|
done
|
|
[[ -f "$ENCRYPTION_SCRIPT" && ! -L "$ENCRYPTION_SCRIPT" && -r "$ENCRYPTION_SCRIPT" ]] ||
|
|
plain_fail 'k3s encryption validator is unavailable'
|
|
[[ -f "$RESTORE_SCRIPT" && ! -L "$RESTORE_SCRIPT" && -r "$RESTORE_SCRIPT" ]] ||
|
|
plain_fail 'k3s restore evidence validator is unavailable'
|
|
}
|
|
|
|
make_temporary_dir() {
|
|
umask 077
|
|
temporary_dir="$(mktemp -d /tmp/platform-observability-secrets.XXXXXX)" ||
|
|
plain_fail 'could not create private temporary directory'
|
|
[[ "$temporary_dir" == /tmp/platform-observability-secrets.?????? && ! -L "$temporary_dir" ]] ||
|
|
plain_fail 'private temporary directory boundary is invalid'
|
|
chmod 0700 -- "$temporary_dir"
|
|
}
|
|
|
|
validate_private_input_metadata() {
|
|
local file=$1 description=$2 metadata
|
|
[[ "$file" == /* && -f "$file" && ! -L "$file" && -O "$file" && -r "$file" && -s "$file" ]] ||
|
|
plain_fail "$description must be a readable non-empty current-user-owned non-symlink regular file"
|
|
metadata="$(stat --format='%u:%a:%h:%F' -- "$file")" || plain_fail "cannot stat $description"
|
|
[[ "$metadata" == "${EUID}:600:1:regular file" ]] ||
|
|
plain_fail "$description must be current-user-owned mode 0600 with one hard link"
|
|
}
|
|
|
|
validate_selected_input_metadata() {
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
validate_private_input_metadata "$grafana_user_file" 'Grafana admin user file'
|
|
validate_private_input_metadata "$grafana_password_file" 'Grafana admin password file'
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
validate_private_input_metadata "$slack_webhook_file" 'Slack webhook file'
|
|
fi
|
|
}
|
|
|
|
snapshot_private_input() {
|
|
local source=$1 destination=$2 description=$3
|
|
python3 -I -S - "$source" "$destination" <<'PY' ||
|
|
import os, stat, sys
|
|
source, destination = sys.argv[1:]
|
|
required = ("O_NOFOLLOW", "O_CLOEXEC")
|
|
if any(not hasattr(os, name) for name in required):
|
|
raise SystemExit(1)
|
|
source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
|
try:
|
|
before = os.fstat(source_fd)
|
|
if (
|
|
not stat.S_ISREG(before.st_mode)
|
|
or stat.S_IMODE(before.st_mode) != 0o600
|
|
or before.st_uid != os.geteuid()
|
|
or before.st_size <= 0
|
|
or before.st_nlink != 1
|
|
):
|
|
raise SystemExit(1)
|
|
chunks = []
|
|
while True:
|
|
chunk = os.read(source_fd, 65536)
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
after = os.fstat(source_fd)
|
|
identity_before = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns)
|
|
identity_after = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns)
|
|
if identity_before != identity_after:
|
|
raise SystemExit(1)
|
|
payload = b"".join(chunks)
|
|
if len(payload) != before.st_size:
|
|
raise SystemExit(1)
|
|
finally:
|
|
os.close(source_fd)
|
|
destination_fd = os.open(
|
|
destination,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC,
|
|
0o600,
|
|
)
|
|
try:
|
|
view = memoryview(payload)
|
|
while view:
|
|
written = os.write(destination_fd, view)
|
|
if written <= 0:
|
|
raise SystemExit(1)
|
|
view = view[written:]
|
|
finally:
|
|
os.close(destination_fd)
|
|
PY
|
|
plain_fail "could not take a stable private snapshot of $description"
|
|
}
|
|
|
|
capture_selected_inputs() {
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
grafana_user_snapshot="$temporary_dir/input-grafana-user"
|
|
grafana_password_snapshot="$temporary_dir/input-grafana-password"
|
|
snapshot_private_input "$grafana_user_file" "$grafana_user_snapshot" 'Grafana admin user file'
|
|
snapshot_private_input "$grafana_password_file" "$grafana_password_snapshot" 'Grafana admin password file'
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
slack_webhook_snapshot="$temporary_dir/input-slack-webhook"
|
|
snapshot_private_input "$slack_webhook_file" "$slack_webhook_snapshot" 'Slack webhook file'
|
|
fi
|
|
}
|
|
|
|
recheck_selected_inputs() {
|
|
local second
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
second="$temporary_dir/recheck-grafana-user"
|
|
snapshot_private_input "$grafana_user_file" "$second" 'Grafana admin user file'
|
|
cmp --silent -- "$grafana_user_snapshot" "$second" ||
|
|
plain_fail 'Grafana admin user input changed during confirmation'
|
|
second="$temporary_dir/recheck-grafana-password"
|
|
snapshot_private_input "$grafana_password_file" "$second" 'Grafana admin password file'
|
|
cmp --silent -- "$grafana_password_snapshot" "$second" ||
|
|
plain_fail 'Grafana admin password input changed during confirmation'
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
second="$temporary_dir/recheck-slack-webhook"
|
|
snapshot_private_input "$slack_webhook_file" "$second" 'Slack webhook file'
|
|
cmp --silent -- "$slack_webhook_snapshot" "$second" ||
|
|
plain_fail 'Slack webhook input changed during confirmation'
|
|
fi
|
|
}
|
|
|
|
validate_admin_user_content() {
|
|
python3 -I -S - "$grafana_user_snapshot" <<'PY'
|
|
import pathlib, sys
|
|
data = pathlib.Path(sys.argv[1]).read_bytes()
|
|
if not data or any(byte < 0x20 or byte > 0x7e for byte in data):
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_admin_password_content() {
|
|
python3 -I -S - "$grafana_password_snapshot" <<'PY'
|
|
import pathlib, sys
|
|
data = pathlib.Path(sys.argv[1]).read_bytes()
|
|
if len(data) < 20 or b"\x00" in data or b"\r" in data or b"\n" in data:
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_slack_webhook_content() {
|
|
python3 -I -S - "$slack_webhook_snapshot" <<'PY'
|
|
import pathlib, re, sys
|
|
data = pathlib.Path(sys.argv[1]).read_bytes()
|
|
try:
|
|
value = data.decode("ascii")
|
|
except UnicodeDecodeError:
|
|
raise SystemExit(1)
|
|
pattern = r"https://hooks[.]slack[.]com/services/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+"
|
|
if re.fullmatch(pattern, value) is None:
|
|
raise SystemExit(1)
|
|
PY
|
|
}
|
|
|
|
validate_selected_input_content() {
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
validate_admin_user_content || plain_fail 'Grafana admin user must be one non-empty printable ASCII line'
|
|
validate_admin_password_content || plain_fail 'Grafana admin password must be at least 20 bytes with no NUL or line break'
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
validate_slack_webhook_content ||
|
|
plain_fail 'Slack webhook must have the exact hooks.slack.com/services prefix and three path components'
|
|
fi
|
|
}
|
|
|
|
run_common_gates() {
|
|
local -a safe_environment=(
|
|
/usr/bin/env -i
|
|
"PATH=$VALIDATOR_PATH"
|
|
"HOME=$VALIDATOR_HOME"
|
|
)
|
|
if [[ "$TEST_MODE" == true ]]; then
|
|
safe_environment+=(
|
|
"PLATFORM_TEST_COMMAND_LOG=${PLATFORM_TEST_COMMAND_LOG:-}"
|
|
"PLATFORM_TEST_ENCRYPTION_COUNT=${PLATFORM_TEST_ENCRYPTION_COUNT:-}"
|
|
"PLATFORM_TEST_RESTORE_COUNT=${PLATFORM_TEST_RESTORE_COUNT:-}"
|
|
"PLATFORM_TEST_ENCRYPTION_FAIL_AT=${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}"
|
|
"PLATFORM_TEST_RESTORE_FAIL_AT=${PLATFORM_TEST_RESTORE_FAIL_AT:-0}"
|
|
"PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE=${PLATFORM_TEST_MUTATE_INPUT_ON_LAST_GATE:-}"
|
|
"PLATFORM_TEST_VALIDATOR_ENV_LOG=${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}"
|
|
"PLATFORM_TEST_STATE=${PLATFORM_TEST_STATE:-}"
|
|
)
|
|
fi
|
|
"${safe_environment[@]}" /usr/bin/bash "$ENCRYPTION_SCRIPT" --expect-reencrypted >/dev/null ||
|
|
plain_fail 'k3s Secret encryption is not fully re-encrypted'
|
|
"${safe_environment[@]}" /usr/bin/bash "$RESTORE_SCRIPT" --check >/dev/null ||
|
|
plain_fail 'current k3s restore evidence is unavailable'
|
|
}
|
|
|
|
require_context_and_namespace() {
|
|
local context
|
|
context="$(kubectl_bounded config current-context)" || plain_fail 'cannot read current Kubernetes context'
|
|
[[ "$context" == "$EXPECTED_CONTEXT" ]] || plain_fail 'current Kubernetes context is not default'
|
|
kubectl_bounded get namespace "$NAMESPACE" >/dev/null ||
|
|
plain_fail 'observability namespace is absent'
|
|
}
|
|
|
|
require_current_context() {
|
|
local context
|
|
context="$(kubectl_bounded config current-context)" ||
|
|
plain_fail 'cannot read current Kubernetes context'
|
|
[[ "$context" == "$EXPECTED_CONTEXT" ]] ||
|
|
plain_fail 'current Kubernetes context is not default'
|
|
}
|
|
|
|
kubectl_bounded() {
|
|
local process_timeout=$KUBECTL_PROCESS_TIMEOUT
|
|
[[ "${1:-}" != proxy ]] || process_timeout=$KUBECTL_PROXY_PROCESS_TIMEOUT
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "$process_timeout" \
|
|
"$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" "$@"
|
|
}
|
|
|
|
secret_exists() {
|
|
local name=$1 result
|
|
result="$(kubectl_bounded --namespace "$NAMESPACE" get secret "$name" \
|
|
--ignore-not-found --output=name)" || return 2
|
|
if [[ -z "$result" ]]; then return 1; fi
|
|
[[ "$result" == "secret/$name" ]] || return 2
|
|
return 0
|
|
}
|
|
|
|
validate_secret_json() {
|
|
local json_file=$1 name=$2 expected_keys=$3 decoded_prefix=$4 expected_token=${5:-}
|
|
python3 -I -S - "$json_file" "$name" "$expected_keys" "$decoded_prefix" "$expected_token" <<'PY'
|
|
import base64, binascii, json, os, pathlib, re, sys
|
|
json_file, expected_name, expected_keys_raw, decoded_prefix, expected_token = sys.argv[1:]
|
|
try:
|
|
item = json.loads(pathlib.Path(json_file).read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(1)
|
|
expected_keys = expected_keys_raw.split(",")
|
|
metadata = item.get("metadata") or {}
|
|
data = item.get("data")
|
|
uid = metadata.get("uid")
|
|
annotations = metadata.get("annotations") or {}
|
|
if (
|
|
item.get("apiVersion") != "v1"
|
|
or item.get("kind") != "Secret"
|
|
or item.get("type") != "Opaque"
|
|
or metadata.get("namespace") != "observability"
|
|
or metadata.get("name") != expected_name
|
|
or not isinstance(data, dict)
|
|
or sorted(data) != sorted(expected_keys)
|
|
or not isinstance(uid, str)
|
|
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", uid) is None
|
|
):
|
|
raise SystemExit(1)
|
|
if expected_token and annotations.get("observability.hyeonworks.com/create-transaction") != expected_token:
|
|
raise SystemExit(1)
|
|
for key in expected_keys:
|
|
encoded = data.get(key)
|
|
if not isinstance(encoded, str):
|
|
raise SystemExit(1)
|
|
try:
|
|
decoded = base64.b64decode(encoded, validate=True)
|
|
except (binascii.Error, ValueError):
|
|
raise SystemExit(1)
|
|
destination = pathlib.Path(decoded_prefix + "." + key)
|
|
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
descriptor = os.open(destination, flags, 0o600)
|
|
try:
|
|
os.write(descriptor, decoded)
|
|
finally:
|
|
os.close(descriptor)
|
|
print(uid, end="")
|
|
PY
|
|
}
|
|
|
|
new_transaction_token() {
|
|
python3 -I -S -c 'import secrets; print(secrets.token_hex(16))'
|
|
}
|
|
|
|
build_secret_manifest() {
|
|
local asset=$1 token=$2 destination=$3 name
|
|
case "$asset" in
|
|
grafana) name='grafana-admin' ;;
|
|
slack) name='alertmanager-slack-webhook' ;;
|
|
*) return 1 ;;
|
|
esac
|
|
python3 -I -S - "$asset" "$name" "$token" "$destination" \
|
|
"$grafana_user_snapshot" "$grafana_password_snapshot" "$slack_webhook_snapshot" <<'PY'
|
|
import base64, json, os, pathlib, re, sys
|
|
asset, name, token, destination, user_file, password_file, slack_file = sys.argv[1:]
|
|
if re.fullmatch(r"[0-9a-f]{32}", token) is None:
|
|
raise SystemExit(1)
|
|
if asset == "grafana":
|
|
sources = {"admin-user": user_file, "admin-password": password_file}
|
|
elif asset == "slack":
|
|
sources = {"url": slack_file}
|
|
else:
|
|
raise SystemExit(1)
|
|
data = {
|
|
key: base64.b64encode(pathlib.Path(path).read_bytes()).decode("ascii")
|
|
for key, path in sources.items()
|
|
}
|
|
item = {
|
|
"apiVersion": "v1",
|
|
"kind": "Secret",
|
|
"type": "Opaque",
|
|
"metadata": {
|
|
"namespace": "observability",
|
|
"name": name,
|
|
"annotations": {
|
|
"observability.hyeonworks.com/create-transaction": token,
|
|
},
|
|
},
|
|
"data": data,
|
|
}
|
|
descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, 0o600)
|
|
try:
|
|
encoded = json.dumps(item, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
|
view = memoryview(encoded)
|
|
while view:
|
|
written = os.write(descriptor, view)
|
|
if written <= 0:
|
|
raise SystemExit(1)
|
|
view = view[written:]
|
|
finally:
|
|
os.close(descriptor)
|
|
PY
|
|
}
|
|
|
|
inspect_contract() {
|
|
local asset=$1 phase=$2 state_var=$3 uid_var=$4
|
|
local name expected_keys json_file decoded_prefix uid
|
|
case "$asset" in
|
|
grafana)
|
|
name='grafana-admin'; expected_keys='admin-user,admin-password'
|
|
;;
|
|
slack)
|
|
name='alertmanager-slack-webhook'; expected_keys='url'
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
|
|
if secret_exists "$name"; then
|
|
json_file="$temporary_dir/${phase}-${asset}.json"
|
|
decoded_prefix="$temporary_dir/${phase}-${asset}"
|
|
kubectl_bounded --namespace "$NAMESPACE" get secret "$name" --output=json >"$json_file" ||
|
|
plain_fail "cannot read existing $asset Secret contract"
|
|
chmod 0600 -- "$json_file"
|
|
uid="$(validate_secret_json "$json_file" "$name" "$expected_keys" "$decoded_prefix")" ||
|
|
plain_fail "existing $asset Secret has wrong type, keys, metadata, or encoding"
|
|
case "$asset" in
|
|
grafana)
|
|
cmp --silent -- "$grafana_user_snapshot" "$decoded_prefix.admin-user" ||
|
|
plain_fail 'existing Grafana admin user differs; rotation was refused'
|
|
cmp --silent -- "$grafana_password_snapshot" "$decoded_prefix.admin-password" ||
|
|
plain_fail 'existing Grafana admin password differs; rotation was refused'
|
|
;;
|
|
slack)
|
|
cmp --silent -- "$slack_webhook_snapshot" "$decoded_prefix.url" ||
|
|
plain_fail 'existing Slack webhook differs; rotation was refused'
|
|
;;
|
|
esac
|
|
printf -v "$state_var" '%s' existing
|
|
printf -v "$uid_var" '%s' "$uid"
|
|
else
|
|
case $? in
|
|
1)
|
|
printf -v "$state_var" '%s' absent
|
|
printf -v "$uid_var" '%s' ''
|
|
;;
|
|
*) plain_fail "cannot determine $asset Secret presence" ;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
read_confirmation() {
|
|
local prompt=$1 expected=$2 answer=''
|
|
printf '%s' "$prompt" >&2
|
|
if [[ "$TEST_MODE" == true ]]; then
|
|
(( confirmation_index < ${#confirmation_answers[@]} )) || plain_fail 'confirmation input is missing'
|
|
answer="${confirmation_answers[$confirmation_index]}"
|
|
confirmation_index=$((confirmation_index + 1))
|
|
else
|
|
[[ -t 0 ]] || plain_fail 'an interactive terminal is required for execute confirmation'
|
|
IFS= read -r answer
|
|
fi
|
|
[[ "$answer" == "$expected" ]] || plain_fail 'confirmation was not exact'
|
|
}
|
|
|
|
now_utc() {
|
|
if [[ "$TEST_MODE" == true ]]; then printf '%s\n' "$TEST_NOW_UTC"; else date -u +%Y-%m-%dT%H:%M:%SZ; fi
|
|
}
|
|
|
|
timestamp_epoch() {
|
|
local timestamp=$1 normalized
|
|
normalized="$(date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1
|
|
[[ "$normalized" == "$timestamp" ]] || return 1
|
|
date -u -d "$timestamp" +%s
|
|
}
|
|
|
|
now_epoch() {
|
|
timestamp_epoch "$(now_utc)"
|
|
}
|
|
|
|
evidence_asset_fields() {
|
|
local asset=$1 path_var=$2 resource_var=$3
|
|
case "$asset" in
|
|
grafana)
|
|
printf -v "$path_var" '%s' "$EVIDENCE_DIR/grafana-local.env"
|
|
printf -v "$resource_var" '%s' 'observability/grafana-admin'
|
|
;;
|
|
slack)
|
|
printf -v "$path_var" '%s' "$EVIDENCE_DIR/slack.env"
|
|
printf -v "$resource_var" '%s' 'observability/alertmanager-slack-webhook'
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
sudo_refresh() {
|
|
"$SUDO_BIN" -v || plain_fail 'sudo credential refresh failed'
|
|
"$SUDO_BIN" -n /usr/bin/true || plain_fail 'narrow non-interactive sudo is unavailable'
|
|
}
|
|
|
|
validate_recovery_evidence_stream() {
|
|
local file=$1 expected_resource=$2 timestamp_var=$3
|
|
local -a lines=()
|
|
local parsed_timestamp marker_epoch current_epoch
|
|
mapfile -t lines <"$file"
|
|
[[ "$(wc -l <"$file" | tr -d '[:space:]')" == 4 && ${#lines[@]} -eq 4 ]] || return 1
|
|
[[ "${lines[0]}" == "schema=$EVIDENCE_SCHEMA" ]] || return 1
|
|
[[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1
|
|
[[ "${lines[2]}" == "resource=$expected_resource" ]] || return 1
|
|
[[ "${lines[3]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1
|
|
parsed_timestamp="${BASH_REMATCH[1]}"
|
|
timestamp_epoch "$parsed_timestamp" >/dev/null || return 1
|
|
printf -v "$timestamp_var" '%s' "$parsed_timestamp"
|
|
}
|
|
|
|
validate_slack_risk_evidence_stream() {
|
|
local file=$1 timestamp_var=$2
|
|
local -a lines=()
|
|
local parsed_timestamp marker_epoch current_epoch
|
|
mapfile -t lines <"$file"
|
|
[[ "$(wc -l <"$file" | tr -d '[:space:]')" == 8 && ${#lines[@]} -eq 8 ]] || return 1
|
|
[[ "${lines[0]}" == "schema=$RISK_EVIDENCE_SCHEMA" ]] || return 1
|
|
[[ "${lines[1]}" == "context=$EXPECTED_CONTEXT" ]] || return 1
|
|
[[ "${lines[2]}" == 'resource=observability/alertmanager-slack-webhook' ]] || return 1
|
|
[[ "${lines[3]}" == 'reason=off-host-slack-recovery-deferred' ]] || return 1
|
|
[[ "${lines[4]}" == "accepted_by_uid=$RISK_ACCEPTED_BY_UID" ]] || return 1
|
|
[[ "${lines[5]}" == 'approval_method=interactive-exact-confirmation' ]] || return 1
|
|
[[ "${lines[6]}" == "approval_ref=$RISK_APPROVAL_REF" ]] || return 1
|
|
[[ "${lines[7]}" =~ ^checked_at_utc=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$ ]] || return 1
|
|
parsed_timestamp="${BASH_REMATCH[1]}"
|
|
timestamp_epoch "$parsed_timestamp" >/dev/null || return 1
|
|
marker_epoch="$(timestamp_epoch "$parsed_timestamp")" || return 1
|
|
current_epoch="$(now_epoch)" || return 1
|
|
(( marker_epoch <= current_epoch )) || return 1
|
|
printf -v "$timestamp_var" '%s' "$parsed_timestamp"
|
|
}
|
|
|
|
classify_slack_evidence_stream() {
|
|
local file=$1 kind_var=$2 timestamp_var=$3 timestamp
|
|
if validate_recovery_evidence_stream "$file" 'observability/alertmanager-slack-webhook' timestamp; then
|
|
printf -v "$kind_var" '%s' RECOVERY
|
|
elif validate_slack_risk_evidence_stream "$file" timestamp; then
|
|
printf -v "$kind_var" '%s' RISK_ACCEPTED
|
|
else
|
|
return 1
|
|
fi
|
|
printf -v "$timestamp_var" '%s' "$timestamp"
|
|
}
|
|
|
|
validate_evidence_stream() {
|
|
validate_recovery_evidence_stream "$@"
|
|
}
|
|
|
|
root_marker_helper() {
|
|
local output=$1 failure_phase=''
|
|
shift
|
|
if [[ "${1:-}" == install && "$TEST_MODE" == true ]]; then
|
|
failure_phase=$MARKER_TEST_FAILURE_PHASE
|
|
fi
|
|
"$SUDO_BIN" -n /usr/bin/python3 -I -S - platform-observability-marker-helper "$@" \
|
|
"$failure_phase" >"$output" 2>/dev/null <<'PY'
|
|
import errno
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import stat
|
|
import sys
|
|
|
|
(
|
|
tag,
|
|
action,
|
|
directory,
|
|
basename,
|
|
expected_uid_raw,
|
|
expected_gid_raw,
|
|
source,
|
|
expected_kind,
|
|
expected_source,
|
|
source_uid_raw,
|
|
failure_phase,
|
|
) = sys.argv[1:]
|
|
if tag != "platform-observability-marker-helper":
|
|
raise SystemExit(2)
|
|
if action not in {"check-dir", "ensure-dir", "read", "install", "unlink-exact"}:
|
|
raise SystemExit(2)
|
|
if failure_phase not in {"", "first-write", "pre-rename"}:
|
|
raise SystemExit(2)
|
|
if failure_phase and action != "install":
|
|
raise SystemExit(2)
|
|
if not directory.startswith("/") or "\x00" in directory:
|
|
raise SystemExit(2)
|
|
expected_uid = int(expected_uid_raw)
|
|
expected_gid = int(expected_gid_raw)
|
|
source_uid = int(source_uid_raw)
|
|
if min(expected_uid, expected_gid, source_uid) < 0:
|
|
raise SystemExit(2)
|
|
if action in {"check-dir", "ensure-dir"}:
|
|
if basename not in {"production-hierarchy", "fixture-hierarchy"}:
|
|
raise SystemExit(2)
|
|
if source or expected_kind or expected_source or failure_phase:
|
|
raise SystemExit(2)
|
|
else:
|
|
if basename not in {"grafana-local.env", "slack.env"}:
|
|
raise SystemExit(2)
|
|
fixture_marker_directory = re.fullmatch(
|
|
r"/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+/evidence-parent/platform/recovery-evidence",
|
|
directory,
|
|
)
|
|
if directory == "/etc/hyeonworks/platform/recovery-evidence":
|
|
if expected_uid != 0 or expected_gid != 0:
|
|
raise SystemExit(2)
|
|
elif fixture_marker_directory is not None:
|
|
if expected_uid != source_uid:
|
|
raise SystemExit(2)
|
|
else:
|
|
raise SystemExit(2)
|
|
directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
|
|
|
|
|
def open_directory(path):
|
|
if path == "/etc/hyeonworks/platform/recovery-evidence":
|
|
profile = "production-hierarchy"
|
|
anchor = "/etc"
|
|
components = ("hyeonworks", "platform", "recovery-evidence")
|
|
else:
|
|
fixture_suffix = "/evidence-parent/platform/recovery-evidence"
|
|
if not path.endswith(fixture_suffix):
|
|
raise OSError(errno.EINVAL, "invalid marker directory")
|
|
profile = "fixture-hierarchy"
|
|
anchor = path[: -len(fixture_suffix)]
|
|
components = ("evidence-parent", "platform", "recovery-evidence")
|
|
|
|
descriptors = [open_hierarchy_anchor(anchor, profile)]
|
|
identities = []
|
|
try:
|
|
for component in components:
|
|
parent_descriptor = descriptors[-1]
|
|
following = os.open(component, directory_flags, dir_fd=parent_descriptor)
|
|
try:
|
|
identity = attest_managed_directory(following)
|
|
attest_managed_entry(parent_descriptor, component, identity)
|
|
except BaseException:
|
|
os.close(following)
|
|
raise
|
|
descriptors.append(following)
|
|
identities.append(identity)
|
|
for index, component in enumerate(components):
|
|
attest_managed_entry(descriptors[index], component, identities[index])
|
|
result = descriptors.pop()
|
|
for descriptor in reversed(descriptors):
|
|
os.close(descriptor)
|
|
return result
|
|
except BaseException:
|
|
for descriptor in reversed(descriptors):
|
|
os.close(descriptor)
|
|
raise
|
|
|
|
|
|
def require_managed_metadata(metadata):
|
|
if (
|
|
not stat.S_ISDIR(metadata.st_mode)
|
|
or stat.S_IMODE(metadata.st_mode) != 0o700
|
|
or metadata.st_uid != expected_uid
|
|
or metadata.st_gid != expected_gid
|
|
):
|
|
raise OSError(errno.EPERM, "managed directory metadata mismatch")
|
|
|
|
|
|
def attest_managed_directory(descriptor):
|
|
metadata = os.fstat(descriptor)
|
|
require_managed_metadata(metadata)
|
|
return metadata.st_dev, metadata.st_ino
|
|
|
|
|
|
def attest_managed_entry(parent_descriptor, component, expected_identity):
|
|
metadata = os.stat(component, dir_fd=parent_descriptor, follow_symlinks=False)
|
|
require_managed_metadata(metadata)
|
|
if (metadata.st_dev, metadata.st_ino) != expected_identity:
|
|
raise OSError(errno.ESTALE, "managed directory identity changed")
|
|
|
|
|
|
def open_hierarchy_anchor(path, profile):
|
|
descriptor = os.open(path, directory_flags)
|
|
try:
|
|
metadata = os.fstat(descriptor)
|
|
if not stat.S_ISDIR(metadata.st_mode):
|
|
raise OSError(errno.ENOTDIR, "hierarchy anchor is not a directory")
|
|
if profile == "production-hierarchy":
|
|
if metadata.st_uid != 0 or metadata.st_gid != 0:
|
|
raise OSError(errno.EPERM, "production hierarchy anchor metadata mismatch")
|
|
elif (
|
|
stat.S_IMODE(metadata.st_mode) != 0o700
|
|
or metadata.st_uid != expected_uid
|
|
or metadata.st_gid != expected_gid
|
|
):
|
|
raise OSError(errno.EPERM, "fixture hierarchy anchor metadata mismatch")
|
|
return descriptor
|
|
except BaseException:
|
|
os.close(descriptor)
|
|
raise
|
|
|
|
|
|
def manage_evidence_hierarchy(profile, anchor, create):
|
|
if profile == "production-hierarchy":
|
|
if anchor != "/etc" or expected_uid != 0 or expected_gid != 0:
|
|
raise OSError(errno.EINVAL, "invalid production hierarchy profile")
|
|
components = ("hyeonworks", "platform", "recovery-evidence")
|
|
elif profile == "fixture-hierarchy":
|
|
if (
|
|
expected_uid != source_uid
|
|
or re.fullmatch(
|
|
r"/tmp/platform-observability-secrets-test[.][A-Za-z0-9]{6}/[A-Za-z0-9._-]+",
|
|
anchor,
|
|
)
|
|
is None
|
|
):
|
|
raise OSError(errno.EINVAL, "invalid fixture hierarchy profile")
|
|
components = ("evidence-parent", "platform", "recovery-evidence")
|
|
else:
|
|
raise OSError(errno.EINVAL, "unknown hierarchy profile")
|
|
|
|
descriptors = [open_hierarchy_anchor(anchor, profile)]
|
|
identities = []
|
|
|
|
def revalidate_opened_prefix():
|
|
for index, opened_component in enumerate(components[: len(identities)]):
|
|
attest_managed_entry(descriptors[index], opened_component, identities[index])
|
|
|
|
try:
|
|
for component in components:
|
|
parent_descriptor = descriptors[-1]
|
|
needs_parent_fsync = False
|
|
try:
|
|
following = os.open(component, directory_flags, dir_fd=parent_descriptor)
|
|
except FileNotFoundError:
|
|
if not create:
|
|
revalidate_opened_prefix()
|
|
return 4
|
|
try:
|
|
previous_umask = os.umask(0)
|
|
try:
|
|
os.mkdir(component, 0o700, dir_fd=parent_descriptor)
|
|
finally:
|
|
os.umask(previous_umask)
|
|
except FileExistsError:
|
|
needs_parent_fsync = True
|
|
else:
|
|
needs_parent_fsync = True
|
|
following = os.open(component, directory_flags, dir_fd=parent_descriptor)
|
|
try:
|
|
identity = attest_managed_directory(following)
|
|
attest_managed_entry(parent_descriptor, component, identity)
|
|
if needs_parent_fsync:
|
|
os.fsync(parent_descriptor)
|
|
attest_managed_entry(parent_descriptor, component, identity)
|
|
except BaseException:
|
|
os.close(following)
|
|
raise
|
|
descriptors.append(following)
|
|
identities.append(identity)
|
|
revalidate_opened_prefix()
|
|
return 0
|
|
finally:
|
|
for descriptor in reversed(descriptors):
|
|
os.close(descriptor)
|
|
|
|
|
|
def read_open_file(directory_fd, name):
|
|
descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=directory_fd)
|
|
try:
|
|
before = os.fstat(descriptor)
|
|
if (
|
|
not stat.S_ISREG(before.st_mode)
|
|
or stat.S_IMODE(before.st_mode) != 0o600
|
|
or before.st_uid != expected_uid
|
|
or before.st_gid != expected_gid
|
|
or before.st_nlink != 1
|
|
):
|
|
raise OSError(errno.EPERM, "file metadata mismatch")
|
|
chunks = []
|
|
while True:
|
|
chunk = os.read(descriptor, 65536)
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
after = os.fstat(descriptor)
|
|
identity_before = (
|
|
before.st_dev,
|
|
before.st_ino,
|
|
before.st_size,
|
|
before.st_mtime_ns,
|
|
before.st_ctime_ns,
|
|
before.st_nlink,
|
|
)
|
|
identity_after = (
|
|
after.st_dev,
|
|
after.st_ino,
|
|
after.st_size,
|
|
after.st_mtime_ns,
|
|
after.st_ctime_ns,
|
|
after.st_nlink,
|
|
)
|
|
if identity_before != identity_after:
|
|
raise OSError(errno.ESTALE, "file changed while open")
|
|
payload = b"".join(chunks)
|
|
if len(payload) != before.st_size:
|
|
raise OSError(errno.EIO, "short read")
|
|
return payload
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def read_source(path):
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
|
try:
|
|
before = os.fstat(descriptor)
|
|
if (
|
|
not stat.S_ISREG(before.st_mode)
|
|
or stat.S_IMODE(before.st_mode) != 0o600
|
|
or before.st_uid != source_uid
|
|
or before.st_nlink != 1
|
|
):
|
|
raise OSError(errno.EPERM, "source metadata mismatch")
|
|
chunks = []
|
|
while True:
|
|
chunk = os.read(descriptor, 65536)
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
after = os.fstat(descriptor)
|
|
before_identity = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, before.st_nlink)
|
|
after_identity = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, after.st_nlink)
|
|
if before_identity != after_identity:
|
|
raise OSError(errno.ESTALE, "source changed")
|
|
payload = b"".join(chunks)
|
|
if len(payload) != before.st_size:
|
|
raise OSError(errno.EIO, "short source read")
|
|
return payload
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
if action in {"check-dir", "ensure-dir"}:
|
|
raise SystemExit(
|
|
manage_evidence_hierarchy(
|
|
basename,
|
|
directory,
|
|
action == "ensure-dir",
|
|
)
|
|
)
|
|
|
|
try:
|
|
directory_fd = open_directory(directory)
|
|
except FileNotFoundError:
|
|
if action == "read":
|
|
raise SystemExit(4)
|
|
raise
|
|
try:
|
|
if action == "read":
|
|
try:
|
|
payload = read_open_file(directory_fd, basename)
|
|
except FileNotFoundError:
|
|
raise SystemExit(4)
|
|
os.write(1, payload)
|
|
raise SystemExit(0)
|
|
|
|
desired = read_source(source)
|
|
if expected_kind == "absent":
|
|
try:
|
|
read_open_file(directory_fd, basename)
|
|
except FileNotFoundError:
|
|
pass
|
|
else:
|
|
raise OSError(errno.EEXIST, "expected absent marker")
|
|
elif expected_kind == "file":
|
|
if read_open_file(directory_fd, basename) != read_source(expected_source):
|
|
raise OSError(errno.ESTALE, "marker precondition mismatch")
|
|
else:
|
|
raise OSError(errno.EINVAL, "invalid marker precondition")
|
|
|
|
if action == "unlink-exact":
|
|
if read_open_file(directory_fd, basename) != desired:
|
|
raise OSError(errno.ESTALE, "unlink precondition mismatch")
|
|
os.unlink(basename, dir_fd=directory_fd)
|
|
os.fsync(directory_fd)
|
|
try:
|
|
os.stat(basename, dir_fd=directory_fd, follow_symlinks=False)
|
|
except FileNotFoundError:
|
|
raise SystemExit(0)
|
|
raise OSError(errno.EEXIST, "marker remained after unlink")
|
|
|
|
temporary_name = f".{basename}.new.{os.getpid():x}"
|
|
temporary_fd = os.open(
|
|
temporary_name,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
|
0o600,
|
|
dir_fd=directory_fd,
|
|
)
|
|
temporary_exists = True
|
|
try:
|
|
try:
|
|
os.fchown(temporary_fd, expected_uid, expected_gid)
|
|
os.fchmod(temporary_fd, 0o600)
|
|
view = memoryview(desired)
|
|
while view:
|
|
if failure_phase == "first-write":
|
|
written = 0
|
|
failure_phase = ""
|
|
else:
|
|
written = os.write(temporary_fd, view)
|
|
if written <= 0:
|
|
raise OSError(errno.EIO, "short marker write")
|
|
view = view[written:]
|
|
os.fsync(temporary_fd)
|
|
metadata = os.fstat(temporary_fd)
|
|
if metadata.st_nlink != 1:
|
|
raise OSError(errno.EPERM, "temporary marker link count mismatch")
|
|
if failure_phase == "pre-rename":
|
|
raise OSError(errno.EIO, "injected marker pre-rename failure")
|
|
finally:
|
|
os.close(temporary_fd)
|
|
os.replace(temporary_name, basename, src_dir_fd=directory_fd, dst_dir_fd=directory_fd)
|
|
temporary_exists = False
|
|
os.fsync(directory_fd)
|
|
if read_open_file(directory_fd, basename) != desired:
|
|
raise OSError(errno.EIO, "installed marker verification failed")
|
|
finally:
|
|
if temporary_exists:
|
|
try:
|
|
os.unlink(temporary_name, dir_fd=directory_fd)
|
|
except FileNotFoundError:
|
|
pass
|
|
finally:
|
|
os.close(directory_fd)
|
|
PY
|
|
}
|
|
|
|
root_marker_read() {
|
|
local asset=$1 destination=$2 path resource rc
|
|
evidence_asset_fields "$asset" path resource || return 1
|
|
if root_marker_helper "$destination" read "$EVIDENCE_DIR" "${path##*/}" \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID"; then
|
|
chmod 0600 -- "$destination"
|
|
return 0
|
|
else
|
|
rc=$?
|
|
rm -f -- "$destination"
|
|
return "$rc"
|
|
fi
|
|
}
|
|
|
|
read_root_evidence() {
|
|
local asset=$1 destination=$2 resource path timestamp
|
|
evidence_asset_fields "$asset" path resource || return 1
|
|
root_marker_read "$asset" "$destination" || return 1
|
|
validate_evidence_stream "$destination" "$resource" timestamp
|
|
}
|
|
|
|
check_recovery_evidence() {
|
|
local asset=$1 marker timestamp marker_epoch current_epoch age
|
|
require_current_context
|
|
sudo_refresh
|
|
marker="$temporary_dir/check-${asset}.env"
|
|
read_root_evidence "$asset" "$marker" || plain_fail "$asset recovery evidence is missing or malformed"
|
|
case "$asset" in
|
|
grafana) validate_evidence_stream "$marker" 'observability/grafana-admin' timestamp ;;
|
|
slack) validate_evidence_stream "$marker" 'observability/alertmanager-slack-webhook' timestamp ;;
|
|
esac || plain_fail "$asset recovery evidence is malformed"
|
|
marker_epoch="$(timestamp_epoch "$timestamp")" || plain_fail 'recovery timestamp is invalid'
|
|
current_epoch="$(now_epoch)" || plain_fail 'current UTC clock is invalid'
|
|
age=$((current_epoch - marker_epoch))
|
|
(( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) ||
|
|
plain_fail "$asset recovery evidence is outside the 30-day window"
|
|
case "$asset" in
|
|
grafana) printf 'GRAFANA_RECOVERY_EVIDENCE=PASS\n' ;;
|
|
slack) printf 'SLACK_RECOVERY_EVIDENCE=PASS\n' ;;
|
|
esac
|
|
}
|
|
|
|
check_slack_deployment_evidence() {
|
|
local marker kind timestamp marker_epoch current_epoch age
|
|
require_current_context
|
|
sudo_refresh
|
|
marker="$temporary_dir/check-slack-deployment.env"
|
|
root_marker_read slack "$marker" || plain_fail 'Slack deployment evidence is missing or malformed'
|
|
classify_slack_evidence_stream "$marker" kind timestamp ||
|
|
plain_fail 'Slack deployment evidence is malformed'
|
|
if [[ "$kind" == RECOVERY ]]; then
|
|
marker_epoch="$(timestamp_epoch "$timestamp")" || plain_fail 'recovery timestamp is invalid'
|
|
current_epoch="$(now_epoch)" || plain_fail 'current UTC clock is invalid'
|
|
age=$((current_epoch - marker_epoch))
|
|
(( age >= 0 && age <= EVIDENCE_MAX_AGE_SECONDS )) ||
|
|
plain_fail 'Slack recovery evidence is outside the 30-day window'
|
|
printf 'SLACK_DEPLOYMENT_GATE=RECOVERY\n'
|
|
else
|
|
printf 'SLACK_DEPLOYMENT_GATE=RISK_ACCEPTED\n'
|
|
fi
|
|
}
|
|
|
|
ensure_evidence_directory() {
|
|
root_marker_helper /dev/null ensure-dir "$EVIDENCE_HIERARCHY_ANCHOR" \
|
|
"$EVIDENCE_HIERARCHY_PROFILE" \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID" ||
|
|
plain_fail 'cannot safely create or verify the recovery evidence directory'
|
|
}
|
|
|
|
preflight_evidence_directory() {
|
|
local rc
|
|
if root_marker_helper /dev/null check-dir "$EVIDENCE_HIERARCHY_ANCHOR" \
|
|
"$EVIDENCE_HIERARCHY_PROFILE" \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" '' '' '' "$EUID"; then
|
|
return 0
|
|
else
|
|
rc=$?
|
|
[[ "$rc" == 4 ]] || plain_fail 'recovery evidence directory boundary is unsafe or drifted'
|
|
fi
|
|
}
|
|
|
|
prepare_marker() {
|
|
local asset=$1 kind=${2:-RECOVERY} path resource snapshot timestamp desired read_rc prior_kind
|
|
evidence_asset_fields "$asset" path resource || plain_fail 'unknown recovery evidence asset'
|
|
snapshot="$temporary_dir/prior-${asset}.env"
|
|
desired="$temporary_dir/desired-${asset}.env"
|
|
MARKER_PRIOR_FILE["$asset"]="$snapshot"
|
|
MARKER_DESIRED_FILE["$asset"]="$desired"
|
|
MARKER_KIND["$asset"]="$kind"
|
|
MARKER_HAD_PRIOR["$asset"]=false
|
|
|
|
if root_marker_read "$asset" "$snapshot"; then
|
|
if [[ "$asset" == slack ]]; then
|
|
classify_slack_evidence_stream "$snapshot" prior_kind timestamp ||
|
|
plain_fail 'Slack recovery marker schema drifted'
|
|
if [[ "$kind" == RISK && "$prior_kind" == RECOVERY ]]; then
|
|
plain_fail 'Slack recovery evidence cannot be downgraded to risk acceptance'
|
|
fi
|
|
if [[ "$kind" == RISK && "$prior_kind" == RISK_ACCEPTED ]]; then
|
|
cp -- "$snapshot" "$desired"
|
|
chmod 0600 -- "$desired"
|
|
MARKER_HAD_PRIOR["$asset"]=true
|
|
return
|
|
fi
|
|
else
|
|
validate_recovery_evidence_stream "$snapshot" "$resource" timestamp ||
|
|
plain_fail "$asset recovery marker schema drifted"
|
|
fi
|
|
MARKER_HAD_PRIOR["$asset"]=true
|
|
else
|
|
read_rc=$?
|
|
[[ "$read_rc" == 4 ]] || plain_fail "cannot safely read $asset recovery marker"
|
|
fi
|
|
|
|
if [[ "$kind" == RISK ]]; then
|
|
printf '%s\n' \
|
|
"schema=$RISK_EVIDENCE_SCHEMA" \
|
|
"context=$EXPECTED_CONTEXT" \
|
|
"resource=$resource" \
|
|
'reason=off-host-slack-recovery-deferred' \
|
|
"accepted_by_uid=$RISK_ACCEPTED_BY_UID" \
|
|
'approval_method=interactive-exact-confirmation' \
|
|
"approval_ref=$RISK_APPROVAL_REF" \
|
|
"checked_at_utc=$(now_utc)" >"$desired"
|
|
else
|
|
printf '%s\n' \
|
|
"schema=$EVIDENCE_SCHEMA" \
|
|
"context=$EXPECTED_CONTEXT" \
|
|
"resource=$resource" \
|
|
"checked_at_utc=$(now_utc)" >"$desired"
|
|
fi
|
|
chmod 0600 -- "$desired"
|
|
if [[ "$kind" == RISK ]]; then
|
|
validate_slack_risk_evidence_stream "$desired" timestamp ||
|
|
plain_fail 'generated Slack risk acceptance evidence is invalid'
|
|
else
|
|
validate_recovery_evidence_stream "$desired" "$resource" timestamp ||
|
|
plain_fail 'generated recovery evidence is invalid'
|
|
fi
|
|
}
|
|
|
|
root_marker_install() {
|
|
local asset=$1 source=$2 expected_kind=$3 expected_source=${4:-} path resource
|
|
evidence_asset_fields "$asset" path resource || return 1
|
|
root_marker_helper /dev/null install "$EVIDENCE_DIR" "${path##*/}" \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$source" "$expected_kind" \
|
|
"$expected_source" "$EUID"
|
|
}
|
|
|
|
marker_prestate_unchanged() {
|
|
local asset=$1 current="$temporary_dir/prestate-${asset}-$RANDOM" read_rc
|
|
if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then
|
|
root_marker_read "$asset" "$current" || return 1
|
|
cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "$current"
|
|
else
|
|
if root_marker_read "$asset" "$current"; then
|
|
return 1
|
|
else
|
|
read_rc=$?
|
|
[[ "$read_rc" == 4 ]]
|
|
fi
|
|
fi
|
|
}
|
|
|
|
marker_record_written() {
|
|
local asset=$1 existing
|
|
for existing in "${WRITTEN_MARKERS[@]}"; do
|
|
[[ "$existing" != "$asset" ]] || return 0
|
|
done
|
|
WRITTEN_MARKERS+=("$asset")
|
|
}
|
|
|
|
clear_pending_marker() {
|
|
pending_marker_asset=''
|
|
}
|
|
|
|
classify_pending_marker() {
|
|
local asset=$pending_marker_asset current read_rc
|
|
[[ -n "$asset" ]] || return 0
|
|
current="$temporary_dir/classify-marker-${asset}-$RANDOM"
|
|
if root_marker_read "$asset" "$current"; then
|
|
if cmp --silent -- "${MARKER_DESIRED_FILE[$asset]}" "$current"; then
|
|
marker_record_written "$asset"
|
|
clear_pending_marker
|
|
return 0
|
|
fi
|
|
if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]] &&
|
|
cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "$current"; then
|
|
clear_pending_marker
|
|
return 1
|
|
fi
|
|
return 2
|
|
else
|
|
read_rc=$?
|
|
if [[ "$read_rc" == 4 && "${MARKER_HAD_PRIOR[$asset]}" == false ]]; then
|
|
clear_pending_marker
|
|
return 1
|
|
fi
|
|
return 2
|
|
fi
|
|
}
|
|
|
|
install_marker() {
|
|
local asset=$1 install_rc classification
|
|
if [[ "$asset" == slack && "${MARKER_HAD_PRIOR[$asset]}" == true ]] &&
|
|
cmp --silent -- "${MARKER_PRIOR_FILE[$asset]}" "${MARKER_DESIRED_FILE[$asset]}"; then
|
|
return 0
|
|
fi
|
|
marker_prestate_unchanged "$asset" || return 1
|
|
pending_marker_asset=$asset
|
|
if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then
|
|
root_marker_install "$asset" "${MARKER_DESIRED_FILE[$asset]}" file \
|
|
"${MARKER_PRIOR_FILE[$asset]}" || install_rc=$?
|
|
else
|
|
root_marker_install "$asset" "${MARKER_DESIRED_FILE[$asset]}" absent '' || install_rc=$?
|
|
fi
|
|
if [[ -n "${install_rc:-}" ]]; then
|
|
classify_pending_marker
|
|
classification=$?
|
|
if [[ "$asset" == slack && "${MARKER_KIND[$asset]}" == RISK &&
|
|
"$classification" == 0 ]]; then
|
|
transaction_active=false
|
|
emit_manual_recovery
|
|
printf 'ERROR: Slack risk marker installation outcome is uncertain\n' >&2
|
|
exit "$install_rc"
|
|
fi
|
|
[[ "$classification" != 2 ]] || emit_manual_recovery
|
|
return "$install_rc"
|
|
fi
|
|
classify_pending_marker || return 1
|
|
}
|
|
|
|
restore_written_markers() {
|
|
local index asset current desired prior read_rc rc=0
|
|
for ((index=${#WRITTEN_MARKERS[@]} - 1; index >= 0; index--)); do
|
|
asset="${WRITTEN_MARKERS[$index]}"
|
|
desired="${MARKER_DESIRED_FILE[$asset]}"
|
|
current="$temporary_dir/rollback-marker-${asset}-$RANDOM"
|
|
if ! root_marker_read "$asset" "$current" || ! cmp --silent -- "$desired" "$current"; then
|
|
rc=1
|
|
continue
|
|
fi
|
|
if [[ "${MARKER_HAD_PRIOR[$asset]}" == true ]]; then
|
|
prior="${MARKER_PRIOR_FILE[$asset]}"
|
|
root_marker_install "$asset" "$prior" file "$desired" || rc=1
|
|
else
|
|
root_marker_helper /dev/null unlink-exact "$EVIDENCE_DIR" \
|
|
"$(case "$asset" in grafana) printf grafana-local.env ;; slack) printf slack.env ;; esac)" \
|
|
"$EVIDENCE_EXPECTED_UID" "$EVIDENCE_EXPECTED_GID" "$desired" file "$desired" "$EUID" || rc=1
|
|
fi
|
|
done
|
|
return "$rc"
|
|
}
|
|
|
|
stop_proxy() {
|
|
local pid=$1 attempt
|
|
/bin/kill -TERM "$pid" 2>/dev/null || true
|
|
for ((attempt=0; attempt<20; attempt++)); do
|
|
if ! /bin/kill -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
/bin/kill -KILL "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
}
|
|
|
|
delete_secret_with_uid_precondition() {
|
|
local name=$1 uid=$2 socket options response proxy_log proxy_pid attempt http_code curl_rc=0
|
|
[[ "$name" == grafana-admin || "$name" == alertmanager-slack-webhook ]] || return 1
|
|
[[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
|
|
socket="$temporary_dir/proxy-${name}.sock"
|
|
options="$temporary_dir/delete-${name}.json"
|
|
response="$temporary_dir/delete-${name}.response"
|
|
proxy_log="$temporary_dir/proxy-${name}.log"
|
|
printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' \
|
|
"$uid" >"$options"
|
|
chmod 0600 -- "$options" "$response" 2>/dev/null || true
|
|
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "$KUBECTL_PROXY_PROCESS_TIMEOUT" \
|
|
"$KUBECTL_BIN" --request-timeout="$KUBECTL_REQUEST_TIMEOUT" proxy \
|
|
--unix-socket="$socket" \
|
|
--api-prefix=/ \
|
|
--accept-paths="^/api/v1/namespaces/${NAMESPACE}/secrets/${name}$" \
|
|
--reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 &
|
|
proxy_pid=$!
|
|
for ((attempt=0; attempt<50; attempt++)); do
|
|
[[ -S "$socket" ]] && break
|
|
/bin/kill -0 "$proxy_pid" 2>/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; }
|
|
/usr/bin/sleep 0.05
|
|
done
|
|
[[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; }
|
|
|
|
http_code="$(curl --silent --show-error --max-time 5 \
|
|
--unix-socket "$socket" \
|
|
--output "$response" \
|
|
--write-out '%{http_code}' \
|
|
--request DELETE \
|
|
--header 'Content-Type: application/json' \
|
|
--data-binary "@$options" \
|
|
"http://localhost/api/v1/namespaces/${NAMESPACE}/secrets/${name}")" || curl_rc=$?
|
|
stop_proxy "$proxy_pid"
|
|
(( curl_rc == 0 )) || return 1
|
|
[[ "$http_code" == 200 || "$http_code" == 202 ]] || return 1
|
|
if secret_exists "$name"; then return 1; else [[ $? == 1 ]]; fi
|
|
}
|
|
|
|
rollback_owned_secrets() {
|
|
local index rc=0
|
|
for ((index=${#OWNED_NAMES[@]} - 1; index >= 0; index--)); do
|
|
if ! delete_secret_with_uid_precondition "${OWNED_NAMES[$index]}" "${OWNED_UIDS[$index]}"; then
|
|
rc=1
|
|
fi
|
|
done
|
|
return "$rc"
|
|
}
|
|
|
|
rollback_transaction() {
|
|
local rc=0
|
|
[[ "$rollback_in_progress" == false ]] || return 1
|
|
rollback_in_progress=true
|
|
restore_written_markers || rc=1
|
|
rollback_owned_secrets || rc=1
|
|
if (( rc == 0 )); then
|
|
printf 'OBSERVABILITY_SECRET_ROLLBACK=PASS\n' >&2
|
|
else
|
|
printf 'OBSERVABILITY_SECRET_ROLLBACK=FAIL\n' >&2
|
|
emit_manual_recovery
|
|
fi
|
|
rollback_in_progress=false
|
|
return "$rc"
|
|
}
|
|
|
|
clear_pending_create() {
|
|
pending_asset=''
|
|
pending_name=''
|
|
pending_token=''
|
|
pending_manifest=''
|
|
pending_response=''
|
|
}
|
|
|
|
classify_pending_create() {
|
|
local live_json live_prefix response_prefix response_uid live_uid presence_rc expected_keys
|
|
[[ -n "$pending_asset" && -n "$pending_name" && -n "$pending_token" && -n "$pending_manifest" ]] || return 0
|
|
case "$pending_asset" in
|
|
grafana) expected_keys='admin-user,admin-password' ;;
|
|
slack) expected_keys='url' ;;
|
|
*) return 2 ;;
|
|
esac
|
|
|
|
if [[ -n "$pending_response" && -f "$pending_response" && ! -L "$pending_response" && -s "$pending_response" ]]; then
|
|
response_prefix="$temporary_dir/pending-${pending_asset}-response"
|
|
response_uid="$(validate_secret_json "$pending_response" "$pending_name" "$expected_keys" \
|
|
"$response_prefix" "$pending_token")" || response_uid=''
|
|
if [[ -n "$response_uid" ]]; then
|
|
case "$pending_asset" in
|
|
grafana)
|
|
cmp --silent -- "$grafana_user_snapshot" "$response_prefix.admin-user" || response_uid=''
|
|
cmp --silent -- "$grafana_password_snapshot" "$response_prefix.admin-password" || response_uid=''
|
|
;;
|
|
slack)
|
|
cmp --silent -- "$slack_webhook_snapshot" "$response_prefix.url" || response_uid=''
|
|
;;
|
|
esac
|
|
fi
|
|
else
|
|
response_uid=''
|
|
fi
|
|
if secret_exists "$pending_name"; then
|
|
live_json="$temporary_dir/pending-${pending_asset}-live.json"
|
|
live_prefix="$temporary_dir/pending-${pending_asset}-live"
|
|
kubectl_bounded --namespace "$NAMESPACE" get secret "$pending_name" --output=json >"$live_json" || return 2
|
|
chmod 0600 -- "$live_json"
|
|
live_uid="$(validate_secret_json "$live_json" "$pending_name" "$expected_keys" \
|
|
"$live_prefix" "$pending_token")" || return 2
|
|
[[ -n "$response_uid" && "$live_uid" == "$response_uid" ]] || return 2
|
|
case "$pending_asset" in
|
|
grafana)
|
|
cmp --silent -- "$grafana_user_snapshot" "$live_prefix.admin-user" || return 2
|
|
cmp --silent -- "$grafana_password_snapshot" "$live_prefix.admin-password" || return 2
|
|
;;
|
|
slack) cmp --silent -- "$slack_webhook_snapshot" "$live_prefix.url" || return 2 ;;
|
|
esac
|
|
OWNED_ASSETS+=("$pending_asset")
|
|
OWNED_NAMES+=("$pending_name")
|
|
OWNED_UIDS+=("$response_uid")
|
|
clear_pending_create
|
|
return 0
|
|
else
|
|
presence_rc=$?
|
|
[[ "$presence_rc" == 1 ]] || return 2
|
|
clear_pending_create
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
transaction_fail() {
|
|
local message=$1 ambiguous=${2:-false} exit_status=${3:-1} classification
|
|
trap - ERR HUP INT TERM
|
|
set +e
|
|
if [[ -n "$pending_marker_asset" ]]; then
|
|
classify_pending_marker
|
|
classification=$?
|
|
[[ "$classification" != 2 ]] || ambiguous=true
|
|
fi
|
|
if [[ -n "$pending_asset" ]]; then
|
|
classify_pending_create
|
|
case $? in
|
|
0|1) ;;
|
|
*) ambiguous=true ;;
|
|
esac
|
|
fi
|
|
[[ "$ambiguous" != true ]] || emit_manual_recovery
|
|
if [[ "$transaction_active" == true ]]; then rollback_transaction || true; fi
|
|
transaction_active=false
|
|
printf 'ERROR: %s\n' "$message" >&2
|
|
exit "$exit_status"
|
|
}
|
|
|
|
on_error() {
|
|
local rc=$1 line=$2
|
|
transaction_fail "unexpected failure at line $line (exit $rc)"
|
|
}
|
|
|
|
on_signal() {
|
|
local rc=$1
|
|
transaction_fail "interrupted (exit $rc)" false "$rc"
|
|
}
|
|
|
|
create_asset_secret() {
|
|
local asset=$1 name uid ambiguous=false response decoded_prefix create_rc=0
|
|
case "$asset" in
|
|
grafana) name='grafana-admin' ;;
|
|
slack) name='alertmanager-slack-webhook' ;;
|
|
*) transaction_fail 'unknown Secret asset' ;;
|
|
esac
|
|
|
|
pending_asset="$asset"
|
|
pending_name="$name"
|
|
pending_token="$(new_transaction_token)" || transaction_fail 'could not create transaction ownership token'
|
|
[[ "$pending_token" =~ ^[0-9a-f]{32}$ ]] || transaction_fail 'transaction ownership token is invalid'
|
|
pending_manifest="$temporary_dir/create-${asset}.json"
|
|
response="$temporary_dir/create-${asset}-response.json"
|
|
pending_response="$response"
|
|
decoded_prefix="$temporary_dir/create-${asset}-response"
|
|
build_secret_manifest "$asset" "$pending_token" "$pending_manifest" ||
|
|
transaction_fail "could not build $asset Secret create request"
|
|
|
|
if kubectl_bounded create --filename="$pending_manifest" --output=json >"$response"; then
|
|
:
|
|
else
|
|
create_rc=$?
|
|
transaction_fail "create outcome for $NAMESPACE/$name was not confirmed (exit $create_rc)"
|
|
fi
|
|
chmod 0600 -- "$response"
|
|
case "$asset" in
|
|
grafana)
|
|
uid="$(validate_secret_json "$response" "$name" 'admin-user,admin-password' \
|
|
"$decoded_prefix" "$pending_token")" || transaction_fail 'Grafana create response contract is invalid'
|
|
cmp --silent -- "$grafana_user_snapshot" "$decoded_prefix.admin-user" ||
|
|
transaction_fail 'Grafana create response user differs'
|
|
cmp --silent -- "$grafana_password_snapshot" "$decoded_prefix.admin-password" ||
|
|
transaction_fail 'Grafana create response password differs'
|
|
;;
|
|
slack)
|
|
uid="$(validate_secret_json "$response" "$name" url "$decoded_prefix" "$pending_token")" ||
|
|
transaction_fail 'Slack create response contract is invalid'
|
|
cmp --silent -- "$slack_webhook_snapshot" "$decoded_prefix.url" ||
|
|
transaction_fail 'Slack create response webhook differs'
|
|
;;
|
|
esac
|
|
OWNED_ASSETS+=("$asset")
|
|
OWNED_NAMES+=("$name")
|
|
OWNED_UIDS+=("$uid")
|
|
clear_pending_create
|
|
}
|
|
|
|
verify_reinspection() {
|
|
local asset=$1 expected_state=$2 expected_uid=$3 actual_state actual_uid
|
|
inspect_contract "$asset" postprompt actual_state actual_uid
|
|
[[ "$actual_state" == "$expected_state" && "$actual_uid" == "$expected_uid" ]] ||
|
|
plain_fail "$asset Secret state changed before the last gate"
|
|
}
|
|
|
|
execute_transaction() {
|
|
local grafana_state='' grafana_uid='' slack_state='' slack_uid=''
|
|
local post_state post_uid index asset
|
|
|
|
validate_selected_input_metadata
|
|
run_common_gates
|
|
validate_selected_input_metadata
|
|
capture_selected_inputs
|
|
validate_selected_input_content
|
|
require_context_and_namespace
|
|
|
|
if [[ "$grafana_requested" == true ]]; then inspect_contract grafana preprompt grafana_state grafana_uid; fi
|
|
if [[ "$slack_requested" == true ]]; then inspect_contract slack preprompt slack_state slack_uid; fi
|
|
|
|
sudo_refresh
|
|
read_confirmation 'Type APPLY default: ' 'APPLY default'
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
read_confirmation 'Type RECOVERY GRAFANA-LOCAL default: ' 'RECOVERY GRAFANA-LOCAL default'
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
if [[ "$accept_slack_risk_requested" == true ]]; then
|
|
read_confirmation 'Type ACCEPT NO OFF-HOST SLACK RECOVERY default: ' \
|
|
'ACCEPT NO OFF-HOST SLACK RECOVERY default'
|
|
else
|
|
read_confirmation 'Type RECOVERY SLACK default: ' 'RECOVERY SLACK default'
|
|
fi
|
|
fi
|
|
|
|
preflight_evidence_directory
|
|
if [[ "$grafana_requested" == true ]]; then prepare_marker grafana; fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
if [[ "$accept_slack_risk_requested" == true ]]; then
|
|
prepare_marker slack RISK
|
|
else
|
|
prepare_marker slack RECOVERY
|
|
fi
|
|
fi
|
|
recheck_selected_inputs
|
|
require_context_and_namespace
|
|
if [[ "$grafana_requested" == true ]]; then verify_reinspection grafana "$grafana_state" "$grafana_uid"; fi
|
|
if [[ "$slack_requested" == true ]]; then verify_reinspection slack "$slack_state" "$slack_uid"; fi
|
|
|
|
run_common_gates
|
|
transaction_active=true
|
|
trap 'on_error "$?" "$LINENO"' ERR
|
|
trap 'on_signal 129' HUP
|
|
trap 'on_signal 130' INT
|
|
trap 'on_signal 143' TERM
|
|
|
|
if [[ "$grafana_requested" == true && "$grafana_state" == absent ]]; then create_asset_secret grafana; fi
|
|
if [[ "$slack_requested" == true && "$slack_state" == absent ]]; then create_asset_secret slack; fi
|
|
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
inspect_contract grafana postcreate post_state post_uid
|
|
if [[ "$grafana_state" == absent ]]; then
|
|
[[ "$post_uid" == "${OWNED_UIDS[0]}" ]] || transaction_fail 'Grafana Secret UID changed after creation' true
|
|
else
|
|
[[ "$post_uid" == "$grafana_uid" ]] || transaction_fail 'existing Grafana Secret UID changed' true
|
|
fi
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
inspect_contract slack postcreate post_state post_uid
|
|
if [[ "$slack_state" == absent ]]; then
|
|
for ((index=0; index<${#OWNED_ASSETS[@]}; index++)); do
|
|
if [[ "${OWNED_ASSETS[$index]}" == slack ]]; then
|
|
[[ "$post_uid" == "${OWNED_UIDS[$index]}" ]] || transaction_fail 'Slack Secret UID changed after creation' true
|
|
fi
|
|
done
|
|
else
|
|
[[ "$post_uid" == "$slack_uid" ]] || transaction_fail 'existing Slack Secret UID changed' true
|
|
fi
|
|
fi
|
|
|
|
ensure_evidence_directory
|
|
if [[ "$grafana_requested" == true ]]; then install_marker grafana; fi
|
|
if [[ "$slack_requested" == true ]]; then install_marker slack; fi
|
|
|
|
trap - ERR HUP INT TERM
|
|
transaction_active=false
|
|
if [[ "$grafana_requested" == true ]]; then
|
|
if [[ "$grafana_state" == absent ]]; then
|
|
printf 'OBSERVABILITY_GRAFANA_ADMIN=CREATE_CONFIRMED\n'
|
|
else
|
|
printf 'OBSERVABILITY_GRAFANA_ADMIN=REUSED_UNCHANGED\n'
|
|
fi
|
|
fi
|
|
if [[ "$slack_requested" == true ]]; then
|
|
if [[ "$slack_state" == absent ]]; then
|
|
printf 'OBSERVABILITY_SLACK_WEBHOOK=CREATE_CONFIRMED\n'
|
|
else
|
|
printf 'OBSERVABILITY_SLACK_WEBHOOK=REUSED_UNCHANGED\n'
|
|
fi
|
|
fi
|
|
printf 'OBSERVABILITY_SECRETS_TRANSACTION=PASS\n'
|
|
}
|
|
|
|
observability_secrets_main() {
|
|
parse_arguments "$@"
|
|
|
|
if [[ "$execute_requested" == false && "$check_grafana_requested" == false &&
|
|
"$check_slack_requested" == false && "$check_slack_deployment_requested" == false ]]; then
|
|
printf '%s\n' \
|
|
'OBSERVABILITY_SECRETS_DRY_RUN=PASS' \
|
|
'CONTEXT=default' \
|
|
'GRAFANA_ADMIN_CONTRACT=observability/grafana-admin:Opaque:admin-user,admin-password' \
|
|
'SLACK_WEBHOOK_CONTRACT=observability/alertmanager-slack-webhook:Opaque:url' \
|
|
'MUTATION=NOT_REQUESTED'
|
|
return 0
|
|
fi
|
|
|
|
(( EUID != 0 )) || plain_fail 'run this script as the current user, not through sudo'
|
|
require_commands
|
|
make_temporary_dir
|
|
|
|
if [[ "$check_grafana_requested" == true ]]; then
|
|
check_recovery_evidence grafana
|
|
elif [[ "$check_slack_requested" == true ]]; then
|
|
check_recovery_evidence slack
|
|
elif [[ "$check_slack_deployment_requested" == true ]]; then
|
|
check_slack_deployment_evidence
|
|
else
|
|
execute_transaction
|
|
fi
|
|
}
|
|
|
|
platform_observability_secrets_fixture_main() {
|
|
local fixture_root=${1:-}
|
|
[[ "${BASH_SOURCE[0]}" != "$0" ]] || plain_fail 'test fixture entrypoint must be sourced'
|
|
[[ -n "$fixture_root" ]] || plain_fail 'test fixture root is required'
|
|
shift
|
|
TEST_MODE=true
|
|
configure_test_boundaries "$fixture_root"
|
|
observability_secrets_main "$@"
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
reject_production_overrides
|
|
observability_secrets_main "$@"
|
|
fi
|