#!/usr/bin/env bash # Regression coverage for parser branches that must fail closed. No command in # this file contacts a k3s service, systemd, or the Kubernetes API. set -Eeuo pipefail readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly LIBRARY_PATH="${REPOSITORY_ROOT}/scripts/lib/k3s-secret-encryption.sh" readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-secret-encryption.sh" readonly ORIGINAL_PATH="$PATH" fail() { printf 'TEST FAILURE: %s\n' "$*" >&2 exit 1 } assert_eq() { local expected="$1" local actual="$2" local description="$3" [[ "$actual" == "$expected" ]] || fail "$description" } assert_succeeds() { "$@" || fail "expected success: $*" } assert_fails() { if "$@"; then fail "expected failure: $*" fi } assert_exit() { local expected="$1" shift local actual=0 "$@" || actual=$? [[ "$actual" == "$expected" ]] || fail "expected exit ${expected}, got ${actual}: $*" } fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-secret-encryption-test.XXXXXX")" verified_output_dir='' rejected_output_dir='' race_output_dir='' finished_output_dir='' transition_output_dir='' enabled_output_dir='' symlink_output_parent='' cleanup() { if [[ -n "$verified_output_dir" ]]; then rm -rf -- "$verified_output_dir" fi if [[ -n "$rejected_output_dir" ]]; then rm -rf -- "$rejected_output_dir" fi if [[ -n "$race_output_dir" ]]; then rm -rf -- "$race_output_dir" fi if [[ -n "$finished_output_dir" ]]; then rm -rf -- "$finished_output_dir" fi if [[ -n "$transition_output_dir" ]]; then rm -rf -- "$transition_output_dir" fi if [[ -n "$enabled_output_dir" ]]; then rm -rf -- "$enabled_output_dir" fi if [[ -n "$symlink_output_parent" && -L "$symlink_output_parent" ]]; then unlink -- "$symlink_output_parent" fi case "$fixture_root" in /tmp/k3s-secret-encryption-test.*|"${TMPDIR:-/tmp}"/k3s-secret-encryption-test.*) rm -rf -- "$fixture_root" ;; *) fail 'refusing to remove an unexpected fixture directory' ;; esac } trap cleanup EXIT # The production change that this test catches is a missing status parser. # This source must fail RED until the source-only library is created. # shellcheck source=/dev/null source "$LIBRARY_PATH" # Regression: GNU stat describes a zero-byte regular file as "regular empty # file". The root text reader must accept the file by inode kind, ownership, # and mode instead of depending on that presentation string. assert_succeeds /usr/bin/bash -c ' set -Eeuo pipefail source "$1" _kse_privileged_capture() { local binary="$1" shift if [[ "$binary" == "$KSE_STAT" ]]; then case "${1:-}" in --format=%u:%a:%F) printf "0:600:regular empty file\n" ;; --format=%u:%a) printf "0:600\n" ;; *) return 1 ;; esac return 0 fi if [[ "$binary" == "$KSE_CAT" && "${*: -1}" == /etc/systemd/system/k3s.service.env ]]; then return 0 fi return 1 } _kse_privileged_quiet() { [[ "$1" == "$KSE_TEST" ]] || return 1 shift case "$*" in "-f /etc/systemd/system/k3s.service.env"|"! -L /etc/systemd/system/k3s.service.env") return 0 ;; *) return 1 ;; esac } value="$(_kse_read_root_text /etc/systemd/system/k3s.service.env)" [[ -z "$value" ]] ' bash "$VALIDATOR_PATH" disabled_json='{"stage":"","activekey":""}' start_json='{"stage":"start","activekey":"","enable":false,"hashmatch":true}' enabled_start_json='{"stage":"start","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' enabled_finished_json='{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' active_json='{"stage":"reencrypt_active","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' mismatch_json='{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hasherror":"hash mismatch"}' secretbox_json='{"stage":"start","activekey":"XSalsa20-POLY1305 fixture-key","enable":true,"hashmatch":true}' # Wrong branches here would weaken the state-machine fail-stop gate. assert_eq disabled_no_config "$(classify_encryption_status "$disabled_json")" 'disabled fixture classification' assert_eq transition_start "$(classify_encryption_status "$start_json")" 'late-enable transition classification' assert_eq enabled_stable "$(classify_encryption_status "$enabled_start_json")" 'enabled start classification' assert_eq enabled_stable "$(classify_encryption_status "$enabled_finished_json")" 'enabled finished classification' assert_eq unsafe_transition "$(classify_encryption_status "$active_json")" 'active re-encryption classification' assert_eq hash_mismatch "$(classify_encryption_status "$mismatch_json")" 'hash mismatch classification' assert_eq invalid "$(classify_encryption_status '{"stage":"start","activekey":"","enable":false,"hashmatch":true,"inactivekeys":["unexpected"]}')" 'non-empty transition inactive keys rejection' assert_eq invalid "$(classify_encryption_status '{"stage":"","activekey":"","hasherror":"unexpected"}')" 'disabled status hash error rejection' assert_eq invalid "$(classify_encryption_status "${disabled_json}"$'\n'"${disabled_json}")" 'concatenated JSON documents rejection' assert_eq invalid "$(classify_encryption_status "${disabled_json} trailing")" 'JSON trailing garbage rejection' assert_eq aescbc "$(classify_encryption_provider "$enabled_start_json")" 'AES-CBC provider classification' assert_eq secretbox "$(classify_encryption_provider "$secretbox_json")" 'secretbox provider classification' assert_eq invalid "$(classify_encryption_provider '{"stage":"start","activekey":"AES-CBC","enable":true,"hashmatch":true}')" 'provider exact-prefix rejection' assert_succeeds version_supports_late_enable v1.33.10+k3s1 assert_succeeds version_supports_late_enable v1.34.6+k3s1 assert_succeeds version_supports_late_enable v1.35.3+k3s1 assert_succeeds version_supports_late_enable v1.36.2+k3s1 assert_fails version_supports_late_enable v1.33.9+k3s1 assert_fails version_supports_late_enable v1.34.5+k3s1 assert_fails version_supports_late_enable v1.35.2+k3s1 assert_succeeds require_exact_encryption_state enabled_stable "$enabled_finished_json" assert_fails require_exact_encryption_state enabled_stable "$active_json" fixture_config="${fixture_root}/encryption-config.json" fixture_state="${fixture_root}/encryption-state.json" fixture_stale_state="${fixture_root}/encryption-state-stale.json" printf '{"fixture":"config"}\n' >"$fixture_config" fixture_hash="$(sha256sum -- "$fixture_config" | awk '{print $1}')" fixture_annotation="reencrypt_finished-${fixture_hash}" printf '%s' "$fixture_annotation" >"$fixture_state" printf '%s' "start-${fixture_hash}" >"$fixture_stale_state" chmod 0600 "$fixture_config" "$fixture_state" "$fixture_stale_state" assert_eq match "$(verify_local_encryption_config_integrity_evidence \ '0:600:regular file' '0:600:regular file' "$fixture_hash" \ "$fixture_annotation" "$fixture_annotation" reencrypt_finished)" \ 'privileged evidence integrity match' # The shim changes only the metadata an unprivileged fixture cannot create; # content, hashes, symlink checks, and all parser behavior stay real. mkdir -p "${fixture_root}/stat-shim" printf '%s\n' '#!/usr/bin/env bash' \ 'if [[ "$1" == "--format=%u:%a:%F" ]]; then printf "0:600:regular file\\n"; exit 0; fi' \ 'exec /usr/bin/stat "$@"' >"${fixture_root}/stat-shim/stat" chmod 0700 "${fixture_root}/stat-shim/stat" PATH="${fixture_root}/stat-shim:${ORIGINAL_PATH}" assert_eq match "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_state" "$fixture_annotation" reencrypt_finished)" 'matching root metadata fixture' assert_eq mismatch "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_stale_state" "$fixture_annotation" reencrypt_finished || true)" 'stale state rejection' PATH="$ORIGINAL_PATH" assert_eq mismatch "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_state" "$fixture_annotation" reencrypt_finished || true)" 'user-owned config rejection' ln -s -- "$fixture_config" "${fixture_root}/config-link" assert_eq mismatch "$(verify_local_encryption_config_integrity "${fixture_root}/config-link" "$fixture_state" "$fixture_annotation" reencrypt_finished || true)" 'symlink config rejection' fake_bin="${fixture_root}/bin" mkdir -p "$fake_bin" printf '%s\n' '#!/usr/bin/env bash' \ 'printf "%s\n" "${SYSTEMCTL_SHOW_FIXTURE:-}"; [[ "${SYSTEMCTL_SHOW_FIXTURE:-}" == *$'"'"'\nEnvironment='"'"'* ]] || printf "Environment=\n"; [[ "${SYSTEMCTL_SHOW_FIXTURE:-}" == *$'"'"'\nEnvironmentFiles='"'"'* ]] || printf "EnvironmentFiles=\n"' >"${fake_bin}/systemctl" printf '%s\n' '#!/usr/bin/env bash' \ 'if [[ "${K3S_HANG:-0}" == 1 ]]; then while :; do /usr/bin/sleep 1; done; fi; if [[ "${1:-}" == --version ]]; then printf "k3s version v1.36.2+k3s1 (fixture)\n"; exit 0; fi; if [[ "${1:-}" == kubectl ]]; then case "${2:-}" in get) if [[ "${3:-}" == nodes ]]; then printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"fixture-server\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{\"k3s.io/encryption-config-hash\":\"token-password-encryption-config\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}"; exit 0; fi; if [[ "${3:-}" == --raw=/readyz ]]; then printf "token-password-encryption-config\n"; [[ "${K3S_API_RC:-0}" == 0 ]] && exit 0 || exit 1; fi ;; esac; exit 1; fi; case "${K3S_STATUS_FIXTURE:-}" in disabled) printf "%s\n" "{\"stage\":\"\",\"activekey\":\"\"}"; exit 0 ;; finished) stage=reencrypt_finished ;; start) stage=start ;; unsupported) stage=unexpected_stage ;; mismatch) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC fixture-key\",\"enable\":true,\"hasherror\":\"mismatch\"}"; exit 0 ;; malformed) printf "%s\n" "{"; exit 0 ;; active_then_finished) if [[ -e "${K3S_STATUS_COUNTER:?}" ]]; then stage=reencrypt_finished; else : >"$K3S_STATUS_COUNTER"; stage=reencrypt_active; fi ;; *) exit 1 ;; esac; printf "%s\n" "{\"stage\":\"${stage}\",\"activekey\":\"AES-CBC fixture-key\",\"enable\":true,\"hashmatch\":true}"' >"${fake_bin}/k3s" printf '%s\n' '#!/usr/bin/env bash' \ '{ printf "sudo"; printf " <%s>" "$@"; printf "\n"; } >>"${FAKE_ARG_LOG:?}"' \ 'if [[ "${SUDO_HANG:-0}" == 1 ]]; then while :; do /usr/bin/sleep 1; done; fi' \ '[[ "${1:-}" == --non-interactive ]] && shift' \ 'if [[ "${1:-}" == /usr/bin/env && " $* " == *" --privileged-probe "* && -n "${K3S_VALIDATOR_FAKE_PATH:-}" ]]; then shift; [[ "${1:-}" == -i ]] && shift; [[ "${1:-}" == PATH=* ]] && shift; [[ "${1:-}" == LC_ALL=* ]] && shift; exec /usr/bin/env PATH="$K3S_VALIDATOR_FAKE_PATH" "$@"; fi' \ 'exec "$@"' >"${fake_bin}/sudo" printf '%s\n' '#!/usr/bin/env bash' \ '{ printf "timeout"; printf " <%s>" "$@"; printf "\n"; } >>"${FAKE_ARG_LOG:?}"' \ 'shift 3' \ 'if [[ "${SUDO_HANG:-0}" == 1 ]]; then exec /usr/bin/timeout --signal=TERM --kill-after=0.1s 0.05s "$@"; fi' \ 'if [[ "${K3S_HANG:-0}" == 1 ]]; then if [[ -e "${K3S_TIMEOUT_COUNTER:?}" ]]; then exit 124; fi; : >"$K3S_TIMEOUT_COUNTER"; exec /usr/bin/timeout --signal=TERM --kill-after=0.1s 0.05s "$@"; fi' \ 'exec "$@"' >"${fake_bin}/timeout" printf '%s\n' '#!/usr/bin/env bash' 'exit 0' >"${fake_bin}/sleep" printf '%s\n' '#!/usr/bin/env bash' 'exit 1' >"${fake_bin}/kubectl" printf '%s\n' '#!/usr/bin/env bash' \ 'if [[ "${HANDOFF_RACE:-0}" == 1 && " $* " == *" inventory.env status.sha256 "* ]]; then : >unexpected-race; fi' \ 'exec /usr/bin/stat "$@"' >"${fake_bin}/stat" chmod 0700 "${fake_bin}/systemctl" "${fake_bin}/k3s" "${fake_bin}/sudo" "${fake_bin}/timeout" "${fake_bin}/sleep" "${fake_bin}/kubectl" "${fake_bin}/stat" PATH="${fake_bin}:${ORIGINAL_PATH}" FAKE_ARG_LOG="${fixture_root}/fake-argv.log" export SYSTEMCTL_SHOW_FIXTURE K3S_STATUS_FIXTURE FAKE_ARG_LOG K3S_API_RC config_root="${fixture_root}/config-root" mkdir -p "${config_root}/config.yaml.d" printf 'secrets-encryption: false\n' >"${config_root}/config.yaml" printf 'secrets-encryption: true\n' >"${config_root}/config.yaml.d/40-encryption.yaml" export K3S_CONFIG_DIR="$config_root" SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'default YAML drop-in merge and implicit provider' cli_config="${fixture_root}/cli.yaml" env_config="${fixture_root}/env.yaml" printf 'secrets-encryption: true\nsecrets-encryption-provider: aescbc\n' >"$cli_config" printf 'secrets-encryption: false\n' >"$env_config" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${cli_config} Environment=K3S_CONFIG_FILE=${env_config}" assert_eq "aescbc/${cli_config}" "$(detect_effective_encryption_config_owner)" 'CLI config precedence' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server Environment=K3S_CONFIG_FILE=${cli_config}" assert_eq "aescbc/${cli_config}" "$(detect_effective_encryption_config_owner)" 'K3S_CONFIG_FILE config selection' printf 'secrets-encryption: true\n' >"$env_config" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config} Environment=K3S_SECRETS_ENCRYPTION_PROVIDER=secretbox" assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'environment provider overrides YAML provider absence' printf 'secrets-encryption-provider: secretbox\n' >"$env_config" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config} Environment=K3S_SECRETS_ENCRYPTION=true" assert_eq "secretbox/${env_config}" "$(detect_effective_encryption_config_owner)" 'YAML provider remains effective when environment only enables' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --secrets-encryption=true --secrets-encryption-provider=aescbc' assert_eq aescbc/command-line "$(detect_effective_encryption_config_owner)" 'ExecStart encryption key precedence' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --secrets-encryption=true ; ignore_errors=no ; }' assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'systemctl show ExecStart serialization' # These fixtures catch accepting anything except one systemd ExecStart record # whose known metadata appears once, in the production order, with valid types. SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'production systemctl show ExecStart serialization' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } { path=/usr/bin/false ; argv[]=/usr/bin/false ; ignore_errors=no ; } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart extra record rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; argv[]=/usr/local/bin/k3s server ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart duplicate argv rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; unknown=0 ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart unknown metadata rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; pid=0 ; code=(null) ; status=0/0 } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart duplicate metadata rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; stop_time=[n/a] ; start_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart metadata order rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=not-a-number ; code=(null) ; status=0/0 } Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart malformed metadata rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart malformed brace rejection' env_file="${fixture_root}/k3s.env" printf 'K3S_SECRETS_ENCRYPTION=true\n' >"$env_file" K3S_CONFIG_DIR="${fixture_root}/empty-config-root" mkdir -p "$K3S_CONFIG_DIR" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${env_file}" assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile enable source' printf '# comment\nK3S_SECRETS_ENCRYPTION=true\n' >"${fixture_root}/comment.env" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/comment.env" assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile comments ignored' printf 'K3S_SECRETS_ENCRYPTION="true"\n' >"${fixture_root}/quoted.env" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/quoted.env" assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'quoted EnvironmentFile assignment rejection' printf 'K3S_SECRETS_ENCRYPTION\n' >"${fixture_root}/reset.env" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/reset.env" assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'EnvironmentFile reset rejection' printf "OTHER='multiline\nK3S_SECRETS_ENCRYPTION=true\nclosed'\n" >"${fixture_root}/multiline-other.env" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/multiline-other.env (ignore_errors=no)" assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'non-target multiline record does not create assignment' printf '%s\n' 'OTHER="first \"' 'K3S_SECRETS_ENCRYPTION=true' 'continued"' >"${fixture_root}/escaped-quote-other.env" assert_eq '' "$(_k3s_envfile_values "${fixture_root}/escaped-quote-other.env" K3S_SECRETS_ENCRYPTION || true)" 'escaped quote keeps target-looking line inside non-target logical record' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/escaped-quote-other.env (ignore_errors=no)" assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'escaped quoted EnvironmentFile record fails closed without target assignment' printf '%s\n' 'OTHER=prefix\' 'K3S_SECRETS_ENCRYPTION=true' >"${fixture_root}/continued-other.env" assert_eq '' "$(_k3s_envfile_values "${fixture_root}/continued-other.env" K3S_SECRETS_ENCRYPTION || true)" 'backslash continuation keeps target-looking line inside non-target logical record' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/continued-other.env (ignore_errors=no)" assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'continued EnvironmentFile record fails closed without target assignment' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${env_file} (ignore_errors=no) ${fixture_root}/comment.env (ignore_errors=no)" assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'later EnvironmentFile assignment wins' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server Environment=K3S_SECRETS_ENCRYPTION=false EnvironmentFiles=${env_file} (ignore_errors=no)" assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile overrides Environment assignment' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/absent.env (ignore_errors=yes) ${env_file} (ignore_errors=no)" assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'optional missing EnvironmentFile skipped' provider_env_file="${fixture_root}/k3s-provider.env" printf 'K3S_SECRETS_ENCRYPTION=true\nK3S_SECRETS_ENCRYPTION_PROVIDER=secretbox\n' >"$provider_env_file" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${provider_env_file}" assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'EnvironmentFile provider source' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'direct environment enable source' dropin_dir="${fixture_root}/k3s.service.d" mkdir -p "$dropin_dir" printf '[Service]\nEnvironment=K3S_SECRETS_ENCRYPTION=true\n' >"${dropin_dir}/40-encryption.conf" K3S_SYSTEMD_DROPIN_DIR="$dropin_dir" SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'systemd drop-in environment source' K3S_SYSTEMD_DROPIN_DIR="${fixture_root}/no-dropins" SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server Environment=K3S_SECRETS_ENCRYPTION=true K3S_SECRETS_ENCRYPTION=false' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'same precedence environment conflict' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server Environment=K3S_SECRETS_ENCRYPTION=true K3S_SECRETS_ENCRYPTION_PROVIDER=aescbc K3S_SECRETS_ENCRYPTION_PROVIDER=secretbox' assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'later Environment provider assignment wins' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --config=$CONFIG Environment=K3S_SECRETS_ENCRYPTION=true' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'dynamic config expansion rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --config=' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'empty CLI config rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --secrets-encryption-provider=' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'empty CLI provider rejection' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/missing.env" assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'unreadable EnvironmentFile rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart=' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'blank ExecStart rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/bin/false' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'unrelated ExecStart rejection' SYSTEMCTL_SHOW_FIXTURE=$'ExecStart=/usr/local/bin/k3s server\nExecStart=/usr/local/bin/k3s server' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'multiple ExecStart rejection' SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --secrets-encryption=true ; ignore_errors=no ; } { path=/usr/bin/false ; argv[]=/usr/bin/false ; ignore_errors=no ; } Environment= EnvironmentFiles=' assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'multiple serialized ExecStart command records rejection' data_dir="${fixture_root}/data" mkdir -p "${data_dir}/server/db" touch "${data_dir}/server/db/state.db" ( _k3s_datastore_local_evidence() { printf 'sqlite\n'; } SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' assert_eq sqlite "$(detect_k3s_datastore)" 'default data-dir sqlite evidence' ) ( _k3s_datastore_local_evidence() { printf 'embedded-etcd\n'; } SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' assert_eq embedded-etcd "$(detect_k3s_datastore)" 'default data-dir embedded etcd evidence' ) SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}" assert_eq ambiguous "$(detect_k3s_datastore)" 'custom CLI data-dir rejection' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} --datastore-endpoint=" assert_eq ambiguous "$(detect_k3s_datastore)" 'empty CLI datastore endpoint rejection' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} --datastore-endpoint=fixture" assert_eq ambiguous "$(detect_k3s_datastore)" 'ExecStart external endpoint conflict rejection' alt_data_dir="${fixture_root}/alt-data" mkdir -p "${alt_data_dir}/server/db" touch "${alt_data_dir}/server/db/state.db" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server Environment=K3S_DATA_DIR=${alt_data_dir}" assert_eq ambiguous "$(detect_k3s_datastore)" 'custom environment data-dir rejection' printf 'data-dir: %s\ndatastore-endpoint: fixture\n' "$alt_data_dir" >"$env_config" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config}" assert_eq ambiguous "$(detect_k3s_datastore)" 'merged YAML data-dir and endpoint conflict rejection' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server EnvironmentFiles=${fixture_root}/config-path.env" printf 'K3S_CONFIG_FILE=%s\n' "$env_config" >"${fixture_root}/config-path.env" assert_eq ambiguous "$(detect_k3s_datastore)" 'EnvironmentFile config selection' datastore_env_file="${fixture_root}/datastore.env" printf 'K3S_DATASTORE_ENDPOINT=fixture\n' >"$datastore_env_file" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} EnvironmentFiles=${datastore_env_file}" assert_eq ambiguous "$(detect_k3s_datastore)" 'EnvironmentFile external datastore conflict rejection' SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}" rm -f -- "${data_dir}/server/db/state.db" mkdir -p "${data_dir}/server/db/etcd" assert_eq ambiguous "$(detect_k3s_datastore)" 'custom embedded-etcd path rejection' rm -rf -- "${data_dir}/server/db/etcd" SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} Environment=K3S_DATASTORE_ENDPOINT=fixture" assert_eq ambiguous "$(detect_k3s_datastore)" 'custom path takes precedence over external classification' touch "${data_dir}/server/db/state.db" mkdir -p "${data_dir}/server/db/etcd" assert_eq ambiguous "$(detect_k3s_datastore)" 'conflicting datastore evidence rejection' printf 'secrets-encryption: true\nsecrets-encryption-provider: aescbc\n' >"${config_root}/config.yaml" printf 'secrets-encryption-provider: secretbox\n' >"${config_root}/config.yaml.d/40-provider.yaml" K3S_CONFIG_DIR="$config_root" SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server' assert_eq "secretbox/${config_root}/config.yaml.d/40-provider.yaml" "$(detect_effective_encryption_config_owner)" 'provider fragment owner' virtual_config="${fixture_root}/not-user-listable/config.yaml" ( _k3s_list_yaml_files() { printf '%s\n' "${config_root}/config.yaml.d/40-provider.yaml"; } assert_eq "${config_root}/config.yaml.d/40-provider.yaml|secretbox" \ "$(_k3s_yaml_value "$virtual_config" secrets-encryption-provider)" \ 'privileged YAML listing callback' ) # A correct helper returns non-zero before this test-only outer guard fires. # Exit 124 means the helper did not bound its own privileged command. assert_bounded_hang_failure() { local helper="$1" argument="$2" counter rc=0 counter="${fixture_root}/${helper}-timeout-counter" rm -f -- "$counter" /usr/bin/timeout --signal=TERM --kill-after=0.1s 1s env \ PATH="$PATH" K3S_HANG=1 K3S_TIMEOUT_COUNTER="$counter" \ FAKE_ARG_LOG="$FAKE_ARG_LOG" \ bash -c 'set -Eeuo pipefail; source "$1"; "$2" "$3"' \ bash "$LIBRARY_PATH" "$helper" "$argument" || rc=$? [[ "$rc" -ne 0 && "$rc" -ne 124 ]] || fail "$helper did not bound a hanging privileged command" } assert_bounded_hang_failure wait_for_reencrypt_finished 0 assert_bounded_hang_failure wait_for_k3s_api ignored # A 599-second first call leaves too little budget for another bounded call. # The fake clock makes the ten-minute deadline boundary deterministic and fast. assert_deadline_stops_second_call() { local helper="$1" argument="$2" local clock_file="${fixture_root}/${helper}-clock" calls_file="${fixture_root}/${helper}-deadline-calls" call_count printf '0\n' >"$clock_file" : >"$calls_file" ( _k3s_now_seconds() { printf '%s\n' "$(<"$clock_file")"; } timeout() { local now printf 'call\n' >>"$calls_file" now="$(<"$clock_file")" printf '%s\n' "$((now + 599))" >"$clock_file" if [[ " $* " == *' secrets-encrypt '* ]]; then printf '%s\n' '{"stage":"reencrypt_active","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' return 0 fi return 1 } sleep() { local now now="$(<"$clock_file")" printf '%s\n' "$((now + $1))" >"$clock_file" } assert_fails "$helper" "$argument" ) call_count="$(wc -l <"$calls_file" | tr -d '[:space:]')" assert_eq 1 "$call_count" "$helper respects the overall ten-minute deadline (calls=$call_count)" } assert_deadline_stops_second_call wait_for_reencrypt_finished 0 assert_deadline_stops_second_call wait_for_k3s_api ignored K3S_STATUS_FIXTURE=finished : >"$FAKE_ARG_LOG" assert_succeeds wait_for_reencrypt_finished 0 assert_eq $'timeout <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--output> \nsudo <--non-interactive> <--output> ' "$(<"$FAKE_ARG_LOG")" 'bounded authoritative status reader argv' K3S_STATUS_FIXTURE=active_then_finished K3S_STATUS_COUNTER="${fixture_root}/status-counter" export K3S_STATUS_COUNTER assert_succeeds wait_for_reencrypt_finished 1 K3S_STATUS_FIXTURE=start assert_fails wait_for_reencrypt_finished 0 K3S_STATUS_FIXTURE=mismatch assert_fails wait_for_reencrypt_finished 0 K3S_STATUS_FIXTURE=malformed assert_fails wait_for_reencrypt_finished 0 K3S_STATUS_FIXTURE=unsupported assert_fails wait_for_reencrypt_finished 0 K3S_API_RC=0 : >"$FAKE_ARG_LOG" assert_succeeds wait_for_k3s_api assert_eq $'timeout <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--raw=/readyz>\nsudo <--non-interactive> <--raw=/readyz>' "$(<"$FAKE_ARG_LOG")" 'bounded credentialed readyz argv' K3S_API_RC=1 assert_fails wait_for_k3s_api # Validator entrypoint fixtures source the production script in a fresh shell, # override only its privileged command boundary, and call the real main. There # is no production environment switch that enables this harness. run_validator_fixture() { /usr/bin/env \ KSE_FIX_STATUS="${KSE_FIX_STATUS:-disabled}" \ KSE_FIX_INTEGRITY="${KSE_FIX_INTEGRITY:-match}" \ KSE_FIX_VERSION="${KSE_FIX_VERSION:-v1.36.2+k3s1}" \ KSE_FIX_NODES="${KSE_FIX_NODES:-valid}" \ KSE_FIX_API_RC="${KSE_FIX_API_RC:-0}" \ KSE_FIX_SUDO_RC="${KSE_FIX_SUDO_RC:-0}" \ KSE_FIX_ARG_LOG="$FAKE_ARG_LOG" \ /usr/bin/bash -c ' set -Eeuo pipefail source "$1" _kse_validate_runtime_dependencies() { return 0; } _kse_validate_sudo_credentials() { printf " <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--validate>\n" >>"$KSE_FIX_ARG_LOG" [[ "$KSE_FIX_SUDO_RC" == 0 ]] } _kse_detect_datastore() { printf "sqlite\n"; } _kse_privileged_capture() { local binary="$1" fixture_hash stage annotation shift { printf "<%s>" "$binary"; printf " <%s>" "$@"; printf "\n"; } >>"$KSE_FIX_ARG_LOG" fixture_hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa if [[ "$binary" == "$KSE_K3S" && "${1:-}" == --version ]]; then printf "k3s version %s (fixture)\n" "$KSE_FIX_VERSION"; return 0 fi if [[ "$binary" == "$KSE_K3S" && " ${*} " == *" secrets-encrypt status --output json "* ]]; then case "$KSE_FIX_STATUS" in disabled) printf "%s\n" "{\"stage\":\"\",\"activekey\":\"\"}" ;; transition) printf "%s\n" "{\"stage\":\"start\",\"activekey\":\"\",\"enable\":false,\"hashmatch\":true}" ;; enabled_start) printf "%s\n" "{\"stage\":\"start\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;; finished) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;; active) printf "%s\n" "{\"stage\":\"reencrypt_active\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;; mismatch) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hasherror\":\"password-token\"}" ;; malformed) printf "{\n" ;; *) return 1 ;; esac return 0 fi if [[ "$binary" == "$KSE_K3S" && " ${*} " == *" kubectl get nodes -o json "* ]]; then case "$KSE_FIX_STATUS" in disabled) annotation="" ;; transition|enabled_start) annotation="start-${fixture_hash}" ;; *) annotation="reencrypt_finished-${fixture_hash}" ;; esac case "$KSE_FIX_NODES" in valid) printf "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{%s}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}\n" "${annotation:+\"k3s.io/encryption-config-hash\":\"${annotation}\"}" ;; different_ready) printf "{\"items\":[{\"metadata\":{\"name\":\"isolated-restore\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{%s}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}\n" "${annotation:+\"k3s.io/encryption-config-hash\":\"${annotation}\"}" ;; wrong_node) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"wrong-node\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}" ;; not_ready) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"False\"}]}}]}" ;; two_servers) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}},{\"metadata\":{\"name\":\"second\",\"uid\":\"11111111-2222-4333-8444-555555555555\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}" ;; esac return 0 fi if [[ "$binary" == "$KSE_SYSTEMCTL" ]]; then printf "%s\n" "ExecStart=/usr/local/bin/k3s server" "Environment=K3S_TOKEN=token-password-encryption-config" "EnvironmentFiles="; return 0 fi if [[ "$binary" == "$KSE_STAT" ]]; then printf "0:600:regular file\n"; return 0; fi if [[ "$binary" == "$KSE_SHA256SUM" ]]; then printf "%s %s\n" "$fixture_hash" "$KSE_CONFIG_PATH"; return 0; fi if [[ "$binary" == "$KSE_CAT" && " ${*} " == *" $KSE_STATE_PATH "* ]]; then case "$KSE_FIX_STATUS" in transition|enabled_start) stage=start ;; *) stage=reencrypt_finished ;; esac [[ "$KSE_FIX_INTEGRITY" == match ]] || fixture_hash=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb printf "%s-%s" "$stage" "$fixture_hash"; return 0 fi return 1 } _kse_privileged_quiet() { if [[ "$1" == "$KSE_K3S" && " ${*:2} " == *" kubectl get --raw=/readyz "* ]]; then { printf "<%s>" "$1"; shift; printf " <%s>" "$@"; printf "\n"; } >>"$KSE_FIX_ARG_LOG" [[ "$KSE_FIX_API_RC" == 0 ]]; return fi _kse_privileged_capture "$@" >/dev/null } k3s_secret_encryption_main "${@:2}" ' bash "$VALIDATOR_PATH" "$@" } # A user-owned executable can never satisfy the production root-owned binary # gate, even when it is absolute and executable. assert_fails /usr/bin/bash -c 'source "$1"; _kse_trusted_executable "$2"' \ bash "$VALIDATOR_PATH" "${fake_bin}/k3s" assert_validator_fixture_fails() { if run_validator_fixture "$@" >"${fixture_root}/validator-failure.out" 2>"${fixture_root}/validator-failure.err"; then fail "validator fixture unexpectedly succeeded: $*" fi } assert_sensitive_free() { local text="$1" [[ "$text" != *token* && "$text" != *password* && "$text" != *encryption-config* && "$text" != *'AES-CBC '* ]] } # Mutation probe: the sensitive-output assertion must reject this literal. assert_fails assert_sensitive_free 'token-password-encryption-config' 'sensitive assertion mutation probe' # A denied noninteractive sudo credential gate must be actionable and must stop # before the first privileged evidence command. : >"$FAKE_ARG_LOG" sudo_denial_rc=0 KSE_FIX_SUDO_RC=1 run_validator_fixture --expect-disabled \ >"${fixture_root}/sudo-denial.out" 2>"${fixture_root}/sudo-denial.err" || sudo_denial_rc=$? [[ "$sudo_denial_rc" -ne 0 ]] || fail 'sudo denial fixture unexpectedly succeeded' assert_eq 'ERROR: noninteractive sudo authorization is unavailable; run sudo -v in the same terminal, then rerun the validator' \ "$(<"${fixture_root}/sudo-denial.err")" 'sudo denial actionable error' assert_eq ' <--signal=TERM> <--kill-after=1s> <9s> <--non-interactive> <--validate>' \ "$(<"$FAKE_ARG_LOG")" 'sudo denial stops before privileged evidence' assert_fixture_stdout_safe() { local status="$1" output expected shift output="$(KSE_FIX_STATUS="$status" run_validator_fixture "$@" 2>"${fixture_root}/validator-safe.err")" || fail "$status safe stdout fixture" assert_sensitive_free "$output" || fail "$status stdout leaks sensitive fixture data" case "$status" in disabled) expected=$'Encryption status: Disabled\nRotation stage: none' ;; transition) expected=$'Encryption status: Disabled\nRotation stage: start' ;; enabled_start) expected=$'Encryption status: Enabled\nRotation stage: start' ;; finished) expected=$'Encryption status: Enabled\nRotation stage: reencrypt_finished' ;; esac [[ "$output" == *"$expected"* ]] || fail "$status stdout inventory class" } assert_failed_fixture_stdout_safe() { local status="$1" output rc=0 expected shift output="$(KSE_FIX_STATUS="$status" run_validator_fixture "$@" 2>"${fixture_root}/validator-safe-failure.err")" || rc=$? [[ "$rc" -ne 0 ]] || fail "$status fixture unexpectedly succeeds" assert_sensitive_free "$output" || fail "$status failing stdout leaks sensitive fixture data" case "$status" in active|mismatch) expected=$'Encryption status: Enabled\nRotation stage: unsafe' [[ "$output" == *"$expected"* ]] || fail "$status failing stdout inventory class" ;; malformed) [[ -z "$output" ]] || fail 'malformed status writes inventory stdout' ;; esac } validator_stdout='' : >"$FAKE_ARG_LOG" KSE_FIX_STATUS=disabled validator_stdout="$(run_validator_fixture --expect-disabled)" || fail 'disabled fixture accepts --expect-disabled' [[ "$validator_stdout" != *token* && "$validator_stdout" != *password* && "$validator_stdout" != *encryption-config* ]] || fail 'validator stdout leaks fixture secret material' [[ "$(<"$FAKE_ARG_LOG")" != *"${VALIDATOR_PATH}"* ]] || fail 'validator sudo-executes workspace code' [[ "$(<"$FAKE_ARG_LOG")" == *' <--version>'* ]] || fail 'validator does not use the absolute k3s binary' [[ "$(<"$FAKE_ARG_LOG")" == *' <-o> '* ]] || fail 'validator node inventory argv' [[ "$(<"$FAKE_ARG_LOG")" == *' <--raw=/readyz>'* ]] || fail 'validator readyz argv' KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-enabled KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-transition-start KSE_FIX_STATUS=transition run_validator_fixture --expect-transition-start >/dev/null || fail 'transition fixture expectation' KSE_FIX_STATUS=transition assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=enabled_start run_validator_fixture --expect-enabled >/dev/null || fail 'enabled/start fixture expectation' KSE_FIX_STATUS=enabled_start assert_validator_fixture_fails --expect-reencrypted KSE_FIX_STATUS=finished run_validator_fixture --expect-enabled >/dev/null || fail 'enabled/finished fixture expectation' KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match run_validator_fixture --expect-reencrypted >/dev/null || fail 'reencrypted integrity match fixture' KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match KSE_FIX_NODES=different_ready \ run_validator_fixture --expect-reencrypted-restore >/dev/null || fail 'isolated restore accepts a different ready server hostname' KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match KSE_FIX_NODES=not_ready \ assert_validator_fixture_fails --expect-reencrypted-restore KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=mismatch assert_validator_fixture_fails --expect-reencrypted KSE_FIX_STATUS=active assert_validator_fixture_fails KSE_FIX_STATUS=mismatch assert_validator_fixture_fails KSE_FIX_STATUS=malformed assert_validator_fixture_fails KSE_FIX_STATUS=disabled KSE_FIX_VERSION=v1.36.1+k3s1 assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=disabled KSE_FIX_VERSION=v1.36.2+k3s1suffix assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=disabled KSE_FIX_NODES=two_servers assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=disabled KSE_FIX_NODES=wrong_node assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=disabled KSE_FIX_NODES=not_ready assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=disabled KSE_FIX_API_RC=1 assert_validator_fixture_fails --expect-disabled KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir '' # Exercise the sensitive-output contract for every validator inventory class, # including states that correctly return non-zero. assert_fixture_stdout_safe disabled --expect-disabled assert_fixture_stdout_safe transition --expect-transition-start assert_fixture_stdout_safe enabled_start --expect-enabled assert_fixture_stdout_safe finished --expect-enabled assert_failed_fixture_stdout_safe active assert_failed_fixture_stdout_safe mismatch assert_failed_fixture_stdout_safe malformed # Failed expectations must leave a valid empty handoff untouched. rejected_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0700 "$rejected_output_dir" KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-enabled --verified-output-dir "$rejected_output_dir" [[ -z "$(find "$rejected_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'failed expectation leaves verified output' # Only a physical direct child of /tmp is accepted. mkdir -p "${rejected_output_dir}/nested" KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "$rejected_output_dir" KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "${rejected_output_dir}/nested" symlink_output_parent="/tmp/platform-k3s-encryption.link${RANDOM}${RANDOM}" ln -s -- "$fixture_root" "$symlink_output_parent" KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "${symlink_output_parent}/nested" race_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0750 "$race_output_dir" KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "$race_output_dir" verified_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0700 "$verified_output_dir" KSE_FIX_STATUS=disabled validator_stdout="$(run_validator_fixture --expect-disabled --verified-output-dir "$verified_output_dir")" || fail 'verified output fixture accepts safe directory' assert_eq 600 "$(stat --format='%a' "$verified_output_dir/inventory.env")" 'inventory handoff mode' assert_eq 600 "$(stat --format='%a' "$verified_output_dir/status.sha256")" 'status SHA handoff mode' [[ "$(<"$verified_output_dir/inventory.env")" == *'node_uid=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'* ]] || fail 'verified inventory omits immutable node UID' assert_eq 4786ad5101dd5c0de42b13d276e2cb0968448fd2b5c16abb6164ffeb69fafd2b "$(<"$verified_output_dir/status.sha256")" 'canonical disabled status SHA' [[ "$(<"$verified_output_dir/inventory.env")" != *token* && "$(<"$verified_output_dir/inventory.env")" != *password* && "$(<"$verified_output_dir/inventory.env")" != *encryption-config* ]] || fail 'verified inventory leaks fixture secret material' assert_sensitive_free "$(<"$verified_output_dir/inventory.env")$'\n'$(<"$verified_output_dir/status.sha256")" || fail 'disabled handoff leaks sensitive fixture data' transition_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0700 "$transition_output_dir" KSE_FIX_STATUS=transition run_validator_fixture --expect-transition-start --verified-output-dir "$transition_output_dir" >/dev/null || fail 'transition verified output fixture' assert_sensitive_free "$(<"$transition_output_dir/inventory.env")$'\n'$(<"$transition_output_dir/status.sha256")" || fail 'transition handoff leaks sensitive fixture data' enabled_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0700 "$enabled_output_dir" KSE_FIX_STATUS=enabled_start run_validator_fixture --expect-enabled --verified-output-dir "$enabled_output_dir" >/dev/null || fail 'enabled/start verified output fixture' assert_sensitive_free "$(<"$enabled_output_dir/inventory.env")$'\n'$(<"$enabled_output_dir/status.sha256")" || fail 'enabled/start handoff leaks sensitive fixture data' finished_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0700 "$finished_output_dir" KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match run_validator_fixture --expect-reencrypted --verified-output-dir "$finished_output_dir" >/dev/null || fail 'reencrypted verified output fixture' assert_eq 9f963cbcdaa79fd76d8c6df50d1108cee96be0e9e947308cb18a033c8eb24595 "$(<"$finished_output_dir/status.sha256")" 'canonical reencrypted status SHA' [[ "$(<"$finished_output_dir/inventory.env")" != *token* && "$(<"$finished_output_dir/inventory.env")" != *password* && "$(<"$finished_output_dir/inventory.env")" != *encryption-config* ]] || fail 'reencrypted handoff leaks fixture secret material' assert_sensitive_free "$(<"$finished_output_dir/inventory.env")$'\n'$(<"$finished_output_dir/status.sha256")" || fail 'reencrypted handoff leaks sensitive fixture data' # Caller PATH commands must never participate in validation or handoff. malicious_bin="${fixture_root}/malicious-bin" malicious_marker="${fixture_root}/malicious-path-used" mkdir -p "$malicious_bin" for command_name in dirname stat find id timeout sudo; do printf '%s\n' '#!/usr/bin/env bash' ": >\"${malicious_marker}\"" 'exit 99' >"${malicious_bin}/${command_name}" chmod 0700 "${malicious_bin}/${command_name}" done malicious_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" chmod 0700 "$malicious_output_dir" PATH="${malicious_bin}:${ORIGINAL_PATH}" KSE_FIX_STATUS=disabled run_validator_fixture --expect-disabled --verified-output-dir "$malicious_output_dir" >/dev/null || fail 'validator ignores caller PATH commands' [[ ! -e "$malicious_marker" ]] || fail 'validator trusted a caller PATH command' rm -rf -- "$malicious_output_dir" # The host drop-in must contain exactly the two encryption settings and no # Kubernetes resource metadata or unrelated configuration. awk ' /^[[:space:]]*($|#)/ { next } $0 == "secrets-encryption: true" { enabled++; next } $0 == "secrets-encryption-provider: aescbc" { provider++; next } { unexpected++ } END { exit !(enabled == 1 && provider == 1 && unexpected == 0) } ' infrastructure/security/k3s/40-secrets-encryption.yaml # Bootstrap state-machine regression coverage. The production script is # sourced and its host boundaries are replaced in this subshell only; there is # deliberately no environment-controlled fake path in production. readonly BOOTSTRAP_PATH="${REPOSITORY_ROOT}/scripts/bootstrap/apply-k3s-secret-encryption.sh" bootstrap_fixture_root="${fixture_root}/bootstrap" bootstrap_backup_root="${bootstrap_fixture_root}/recovery" mkdir -p "$bootstrap_backup_root" run_bootstrap_fixture() ( local scenario="$1" shift local log="${bootstrap_fixture_root}/${scenario}.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_load_inventory() { printf 'inventory\n' >>"$log" case "$scenario" in disabled_sqlite|local_success|local_plain_ext4|local_wrong_outer_disk|pre_capacity_failure|post_capacity_failure|backup_failure|service_recovery_failure|post_backup_failure|marker_failure|final_validation_failure|timeout_finished|timeout_start|root_failure|enable_failure|dropin_failure|initial_restart_failure|final_restart_failure|estimate|estimate_invalid) printf 'disabled_no_config|none|sqlite|none|none|mismatch|mismatch\n' ;; disabled_etcd) printf 'disabled_no_config|none|embedded-etcd|none|none|mismatch|mismatch\n' ;; disabled_backend_drift_external|disabled_backend_drift_ambiguous) if [[ -e "${bootstrap_fixture_root}/${scenario}.counter" ]]; then if [[ "$scenario" == disabled_backend_drift_external ]]; then printf 'disabled_no_config|none|external|none|none|mismatch|mismatch\n' else printf 'disabled_no_config|none|ambiguous|none|none|mismatch|mismatch\n' fi else : >"${bootstrap_fixture_root}/${scenario}.counter" printf 'disabled_no_config|none|sqlite|none|none|mismatch|mismatch\n' fi ;; mismatch) printf 'hash_mismatch|unsafe|sqlite|invalid|ambiguous|mismatch|mismatch\n' ;; external) printf 'disabled_no_config|none|external|none|none|mismatch|mismatch\n' ;; ambiguous_backend) printf 'disabled_no_config|none|ambiguous|none|none|mismatch|mismatch\n' ;; enabled_finished) printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|match|match\n' ;; enabled_finished_drift_provider|enabled_finished_drift_owner) if [[ -e "${bootstrap_fixture_root}/${scenario}.counter" ]]; then if [[ "$scenario" == enabled_finished_drift_provider ]]; then printf 'enabled_stable|reencrypt_finished|sqlite|secretbox|secretbox/config-owner|match|match\n' else printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|ambiguous|match|match\n' fi else : >"${bootstrap_fixture_root}/${scenario}.counter" printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|match|match\n' fi ;; enabled_wrong_provider) printf 'enabled_stable|reencrypt_finished|sqlite|secretbox|secretbox/config-owner|match|match\n' ;; enabled_wrong_owner) printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|ambiguous|match|match\n' ;; enabled_bad_integrity) printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|mismatch|match\n' ;; enabled_start|enabled_start_rotate) printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|match|match\n' ;; enabled_start_drift) if [[ -e "${bootstrap_fixture_root}/enabled-start-drift.counter" ]]; then printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|mismatch|mismatch\n' else : >"${bootstrap_fixture_root}/enabled-start-drift.counter" printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|match|match\n' fi ;; *) return 97 ;; esac } # Keep the legacy override during the RED stage so the old implementation # cannot reach a real terminal. The new implementation must use the split # confirmation boundaries below instead. _kseb_confirm_execute() { printf 'legacy-confirm\n' >>"$log"; } _kseb_prepare_execute_context() { printf 'default\n'; } _kseb_confirm_apply() { printf 'APPLY\n' >>"$log"; } _kseb_validate_backup_root() { printf 'generic-root\n' >>"$log" [[ "$scenario" != root_failure ]] } _kseb_validate_recovery_policy() { local policy="$1" printf 'policy-root:%s\n' "$policy" >>"$log" case "$scenario" in local_plain_ext4|local_wrong_outer_disk) return 1 ;; *) return 0 ;; esac } _kseb_confirm_recovery() { local context="$1" policy="$2" [[ "$context" == default ]] printf 'RECOVERY\nENCRYPTED\n' >>"$log" if [[ "$policy" == local-separate-disk-luks ]]; then printf 'LOCAL_RISK_ACCEPTED\n' >>"$log" fi } _kseb_backup_phase() { local phase="$1" backend="$2" if [[ "$scenario" == pre_capacity_failure && "$phase" == pre ]]; then printf 'capacity:pre\n' >>"$log" return 1 fi if [[ "$scenario" == post_capacity_failure && "$phase" == post ]]; then printf 'capacity:post\n' >>"$log" return 1 fi printf 'backup:%s:%s\n' "$phase" "$backend" >>"$log" [[ "$scenario" != backup_failure || "$phase" != pre ]] && [[ "$scenario" != post_backup_failure || "$phase" != post ]] || return $? if [[ "$scenario" == service_recovery_failure && "$phase" == pre ]]; then return 75; fi if [[ "$scenario" == marker_failure && "$phase" == post ]]; then return 76; fi } _kseb_enable() { printf 'enable\n' >>"$log"; [[ "$scenario" != enable_failure ]]; } _kseb_install_dropin() { printf 'install-dropin\n' >>"$log"; [[ "$scenario" != dropin_failure ]]; } _kseb_restart() { local counter="${bootstrap_fixture_root}/${scenario}.restart-counter" printf 'restart\n' >>"$log" if [[ "$scenario" == initial_restart_failure ]]; then return 1; fi if [[ "$scenario" == final_restart_failure ]]; then if [[ -e "$counter" ]]; then return 1; fi : >"$counter" fi } _kseb_rotate_keys() { printf 'rotate\n' >>"$log" case "$scenario" in timeout_finished|timeout_start) return 124 ;; *) return 0 ;; esac } _kseb_read_status_json() { local stage counter="${bootstrap_fixture_root}/${scenario}.status-counter" case "$scenario" in timeout_finished) if [[ -e "$counter" ]]; then stage=reencrypt_finished; else stage=reencrypt_active; : >"$counter"; fi ;; timeout_start) stage=start ;; *) stage=reencrypt_finished ;; esac printf 'poll:%s\n' "$stage" >>"$log" printf '{"stage":"%s","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}\n' "$stage" } sleep() { :; } _kseb_validate_expectation() { printf 'validate:%s\n' "$1" >>"$log" [[ "$scenario" != final_validation_failure || "$1" != --expect-reencrypted ]] } _kseb_dry_run() { printf 'dry-run\n' >>"$log"; } _kseb_estimate_phase_bytes() { printf 'estimate\n' >>"$log" if [[ "$scenario" == estimate_invalid ]]; then printf '9223372036854775808\n' else printf '1048576\n' fi } apply_k3s_secret_encryption_main "$@" \ >"${bootstrap_fixture_root}/${scenario}.stdout" \ 2>"${bootstrap_fixture_root}/${scenario}.stderr" ) assert_bootstrap_order() { local scenario="$1" expected="$2" assert_eq "$expected" "$(<"${bootstrap_fixture_root}/${scenario}.log")" "$scenario bootstrap order" } # These tests protect the required, closed recovery-policy parser and split # confirmation dispatch. Missing or malformed policy input must stop before # inventory collection, confirmation, or mutation dispatch. assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" assert_bootstrap_order disabled_sqlite '' assert_exit 2 run_bootstrap_fixture disabled_sqlite --recovery-policy encrypted-off-host assert_bootstrap_order disabled_sqlite '' assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy unknown assert_bootstrap_order disabled_sqlite '' assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host --recovery-policy encrypted-off-host assert_bootstrap_order disabled_sqlite '' assert_succeeds run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order disabled_sqlite $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' assert_succeeds run_bootstrap_fixture local_success --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy local-separate-disk-luks assert_bootstrap_order local_success $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks\nRECOVERY\nENCRYPTED\nLOCAL_RISK_ACCEPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' assert_fails run_bootstrap_fixture local_plain_ext4 --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy local-separate-disk-luks assert_bootstrap_order local_plain_ext4 $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks' assert_fails run_bootstrap_fixture local_wrong_outer_disk --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy local-separate-disk-luks assert_bootstrap_order local_wrong_outer_disk $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks' # The production change that makes these probes pass is the real local policy # boundary: exact inner root, one validator call, then pinned-root revalidation. probe_recovery_policy_dispatch() ( local policy="$1" root="$2" scenario="$3" rc=0 local log="${bootstrap_fixture_root}/policy-dispatch-${scenario}.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_local_recovery_root() { printf '/fixture/local-recovery\n'; } _kseb_run_local_recovery_validator() { printf 'local-validator\n' >>"$log" [[ "$scenario" != validator_failure ]] } _kseb_revalidate_backup_root() { printf 'root-revalidated\n' >>"$log" [[ "$scenario" != identity_drift ]] } _kseb_validate_recovery_policy "$policy" "$root" || rc=$? printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'local-validator\nroot-revalidated\nrc=0' \ "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery success)" \ 'local recovery policy validates exact root and revalidates identity' assert_eq 'rc=1' \ "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery/child child_root)" \ 'local recovery policy rejects a child root before validation' assert_eq $'local-validator\nrc=1' \ "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery validator_failure)" \ 'local recovery policy rejects validator failure before root revalidation' assert_eq $'local-validator\nroot-revalidated\nrc=1' \ "$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery identity_drift)" \ 'local recovery policy rejects root identity drift' assert_eq 'rc=0' \ "$(probe_recovery_policy_dispatch encrypted-off-host /fixture/off-host off_host)" \ 'off-host policy never dispatches the local validator' local_contract_fixture="${bootstrap_fixture_root}/local-recovery-contract.env" awk -F= ' $1 == "K3SLR_INNER_MOUNT" { print "K3SLR_INNER_MOUNT=/fixture/contract-root"; next } { print } ' infrastructure/security/k3s/local-recovery.env >"$local_contract_fixture" probe_local_recovery_contract_binding() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_local_recovery_contract() { printf '%s\n' "$local_contract_fixture"; } _kseb_local_recovery_root ) assert_eq '/fixture/contract-root' "$(probe_local_recovery_contract_binding)" \ 'local policy root is read from the authoritative recovery contract' probe_recovery_policy_contract_drift() ( local marker="${bootstrap_fixture_root}/contract-drift.marker" rc=0 local log="${bootstrap_fixture_root}/contract-drift.log" rm -f -- "$marker"; : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_local_recovery_root() { if [[ -e "$marker" ]]; then printf '/fixture/root-after\n' else : >"$marker" printf '/fixture/root-before\n' fi } _kseb_run_local_recovery_validator() { printf 'local-validator\n' >>"$log"; } _kseb_revalidate_backup_root() { printf 'unexpected-root-revalidation\n' >>"$log"; } _kseb_validate_recovery_policy local-separate-disk-luks /fixture/root-before || rc=$? printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'local-validator\nrc=1' "$(probe_recovery_policy_contract_drift)" \ 'local policy rejects contract root drift after validator success' duplicate_contract_fixture="${bootstrap_fixture_root}/local-recovery-contract-duplicate.env" awk ' { print } END { print "K3SLR_INNER_MOUNT=/fixture/duplicate-root" } ' infrastructure/security/k3s/local-recovery.env >"$duplicate_contract_fixture" probe_duplicate_local_recovery_contract() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_local_recovery_contract() { printf '%s\n' "$duplicate_contract_fixture"; } _kseb_local_recovery_root ) assert_fails probe_duplicate_local_recovery_contract probe_local_validator_boundary() ( local scenario="$1" rc=0 local command_log="${bootstrap_fixture_root}/local-validator-${scenario}.argv" local public_out="${bootstrap_fixture_root}/local-validator-${scenario}.stdout" local public_err="${bootstrap_fixture_root}/local-validator-${scenario}.stderr" : >"$command_log"; : >"$public_out"; : >"$public_err" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" if [[ "$scenario" == cleanup_failure ]]; then _kseb_safe_remove_local_validator_handoff() { /usr/bin/rm -- "$1/stdout" "$1/stderr" >/dev/null 2>&1 || true /usr/bin/rmdir -- "$1" >/dev/null 2>&1 || true return 1 } fi _kseb_local_validator_command() { printf '%s\n' "$@" >"$command_log" case "$scenario" in success|cleanup_failure) printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' ;; extra_stdout) printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\nUNEXPECTED\n' ;; stderr_output) printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' printf 'RAW-IDENTIFIER-SENTINEL\n' >&2 ;; command_failure) return 1 ;; oversize) /usr/bin/yes X | /usr/bin/head --bytes=16384 ;; *) return 97 ;; esac } _kseb_run_local_recovery_validator >"$public_out" 2>"$public_err" || rc=$? [[ ! -s "$public_out" && ! -s "$public_err" ]] || return 1 printf 'rc=%s\n' "$rc" ) assert_eq 'rc=0' "$(probe_local_validator_boundary success)" \ 'local validator accepts only its exact safe success record' assert_eq $'/usr/bin/env\n-i\nPATH=/usr/sbin:/usr/bin:/sbin:/bin\nLC_ALL=C\n/usr/bin/timeout\n--signal=TERM\n--kill-after=1s\n60s\n/usr/bin/bash\n--noprofile\n--norc\n--\n'"${REPOSITORY_ROOT}"$'/scripts/validate/k3s-local-recovery.sh\n--expect-open' \ "$(<"${bootstrap_fixture_root}/local-validator-success.argv")" \ 'local validator command uses fixed binaries, clean environment, and expect-open' assert_eq 'rc=1' "$(probe_local_validator_boundary extra_stdout)" \ 'local validator rejects extra stdout without exposing it' assert_eq 'rc=1' "$(probe_local_validator_boundary stderr_output)" \ 'local validator rejects stderr without exposing it' assert_eq 'rc=1' "$(probe_local_validator_boundary command_failure)" \ 'local validator rejects a nonzero command result' assert_eq 'rc=1' "$(probe_local_validator_boundary cleanup_failure)" \ 'local validator treats capture cleanup failure as policy failure' local_validator_tmp_count() { /usr/bin/find /tmp -maxdepth 1 -type d -name 'platform-k3s-local-validator.*' -printf . | /usr/bin/wc -c } local_validator_tmp_before="$(local_validator_tmp_count)" assert_eq 'rc=1' "$(probe_local_validator_boundary oversize)" \ 'local validator bounds oversized output and returns failure' assert_eq "$local_validator_tmp_before" "$(local_validator_tmp_count)" \ 'oversized local validator output leaves no temporary directory' probe_local_validator_result_matrix() ( local scenario="$1" directory="${bootstrap_fixture_root}/validator-result-${1}" local stdout_file="$directory/stdout" stderr_file="$directory/stderr" command_rc=0 rm -rf -- "$directory"; mkdir -m 0700 -- "$directory" : >"$stderr_file" case "$scenario" in exact|timeout|signal|stderr) printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" ;; embedded_nul) printf 'Recovery device: matc\0\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" ;; carriage_return) printf 'Recovery device: matc\r\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" ;; short) printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file" /usr/bin/truncate --size=85 -- "$stdout_file" ;; long) printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\nX' >"$stdout_file" ;; *) return 97 ;; esac case "$scenario" in timeout) command_rc=124 ;; signal) command_rc=137 ;; stderr) printf 'unexpected\n' >"$stderr_file" ;; esac chmod 0600 -- "$stdout_file" "$stderr_file" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_local_validator_result_is_exact "$command_rc" "$stdout_file" "$stderr_file" ) assert_succeeds probe_local_validator_result_matrix exact assert_fails probe_local_validator_result_matrix embedded_nul assert_fails probe_local_validator_result_matrix carriage_return assert_fails probe_local_validator_result_matrix short assert_fails probe_local_validator_result_matrix long assert_fails probe_local_validator_result_matrix timeout assert_fails probe_local_validator_result_matrix signal assert_fails probe_local_validator_result_matrix stderr probe_local_validator_exact_cleanup() ( local scenario="$1" directory directory="$(/usr/bin/mktemp -d /tmp/platform-k3s-local-validator.XXXXXX)" || return 1 case "$directory" in /tmp/platform-k3s-local-validator.[A-Za-z0-9]*) ;; *) return 1 ;; esac trap '/usr/bin/rm -f -- "$directory/stdout" "$directory/stderr" "$directory/extra"; /usr/bin/rmdir -- "$directory" 2>/dev/null || true' EXIT : >"$directory/stdout"; : >"$directory/stderr" if [[ "$scenario" == extra ]]; then : >"$directory/extra"; fi chmod 0600 -- "$directory"/* # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" if [[ "$scenario" == success ]]; then _kseb_safe_remove_local_validator_handoff "$directory" || return 1 [[ ! -e "$directory" ]] else if _kseb_safe_remove_local_validator_handoff "$directory"; then return 1; fi [[ -d "$directory" && -f "$directory/extra" && ! -e "$directory/stdout" && ! -e "$directory/stderr" ]] fi ) assert_succeeds probe_local_validator_exact_cleanup success assert_succeeds probe_local_validator_exact_cleanup extra probe_recovery_confirmation() ( local policy="$1" answers="$2" context_after="${3:-default}" rc=0 output # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_current_context() { printf '%s\n' "$context_after"; } output="$(_kseb_confirm_recovery default "$policy" <<<"$answers" 2>&1)" || rc=$? printf 'rc=%s|offhost=%s|localrisk=%s\n' "$rc" \ "$([[ "$output" == *off-host* ]] && printf 1 || printf 0)" \ "$([[ "$output" == *LOCAL_RISK_ACCEPTED* ]] && printf 1 || printf 0)" ) assert_eq 'rc=0|offhost=1|localrisk=0' \ "$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nENCRYPTED default')" \ 'off-host policy keeps its two explicit confirmations' assert_eq 'rc=0|offhost=0|localrisk=1' \ "$(probe_recovery_confirmation local-separate-disk-luks $'RECOVERY default\nENCRYPTED default\nLOCAL_RISK_ACCEPTED default')" \ 'local policy uses truthful encryption and local-risk confirmations' assert_eq 'rc=1|offhost=0|localrisk=1' \ "$(probe_recovery_confirmation local-separate-disk-luks $'RECOVERY default\nENCRYPTED default')" \ 'local policy refuses a missing local-risk confirmation' assert_eq 'rc=1|offhost=0|localrisk=0' \ "$(probe_recovery_confirmation encrypted-off-host $'WRONG default\nENCRYPTED default')" \ 'recovery confirmation rejects the wrong RECOVERY record' assert_eq 'rc=1|offhost=1|localrisk=0' \ "$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nWRONG default')" \ 'recovery confirmation rejects the wrong ENCRYPTED record' assert_eq 'rc=1|offhost=1|localrisk=0' \ "$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nENCRYPTED default' changed)" \ 'recovery confirmation rejects context drift' probe_apply_confirmation() ( local answer="$1" rc=0 # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_confirm_apply default <<<"$answer" >/dev/null 2>&1 || rc=$? printf 'rc=%s\n' "$rc" ) assert_eq 'rc=0' "$(probe_apply_confirmation 'APPLY default')" 'APPLY confirmation exact success' assert_eq 'rc=1' "$(probe_apply_confirmation 'APPLY other')" 'APPLY confirmation rejects another context' assert_eq 'rc=1' "$(probe_apply_confirmation 'apply default')" 'APPLY confirmation is case-sensitive' # Existing bootstrap state-machine regressions continue below. Once the # implementation exists, these literals protect the entire mutation order. assert_succeeds run_bootstrap_fixture disabled_etcd --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host [[ "$(<"${bootstrap_fixture_root}/disabled_etcd.log")" == *$'backup:pre:embedded-etcd'*$'backup:post:embedded-etcd'* ]] || fail 'embedded-etcd backend routing' assert_succeeds run_bootstrap_fixture timeout_finished --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order timeout_finished $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_active\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' assert_eq 1 "$(awk '$0 == "rotate" { count++ } END { print count + 0 }' "${bootstrap_fixture_root}/timeout_finished.log")" 'timeout never retries rotate-keys' assert_fails run_bootstrap_fixture timeout_start --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order timeout_start $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:start' assert_fails run_bootstrap_fixture mismatch --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order mismatch 'inventory' assert_fails run_bootstrap_fixture external --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order external 'inventory' assert_fails run_bootstrap_fixture ambiguous_backend --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order ambiguous_backend 'inventory' assert_fails run_bootstrap_fixture backup_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order backup_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite' assert_fails run_bootstrap_fixture root_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order root_failure $'inventory\nAPPLY\ngeneric-root' assert_fails run_bootstrap_fixture final_validation_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host [[ "$(<"${bootstrap_fixture_root}/final_validation_failure.log")" != *'backup:post:'* ]] || fail 'post backup before final verification' assert_succeeds run_bootstrap_fixture enabled_finished --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_finished $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory\nbackup:post:sqlite' assert_fails run_bootstrap_fixture enabled_wrong_provider --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_wrong_provider 'inventory' assert_fails run_bootstrap_fixture enabled_wrong_owner --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_wrong_owner 'inventory' assert_fails run_bootstrap_fixture enabled_bad_integrity --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_bad_integrity 'inventory' assert_fails run_bootstrap_fixture enabled_start --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_start 'inventory' assert_succeeds run_bootstrap_fixture enabled_start_rotate --execute --rotate-existing --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order enabled_start_rotate $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-enabled\nbackup:pre:sqlite\ninventory\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite' assert_succeeds run_bootstrap_fixture disabled_sqlite assert_bootstrap_order disabled_sqlite $'inventory\ndry-run' assert_fails run_bootstrap_fixture disabled_sqlite --rotate-existing assert_fails run_bootstrap_fixture disabled_sqlite --execute --backup-root relative/path --recovery-policy encrypted-off-host assert_fails run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \ --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host # These CLI regressions catch estimator dispatch through any mutating branch, # accepting an invalid estimate, or emitting anything except the public line. assert_succeeds run_bootstrap_fixture estimate --estimate-recovery-bytes assert_eq 'phase_bytes=1048576' "$(<"${bootstrap_fixture_root}/estimate.stdout")" \ 'read-only phase estimator output' assert_bootstrap_order estimate $'inventory\nestimate' assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --execute assert_exit 2 run_bootstrap_fixture estimate --execute --estimate-recovery-bytes assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --rotate-existing assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --backup-root "$bootstrap_backup_root" assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --estimate-recovery-bytes assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --help assert_exit 2 run_bootstrap_fixture estimate --help --estimate-recovery-bytes assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy auto assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy encrypted-off-host assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy local-separate-disk-luks assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --unknown assert_fails run_bootstrap_fixture estimate_invalid --estimate-recovery-bytes assert_eq '' "$(<"${bootstrap_fixture_root}/estimate_invalid.stdout")" \ 'invalid estimate has no public output' probe_estimator_fixed_sources() ( local backend="$1" log="${bootstrap_fixture_root}/estimate-${1}-sources.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_estimator_source_kind() { printf 'present\n'; } _kseb_estimator_source_identity() { case "$1" in /usr/lib/systemd/system/k3s.service|/lib/systemd/system/k3s.service) printf '41:8\n' ;; *) printf '41:%s\n' "$(printf '%s' "$1" | /usr/bin/cksum | awk '{print $1}')" ;; esac } _kseb_privileged_estimator_du() { printf '%s\n' "$1" >>"$log" printf '1\t%s\n' "$1" } _kseb_estimate_phase_bytes "$backend" ) assert_eq 16777224 "$(probe_estimator_fixed_sources sqlite)" \ 'SQLite estimator sums fixed backup inputs once' assert_eq $'/var/lib/rancher/k3s/server/db\n/var/lib/rancher/k3s/server/token\n/etc/rancher/k3s\n/etc/systemd/system/k3s.service\n/etc/systemd/system/k3s.service.env\n/etc/systemd/system/k3s.service.d\n/usr/lib/systemd/system/k3s.service\n/var/lib/rancher/k3s/server/cred/encryption-config.json' \ "$(<"${bootstrap_fixture_root}/estimate-sqlite-sources.log")" \ 'SQLite estimator reads each effective fixed source once' assert_eq 16777224 "$(probe_estimator_fixed_sources embedded-etcd)" \ 'embedded-etcd estimator sums fixed backup inputs once' assert_eq '/var/lib/rancher/k3s/server/db/etcd' \ "$(sed -n '1p' "${bootstrap_fixture_root}/estimate-embedded-etcd-sources.log")" \ 'embedded-etcd estimator uses current local data directory' probe_estimator_du_contract() ( local scenario="$1" marker="${bootstrap_fixture_root}/estimate-identity-${1}.counter" rm -f -- "$marker" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_estimator_source_kind() { printf 'present\n'; } _kseb_estimator_source_identity() { if [[ "$scenario" == identity_change && -e "$marker" ]]; then printf '41:2\n' else printf '41:1\n'; : >"$marker"; fi } _kseb_privileged_estimator_du() { case "$scenario" in malformed) printf 'not-a-number\t%s\n' "$1" ;; multiline) printf '1\t%s\n2\t%s\n' "$1" "$1" ;; trailing_blank) printf '1\t%s\n\n' "$1" ;; wrong_path) printf '1\t/etc/shadow\n' ;; *) printf '1\t%s\n' "$1" ;; esac } _kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/token required ) assert_fails probe_estimator_du_contract malformed assert_fails probe_estimator_du_contract multiline assert_fails probe_estimator_du_contract trailing_blank assert_fails probe_estimator_du_contract wrong_path assert_fails probe_estimator_du_contract identity_change probe_estimator_symlink_parent() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_estimator_source_kind() { printf 'present\n'; } _kseb_privileged_estimator_lstat() { case "$1" in /|/var) printf '41:1|41ed\n' ;; /var/lib) printf '41:2|a1ff\n' ;; *) printf '41:3|41ed\n' ;; esac } _kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; } _kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/db required ) assert_fails probe_estimator_symlink_parent probe_estimator_numeric_mode_metadata() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_estimator_source_kind() { printf 'present\n'; } _kseb_privileged_estimator_lstat() { if [[ "$1" == /var/lib/rancher/k3s/server/token ]]; then printf '41:9|81a4\n' else printf '41:8|41ed\n' fi } _kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; } _kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/token required ) assert_eq 1 "$(probe_estimator_numeric_mode_metadata)" \ 'numeric stat mode accepts directories and a regular-file source' probe_estimator_numeric_stat_argv() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_estimator_capture() { printf '<%s>' "$@" printf '\n' } _kseb_privileged_estimator_lstat /var/lib/rancher/k3s/server/token ) assert_eq '<--format=%d:%i|%f><-->' \ "$(probe_estimator_numeric_stat_argv 2>/dev/null || true)" \ 'privileged stat uses locale-independent numeric mode data' probe_estimator_out_of_bound() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_estimator_source_kind() { printf 'present\n'; } _kseb_estimator_source_identity() { printf '41:1\n'; } _kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; } _kseb_estimate_source_bytes sqlite /tmp/not-a-k3s-source required ) assert_fails probe_estimator_out_of_bound probe_estimator_overflow() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_estimator_source_records() { printf 'required|/var/lib/rancher/k3s/server/db\n' printf 'required|/var/lib/rancher/k3s/server/token\n' } _kseb_estimator_source_probe() { [[ "$1" == */server/db ]] && printf '41:1\n' || printf '41:2\n' } _kseb_estimate_source_bytes() { [[ "$2" == */server/db ]] && printf '9223372036854775800\n' || printf '8\n' } _kseb_estimate_phase_bytes sqlite ) assert_fails probe_estimator_overflow assert_fails probe_estimator_fixed_sources external # The production change that makes these arithmetic cases pass is an # overflow-safe P + ceil(P / 4) reserve calculation. Expected values are # hand-derived literals rather than values computed with production helpers. probe_required_recovery_bytes() ( local phase="$1" phase_bytes="$2" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_required_recovery_bytes "$phase" "$phase_bytes" ) assert_eq 10737418240 "$(probe_required_recovery_bytes post 0)" 'zero-byte post reserve' assert_eq 10737418240 "$(probe_required_recovery_bytes pre 0)" 'zero-byte pre reserve' assert_eq 10737418242 "$(probe_required_recovery_bytes post 1)" 'one-byte post reserve rounds upward' assert_eq 10737418244 "$(probe_required_recovery_bytes pre 1)" 'one-byte pre reserve rounds upward twice' assert_eq 10737418245 "$(probe_required_recovery_bytes post 0004)" 'normalized four-byte post reserve' assert_eq 10737418250 "$(probe_required_recovery_bytes pre 4)" 'four-byte pre reserve' assert_eq 11811160065 "$(probe_required_recovery_bytes post 858993460)" 'post capacity literal' assert_eq 12884901890 "$(probe_required_recovery_bytes pre 858993460)" 'pre capacity literal' assert_eq 9223372036854775807 "$(probe_required_recovery_bytes post 7378697620893886053)" \ 'maximum signed post reserve' assert_eq 9223372036854775806 "$(probe_required_recovery_bytes pre 3689348810446943026)" \ 'maximum signed pre reserve' assert_fails probe_required_recovery_bytes post 7378697620893886054 assert_fails probe_required_recovery_bytes pre 3689348810446943027 assert_fails probe_required_recovery_bytes post 7378697629483820646 assert_fails probe_required_recovery_bytes post 9223372036854775807 assert_fails probe_required_recovery_bytes invalid 1 assert_fails probe_required_recovery_bytes pre -1 assert_fails probe_required_recovery_bytes post 1.25 assert_fails probe_required_recovery_bytes post 9223372036854775808 probe_phase_capacity_gate() ( local phase="$1" fixture_phase_bytes="$2" fixture_free_bytes="$3" rc=0 local log="${bootstrap_fixture_root}/capacity-${phase}-${fixture_free_bytes}.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_estimate_phase_bytes() { printf 'estimate:%s\n' "$1" >>"$log"; printf '%s\n' "$fixture_phase_bytes"; } _kseb_privileged_free_bytes() { printf 'free:%s\n' "$1" >>"$log"; printf '%s\n' "$fixture_free_bytes"; } _kseb_require_phase_capacity "$phase" sqlite /fixture-root || rc=$? printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \ "$(probe_phase_capacity_gate pre 858993460 12884901890)" 'pre capacity exact threshold' assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \ "$(probe_phase_capacity_gate pre 858993460 12884901889)" 'pre capacity one byte short' assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \ "$(probe_phase_capacity_gate post 858993460 11811160065)" 'post capacity exact threshold' assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \ "$(probe_phase_capacity_gate post 858993460 11811160064)" 'post capacity one byte short' assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \ "$(probe_phase_capacity_gate post 858993460 00011811160065)" 'free-byte normalization' assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \ "$(probe_phase_capacity_gate post 858993460 9223372036854775808)" 'free-byte signed overflow rejection' # Focused regression: API-derived metadata must be captured before entering the # backend routine that stops SQLite. The backend deliberately returns failure # before any filesystem command, so this exercises production orchestration # without a backup write or host call. probe_backup_metadata_order() ( local log="${bootstrap_fixture_root}/metadata-order.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_require_phase_capacity() { printf 'capacity:%s:%s:%s\n' "$1" "$2" "$3" >>"$log" } _kseb_secret_count() { printf 'secret-count\n' >>"$log"; printf '7\n'; } _kseb_version() { printf 'version\n' >>"$log"; printf 'v1.36.2+k3s1\n'; } _kseb_revalidate_backup_root() { :; } _kseb_pinned_phase_transaction() { printf 'transaction:%s:count=%s:version=%s\n' "${5:-missing}" "${7:-missing}" "${8:-missing}" >>"$log" return 1 } _kseb_backup_phase pre sqlite /fixture-root /fixture-bundle \ 00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || true _kseb_backup_phase post sqlite /fixture-root /fixture-bundle \ 00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || true command cat -- "$log" ) assert_eq $'capacity:pre:sqlite:/fixture-root\nsecret-count\nversion\ntransaction:pre:count=7:version=v1.36.2+k3s1\ncapacity:post:sqlite:/fixture-root\nsecret-count\nversion\ntransaction:post:count=7:version=v1.36.2+k3s1' \ "$(probe_backup_metadata_order)" 'pre and post each recalculate capacity before stopped-service backend' probe_backup_capacity_failure() ( local phase="$1" log="${bootstrap_fixture_root}/capacity-failure-${1}.log" rc=0 : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_revalidate_backup_root() { :; } _kseb_require_phase_capacity() { printf 'capacity:%s\n' "$1" >>"$log"; return 1; } _kseb_secret_count() { printf 'unexpected-secret-count\n' >>"$log"; } _kseb_version() { printf 'unexpected-version\n' >>"$log"; } _kseb_pinned_phase_transaction() { printf 'unexpected-transaction\n' >>"$log"; } _kseb_backup_phase "$phase" sqlite /fixture-root /fixture-bundle \ 00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || rc=$? printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'capacity:pre\nrc=1' "$(probe_backup_capacity_failure pre)" \ 'pre capacity failure stops before API metadata and backup transaction' assert_eq $'capacity:post\nrc=1' "$(probe_backup_capacity_failure post)" \ 'post capacity failure stops before API metadata and backup transaction' # Focused regression: the bootstrap owns a bounded API waiter whose command # boundary can use the absolute k3s binary. The fake readiness probe is the # only injected dependency and succeeds on its second call. probe_bootstrap_api_waiter() ( local log="${bootstrap_fixture_root}/api-wait.log" counter="${bootstrap_fixture_root}/api-wait.counter" : >"$log" rm -f -- "$counter" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_api_ready_once() { printf 'probe\n' >>"$log" if [[ -e "$counter" ]]; then return 0; fi : >"$counter" return 1 } sleep() { printf 'sleep:%s\n' "$1" >>"$log"; } _kseb_wait_for_api command cat -- "$log" ) assert_eq $'probe\nsleep:10\nprobe' "$(probe_bootstrap_api_waiter 2>/dev/null || true)" \ 'bootstrap bounded absolute-command API waiter' # A root-only recovery mount is not traversable by the invoking user, so phase # collision decisions must be made from privileged evidence rather than shell # -e checks. This pure production helper makes those decisions testable. probe_phase_layout_policy() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_phase_layout_allowed pre absent absent absent || return 1 if _kseb_phase_layout_allowed pre directory '0:0:700:directory' absent; then return 1; fi _kseb_phase_layout_allowed post absent absent absent || return 1 _kseb_phase_layout_allowed post directory '0:0:700:directory' directory || return 1 if _kseb_phase_layout_allowed post directory '0:0:755:directory' directory; then return 1; fi if _kseb_phase_layout_allowed post directory '0:0:700:directory' absent; then return 1; fi ) assert_succeeds probe_phase_layout_policy # Enabled/start can drift during confirmation or the pre-backup. The second # inventory is the immediate pre-rotation gate and must stop all later mutation. assert_fails run_bootstrap_fixture enabled_start_drift --execute --rotate-existing --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order enabled_start_drift $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-enabled\nbackup:pre:sqlite\ninventory' # The guarded SQLite lifecycle must attempt start after every stop attempt, # including a stop error or copy failure, and only run recovery after a # confirmed start. These dependencies are local functions in the probe. probe_sqlite_guard() ( local scenario="$1" log="${bootstrap_fixture_root}/sqlite-guard-${1}.log" rc=0 : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_stop_k3s() { printf 'stop\n' >>"$log"; [[ "$scenario" != stop_failure ]]; } _kseb_start_k3s() { printf 'start\n' >>"$log"; [[ "$scenario" != start_failure ]]; } _kseb_sqlite_copy_and_verify() { printf 'copy\n' >>"$log"; [[ "$scenario" != copy_failure ]]; } _kseb_before_start_dispatch() { if [[ "$scenario" == pre_start_term ]]; then kill -s TERM "$BASHPID"; fi } _kseb_recovery_check() { printf 'recovery\n' >>"$log"; } _kseb_backup_sqlite_guarded /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'stop\ncopy\nstart\nrecovery\nrc=0' "$(probe_sqlite_guard success 2>/dev/null || true)" \ 'SQLite guarded success lifecycle' assert_eq $'stop\ncopy\nstart\nrc=1' "$(probe_sqlite_guard copy_failure 2>/dev/null || true)" \ 'SQLite copy failure restarts service' assert_eq $'stop\nstart\nrc=1' "$(probe_sqlite_guard stop_failure 2>/dev/null || true)" \ 'SQLite stop error still attempts service start' assert_eq $'stop\ncopy\nstart\nrc=75' "$(probe_sqlite_guard start_failure 2>/dev/null || true)" \ 'SQLite start failure is not reported as recovery success' assert_eq $'stop\ncopy\nstart\nrc=143' "$(probe_sqlite_guard pre_start_term 2>/dev/null || true)" \ 'SQLite pre-start TERM dispatches start once and stops flow' probe_quiet_command_boundary() ( local output rc=0 # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" noisy_backup_failure() { printf 'sentinel-recovery-absolute-path\n' printf 'sentinel-server-token-and-encryption-config\n' >&2 return 42 } output="$(_kseb_run_quiet noisy_backup_failure 2>&1)" || rc=$? [[ "$rc" == 42 && -z "$output" ]] ) assert_succeeds probe_quiet_command_boundary probe_relative_manifest_builder() ( local fixture="${bootstrap_fixture_root}/manifest-fixture" output="${bootstrap_fixture_root}/manifest-output" mkdir -p "$fixture/sub" : >"$fixture/alpha"; : >"$fixture/sub/beta" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_manifest_list_files() { printf '%s\0%s\0' "$fixture/alpha" "$fixture/sub/beta"; } _kseb_manifest_hash_file() { case "$1" in */alpha) printf '%064d\n' 1 ;; */beta) printf '%064d\n' 2 ;; *) return 1 ;; esac } _kseb_build_relative_manifest "$fixture" "$output" assert_eq $'0000000000000000000000000000000000000000000000000000000000000001 ./alpha\n0000000000000000000000000000000000000000000000000000000000000002 ./sub/beta' \ "$(<"$output")" 'relative manifest entries' ) assert_succeeds probe_relative_manifest_builder probe_manifest_hash_failure() ( local fixture="${bootstrap_fixture_root}/manifest-failure" output="${bootstrap_fixture_root}/manifest-failure-output" mkdir -p "$fixture" : >"$fixture/alpha" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_manifest_list_files() { printf '%s\0' "$fixture/alpha"; } _kseb_manifest_hash_file() { return 1; } ! _kseb_build_relative_manifest "$fixture" "$output" ) assert_succeeds probe_manifest_hash_failure probe_manifest_list_failure() ( local fixture="${bootstrap_fixture_root}/manifest-list-failure" output="${bootstrap_fixture_root}/manifest-list-failure-output" mkdir -p "$fixture" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_manifest_list_files() { return 1; } if _kseb_build_relative_manifest "$fixture" "$output"; then return 1; fi ) assert_succeeds probe_manifest_list_failure probe_manifest_install_check() ( local scenario="$1" log="${bootstrap_fixture_root}/manifest-check-${1}.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_manifest_lifecycle() { printf 'build\ninstall\ncheck\n' >>"$log" [[ "$scenario" != check_failure ]] } if [[ "$scenario" == check_failure ]]; then ! _kseb_write_verification_manifest /phase else _kseb_write_verification_manifest /phase fi command cat -- "$log" ) assert_eq $'build\ninstall\ncheck' "$(probe_manifest_install_check success 2>/dev/null || true)" \ 'manifest installed then checked' assert_eq $'build\ninstall\ncheck' "$(probe_manifest_install_check check_failure 2>/dev/null || true)" \ 'manifest check failure propagates' probe_backup_root_chain_policy() ( local scenario="$1" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_path_lstat() { case "$scenario/$1" in writable//safe) printf '0:0:777:directory\n' ;; symlink//safe) printf '0:0:777:symbolic link\n' ;; *) printf '0:0:755:directory\n' ;; esac } _kseb_backup_root_chain_safe /safe/recovery ) assert_succeeds probe_backup_root_chain_policy safe assert_fails probe_backup_root_chain_policy writable assert_fails probe_backup_root_chain_policy symlink probe_backup_root_identity_policy() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_backup_root_identity_matches '41:99' '41:99' || return 1 if _kseb_backup_root_identity_matches '41:99' '41:100'; then return 1; fi ) assert_succeeds probe_backup_root_identity_policy probe_backend_command_contract() ( local log="${bootstrap_fixture_root}/backend-command-contract.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_exec() { printf '<%s>\n' "$*" >>"$log"; } _kseb_etcd_snapshot_save pre 20260801T000000Z /phase-pre _kseb_etcd_snapshot_list /phase-pre _kseb_etcd_snapshot_save post 20260801T000000Z /phase-post _kseb_etcd_snapshot_list /phase-post _kseb_archive_host_config /phase-pre _kseb_copy_sqlite_datastore /phase-pre _kseb_compare_sqlite_datastore /phase-pre command cat -- "$log" ) assert_eq $'\n\n\n\n\n\n' \ "$(probe_backend_command_contract 2>/dev/null || true)" 'production backend exact argv contract' probe_production_backend_order() ( local backend="$1" scenario="$2" log="${bootstrap_fixture_root}/production-${1}-${2}.log" rc=0 : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_copy_sqlite_datastore() { printf 'sqlite-copy\n' >>"$log"; } _kseb_compare_sqlite_datastore() { printf 'sqlite-compare\n' >>"$log"; } _kseb_etcd_snapshot_save() { printf 'etcd-save:%s\n' "$1" >>"$log"; } _kseb_etcd_snapshot_list() { printf 'etcd-list\n' >>"$log"; } _kseb_copy_common_recovery() { printf 'common-copy\n' >>"$log"; } _kseb_write_bundle_metadata() { printf 'metadata\n' >>"$log"; } _kseb_write_verification_manifest() { printf 'manifest\n' >>"$log" [[ "$scenario" != manifest_failure ]] } _kseb_recovery_check() { printf 'recovery\n' >>"$log"; } if [[ "$backend" == sqlite ]]; then _kseb_sqlite_copy_and_verify /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? else _kseb_backup_etcd /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? fi printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'sqlite-copy\ncommon-copy\nsqlite-compare\nmetadata\nmanifest\nrc=0' \ "$(probe_production_backend_order sqlite success)" 'production SQLite helper order' assert_eq $'sqlite-copy\ncommon-copy\nsqlite-compare\nmetadata\nmanifest\nrc=1' \ "$(probe_production_backend_order sqlite manifest_failure)" 'SQLite manifest failure propagation' assert_eq $'etcd-save:pre\netcd-list\ncommon-copy\nmetadata\nmanifest\nrecovery\nrc=0' \ "$(probe_production_backend_order etcd success)" 'production embedded-etcd helper order' assert_eq $'etcd-save:pre\netcd-list\ncommon-copy\nmetadata\nmanifest\nrc=1' \ "$(probe_production_backend_order etcd manifest_failure)" 'etcd manifest failure blocks recovery' probe_terminal_guidance_contract() ( local output rc=0 injected # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" output="$(_kseb_terminal_failure reencrypt_stage_start 2>&1)" || rc=$? [[ "$rc" == 1 && "$output" == 'ERROR: current-state=reencrypt_stage_start recovery=bootstrap/manual/k3s-secret-encryption.md' ]] || return 1 injected="$(_kseb_terminal_failure 'unsafe-/sentinel-recovery-path-server-token' 2>&1 || true)" [[ "$injected" == 'ERROR: current-state=unknown recovery=bootstrap/manual/k3s-secret-encryption.md' ]] ) assert_succeeds probe_terminal_guidance_contract assert_public_failure_guidance() { local scenario="$1" expected_state="$2" output output="$(<"${bootstrap_fixture_root}/${scenario}.stderr")" [[ "$output" == *"ERROR: current-state=${expected_state} recovery=bootstrap/manual/k3s-secret-encryption.md"* ]] || fail "$scenario terminal guidance" [[ "$output" != *'sentinel-recovery-absolute-path'* && "$output" != *'server-token'* && "$output" != *'encryption-config'* ]] || fail "$scenario terminal guidance leak" } assert_public_failure_guidance timeout_start reencrypt_stage_start assert_public_failure_guidance mismatch hash_mismatch assert_public_failure_guidance final_validation_failure final_validation_failed assert_public_failure_guidance backup_failure pre_backup_failed assert_fails run_bootstrap_fixture pre_capacity_failure --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order pre_capacity_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\ncapacity:pre' assert_public_failure_guidance pre_capacity_failure pre_backup_failed assert_fails run_bootstrap_fixture post_capacity_failure --execute --backup-root "$bootstrap_backup_root" \ --recovery-policy encrypted-off-host assert_bootstrap_order post_capacity_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\ncapacity:post' assert_public_failure_guidance post_capacity_failure post_backup_failed assert_fails run_bootstrap_fixture service_recovery_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance service_recovery_failure service_recovery_failed assert_fails run_bootstrap_fixture post_backup_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance post_backup_failure post_backup_failed assert_fails run_bootstrap_fixture marker_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance marker_failure marker_install_failed assert_fails run_bootstrap_fixture enable_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance enable_failure partial_enable_failed assert_fails run_bootstrap_fixture dropin_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance dropin_failure partial_dropin_failed assert_fails run_bootstrap_fixture initial_restart_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance initial_restart_failure partial_initial_restart_failed assert_fails run_bootstrap_fixture final_restart_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_public_failure_guidance final_restart_failure partial_final_restart_failed probe_recovery_budget_policy() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_recovery_budget_valid 430 130 30 || return 1 if _kseb_recovery_budget_valid 441 130 30; then return 1; fi ) assert_succeeds probe_recovery_budget_policy probe_workload_ready_policy() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" local empty_status='{"items":[{"status":{"phase":"Running","containerStatuses":[]},"spec":{"containers":[{"name":"app"}],"initContainers":[]}}]}' local ready='{"items":[{"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":true}],"initContainerStatuses":[]},"spec":{"containers":[{"name":"app"}],"initContainers":[]}}]}' if _kseb_workloads_ready_json "$empty_status"; then return 1; fi _kseb_workloads_ready_json "$ready" ) assert_succeeds probe_workload_ready_policy # Regression: after a K3s restart, a transient non-ready Pod must be retried # inside the existing 30-second workload budget instead of failing the backup # on the first observation. Clock, sleep, and API reads are file-backed # because command substitutions execute the read seam in a child shell. probe_workload_wait_transient() ( local state_dir="${bootstrap_fixture_root}/workload-wait-transient" local clock_file="${state_dir}/clock" counter_file="${state_dir}/counter" local log="${state_dir}/log" local nonready ready mkdir -p "$state_dir" printf '0\n' >"$clock_file" printf '0\n' >"$counter_file" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" nonready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":false}],"initContainerStatuses":[]}}]}' ready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":true}],"initContainerStatuses":[]}}]}' _kseb_now_seconds() { command cat -- "$clock_file"; } _kseb_read_workloads_json() { local command_timeout="$1" count printf 'read:%s\n' "$command_timeout" >>"$log" count="$(command cat -- "$counter_file")" count=$((count + 1)) printf '%s\n' "$count" >"$counter_file" if (( count == 1 )); then printf '%s\n' "$nonready"; else printf '%s\n' "$ready"; fi } sleep() { local delay="$1" now printf 'sleep:%s\n' "$delay" >>"$log" now="$(command cat -- "$clock_file")" printf '%s\n' "$((now + delay))" >"$clock_file" } _kseb_wait_for_workloads || return 1 [[ "$(command cat -- "$log")" == $'read:9\nsleep:5\nread:9' ]] ) assert_succeeds probe_workload_wait_transient probe_workload_wait_deadline() ( local state_dir="${bootstrap_fixture_root}/workload-wait-deadline" local clock_file="${state_dir}/clock" log="${state_dir}/log" local nonready mkdir -p "$state_dir" printf '0\n' >"$clock_file" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" nonready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":false}],"initContainerStatuses":[]}}]}' _kseb_now_seconds() { command cat -- "$clock_file"; } _kseb_read_workloads_json() { printf 'read:%s\n' "$1" >>"$log" printf '%s\n' "$nonready" } sleep() { local delay="$1" now printf 'sleep:%s\n' "$delay" >>"$log" now="$(command cat -- "$clock_file")" printf '%s\n' "$((now + delay))" >"$clock_file" } if _kseb_wait_for_workloads; then return 1; fi [[ "$(command cat -- "$log")" == $'read:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:4\nsleep:5' ]] ) assert_succeeds probe_workload_wait_deadline if [[ "${KSEB_WORKLOAD_WAIT_FOCUS:-}" == source ]]; then printf 'K3S WORKLOAD WAIT SOURCE TEST PASS\n' exit 0 fi probe_sqlite_signal_guard() ( local signal="$1" log="${bootstrap_fixture_root}/sqlite-signal-${1}.log" rc=0 : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_stop_k3s() { printf 'stop\n' >>"$log"; } _kseb_start_k3s() { printf 'start\n' >>"$log"; } _kseb_sqlite_copy_and_verify() { printf 'copy\n' >>"$log" kill -s "$signal" "$BASHPID" printf 'continued-after-signal\n' >>"$log" } _kseb_recovery_check() { printf 'recovery\n' >>"$log"; } _kseb_backup_sqlite_guarded /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$? printf 'rc=%s\n' "$rc" >>"$log" command cat -- "$log" ) assert_eq $'stop\ncopy\nstart\nrc=143' "$(probe_sqlite_signal_guard TERM 2>/dev/null || true)" \ 'SQLite TERM guard starts once and stops flow' assert_eq $'stop\ncopy\nstart\nrc=130' "$(probe_sqlite_signal_guard INT 2>/dev/null || true)" \ 'SQLite INT guard starts once and stops flow' # Round 1: a documented root:root 0700 mount cannot be traversed by the # invoking user. Exercise the production decision with only privileged # evidence readers replaced. probe_privileged_root0700_validation() ( # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_privileged_root_kind() { printf 'directory\n'; } _kseb_privileged_realpath() { printf '%s\n' "$1"; } _kseb_privileged_root_metadata() { printf '0:0:700:directory\n'; } _kseb_privileged_backup_root_chain_safe() { return 0; } _kseb_privileged_mount_source() { [[ "$1" == "$KSEB_DATA_DIR" ]] && printf 'data-source\n' || printf 'backup-source\n'; } _kseb_privileged_free_bytes() { printf '10737418240\n'; } _kseb_privileged_root_identity() { printf '41:99\n'; } _kseb_validate_backup_root /secure/recovery [[ "$KSEB_BACKUP_ROOT_IDENTITY" == '41:99' ]] ) assert_succeeds probe_privileged_root0700_validation probe_pinned_root_swap_boundary() ( local scenario="$1" dispatched=false # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY='41:99' _kseb_privileged_backup_root_chain_safe() { return 0; } _kseb_privileged_root_identity() { [[ "$scenario" == swapped ]] && printf '41:100\n' || printf '41:99\n' } _kseb_privileged_relative_components_safe() { return 0; } _kseb_privileged_pinned_dispatch() { dispatched=true; } if [[ "$scenario" == swapped ]]; then if _kseb_pinned_root_operation /secure/recovery bundle/pre verify; then return 1; fi [[ "$dispatched" == false ]] else _kseb_pinned_root_operation /secure/recovery bundle/pre verify [[ "$dispatched" == true ]] fi ) assert_succeeds probe_pinned_root_swap_boundary stable assert_succeeds probe_pinned_root_swap_boundary swapped probe_actual_pinned_root_swap() ( local parent="${bootstrap_fixture_root}/actual-swap" root dispatched=false mkdir -p "$parent/recovery/bundle/pre" "$parent/replacement/bundle/pre" root="$parent/recovery" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" _kseb_privileged_backup_root_chain_safe() { mv -- "$root" "$parent/original" mv -- "$parent/replacement" "$root" } _kseb_privileged_root_identity() { stat --format='%d:%i' -- "$1"; } _kseb_privileged_relative_components_safe() { return 0; } _kseb_privileged_pinned_dispatch() { dispatched=true; } if _kseb_pinned_root_operation "$root" bundle/pre verify; then return 1; fi [[ "$dispatched" == false ]] ) assert_succeeds probe_actual_pinned_root_swap probe_root_only_manifest_entrypoint() ( local log="${bootstrap_fixture_root}/root-only-manifest.log" : >"$log" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" mktemp() { printf 'user-mktemp\n' >>"$log"; return 1; } _kseb_privileged_manifest_lifecycle() { printf 'root-lifecycle\n' >>"$log"; } _kseb_write_verification_manifest /secure/recovery/bundle/pre assert_eq 'root-lifecycle' "$(<"$log")" 'manifest lifecycle remains root-only' ) assert_succeeds probe_root_only_manifest_entrypoint probe_production_mismatch_load_boundary() ( local inventory # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" bash() { return 1; } _kseb_read_status_json() { printf '%s\n' '{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hasherror":"mismatch"}' } inventory="$(_kseb_load_inventory)" || return 1 [[ "$inventory" == hash_mismatch\|unsafe\|ambiguous\|invalid\|ambiguous\|mismatch\|mismatch ]] ) assert_succeeds probe_production_mismatch_load_boundary probe_unexpected_stage_output_safety() ( local output rc=0 output_file="${bootstrap_fixture_root}/unexpected-stage.out" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" _kseb_read_status_json() { printf '%s\n' '{"stage":"unexpected\nSENTINEL-STAGE","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}' } : >"$output_file" _kseb_wait_for_reencrypt 124 >"$output_file" 2>&1 || rc=$? output="$(<"$output_file")" [[ "$rc" != 0 && -z "$output" && "$KSEB_LAST_SAFE_STATE" == reencrypt_stage_unexpected ]] ) assert_succeeds probe_unexpected_stage_output_safety assert_fails run_bootstrap_fixture enabled_finished_drift_provider --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_finished_drift_provider $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory' assert_fails run_bootstrap_fixture enabled_finished_drift_owner --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order enabled_finished_drift_owner $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory' assert_fails run_bootstrap_fixture disabled_backend_drift_external --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order disabled_backend_drift_external $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory' assert_fails run_bootstrap_fixture disabled_backend_drift_ambiguous --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host assert_bootstrap_order disabled_backend_drift_ambiguous $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory' # Round 2: the production privileged dispatcher must dereference both the root # and phase directory FDs. Only privilege elevation is replaced; the fixed # dispatcher and its stat decisions remain production code. probe_production_fd_dereference() ( local root="${bootstrap_fixture_root}/fd-dereference" identity metadata mkdir -p "$root/bundle/pre" chmod 0700 "$root" "$root/bundle" "$root/bundle/pre" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" identity="$(stat --format='%d:%i' -- "$root")" declare -F _kseb_run_privileged_shell >/dev/null || return 1 _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } metadata="$(_kseb_privileged_pinned_dispatch "$root" "$identity" bundle/pre inspect)" [[ "$metadata" == "$identity|$(id -u):$(id -g):700:directory" ]] ) if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == fd ]]; then assert_succeeds probe_production_fd_dereference fi write_round2_fake_k3s() { local path="$1" printf '%s\n' '#!/usr/bin/env bash' \ 'set -Eeuo pipefail' \ 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \ ' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \ ' if [[ -n "${KSEB_SWAP_ROOT:-}" ]]; then mv -- "$KSEB_SWAP_ROOT" "$KSEB_SWAP_ORIGINAL"; mv -- "$KSEB_SWAP_REPLACEMENT" "$KSEB_SWAP_ROOT"; fi' \ ' if [[ -n "${KSEB_SWAP_PHASE:-}" ]]; then mv -- "$KSEB_SWAP_PHASE" "$KSEB_SWAP_PHASE_ORIGINAL"; mv -- "$KSEB_SWAP_PHASE_REPLACEMENT" "$KSEB_SWAP_PHASE"; fi' \ ' : >"${destination:?}/fixture-snapshot"; exit 0' \ 'fi' \ 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then printf "%s\\n" "{\"items\":[]}"; exit 0; fi' \ 'exit 1' >"$path" chmod 0700 "$path" } # This fixture renames and replaces the public root after the production # transaction has opened it. Snapshot, common copy, manifest, sizing and the # post marker must all stay on the original FD-derived tree. probe_complete_transaction_actual_swap() ( local parent="${bootstrap_fixture_root}/round2-swap" root original replacement local data config fake_k3s marker evidence output expected_metadata parent="$(mktemp -d "${bootstrap_fixture_root}/round2-swap.XXXXXX")" root="$parent/recovery"; original="$parent/original"; replacement="$parent/replacement" data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s" marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env" mkdir -p "$root" "$replacement" "$data/server/cred" "$config" chmod 0700 "$root" "$replacement" printf 'token\n' >"$data/server/token" printf '{}\n' >"$data/server/cred/encryption-config.json" printf 'config\n' >"$config/config.yaml" : >"$evidence" write_round2_fake_k3s "$fake_k3s" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" declare -F _kseb_pinned_phase_transaction >/dev/null || return 1 _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } _kseb_transaction_data_dir() { printf '%s\n' "$data"; } _kseb_transaction_config_dir() { printf '%s\n' "$config"; } _kseb_transaction_systemd_paths() { :; } _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } _kseb_transaction_post_marker() { printf '%s\n' "$marker"; } _kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; } export KSEB_SWAP_ROOT="$root" KSEB_SWAP_ORIGINAL="$original" KSEB_SWAP_REPLACEMENT="$replacement" output="$(_kseb_pinned_phase_transaction "$root" bundle/post embedded-etcd \ 01234567-89ab-4cde-8fab-0123456789ab post 20260801T000000Z 7 v1.36.2+k3s1)" expected_metadata=$'schema=platform-k3s-bundle-v1\nbundle_id=01234567-89ab-4cde-8fab-0123456789ab\nphase=post\nk3s_version=v1.36.2+k3s1\ndatastore=embedded-etcd\ncreated_at_utc=2026-08-01T00:00:00Z\nsecret_count=7' [[ "$output" =~ ^[0-9]+$ && -f "$original/bundle/post/fixture-snapshot" && -f "$original/bundle/post/server-token" && -f "$original/bundle/post/verification.manifest" && -f "$marker" && "$(<"$marker")" == "$expected_metadata" && "$(<"$original/bundle/post/bundle.env")" == "$expected_metadata" && ! -e "$root/bundle" ]] ) if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == transaction ]]; then assert_succeeds probe_complete_transaction_actual_swap fi # The manifest lifecycle must open its phase directory relative to the pinned # root FD and validate the dereferenced directory target before hashing. probe_manifest_from_pinned_handle() ( local parent root data config fake_k3s phase original_phase replacement_phase output local rc=0 parent="$(mktemp -d "${bootstrap_fixture_root}/round2-manifest.XXXXXX")" root="$parent/recovery"; data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s" phase="$root/bundle/pre"; original_phase="$root/bundle/original-pre" replacement_phase="$parent/replacement-pre" mkdir -p "$root" "$data/server/cred" "$config" "$replacement_phase" chmod 0700 "$root" printf 'token\n' >"$data/server/token" printf '{}\n' >"$data/server/cred/encryption-config.json" printf 'config\n' >"$config/config.yaml" write_round2_fake_k3s "$fake_k3s" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" declare -F _kseb_pinned_phase_transaction >/dev/null || return 1 _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } _kseb_transaction_data_dir() { printf '%s\n' "$data"; } _kseb_transaction_config_dir() { printf '%s\n' "$config"; } _kseb_transaction_systemd_paths() { :; } _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } export KSEB_SWAP_ROOT='' KSEB_SWAP_PHASE="$phase" KSEB_SWAP_PHASE_ORIGINAL="$original_phase" export KSEB_SWAP_PHASE_REPLACEMENT="$replacement_phase" output="$(_kseb_pinned_phase_transaction "$root" bundle/pre embedded-etcd bundle pre 20260801T000000Z 7 v1.36.2+k3s1 2>&1)" || rc=$? [[ "$rc" != 0 && "$output" != *"$parent"* && -d "$original_phase" && -z "$(find "$original_phase" -mindepth 1 -print -quit)" && -d "$phase" && ! -e "$phase/verification.manifest" ]] ) if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == manifest ]]; then assert_succeeds probe_manifest_from_pinned_handle fi write_round2_fake_systemctl() { local path="$1" printf '%s\n' '#!/usr/bin/env bash' \ 'set -Eeuo pipefail' \ 'printf "%s\\n" "$1" >>"${KSEB_SYSTEMCTL_LOG:?}"' \ 'if [[ "$1" == start && "${KSEB_SIGNAL_DURING_START:-0}" == 1 ]]; then kill -s TERM "$PPID"; fi' \ 'exit 0' >"$path" chmod 0700 "$path" } write_workload_wait_fake_k3s() { local path="$1" printf '%s\n' '#!/usr/bin/env bash' \ 'set -Eeuo pipefail' \ 'printf "%s|%s %s %s\n" "${KSEB_WORKLOAD_SCENARIO:?}" "${1:-}" "${2:-}" "${3:-}" >>"${KSEB_K3S_CALL_LOG:?}"' \ 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \ ' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \ ' : >"${destination:?}/fixture-snapshot"' \ ' exit 0' \ 'fi' \ 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then' \ ' if [[ "${KSEB_WORKLOAD_SCENARIO:?}" == api_failure ]]; then exit 9; fi' \ ' exit 0' \ 'fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then' \ ' count=$(<"${KSEB_WORKLOAD_COUNTER:?}")' \ ' count=$((count + 1))' \ ' printf "%s\n" "$count" >"$KSEB_WORKLOAD_COUNTER"' \ ' printf "read\n" >>"${KSEB_WORKLOAD_LOG:?}"' \ ' if [[ "$KSEB_WORKLOAD_SCENARIO" == transient && "$count" == 1 ]]; then' \ ' ready=false' \ ' elif [[ "$KSEB_WORKLOAD_SCENARIO" == persistent ]]; then' \ ' ready=false' \ ' else' \ ' ready=true' \ ' fi' \ ' printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"fixture\",\"namespace\":\"default\"},\"spec\":{\"containers\":[{\"name\":\"app\"}],\"initContainers\":[]},\"status\":{\"phase\":\"Running\",\"containerStatuses\":[{\"name\":\"app\",\"ready\":${ready}}],\"initContainerStatuses\":[]}}]}"' \ ' exit 0' \ 'fi' \ 'exit 1' >"$path" chmod 0700 "$path" } probe_pinned_workload_recovery() ( local scenario="$1" parent root data config fake_k3s fake_systemctl local systemctl_log workload_log workload_counter k3s_call_log marker evidence stderr_log output='' rc=0 parent="$(mktemp -d "${bootstrap_fixture_root}/pinned-workload-${scenario}.XXXXXX")" root="$parent/recovery"; data="$parent/data"; config="$parent/config" fake_k3s="$parent/k3s"; fake_systemctl="$parent/systemctl" systemctl_log="$parent/systemctl.log"; workload_log="$parent/workload.log" k3s_call_log="$parent/k3s-call.log" workload_counter="$parent/workload.counter" marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env" stderr_log="$parent/transaction.stderr" mkdir -p "$root" "$data/server/db" "$config" chmod 0700 "$root" printf 'state\n' >"$data/server/db/state.db" printf 'token\n' >"$data/server/token" printf 'config\n' >"$config/config.yaml" : >"$systemctl_log"; : >"$workload_log"; : >"$k3s_call_log"; printf '0\n' >"$workload_counter" write_workload_wait_fake_k3s "$fake_k3s" write_round2_fake_systemctl "$fake_systemctl" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } _kseb_transaction_data_dir() { printf '%s\n' "$data"; } _kseb_transaction_config_dir() { printf '%s\n' "$config"; } _kseb_transaction_systemd_paths() { :; } _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } _kseb_transaction_systemctl_binary() { printf '%s\n' "$fake_systemctl"; } _kseb_transaction_post_marker() { printf '%s\n' "$marker"; } _kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; } export KSEB_SYSTEMCTL_LOG="$systemctl_log" KSEB_SIGNAL_DURING_START=0 export KSEB_WORKLOAD_SCENARIO="$scenario" KSEB_WORKLOAD_COUNTER="$workload_counter" export KSEB_WORKLOAD_LOG="$workload_log" KSEB_K3S_CALL_LOG="$k3s_call_log" output="$(_kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" \ bundle/pre transaction "$data" "$config" "$fake_k3s" "$fake_systemctl" \ "$marker" "$evidence" '' sqlite 01234567-89ab-4cde-8fab-0123456789ab \ pre 20260801T000000Z 7 v1.36.2+k3s1 pre 2>"$stderr_log")" || rc=$? case "$scenario" in transient) if [[ "$rc" != 0 || ! "$output" =~ ^[0-9]+$ || "$(command cat -- "$systemctl_log")" != $'stop\nstart' || "$(command cat -- "$workload_log")" != $'read\nread' || ! -f "$root/bundle/pre/datastore/state.db" || ! -f "$root/bundle/pre/verification.manifest" ]]; then printf 'PINNED_TRANSIENT_DIAG rc=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s manifest=%s stderr_lines=%s\n' \ "$rc" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \ "$(wc -l <"$workload_log" | tr -d '[:space:]')" \ "$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \ "$([[ -f "$root/bundle/pre/verification.manifest" ]] && printf yes || printf no)" \ "$(wc -l <"$stderr_log" | tr -d '[:space:]')" >&2 command cat -- "$k3s_call_log" >&2 return 1 fi ;; persistent) if [[ "$rc" != 75 || -n "$output" || "$(command cat -- "$systemctl_log")" != $'stop\nstart' || "$(wc -l <"$workload_log" | tr -d '[:space:]')" -lt 2 || ! -d "$root/bundle/pre" || -n "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]; then printf 'PINNED_PERSISTENT_DIAG rc=%s output_bytes=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s phase_exists=%s phase_entries=%s\n' \ "$rc" "${#output}" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \ "$(wc -l <"$workload_log" | tr -d '[:space:]')" \ "$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \ "$([[ -d "$root/bundle/pre" ]] && printf yes || printf no)" \ "$(find "$root/bundle/pre" -mindepth 1 -printf . 2>/dev/null | wc -c | tr -d '[:space:]')" >&2 command cat -- "$k3s_call_log" >&2 return 1 fi ;; api_failure) if [[ "$rc" != 75 || -n "$output" || "$(command cat -- "$systemctl_log")" != $'stop\nstart' || -s "$workload_log" || ! -d "$root/bundle/pre" || -n "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]; then printf 'PINNED_API_FAILURE_DIAG rc=%s output_bytes=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s phase_exists=%s phase_entries=%s\n' \ "$rc" "${#output}" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \ "$(wc -l <"$workload_log" | tr -d '[:space:]')" \ "$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \ "$([[ -d "$root/bundle/pre" ]] && printf yes || printf no)" \ "$(find "$root/bundle/pre" -mindepth 1 -printf . 2>/dev/null | wc -c | tr -d '[:space:]')" >&2 command cat -- "$k3s_call_log" >&2 return 1 fi ;; *) return 1 ;; esac ) assert_succeeds probe_pinned_workload_recovery transient assert_succeeds probe_pinned_workload_recovery persistent assert_succeeds probe_pinned_workload_recovery api_failure if [[ "${KSEB_WORKLOAD_WAIT_FOCUS:-}" == pinned ]]; then printf 'K3S WORKLOAD WAIT PINNED TEST PASS\n' exit 0 fi # Trigger failure after stop so EXIT cleanup performs the real production start # dispatch. TERM arrives from inside that command; cleanup must not be cut off # and start must be attempted exactly once. probe_cleanup_window_signal() ( local parent data root systemctl_log fake_systemctl rc=0 parent="$(mktemp -d "${bootstrap_fixture_root}/round2-cleanup.XXXXXX")" data="$parent/data"; root="$parent/recovery"; systemctl_log="$parent/systemctl.log" fake_systemctl="$parent/systemctl" mkdir -p "$data/server/db" "$root" chmod 0700 "$root" : >"$systemctl_log" write_round2_fake_systemctl "$fake_systemctl" # Deliberately omit server/token so the pinned transaction enters cleanup. # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } _kseb_transaction_data_dir() { printf '%s\n' "$data"; } _kseb_transaction_config_dir() { printf '%s\n' "$parent/no-config"; } _kseb_transaction_systemd_paths() { :; } _kseb_transaction_systemctl_binary() { printf '%s\n' "$fake_systemctl"; } export KSEB_SYSTEMCTL_LOG="$systemctl_log" KSEB_SIGNAL_DURING_START=1 _kseb_pinned_phase_transaction "$root" bundle/pre sqlite bundle pre 20260801T000000Z 7 v1.36.2+k3s1 >/dev/null 2>&1 || rc=$? [[ "$rc" != 0 && "$(<"$systemctl_log")" == $'stop\nstart' && -d "$root/bundle/pre" && -z "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]] ) if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == cleanup ]]; then assert_succeeds probe_cleanup_window_signal fi write_round3_fake_k3s() { local path="$1" printf '%s\n' '#!/usr/bin/env bash' \ 'set -Eeuo pipefail' \ 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \ ' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \ ' : >"${destination:?}/partial-before-swap"' \ ' mv -- "${KSEB_R3_PHASE:?}" "${KSEB_R3_ORIGINAL:?}"' \ ' mv -- "${KSEB_R3_REPLACEMENT:?}" "${KSEB_R3_PHASE:?}"' \ ' case "${KSEB_R3_SCENARIO:?}" in' \ ' snapshot_failure) exit 9 ;;' \ ' manifest_failure) : >"${destination}/invalid manifest name" ;;' \ ' commit_swap) : >"${destination}/valid-after-swap" ;;' \ ' *) exit 91 ;;' \ ' esac' \ ' exit 0' \ 'fi' \ 'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \ 'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then printf "%s\\n" "{\"items\":[]}"; exit 0; fi' \ 'exit 1' >"$path" chmod 0700 "$path" } # A production transaction must never clean or commit through the old phase # pathname after that pathname has been replaced. Only privilege elevation # and the k3s external effect are replaced; FD creation, manifest, cleanup, # recovery, marker and commit decisions are the real fixed dispatcher. probe_round3_phase_swap_identity() ( local scenario="$1" parent root data config fake_k3s phase original_phase replacement_phase local marker evidence output rc=0 parent="$(mktemp -d "${bootstrap_fixture_root}/round3-${scenario}.XXXXXX")" root="$parent/recovery"; data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s" phase="$root/bundle/post"; original_phase="$root/bundle/original-post" replacement_phase="$parent/replacement-post" marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env" mkdir -p "$root" "$data/server/cred" "$config" "$replacement_phase" chmod 0700 "$root" "$replacement_phase" printf 'token\n' >"$data/server/token" printf '{}\n' >"$data/server/cred/encryption-config.json" printf 'config\n' >"$config/config.yaml" printf 'replacement-untouched\n' >"$replacement_phase/replacement-sentinel" printf 'restore-untouched\n' >"$evidence" write_round3_fake_k3s "$fake_k3s" # shellcheck source=/dev/null source "$BOOTSTRAP_PATH" KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")" _kseb_run_privileged_shell() { /usr/bin/bash "$@"; } _kseb_transaction_data_dir() { printf '%s\n' "$data"; } _kseb_transaction_config_dir() { printf '%s\n' "$config"; } _kseb_transaction_systemd_paths() { :; } _kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; } _kseb_transaction_post_marker() { printf '%s\n' "$marker"; } _kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; } export KSEB_R3_SCENARIO="$scenario" KSEB_R3_PHASE="$phase" KSEB_R3_ORIGINAL="$original_phase" export KSEB_R3_REPLACEMENT="$replacement_phase" output="$(_kseb_pinned_phase_transaction "$root" bundle/post embedded-etcd bundle post 20260801T000000Z 7 v1.36.2+k3s1 2>&1)" || rc=$? [[ "$rc" != 0 && "$output" != *"$parent"* && -d "$phase" && "$(<"$phase/replacement-sentinel")" == replacement-untouched && -d "$original_phase" && -z "$(find "$original_phase" -mindepth 1 -print -quit)" && ! -e "$marker" && "$(<"$evidence")" == restore-untouched ]] ) if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == failure ]]; then assert_succeeds probe_round3_phase_swap_identity snapshot_failure fi if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == manifest ]]; then assert_succeeds probe_round3_phase_swap_identity manifest_failure fi if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == commit ]]; then assert_succeeds probe_round3_phase_swap_identity commit_swap fi bash "${REPOSITORY_ROOT}/scripts/validate/test-k3s-secret-encryption-restore-evidence.sh" printf 'K3S SECRET ENCRYPTION STATUS TEST PASS\n'