2361 lines
131 KiB
Bash
2361 lines
131 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Regression coverage for parser branches that must fail closed. No command in
|
|
# this file contacts a k3s service, systemd, or the Kubernetes API.
|
|
set -Eeuo pipefail
|
|
|
|
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly LIBRARY_PATH="${REPOSITORY_ROOT}/scripts/lib/k3s-secret-encryption.sh"
|
|
readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-secret-encryption.sh"
|
|
readonly ORIGINAL_PATH="$PATH"
|
|
|
|
fail() {
|
|
printf 'TEST FAILURE: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
assert_eq() {
|
|
local expected="$1"
|
|
local actual="$2"
|
|
local description="$3"
|
|
[[ "$actual" == "$expected" ]] || fail "$description"
|
|
}
|
|
|
|
assert_succeeds() {
|
|
"$@" || fail "expected success: $*"
|
|
}
|
|
|
|
assert_fails() {
|
|
if "$@"; then
|
|
fail "expected failure: $*"
|
|
fi
|
|
}
|
|
|
|
assert_exit() {
|
|
local expected="$1"
|
|
shift
|
|
local actual=0
|
|
"$@" || actual=$?
|
|
[[ "$actual" == "$expected" ]] || fail "expected exit ${expected}, got ${actual}: $*"
|
|
}
|
|
|
|
fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-secret-encryption-test.XXXXXX")"
|
|
verified_output_dir=''
|
|
rejected_output_dir=''
|
|
race_output_dir=''
|
|
finished_output_dir=''
|
|
transition_output_dir=''
|
|
enabled_output_dir=''
|
|
symlink_output_parent=''
|
|
cleanup() {
|
|
if [[ -n "$verified_output_dir" ]]; then
|
|
rm -rf -- "$verified_output_dir"
|
|
fi
|
|
if [[ -n "$rejected_output_dir" ]]; then
|
|
rm -rf -- "$rejected_output_dir"
|
|
fi
|
|
if [[ -n "$race_output_dir" ]]; then
|
|
rm -rf -- "$race_output_dir"
|
|
fi
|
|
if [[ -n "$finished_output_dir" ]]; then
|
|
rm -rf -- "$finished_output_dir"
|
|
fi
|
|
if [[ -n "$transition_output_dir" ]]; then
|
|
rm -rf -- "$transition_output_dir"
|
|
fi
|
|
if [[ -n "$enabled_output_dir" ]]; then
|
|
rm -rf -- "$enabled_output_dir"
|
|
fi
|
|
if [[ -n "$symlink_output_parent" && -L "$symlink_output_parent" ]]; then
|
|
unlink -- "$symlink_output_parent"
|
|
fi
|
|
case "$fixture_root" in
|
|
/tmp/k3s-secret-encryption-test.*|"${TMPDIR:-/tmp}"/k3s-secret-encryption-test.*)
|
|
rm -rf -- "$fixture_root"
|
|
;;
|
|
*)
|
|
fail 'refusing to remove an unexpected fixture directory'
|
|
;;
|
|
esac
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# The production change that this test catches is a missing status parser.
|
|
# This source must fail RED until the source-only library is created.
|
|
# shellcheck source=/dev/null
|
|
source "$LIBRARY_PATH"
|
|
|
|
# Regression: GNU stat describes a zero-byte regular file as "regular empty
|
|
# file". The root text reader must accept the file by inode kind, ownership,
|
|
# and mode instead of depending on that presentation string.
|
|
assert_succeeds /usr/bin/bash -c '
|
|
set -Eeuo pipefail
|
|
source "$1"
|
|
_kse_privileged_capture() {
|
|
local binary="$1"
|
|
shift
|
|
if [[ "$binary" == "$KSE_STAT" ]]; then
|
|
case "${1:-}" in
|
|
--format=%u:%a:%F) printf "0:600:regular empty file\n" ;;
|
|
--format=%u:%a) printf "0:600\n" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
return 0
|
|
fi
|
|
if [[ "$binary" == "$KSE_CAT" && "${*: -1}" == /etc/systemd/system/k3s.service.env ]]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
_kse_privileged_quiet() {
|
|
[[ "$1" == "$KSE_TEST" ]] || return 1
|
|
shift
|
|
case "$*" in
|
|
"-f /etc/systemd/system/k3s.service.env"|"! -L /etc/systemd/system/k3s.service.env") return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
value="$(_kse_read_root_text /etc/systemd/system/k3s.service.env)"
|
|
[[ -z "$value" ]]
|
|
' bash "$VALIDATOR_PATH"
|
|
|
|
disabled_json='{"stage":"","activekey":""}'
|
|
start_json='{"stage":"start","activekey":"","enable":false,"hashmatch":true}'
|
|
enabled_start_json='{"stage":"start","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}'
|
|
enabled_finished_json='{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}'
|
|
active_json='{"stage":"reencrypt_active","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}'
|
|
mismatch_json='{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hasherror":"hash mismatch"}'
|
|
secretbox_json='{"stage":"start","activekey":"XSalsa20-POLY1305 fixture-key","enable":true,"hashmatch":true}'
|
|
|
|
# Wrong branches here would weaken the state-machine fail-stop gate.
|
|
assert_eq disabled_no_config "$(classify_encryption_status "$disabled_json")" 'disabled fixture classification'
|
|
assert_eq transition_start "$(classify_encryption_status "$start_json")" 'late-enable transition classification'
|
|
assert_eq enabled_stable "$(classify_encryption_status "$enabled_start_json")" 'enabled start classification'
|
|
assert_eq enabled_stable "$(classify_encryption_status "$enabled_finished_json")" 'enabled finished classification'
|
|
assert_eq unsafe_transition "$(classify_encryption_status "$active_json")" 'active re-encryption classification'
|
|
assert_eq hash_mismatch "$(classify_encryption_status "$mismatch_json")" 'hash mismatch classification'
|
|
assert_eq invalid "$(classify_encryption_status '{"stage":"start","activekey":"","enable":false,"hashmatch":true,"inactivekeys":["unexpected"]}')" 'non-empty transition inactive keys rejection'
|
|
assert_eq invalid "$(classify_encryption_status '{"stage":"","activekey":"","hasherror":"unexpected"}')" 'disabled status hash error rejection'
|
|
assert_eq invalid "$(classify_encryption_status "${disabled_json}"$'\n'"${disabled_json}")" 'concatenated JSON documents rejection'
|
|
assert_eq invalid "$(classify_encryption_status "${disabled_json} trailing")" 'JSON trailing garbage rejection'
|
|
assert_eq aescbc "$(classify_encryption_provider "$enabled_start_json")" 'AES-CBC provider classification'
|
|
assert_eq secretbox "$(classify_encryption_provider "$secretbox_json")" 'secretbox provider classification'
|
|
assert_eq invalid "$(classify_encryption_provider '{"stage":"start","activekey":"AES-CBC","enable":true,"hashmatch":true}')" 'provider exact-prefix rejection'
|
|
|
|
assert_succeeds version_supports_late_enable v1.33.10+k3s1
|
|
assert_succeeds version_supports_late_enable v1.34.6+k3s1
|
|
assert_succeeds version_supports_late_enable v1.35.3+k3s1
|
|
assert_succeeds version_supports_late_enable v1.36.2+k3s1
|
|
assert_fails version_supports_late_enable v1.33.9+k3s1
|
|
assert_fails version_supports_late_enable v1.34.5+k3s1
|
|
assert_fails version_supports_late_enable v1.35.2+k3s1
|
|
|
|
assert_succeeds require_exact_encryption_state enabled_stable "$enabled_finished_json"
|
|
assert_fails require_exact_encryption_state enabled_stable "$active_json"
|
|
|
|
fixture_config="${fixture_root}/encryption-config.json"
|
|
fixture_state="${fixture_root}/encryption-state.json"
|
|
fixture_stale_state="${fixture_root}/encryption-state-stale.json"
|
|
printf '{"fixture":"config"}\n' >"$fixture_config"
|
|
fixture_hash="$(sha256sum -- "$fixture_config" | awk '{print $1}')"
|
|
fixture_annotation="reencrypt_finished-${fixture_hash}"
|
|
printf '%s' "$fixture_annotation" >"$fixture_state"
|
|
printf '%s' "start-${fixture_hash}" >"$fixture_stale_state"
|
|
chmod 0600 "$fixture_config" "$fixture_state" "$fixture_stale_state"
|
|
|
|
assert_eq match "$(verify_local_encryption_config_integrity_evidence \
|
|
'0:600:regular file' '0:600:regular file' "$fixture_hash" \
|
|
"$fixture_annotation" "$fixture_annotation" reencrypt_finished)" \
|
|
'privileged evidence integrity match'
|
|
|
|
# The shim changes only the metadata an unprivileged fixture cannot create;
|
|
# content, hashes, symlink checks, and all parser behavior stay real.
|
|
mkdir -p "${fixture_root}/stat-shim"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'if [[ "$1" == "--format=%u:%a:%F" ]]; then printf "0:600:regular file\\n"; exit 0; fi' \
|
|
'exec /usr/bin/stat "$@"' >"${fixture_root}/stat-shim/stat"
|
|
chmod 0700 "${fixture_root}/stat-shim/stat"
|
|
PATH="${fixture_root}/stat-shim:${ORIGINAL_PATH}"
|
|
assert_eq match "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_state" "$fixture_annotation" reencrypt_finished)" 'matching root metadata fixture'
|
|
assert_eq mismatch "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_stale_state" "$fixture_annotation" reencrypt_finished || true)" 'stale state rejection'
|
|
PATH="$ORIGINAL_PATH"
|
|
assert_eq mismatch "$(verify_local_encryption_config_integrity "$fixture_config" "$fixture_state" "$fixture_annotation" reencrypt_finished || true)" 'user-owned config rejection'
|
|
ln -s -- "$fixture_config" "${fixture_root}/config-link"
|
|
assert_eq mismatch "$(verify_local_encryption_config_integrity "${fixture_root}/config-link" "$fixture_state" "$fixture_annotation" reencrypt_finished || true)" 'symlink config rejection'
|
|
|
|
fake_bin="${fixture_root}/bin"
|
|
mkdir -p "$fake_bin"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'printf "%s\n" "${SYSTEMCTL_SHOW_FIXTURE:-}"; [[ "${SYSTEMCTL_SHOW_FIXTURE:-}" == *$'"'"'\nEnvironment='"'"'* ]] || printf "Environment=\n"; [[ "${SYSTEMCTL_SHOW_FIXTURE:-}" == *$'"'"'\nEnvironmentFiles='"'"'* ]] || printf "EnvironmentFiles=\n"' >"${fake_bin}/systemctl"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'if [[ "${K3S_HANG:-0}" == 1 ]]; then while :; do /usr/bin/sleep 1; done; fi; if [[ "${1:-}" == --version ]]; then printf "k3s version v1.36.2+k3s1 (fixture)\n"; exit 0; fi; if [[ "${1:-}" == kubectl ]]; then case "${2:-}" in get) if [[ "${3:-}" == nodes ]]; then printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"fixture-server\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{\"k3s.io/encryption-config-hash\":\"token-password-encryption-config\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}"; exit 0; fi; if [[ "${3:-}" == --raw=/readyz ]]; then printf "token-password-encryption-config\n"; [[ "${K3S_API_RC:-0}" == 0 ]] && exit 0 || exit 1; fi ;; esac; exit 1; fi; case "${K3S_STATUS_FIXTURE:-}" in disabled) printf "%s\n" "{\"stage\":\"\",\"activekey\":\"\"}"; exit 0 ;; finished) stage=reencrypt_finished ;; start) stage=start ;; unsupported) stage=unexpected_stage ;; mismatch) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC fixture-key\",\"enable\":true,\"hasherror\":\"mismatch\"}"; exit 0 ;; malformed) printf "%s\n" "{"; exit 0 ;; active_then_finished) if [[ -e "${K3S_STATUS_COUNTER:?}" ]]; then stage=reencrypt_finished; else : >"$K3S_STATUS_COUNTER"; stage=reencrypt_active; fi ;; *) exit 1 ;; esac; printf "%s\n" "{\"stage\":\"${stage}\",\"activekey\":\"AES-CBC fixture-key\",\"enable\":true,\"hashmatch\":true}"' >"${fake_bin}/k3s"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'{ printf "sudo"; printf " <%s>" "$@"; printf "\n"; } >>"${FAKE_ARG_LOG:?}"' \
|
|
'if [[ "${SUDO_HANG:-0}" == 1 ]]; then while :; do /usr/bin/sleep 1; done; fi' \
|
|
'[[ "${1:-}" == --non-interactive ]] && shift' \
|
|
'if [[ "${1:-}" == /usr/bin/env && " $* " == *" --privileged-probe "* && -n "${K3S_VALIDATOR_FAKE_PATH:-}" ]]; then shift; [[ "${1:-}" == -i ]] && shift; [[ "${1:-}" == PATH=* ]] && shift; [[ "${1:-}" == LC_ALL=* ]] && shift; exec /usr/bin/env PATH="$K3S_VALIDATOR_FAKE_PATH" "$@"; fi' \
|
|
'exec "$@"' >"${fake_bin}/sudo"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'{ printf "timeout"; printf " <%s>" "$@"; printf "\n"; } >>"${FAKE_ARG_LOG:?}"' \
|
|
'shift 3' \
|
|
'if [[ "${SUDO_HANG:-0}" == 1 ]]; then exec /usr/bin/timeout --signal=TERM --kill-after=0.1s 0.05s "$@"; fi' \
|
|
'if [[ "${K3S_HANG:-0}" == 1 ]]; then if [[ -e "${K3S_TIMEOUT_COUNTER:?}" ]]; then exit 124; fi; : >"$K3S_TIMEOUT_COUNTER"; exec /usr/bin/timeout --signal=TERM --kill-after=0.1s 0.05s "$@"; fi' \
|
|
'exec "$@"' >"${fake_bin}/timeout"
|
|
printf '%s\n' '#!/usr/bin/env bash' 'exit 0' >"${fake_bin}/sleep"
|
|
printf '%s\n' '#!/usr/bin/env bash' 'exit 1' >"${fake_bin}/kubectl"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'if [[ "${HANDOFF_RACE:-0}" == 1 && " $* " == *" inventory.env status.sha256 "* ]]; then : >unexpected-race; fi' \
|
|
'exec /usr/bin/stat "$@"' >"${fake_bin}/stat"
|
|
chmod 0700 "${fake_bin}/systemctl" "${fake_bin}/k3s" "${fake_bin}/sudo" "${fake_bin}/timeout" "${fake_bin}/sleep" "${fake_bin}/kubectl" "${fake_bin}/stat"
|
|
PATH="${fake_bin}:${ORIGINAL_PATH}"
|
|
FAKE_ARG_LOG="${fixture_root}/fake-argv.log"
|
|
export SYSTEMCTL_SHOW_FIXTURE K3S_STATUS_FIXTURE FAKE_ARG_LOG K3S_API_RC
|
|
|
|
config_root="${fixture_root}/config-root"
|
|
mkdir -p "${config_root}/config.yaml.d"
|
|
printf 'secrets-encryption: false\n' >"${config_root}/config.yaml"
|
|
printf 'secrets-encryption: true\n' >"${config_root}/config.yaml.d/40-encryption.yaml"
|
|
export K3S_CONFIG_DIR="$config_root"
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server'
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'default YAML drop-in merge and implicit provider'
|
|
|
|
cli_config="${fixture_root}/cli.yaml"
|
|
env_config="${fixture_root}/env.yaml"
|
|
printf 'secrets-encryption: true\nsecrets-encryption-provider: aescbc\n' >"$cli_config"
|
|
printf 'secrets-encryption: false\n' >"$env_config"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${cli_config}
|
|
Environment=K3S_CONFIG_FILE=${env_config}"
|
|
assert_eq "aescbc/${cli_config}" "$(detect_effective_encryption_config_owner)" 'CLI config precedence'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
Environment=K3S_CONFIG_FILE=${cli_config}"
|
|
assert_eq "aescbc/${cli_config}" "$(detect_effective_encryption_config_owner)" 'K3S_CONFIG_FILE config selection'
|
|
printf 'secrets-encryption: true\n' >"$env_config"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config}
|
|
Environment=K3S_SECRETS_ENCRYPTION_PROVIDER=secretbox"
|
|
assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'environment provider overrides YAML provider absence'
|
|
printf 'secrets-encryption-provider: secretbox\n' >"$env_config"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config}
|
|
Environment=K3S_SECRETS_ENCRYPTION=true"
|
|
assert_eq "secretbox/${env_config}" "$(detect_effective_encryption_config_owner)" 'YAML provider remains effective when environment only enables'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --secrets-encryption=true --secrets-encryption-provider=aescbc'
|
|
assert_eq aescbc/command-line "$(detect_effective_encryption_config_owner)" 'ExecStart encryption key precedence'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --secrets-encryption=true ; ignore_errors=no ; }'
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'systemctl show ExecStart serialization'
|
|
|
|
# These fixtures catch accepting anything except one systemd ExecStart record
|
|
# whose known metadata appears once, in the production order, with valid types.
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'production systemctl show ExecStart serialization'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } { path=/usr/bin/false ; argv[]=/usr/bin/false ; ignore_errors=no ; }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart extra record rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; argv[]=/usr/local/bin/k3s server ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart duplicate argv rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; unknown=0 ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart unknown metadata rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; pid=0 ; code=(null) ; status=0/0 }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart duplicate metadata rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; stop_time=[n/a] ; start_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart metadata order rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=not-a-number ; code=(null) ; status=0/0 }
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart malformed metadata rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --disable=servicelb ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'production ExecStart malformed brace rejection'
|
|
|
|
env_file="${fixture_root}/k3s.env"
|
|
printf 'K3S_SECRETS_ENCRYPTION=true\n' >"$env_file"
|
|
K3S_CONFIG_DIR="${fixture_root}/empty-config-root"
|
|
mkdir -p "$K3S_CONFIG_DIR"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${env_file}"
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile enable source'
|
|
printf '# comment\nK3S_SECRETS_ENCRYPTION=true\n' >"${fixture_root}/comment.env"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/comment.env"
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile comments ignored'
|
|
printf 'K3S_SECRETS_ENCRYPTION="true"\n' >"${fixture_root}/quoted.env"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/quoted.env"
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'quoted EnvironmentFile assignment rejection'
|
|
printf 'K3S_SECRETS_ENCRYPTION\n' >"${fixture_root}/reset.env"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/reset.env"
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'EnvironmentFile reset rejection'
|
|
printf "OTHER='multiline\nK3S_SECRETS_ENCRYPTION=true\nclosed'\n" >"${fixture_root}/multiline-other.env"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/multiline-other.env (ignore_errors=no)"
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'non-target multiline record does not create assignment'
|
|
printf '%s\n' 'OTHER="first \"' 'K3S_SECRETS_ENCRYPTION=true' 'continued"' >"${fixture_root}/escaped-quote-other.env"
|
|
assert_eq '' "$(_k3s_envfile_values "${fixture_root}/escaped-quote-other.env" K3S_SECRETS_ENCRYPTION || true)" 'escaped quote keeps target-looking line inside non-target logical record'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/escaped-quote-other.env (ignore_errors=no)"
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'escaped quoted EnvironmentFile record fails closed without target assignment'
|
|
printf '%s\n' 'OTHER=prefix\' 'K3S_SECRETS_ENCRYPTION=true' >"${fixture_root}/continued-other.env"
|
|
assert_eq '' "$(_k3s_envfile_values "${fixture_root}/continued-other.env" K3S_SECRETS_ENCRYPTION || true)" 'backslash continuation keeps target-looking line inside non-target logical record'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/continued-other.env (ignore_errors=no)"
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'continued EnvironmentFile record fails closed without target assignment'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${env_file} (ignore_errors=no) ${fixture_root}/comment.env (ignore_errors=no)"
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'later EnvironmentFile assignment wins'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
Environment=K3S_SECRETS_ENCRYPTION=false
|
|
EnvironmentFiles=${env_file} (ignore_errors=no)"
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'EnvironmentFile overrides Environment assignment'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/absent.env (ignore_errors=yes) ${env_file} (ignore_errors=no)"
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'optional missing EnvironmentFile skipped'
|
|
provider_env_file="${fixture_root}/k3s-provider.env"
|
|
printf 'K3S_SECRETS_ENCRYPTION=true\nK3S_SECRETS_ENCRYPTION_PROVIDER=secretbox\n' >"$provider_env_file"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${provider_env_file}"
|
|
assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'EnvironmentFile provider source'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'direct environment enable source'
|
|
dropin_dir="${fixture_root}/k3s.service.d"
|
|
mkdir -p "$dropin_dir"
|
|
printf '[Service]\nEnvironment=K3S_SECRETS_ENCRYPTION=true\n' >"${dropin_dir}/40-encryption.conf"
|
|
K3S_SYSTEMD_DROPIN_DIR="$dropin_dir"
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server'
|
|
assert_eq aescbc/implicit-default "$(detect_effective_encryption_config_owner)" 'systemd drop-in environment source'
|
|
K3S_SYSTEMD_DROPIN_DIR="${fixture_root}/no-dropins"
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server
|
|
Environment=K3S_SECRETS_ENCRYPTION=true K3S_SECRETS_ENCRYPTION=false'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'same precedence environment conflict'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server
|
|
Environment=K3S_SECRETS_ENCRYPTION=true K3S_SECRETS_ENCRYPTION_PROVIDER=aescbc K3S_SECRETS_ENCRYPTION_PROVIDER=secretbox'
|
|
assert_eq secretbox/environment "$(detect_effective_encryption_config_owner)" 'later Environment provider assignment wins'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --config=$CONFIG
|
|
Environment=K3S_SECRETS_ENCRYPTION=true'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'dynamic config expansion rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --config='
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'empty CLI config rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server --secrets-encryption-provider='
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'empty CLI provider rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/missing.env"
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'unreadable EnvironmentFile rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart='
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'blank ExecStart rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/bin/false'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'unrelated ExecStart rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE=$'ExecStart=/usr/local/bin/k3s server\nExecStart=/usr/local/bin/k3s server'
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'multiple ExecStart rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart={ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s server --secrets-encryption=true ; ignore_errors=no ; } { path=/usr/bin/false ; argv[]=/usr/bin/false ; ignore_errors=no ; }
|
|
Environment=
|
|
EnvironmentFiles='
|
|
assert_eq ambiguous "$(detect_effective_encryption_config_owner)" 'multiple serialized ExecStart command records rejection'
|
|
|
|
data_dir="${fixture_root}/data"
|
|
mkdir -p "${data_dir}/server/db"
|
|
touch "${data_dir}/server/db/state.db"
|
|
(
|
|
_k3s_datastore_local_evidence() { printf 'sqlite\n'; }
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server'
|
|
assert_eq sqlite "$(detect_k3s_datastore)" 'default data-dir sqlite evidence'
|
|
)
|
|
(
|
|
_k3s_datastore_local_evidence() { printf 'embedded-etcd\n'; }
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server'
|
|
assert_eq embedded-etcd "$(detect_k3s_datastore)" 'default data-dir embedded etcd evidence'
|
|
)
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'custom CLI data-dir rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} --datastore-endpoint="
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'empty CLI datastore endpoint rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir} --datastore-endpoint=fixture"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'ExecStart external endpoint conflict rejection'
|
|
alt_data_dir="${fixture_root}/alt-data"
|
|
mkdir -p "${alt_data_dir}/server/db"
|
|
touch "${alt_data_dir}/server/db/state.db"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
Environment=K3S_DATA_DIR=${alt_data_dir}"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'custom environment data-dir rejection'
|
|
printf 'data-dir: %s\ndatastore-endpoint: fixture\n' "$alt_data_dir" >"$env_config"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --config ${env_config}"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'merged YAML data-dir and endpoint conflict rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server
|
|
EnvironmentFiles=${fixture_root}/config-path.env"
|
|
printf 'K3S_CONFIG_FILE=%s\n' "$env_config" >"${fixture_root}/config-path.env"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'EnvironmentFile config selection'
|
|
datastore_env_file="${fixture_root}/datastore.env"
|
|
printf 'K3S_DATASTORE_ENDPOINT=fixture\n' >"$datastore_env_file"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}
|
|
EnvironmentFiles=${datastore_env_file}"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'EnvironmentFile external datastore conflict rejection'
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}"
|
|
rm -f -- "${data_dir}/server/db/state.db"
|
|
mkdir -p "${data_dir}/server/db/etcd"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'custom embedded-etcd path rejection'
|
|
rm -rf -- "${data_dir}/server/db/etcd"
|
|
SYSTEMCTL_SHOW_FIXTURE="ExecStart=/usr/local/bin/k3s server --data-dir ${data_dir}
|
|
Environment=K3S_DATASTORE_ENDPOINT=fixture"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'custom path takes precedence over external classification'
|
|
touch "${data_dir}/server/db/state.db"
|
|
mkdir -p "${data_dir}/server/db/etcd"
|
|
assert_eq ambiguous "$(detect_k3s_datastore)" 'conflicting datastore evidence rejection'
|
|
|
|
printf 'secrets-encryption: true\nsecrets-encryption-provider: aescbc\n' >"${config_root}/config.yaml"
|
|
printf 'secrets-encryption-provider: secretbox\n' >"${config_root}/config.yaml.d/40-provider.yaml"
|
|
K3S_CONFIG_DIR="$config_root"
|
|
SYSTEMCTL_SHOW_FIXTURE='ExecStart=/usr/local/bin/k3s server'
|
|
assert_eq "secretbox/${config_root}/config.yaml.d/40-provider.yaml" "$(detect_effective_encryption_config_owner)" 'provider fragment owner'
|
|
|
|
virtual_config="${fixture_root}/not-user-listable/config.yaml"
|
|
(
|
|
_k3s_list_yaml_files() { printf '%s\n' "${config_root}/config.yaml.d/40-provider.yaml"; }
|
|
assert_eq "${config_root}/config.yaml.d/40-provider.yaml|secretbox" \
|
|
"$(_k3s_yaml_value "$virtual_config" secrets-encryption-provider)" \
|
|
'privileged YAML listing callback'
|
|
)
|
|
|
|
# A correct helper returns non-zero before this test-only outer guard fires.
|
|
# Exit 124 means the helper did not bound its own privileged command.
|
|
assert_bounded_hang_failure() {
|
|
local helper="$1" argument="$2" counter rc=0
|
|
counter="${fixture_root}/${helper}-timeout-counter"
|
|
rm -f -- "$counter"
|
|
/usr/bin/timeout --signal=TERM --kill-after=0.1s 1s env \
|
|
PATH="$PATH" K3S_HANG=1 K3S_TIMEOUT_COUNTER="$counter" \
|
|
FAKE_ARG_LOG="$FAKE_ARG_LOG" \
|
|
bash -c 'set -Eeuo pipefail; source "$1"; "$2" "$3"' \
|
|
bash "$LIBRARY_PATH" "$helper" "$argument" || rc=$?
|
|
[[ "$rc" -ne 0 && "$rc" -ne 124 ]] || fail "$helper did not bound a hanging privileged command"
|
|
}
|
|
|
|
assert_bounded_hang_failure wait_for_reencrypt_finished 0
|
|
assert_bounded_hang_failure wait_for_k3s_api ignored
|
|
|
|
# A 599-second first call leaves too little budget for another bounded call.
|
|
# The fake clock makes the ten-minute deadline boundary deterministic and fast.
|
|
assert_deadline_stops_second_call() {
|
|
local helper="$1" argument="$2"
|
|
local clock_file="${fixture_root}/${helper}-clock" calls_file="${fixture_root}/${helper}-deadline-calls" call_count
|
|
printf '0\n' >"$clock_file"
|
|
: >"$calls_file"
|
|
(
|
|
_k3s_now_seconds() { printf '%s\n' "$(<"$clock_file")"; }
|
|
timeout() {
|
|
local now
|
|
printf 'call\n' >>"$calls_file"
|
|
now="$(<"$clock_file")"
|
|
printf '%s\n' "$((now + 599))" >"$clock_file"
|
|
if [[ " $* " == *' secrets-encrypt '* ]]; then
|
|
printf '%s\n' '{"stage":"reencrypt_active","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}'
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
sleep() {
|
|
local now
|
|
now="$(<"$clock_file")"
|
|
printf '%s\n' "$((now + $1))" >"$clock_file"
|
|
}
|
|
assert_fails "$helper" "$argument"
|
|
)
|
|
call_count="$(wc -l <"$calls_file" | tr -d '[:space:]')"
|
|
assert_eq 1 "$call_count" "$helper respects the overall ten-minute deadline (calls=$call_count)"
|
|
}
|
|
|
|
assert_deadline_stops_second_call wait_for_reencrypt_finished 0
|
|
assert_deadline_stops_second_call wait_for_k3s_api ignored
|
|
|
|
K3S_STATUS_FIXTURE=finished
|
|
: >"$FAKE_ARG_LOG"
|
|
assert_succeeds wait_for_reencrypt_finished 0
|
|
assert_eq $'timeout <--signal=TERM> <--kill-after=1s> <9s> <sudo> <--non-interactive> <k3s> <secrets-encrypt> <status> <--output> <json>\nsudo <--non-interactive> <k3s> <secrets-encrypt> <status> <--output> <json>' "$(<"$FAKE_ARG_LOG")" 'bounded authoritative status reader argv'
|
|
K3S_STATUS_FIXTURE=active_then_finished
|
|
K3S_STATUS_COUNTER="${fixture_root}/status-counter"
|
|
export K3S_STATUS_COUNTER
|
|
assert_succeeds wait_for_reencrypt_finished 1
|
|
K3S_STATUS_FIXTURE=start
|
|
assert_fails wait_for_reencrypt_finished 0
|
|
K3S_STATUS_FIXTURE=mismatch
|
|
assert_fails wait_for_reencrypt_finished 0
|
|
K3S_STATUS_FIXTURE=malformed
|
|
assert_fails wait_for_reencrypt_finished 0
|
|
K3S_STATUS_FIXTURE=unsupported
|
|
assert_fails wait_for_reencrypt_finished 0
|
|
K3S_API_RC=0
|
|
: >"$FAKE_ARG_LOG"
|
|
assert_succeeds wait_for_k3s_api
|
|
assert_eq $'timeout <--signal=TERM> <--kill-after=1s> <9s> <sudo> <--non-interactive> <k3s> <kubectl> <get> <--raw=/readyz>\nsudo <--non-interactive> <k3s> <kubectl> <get> <--raw=/readyz>' "$(<"$FAKE_ARG_LOG")" 'bounded credentialed readyz argv'
|
|
K3S_API_RC=1
|
|
assert_fails wait_for_k3s_api
|
|
|
|
# Validator entrypoint fixtures source the production script in a fresh shell,
|
|
# override only its privileged command boundary, and call the real main. There
|
|
# is no production environment switch that enables this harness.
|
|
run_validator_fixture() {
|
|
/usr/bin/env \
|
|
KSE_FIX_STATUS="${KSE_FIX_STATUS:-disabled}" \
|
|
KSE_FIX_INTEGRITY="${KSE_FIX_INTEGRITY:-match}" \
|
|
KSE_FIX_VERSION="${KSE_FIX_VERSION:-v1.36.2+k3s1}" \
|
|
KSE_FIX_NODES="${KSE_FIX_NODES:-valid}" \
|
|
KSE_FIX_API_RC="${KSE_FIX_API_RC:-0}" \
|
|
KSE_FIX_SUDO_RC="${KSE_FIX_SUDO_RC:-0}" \
|
|
KSE_FIX_ARG_LOG="$FAKE_ARG_LOG" \
|
|
/usr/bin/bash -c '
|
|
set -Eeuo pipefail
|
|
source "$1"
|
|
_kse_validate_runtime_dependencies() { return 0; }
|
|
_kse_validate_sudo_credentials() {
|
|
printf "</usr/bin/timeout> <--signal=TERM> <--kill-after=1s> <9s> </usr/bin/sudo> <--non-interactive> <--validate>\n" >>"$KSE_FIX_ARG_LOG"
|
|
[[ "$KSE_FIX_SUDO_RC" == 0 ]]
|
|
}
|
|
_kse_detect_datastore() { printf "sqlite\n"; }
|
|
_kse_privileged_capture() {
|
|
local binary="$1" fixture_hash stage annotation
|
|
shift
|
|
{ printf "<%s>" "$binary"; printf " <%s>" "$@"; printf "\n"; } >>"$KSE_FIX_ARG_LOG"
|
|
fixture_hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
|
if [[ "$binary" == "$KSE_K3S" && "${1:-}" == --version ]]; then
|
|
printf "k3s version %s (fixture)\n" "$KSE_FIX_VERSION"; return 0
|
|
fi
|
|
if [[ "$binary" == "$KSE_K3S" && " ${*} " == *" secrets-encrypt status --output json "* ]]; then
|
|
case "$KSE_FIX_STATUS" in
|
|
disabled) printf "%s\n" "{\"stage\":\"\",\"activekey\":\"\"}" ;;
|
|
transition) printf "%s\n" "{\"stage\":\"start\",\"activekey\":\"\",\"enable\":false,\"hashmatch\":true}" ;;
|
|
enabled_start) printf "%s\n" "{\"stage\":\"start\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;;
|
|
finished) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;;
|
|
active) printf "%s\n" "{\"stage\":\"reencrypt_active\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hashmatch\":true}" ;;
|
|
mismatch) printf "%s\n" "{\"stage\":\"reencrypt_finished\",\"activekey\":\"AES-CBC token-password-encryption-config\",\"enable\":true,\"hasherror\":\"password-token\"}" ;;
|
|
malformed) printf "{\n" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
return 0
|
|
fi
|
|
if [[ "$binary" == "$KSE_K3S" && " ${*} " == *" kubectl get nodes -o json "* ]]; then
|
|
case "$KSE_FIX_STATUS" in disabled) annotation="" ;; transition|enabled_start) annotation="start-${fixture_hash}" ;; *) annotation="reencrypt_finished-${fixture_hash}" ;; esac
|
|
case "$KSE_FIX_NODES" in
|
|
valid) printf "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{%s}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}\n" "${annotation:+\"k3s.io/encryption-config-hash\":\"${annotation}\"}" ;;
|
|
different_ready) printf "{\"items\":[{\"metadata\":{\"name\":\"isolated-restore\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"},\"annotations\":{%s}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}\n" "${annotation:+\"k3s.io/encryption-config-hash\":\"${annotation}\"}" ;;
|
|
wrong_node) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"wrong-node\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}" ;;
|
|
not_ready) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"False\"}]}}]}" ;;
|
|
two_servers) printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"donghyeon-system-product-name\",\"uid\":\"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}},{\"metadata\":{\"name\":\"second\",\"uid\":\"11111111-2222-4333-8444-555555555555\",\"labels\":{\"node-role.kubernetes.io/control-plane\":\"true\"}},\"status\":{\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}]}" ;;
|
|
esac
|
|
return 0
|
|
fi
|
|
if [[ "$binary" == "$KSE_SYSTEMCTL" ]]; then
|
|
printf "%s\n" "ExecStart=/usr/local/bin/k3s server" "Environment=K3S_TOKEN=token-password-encryption-config" "EnvironmentFiles="; return 0
|
|
fi
|
|
if [[ "$binary" == "$KSE_STAT" ]]; then printf "0:600:regular file\n"; return 0; fi
|
|
if [[ "$binary" == "$KSE_SHA256SUM" ]]; then printf "%s %s\n" "$fixture_hash" "$KSE_CONFIG_PATH"; return 0; fi
|
|
if [[ "$binary" == "$KSE_CAT" && " ${*} " == *" $KSE_STATE_PATH "* ]]; then
|
|
case "$KSE_FIX_STATUS" in transition|enabled_start) stage=start ;; *) stage=reencrypt_finished ;; esac
|
|
[[ "$KSE_FIX_INTEGRITY" == match ]] || fixture_hash=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
|
|
printf "%s-%s" "$stage" "$fixture_hash"; return 0
|
|
fi
|
|
return 1
|
|
}
|
|
_kse_privileged_quiet() {
|
|
if [[ "$1" == "$KSE_K3S" && " ${*:2} " == *" kubectl get --raw=/readyz "* ]]; then
|
|
{ printf "<%s>" "$1"; shift; printf " <%s>" "$@"; printf "\n"; } >>"$KSE_FIX_ARG_LOG"
|
|
[[ "$KSE_FIX_API_RC" == 0 ]]; return
|
|
fi
|
|
_kse_privileged_capture "$@" >/dev/null
|
|
}
|
|
k3s_secret_encryption_main "${@:2}"
|
|
' bash "$VALIDATOR_PATH" "$@"
|
|
}
|
|
|
|
# A user-owned executable can never satisfy the production root-owned binary
|
|
# gate, even when it is absolute and executable.
|
|
assert_fails /usr/bin/bash -c 'source "$1"; _kse_trusted_executable "$2"' \
|
|
bash "$VALIDATOR_PATH" "${fake_bin}/k3s"
|
|
|
|
assert_validator_fixture_fails() {
|
|
if run_validator_fixture "$@" >"${fixture_root}/validator-failure.out" 2>"${fixture_root}/validator-failure.err"; then
|
|
fail "validator fixture unexpectedly succeeded: $*"
|
|
fi
|
|
}
|
|
|
|
assert_sensitive_free() {
|
|
local text="$1"
|
|
[[ "$text" != *token* && "$text" != *password* &&
|
|
"$text" != *encryption-config* && "$text" != *'AES-CBC '* ]]
|
|
}
|
|
|
|
# Mutation probe: the sensitive-output assertion must reject this literal.
|
|
assert_fails assert_sensitive_free 'token-password-encryption-config' 'sensitive assertion mutation probe'
|
|
|
|
# A denied noninteractive sudo credential gate must be actionable and must stop
|
|
# before the first privileged evidence command.
|
|
: >"$FAKE_ARG_LOG"
|
|
sudo_denial_rc=0
|
|
KSE_FIX_SUDO_RC=1 run_validator_fixture --expect-disabled \
|
|
>"${fixture_root}/sudo-denial.out" 2>"${fixture_root}/sudo-denial.err" || sudo_denial_rc=$?
|
|
[[ "$sudo_denial_rc" -ne 0 ]] || fail 'sudo denial fixture unexpectedly succeeded'
|
|
assert_eq 'ERROR: noninteractive sudo authorization is unavailable; run sudo -v in the same terminal, then rerun the validator' \
|
|
"$(<"${fixture_root}/sudo-denial.err")" 'sudo denial actionable error'
|
|
assert_eq '</usr/bin/timeout> <--signal=TERM> <--kill-after=1s> <9s> </usr/bin/sudo> <--non-interactive> <--validate>' \
|
|
"$(<"$FAKE_ARG_LOG")" 'sudo denial stops before privileged evidence'
|
|
|
|
assert_fixture_stdout_safe() {
|
|
local status="$1" output expected
|
|
shift
|
|
output="$(KSE_FIX_STATUS="$status" run_validator_fixture "$@" 2>"${fixture_root}/validator-safe.err")" || fail "$status safe stdout fixture"
|
|
assert_sensitive_free "$output" || fail "$status stdout leaks sensitive fixture data"
|
|
case "$status" in
|
|
disabled) expected=$'Encryption status: Disabled\nRotation stage: none' ;;
|
|
transition) expected=$'Encryption status: Disabled\nRotation stage: start' ;;
|
|
enabled_start) expected=$'Encryption status: Enabled\nRotation stage: start' ;;
|
|
finished) expected=$'Encryption status: Enabled\nRotation stage: reencrypt_finished' ;;
|
|
esac
|
|
[[ "$output" == *"$expected"* ]] || fail "$status stdout inventory class"
|
|
}
|
|
|
|
assert_failed_fixture_stdout_safe() {
|
|
local status="$1" output rc=0 expected
|
|
shift
|
|
output="$(KSE_FIX_STATUS="$status" run_validator_fixture "$@" 2>"${fixture_root}/validator-safe-failure.err")" || rc=$?
|
|
[[ "$rc" -ne 0 ]] || fail "$status fixture unexpectedly succeeds"
|
|
assert_sensitive_free "$output" || fail "$status failing stdout leaks sensitive fixture data"
|
|
case "$status" in
|
|
active|mismatch)
|
|
expected=$'Encryption status: Enabled\nRotation stage: unsafe'
|
|
[[ "$output" == *"$expected"* ]] || fail "$status failing stdout inventory class"
|
|
;;
|
|
malformed) [[ -z "$output" ]] || fail 'malformed status writes inventory stdout' ;;
|
|
esac
|
|
}
|
|
|
|
validator_stdout=''
|
|
: >"$FAKE_ARG_LOG"
|
|
KSE_FIX_STATUS=disabled validator_stdout="$(run_validator_fixture --expect-disabled)" || fail 'disabled fixture accepts --expect-disabled'
|
|
[[ "$validator_stdout" != *token* && "$validator_stdout" != *password* && "$validator_stdout" != *encryption-config* ]] || fail 'validator stdout leaks fixture secret material'
|
|
[[ "$(<"$FAKE_ARG_LOG")" != *"${VALIDATOR_PATH}"* ]] || fail 'validator sudo-executes workspace code'
|
|
[[ "$(<"$FAKE_ARG_LOG")" == *'</usr/local/bin/k3s> <--version>'* ]] || fail 'validator does not use the absolute k3s binary'
|
|
[[ "$(<"$FAKE_ARG_LOG")" == *'</usr/local/bin/k3s> <kubectl> <get> <nodes> <-o> <json>'* ]] || fail 'validator node inventory argv'
|
|
[[ "$(<"$FAKE_ARG_LOG")" == *'</usr/local/bin/k3s> <kubectl> <get> <--raw=/readyz>'* ]] || fail 'validator readyz argv'
|
|
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-enabled
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-transition-start
|
|
KSE_FIX_STATUS=transition run_validator_fixture --expect-transition-start >/dev/null || fail 'transition fixture expectation'
|
|
KSE_FIX_STATUS=transition assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=enabled_start run_validator_fixture --expect-enabled >/dev/null || fail 'enabled/start fixture expectation'
|
|
KSE_FIX_STATUS=enabled_start assert_validator_fixture_fails --expect-reencrypted
|
|
KSE_FIX_STATUS=finished run_validator_fixture --expect-enabled >/dev/null || fail 'enabled/finished fixture expectation'
|
|
KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match run_validator_fixture --expect-reencrypted >/dev/null || fail 'reencrypted integrity match fixture'
|
|
KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match KSE_FIX_NODES=different_ready \
|
|
run_validator_fixture --expect-reencrypted-restore >/dev/null || fail 'isolated restore accepts a different ready server hostname'
|
|
KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match KSE_FIX_NODES=not_ready \
|
|
assert_validator_fixture_fails --expect-reencrypted-restore
|
|
KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=mismatch assert_validator_fixture_fails --expect-reencrypted
|
|
KSE_FIX_STATUS=active assert_validator_fixture_fails
|
|
KSE_FIX_STATUS=mismatch assert_validator_fixture_fails
|
|
KSE_FIX_STATUS=malformed assert_validator_fixture_fails
|
|
KSE_FIX_STATUS=disabled KSE_FIX_VERSION=v1.36.1+k3s1 assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=disabled KSE_FIX_VERSION=v1.36.2+k3s1suffix assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=disabled KSE_FIX_NODES=two_servers assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=disabled KSE_FIX_NODES=wrong_node assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=disabled KSE_FIX_NODES=not_ready assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=disabled KSE_FIX_API_RC=1 assert_validator_fixture_fails --expect-disabled
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir ''
|
|
|
|
# Exercise the sensitive-output contract for every validator inventory class,
|
|
# including states that correctly return non-zero.
|
|
assert_fixture_stdout_safe disabled --expect-disabled
|
|
assert_fixture_stdout_safe transition --expect-transition-start
|
|
assert_fixture_stdout_safe enabled_start --expect-enabled
|
|
assert_fixture_stdout_safe finished --expect-enabled
|
|
assert_failed_fixture_stdout_safe active
|
|
assert_failed_fixture_stdout_safe mismatch
|
|
assert_failed_fixture_stdout_safe malformed
|
|
|
|
# Failed expectations must leave a valid empty handoff untouched.
|
|
rejected_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0700 "$rejected_output_dir"
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-enabled --verified-output-dir "$rejected_output_dir"
|
|
[[ -z "$(find "$rejected_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'failed expectation leaves verified output'
|
|
|
|
# Only a physical direct child of /tmp is accepted.
|
|
mkdir -p "${rejected_output_dir}/nested"
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "$rejected_output_dir"
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "${rejected_output_dir}/nested"
|
|
symlink_output_parent="/tmp/platform-k3s-encryption.link${RANDOM}${RANDOM}"
|
|
ln -s -- "$fixture_root" "$symlink_output_parent"
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "${symlink_output_parent}/nested"
|
|
race_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0750 "$race_output_dir"
|
|
KSE_FIX_STATUS=disabled assert_validator_fixture_fails --expect-disabled --verified-output-dir "$race_output_dir"
|
|
|
|
verified_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0700 "$verified_output_dir"
|
|
KSE_FIX_STATUS=disabled validator_stdout="$(run_validator_fixture --expect-disabled --verified-output-dir "$verified_output_dir")" || fail 'verified output fixture accepts safe directory'
|
|
assert_eq 600 "$(stat --format='%a' "$verified_output_dir/inventory.env")" 'inventory handoff mode'
|
|
assert_eq 600 "$(stat --format='%a' "$verified_output_dir/status.sha256")" 'status SHA handoff mode'
|
|
[[ "$(<"$verified_output_dir/inventory.env")" == *'node_uid=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'* ]] || fail 'verified inventory omits immutable node UID'
|
|
assert_eq 4786ad5101dd5c0de42b13d276e2cb0968448fd2b5c16abb6164ffeb69fafd2b "$(<"$verified_output_dir/status.sha256")" 'canonical disabled status SHA'
|
|
[[ "$(<"$verified_output_dir/inventory.env")" != *token* && "$(<"$verified_output_dir/inventory.env")" != *password* && "$(<"$verified_output_dir/inventory.env")" != *encryption-config* ]] || fail 'verified inventory leaks fixture secret material'
|
|
assert_sensitive_free "$(<"$verified_output_dir/inventory.env")$'\n'$(<"$verified_output_dir/status.sha256")" || fail 'disabled handoff leaks sensitive fixture data'
|
|
|
|
transition_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0700 "$transition_output_dir"
|
|
KSE_FIX_STATUS=transition run_validator_fixture --expect-transition-start --verified-output-dir "$transition_output_dir" >/dev/null || fail 'transition verified output fixture'
|
|
assert_sensitive_free "$(<"$transition_output_dir/inventory.env")$'\n'$(<"$transition_output_dir/status.sha256")" || fail 'transition handoff leaks sensitive fixture data'
|
|
|
|
enabled_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0700 "$enabled_output_dir"
|
|
KSE_FIX_STATUS=enabled_start run_validator_fixture --expect-enabled --verified-output-dir "$enabled_output_dir" >/dev/null || fail 'enabled/start verified output fixture'
|
|
assert_sensitive_free "$(<"$enabled_output_dir/inventory.env")$'\n'$(<"$enabled_output_dir/status.sha256")" || fail 'enabled/start handoff leaks sensitive fixture data'
|
|
|
|
finished_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0700 "$finished_output_dir"
|
|
KSE_FIX_STATUS=finished KSE_FIX_INTEGRITY=match run_validator_fixture --expect-reencrypted --verified-output-dir "$finished_output_dir" >/dev/null || fail 'reencrypted verified output fixture'
|
|
assert_eq 9f963cbcdaa79fd76d8c6df50d1108cee96be0e9e947308cb18a033c8eb24595 "$(<"$finished_output_dir/status.sha256")" 'canonical reencrypted status SHA'
|
|
[[ "$(<"$finished_output_dir/inventory.env")" != *token* && "$(<"$finished_output_dir/inventory.env")" != *password* && "$(<"$finished_output_dir/inventory.env")" != *encryption-config* ]] || fail 'reencrypted handoff leaks fixture secret material'
|
|
assert_sensitive_free "$(<"$finished_output_dir/inventory.env")$'\n'$(<"$finished_output_dir/status.sha256")" || fail 'reencrypted handoff leaks sensitive fixture data'
|
|
|
|
# Caller PATH commands must never participate in validation or handoff.
|
|
malicious_bin="${fixture_root}/malicious-bin"
|
|
malicious_marker="${fixture_root}/malicious-path-used"
|
|
mkdir -p "$malicious_bin"
|
|
for command_name in dirname stat find id timeout sudo; do
|
|
printf '%s\n' '#!/usr/bin/env bash' ": >\"${malicious_marker}\"" 'exit 99' >"${malicious_bin}/${command_name}"
|
|
chmod 0700 "${malicious_bin}/${command_name}"
|
|
done
|
|
malicious_output_dir="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
|
|
chmod 0700 "$malicious_output_dir"
|
|
PATH="${malicious_bin}:${ORIGINAL_PATH}" KSE_FIX_STATUS=disabled run_validator_fixture --expect-disabled --verified-output-dir "$malicious_output_dir" >/dev/null || fail 'validator ignores caller PATH commands'
|
|
[[ ! -e "$malicious_marker" ]] || fail 'validator trusted a caller PATH command'
|
|
rm -rf -- "$malicious_output_dir"
|
|
|
|
# The host drop-in must contain exactly the two encryption settings and no
|
|
# Kubernetes resource metadata or unrelated configuration.
|
|
awk '
|
|
/^[[:space:]]*($|#)/ { next }
|
|
$0 == "secrets-encryption: true" { enabled++; next }
|
|
$0 == "secrets-encryption-provider: aescbc" { provider++; next }
|
|
{ unexpected++ }
|
|
END { exit !(enabled == 1 && provider == 1 && unexpected == 0) }
|
|
' infrastructure/security/k3s/40-secrets-encryption.yaml
|
|
|
|
# Bootstrap state-machine regression coverage. The production script is
|
|
# sourced and its host boundaries are replaced in this subshell only; there is
|
|
# deliberately no environment-controlled fake path in production.
|
|
readonly BOOTSTRAP_PATH="${REPOSITORY_ROOT}/scripts/bootstrap/apply-k3s-secret-encryption.sh"
|
|
bootstrap_fixture_root="${fixture_root}/bootstrap"
|
|
bootstrap_backup_root="${bootstrap_fixture_root}/recovery"
|
|
mkdir -p "$bootstrap_backup_root"
|
|
|
|
run_bootstrap_fixture() (
|
|
local scenario="$1"
|
|
shift
|
|
local log="${bootstrap_fixture_root}/${scenario}.log"
|
|
: >"$log"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
|
|
_kseb_load_inventory() {
|
|
printf 'inventory\n' >>"$log"
|
|
case "$scenario" in
|
|
disabled_sqlite|local_success|local_plain_ext4|local_wrong_outer_disk|pre_capacity_failure|post_capacity_failure|backup_failure|service_recovery_failure|post_backup_failure|marker_failure|final_validation_failure|timeout_finished|timeout_start|root_failure|enable_failure|dropin_failure|initial_restart_failure|final_restart_failure|estimate|estimate_invalid)
|
|
printf 'disabled_no_config|none|sqlite|none|none|mismatch|mismatch\n'
|
|
;;
|
|
disabled_etcd)
|
|
printf 'disabled_no_config|none|embedded-etcd|none|none|mismatch|mismatch\n'
|
|
;;
|
|
disabled_backend_drift_external|disabled_backend_drift_ambiguous)
|
|
if [[ -e "${bootstrap_fixture_root}/${scenario}.counter" ]]; then
|
|
if [[ "$scenario" == disabled_backend_drift_external ]]; then
|
|
printf 'disabled_no_config|none|external|none|none|mismatch|mismatch\n'
|
|
else
|
|
printf 'disabled_no_config|none|ambiguous|none|none|mismatch|mismatch\n'
|
|
fi
|
|
else
|
|
: >"${bootstrap_fixture_root}/${scenario}.counter"
|
|
printf 'disabled_no_config|none|sqlite|none|none|mismatch|mismatch\n'
|
|
fi
|
|
;;
|
|
mismatch)
|
|
printf 'hash_mismatch|unsafe|sqlite|invalid|ambiguous|mismatch|mismatch\n'
|
|
;;
|
|
external)
|
|
printf 'disabled_no_config|none|external|none|none|mismatch|mismatch\n'
|
|
;;
|
|
ambiguous_backend)
|
|
printf 'disabled_no_config|none|ambiguous|none|none|mismatch|mismatch\n'
|
|
;;
|
|
enabled_finished)
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|match|match\n'
|
|
;;
|
|
enabled_finished_drift_provider|enabled_finished_drift_owner)
|
|
if [[ -e "${bootstrap_fixture_root}/${scenario}.counter" ]]; then
|
|
if [[ "$scenario" == enabled_finished_drift_provider ]]; then
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|secretbox|secretbox/config-owner|match|match\n'
|
|
else
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|ambiguous|match|match\n'
|
|
fi
|
|
else
|
|
: >"${bootstrap_fixture_root}/${scenario}.counter"
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|match|match\n'
|
|
fi
|
|
;;
|
|
enabled_wrong_provider)
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|secretbox|secretbox/config-owner|match|match\n'
|
|
;;
|
|
enabled_wrong_owner)
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|ambiguous|match|match\n'
|
|
;;
|
|
enabled_bad_integrity)
|
|
printf 'enabled_stable|reencrypt_finished|sqlite|aescbc|aescbc/config-owner|mismatch|match\n'
|
|
;;
|
|
enabled_start|enabled_start_rotate)
|
|
printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|match|match\n'
|
|
;;
|
|
enabled_start_drift)
|
|
if [[ -e "${bootstrap_fixture_root}/enabled-start-drift.counter" ]]; then
|
|
printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|mismatch|mismatch\n'
|
|
else
|
|
: >"${bootstrap_fixture_root}/enabled-start-drift.counter"
|
|
printf 'enabled_stable|start|sqlite|aescbc|aescbc/config-owner|match|match\n'
|
|
fi
|
|
;;
|
|
*) return 97 ;;
|
|
esac
|
|
}
|
|
# Keep the legacy override during the RED stage so the old implementation
|
|
# cannot reach a real terminal. The new implementation must use the split
|
|
# confirmation boundaries below instead.
|
|
_kseb_confirm_execute() { printf 'legacy-confirm\n' >>"$log"; }
|
|
_kseb_prepare_execute_context() { printf 'default\n'; }
|
|
_kseb_confirm_apply() { printf 'APPLY\n' >>"$log"; }
|
|
_kseb_validate_backup_root() {
|
|
printf 'generic-root\n' >>"$log"
|
|
[[ "$scenario" != root_failure ]]
|
|
}
|
|
_kseb_validate_recovery_policy() {
|
|
local policy="$1"
|
|
printf 'policy-root:%s\n' "$policy" >>"$log"
|
|
case "$scenario" in
|
|
local_plain_ext4|local_wrong_outer_disk) return 1 ;;
|
|
*) return 0 ;;
|
|
esac
|
|
}
|
|
_kseb_confirm_recovery() {
|
|
local context="$1" policy="$2"
|
|
[[ "$context" == default ]]
|
|
printf 'RECOVERY\nENCRYPTED\n' >>"$log"
|
|
if [[ "$policy" == local-separate-disk-luks ]]; then
|
|
printf 'LOCAL_RISK_ACCEPTED\n' >>"$log"
|
|
fi
|
|
}
|
|
_kseb_backup_phase() {
|
|
local phase="$1" backend="$2"
|
|
if [[ "$scenario" == pre_capacity_failure && "$phase" == pre ]]; then
|
|
printf 'capacity:pre\n' >>"$log"
|
|
return 1
|
|
fi
|
|
if [[ "$scenario" == post_capacity_failure && "$phase" == post ]]; then
|
|
printf 'capacity:post\n' >>"$log"
|
|
return 1
|
|
fi
|
|
printf 'backup:%s:%s\n' "$phase" "$backend" >>"$log"
|
|
[[ "$scenario" != backup_failure || "$phase" != pre ]] &&
|
|
[[ "$scenario" != post_backup_failure || "$phase" != post ]] || return $?
|
|
if [[ "$scenario" == service_recovery_failure && "$phase" == pre ]]; then return 75; fi
|
|
if [[ "$scenario" == marker_failure && "$phase" == post ]]; then return 76; fi
|
|
}
|
|
_kseb_enable() { printf 'enable\n' >>"$log"; [[ "$scenario" != enable_failure ]]; }
|
|
_kseb_install_dropin() { printf 'install-dropin\n' >>"$log"; [[ "$scenario" != dropin_failure ]]; }
|
|
_kseb_restart() {
|
|
local counter="${bootstrap_fixture_root}/${scenario}.restart-counter"
|
|
printf 'restart\n' >>"$log"
|
|
if [[ "$scenario" == initial_restart_failure ]]; then return 1; fi
|
|
if [[ "$scenario" == final_restart_failure ]]; then
|
|
if [[ -e "$counter" ]]; then return 1; fi
|
|
: >"$counter"
|
|
fi
|
|
}
|
|
_kseb_rotate_keys() {
|
|
printf 'rotate\n' >>"$log"
|
|
case "$scenario" in timeout_finished|timeout_start) return 124 ;; *) return 0 ;; esac
|
|
}
|
|
_kseb_read_status_json() {
|
|
local stage counter="${bootstrap_fixture_root}/${scenario}.status-counter"
|
|
case "$scenario" in
|
|
timeout_finished)
|
|
if [[ -e "$counter" ]]; then stage=reencrypt_finished; else stage=reencrypt_active; : >"$counter"; fi
|
|
;;
|
|
timeout_start) stage=start ;;
|
|
*) stage=reencrypt_finished ;;
|
|
esac
|
|
printf 'poll:%s\n' "$stage" >>"$log"
|
|
printf '{"stage":"%s","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}\n' "$stage"
|
|
}
|
|
sleep() { :; }
|
|
_kseb_validate_expectation() {
|
|
printf 'validate:%s\n' "$1" >>"$log"
|
|
[[ "$scenario" != final_validation_failure || "$1" != --expect-reencrypted ]]
|
|
}
|
|
_kseb_dry_run() { printf 'dry-run\n' >>"$log"; }
|
|
_kseb_estimate_phase_bytes() {
|
|
printf 'estimate\n' >>"$log"
|
|
if [[ "$scenario" == estimate_invalid ]]; then
|
|
printf '9223372036854775808\n'
|
|
else
|
|
printf '1048576\n'
|
|
fi
|
|
}
|
|
|
|
apply_k3s_secret_encryption_main "$@" \
|
|
>"${bootstrap_fixture_root}/${scenario}.stdout" \
|
|
2>"${bootstrap_fixture_root}/${scenario}.stderr"
|
|
)
|
|
|
|
assert_bootstrap_order() {
|
|
local scenario="$1" expected="$2"
|
|
assert_eq "$expected" "$(<"${bootstrap_fixture_root}/${scenario}.log")" "$scenario bootstrap order"
|
|
}
|
|
|
|
# These tests protect the required, closed recovery-policy parser and split
|
|
# confirmation dispatch. Missing or malformed policy input must stop before
|
|
# inventory collection, confirmation, or mutation dispatch.
|
|
assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root"
|
|
assert_bootstrap_order disabled_sqlite ''
|
|
assert_exit 2 run_bootstrap_fixture disabled_sqlite --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order disabled_sqlite ''
|
|
assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy unknown
|
|
assert_bootstrap_order disabled_sqlite ''
|
|
assert_exit 2 run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order disabled_sqlite ''
|
|
|
|
assert_succeeds run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order disabled_sqlite $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite'
|
|
|
|
assert_succeeds run_bootstrap_fixture local_success --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy local-separate-disk-luks
|
|
assert_bootstrap_order local_success $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks\nRECOVERY\nENCRYPTED\nLOCAL_RISK_ACCEPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite'
|
|
|
|
assert_fails run_bootstrap_fixture local_plain_ext4 --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy local-separate-disk-luks
|
|
assert_bootstrap_order local_plain_ext4 $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks'
|
|
assert_fails run_bootstrap_fixture local_wrong_outer_disk --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy local-separate-disk-luks
|
|
assert_bootstrap_order local_wrong_outer_disk $'inventory\nAPPLY\ngeneric-root\npolicy-root:local-separate-disk-luks'
|
|
|
|
# The production change that makes these probes pass is the real local policy
|
|
# boundary: exact inner root, one validator call, then pinned-root revalidation.
|
|
probe_recovery_policy_dispatch() (
|
|
local policy="$1" root="$2" scenario="$3" rc=0
|
|
local log="${bootstrap_fixture_root}/policy-dispatch-${scenario}.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_local_recovery_root() { printf '/fixture/local-recovery\n'; }
|
|
_kseb_run_local_recovery_validator() {
|
|
printf 'local-validator\n' >>"$log"
|
|
[[ "$scenario" != validator_failure ]]
|
|
}
|
|
_kseb_revalidate_backup_root() {
|
|
printf 'root-revalidated\n' >>"$log"
|
|
[[ "$scenario" != identity_drift ]]
|
|
}
|
|
_kseb_validate_recovery_policy "$policy" "$root" || rc=$?
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'local-validator\nroot-revalidated\nrc=0' \
|
|
"$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery success)" \
|
|
'local recovery policy validates exact root and revalidates identity'
|
|
assert_eq 'rc=1' \
|
|
"$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery/child child_root)" \
|
|
'local recovery policy rejects a child root before validation'
|
|
assert_eq $'local-validator\nrc=1' \
|
|
"$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery validator_failure)" \
|
|
'local recovery policy rejects validator failure before root revalidation'
|
|
assert_eq $'local-validator\nroot-revalidated\nrc=1' \
|
|
"$(probe_recovery_policy_dispatch local-separate-disk-luks /fixture/local-recovery identity_drift)" \
|
|
'local recovery policy rejects root identity drift'
|
|
assert_eq 'rc=0' \
|
|
"$(probe_recovery_policy_dispatch encrypted-off-host /fixture/off-host off_host)" \
|
|
'off-host policy never dispatches the local validator'
|
|
|
|
local_contract_fixture="${bootstrap_fixture_root}/local-recovery-contract.env"
|
|
awk -F= '
|
|
$1 == "K3SLR_INNER_MOUNT" { print "K3SLR_INNER_MOUNT=/fixture/contract-root"; next }
|
|
{ print }
|
|
' infrastructure/security/k3s/local-recovery.env >"$local_contract_fixture"
|
|
probe_local_recovery_contract_binding() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_local_recovery_contract() { printf '%s\n' "$local_contract_fixture"; }
|
|
_kseb_local_recovery_root
|
|
)
|
|
assert_eq '/fixture/contract-root' "$(probe_local_recovery_contract_binding)" \
|
|
'local policy root is read from the authoritative recovery contract'
|
|
|
|
probe_recovery_policy_contract_drift() (
|
|
local marker="${bootstrap_fixture_root}/contract-drift.marker" rc=0
|
|
local log="${bootstrap_fixture_root}/contract-drift.log"
|
|
rm -f -- "$marker"; : >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_local_recovery_root() {
|
|
if [[ -e "$marker" ]]; then
|
|
printf '/fixture/root-after\n'
|
|
else
|
|
: >"$marker"
|
|
printf '/fixture/root-before\n'
|
|
fi
|
|
}
|
|
_kseb_run_local_recovery_validator() { printf 'local-validator\n' >>"$log"; }
|
|
_kseb_revalidate_backup_root() { printf 'unexpected-root-revalidation\n' >>"$log"; }
|
|
_kseb_validate_recovery_policy local-separate-disk-luks /fixture/root-before || rc=$?
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'local-validator\nrc=1' "$(probe_recovery_policy_contract_drift)" \
|
|
'local policy rejects contract root drift after validator success'
|
|
|
|
duplicate_contract_fixture="${bootstrap_fixture_root}/local-recovery-contract-duplicate.env"
|
|
awk '
|
|
{ print }
|
|
END { print "K3SLR_INNER_MOUNT=/fixture/duplicate-root" }
|
|
' infrastructure/security/k3s/local-recovery.env >"$duplicate_contract_fixture"
|
|
probe_duplicate_local_recovery_contract() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_local_recovery_contract() { printf '%s\n' "$duplicate_contract_fixture"; }
|
|
_kseb_local_recovery_root
|
|
)
|
|
assert_fails probe_duplicate_local_recovery_contract
|
|
|
|
probe_local_validator_boundary() (
|
|
local scenario="$1" rc=0
|
|
local command_log="${bootstrap_fixture_root}/local-validator-${scenario}.argv"
|
|
local public_out="${bootstrap_fixture_root}/local-validator-${scenario}.stdout"
|
|
local public_err="${bootstrap_fixture_root}/local-validator-${scenario}.stderr"
|
|
: >"$command_log"; : >"$public_out"; : >"$public_err"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
if [[ "$scenario" == cleanup_failure ]]; then
|
|
_kseb_safe_remove_local_validator_handoff() {
|
|
/usr/bin/rm -- "$1/stdout" "$1/stderr" >/dev/null 2>&1 || true
|
|
/usr/bin/rmdir -- "$1" >/dev/null 2>&1 || true
|
|
return 1
|
|
}
|
|
fi
|
|
_kseb_local_validator_command() {
|
|
printf '%s\n' "$@" >"$command_log"
|
|
case "$scenario" in
|
|
success|cleanup_failure)
|
|
printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n'
|
|
;;
|
|
extra_stdout)
|
|
printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\nUNEXPECTED\n'
|
|
;;
|
|
stderr_output)
|
|
printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n'
|
|
printf 'RAW-IDENTIFIER-SENTINEL\n' >&2
|
|
;;
|
|
command_failure) return 1 ;;
|
|
oversize) /usr/bin/yes X | /usr/bin/head --bytes=16384 ;;
|
|
*) return 97 ;;
|
|
esac
|
|
}
|
|
_kseb_run_local_recovery_validator >"$public_out" 2>"$public_err" || rc=$?
|
|
[[ ! -s "$public_out" && ! -s "$public_err" ]] || return 1
|
|
printf 'rc=%s\n' "$rc"
|
|
)
|
|
assert_eq 'rc=0' "$(probe_local_validator_boundary success)" \
|
|
'local validator accepts only its exact safe success record'
|
|
assert_eq $'/usr/bin/env\n-i\nPATH=/usr/sbin:/usr/bin:/sbin:/bin\nLC_ALL=C\n/usr/bin/timeout\n--signal=TERM\n--kill-after=1s\n60s\n/usr/bin/bash\n--noprofile\n--norc\n--\n'"${REPOSITORY_ROOT}"$'/scripts/validate/k3s-local-recovery.sh\n--expect-open' \
|
|
"$(<"${bootstrap_fixture_root}/local-validator-success.argv")" \
|
|
'local validator command uses fixed binaries, clean environment, and expect-open'
|
|
assert_eq 'rc=1' "$(probe_local_validator_boundary extra_stdout)" \
|
|
'local validator rejects extra stdout without exposing it'
|
|
assert_eq 'rc=1' "$(probe_local_validator_boundary stderr_output)" \
|
|
'local validator rejects stderr without exposing it'
|
|
assert_eq 'rc=1' "$(probe_local_validator_boundary command_failure)" \
|
|
'local validator rejects a nonzero command result'
|
|
assert_eq 'rc=1' "$(probe_local_validator_boundary cleanup_failure)" \
|
|
'local validator treats capture cleanup failure as policy failure'
|
|
local_validator_tmp_count() {
|
|
/usr/bin/find /tmp -maxdepth 1 -type d -name 'platform-k3s-local-validator.*' -printf . | /usr/bin/wc -c
|
|
}
|
|
local_validator_tmp_before="$(local_validator_tmp_count)"
|
|
assert_eq 'rc=1' "$(probe_local_validator_boundary oversize)" \
|
|
'local validator bounds oversized output and returns failure'
|
|
assert_eq "$local_validator_tmp_before" "$(local_validator_tmp_count)" \
|
|
'oversized local validator output leaves no temporary directory'
|
|
|
|
probe_local_validator_result_matrix() (
|
|
local scenario="$1" directory="${bootstrap_fixture_root}/validator-result-${1}"
|
|
local stdout_file="$directory/stdout" stderr_file="$directory/stderr" command_rc=0
|
|
rm -rf -- "$directory"; mkdir -m 0700 -- "$directory"
|
|
: >"$stderr_file"
|
|
case "$scenario" in
|
|
exact|timeout|signal|stderr)
|
|
printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file"
|
|
;;
|
|
embedded_nul)
|
|
printf 'Recovery device: matc\0\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file"
|
|
;;
|
|
carriage_return)
|
|
printf 'Recovery device: matc\r\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file"
|
|
;;
|
|
short)
|
|
printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\n' >"$stdout_file"
|
|
/usr/bin/truncate --size=85 -- "$stdout_file"
|
|
;;
|
|
long)
|
|
printf 'Recovery device: match\nRecovery state: open\nLineage: match\nLatest bundle: not_checked\nX' >"$stdout_file"
|
|
;;
|
|
*) return 97 ;;
|
|
esac
|
|
case "$scenario" in
|
|
timeout) command_rc=124 ;;
|
|
signal) command_rc=137 ;;
|
|
stderr) printf 'unexpected\n' >"$stderr_file" ;;
|
|
esac
|
|
chmod 0600 -- "$stdout_file" "$stderr_file"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_local_validator_result_is_exact "$command_rc" "$stdout_file" "$stderr_file"
|
|
)
|
|
assert_succeeds probe_local_validator_result_matrix exact
|
|
assert_fails probe_local_validator_result_matrix embedded_nul
|
|
assert_fails probe_local_validator_result_matrix carriage_return
|
|
assert_fails probe_local_validator_result_matrix short
|
|
assert_fails probe_local_validator_result_matrix long
|
|
assert_fails probe_local_validator_result_matrix timeout
|
|
assert_fails probe_local_validator_result_matrix signal
|
|
assert_fails probe_local_validator_result_matrix stderr
|
|
|
|
probe_local_validator_exact_cleanup() (
|
|
local scenario="$1" directory
|
|
directory="$(/usr/bin/mktemp -d /tmp/platform-k3s-local-validator.XXXXXX)" || return 1
|
|
case "$directory" in /tmp/platform-k3s-local-validator.[A-Za-z0-9]*) ;; *) return 1 ;; esac
|
|
trap '/usr/bin/rm -f -- "$directory/stdout" "$directory/stderr" "$directory/extra"; /usr/bin/rmdir -- "$directory" 2>/dev/null || true' EXIT
|
|
: >"$directory/stdout"; : >"$directory/stderr"
|
|
if [[ "$scenario" == extra ]]; then : >"$directory/extra"; fi
|
|
chmod 0600 -- "$directory"/*
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
if [[ "$scenario" == success ]]; then
|
|
_kseb_safe_remove_local_validator_handoff "$directory" || return 1
|
|
[[ ! -e "$directory" ]]
|
|
else
|
|
if _kseb_safe_remove_local_validator_handoff "$directory"; then return 1; fi
|
|
[[ -d "$directory" && -f "$directory/extra" && ! -e "$directory/stdout" && ! -e "$directory/stderr" ]]
|
|
fi
|
|
)
|
|
assert_succeeds probe_local_validator_exact_cleanup success
|
|
assert_succeeds probe_local_validator_exact_cleanup extra
|
|
|
|
probe_recovery_confirmation() (
|
|
local policy="$1" answers="$2" context_after="${3:-default}" rc=0 output
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_current_context() { printf '%s\n' "$context_after"; }
|
|
output="$(_kseb_confirm_recovery default "$policy" <<<"$answers" 2>&1)" || rc=$?
|
|
printf 'rc=%s|offhost=%s|localrisk=%s\n' "$rc" \
|
|
"$([[ "$output" == *off-host* ]] && printf 1 || printf 0)" \
|
|
"$([[ "$output" == *LOCAL_RISK_ACCEPTED* ]] && printf 1 || printf 0)"
|
|
)
|
|
assert_eq 'rc=0|offhost=1|localrisk=0' \
|
|
"$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nENCRYPTED default')" \
|
|
'off-host policy keeps its two explicit confirmations'
|
|
assert_eq 'rc=0|offhost=0|localrisk=1' \
|
|
"$(probe_recovery_confirmation local-separate-disk-luks $'RECOVERY default\nENCRYPTED default\nLOCAL_RISK_ACCEPTED default')" \
|
|
'local policy uses truthful encryption and local-risk confirmations'
|
|
assert_eq 'rc=1|offhost=0|localrisk=1' \
|
|
"$(probe_recovery_confirmation local-separate-disk-luks $'RECOVERY default\nENCRYPTED default')" \
|
|
'local policy refuses a missing local-risk confirmation'
|
|
assert_eq 'rc=1|offhost=0|localrisk=0' \
|
|
"$(probe_recovery_confirmation encrypted-off-host $'WRONG default\nENCRYPTED default')" \
|
|
'recovery confirmation rejects the wrong RECOVERY record'
|
|
assert_eq 'rc=1|offhost=1|localrisk=0' \
|
|
"$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nWRONG default')" \
|
|
'recovery confirmation rejects the wrong ENCRYPTED record'
|
|
assert_eq 'rc=1|offhost=1|localrisk=0' \
|
|
"$(probe_recovery_confirmation encrypted-off-host $'RECOVERY default\nENCRYPTED default' changed)" \
|
|
'recovery confirmation rejects context drift'
|
|
|
|
probe_apply_confirmation() (
|
|
local answer="$1" rc=0
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_confirm_apply default <<<"$answer" >/dev/null 2>&1 || rc=$?
|
|
printf 'rc=%s\n' "$rc"
|
|
)
|
|
assert_eq 'rc=0' "$(probe_apply_confirmation 'APPLY default')" 'APPLY confirmation exact success'
|
|
assert_eq 'rc=1' "$(probe_apply_confirmation 'APPLY other')" 'APPLY confirmation rejects another context'
|
|
assert_eq 'rc=1' "$(probe_apply_confirmation 'apply default')" 'APPLY confirmation is case-sensitive'
|
|
|
|
# Existing bootstrap state-machine regressions continue below. Once the
|
|
# implementation exists, these literals protect the entire mutation order.
|
|
assert_succeeds run_bootstrap_fixture disabled_etcd --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
[[ "$(<"${bootstrap_fixture_root}/disabled_etcd.log")" == *$'backup:pre:embedded-etcd'*$'backup:post:embedded-etcd'* ]] || fail 'embedded-etcd backend routing'
|
|
|
|
assert_succeeds run_bootstrap_fixture timeout_finished --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order timeout_finished $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_active\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite'
|
|
assert_eq 1 "$(awk '$0 == "rotate" { count++ } END { print count + 0 }' "${bootstrap_fixture_root}/timeout_finished.log")" 'timeout never retries rotate-keys'
|
|
|
|
assert_fails run_bootstrap_fixture timeout_start --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order timeout_start $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:start'
|
|
|
|
assert_fails run_bootstrap_fixture mismatch --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order mismatch 'inventory'
|
|
assert_fails run_bootstrap_fixture external --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order external 'inventory'
|
|
assert_fails run_bootstrap_fixture ambiguous_backend --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order ambiguous_backend 'inventory'
|
|
assert_fails run_bootstrap_fixture backup_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order backup_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite'
|
|
assert_fails run_bootstrap_fixture root_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order root_failure $'inventory\nAPPLY\ngeneric-root'
|
|
assert_fails run_bootstrap_fixture final_validation_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
[[ "$(<"${bootstrap_fixture_root}/final_validation_failure.log")" != *'backup:post:'* ]] || fail 'post backup before final verification'
|
|
|
|
assert_succeeds run_bootstrap_fixture enabled_finished --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_finished $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory\nbackup:post:sqlite'
|
|
assert_fails run_bootstrap_fixture enabled_wrong_provider --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_wrong_provider 'inventory'
|
|
assert_fails run_bootstrap_fixture enabled_wrong_owner --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_wrong_owner 'inventory'
|
|
assert_fails run_bootstrap_fixture enabled_bad_integrity --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_bad_integrity 'inventory'
|
|
|
|
assert_fails run_bootstrap_fixture enabled_start --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_start 'inventory'
|
|
assert_succeeds run_bootstrap_fixture enabled_start_rotate --execute --rotate-existing --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_start_rotate $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-enabled\nbackup:pre:sqlite\ninventory\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\nbackup:post:sqlite'
|
|
|
|
assert_succeeds run_bootstrap_fixture disabled_sqlite
|
|
assert_bootstrap_order disabled_sqlite $'inventory\ndry-run'
|
|
assert_fails run_bootstrap_fixture disabled_sqlite --rotate-existing
|
|
assert_fails run_bootstrap_fixture disabled_sqlite --execute --backup-root relative/path --recovery-policy encrypted-off-host
|
|
assert_fails run_bootstrap_fixture disabled_sqlite --execute --backup-root "$bootstrap_backup_root" \
|
|
--backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
|
|
# These CLI regressions catch estimator dispatch through any mutating branch,
|
|
# accepting an invalid estimate, or emitting anything except the public line.
|
|
assert_succeeds run_bootstrap_fixture estimate --estimate-recovery-bytes
|
|
assert_eq 'phase_bytes=1048576' "$(<"${bootstrap_fixture_root}/estimate.stdout")" \
|
|
'read-only phase estimator output'
|
|
assert_bootstrap_order estimate $'inventory\nestimate'
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --execute
|
|
assert_exit 2 run_bootstrap_fixture estimate --execute --estimate-recovery-bytes
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --rotate-existing
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --backup-root "$bootstrap_backup_root"
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --estimate-recovery-bytes
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --help
|
|
assert_exit 2 run_bootstrap_fixture estimate --help --estimate-recovery-bytes
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy auto
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy encrypted-off-host
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --recovery-policy local-separate-disk-luks
|
|
assert_exit 2 run_bootstrap_fixture estimate --estimate-recovery-bytes --unknown
|
|
assert_fails run_bootstrap_fixture estimate_invalid --estimate-recovery-bytes
|
|
assert_eq '' "$(<"${bootstrap_fixture_root}/estimate_invalid.stdout")" \
|
|
'invalid estimate has no public output'
|
|
|
|
probe_estimator_fixed_sources() (
|
|
local backend="$1" log="${bootstrap_fixture_root}/estimate-${1}-sources.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_estimator_source_kind() { printf 'present\n'; }
|
|
_kseb_estimator_source_identity() {
|
|
case "$1" in
|
|
/usr/lib/systemd/system/k3s.service|/lib/systemd/system/k3s.service) printf '41:8\n' ;;
|
|
*) printf '41:%s\n' "$(printf '%s' "$1" | /usr/bin/cksum | awk '{print $1}')" ;;
|
|
esac
|
|
}
|
|
_kseb_privileged_estimator_du() {
|
|
printf '%s\n' "$1" >>"$log"
|
|
printf '1\t%s\n' "$1"
|
|
}
|
|
_kseb_estimate_phase_bytes "$backend"
|
|
)
|
|
assert_eq 16777224 "$(probe_estimator_fixed_sources sqlite)" \
|
|
'SQLite estimator sums fixed backup inputs once'
|
|
assert_eq $'/var/lib/rancher/k3s/server/db\n/var/lib/rancher/k3s/server/token\n/etc/rancher/k3s\n/etc/systemd/system/k3s.service\n/etc/systemd/system/k3s.service.env\n/etc/systemd/system/k3s.service.d\n/usr/lib/systemd/system/k3s.service\n/var/lib/rancher/k3s/server/cred/encryption-config.json' \
|
|
"$(<"${bootstrap_fixture_root}/estimate-sqlite-sources.log")" \
|
|
'SQLite estimator reads each effective fixed source once'
|
|
assert_eq 16777224 "$(probe_estimator_fixed_sources embedded-etcd)" \
|
|
'embedded-etcd estimator sums fixed backup inputs once'
|
|
assert_eq '/var/lib/rancher/k3s/server/db/etcd' \
|
|
"$(sed -n '1p' "${bootstrap_fixture_root}/estimate-embedded-etcd-sources.log")" \
|
|
'embedded-etcd estimator uses current local data directory'
|
|
|
|
probe_estimator_du_contract() (
|
|
local scenario="$1" marker="${bootstrap_fixture_root}/estimate-identity-${1}.counter"
|
|
rm -f -- "$marker"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_estimator_source_kind() { printf 'present\n'; }
|
|
_kseb_estimator_source_identity() {
|
|
if [[ "$scenario" == identity_change && -e "$marker" ]]; then printf '41:2\n'
|
|
else printf '41:1\n'; : >"$marker"; fi
|
|
}
|
|
_kseb_privileged_estimator_du() {
|
|
case "$scenario" in
|
|
malformed) printf 'not-a-number\t%s\n' "$1" ;;
|
|
multiline) printf '1\t%s\n2\t%s\n' "$1" "$1" ;;
|
|
trailing_blank) printf '1\t%s\n\n' "$1" ;;
|
|
wrong_path) printf '1\t/etc/shadow\n' ;;
|
|
*) printf '1\t%s\n' "$1" ;;
|
|
esac
|
|
}
|
|
_kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/token required
|
|
)
|
|
assert_fails probe_estimator_du_contract malformed
|
|
assert_fails probe_estimator_du_contract multiline
|
|
assert_fails probe_estimator_du_contract trailing_blank
|
|
assert_fails probe_estimator_du_contract wrong_path
|
|
assert_fails probe_estimator_du_contract identity_change
|
|
|
|
probe_estimator_symlink_parent() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_estimator_source_kind() { printf 'present\n'; }
|
|
_kseb_privileged_estimator_lstat() {
|
|
case "$1" in
|
|
/|/var) printf '41:1|41ed\n' ;;
|
|
/var/lib) printf '41:2|a1ff\n' ;;
|
|
*) printf '41:3|41ed\n' ;;
|
|
esac
|
|
}
|
|
_kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; }
|
|
_kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/db required
|
|
)
|
|
assert_fails probe_estimator_symlink_parent
|
|
|
|
probe_estimator_numeric_mode_metadata() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_estimator_source_kind() { printf 'present\n'; }
|
|
_kseb_privileged_estimator_lstat() {
|
|
if [[ "$1" == /var/lib/rancher/k3s/server/token ]]; then
|
|
printf '41:9|81a4\n'
|
|
else
|
|
printf '41:8|41ed\n'
|
|
fi
|
|
}
|
|
_kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; }
|
|
_kseb_estimate_source_bytes sqlite /var/lib/rancher/k3s/server/token required
|
|
)
|
|
assert_eq 1 "$(probe_estimator_numeric_mode_metadata)" \
|
|
'numeric stat mode accepts directories and a regular-file source'
|
|
|
|
probe_estimator_numeric_stat_argv() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_estimator_capture() {
|
|
printf '<%s>' "$@"
|
|
printf '\n'
|
|
}
|
|
_kseb_privileged_estimator_lstat /var/lib/rancher/k3s/server/token
|
|
)
|
|
assert_eq '</usr/bin/stat><--format=%d:%i|%f><--></var/lib/rancher/k3s/server/token>' \
|
|
"$(probe_estimator_numeric_stat_argv 2>/dev/null || true)" \
|
|
'privileged stat uses locale-independent numeric mode data'
|
|
|
|
probe_estimator_out_of_bound() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_estimator_source_kind() { printf 'present\n'; }
|
|
_kseb_estimator_source_identity() { printf '41:1\n'; }
|
|
_kseb_privileged_estimator_du() { printf '1\t%s\n' "$1"; }
|
|
_kseb_estimate_source_bytes sqlite /tmp/not-a-k3s-source required
|
|
)
|
|
assert_fails probe_estimator_out_of_bound
|
|
|
|
probe_estimator_overflow() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_estimator_source_records() {
|
|
printf 'required|/var/lib/rancher/k3s/server/db\n'
|
|
printf 'required|/var/lib/rancher/k3s/server/token\n'
|
|
}
|
|
_kseb_estimator_source_probe() {
|
|
[[ "$1" == */server/db ]] && printf '41:1\n' || printf '41:2\n'
|
|
}
|
|
_kseb_estimate_source_bytes() {
|
|
[[ "$2" == */server/db ]] && printf '9223372036854775800\n' || printf '8\n'
|
|
}
|
|
_kseb_estimate_phase_bytes sqlite
|
|
)
|
|
assert_fails probe_estimator_overflow
|
|
assert_fails probe_estimator_fixed_sources external
|
|
|
|
# The production change that makes these arithmetic cases pass is an
|
|
# overflow-safe P + ceil(P / 4) reserve calculation. Expected values are
|
|
# hand-derived literals rather than values computed with production helpers.
|
|
probe_required_recovery_bytes() (
|
|
local phase="$1" phase_bytes="$2"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_required_recovery_bytes "$phase" "$phase_bytes"
|
|
)
|
|
assert_eq 10737418240 "$(probe_required_recovery_bytes post 0)" 'zero-byte post reserve'
|
|
assert_eq 10737418240 "$(probe_required_recovery_bytes pre 0)" 'zero-byte pre reserve'
|
|
assert_eq 10737418242 "$(probe_required_recovery_bytes post 1)" 'one-byte post reserve rounds upward'
|
|
assert_eq 10737418244 "$(probe_required_recovery_bytes pre 1)" 'one-byte pre reserve rounds upward twice'
|
|
assert_eq 10737418245 "$(probe_required_recovery_bytes post 0004)" 'normalized four-byte post reserve'
|
|
assert_eq 10737418250 "$(probe_required_recovery_bytes pre 4)" 'four-byte pre reserve'
|
|
assert_eq 11811160065 "$(probe_required_recovery_bytes post 858993460)" 'post capacity literal'
|
|
assert_eq 12884901890 "$(probe_required_recovery_bytes pre 858993460)" 'pre capacity literal'
|
|
assert_eq 9223372036854775807 "$(probe_required_recovery_bytes post 7378697620893886053)" \
|
|
'maximum signed post reserve'
|
|
assert_eq 9223372036854775806 "$(probe_required_recovery_bytes pre 3689348810446943026)" \
|
|
'maximum signed pre reserve'
|
|
assert_fails probe_required_recovery_bytes post 7378697620893886054
|
|
assert_fails probe_required_recovery_bytes pre 3689348810446943027
|
|
assert_fails probe_required_recovery_bytes post 7378697629483820646
|
|
assert_fails probe_required_recovery_bytes post 9223372036854775807
|
|
assert_fails probe_required_recovery_bytes invalid 1
|
|
assert_fails probe_required_recovery_bytes pre -1
|
|
assert_fails probe_required_recovery_bytes post 1.25
|
|
assert_fails probe_required_recovery_bytes post 9223372036854775808
|
|
|
|
probe_phase_capacity_gate() (
|
|
local phase="$1" fixture_phase_bytes="$2" fixture_free_bytes="$3" rc=0
|
|
local log="${bootstrap_fixture_root}/capacity-${phase}-${fixture_free_bytes}.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_estimate_phase_bytes() { printf 'estimate:%s\n' "$1" >>"$log"; printf '%s\n' "$fixture_phase_bytes"; }
|
|
_kseb_privileged_free_bytes() { printf 'free:%s\n' "$1" >>"$log"; printf '%s\n' "$fixture_free_bytes"; }
|
|
_kseb_require_phase_capacity "$phase" sqlite /fixture-root || rc=$?
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \
|
|
"$(probe_phase_capacity_gate pre 858993460 12884901890)" 'pre capacity exact threshold'
|
|
assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \
|
|
"$(probe_phase_capacity_gate pre 858993460 12884901889)" 'pre capacity one byte short'
|
|
assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \
|
|
"$(probe_phase_capacity_gate post 858993460 11811160065)" 'post capacity exact threshold'
|
|
assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \
|
|
"$(probe_phase_capacity_gate post 858993460 11811160064)" 'post capacity one byte short'
|
|
assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=0' \
|
|
"$(probe_phase_capacity_gate post 858993460 00011811160065)" 'free-byte normalization'
|
|
assert_eq $'estimate:sqlite\nfree:/fixture-root\nrc=1' \
|
|
"$(probe_phase_capacity_gate post 858993460 9223372036854775808)" 'free-byte signed overflow rejection'
|
|
|
|
# Focused regression: API-derived metadata must be captured before entering the
|
|
# backend routine that stops SQLite. The backend deliberately returns failure
|
|
# before any filesystem command, so this exercises production orchestration
|
|
# without a backup write or host call.
|
|
probe_backup_metadata_order() (
|
|
local log="${bootstrap_fixture_root}/metadata-order.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_require_phase_capacity() {
|
|
printf 'capacity:%s:%s:%s\n' "$1" "$2" "$3" >>"$log"
|
|
}
|
|
_kseb_secret_count() { printf 'secret-count\n' >>"$log"; printf '7\n'; }
|
|
_kseb_version() { printf 'version\n' >>"$log"; printf 'v1.36.2+k3s1\n'; }
|
|
_kseb_revalidate_backup_root() { :; }
|
|
_kseb_pinned_phase_transaction() {
|
|
printf 'transaction:%s:count=%s:version=%s\n' "${5:-missing}" "${7:-missing}" "${8:-missing}" >>"$log"
|
|
return 1
|
|
}
|
|
_kseb_backup_phase pre sqlite /fixture-root /fixture-bundle \
|
|
00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || true
|
|
_kseb_backup_phase post sqlite /fixture-root /fixture-bundle \
|
|
00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || true
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'capacity:pre:sqlite:/fixture-root\nsecret-count\nversion\ntransaction:pre:count=7:version=v1.36.2+k3s1\ncapacity:post:sqlite:/fixture-root\nsecret-count\nversion\ntransaction:post:count=7:version=v1.36.2+k3s1' \
|
|
"$(probe_backup_metadata_order)" 'pre and post each recalculate capacity before stopped-service backend'
|
|
|
|
probe_backup_capacity_failure() (
|
|
local phase="$1" log="${bootstrap_fixture_root}/capacity-failure-${1}.log" rc=0
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_revalidate_backup_root() { :; }
|
|
_kseb_require_phase_capacity() { printf 'capacity:%s\n' "$1" >>"$log"; return 1; }
|
|
_kseb_secret_count() { printf 'unexpected-secret-count\n' >>"$log"; }
|
|
_kseb_version() { printf 'unexpected-version\n' >>"$log"; }
|
|
_kseb_pinned_phase_transaction() { printf 'unexpected-transaction\n' >>"$log"; }
|
|
_kseb_backup_phase "$phase" sqlite /fixture-root /fixture-bundle \
|
|
00000000-0000-0000-0000-000000000000 20260801T000000Z >/dev/null 2>&1 || rc=$?
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'capacity:pre\nrc=1' "$(probe_backup_capacity_failure pre)" \
|
|
'pre capacity failure stops before API metadata and backup transaction'
|
|
assert_eq $'capacity:post\nrc=1' "$(probe_backup_capacity_failure post)" \
|
|
'post capacity failure stops before API metadata and backup transaction'
|
|
|
|
# Focused regression: the bootstrap owns a bounded API waiter whose command
|
|
# boundary can use the absolute k3s binary. The fake readiness probe is the
|
|
# only injected dependency and succeeds on its second call.
|
|
probe_bootstrap_api_waiter() (
|
|
local log="${bootstrap_fixture_root}/api-wait.log" counter="${bootstrap_fixture_root}/api-wait.counter"
|
|
: >"$log"
|
|
rm -f -- "$counter"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_api_ready_once() {
|
|
printf 'probe\n' >>"$log"
|
|
if [[ -e "$counter" ]]; then return 0; fi
|
|
: >"$counter"
|
|
return 1
|
|
}
|
|
sleep() { printf 'sleep:%s\n' "$1" >>"$log"; }
|
|
_kseb_wait_for_api
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'probe\nsleep:10\nprobe' "$(probe_bootstrap_api_waiter 2>/dev/null || true)" \
|
|
'bootstrap bounded absolute-command API waiter'
|
|
|
|
# A root-only recovery mount is not traversable by the invoking user, so phase
|
|
# collision decisions must be made from privileged evidence rather than shell
|
|
# -e checks. This pure production helper makes those decisions testable.
|
|
probe_phase_layout_policy() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_phase_layout_allowed pre absent absent absent || return 1
|
|
if _kseb_phase_layout_allowed pre directory '0:0:700:directory' absent; then return 1; fi
|
|
_kseb_phase_layout_allowed post absent absent absent || return 1
|
|
_kseb_phase_layout_allowed post directory '0:0:700:directory' directory || return 1
|
|
if _kseb_phase_layout_allowed post directory '0:0:755:directory' directory; then return 1; fi
|
|
if _kseb_phase_layout_allowed post directory '0:0:700:directory' absent; then return 1; fi
|
|
)
|
|
assert_succeeds probe_phase_layout_policy
|
|
|
|
# Enabled/start can drift during confirmation or the pre-backup. The second
|
|
# inventory is the immediate pre-rotation gate and must stop all later mutation.
|
|
assert_fails run_bootstrap_fixture enabled_start_drift --execute --rotate-existing --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_start_drift $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-enabled\nbackup:pre:sqlite\ninventory'
|
|
|
|
# The guarded SQLite lifecycle must attempt start after every stop attempt,
|
|
# including a stop error or copy failure, and only run recovery after a
|
|
# confirmed start. These dependencies are local functions in the probe.
|
|
probe_sqlite_guard() (
|
|
local scenario="$1" log="${bootstrap_fixture_root}/sqlite-guard-${1}.log" rc=0
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_stop_k3s() { printf 'stop\n' >>"$log"; [[ "$scenario" != stop_failure ]]; }
|
|
_kseb_start_k3s() { printf 'start\n' >>"$log"; [[ "$scenario" != start_failure ]]; }
|
|
_kseb_sqlite_copy_and_verify() { printf 'copy\n' >>"$log"; [[ "$scenario" != copy_failure ]]; }
|
|
_kseb_before_start_dispatch() {
|
|
if [[ "$scenario" == pre_start_term ]]; then kill -s TERM "$BASHPID"; fi
|
|
}
|
|
_kseb_recovery_check() { printf 'recovery\n' >>"$log"; }
|
|
_kseb_backup_sqlite_guarded /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$?
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'stop\ncopy\nstart\nrecovery\nrc=0' "$(probe_sqlite_guard success 2>/dev/null || true)" \
|
|
'SQLite guarded success lifecycle'
|
|
assert_eq $'stop\ncopy\nstart\nrc=1' "$(probe_sqlite_guard copy_failure 2>/dev/null || true)" \
|
|
'SQLite copy failure restarts service'
|
|
assert_eq $'stop\nstart\nrc=1' "$(probe_sqlite_guard stop_failure 2>/dev/null || true)" \
|
|
'SQLite stop error still attempts service start'
|
|
assert_eq $'stop\ncopy\nstart\nrc=75' "$(probe_sqlite_guard start_failure 2>/dev/null || true)" \
|
|
'SQLite start failure is not reported as recovery success'
|
|
assert_eq $'stop\ncopy\nstart\nrc=143' "$(probe_sqlite_guard pre_start_term 2>/dev/null || true)" \
|
|
'SQLite pre-start TERM dispatches start once and stops flow'
|
|
|
|
probe_quiet_command_boundary() (
|
|
local output rc=0
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
noisy_backup_failure() {
|
|
printf 'sentinel-recovery-absolute-path\n'
|
|
printf 'sentinel-server-token-and-encryption-config\n' >&2
|
|
return 42
|
|
}
|
|
output="$(_kseb_run_quiet noisy_backup_failure 2>&1)" || rc=$?
|
|
[[ "$rc" == 42 && -z "$output" ]]
|
|
)
|
|
assert_succeeds probe_quiet_command_boundary
|
|
|
|
probe_relative_manifest_builder() (
|
|
local fixture="${bootstrap_fixture_root}/manifest-fixture" output="${bootstrap_fixture_root}/manifest-output"
|
|
mkdir -p "$fixture/sub"
|
|
: >"$fixture/alpha"; : >"$fixture/sub/beta"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_manifest_list_files() { printf '%s\0%s\0' "$fixture/alpha" "$fixture/sub/beta"; }
|
|
_kseb_manifest_hash_file() {
|
|
case "$1" in */alpha) printf '%064d\n' 1 ;; */beta) printf '%064d\n' 2 ;; *) return 1 ;; esac
|
|
}
|
|
_kseb_build_relative_manifest "$fixture" "$output"
|
|
assert_eq $'0000000000000000000000000000000000000000000000000000000000000001 ./alpha\n0000000000000000000000000000000000000000000000000000000000000002 ./sub/beta' \
|
|
"$(<"$output")" 'relative manifest entries'
|
|
)
|
|
assert_succeeds probe_relative_manifest_builder
|
|
|
|
probe_manifest_hash_failure() (
|
|
local fixture="${bootstrap_fixture_root}/manifest-failure" output="${bootstrap_fixture_root}/manifest-failure-output"
|
|
mkdir -p "$fixture"
|
|
: >"$fixture/alpha"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_manifest_list_files() { printf '%s\0' "$fixture/alpha"; }
|
|
_kseb_manifest_hash_file() { return 1; }
|
|
! _kseb_build_relative_manifest "$fixture" "$output"
|
|
)
|
|
assert_succeeds probe_manifest_hash_failure
|
|
|
|
probe_manifest_list_failure() (
|
|
local fixture="${bootstrap_fixture_root}/manifest-list-failure" output="${bootstrap_fixture_root}/manifest-list-failure-output"
|
|
mkdir -p "$fixture"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_manifest_list_files() { return 1; }
|
|
if _kseb_build_relative_manifest "$fixture" "$output"; then return 1; fi
|
|
)
|
|
assert_succeeds probe_manifest_list_failure
|
|
|
|
probe_manifest_install_check() (
|
|
local scenario="$1" log="${bootstrap_fixture_root}/manifest-check-${1}.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_manifest_lifecycle() {
|
|
printf 'build\ninstall\ncheck\n' >>"$log"
|
|
[[ "$scenario" != check_failure ]]
|
|
}
|
|
if [[ "$scenario" == check_failure ]]; then
|
|
! _kseb_write_verification_manifest /phase
|
|
else
|
|
_kseb_write_verification_manifest /phase
|
|
fi
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'build\ninstall\ncheck' "$(probe_manifest_install_check success 2>/dev/null || true)" \
|
|
'manifest installed then checked'
|
|
assert_eq $'build\ninstall\ncheck' "$(probe_manifest_install_check check_failure 2>/dev/null || true)" \
|
|
'manifest check failure propagates'
|
|
|
|
probe_backup_root_chain_policy() (
|
|
local scenario="$1"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_path_lstat() {
|
|
case "$scenario/$1" in
|
|
writable//safe) printf '0:0:777:directory\n' ;;
|
|
symlink//safe) printf '0:0:777:symbolic link\n' ;;
|
|
*) printf '0:0:755:directory\n' ;;
|
|
esac
|
|
}
|
|
_kseb_backup_root_chain_safe /safe/recovery
|
|
)
|
|
assert_succeeds probe_backup_root_chain_policy safe
|
|
assert_fails probe_backup_root_chain_policy writable
|
|
assert_fails probe_backup_root_chain_policy symlink
|
|
|
|
probe_backup_root_identity_policy() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_backup_root_identity_matches '41:99' '41:99' || return 1
|
|
if _kseb_backup_root_identity_matches '41:99' '41:100'; then return 1; fi
|
|
)
|
|
assert_succeeds probe_backup_root_identity_policy
|
|
|
|
probe_backend_command_contract() (
|
|
local log="${bootstrap_fixture_root}/backend-command-contract.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_exec() { printf '<%s>\n' "$*" >>"$log"; }
|
|
_kseb_etcd_snapshot_save pre 20260801T000000Z /phase-pre
|
|
_kseb_etcd_snapshot_list /phase-pre
|
|
_kseb_etcd_snapshot_save post 20260801T000000Z /phase-post
|
|
_kseb_etcd_snapshot_list /phase-post
|
|
_kseb_archive_host_config /phase-pre
|
|
_kseb_copy_sqlite_datastore /phase-pre
|
|
_kseb_compare_sqlite_datastore /phase-pre
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'</usr/local/bin/k3s etcd-snapshot save --name=pre-secrets-encryption-20260801T000000Z --etcd-snapshot-compress --dir=/phase-pre>\n</usr/local/bin/k3s etcd-snapshot list --dir=/phase-pre>\n</usr/local/bin/k3s etcd-snapshot save --name=post-secrets-encryption-20260801T000000Z --etcd-snapshot-compress --dir=/phase-post>\n</usr/local/bin/k3s etcd-snapshot list --dir=/phase-post>\n</usr/bin/tar --exclude=./platform-post-bundle.env --exclude=./platform-restore-evidence.env -C /etc/rancher/k3s -cpf /phase-pre/host-config.tar .>\n</bin/cp -a -- /var/lib/rancher/k3s/server/db /phase-pre/datastore>\n</usr/bin/diff --no-dereference --recursive --brief /var/lib/rancher/k3s/server/db /phase-pre/datastore>' \
|
|
"$(probe_backend_command_contract 2>/dev/null || true)" 'production backend exact argv contract'
|
|
|
|
probe_production_backend_order() (
|
|
local backend="$1" scenario="$2" log="${bootstrap_fixture_root}/production-${1}-${2}.log" rc=0
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_copy_sqlite_datastore() { printf 'sqlite-copy\n' >>"$log"; }
|
|
_kseb_compare_sqlite_datastore() { printf 'sqlite-compare\n' >>"$log"; }
|
|
_kseb_etcd_snapshot_save() { printf 'etcd-save:%s\n' "$1" >>"$log"; }
|
|
_kseb_etcd_snapshot_list() { printf 'etcd-list\n' >>"$log"; }
|
|
_kseb_copy_common_recovery() { printf 'common-copy\n' >>"$log"; }
|
|
_kseb_write_bundle_metadata() { printf 'metadata\n' >>"$log"; }
|
|
_kseb_write_verification_manifest() {
|
|
printf 'manifest\n' >>"$log"
|
|
[[ "$scenario" != manifest_failure ]]
|
|
}
|
|
_kseb_recovery_check() { printf 'recovery\n' >>"$log"; }
|
|
if [[ "$backend" == sqlite ]]; then
|
|
_kseb_sqlite_copy_and_verify /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$?
|
|
else
|
|
_kseb_backup_etcd /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$?
|
|
fi
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'sqlite-copy\ncommon-copy\nsqlite-compare\nmetadata\nmanifest\nrc=0' \
|
|
"$(probe_production_backend_order sqlite success)" 'production SQLite helper order'
|
|
assert_eq $'sqlite-copy\ncommon-copy\nsqlite-compare\nmetadata\nmanifest\nrc=1' \
|
|
"$(probe_production_backend_order sqlite manifest_failure)" 'SQLite manifest failure propagation'
|
|
assert_eq $'etcd-save:pre\netcd-list\ncommon-copy\nmetadata\nmanifest\nrecovery\nrc=0' \
|
|
"$(probe_production_backend_order etcd success)" 'production embedded-etcd helper order'
|
|
assert_eq $'etcd-save:pre\netcd-list\ncommon-copy\nmetadata\nmanifest\nrc=1' \
|
|
"$(probe_production_backend_order etcd manifest_failure)" 'etcd manifest failure blocks recovery'
|
|
|
|
probe_terminal_guidance_contract() (
|
|
local output rc=0 injected
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
output="$(_kseb_terminal_failure reencrypt_stage_start 2>&1)" || rc=$?
|
|
[[ "$rc" == 1 && "$output" == 'ERROR: current-state=reencrypt_stage_start recovery=bootstrap/manual/k3s-secret-encryption.md' ]] || return 1
|
|
injected="$(_kseb_terminal_failure 'unsafe-/sentinel-recovery-path-server-token' 2>&1 || true)"
|
|
[[ "$injected" == 'ERROR: current-state=unknown recovery=bootstrap/manual/k3s-secret-encryption.md' ]]
|
|
)
|
|
assert_succeeds probe_terminal_guidance_contract
|
|
|
|
assert_public_failure_guidance() {
|
|
local scenario="$1" expected_state="$2" output
|
|
output="$(<"${bootstrap_fixture_root}/${scenario}.stderr")"
|
|
[[ "$output" == *"ERROR: current-state=${expected_state} recovery=bootstrap/manual/k3s-secret-encryption.md"* ]] ||
|
|
fail "$scenario terminal guidance"
|
|
[[ "$output" != *'sentinel-recovery-absolute-path'* && "$output" != *'server-token'* &&
|
|
"$output" != *'encryption-config'* ]] || fail "$scenario terminal guidance leak"
|
|
}
|
|
assert_public_failure_guidance timeout_start reencrypt_stage_start
|
|
assert_public_failure_guidance mismatch hash_mismatch
|
|
assert_public_failure_guidance final_validation_failure final_validation_failed
|
|
assert_public_failure_guidance backup_failure pre_backup_failed
|
|
assert_fails run_bootstrap_fixture pre_capacity_failure --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order pre_capacity_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\ncapacity:pre'
|
|
assert_public_failure_guidance pre_capacity_failure pre_backup_failed
|
|
assert_fails run_bootstrap_fixture post_capacity_failure --execute --backup-root "$bootstrap_backup_root" \
|
|
--recovery-policy encrypted-off-host
|
|
assert_bootstrap_order post_capacity_failure $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory\nbackup:pre:sqlite\nenable\ninstall-dropin\nrestart\nvalidate:--expect-transition-start\nrotate\npoll:reencrypt_finished\nrestart\nvalidate:--expect-reencrypted\ncapacity:post'
|
|
assert_public_failure_guidance post_capacity_failure post_backup_failed
|
|
assert_fails run_bootstrap_fixture service_recovery_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance service_recovery_failure service_recovery_failed
|
|
assert_fails run_bootstrap_fixture post_backup_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance post_backup_failure post_backup_failed
|
|
assert_fails run_bootstrap_fixture marker_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance marker_failure marker_install_failed
|
|
|
|
assert_fails run_bootstrap_fixture enable_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance enable_failure partial_enable_failed
|
|
assert_fails run_bootstrap_fixture dropin_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance dropin_failure partial_dropin_failed
|
|
assert_fails run_bootstrap_fixture initial_restart_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance initial_restart_failure partial_initial_restart_failed
|
|
assert_fails run_bootstrap_fixture final_restart_failure --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_public_failure_guidance final_restart_failure partial_final_restart_failed
|
|
|
|
probe_recovery_budget_policy() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_recovery_budget_valid 430 130 30 || return 1
|
|
if _kseb_recovery_budget_valid 441 130 30; then return 1; fi
|
|
)
|
|
assert_succeeds probe_recovery_budget_policy
|
|
|
|
probe_workload_ready_policy() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
local empty_status='{"items":[{"status":{"phase":"Running","containerStatuses":[]},"spec":{"containers":[{"name":"app"}],"initContainers":[]}}]}'
|
|
local ready='{"items":[{"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":true}],"initContainerStatuses":[]},"spec":{"containers":[{"name":"app"}],"initContainers":[]}}]}'
|
|
if _kseb_workloads_ready_json "$empty_status"; then return 1; fi
|
|
_kseb_workloads_ready_json "$ready"
|
|
)
|
|
assert_succeeds probe_workload_ready_policy
|
|
|
|
# Regression: after a K3s restart, a transient non-ready Pod must be retried
|
|
# inside the existing 30-second workload budget instead of failing the backup
|
|
# on the first observation. Clock, sleep, and API reads are file-backed
|
|
# because command substitutions execute the read seam in a child shell.
|
|
probe_workload_wait_transient() (
|
|
local state_dir="${bootstrap_fixture_root}/workload-wait-transient"
|
|
local clock_file="${state_dir}/clock" counter_file="${state_dir}/counter"
|
|
local log="${state_dir}/log"
|
|
local nonready ready
|
|
mkdir -p "$state_dir"
|
|
printf '0\n' >"$clock_file"
|
|
printf '0\n' >"$counter_file"
|
|
: >"$log"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
nonready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":false}],"initContainerStatuses":[]}}]}'
|
|
ready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":true}],"initContainerStatuses":[]}}]}'
|
|
|
|
_kseb_now_seconds() { command cat -- "$clock_file"; }
|
|
_kseb_read_workloads_json() {
|
|
local command_timeout="$1" count
|
|
printf 'read:%s\n' "$command_timeout" >>"$log"
|
|
count="$(command cat -- "$counter_file")"
|
|
count=$((count + 1))
|
|
printf '%s\n' "$count" >"$counter_file"
|
|
if (( count == 1 )); then printf '%s\n' "$nonready"; else printf '%s\n' "$ready"; fi
|
|
}
|
|
sleep() {
|
|
local delay="$1" now
|
|
printf 'sleep:%s\n' "$delay" >>"$log"
|
|
now="$(command cat -- "$clock_file")"
|
|
printf '%s\n' "$((now + delay))" >"$clock_file"
|
|
}
|
|
|
|
_kseb_wait_for_workloads || return 1
|
|
[[ "$(command cat -- "$log")" == $'read:9\nsleep:5\nread:9' ]]
|
|
)
|
|
assert_succeeds probe_workload_wait_transient
|
|
|
|
probe_workload_wait_deadline() (
|
|
local state_dir="${bootstrap_fixture_root}/workload-wait-deadline"
|
|
local clock_file="${state_dir}/clock" log="${state_dir}/log"
|
|
local nonready
|
|
mkdir -p "$state_dir"
|
|
printf '0\n' >"$clock_file"
|
|
: >"$log"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
nonready='{"items":[{"metadata":{"name":"fixture","namespace":"default"},"spec":{"containers":[{"name":"app"}],"initContainers":[]},"status":{"phase":"Running","containerStatuses":[{"name":"app","ready":false}],"initContainerStatuses":[]}}]}'
|
|
|
|
_kseb_now_seconds() { command cat -- "$clock_file"; }
|
|
_kseb_read_workloads_json() {
|
|
printf 'read:%s\n' "$1" >>"$log"
|
|
printf '%s\n' "$nonready"
|
|
}
|
|
sleep() {
|
|
local delay="$1" now
|
|
printf 'sleep:%s\n' "$delay" >>"$log"
|
|
now="$(command cat -- "$clock_file")"
|
|
printf '%s\n' "$((now + delay))" >"$clock_file"
|
|
}
|
|
|
|
if _kseb_wait_for_workloads; then return 1; fi
|
|
[[ "$(command cat -- "$log")" == $'read:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:9\nsleep:5\nread:4\nsleep:5' ]]
|
|
)
|
|
assert_succeeds probe_workload_wait_deadline
|
|
|
|
if [[ "${KSEB_WORKLOAD_WAIT_FOCUS:-}" == source ]]; then
|
|
printf 'K3S WORKLOAD WAIT SOURCE TEST PASS\n'
|
|
exit 0
|
|
fi
|
|
|
|
probe_sqlite_signal_guard() (
|
|
local signal="$1" log="${bootstrap_fixture_root}/sqlite-signal-${1}.log" rc=0
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_stop_k3s() { printf 'stop\n' >>"$log"; }
|
|
_kseb_start_k3s() { printf 'start\n' >>"$log"; }
|
|
_kseb_sqlite_copy_and_verify() {
|
|
printf 'copy\n' >>"$log"
|
|
kill -s "$signal" "$BASHPID"
|
|
printf 'continued-after-signal\n' >>"$log"
|
|
}
|
|
_kseb_recovery_check() { printf 'recovery\n' >>"$log"; }
|
|
_kseb_backup_sqlite_guarded /phase bundle pre stamp 7 v1.36.2+k3s1 || rc=$?
|
|
printf 'rc=%s\n' "$rc" >>"$log"
|
|
command cat -- "$log"
|
|
)
|
|
assert_eq $'stop\ncopy\nstart\nrc=143' "$(probe_sqlite_signal_guard TERM 2>/dev/null || true)" \
|
|
'SQLite TERM guard starts once and stops flow'
|
|
assert_eq $'stop\ncopy\nstart\nrc=130' "$(probe_sqlite_signal_guard INT 2>/dev/null || true)" \
|
|
'SQLite INT guard starts once and stops flow'
|
|
|
|
# Round 1: a documented root:root 0700 mount cannot be traversed by the
|
|
# invoking user. Exercise the production decision with only privileged
|
|
# evidence readers replaced.
|
|
probe_privileged_root0700_validation() (
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_privileged_root_kind() { printf 'directory\n'; }
|
|
_kseb_privileged_realpath() { printf '%s\n' "$1"; }
|
|
_kseb_privileged_root_metadata() { printf '0:0:700:directory\n'; }
|
|
_kseb_privileged_backup_root_chain_safe() { return 0; }
|
|
_kseb_privileged_mount_source() { [[ "$1" == "$KSEB_DATA_DIR" ]] && printf 'data-source\n' || printf 'backup-source\n'; }
|
|
_kseb_privileged_free_bytes() { printf '10737418240\n'; }
|
|
_kseb_privileged_root_identity() { printf '41:99\n'; }
|
|
_kseb_validate_backup_root /secure/recovery
|
|
[[ "$KSEB_BACKUP_ROOT_IDENTITY" == '41:99' ]]
|
|
)
|
|
assert_succeeds probe_privileged_root0700_validation
|
|
|
|
probe_pinned_root_swap_boundary() (
|
|
local scenario="$1" dispatched=false
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY='41:99'
|
|
_kseb_privileged_backup_root_chain_safe() { return 0; }
|
|
_kseb_privileged_root_identity() {
|
|
[[ "$scenario" == swapped ]] && printf '41:100\n' || printf '41:99\n'
|
|
}
|
|
_kseb_privileged_relative_components_safe() { return 0; }
|
|
_kseb_privileged_pinned_dispatch() { dispatched=true; }
|
|
if [[ "$scenario" == swapped ]]; then
|
|
if _kseb_pinned_root_operation /secure/recovery bundle/pre verify; then return 1; fi
|
|
[[ "$dispatched" == false ]]
|
|
else
|
|
_kseb_pinned_root_operation /secure/recovery bundle/pre verify
|
|
[[ "$dispatched" == true ]]
|
|
fi
|
|
)
|
|
assert_succeeds probe_pinned_root_swap_boundary stable
|
|
assert_succeeds probe_pinned_root_swap_boundary swapped
|
|
|
|
probe_actual_pinned_root_swap() (
|
|
local parent="${bootstrap_fixture_root}/actual-swap" root dispatched=false
|
|
mkdir -p "$parent/recovery/bundle/pre" "$parent/replacement/bundle/pre"
|
|
root="$parent/recovery"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
|
|
_kseb_privileged_backup_root_chain_safe() {
|
|
mv -- "$root" "$parent/original"
|
|
mv -- "$parent/replacement" "$root"
|
|
}
|
|
_kseb_privileged_root_identity() { stat --format='%d:%i' -- "$1"; }
|
|
_kseb_privileged_relative_components_safe() { return 0; }
|
|
_kseb_privileged_pinned_dispatch() { dispatched=true; }
|
|
if _kseb_pinned_root_operation "$root" bundle/pre verify; then return 1; fi
|
|
[[ "$dispatched" == false ]]
|
|
)
|
|
assert_succeeds probe_actual_pinned_root_swap
|
|
|
|
probe_root_only_manifest_entrypoint() (
|
|
local log="${bootstrap_fixture_root}/root-only-manifest.log"
|
|
: >"$log"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
mktemp() { printf 'user-mktemp\n' >>"$log"; return 1; }
|
|
_kseb_privileged_manifest_lifecycle() { printf 'root-lifecycle\n' >>"$log"; }
|
|
_kseb_write_verification_manifest /secure/recovery/bundle/pre
|
|
assert_eq 'root-lifecycle' "$(<"$log")" 'manifest lifecycle remains root-only'
|
|
)
|
|
assert_succeeds probe_root_only_manifest_entrypoint
|
|
|
|
probe_production_mismatch_load_boundary() (
|
|
local inventory
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
bash() { return 1; }
|
|
_kseb_read_status_json() {
|
|
printf '%s\n' '{"stage":"reencrypt_finished","activekey":"AES-CBC fixture-key","enable":true,"hasherror":"mismatch"}'
|
|
}
|
|
inventory="$(_kseb_load_inventory)" || return 1
|
|
[[ "$inventory" == hash_mismatch\|unsafe\|ambiguous\|invalid\|ambiguous\|mismatch\|mismatch ]]
|
|
)
|
|
assert_succeeds probe_production_mismatch_load_boundary
|
|
|
|
probe_unexpected_stage_output_safety() (
|
|
local output rc=0 output_file="${bootstrap_fixture_root}/unexpected-stage.out"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
_kseb_read_status_json() {
|
|
printf '%s\n' '{"stage":"unexpected\nSENTINEL-STAGE","activekey":"AES-CBC fixture-key","enable":true,"hashmatch":true}'
|
|
}
|
|
: >"$output_file"
|
|
_kseb_wait_for_reencrypt 124 >"$output_file" 2>&1 || rc=$?
|
|
output="$(<"$output_file")"
|
|
[[ "$rc" != 0 && -z "$output" && "$KSEB_LAST_SAFE_STATE" == reencrypt_stage_unexpected ]]
|
|
)
|
|
assert_succeeds probe_unexpected_stage_output_safety
|
|
|
|
assert_fails run_bootstrap_fixture enabled_finished_drift_provider --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_finished_drift_provider $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory'
|
|
assert_fails run_bootstrap_fixture enabled_finished_drift_owner --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order enabled_finished_drift_owner $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-reencrypted\ninventory'
|
|
|
|
assert_fails run_bootstrap_fixture disabled_backend_drift_external --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order disabled_backend_drift_external $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory'
|
|
assert_fails run_bootstrap_fixture disabled_backend_drift_ambiguous --execute --backup-root "$bootstrap_backup_root" --recovery-policy encrypted-off-host
|
|
assert_bootstrap_order disabled_backend_drift_ambiguous $'inventory\nAPPLY\ngeneric-root\npolicy-root:encrypted-off-host\nRECOVERY\nENCRYPTED\nvalidate:--expect-disabled\ninventory'
|
|
|
|
# Round 2: the production privileged dispatcher must dereference both the root
|
|
# and phase directory FDs. Only privilege elevation is replaced; the fixed
|
|
# dispatcher and its stat decisions remain production code.
|
|
probe_production_fd_dereference() (
|
|
local root="${bootstrap_fixture_root}/fd-dereference" identity metadata
|
|
mkdir -p "$root/bundle/pre"
|
|
chmod 0700 "$root" "$root/bundle" "$root/bundle/pre"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
identity="$(stat --format='%d:%i' -- "$root")"
|
|
declare -F _kseb_run_privileged_shell >/dev/null || return 1
|
|
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
|
|
metadata="$(_kseb_privileged_pinned_dispatch "$root" "$identity" bundle/pre inspect)"
|
|
[[ "$metadata" == "$identity|$(id -u):$(id -g):700:directory" ]]
|
|
)
|
|
if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == fd ]]; then
|
|
assert_succeeds probe_production_fd_dereference
|
|
fi
|
|
|
|
write_round2_fake_k3s() {
|
|
local path="$1"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \
|
|
' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \
|
|
' if [[ -n "${KSEB_SWAP_ROOT:-}" ]]; then mv -- "$KSEB_SWAP_ROOT" "$KSEB_SWAP_ORIGINAL"; mv -- "$KSEB_SWAP_REPLACEMENT" "$KSEB_SWAP_ROOT"; fi' \
|
|
' if [[ -n "${KSEB_SWAP_PHASE:-}" ]]; then mv -- "$KSEB_SWAP_PHASE" "$KSEB_SWAP_PHASE_ORIGINAL"; mv -- "$KSEB_SWAP_PHASE_REPLACEMENT" "$KSEB_SWAP_PHASE"; fi' \
|
|
' : >"${destination:?}/fixture-snapshot"; exit 0' \
|
|
'fi' \
|
|
'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then printf "%s\\n" "{\"items\":[]}"; exit 0; fi' \
|
|
'exit 1' >"$path"
|
|
chmod 0700 "$path"
|
|
}
|
|
|
|
# This fixture renames and replaces the public root after the production
|
|
# transaction has opened it. Snapshot, common copy, manifest, sizing and the
|
|
# post marker must all stay on the original FD-derived tree.
|
|
probe_complete_transaction_actual_swap() (
|
|
local parent="${bootstrap_fixture_root}/round2-swap" root original replacement
|
|
local data config fake_k3s marker evidence output expected_metadata
|
|
parent="$(mktemp -d "${bootstrap_fixture_root}/round2-swap.XXXXXX")"
|
|
root="$parent/recovery"; original="$parent/original"; replacement="$parent/replacement"
|
|
data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s"
|
|
marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env"
|
|
mkdir -p "$root" "$replacement" "$data/server/cred" "$config"
|
|
chmod 0700 "$root" "$replacement"
|
|
printf 'token\n' >"$data/server/token"
|
|
printf '{}\n' >"$data/server/cred/encryption-config.json"
|
|
printf 'config\n' >"$config/config.yaml"
|
|
: >"$evidence"
|
|
write_round2_fake_k3s "$fake_k3s"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
|
|
declare -F _kseb_pinned_phase_transaction >/dev/null || return 1
|
|
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
|
|
_kseb_transaction_data_dir() { printf '%s\n' "$data"; }
|
|
_kseb_transaction_config_dir() { printf '%s\n' "$config"; }
|
|
_kseb_transaction_systemd_paths() { :; }
|
|
_kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; }
|
|
_kseb_transaction_post_marker() { printf '%s\n' "$marker"; }
|
|
_kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; }
|
|
export KSEB_SWAP_ROOT="$root" KSEB_SWAP_ORIGINAL="$original" KSEB_SWAP_REPLACEMENT="$replacement"
|
|
output="$(_kseb_pinned_phase_transaction "$root" bundle/post embedded-etcd \
|
|
01234567-89ab-4cde-8fab-0123456789ab post 20260801T000000Z 7 v1.36.2+k3s1)"
|
|
expected_metadata=$'schema=platform-k3s-bundle-v1\nbundle_id=01234567-89ab-4cde-8fab-0123456789ab\nphase=post\nk3s_version=v1.36.2+k3s1\ndatastore=embedded-etcd\ncreated_at_utc=2026-08-01T00:00:00Z\nsecret_count=7'
|
|
[[ "$output" =~ ^[0-9]+$ && -f "$original/bundle/post/fixture-snapshot" &&
|
|
-f "$original/bundle/post/server-token" && -f "$original/bundle/post/verification.manifest" &&
|
|
-f "$marker" && "$(<"$marker")" == "$expected_metadata" &&
|
|
"$(<"$original/bundle/post/bundle.env")" == "$expected_metadata" &&
|
|
! -e "$root/bundle" ]]
|
|
)
|
|
if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == transaction ]]; then
|
|
assert_succeeds probe_complete_transaction_actual_swap
|
|
fi
|
|
|
|
# The manifest lifecycle must open its phase directory relative to the pinned
|
|
# root FD and validate the dereferenced directory target before hashing.
|
|
probe_manifest_from_pinned_handle() (
|
|
local parent root data config fake_k3s phase original_phase replacement_phase output
|
|
local rc=0
|
|
parent="$(mktemp -d "${bootstrap_fixture_root}/round2-manifest.XXXXXX")"
|
|
root="$parent/recovery"; data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s"
|
|
phase="$root/bundle/pre"; original_phase="$root/bundle/original-pre"
|
|
replacement_phase="$parent/replacement-pre"
|
|
mkdir -p "$root" "$data/server/cred" "$config" "$replacement_phase"
|
|
chmod 0700 "$root"
|
|
printf 'token\n' >"$data/server/token"
|
|
printf '{}\n' >"$data/server/cred/encryption-config.json"
|
|
printf 'config\n' >"$config/config.yaml"
|
|
write_round2_fake_k3s "$fake_k3s"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
|
|
declare -F _kseb_pinned_phase_transaction >/dev/null || return 1
|
|
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
|
|
_kseb_transaction_data_dir() { printf '%s\n' "$data"; }
|
|
_kseb_transaction_config_dir() { printf '%s\n' "$config"; }
|
|
_kseb_transaction_systemd_paths() { :; }
|
|
_kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; }
|
|
export KSEB_SWAP_ROOT='' KSEB_SWAP_PHASE="$phase" KSEB_SWAP_PHASE_ORIGINAL="$original_phase"
|
|
export KSEB_SWAP_PHASE_REPLACEMENT="$replacement_phase"
|
|
output="$(_kseb_pinned_phase_transaction "$root" bundle/pre embedded-etcd bundle pre 20260801T000000Z 7 v1.36.2+k3s1 2>&1)" || rc=$?
|
|
[[ "$rc" != 0 && "$output" != *"$parent"* && -d "$original_phase" &&
|
|
-z "$(find "$original_phase" -mindepth 1 -print -quit)" && -d "$phase" &&
|
|
! -e "$phase/verification.manifest" ]]
|
|
)
|
|
if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == manifest ]]; then
|
|
assert_succeeds probe_manifest_from_pinned_handle
|
|
fi
|
|
|
|
write_round2_fake_systemctl() {
|
|
local path="$1"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'printf "%s\\n" "$1" >>"${KSEB_SYSTEMCTL_LOG:?}"' \
|
|
'if [[ "$1" == start && "${KSEB_SIGNAL_DURING_START:-0}" == 1 ]]; then kill -s TERM "$PPID"; fi' \
|
|
'exit 0' >"$path"
|
|
chmod 0700 "$path"
|
|
}
|
|
|
|
write_workload_wait_fake_k3s() {
|
|
local path="$1"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'printf "%s|%s %s %s\n" "${KSEB_WORKLOAD_SCENARIO:?}" "${1:-}" "${2:-}" "${3:-}" >>"${KSEB_K3S_CALL_LOG:?}"' \
|
|
'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \
|
|
' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \
|
|
' : >"${destination:?}/fixture-snapshot"' \
|
|
' exit 0' \
|
|
'fi' \
|
|
'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then' \
|
|
' if [[ "${KSEB_WORKLOAD_SCENARIO:?}" == api_failure ]]; then exit 9; fi' \
|
|
' exit 0' \
|
|
'fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then' \
|
|
' count=$(<"${KSEB_WORKLOAD_COUNTER:?}")' \
|
|
' count=$((count + 1))' \
|
|
' printf "%s\n" "$count" >"$KSEB_WORKLOAD_COUNTER"' \
|
|
' printf "read\n" >>"${KSEB_WORKLOAD_LOG:?}"' \
|
|
' if [[ "$KSEB_WORKLOAD_SCENARIO" == transient && "$count" == 1 ]]; then' \
|
|
' ready=false' \
|
|
' elif [[ "$KSEB_WORKLOAD_SCENARIO" == persistent ]]; then' \
|
|
' ready=false' \
|
|
' else' \
|
|
' ready=true' \
|
|
' fi' \
|
|
' printf "%s\n" "{\"items\":[{\"metadata\":{\"name\":\"fixture\",\"namespace\":\"default\"},\"spec\":{\"containers\":[{\"name\":\"app\"}],\"initContainers\":[]},\"status\":{\"phase\":\"Running\",\"containerStatuses\":[{\"name\":\"app\",\"ready\":${ready}}],\"initContainerStatuses\":[]}}]}"' \
|
|
' exit 0' \
|
|
'fi' \
|
|
'exit 1' >"$path"
|
|
chmod 0700 "$path"
|
|
}
|
|
|
|
probe_pinned_workload_recovery() (
|
|
local scenario="$1" parent root data config fake_k3s fake_systemctl
|
|
local systemctl_log workload_log workload_counter k3s_call_log marker evidence stderr_log output='' rc=0
|
|
parent="$(mktemp -d "${bootstrap_fixture_root}/pinned-workload-${scenario}.XXXXXX")"
|
|
root="$parent/recovery"; data="$parent/data"; config="$parent/config"
|
|
fake_k3s="$parent/k3s"; fake_systemctl="$parent/systemctl"
|
|
systemctl_log="$parent/systemctl.log"; workload_log="$parent/workload.log"
|
|
k3s_call_log="$parent/k3s-call.log"
|
|
workload_counter="$parent/workload.counter"
|
|
marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env"
|
|
stderr_log="$parent/transaction.stderr"
|
|
mkdir -p "$root" "$data/server/db" "$config"
|
|
chmod 0700 "$root"
|
|
printf 'state\n' >"$data/server/db/state.db"
|
|
printf 'token\n' >"$data/server/token"
|
|
printf 'config\n' >"$config/config.yaml"
|
|
: >"$systemctl_log"; : >"$workload_log"; : >"$k3s_call_log"; printf '0\n' >"$workload_counter"
|
|
write_workload_wait_fake_k3s "$fake_k3s"
|
|
write_round2_fake_systemctl "$fake_systemctl"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
|
|
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
|
|
_kseb_transaction_data_dir() { printf '%s\n' "$data"; }
|
|
_kseb_transaction_config_dir() { printf '%s\n' "$config"; }
|
|
_kseb_transaction_systemd_paths() { :; }
|
|
_kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; }
|
|
_kseb_transaction_systemctl_binary() { printf '%s\n' "$fake_systemctl"; }
|
|
_kseb_transaction_post_marker() { printf '%s\n' "$marker"; }
|
|
_kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; }
|
|
export KSEB_SYSTEMCTL_LOG="$systemctl_log" KSEB_SIGNAL_DURING_START=0
|
|
export KSEB_WORKLOAD_SCENARIO="$scenario" KSEB_WORKLOAD_COUNTER="$workload_counter"
|
|
export KSEB_WORKLOAD_LOG="$workload_log" KSEB_K3S_CALL_LOG="$k3s_call_log"
|
|
|
|
output="$(_kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" \
|
|
bundle/pre transaction "$data" "$config" "$fake_k3s" "$fake_systemctl" \
|
|
"$marker" "$evidence" '' sqlite 01234567-89ab-4cde-8fab-0123456789ab \
|
|
pre 20260801T000000Z 7 v1.36.2+k3s1 pre 2>"$stderr_log")" || rc=$?
|
|
|
|
case "$scenario" in
|
|
transient)
|
|
if [[ "$rc" != 0 || ! "$output" =~ ^[0-9]+$ ||
|
|
"$(command cat -- "$systemctl_log")" != $'stop\nstart' ||
|
|
"$(command cat -- "$workload_log")" != $'read\nread' ||
|
|
! -f "$root/bundle/pre/datastore/state.db" ||
|
|
! -f "$root/bundle/pre/verification.manifest" ]]; then
|
|
printf 'PINNED_TRANSIENT_DIAG rc=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s manifest=%s stderr_lines=%s\n' \
|
|
"$rc" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \
|
|
"$(wc -l <"$workload_log" | tr -d '[:space:]')" \
|
|
"$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \
|
|
"$([[ -f "$root/bundle/pre/verification.manifest" ]] && printf yes || printf no)" \
|
|
"$(wc -l <"$stderr_log" | tr -d '[:space:]')" >&2
|
|
command cat -- "$k3s_call_log" >&2
|
|
return 1
|
|
fi
|
|
;;
|
|
persistent)
|
|
if [[ "$rc" != 75 || -n "$output" ||
|
|
"$(command cat -- "$systemctl_log")" != $'stop\nstart' ||
|
|
"$(wc -l <"$workload_log" | tr -d '[:space:]')" -lt 2 ||
|
|
! -d "$root/bundle/pre" ||
|
|
-n "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]; then
|
|
printf 'PINNED_PERSISTENT_DIAG rc=%s output_bytes=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s phase_exists=%s phase_entries=%s\n' \
|
|
"$rc" "${#output}" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \
|
|
"$(wc -l <"$workload_log" | tr -d '[:space:]')" \
|
|
"$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \
|
|
"$([[ -d "$root/bundle/pre" ]] && printf yes || printf no)" \
|
|
"$(find "$root/bundle/pre" -mindepth 1 -printf . 2>/dev/null | wc -c | tr -d '[:space:]')" >&2
|
|
command cat -- "$k3s_call_log" >&2
|
|
return 1
|
|
fi
|
|
;;
|
|
api_failure)
|
|
if [[ "$rc" != 75 || -n "$output" ||
|
|
"$(command cat -- "$systemctl_log")" != $'stop\nstart' ||
|
|
-s "$workload_log" || ! -d "$root/bundle/pre" ||
|
|
-n "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]; then
|
|
printf 'PINNED_API_FAILURE_DIAG rc=%s output_bytes=%s systemctl_lines=%s workload_lines=%s k3s_calls=%s phase_exists=%s phase_entries=%s\n' \
|
|
"$rc" "${#output}" "$(wc -l <"$systemctl_log" | tr -d '[:space:]')" \
|
|
"$(wc -l <"$workload_log" | tr -d '[:space:]')" \
|
|
"$(wc -l <"$k3s_call_log" | tr -d '[:space:]')" \
|
|
"$([[ -d "$root/bundle/pre" ]] && printf yes || printf no)" \
|
|
"$(find "$root/bundle/pre" -mindepth 1 -printf . 2>/dev/null | wc -c | tr -d '[:space:]')" >&2
|
|
command cat -- "$k3s_call_log" >&2
|
|
return 1
|
|
fi
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
)
|
|
assert_succeeds probe_pinned_workload_recovery transient
|
|
assert_succeeds probe_pinned_workload_recovery persistent
|
|
assert_succeeds probe_pinned_workload_recovery api_failure
|
|
|
|
if [[ "${KSEB_WORKLOAD_WAIT_FOCUS:-}" == pinned ]]; then
|
|
printf 'K3S WORKLOAD WAIT PINNED TEST PASS\n'
|
|
exit 0
|
|
fi
|
|
|
|
# Trigger failure after stop so EXIT cleanup performs the real production start
|
|
# dispatch. TERM arrives from inside that command; cleanup must not be cut off
|
|
# and start must be attempted exactly once.
|
|
probe_cleanup_window_signal() (
|
|
local parent data root systemctl_log fake_systemctl rc=0
|
|
parent="$(mktemp -d "${bootstrap_fixture_root}/round2-cleanup.XXXXXX")"
|
|
data="$parent/data"; root="$parent/recovery"; systemctl_log="$parent/systemctl.log"
|
|
fake_systemctl="$parent/systemctl"
|
|
mkdir -p "$data/server/db" "$root"
|
|
chmod 0700 "$root"
|
|
: >"$systemctl_log"
|
|
write_round2_fake_systemctl "$fake_systemctl"
|
|
# Deliberately omit server/token so the pinned transaction enters cleanup.
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
|
|
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
|
|
_kseb_transaction_data_dir() { printf '%s\n' "$data"; }
|
|
_kseb_transaction_config_dir() { printf '%s\n' "$parent/no-config"; }
|
|
_kseb_transaction_systemd_paths() { :; }
|
|
_kseb_transaction_systemctl_binary() { printf '%s\n' "$fake_systemctl"; }
|
|
export KSEB_SYSTEMCTL_LOG="$systemctl_log" KSEB_SIGNAL_DURING_START=1
|
|
_kseb_pinned_phase_transaction "$root" bundle/pre sqlite bundle pre 20260801T000000Z 7 v1.36.2+k3s1 >/dev/null 2>&1 || rc=$?
|
|
[[ "$rc" != 0 && "$(<"$systemctl_log")" == $'stop\nstart' &&
|
|
-d "$root/bundle/pre" && -z "$(find "$root/bundle/pre" -mindepth 1 -print -quit)" ]]
|
|
)
|
|
if [[ -z "${KSEB_ROUND2_FOCUS:-}" || "${KSEB_ROUND2_FOCUS:-}" == cleanup ]]; then
|
|
assert_succeeds probe_cleanup_window_signal
|
|
fi
|
|
|
|
write_round3_fake_k3s() {
|
|
local path="$1"
|
|
printf '%s\n' '#!/usr/bin/env bash' \
|
|
'set -Eeuo pipefail' \
|
|
'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == save ]]; then' \
|
|
' for argument in "$@"; do case "$argument" in --dir=*) destination=${argument#--dir=} ;; esac; done' \
|
|
' : >"${destination:?}/partial-before-swap"' \
|
|
' mv -- "${KSEB_R3_PHASE:?}" "${KSEB_R3_ORIGINAL:?}"' \
|
|
' mv -- "${KSEB_R3_REPLACEMENT:?}" "${KSEB_R3_PHASE:?}"' \
|
|
' case "${KSEB_R3_SCENARIO:?}" in' \
|
|
' snapshot_failure) exit 9 ;;' \
|
|
' manifest_failure) : >"${destination}/invalid manifest name" ;;' \
|
|
' commit_swap) : >"${destination}/valid-after-swap" ;;' \
|
|
' *) exit 91 ;;' \
|
|
' esac' \
|
|
' exit 0' \
|
|
'fi' \
|
|
'if [[ "${1:-}" == etcd-snapshot && "${2:-}" == list ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == --raw=/readyz ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == wait ]]; then exit 0; fi' \
|
|
'if [[ "${1:-}" == kubectl && "${2:-}" == get && "${3:-}" == pods ]]; then printf "%s\\n" "{\"items\":[]}"; exit 0; fi' \
|
|
'exit 1' >"$path"
|
|
chmod 0700 "$path"
|
|
}
|
|
|
|
# A production transaction must never clean or commit through the old phase
|
|
# pathname after that pathname has been replaced. Only privilege elevation
|
|
# and the k3s external effect are replaced; FD creation, manifest, cleanup,
|
|
# recovery, marker and commit decisions are the real fixed dispatcher.
|
|
probe_round3_phase_swap_identity() (
|
|
local scenario="$1" parent root data config fake_k3s phase original_phase replacement_phase
|
|
local marker evidence output rc=0
|
|
parent="$(mktemp -d "${bootstrap_fixture_root}/round3-${scenario}.XXXXXX")"
|
|
root="$parent/recovery"; data="$parent/data"; config="$parent/config"; fake_k3s="$parent/k3s"
|
|
phase="$root/bundle/post"; original_phase="$root/bundle/original-post"
|
|
replacement_phase="$parent/replacement-post"
|
|
marker="$parent/platform-post-bundle.env"; evidence="$parent/platform-restore-evidence.env"
|
|
mkdir -p "$root" "$data/server/cred" "$config" "$replacement_phase"
|
|
chmod 0700 "$root" "$replacement_phase"
|
|
printf 'token\n' >"$data/server/token"
|
|
printf '{}\n' >"$data/server/cred/encryption-config.json"
|
|
printf 'config\n' >"$config/config.yaml"
|
|
printf 'replacement-untouched\n' >"$replacement_phase/replacement-sentinel"
|
|
printf 'restore-untouched\n' >"$evidence"
|
|
write_round3_fake_k3s "$fake_k3s"
|
|
# shellcheck source=/dev/null
|
|
source "$BOOTSTRAP_PATH"
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
|
|
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
|
|
_kseb_transaction_data_dir() { printf '%s\n' "$data"; }
|
|
_kseb_transaction_config_dir() { printf '%s\n' "$config"; }
|
|
_kseb_transaction_systemd_paths() { :; }
|
|
_kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; }
|
|
_kseb_transaction_post_marker() { printf '%s\n' "$marker"; }
|
|
_kseb_transaction_restore_evidence() { printf '%s\n' "$evidence"; }
|
|
export KSEB_R3_SCENARIO="$scenario" KSEB_R3_PHASE="$phase" KSEB_R3_ORIGINAL="$original_phase"
|
|
export KSEB_R3_REPLACEMENT="$replacement_phase"
|
|
output="$(_kseb_pinned_phase_transaction "$root" bundle/post embedded-etcd bundle post 20260801T000000Z 7 v1.36.2+k3s1 2>&1)" || rc=$?
|
|
[[ "$rc" != 0 && "$output" != *"$parent"* && -d "$phase" &&
|
|
"$(<"$phase/replacement-sentinel")" == replacement-untouched &&
|
|
-d "$original_phase" && -z "$(find "$original_phase" -mindepth 1 -print -quit)" &&
|
|
! -e "$marker" && "$(<"$evidence")" == restore-untouched ]]
|
|
)
|
|
|
|
if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == failure ]]; then
|
|
assert_succeeds probe_round3_phase_swap_identity snapshot_failure
|
|
fi
|
|
if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == manifest ]]; then
|
|
assert_succeeds probe_round3_phase_swap_identity manifest_failure
|
|
fi
|
|
if [[ -z "${KSEB_ROUND3_FOCUS:-}" || "${KSEB_ROUND3_FOCUS:-}" == commit ]]; then
|
|
assert_succeeds probe_round3_phase_swap_identity commit_swap
|
|
fi
|
|
|
|
bash "${REPOSITORY_ROOT}/scripts/validate/test-k3s-secret-encryption-restore-evidence.sh"
|
|
|
|
printf 'K3S SECRET ENCRYPTION STATUS TEST PASS\n'
|