1625 lines
64 KiB
Bash
1625 lines
64 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Fail-stop bootstrap for enabling k3s Secret encryption. This file is
|
|
# source-safe so tests can replace host-operation functions without exposing a
|
|
# production environment switch for fake execution.
|
|
set -Eeuo pipefail
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
|
|
readonly KSEB_SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
|
else
|
|
readonly KSEB_SCRIPT_DIR='.'
|
|
fi
|
|
readonly KSEB_ROOT="$(cd -- "${KSEB_SCRIPT_DIR}/../.." && pwd -P)"
|
|
readonly KSEB_LIBRARY="${KSEB_ROOT}/scripts/lib/k3s-secret-encryption.sh"
|
|
readonly KSEB_LOCAL_RECOVERY_LIBRARY="${KSEB_ROOT}/scripts/lib/k3s-local-recovery.sh"
|
|
readonly KSEB_VALIDATOR="${KSEB_ROOT}/scripts/validate/k3s-secret-encryption.sh"
|
|
readonly KSEB_LOCAL_RECOVERY_VALIDATOR="${KSEB_ROOT}/scripts/validate/k3s-local-recovery.sh"
|
|
readonly KSEB_LOCAL_RECOVERY_CONTRACT="${KSEB_ROOT}/infrastructure/security/k3s/local-recovery.env"
|
|
readonly KSEB_DROPIN_SOURCE="${KSEB_ROOT}/infrastructure/security/k3s/40-secrets-encryption.yaml"
|
|
readonly KSEB_DATA_DIR='/var/lib/rancher/k3s'
|
|
readonly KSEB_CONFIG_DIR='/etc/rancher/k3s'
|
|
readonly KSEB_POST_MARKER='/etc/rancher/k3s/platform-post-bundle.env'
|
|
readonly KSEB_RESTORE_EVIDENCE='/etc/rancher/k3s/platform-restore-evidence.env'
|
|
readonly KSEB_RECOVERY_DOCUMENT='bootstrap/manual/k3s-secret-encryption.md'
|
|
readonly KSEB_MINIMUM_FREE_BYTES=10737418240
|
|
readonly KSEB_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin'
|
|
readonly KSEB_API_RECOVERY_BUDGET=430
|
|
readonly KSEB_NODE_RECOVERY_BUDGET=130
|
|
readonly KSEB_WORKLOAD_RECOVERY_BUDGET=30
|
|
readonly KSEB_ESTIMATE_METADATA_BYTES=16777216
|
|
readonly KSEB_SIGNED_MAX=9223372036854775807
|
|
KSEB_BACKUP_ROOT_IDENTITY=''
|
|
KSEB_LAST_SAFE_STATE='unknown'
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$KSEB_LIBRARY"
|
|
# shellcheck source=/dev/null
|
|
source "$KSEB_LOCAL_RECOVERY_LIBRARY"
|
|
|
|
_kseb_fail() {
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
return 1
|
|
}
|
|
|
|
_kseb_terminal_failure() {
|
|
local state="$1"
|
|
case "$state" in
|
|
unknown|initial_unsafe|datastore_unsafe|backup_root_unsafe|disabled_state_drift|enabled_start_drift|enabled_finished_drift|\
|
|
reencrypt_stage_start|reencrypt_stage_unexpected|reencrypt_active_timeout|hash_mismatch|\
|
|
expectation_failed|final_validation_failed|pre_backup_failed|post_backup_failed|\
|
|
service_recovery_failed|marker_install_failed|partial_enable_failed|partial_dropin_failed|\
|
|
partial_initial_restart_failed|partial_final_restart_failed) ;;
|
|
*) state=unknown ;;
|
|
esac
|
|
printf 'ERROR: current-state=%s recovery=%s\n' "$state" "$KSEB_RECOVERY_DOCUMENT" >&2
|
|
return 1
|
|
}
|
|
|
|
_kseb_run_quiet() {
|
|
"$@" >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_privileged_exec() {
|
|
/usr/bin/sudo "$@" >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_usage() {
|
|
cat <<'USAGE'
|
|
사용법:
|
|
bash scripts/bootstrap/apply-k3s-secret-encryption.sh
|
|
bash scripts/bootstrap/apply-k3s-secret-encryption.sh \
|
|
--execute --backup-root "$K3S_RECOVERY_ROOT" \
|
|
--recovery-policy encrypted-off-host
|
|
bash scripts/bootstrap/apply-k3s-secret-encryption.sh \
|
|
--execute --rotate-existing --backup-root "$K3S_RECOVERY_ROOT" \
|
|
--recovery-policy encrypted-off-host
|
|
bash scripts/bootstrap/apply-k3s-secret-encryption.sh \
|
|
--execute --backup-root /srv/recovery/k3s \
|
|
--recovery-policy local-separate-disk-luks
|
|
bash scripts/bootstrap/apply-k3s-secret-encryption.sh \
|
|
--estimate-recovery-bytes
|
|
|
|
인자 없이 실행하면 읽기 전용 상태와 필요한 운영자 확인만 출력합니다.
|
|
실행에는 명시적인 recovery policy와 해당 정책의 암호화 recovery mount 검증이 필요합니다.
|
|
USAGE
|
|
}
|
|
|
|
_kseb_safe_remove_handoff() {
|
|
local directory="$1"
|
|
case "$directory" in
|
|
/tmp/platform-k3s-encryption.[A-Za-z0-9]*) rm -rf -- "$directory" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
_kseb_inventory_value() {
|
|
local file="$1" key="$2" value count
|
|
count="$(awk -F= -v key="$key" '$1 == key { count++ } END { print count + 0 }' "$file")" || return 1
|
|
[[ "$count" == 1 ]] || return 1
|
|
value="$(awk -F= -v key="$key" '$1 == key { sub(/^[^=]*=/, ""); print }' "$file")" || return 1
|
|
[[ "$value" =~ ^[A-Za-z0-9_.+-]+$ ]] || return 1
|
|
printf '%s\n' "$value"
|
|
}
|
|
|
|
_kseb_read_status_json() {
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 9s \
|
|
/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s \
|
|
secrets-encrypt status --output json 2>/dev/null
|
|
}
|
|
|
|
_kseb_load_inventory() {
|
|
local handoff inventory encryption rotation datastore integrity server_hashes
|
|
local status_class provider=none owner=none status_json
|
|
umask 077
|
|
handoff="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" || return 1
|
|
chmod 0700 "$handoff" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
if ! bash "$KSEB_VALIDATOR" --verified-output-dir "$handoff" >/dev/null; then
|
|
status_json="$(_kseb_read_status_json)" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
status_class="$(classify_encryption_status "$status_json")"
|
|
_kseb_safe_remove_handoff "$handoff"
|
|
if [[ "$status_class" == hash_mismatch ]]; then
|
|
printf 'hash_mismatch|unsafe|ambiguous|invalid|ambiguous|mismatch|mismatch\n'
|
|
return 0
|
|
fi
|
|
return 1
|
|
fi
|
|
inventory="$handoff/inventory.env"
|
|
encryption="$(_kseb_inventory_value "$inventory" encryption)" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
rotation="$(_kseb_inventory_value "$inventory" rotation)" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
datastore="$(_kseb_inventory_value "$inventory" datastore)" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
integrity="$(_kseb_inventory_value "$inventory" integrity)" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
server_hashes="$(_kseb_inventory_value "$inventory" server_hashes)" || { _kseb_safe_remove_handoff "$handoff"; return 1; }
|
|
_kseb_safe_remove_handoff "$handoff" || return 1
|
|
|
|
case "${encryption}/${rotation}" in
|
|
Disabled/none) status_class=disabled_no_config ;;
|
|
Disabled/start) status_class=transition_start ;;
|
|
Enabled/start|Enabled/reencrypt_finished) status_class=enabled_stable ;;
|
|
*) status_class=invalid ;;
|
|
esac
|
|
if [[ "$status_class" == enabled_stable ]]; then
|
|
status_json="$(_kseb_read_status_json)" || return 1
|
|
provider="$(classify_encryption_provider "$status_json")"
|
|
owner="$(detect_effective_encryption_config_owner)" || owner=ambiguous
|
|
fi
|
|
printf '%s|%s|%s|%s|%s|%s|%s\n' \
|
|
"$status_class" "$rotation" "$datastore" "$provider" "$owner" "$integrity" "$server_hashes"
|
|
}
|
|
|
|
_kseb_require_exact_enabled_start() {
|
|
local expected_backend="$1" inventory status rotation backend provider owner integrity server_hashes
|
|
inventory="$(_kseb_load_inventory)" || return 1
|
|
IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory"
|
|
[[ "$status" == enabled_stable && "$rotation" == start && "$backend" == "$expected_backend" &&
|
|
"$provider" == aescbc && "$owner" == aescbc/* && "$owner" != aescbc/ambiguous &&
|
|
"$integrity" == match && "$server_hashes" == match ]]
|
|
}
|
|
|
|
_kseb_require_exact_disabled() {
|
|
local expected_backend="$1" inventory status rotation backend provider owner integrity server_hashes
|
|
inventory="$(_kseb_load_inventory)" || return 1
|
|
IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory"
|
|
[[ "$status" == disabled_no_config && "$rotation" == none &&
|
|
"$backend" == "$expected_backend" && ( "$backend" == sqlite || "$backend" == embedded-etcd ) &&
|
|
"$provider" == none && "$owner" == none ]]
|
|
}
|
|
|
|
_kseb_require_exact_finished() {
|
|
local expected_backend="$1" inventory status rotation backend provider owner integrity server_hashes
|
|
inventory="$(_kseb_load_inventory)" || return 1
|
|
IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory"
|
|
[[ "$status" == enabled_stable && "$rotation" == reencrypt_finished &&
|
|
"$backend" == "$expected_backend" && "$provider" == aescbc &&
|
|
"$owner" == aescbc/* && "$owner" != aescbc/ambiguous &&
|
|
"$integrity" == match && "$server_hashes" == match ]]
|
|
}
|
|
|
|
_kseb_dry_run() {
|
|
local inventory="$1" status rotation backend ignored
|
|
IFS='|' read -r status rotation backend ignored <<<"$inventory"
|
|
printf '현재 상태: %s (rotation=%s, datastore=%s)\n' "$status" "$rotation" "$backend"
|
|
printf '실행 시 --recovery-policy와 현재 context의 정책별 확인이 필요합니다.\n'
|
|
printf 'DRY RUN PASS: 변경하지 않았습니다.\n'
|
|
}
|
|
|
|
_kseb_current_context() {
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 9s \
|
|
/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s \
|
|
kubectl config current-context 2>/dev/null
|
|
}
|
|
|
|
_kseb_prepare_execute_context() {
|
|
local context
|
|
[[ -t 0 ]] || { _kseb_fail '--execute는 대화형 터미널이 필요합니다'; return 1; }
|
|
/usr/bin/sudo -v || return 1
|
|
context="$(_kseb_current_context)" || return 1
|
|
[[ -n "$context" && "$context" != *$'\n'* ]] || return 1
|
|
printf '%s\n' "$context"
|
|
}
|
|
|
|
_kseb_confirm_apply() {
|
|
local context="$1" answer
|
|
[[ -n "$context" && "$context" != *$'\n'* ]] || return 1
|
|
printf 'Type APPLY %s to continue: ' "$context"
|
|
read -r answer
|
|
[[ "$answer" == "APPLY $context" ]] || { _kseb_fail '취소했습니다'; return 1; }
|
|
}
|
|
|
|
_kseb_local_recovery_contract() {
|
|
printf '%s\n' "$KSEB_LOCAL_RECOVERY_CONTRACT"
|
|
}
|
|
|
|
_kseb_local_recovery_root() {
|
|
local contract root
|
|
[[ "$-" != *x* ]] || return 1
|
|
contract="$(_kseb_local_recovery_contract)" || return 1
|
|
[[ -n "$contract" && "$contract" != *$'\n'* ]] || return 1
|
|
root="$(_k3slr_config_value "$contract" K3SLR_INNER_MOUNT)" || return 1
|
|
[[ -n "$root" && "$root" != *$'\n'* ]] || return 1
|
|
printf '%s\n' "$root"
|
|
}
|
|
|
|
_kseb_local_validator_command() {
|
|
"$@"
|
|
}
|
|
|
|
_kseb_safe_remove_local_validator_handoff() {
|
|
local directory="$1" path cleanup_rc=0
|
|
case "$directory" in
|
|
/tmp/platform-k3s-local-validator.[A-Za-z0-9]*)
|
|
[[ -d "$directory" && ! -L "$directory" ]] || return 1
|
|
for path in "$directory/stdout" "$directory/stderr"; do
|
|
if [[ -e "$path" || -L "$path" ]]; then
|
|
[[ -f "$path" && ! -L "$path" ]] || return 1
|
|
/usr/bin/rm -- "$path" >/dev/null 2>&1 || cleanup_rc=1
|
|
fi
|
|
done
|
|
/usr/bin/rmdir -- "$directory" >/dev/null 2>&1 || cleanup_rc=1
|
|
return "$cleanup_rc"
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
_kseb_local_validator_result_is_exact() {
|
|
local command_rc="$1" stdout_file="$2" stderr_file="$3" output='' expected stdout_size stderr_size
|
|
[[ "$command_rc" =~ ^(0|[1-9][0-9]{0,2})$ ]] || return 1
|
|
(( command_rc == 0 )) || return 1
|
|
[[ -f "$stdout_file" && ! -L "$stdout_file" && -f "$stderr_file" && ! -L "$stderr_file" ]] || return 1
|
|
stdout_size="$(/usr/bin/stat --format='%s' -- "$stdout_file")" || return 1
|
|
stderr_size="$(/usr/bin/stat --format='%s' -- "$stderr_file")" || return 1
|
|
[[ "$stdout_size" == 86 && "$stderr_size" == 0 ]] || return 1
|
|
if IFS= read -r -d '' output <"$stdout_file"; then
|
|
return 1
|
|
fi
|
|
printf -v expected '%s\n' \
|
|
'Recovery device: match' \
|
|
'Recovery state: open' \
|
|
'Lineage: match' \
|
|
'Latest bundle: not_checked'
|
|
[[ "$output" == "$expected" ]]
|
|
}
|
|
|
|
_kseb_run_local_recovery_validator() (
|
|
local handoff='' stdout_file stderr_file command_rc=0 result_rc=1 cleanup_rc=0
|
|
[[ "$-" != *x* ]] || return 1
|
|
umask 077
|
|
handoff="$(/usr/bin/mktemp -d /tmp/platform-k3s-local-validator.XXXXXX)" || return 1
|
|
/usr/bin/chmod 0700 -- "$handoff" || {
|
|
_kseb_safe_remove_local_validator_handoff "$handoff" >/dev/null 2>&1 || true
|
|
return 1
|
|
}
|
|
trap '_kseb_safe_remove_local_validator_handoff "$handoff" >/dev/null 2>&1 || true' EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
stdout_file="$handoff/stdout"
|
|
stderr_file="$handoff/stderr"
|
|
: >"$stdout_file"
|
|
: >"$stderr_file"
|
|
/usr/bin/chmod 0600 -- "$stdout_file" "$stderr_file" || return 1
|
|
ulimit -f 8 || return 1
|
|
set +e
|
|
_kseb_local_validator_command \
|
|
/usr/bin/env -i \
|
|
PATH="$KSEB_SECURE_PATH" \
|
|
LC_ALL=C \
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 60s \
|
|
/usr/bin/bash --noprofile --norc -- \
|
|
"$KSEB_LOCAL_RECOVERY_VALIDATOR" --expect-open \
|
|
>"$stdout_file" 2>"$stderr_file"
|
|
command_rc=$?
|
|
set -e
|
|
if _kseb_local_validator_result_is_exact "$command_rc" "$stdout_file" "$stderr_file"; then
|
|
result_rc=0
|
|
fi
|
|
_kseb_safe_remove_local_validator_handoff "$handoff" >/dev/null 2>&1 || cleanup_rc=1
|
|
if (( cleanup_rc == 0 )); then
|
|
trap - EXIT
|
|
else
|
|
result_rc=1
|
|
fi
|
|
trap - HUP INT TERM
|
|
return "$result_rc"
|
|
)
|
|
|
|
_kseb_validate_recovery_policy() {
|
|
local policy="$1" root="$2" expected_root confirmed_root
|
|
[[ -n "$root" ]] || return 1
|
|
case "$policy" in
|
|
encrypted-off-host) return 0 ;;
|
|
local-separate-disk-luks)
|
|
expected_root="$(_kseb_local_recovery_root)" || return 1
|
|
[[ -n "$expected_root" && "$expected_root" != *$'\n'* && "$root" == "$expected_root" ]] || return 1
|
|
_kseb_run_local_recovery_validator >/dev/null 2>&1 || return 1
|
|
confirmed_root="$(_kseb_local_recovery_root)" || return 1
|
|
[[ "$confirmed_root" == "$expected_root" ]] || return 1
|
|
_kseb_revalidate_backup_root "$root"
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
_kseb_confirm_recovery() {
|
|
local context="$1" policy="$2" answer check
|
|
[[ -n "$context" && "$context" != *$'\n'* ]] || return 1
|
|
printf 'Type RECOVERY %s after testing escrow decryption access now: ' "$context"
|
|
read -r answer
|
|
[[ "$answer" == "RECOVERY $context" ]] || { _kseb_fail 'recovery 접근 확인이 없습니다'; return 1; }
|
|
if [[ "$policy" == encrypted-off-host ]]; then
|
|
printf 'Type ENCRYPTED %s to attest the mount is encrypted off-host media: ' "$context"
|
|
elif [[ "$policy" == local-separate-disk-luks ]]; then
|
|
printf 'Type ENCRYPTED %s to attest the mounted local recovery volume is LUKS2 encrypted: ' "$context"
|
|
else
|
|
return 1
|
|
fi
|
|
read -r answer
|
|
[[ "$answer" == "ENCRYPTED $context" ]] || { _kseb_fail '암호화 recovery media 확인이 없습니다'; return 1; }
|
|
if [[ "$policy" == local-separate-disk-luks ]]; then
|
|
printf 'Type LOCAL_RISK_ACCEPTED %s to accept same-host local disk failure risk: ' "$context"
|
|
read -r answer
|
|
[[ "$answer" == "LOCAL_RISK_ACCEPTED $context" ]] || {
|
|
_kseb_fail 'local recovery disk 위험 승인이 없습니다'
|
|
return 1
|
|
}
|
|
fi
|
|
check="$(_kseb_current_context)" || return 1
|
|
[[ "$check" == "$context" ]] || { _kseb_fail 'context가 바뀌었습니다'; return 1; }
|
|
}
|
|
|
|
_kseb_validate_backup_root() {
|
|
local root="$1" kind physical metadata backup_source data_source free_bytes
|
|
[[ "$root" =~ ^/[A-Za-z0-9._/-]+$ && "$root" != *'//'*
|
|
&& "$root" != */../* && "$root" != */./* ]] || return 1
|
|
kind="$(_kseb_privileged_root_kind "$root")" || return 1
|
|
[[ "$kind" == directory ]] || return 1
|
|
_kseb_privileged_backup_root_chain_safe "$root" || return 1
|
|
physical="$(_kseb_privileged_realpath "$root")" || return 1
|
|
[[ "$physical" == "$root" ]] || return 1
|
|
metadata="$(_kseb_privileged_root_metadata "$root")" || return 1
|
|
[[ "$metadata" == '0:0:700:directory' ]] || return 1
|
|
backup_source="$(_kseb_privileged_mount_source "$root")" || return 1
|
|
data_source="$(_kseb_privileged_mount_source "$KSEB_DATA_DIR")" || return 1
|
|
[[ -n "$backup_source" && -n "$data_source" && "$backup_source" != "$data_source" ]] || return 1
|
|
free_bytes="$(_kseb_privileged_free_bytes "$root")" || return 1
|
|
[[ "$free_bytes" =~ ^[0-9]+$ ]] && (( free_bytes >= KSEB_MINIMUM_FREE_BYTES )) || return 1
|
|
KSEB_BACKUP_ROOT_IDENTITY="$(_kseb_privileged_root_identity "$root")" || return 1
|
|
[[ "$KSEB_BACKUP_ROOT_IDENTITY" =~ ^[0-9]+:[0-9]+$ ]]
|
|
}
|
|
|
|
_kseb_privileged_root_kind() {
|
|
local path="$1"
|
|
if /usr/bin/sudo /usr/bin/test -L "$path"; then printf 'symlink\n'
|
|
elif /usr/bin/sudo /usr/bin/test -d "$path"; then printf 'directory\n'
|
|
elif /usr/bin/sudo /usr/bin/test -e "$path"; then printf 'other\n'
|
|
else printf 'absent\n'; fi
|
|
}
|
|
|
|
_kseb_privileged_realpath() {
|
|
/usr/bin/sudo /usr/bin/realpath --canonicalize-existing -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_privileged_root_metadata() {
|
|
/usr/bin/sudo /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_privileged_mount_source() {
|
|
/usr/bin/sudo /usr/bin/findmnt --noheadings --output SOURCE --target "$1" 2>/dev/null |
|
|
awk 'NF == 1 { print }'
|
|
}
|
|
|
|
_kseb_privileged_free_bytes() {
|
|
/usr/bin/sudo /usr/bin/df --block-size=1 --output=avail "$1" 2>/dev/null |
|
|
awk 'NR == 2 && $1 ~ /^[0-9]+$/ { print $1 }'
|
|
}
|
|
|
|
_kseb_privileged_root_identity() {
|
|
/usr/bin/sudo /usr/bin/stat --format='%d:%i' -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_privileged_path_lstat() {
|
|
/usr/bin/sudo /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_privileged_backup_root_chain_safe() {
|
|
_kseb_backup_root_chain_safe "$1" _kseb_privileged_path_lstat
|
|
}
|
|
|
|
_kseb_path_lstat() {
|
|
/usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_backup_root_chain_safe() {
|
|
local root="$1" reader="${2:-_kseb_path_lstat}" current='/' component metadata uid gid mode file_type
|
|
local -a components=()
|
|
[[ "$root" =~ ^/[A-Za-z0-9._/-]+$ && "$root" != *'//'*
|
|
&& "$root" != */../* && "$root" != */./* ]] || return 1
|
|
IFS='/' read -r -a components <<<"${root#/}"
|
|
for current in /; do
|
|
metadata="$("$reader" "$current")" || return 1
|
|
IFS=: read -r uid gid mode file_type <<<"$metadata"
|
|
[[ "$uid" == 0 && "$gid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == directory ]] || return 1
|
|
(( (8#$mode & 8#022) == 0 )) || return 1
|
|
done
|
|
current=''
|
|
for component in "${components[@]}"; do
|
|
[[ -n "$component" ]] || return 1
|
|
current+="/$component"
|
|
metadata="$("$reader" "$current")" || return 1
|
|
IFS=: read -r uid gid mode file_type <<<"$metadata"
|
|
[[ "$uid" == 0 && "$gid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == directory ]] || return 1
|
|
(( (8#$mode & 8#022) == 0 )) || return 1
|
|
done
|
|
}
|
|
|
|
_kseb_backup_root_identity() {
|
|
/usr/bin/stat --format='%d:%i' -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_backup_root_identity_matches() {
|
|
[[ "$1" =~ ^[0-9]+:[0-9]+$ && "$1" == "$2" ]]
|
|
}
|
|
|
|
_kseb_revalidate_backup_root() {
|
|
local root="$1" identity
|
|
[[ -n "$KSEB_BACKUP_ROOT_IDENTITY" ]] || return 1
|
|
_kseb_privileged_backup_root_chain_safe "$root" || return 1
|
|
identity="$(_kseb_privileged_root_identity "$root")" || return 1
|
|
_kseb_backup_root_identity_matches "$KSEB_BACKUP_ROOT_IDENTITY" "$identity"
|
|
}
|
|
|
|
_kseb_privileged_relative_components_safe() {
|
|
local root="$1" relative="$2" current="$root" component
|
|
[[ "$relative" =~ ^[A-Za-z0-9._/-]+$ && "$relative" != /* &&
|
|
"$relative" != *'..'* && "$relative" != *'//'* ]] || return 1
|
|
IFS='/' read -r -a _kseb_relative_parts <<<"$relative"
|
|
for component in "${_kseb_relative_parts[@]}"; do
|
|
[[ -n "$component" ]] || return 1
|
|
current+="/$component"
|
|
if /usr/bin/sudo /usr/bin/test -L "$current"; then return 1; fi
|
|
/usr/bin/sudo /usr/bin/test -e "$current" || break
|
|
done
|
|
}
|
|
|
|
_kseb_run_privileged_shell() {
|
|
/usr/bin/sudo /usr/bin/bash "$@"
|
|
}
|
|
|
|
_kseb_transaction_data_dir() { printf '%s\n' "$KSEB_DATA_DIR"; }
|
|
_kseb_transaction_config_dir() { printf '%s\n' "$KSEB_CONFIG_DIR"; }
|
|
_kseb_transaction_k3s_binary() { printf '/usr/local/bin/k3s\n'; }
|
|
_kseb_transaction_systemctl_binary() { printf '/usr/bin/systemctl\n'; }
|
|
_kseb_transaction_post_marker() { printf '%s\n' "$KSEB_POST_MARKER"; }
|
|
_kseb_transaction_restore_evidence() { printf '%s\n' "$KSEB_RESTORE_EVIDENCE"; }
|
|
_kseb_transaction_systemd_paths() {
|
|
printf '%s\n' \
|
|
/etc/systemd/system/k3s.service \
|
|
/etc/systemd/system/k3s.service.env \
|
|
/etc/systemd/system/k3s.service.d \
|
|
/usr/lib/systemd/system/k3s.service \
|
|
/lib/systemd/system/k3s.service
|
|
}
|
|
|
|
_kseb_normalize_signed_uint() {
|
|
local value="$1"
|
|
[[ "$value" =~ ^[0-9]+$ ]] || return 1
|
|
while [[ "$value" == 0* && "$value" != 0 ]]; do value="${value#0}"; done
|
|
if (( ${#value} > ${#KSEB_SIGNED_MAX} )); then return 1; fi
|
|
if (( ${#value} == ${#KSEB_SIGNED_MAX} )) && [[ "$value" > "$KSEB_SIGNED_MAX" ]]; then return 1; fi
|
|
printf '%s\n' "$value"
|
|
}
|
|
|
|
_kseb_checked_estimate_add() {
|
|
local left right sum
|
|
left="$(_kseb_normalize_signed_uint "$1")" || return 1
|
|
right="$(_kseb_normalize_signed_uint "$2")" || return 1
|
|
(( right <= KSEB_SIGNED_MAX - left )) || return 1
|
|
sum=$((left + right))
|
|
printf '%s\n' "$sum"
|
|
}
|
|
|
|
_kseb_required_recovery_bytes() {
|
|
local phase="$1" phase_bytes quarter remainder uplift padded required
|
|
case "$phase" in pre|post) ;; *) return 1 ;; esac
|
|
phase_bytes="$(_kseb_normalize_signed_uint "$2")" || return 1
|
|
quarter=$((phase_bytes / 4))
|
|
remainder=$((phase_bytes % 4))
|
|
uplift="$quarter"
|
|
if (( remainder != 0 )); then
|
|
uplift="$(_kseb_checked_estimate_add "$uplift" 1)" || return 1
|
|
fi
|
|
padded="$(_kseb_checked_estimate_add "$phase_bytes" "$uplift")" || return 1
|
|
if [[ "$phase" == pre ]]; then
|
|
required="$(_kseb_checked_estimate_add "$padded" "$padded")" || return 1
|
|
else
|
|
required="$padded"
|
|
fi
|
|
_kseb_checked_estimate_add "$required" "$KSEB_MINIMUM_FREE_BYTES"
|
|
}
|
|
|
|
_kseb_require_phase_capacity() {
|
|
local phase="$1" backend="$2" root="$3" phase_bytes required free_bytes
|
|
phase_bytes="$(_kseb_estimate_phase_bytes "$backend")" || return 1
|
|
phase_bytes="$(_kseb_normalize_signed_uint "$phase_bytes")" || return 1
|
|
required="$(_kseb_required_recovery_bytes "$phase" "$phase_bytes")" || return 1
|
|
free_bytes="$(_kseb_privileged_free_bytes "$root")" || return 1
|
|
free_bytes="$(_kseb_normalize_signed_uint "$free_bytes")" || return 1
|
|
(( free_bytes >= required ))
|
|
}
|
|
|
|
_kseb_estimator_source_allowed() {
|
|
local backend="$1" path="$2"
|
|
case "$backend:$path" in
|
|
sqlite:/var/lib/rancher/k3s/server/db|\
|
|
embedded-etcd:/var/lib/rancher/k3s/server/db/etcd|\
|
|
sqlite:/var/lib/rancher/k3s/server/token|\
|
|
embedded-etcd:/var/lib/rancher/k3s/server/token|\
|
|
sqlite:/etc/rancher/k3s|\
|
|
embedded-etcd:/etc/rancher/k3s|\
|
|
sqlite:/etc/systemd/system/k3s.service|\
|
|
embedded-etcd:/etc/systemd/system/k3s.service|\
|
|
sqlite:/etc/systemd/system/k3s.service.env|\
|
|
embedded-etcd:/etc/systemd/system/k3s.service.env|\
|
|
sqlite:/etc/systemd/system/k3s.service.d|\
|
|
embedded-etcd:/etc/systemd/system/k3s.service.d|\
|
|
sqlite:/usr/lib/systemd/system/k3s.service|\
|
|
embedded-etcd:/usr/lib/systemd/system/k3s.service|\
|
|
sqlite:/lib/systemd/system/k3s.service|\
|
|
embedded-etcd:/lib/systemd/system/k3s.service|\
|
|
sqlite:/var/lib/rancher/k3s/server/cred/encryption-config.json|\
|
|
embedded-etcd:/var/lib/rancher/k3s/server/cred/encryption-config.json) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
_kseb_privileged_estimator_source_kind() {
|
|
local path="$1"
|
|
if /usr/bin/sudo --non-interactive -- /usr/bin/test -L "$path" 2>/dev/null; then
|
|
printf 'symlink\n'
|
|
elif /usr/bin/sudo --non-interactive -- /usr/bin/test -e "$path" 2>/dev/null; then
|
|
printf 'present\n'
|
|
elif /usr/bin/sudo --non-interactive -- /usr/bin/test ! -e "$path" 2>/dev/null; then
|
|
printf 'absent\n'
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
_kseb_privileged_estimator_capture() {
|
|
/usr/bin/sudo --non-interactive -- "$@" 2>/dev/null
|
|
}
|
|
|
|
_kseb_privileged_estimator_lstat() {
|
|
_kseb_privileged_estimator_capture /usr/bin/stat \
|
|
--format='%d:%i|%f' -- "$1"
|
|
}
|
|
|
|
_kseb_privileged_estimator_du() {
|
|
/usr/bin/sudo --non-interactive -- /usr/bin/du \
|
|
--summarize --bytes -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_estimator_source_identity() {
|
|
local path="$1" current='' metadata identity mode_hex mode_type component
|
|
local -a components=()
|
|
[[ "$path" == /* && "$path" != *'//'* && "$path" != */../* && "$path" != */./* ]] || return 1
|
|
metadata="$(_kseb_privileged_estimator_lstat /)" || return 1
|
|
[[ "$metadata" != *$'\n'* ]] || return 1
|
|
identity="${metadata%%|*}"; mode_hex="${metadata#*|}"
|
|
[[ "$identity" =~ ^[0-9]+:[0-9]+$ && "$mode_hex" =~ ^[0-9a-fA-F]{1,8}$ ]] || return 1
|
|
mode_type=$((16#$mode_hex & 8#170000))
|
|
(( mode_type == 8#40000 )) || return 1
|
|
IFS='/' read -r -a components <<<"${path#/}"
|
|
for component in "${components[@]}"; do
|
|
[[ -n "$component" ]] || return 1
|
|
current+="/$component"
|
|
metadata="$(_kseb_privileged_estimator_lstat "$current")" || return 1
|
|
[[ "$metadata" != *$'\n'* ]] || return 1
|
|
identity="${metadata%%|*}"; mode_hex="${metadata#*|}"
|
|
[[ "$identity" =~ ^[0-9]+:[0-9]+$ && "$mode_hex" =~ ^[0-9a-fA-F]{1,8}$ ]] || return 1
|
|
mode_type=$((16#$mode_hex & 8#170000))
|
|
if [[ "$current" == "$path" ]]; then
|
|
(( mode_type == 8#40000 || mode_type == 8#100000 )) || return 1
|
|
else
|
|
(( mode_type == 8#40000 )) || return 1
|
|
fi
|
|
done
|
|
printf '%s\n' "$identity"
|
|
}
|
|
|
|
_kseb_estimator_source_probe() {
|
|
local path="$1" requirement="$2" kind identity
|
|
[[ "$requirement" == required || "$requirement" == optional ]] || return 1
|
|
kind="$(_kseb_privileged_estimator_source_kind "$path")" || return 1
|
|
case "$kind" in
|
|
absent) [[ "$requirement" == optional ]] || return 1; return 0 ;;
|
|
present) ;;
|
|
*) return 1 ;;
|
|
esac
|
|
identity="$(_kseb_estimator_source_identity "$path")" || return 1
|
|
[[ "$identity" =~ ^[0-9]+:[0-9]+$ ]] || return 1
|
|
printf '%s\n' "$identity"
|
|
}
|
|
|
|
_kseb_estimate_source_bytes() {
|
|
local backend="$1" path="$2" requirement="$3" expected_identity="${4:-}"
|
|
local before after raw bytes reported du_rc sentinel=$'\036'
|
|
_kseb_estimator_source_allowed "$backend" "$path" || return 1
|
|
before="$(_kseb_estimator_source_probe "$path" "$requirement")" || return 1
|
|
[[ -n "$before" ]] || { printf '\n'; return 0; }
|
|
[[ -z "$expected_identity" || "$before" == "$expected_identity" ]] || return 1
|
|
raw="$(
|
|
_kseb_privileged_estimator_du "$path"
|
|
du_rc=$?
|
|
printf '%s' "$sentinel"
|
|
exit "$du_rc"
|
|
)" || return 1
|
|
[[ "$raw" == *"$sentinel" ]] || return 1
|
|
raw="${raw%"$sentinel"}"
|
|
[[ "$raw" == *$'\n' ]] || return 1
|
|
raw="${raw%$'\n'}"
|
|
[[ "$raw" != *$'\n'* && "$raw" == *$'\t'* ]] || return 1
|
|
bytes="${raw%%$'\t'*}"; reported="${raw#*$'\t'}"
|
|
[[ "$reported" == "$path" ]] || return 1
|
|
bytes="$(_kseb_normalize_signed_uint "$bytes")" || return 1
|
|
after="$(_kseb_estimator_source_identity "$path")" || return 1
|
|
[[ "$after" == "$before" ]] || return 1
|
|
printf '%s\n' "$bytes"
|
|
}
|
|
|
|
_kseb_estimator_source_records() {
|
|
local backend="$1" path
|
|
case "$backend" in
|
|
sqlite) printf 'required|/var/lib/rancher/k3s/server/db\n' ;;
|
|
embedded-etcd) printf 'required|/var/lib/rancher/k3s/server/db/etcd\n' ;;
|
|
*) return 1 ;;
|
|
esac
|
|
printf 'required|/var/lib/rancher/k3s/server/token\n'
|
|
printf 'optional|/etc/rancher/k3s\n'
|
|
while IFS= read -r path; do
|
|
[[ -n "$path" ]] || continue
|
|
printf 'optional|%s\n' "$path"
|
|
done < <(_kseb_transaction_systemd_paths)
|
|
printf 'optional|/var/lib/rancher/k3s/server/cred/encryption-config.json\n'
|
|
}
|
|
|
|
_kseb_estimate_phase_bytes() {
|
|
local backend="$1" records record requirement path extra identity bytes seen
|
|
local total="$KSEB_ESTIMATE_METADATA_BYTES"
|
|
local -a identities=()
|
|
case "$backend" in sqlite|embedded-etcd) ;; *) return 1 ;; esac
|
|
records="$(_kseb_estimator_source_records "$backend")" || return 1
|
|
[[ -n "$records" ]] || return 1
|
|
while IFS= read -r record; do
|
|
IFS='|' read -r requirement path extra <<<"$record"
|
|
[[ -n "$requirement" && -n "$path" && -z "$extra" ]] || return 1
|
|
_kseb_estimator_source_allowed "$backend" "$path" || return 1
|
|
identity="$(_kseb_estimator_source_probe "$path" "$requirement")" || return 1
|
|
[[ -n "$identity" ]] || continue
|
|
for seen in "${identities[@]}"; do
|
|
[[ "$seen" == "$identity" ]] && continue 2
|
|
done
|
|
bytes="$(_kseb_estimate_source_bytes "$backend" "$path" "$requirement" "$identity")" || return 1
|
|
[[ -n "$bytes" ]] || return 1
|
|
total="$(_kseb_checked_estimate_add "$total" "$bytes")" || return 1
|
|
identities+=("$identity")
|
|
done <<<"$records"
|
|
printf '%s\n' "$total"
|
|
}
|
|
|
|
_kseb_privileged_pinned_dispatch() {
|
|
local root="$1" expected_identity="$2" relative="$3" operation="$4"
|
|
shift 4
|
|
_kseb_run_privileged_shell -c '
|
|
set -Eeuo pipefail
|
|
export LC_ALL=C
|
|
root=$1; expected=$2; relative=$3; operation=$4
|
|
shift 4
|
|
[[ "$relative" =~ ^[A-Za-z0-9._/-]+$ && "$relative" != /* &&
|
|
"$relative" != *".."* && "$relative" != *"//"* ]]
|
|
exec {root_fd}<"$root"
|
|
root_handle="/proc/self/fd/${root_fd}"
|
|
actual=$(/usr/bin/stat --dereference --format="%d:%i" -- "$root_handle")
|
|
[[ "$actual" == "$expected" ]]
|
|
owner_uid=$(/usr/bin/id -u); owner_gid=$(/usr/bin/id -g)
|
|
root_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "$root_handle")
|
|
[[ "$root_metadata" == "$owner_uid:$owner_gid:700:directory" ]]
|
|
|
|
if [[ "$operation" == inspect ]]; then
|
|
exec {phase_fd}<"$root_handle/$relative"
|
|
phase_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "/proc/self/fd/${phase_fd}")
|
|
printf "%s|%s\n" "$actual" "$phase_metadata"
|
|
exit 0
|
|
fi
|
|
[[ "$operation" == transaction && $# == 14 ]]
|
|
|
|
data_dir=$1; config_dir=$2; k3s_binary=$3; systemctl_binary=$4
|
|
post_marker=$5; restore_evidence=$6; systemd_paths=$7
|
|
backend=$8; bundle_id=$9; phase=${10}; utc_stamp=${11}
|
|
secret_count=${12}; version=${13}; requested_phase=${14}
|
|
[[ "$phase" == "$requested_phase" && ( "$phase" == pre || "$phase" == post ) ]]
|
|
[[ "$backend" == sqlite || "$backend" == embedded-etcd ]]
|
|
[[ "$bundle_id" =~ ^[A-Za-z0-9._-]+$ && "$utc_stamp" =~ ^[A-Za-z0-9._-]+$ ]]
|
|
[[ "$secret_count" =~ ^[0-9]+$ && "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]]
|
|
[[ "$utc_stamp" =~ ^([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z$ ]]
|
|
created_at_utc="${BASH_REMATCH[1]}-${BASH_REMATCH[2]}-${BASH_REMATCH[3]}T${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}Z"
|
|
normalized_created_at=$(/usr/bin/date -u -d "$created_at_utc" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)
|
|
[[ "$normalized_created_at" == "$created_at_utc" ]]
|
|
bundle_relative=${relative%/*}; relative_phase=${relative##*/}
|
|
[[ "$bundle_relative" != "$relative" && "$relative_phase" == "$phase" &&
|
|
"$bundle_relative" =~ ^[A-Za-z0-9._/-]+$ ]]
|
|
|
|
bundle_handle="$root_handle/$bundle_relative"
|
|
phase_handle="$root_handle/$relative"
|
|
phase_relative_handle="$phase_handle"
|
|
bundle_created=false; phase_created=false; transaction_committed=false
|
|
stop_attempted=false; start_dispatched=false; pending_signal=0
|
|
marker_tmp=""
|
|
|
|
cleanup_transaction() {
|
|
local rc=$?
|
|
trap "" INT TERM
|
|
if "$stop_attempted" && ! "$start_dispatched"; then
|
|
start_dispatched=true
|
|
"$systemctl_binary" start k3s >/dev/null 2>&1 || rc=75
|
|
fi
|
|
if [[ -n "$marker_tmp" ]]; then
|
|
/bin/rm -f -- "$marker_tmp" >/dev/null 2>&1 || :
|
|
fi
|
|
if "$phase_created" && ! "$transaction_committed"; then
|
|
if cd -- "$phase_handle" >/dev/null 2>&1; then
|
|
/usr/bin/find . -mindepth 1 -depth -delete >/dev/null 2>&1 || :
|
|
fi
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
trap cleanup_transaction EXIT
|
|
trap "pending_signal=130; exit 130" INT
|
|
trap "pending_signal=143; exit 143" TERM
|
|
|
|
if [[ -L "$bundle_handle" || -L "$phase_handle" || -e "$phase_handle" ]]; then exit 1; fi
|
|
if [[ -e "$bundle_handle" ]]; then
|
|
bundle_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "$bundle_handle")
|
|
[[ "$bundle_metadata" == "$owner_uid:$owner_gid:700:directory" ]]
|
|
if [[ "$phase" == post ]]; then
|
|
[[ -d "$bundle_handle/pre" && ! -L "$bundle_handle/pre" ]]
|
|
else
|
|
exit 1
|
|
fi
|
|
else
|
|
/usr/bin/install -d -o "$owner_uid" -g "$owner_gid" -m 0700 -- "$bundle_handle"
|
|
bundle_created=true
|
|
fi
|
|
/usr/bin/install -d -o "$owner_uid" -g "$owner_gid" -m 0700 -- "$phase_handle"
|
|
phase_created=true
|
|
exec {phase_fd}<"$phase_handle"
|
|
phase_handle="/proc/self/fd/${phase_fd}"
|
|
phase_identity=$(/usr/bin/stat --dereference --format="%d:%i" -- "$phase_handle")
|
|
[[ "$phase_identity" =~ ^[0-9]+:[0-9]+$ ]]
|
|
phase_metadata=$(/usr/bin/stat --dereference --format="%u:%g:%a:%F" -- "$phase_handle")
|
|
[[ "$phase_metadata" == "$owner_uid:$owner_gid:700:directory" ]]
|
|
|
|
copy_common_recovery() {
|
|
local target relative_path
|
|
local -a archive_paths=()
|
|
/usr/bin/install -o "$owner_uid" -g "$owner_gid" -m 0600 -- \
|
|
"$data_dir/server/token" "$phase_handle/server-token"
|
|
if [[ -d "$config_dir" ]]; then
|
|
/usr/bin/tar --exclude="./platform-post-bundle.env" \
|
|
--exclude="./platform-restore-evidence.env" -C "$config_dir" \
|
|
-cpf "$phase_handle/host-config.tar" . >/dev/null 2>&1
|
|
/usr/bin/chown "$owner_uid:$owner_gid" "$phase_handle/host-config.tar"
|
|
/usr/bin/chmod 0600 "$phase_handle/host-config.tar"
|
|
fi
|
|
while IFS= read -r target; do
|
|
[[ -n "$target" ]] || continue
|
|
if [[ -e "$target" ]]; then
|
|
relative_path=${target#/}; archive_paths+=("$relative_path")
|
|
fi
|
|
done <<<"$systemd_paths"
|
|
if (( ${#archive_paths[@]} > 0 )); then
|
|
/usr/bin/tar -C / -cpf "$phase_handle/systemd-recovery.tar" \
|
|
"${archive_paths[@]}" >/dev/null 2>&1
|
|
/usr/bin/chown "$owner_uid:$owner_gid" "$phase_handle/systemd-recovery.tar"
|
|
/usr/bin/chmod 0600 "$phase_handle/systemd-recovery.tar"
|
|
fi
|
|
if [[ -f "$data_dir/server/cred/encryption-config.json" ]]; then
|
|
/usr/bin/install -o "$owner_uid" -g "$owner_gid" -m 0600 -- \
|
|
"$data_dir/server/cred/encryption-config.json" \
|
|
"$phase_handle/generated-encryption-config.json"
|
|
fi
|
|
}
|
|
|
|
write_manifest() {
|
|
local file hash manifest_tmp list_tmp
|
|
cd -- "$phase_handle"
|
|
umask 077
|
|
manifest_tmp=".verification.manifest.tmp.$$"
|
|
list_tmp=".verification.files.tmp.$$"
|
|
/bin/rm -f -- "$manifest_tmp" "$list_tmp"
|
|
: >"$manifest_tmp"; : >"$list_tmp"
|
|
/usr/bin/find . -type f ! -name verification.manifest \
|
|
! -name "$manifest_tmp" ! -name "$list_tmp" -print0 >"$list_tmp"
|
|
while IFS= read -r -d "" file; do
|
|
[[ "$file" =~ ^\./[A-Za-z0-9._/-]+$ && "$file" != *".."* ]]
|
|
hash=$(/usr/bin/sha256sum -- "$file"); hash=${hash%% *}
|
|
[[ "$hash" =~ ^[0-9a-f]{64}$ ]]
|
|
printf "%s %s\n" "$hash" "$file" >>"$manifest_tmp"
|
|
done <"$list_tmp"
|
|
/bin/rm -f -- "$list_tmp"
|
|
/usr/bin/chown "$owner_uid:$owner_gid" "$manifest_tmp"
|
|
/usr/bin/chmod 0600 "$manifest_tmp"
|
|
/bin/mv -f -- "$manifest_tmp" verification.manifest
|
|
/usr/bin/sha256sum --check verification.manifest >/dev/null
|
|
}
|
|
|
|
workloads_ready() {
|
|
local workloads=$1
|
|
jq -e '\''
|
|
[.items[] | select(.status.phase != "Succeeded" and .status.phase != "Failed") |
|
|
select(.status.phase != "Running" or
|
|
((.spec.containers // []) | length) == 0 or
|
|
((.status.containerStatuses // []) | length) != ((.spec.containers // []) | length) or
|
|
any((.status.containerStatuses // [])[]; .ready != true) or
|
|
((.status.initContainerStatuses // []) | length) != ((.spec.initContainers // []) | length) or
|
|
any((.status.initContainerStatuses // [])[]; (.state.terminated.exitCode? // -1) != 0))] |
|
|
length == 0'\'' >/dev/null 2>&1 <<<"$workloads"
|
|
}
|
|
|
|
recovery_check() {
|
|
local workloads started deadline now remaining command_timeout delay
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 430s \
|
|
"$k3s_binary" kubectl get --raw=/readyz >/dev/null 2>&1 || return 1
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 130s \
|
|
"$k3s_binary" kubectl wait --for=condition=Ready nodes --all \
|
|
--timeout=120s >/dev/null 2>&1 || return 1
|
|
started=$(/usr/bin/date +%s) || return 1
|
|
[[ "$started" =~ ^[0-9]+$ ]] || return 1
|
|
deadline=$((started + 30))
|
|
while :; do
|
|
now=$(/usr/bin/date +%s) || return 1
|
|
[[ "$now" =~ ^[0-9]+$ ]] || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining > 1 )) || break
|
|
command_timeout=9
|
|
(( command_timeout < remaining )) || command_timeout=$((remaining - 1))
|
|
if workloads=$(/usr/bin/timeout --signal=TERM --kill-after=1s "${command_timeout}s" \
|
|
"$k3s_binary" kubectl get pods --all-namespaces -o json 2>/dev/null); then
|
|
workloads_ready "$workloads" && return 0
|
|
fi
|
|
now=$(/usr/bin/date +%s) || return 1
|
|
[[ "$now" =~ ^[0-9]+$ ]] || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining > 0 )) || break
|
|
delay=5
|
|
(( delay <= remaining )) || delay=$remaining
|
|
/usr/bin/sleep "$delay" || return 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
if [[ "$backend" == sqlite ]]; then
|
|
stop_attempted=true
|
|
"$systemctl_binary" stop k3s >/dev/null 2>&1
|
|
/bin/cp -a -- "$data_dir/server/db" "$phase_handle/datastore"
|
|
else
|
|
"$k3s_binary" etcd-snapshot save \
|
|
--name="${phase}-secrets-encryption-${utc_stamp}" \
|
|
--etcd-snapshot-compress --dir="$phase_handle" >/dev/null 2>&1
|
|
"$k3s_binary" etcd-snapshot list --dir="$phase_handle" >/dev/null 2>&1
|
|
fi
|
|
copy_common_recovery
|
|
if [[ "$backend" == sqlite ]]; then
|
|
/usr/bin/diff --no-dereference --recursive --brief \
|
|
"$data_dir/server/db" "$phase_handle/datastore" >/dev/null 2>&1
|
|
fi
|
|
umask 077
|
|
printf "schema=platform-k3s-bundle-v1\nbundle_id=%s\nphase=%s\nk3s_version=%s\ndatastore=%s\ncreated_at_utc=%s\nsecret_count=%s\n" \
|
|
"$bundle_id" "$phase" "$version" "$backend" "$created_at_utc" "$secret_count" \
|
|
>"$phase_handle/bundle.env"
|
|
/usr/bin/chown "$owner_uid:$owner_gid" "$phase_handle/bundle.env"
|
|
/usr/bin/chmod 0600 "$phase_handle/bundle.env"
|
|
write_manifest
|
|
total_bytes=$(/usr/bin/du --summarize --bytes "$phase_handle" | /usr/bin/awk "{print \$1}")
|
|
[[ "$total_bytes" =~ ^[0-9]+$ ]]
|
|
|
|
if [[ "$backend" == sqlite ]]; then
|
|
trap "pending_signal=130" INT
|
|
trap "pending_signal=143" TERM
|
|
start_dispatched=true
|
|
"$systemctl_binary" start k3s >/dev/null 2>&1 || exit 75
|
|
trap "pending_signal=130; exit 130" INT
|
|
trap "pending_signal=143; exit 143" TERM
|
|
(( pending_signal == 0 )) || exit "$pending_signal"
|
|
fi
|
|
recovery_check || exit 75
|
|
commit_identity=$(/usr/bin/stat --dereference --format="%d:%i" -- "$phase_relative_handle") || exit 1
|
|
[[ "$commit_identity" == "$phase_identity" ]] || exit 1
|
|
if [[ "$phase" == post ]]; then
|
|
marker_tmp="${post_marker}.new.$$"
|
|
/usr/bin/install -o "$owner_uid" -g "$owner_gid" -m 0600 -- \
|
|
"$phase_handle/bundle.env" "$marker_tmp" || exit 76
|
|
/bin/rm -f -- "$restore_evidence" || exit 76
|
|
/bin/mv -f -- "$marker_tmp" "$post_marker" || exit 76
|
|
marker_tmp=""
|
|
fi
|
|
transaction_committed=true
|
|
trap - EXIT INT TERM
|
|
printf "%s\n" "$total_bytes"
|
|
' kseb-pinned "$root" "$expected_identity" "$relative" "$operation" "$@"
|
|
}
|
|
|
|
_kseb_pinned_phase_transaction() {
|
|
local root="$1" relative="$2" backend="$3" bundle_id="$4" phase="$5"
|
|
local utc_stamp="$6" secret_count="$7" version="$8"
|
|
local data_dir config_dir k3s_binary systemctl_binary post_marker restore_evidence systemd_paths
|
|
[[ -n "$KSEB_BACKUP_ROOT_IDENTITY" ]] || return 1
|
|
data_dir="$(_kseb_transaction_data_dir)" || return 1
|
|
config_dir="$(_kseb_transaction_config_dir)" || return 1
|
|
k3s_binary="$(_kseb_transaction_k3s_binary)" || return 1
|
|
systemctl_binary="$(_kseb_transaction_systemctl_binary)" || return 1
|
|
post_marker="$(_kseb_transaction_post_marker)" || return 1
|
|
restore_evidence="$(_kseb_transaction_restore_evidence)" || return 1
|
|
systemd_paths="$(_kseb_transaction_systemd_paths)" || return 1
|
|
_kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" "$relative" transaction \
|
|
"$data_dir" "$config_dir" "$k3s_binary" "$systemctl_binary" \
|
|
"$post_marker" "$restore_evidence" "$systemd_paths" "$backend" "$bundle_id" \
|
|
"$phase" "$utc_stamp" "$secret_count" "$version" "$phase" 2>/dev/null
|
|
}
|
|
|
|
_kseb_pinned_root_operation() {
|
|
local root="$1" relative="$2" operation="$3" identity
|
|
[[ -n "$KSEB_BACKUP_ROOT_IDENTITY" ]] || return 1
|
|
_kseb_privileged_backup_root_chain_safe "$root" || return 1
|
|
identity="$(_kseb_privileged_root_identity "$root")" || return 1
|
|
_kseb_backup_root_identity_matches "$KSEB_BACKUP_ROOT_IDENTITY" "$identity" || return 1
|
|
_kseb_privileged_relative_components_safe "$root" "$relative" || return 1
|
|
_kseb_privileged_pinned_dispatch "$root" "$KSEB_BACKUP_ROOT_IDENTITY" "$relative" "$operation"
|
|
}
|
|
|
|
_kseb_validate_expectation() {
|
|
bash "$KSEB_VALIDATOR" "$1" >/dev/null
|
|
}
|
|
|
|
_kseb_enable() {
|
|
/usr/bin/sudo /usr/local/bin/k3s secrets-encrypt enable >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_install_dropin() {
|
|
/usr/bin/sudo /usr/bin/install -o root -g root -m 0644 \
|
|
"$KSEB_DROPIN_SOURCE" \
|
|
/etc/rancher/k3s/config.yaml.d/40-secrets-encryption.yaml >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_restart() {
|
|
/usr/bin/sudo /usr/bin/systemctl restart k3s >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_rotate_keys() {
|
|
/usr/bin/sudo /usr/local/bin/k3s secrets-encrypt rotate-keys >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_wait_for_reencrypt() {
|
|
local rotate_rc="$1" started now elapsed deadline stage status_json status_class
|
|
[[ "$rotate_rc" =~ ^[0-9]+$ ]] || return 1
|
|
KSEB_LAST_SAFE_STATE=unknown
|
|
started="$SECONDS"
|
|
deadline=$((started + 600))
|
|
while (( SECONDS < deadline )); do
|
|
status_json="$(_kseb_read_status_json)" || { KSEB_LAST_SAFE_STATE=unknown; return 1; }
|
|
status_class="$(classify_encryption_status "$status_json")"
|
|
if [[ "$status_class" == hash_mismatch ]]; then KSEB_LAST_SAFE_STATE=hash_mismatch; return 1; fi
|
|
if [[ "$status_class" == invalid ]]; then KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1; fi
|
|
stage="$(jq -er '.stage' <<<"$status_json" 2>/dev/null)" || { KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1; }
|
|
case "$stage" in
|
|
start|reencrypt_active|reencrypt_finished) ;;
|
|
*) KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1 ;;
|
|
esac
|
|
now="$SECONDS"; elapsed=$((now - started))
|
|
printf '재암호화 상태: stage=%s elapsed=%ss\n' "$stage" "$elapsed"
|
|
[[ "$stage" == reencrypt_finished ]] && return 0
|
|
if [[ "$stage" == start ]]; then KSEB_LAST_SAFE_STATE=reencrypt_stage_start; return 1; fi
|
|
[[ "$stage" == reencrypt_active ]] || { KSEB_LAST_SAFE_STATE=reencrypt_stage_unexpected; return 1; }
|
|
KSEB_LAST_SAFE_STATE=reencrypt_active_timeout
|
|
(( SECONDS + 30 < deadline )) || break
|
|
sleep 30
|
|
done
|
|
return 1
|
|
}
|
|
|
|
_kseb_secret_count() {
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 30s \
|
|
/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \
|
|
get secrets --all-namespaces -o json 2>/dev/null |
|
|
jq -er '.items | length' 2>/dev/null
|
|
}
|
|
|
|
_kseb_version() {
|
|
local raw
|
|
raw="$(/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s --version 2>/dev/null)" || return 1
|
|
sed -n -E 's/^k3s version (v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+)([[:space:]].*)?$/\1/p' <<<"$raw"
|
|
}
|
|
|
|
_kseb_privileged_path_kind() {
|
|
local path="$1"
|
|
if /usr/bin/sudo /usr/bin/test -L "$path"; then printf 'symlink\n'
|
|
elif /usr/bin/sudo /usr/bin/test -d "$path"; then printf 'directory\n'
|
|
elif /usr/bin/sudo /usr/bin/test -e "$path"; then printf 'other\n'
|
|
else printf 'absent\n'; fi
|
|
}
|
|
|
|
_kseb_privileged_directory_metadata() {
|
|
/usr/bin/sudo /usr/bin/stat --format='%u:%g:%a:%F' -- "$1" 2>/dev/null
|
|
}
|
|
|
|
_kseb_phase_layout_allowed() {
|
|
local phase="$1" bundle_kind="$2" bundle_metadata="$3" pre_kind="$4"
|
|
case "$phase/$bundle_kind" in
|
|
pre/absent|post/absent) return 0 ;;
|
|
post/directory)
|
|
[[ "$bundle_metadata" == '0:0:700:directory' && "$pre_kind" == directory ]]
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
_kseb_prepare_phase_dir() {
|
|
local bundle_dir="$1" phase_dir="$2" phase="$3"
|
|
local bundle_kind phase_kind pre_kind=absent bundle_metadata=''
|
|
bundle_kind="$(_kseb_privileged_path_kind "$bundle_dir")" || return 1
|
|
phase_kind="$(_kseb_privileged_path_kind "$phase_dir")" || return 1
|
|
[[ "$phase_kind" == absent ]] || return 1
|
|
if [[ "$bundle_kind" == directory ]]; then
|
|
bundle_metadata="$(_kseb_privileged_directory_metadata "$bundle_dir")" || return 1
|
|
pre_kind="$(_kseb_privileged_path_kind "$bundle_dir/pre")" || return 1
|
|
fi
|
|
_kseb_phase_layout_allowed "$phase" "$bundle_kind" "$bundle_metadata" "$pre_kind" || return 1
|
|
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$bundle_dir" "$phase_dir" 2>/dev/null
|
|
}
|
|
|
|
_kseb_copy_common_recovery() {
|
|
local phase_dir="$1" target relative
|
|
local -a systemd_paths=()
|
|
/usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \
|
|
"$KSEB_DATA_DIR/server/token" "$phase_dir/server-token" || return 1
|
|
|
|
if /usr/bin/sudo /usr/bin/test -d "$KSEB_CONFIG_DIR"; then
|
|
_kseb_archive_host_config "$phase_dir" || return 1
|
|
/usr/bin/sudo /usr/bin/chown root:root "$phase_dir/host-config.tar" || return 1
|
|
/usr/bin/sudo /usr/bin/chmod 0600 "$phase_dir/host-config.tar" || return 1
|
|
fi
|
|
|
|
for target in \
|
|
/etc/systemd/system/k3s.service \
|
|
/etc/systemd/system/k3s.service.env \
|
|
/etc/systemd/system/k3s.service.d \
|
|
/usr/lib/systemd/system/k3s.service \
|
|
/lib/systemd/system/k3s.service; do
|
|
if /usr/bin/sudo /usr/bin/test -e "$target"; then
|
|
relative="${target#/}"
|
|
[[ " ${systemd_paths[*]} " == *" $relative "* ]] || systemd_paths+=("$relative")
|
|
fi
|
|
done
|
|
if (( ${#systemd_paths[@]} > 0 )); then
|
|
/usr/bin/sudo /usr/bin/tar -C / -cpf "$phase_dir/systemd-recovery.tar" \
|
|
"${systemd_paths[@]}" || return 1
|
|
/usr/bin/sudo /usr/bin/chown root:root "$phase_dir/systemd-recovery.tar" || return 1
|
|
/usr/bin/sudo /usr/bin/chmod 0600 "$phase_dir/systemd-recovery.tar" || return 1
|
|
fi
|
|
|
|
target="$KSEB_DATA_DIR/server/cred/encryption-config.json"
|
|
if /usr/bin/sudo /usr/bin/test -f "$target"; then
|
|
/usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \
|
|
"$target" "$phase_dir/generated-encryption-config.json" || return 1
|
|
fi
|
|
}
|
|
|
|
_kseb_archive_host_config() {
|
|
local phase_dir="$1"
|
|
_kseb_privileged_exec /usr/bin/tar \
|
|
--exclude='./platform-post-bundle.env' \
|
|
--exclude='./platform-restore-evidence.env' \
|
|
-C "$KSEB_CONFIG_DIR" -cpf "$phase_dir/host-config.tar" .
|
|
}
|
|
|
|
_kseb_render_bundle_metadata() {
|
|
local bundle_id="$1" phase="$2" datastore="$3" created_at_utc="$4"
|
|
local secret_count="$5" k3s_version="$6"
|
|
[[ "$bundle_id" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]] || return 1
|
|
[[ "$phase" == pre || "$phase" == post ]] || return 1
|
|
[[ "$datastore" == sqlite || "$datastore" == embedded-etcd ]] || return 1
|
|
[[ "$created_at_utc" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || return 1
|
|
[[ "$secret_count" =~ ^[0-9]+$ ]] || return 1
|
|
[[ "$k3s_version" == v1.36.2+k3s1 ]] || return 1
|
|
printf 'schema=platform-k3s-bundle-v1\nbundle_id=%s\nphase=%s\nk3s_version=%s\ndatastore=%s\ncreated_at_utc=%s\nsecret_count=%s\n' \
|
|
"$bundle_id" "$phase" "$k3s_version" "$datastore" "$created_at_utc" "$secret_count"
|
|
}
|
|
|
|
_kseb_rfc3339_from_stamp() {
|
|
local stamp="$1"
|
|
if [[ "$stamp" =~ ^([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z$ ]]; then
|
|
printf '%s-%s-%sT%s:%s:%sZ\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" \
|
|
"${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${BASH_REMATCH[6]}"
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
_kseb_write_bundle_metadata() {
|
|
local phase_dir="$1" bundle_id="$2" phase="$3" backend="$4" utc_stamp="$5"
|
|
local secret_count="$6" version="$7" temporary created_at_utc
|
|
[[ "$secret_count" =~ ^[0-9]+$ ]] || return 1
|
|
[[ "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]] || return 1
|
|
created_at_utc="$(_kseb_rfc3339_from_stamp "$utc_stamp")" || return 1
|
|
temporary="$(mktemp /tmp/k3s-secrets-encryption-metadata.XXXXXX)" || return 1
|
|
chmod 0600 "$temporary" || { rm -f -- "$temporary"; return 1; }
|
|
_kseb_render_bundle_metadata "$bundle_id" "$phase" "$backend" "$created_at_utc" \
|
|
"$secret_count" "$version" >"$temporary" || { rm -f -- "$temporary"; return 1; }
|
|
/usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \
|
|
"$temporary" "$phase_dir/bundle.env" || { rm -f -- "$temporary"; return 1; }
|
|
rm -f -- "$temporary"
|
|
}
|
|
|
|
_kseb_manifest_list_files() {
|
|
/usr/bin/sudo /usr/bin/find "$1" -type f ! -name verification.manifest -print0 2>/dev/null
|
|
}
|
|
|
|
_kseb_manifest_hash_file() {
|
|
local line hash
|
|
line="$(/usr/bin/sudo /usr/bin/sha256sum -- "$1" 2>/dev/null)" || return 1
|
|
hash="${line%% *}"
|
|
[[ "$hash" =~ ^[0-9a-f]{64}$ ]] || return 1
|
|
printf '%s\n' "$hash"
|
|
}
|
|
|
|
_kseb_build_relative_manifest() {
|
|
local phase_dir="$1" output="$2" file relative hash list_file rc=0
|
|
: >"$output" || return 1
|
|
chmod 0600 "$output" || return 1
|
|
list_file="$(mktemp /tmp/k3s-secrets-encryption-files.XXXXXX)" || return 1
|
|
chmod 0600 "$list_file" || { rm -f -- "$list_file"; return 1; }
|
|
if ! _kseb_manifest_list_files "$phase_dir" >"$list_file"; then
|
|
rm -f -- "$list_file"
|
|
return 1
|
|
fi
|
|
while IFS= read -r -d '' file; do
|
|
[[ "$file" == "$phase_dir/"* ]] || { rc=1; break; }
|
|
relative="${file#"$phase_dir/"}"
|
|
[[ -n "$relative" && "$relative" != /* && "$relative" != *'..'* &&
|
|
"$relative" =~ ^[A-Za-z0-9._/-]+$ ]] || { rc=1; break; }
|
|
hash="$(_kseb_manifest_hash_file "$file")" || { rc=1; break; }
|
|
[[ "$hash" =~ ^[0-9a-f]{64}$ ]] || { rc=1; break; }
|
|
printf '%s ./%s\n' "$hash" "$relative" >>"$output" || { rc=1; break; }
|
|
done <"$list_file"
|
|
rm -f -- "$list_file"
|
|
return "$rc"
|
|
}
|
|
|
|
_kseb_manifest_install() {
|
|
/usr/bin/sudo /usr/bin/install -o root -g root -m 0600 \
|
|
"$1" "$2/verification.manifest" >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_manifest_check() {
|
|
/usr/bin/sudo /usr/bin/env --chdir="$1" \
|
|
/usr/bin/sha256sum --check verification.manifest >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_write_verification_manifest() {
|
|
_kseb_privileged_manifest_lifecycle "$1"
|
|
}
|
|
|
|
_kseb_privileged_manifest_lifecycle() {
|
|
local phase_dir="$1"
|
|
/usr/bin/sudo /usr/bin/bash -c '
|
|
set -Eeuo pipefail
|
|
phase=$1
|
|
exec {phase_fd}<"$phase"
|
|
metadata=$(/usr/bin/stat --format="%u:%g:%a:%F" -- "/proc/self/fd/${phase_fd}")
|
|
[[ "$metadata" == "0:0:700:directory" ]]
|
|
cd -- "/proc/self/fd/${phase_fd}"
|
|
umask 077
|
|
manifest_tmp=".verification.manifest.tmp.$$"
|
|
list_tmp=".verification.files.tmp.$$"
|
|
cleanup() { /bin/rm -f -- "$manifest_tmp" "$list_tmp"; }
|
|
trap cleanup EXIT INT TERM
|
|
: >"$manifest_tmp"
|
|
: >"$list_tmp"
|
|
/usr/bin/find . -type f \
|
|
! -name verification.manifest \
|
|
! -name "$manifest_tmp" ! -name "$list_tmp" -print0 >"$list_tmp"
|
|
while IFS= read -r -d "" file; do
|
|
[[ "$file" =~ ^\./[A-Za-z0-9._/-]+$ && "$file" != *".."* ]]
|
|
hash=$(/usr/bin/sha256sum -- "$file")
|
|
hash=${hash%% *}
|
|
[[ "$hash" =~ ^[0-9a-f]{64}$ ]]
|
|
printf "%s %s\n" "$hash" "$file" >>"$manifest_tmp"
|
|
done <"$list_tmp"
|
|
/bin/rm -f -- "$list_tmp"
|
|
/usr/bin/chown root:root "$manifest_tmp"
|
|
/usr/bin/chmod 0600 "$manifest_tmp"
|
|
/bin/mv -f -- "$manifest_tmp" verification.manifest
|
|
/usr/bin/sha256sum --check verification.manifest >/dev/null
|
|
trap - EXIT INT TERM
|
|
' kseb-manifest "$phase_dir" >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_api_ready_once() {
|
|
local command_timeout="${1:-9}"
|
|
[[ "$command_timeout" =~ ^[1-9]$ ]] || return 1
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "${command_timeout}s" \
|
|
/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \
|
|
get --raw=/readyz >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_now_seconds() {
|
|
date +%s
|
|
}
|
|
|
|
_kseb_wait_for_api() {
|
|
local started deadline now remaining command_timeout delay
|
|
started="$(_kseb_now_seconds)" || return 1
|
|
[[ "$started" =~ ^[0-9]+$ ]] || return 1
|
|
deadline=$((started + KSEB_API_RECOVERY_BUDGET))
|
|
while :; do
|
|
now="$(_kseb_now_seconds)" || return 1
|
|
[[ "$now" =~ ^[0-9]+$ ]] || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining > 1 )) || break
|
|
command_timeout=9
|
|
(( command_timeout < remaining )) || command_timeout=$((remaining - 1))
|
|
_kseb_api_ready_once "$command_timeout" && return 0
|
|
now="$(_kseb_now_seconds)" || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining > 0 )) || break
|
|
delay=10
|
|
(( delay <= remaining )) || delay="$remaining"
|
|
sleep "$delay"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
_kseb_node_recovery_check() {
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s 130s \
|
|
/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \
|
|
wait --for=condition=Ready nodes --all --timeout=120s >/dev/null 2>&1
|
|
}
|
|
|
|
_kseb_read_workloads_json() {
|
|
local command_timeout="${1:-30}"
|
|
[[ "$command_timeout" =~ ^([1-9]|[12][0-9]|30)$ ]] || return 1
|
|
/usr/bin/timeout --signal=TERM --kill-after=1s "${command_timeout}s" \
|
|
/usr/bin/sudo --non-interactive -- /usr/local/bin/k3s kubectl \
|
|
get pods --all-namespaces -o json 2>/dev/null
|
|
}
|
|
|
|
_kseb_workloads_ready_json() {
|
|
jq -e '
|
|
[.items[] |
|
|
select(.status.phase != "Succeeded" and .status.phase != "Failed") |
|
|
select(
|
|
.status.phase != "Running" or
|
|
((.spec.containers // []) | length) == 0 or
|
|
((.status.containerStatuses // []) | length) != ((.spec.containers // []) | length) or
|
|
any((.status.containerStatuses // [])[]; .ready != true) or
|
|
((.status.initContainerStatuses // []) | length) != ((.spec.initContainers // []) | length) or
|
|
any((.status.initContainerStatuses // [])[]; (.state.terminated.exitCode? // -1) != 0)
|
|
)] | length == 0
|
|
' >/dev/null 2>&1 <<<"$1"
|
|
}
|
|
|
|
_kseb_recovery_budget_valid() {
|
|
local api="$1" node="$2" workload="$3"
|
|
[[ "$api" =~ ^[0-9]+$ && "$node" =~ ^[0-9]+$ && "$workload" =~ ^[0-9]+$ ]] || return 1
|
|
(( api + node + workload <= 600 ))
|
|
}
|
|
|
|
_kseb_wait_for_workloads() {
|
|
local started deadline now remaining command_timeout delay workloads
|
|
started="$(_kseb_now_seconds)" || return 1
|
|
[[ "$started" =~ ^[0-9]+$ ]] || return 1
|
|
deadline=$((started + KSEB_WORKLOAD_RECOVERY_BUDGET))
|
|
while :; do
|
|
now="$(_kseb_now_seconds)" || return 1
|
|
[[ "$now" =~ ^[0-9]+$ ]] || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining > 1 )) || break
|
|
command_timeout=9
|
|
(( command_timeout < remaining )) || command_timeout=$((remaining - 1))
|
|
workloads="$(_kseb_read_workloads_json "$command_timeout")" || workloads=''
|
|
if [[ -n "$workloads" ]] && _kseb_workloads_ready_json "$workloads"; then
|
|
return 0
|
|
fi
|
|
now="$(_kseb_now_seconds)" || return 1
|
|
[[ "$now" =~ ^[0-9]+$ ]] || return 1
|
|
remaining=$((deadline - now))
|
|
(( remaining > 0 )) || break
|
|
delay=5
|
|
(( delay <= remaining )) || delay="$remaining"
|
|
sleep "$delay"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
_kseb_recovery_check() {
|
|
_kseb_recovery_budget_valid "$KSEB_API_RECOVERY_BUDGET" \
|
|
"$KSEB_NODE_RECOVERY_BUDGET" "$KSEB_WORKLOAD_RECOVERY_BUDGET" || return 1
|
|
_kseb_wait_for_api || return 1
|
|
_kseb_node_recovery_check || return 1
|
|
_kseb_wait_for_workloads
|
|
}
|
|
|
|
_kseb_sqlite_copy_and_verify() {
|
|
local phase_dir="$1" bundle_id="$2" phase="$3" utc_stamp="$4" secret_count="$5" version="$6"
|
|
_kseb_copy_sqlite_datastore "$phase_dir" || return 1
|
|
_kseb_copy_common_recovery "$phase_dir" || return 1
|
|
_kseb_compare_sqlite_datastore "$phase_dir" || return 1
|
|
_kseb_write_bundle_metadata "$phase_dir" "$bundle_id" "$phase" sqlite "$utc_stamp" \
|
|
"$secret_count" "$version" || return 1
|
|
_kseb_write_verification_manifest "$phase_dir"
|
|
}
|
|
|
|
_kseb_copy_sqlite_datastore() {
|
|
_kseb_privileged_exec /bin/cp -a -- "$KSEB_DATA_DIR/server/db" "$1/datastore"
|
|
}
|
|
|
|
_kseb_compare_sqlite_datastore() {
|
|
_kseb_privileged_exec /usr/bin/diff --no-dereference --recursive --brief \
|
|
"$KSEB_DATA_DIR/server/db" "$1/datastore"
|
|
}
|
|
|
|
_kseb_stop_k3s() {
|
|
/usr/bin/sudo /usr/bin/systemctl stop k3s
|
|
}
|
|
|
|
_kseb_start_k3s() {
|
|
/usr/bin/sudo /usr/bin/systemctl start k3s
|
|
}
|
|
|
|
_kseb_backup_sqlite_guarded() (
|
|
local phase_dir="$1" bundle_id="$2" phase="$3" utc_stamp="$4" secret_count="$5" version="$6"
|
|
local stop_attempted=false start_dispatched=false service_recovered=false pending_signal=0
|
|
_kseb_sqlite_exit_recovery() {
|
|
local rc=$?
|
|
trap - EXIT INT TERM
|
|
if "$stop_attempted" && ! "$start_dispatched"; then
|
|
start_dispatched=true
|
|
_kseb_start_k3s >/dev/null 2>&1 || rc=75
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
trap _kseb_sqlite_exit_recovery EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
stop_attempted=true
|
|
_kseb_stop_k3s || return 1
|
|
_kseb_sqlite_copy_and_verify "$phase_dir" "$bundle_id" "$phase" "$utc_stamp" \
|
|
"$secret_count" "$version" || return 1
|
|
_kseb_before_start_dispatch
|
|
trap 'pending_signal=130' INT
|
|
trap 'pending_signal=143' TERM
|
|
start_dispatched=true
|
|
_kseb_start_k3s || return 75
|
|
service_recovered=true
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
(( pending_signal == 0 )) || return "$pending_signal"
|
|
"$service_recovered" || return 1
|
|
_kseb_recovery_check
|
|
)
|
|
|
|
_kseb_before_start_dispatch() {
|
|
:
|
|
}
|
|
|
|
_kseb_backup_sqlite() {
|
|
_kseb_backup_sqlite_guarded "$@"
|
|
}
|
|
|
|
_kseb_backup_etcd() {
|
|
local phase_dir="$1" bundle_id="$2" phase="$3" utc_stamp="$4" secret_count="$5" version="$6"
|
|
_kseb_etcd_snapshot_save "$phase" "$utc_stamp" "$phase_dir" || return 1
|
|
_kseb_etcd_snapshot_list "$phase_dir" || return 1
|
|
_kseb_copy_common_recovery "$phase_dir" || return 1
|
|
_kseb_write_bundle_metadata "$phase_dir" "$bundle_id" "$phase" embedded-etcd "$utc_stamp" \
|
|
"$secret_count" "$version" || return 1
|
|
_kseb_write_verification_manifest "$phase_dir" || return 1
|
|
_kseb_recovery_check
|
|
}
|
|
|
|
_kseb_etcd_snapshot_save() {
|
|
local phase="$1" utc_stamp="$2" phase_dir="$3"
|
|
_kseb_privileged_exec /usr/local/bin/k3s etcd-snapshot save \
|
|
--name="${phase}-secrets-encryption-${utc_stamp}" \
|
|
--etcd-snapshot-compress --dir="$phase_dir"
|
|
}
|
|
|
|
_kseb_etcd_snapshot_list() {
|
|
_kseb_privileged_exec /usr/local/bin/k3s etcd-snapshot list --dir="$1"
|
|
}
|
|
|
|
_kseb_backup_phase() {
|
|
local phase="$1" backend="$2" backup_root="$3" bundle_dir="$4" bundle_id="$5" utc_stamp="$6"
|
|
local bundle_name="${bundle_dir##*/}" relative_phase total_bytes secret_count version transaction_rc
|
|
[[ "$phase" == pre || "$phase" == post ]] || return 1
|
|
_kseb_revalidate_backup_root "$backup_root" || return 1
|
|
_kseb_require_phase_capacity "$phase" "$backend" "$backup_root" || return 1
|
|
secret_count="$(_kseb_secret_count)" || return 1
|
|
version="$(_kseb_version)" || return 1
|
|
relative_phase="$bundle_name/$phase"
|
|
if total_bytes="$(_kseb_pinned_phase_transaction "$backup_root" "$relative_phase" "$backend" \
|
|
"$bundle_id" "$phase" "$utc_stamp" "$secret_count" "$version")"; then
|
|
:
|
|
else
|
|
transaction_rc=$?
|
|
return "$transaction_rc"
|
|
fi
|
|
[[ "$total_bytes" =~ ^[0-9]+$ ]] || return 1
|
|
printf 'Recovery bundle %s 검증 완료: bundle-id=%s, total-bytes=%s\n' \
|
|
"$phase" "$bundle_id" "$total_bytes"
|
|
}
|
|
|
|
_kseb_rotate_wait_restart_validate() {
|
|
local rotate_rc
|
|
set +e
|
|
_kseb_rotate_keys
|
|
rotate_rc=$?
|
|
set -e
|
|
if ! _kseb_wait_for_reencrypt "$rotate_rc"; then
|
|
_kseb_terminal_failure "$KSEB_LAST_SAFE_STATE"
|
|
return 1
|
|
fi
|
|
_kseb_restart || { _kseb_terminal_failure partial_final_restart_failed; return 1; }
|
|
_kseb_validate_expectation --expect-reencrypted || _kseb_terminal_failure final_validation_failed
|
|
}
|
|
|
|
apply_k3s_secret_encryption_main() {
|
|
local execute=false rotate_existing=false backup_root='' backup_seen=false estimate_recovery=false
|
|
local recovery_policy='' recovery_policy_seen=false context
|
|
local inventory status rotation backend provider owner integrity server_hashes
|
|
local utc_stamp bundle_id bundle_dir backup_rc phase_bytes argument estimate_argument_count=0
|
|
|
|
for argument in "$@"; do
|
|
if [[ "$argument" == --estimate-recovery-bytes ]]; then
|
|
estimate_argument_count=$((estimate_argument_count + 1))
|
|
fi
|
|
done
|
|
if (( estimate_argument_count > 0 )) &&
|
|
{ (( $# != 1 )) || [[ "${1:-}" != --estimate-recovery-bytes ]]; }; then
|
|
_kseb_usage >&2
|
|
return 2
|
|
fi
|
|
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--execute) "$execute" && { _kseb_usage >&2; return 2; }; execute=true ;;
|
|
--rotate-existing) "$rotate_existing" && { _kseb_usage >&2; return 2; }; rotate_existing=true ;;
|
|
--estimate-recovery-bytes)
|
|
"$estimate_recovery" && { _kseb_usage >&2; return 2; }
|
|
estimate_recovery=true
|
|
;;
|
|
--backup-root)
|
|
(( $# >= 2 )) && ! "$backup_seen" && [[ "$2" == /* ]] || { _kseb_usage >&2; return 2; }
|
|
backup_root="$2"; backup_seen=true; shift
|
|
;;
|
|
--recovery-policy)
|
|
(( $# >= 2 )) && ! "$recovery_policy_seen" || { _kseb_usage >&2; return 2; }
|
|
case "$2" in
|
|
encrypted-off-host|local-separate-disk-luks) recovery_policy="$2" ;;
|
|
*) _kseb_usage >&2; return 2 ;;
|
|
esac
|
|
recovery_policy_seen=true; shift
|
|
;;
|
|
--help|-h) _kseb_usage; return 0 ;;
|
|
*) _kseb_usage >&2; return 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
if "$estimate_recovery"; then
|
|
! "$execute" && ! "$rotate_existing" && ! "$backup_seen" && ! "$recovery_policy_seen" || {
|
|
_kseb_usage >&2
|
|
return 2
|
|
}
|
|
elif ! "$execute"; then
|
|
! "$rotate_existing" && ! "$backup_seen" && ! "$recovery_policy_seen" || {
|
|
_kseb_usage >&2
|
|
return 2
|
|
}
|
|
else
|
|
"$backup_seen" && "$recovery_policy_seen" || { _kseb_usage >&2; return 2; }
|
|
fi
|
|
|
|
inventory="$(_kseb_load_inventory)" || {
|
|
_kseb_terminal_failure initial_unsafe
|
|
return 1
|
|
}
|
|
IFS='|' read -r status rotation backend provider owner integrity server_hashes <<<"$inventory"
|
|
if [[ "$status" == hash_mismatch ]]; then _kseb_terminal_failure hash_mismatch; return 1; fi
|
|
case "$backend" in sqlite|embedded-etcd) ;; *) _kseb_terminal_failure datastore_unsafe; return 1 ;; esac
|
|
|
|
if [[ "$estimate_recovery" == true ]]; then
|
|
phase_bytes="$(_kseb_estimate_phase_bytes "$backend")" || {
|
|
_kseb_fail 'recovery phase 용량을 안전하게 계산하지 못했습니다'
|
|
return 1
|
|
}
|
|
phase_bytes="$(_kseb_normalize_signed_uint "$phase_bytes")" || {
|
|
_kseb_fail 'recovery phase 용량을 안전하게 계산하지 못했습니다'
|
|
return 1
|
|
}
|
|
printf 'phase_bytes=%s\n' "$phase_bytes"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$execute" == false ]]; then
|
|
_kseb_dry_run "$inventory"
|
|
return 0
|
|
fi
|
|
|
|
case "$status/$rotation" in
|
|
disabled_no_config/none) ;;
|
|
enabled_stable/reencrypt_finished)
|
|
[[ "$provider" == aescbc && "$owner" == aescbc/* && "$owner" != aescbc/ambiguous && "$integrity" == match ]] || {
|
|
_kseb_fail '기존 enabled 상태의 provider owner 또는 local integrity가 안전하지 않습니다'
|
|
return 1
|
|
}
|
|
;;
|
|
enabled_stable/start)
|
|
[[ "$provider" == aescbc && "$owner" == aescbc/* && "$owner" != aescbc/ambiguous && "$integrity" == match ]] || {
|
|
_kseb_fail '기존 enabled/start 상태의 provider owner 또는 local integrity가 안전하지 않습니다'
|
|
return 1
|
|
}
|
|
"$rotate_existing" || {
|
|
_kseb_fail 'enabled/start는 --rotate-existing 명시 승인 없이는 진행할 수 없습니다'
|
|
return 1
|
|
}
|
|
;;
|
|
*) _kseb_terminal_failure initial_unsafe; return 1 ;;
|
|
esac
|
|
|
|
context="$(_kseb_prepare_execute_context)" || return 1
|
|
_kseb_confirm_apply "$context" || return 1
|
|
_kseb_validate_backup_root "$backup_root" || { _kseb_terminal_failure backup_root_unsafe; return 1; }
|
|
_kseb_validate_recovery_policy "$recovery_policy" "$backup_root" || {
|
|
_kseb_terminal_failure backup_root_unsafe
|
|
return 1
|
|
}
|
|
_kseb_confirm_recovery "$context" "$recovery_policy" || return 1
|
|
utc_stamp="$(date -u +%Y%m%dT%H%M%SZ)" || return 1
|
|
bundle_id="$(tr -d '\n' </proc/sys/kernel/random/uuid)" || return 1
|
|
[[ "$bundle_id" =~ ^[0-9a-f-]{36}$ ]] || return 1
|
|
bundle_dir="$backup_root/k3s-secrets-encryption-$utc_stamp"
|
|
|
|
case "$status/$rotation" in
|
|
disabled_no_config/none)
|
|
_kseb_validate_expectation --expect-disabled || return 1
|
|
_kseb_require_exact_disabled "$backend" || { _kseb_terminal_failure disabled_state_drift; return 1; }
|
|
if _kseb_backup_phase pre "$backend" "$backup_root" "$bundle_dir" "$bundle_id" "$utc_stamp"; then :; else
|
|
backup_rc=$?
|
|
if (( backup_rc == 75 )); then _kseb_terminal_failure service_recovery_failed
|
|
else _kseb_terminal_failure pre_backup_failed; fi
|
|
return 1
|
|
fi
|
|
_kseb_run_quiet _kseb_enable || { _kseb_terminal_failure partial_enable_failed; return 1; }
|
|
_kseb_run_quiet _kseb_install_dropin || { _kseb_terminal_failure partial_dropin_failed; return 1; }
|
|
_kseb_run_quiet _kseb_restart || { _kseb_terminal_failure partial_initial_restart_failed; return 1; }
|
|
_kseb_validate_expectation --expect-transition-start || { _kseb_terminal_failure expectation_failed; return 1; }
|
|
_kseb_rotate_wait_restart_validate || return 1
|
|
;;
|
|
enabled_stable/start)
|
|
_kseb_validate_expectation --expect-enabled || return 1
|
|
if _kseb_backup_phase pre "$backend" "$backup_root" "$bundle_dir" "$bundle_id" "$utc_stamp"; then :; else
|
|
backup_rc=$?
|
|
if (( backup_rc == 75 )); then _kseb_terminal_failure service_recovery_failed
|
|
else _kseb_terminal_failure pre_backup_failed; fi
|
|
return 1
|
|
fi
|
|
_kseb_require_exact_enabled_start "$backend" || {
|
|
_kseb_terminal_failure enabled_start_drift
|
|
return 1
|
|
}
|
|
_kseb_rotate_wait_restart_validate || return 1
|
|
;;
|
|
enabled_stable/reencrypt_finished)
|
|
_kseb_validate_expectation --expect-reencrypted || { _kseb_terminal_failure final_validation_failed; return 1; }
|
|
_kseb_require_exact_finished "$backend" || { _kseb_terminal_failure enabled_finished_drift; return 1; }
|
|
;;
|
|
esac
|
|
|
|
if _kseb_backup_phase post "$backend" "$backup_root" "$bundle_dir" "$bundle_id" "$utc_stamp"; then :; else
|
|
backup_rc=$?
|
|
if (( backup_rc == 75 )); then _kseb_terminal_failure service_recovery_failed
|
|
elif (( backup_rc == 76 )); then _kseb_terminal_failure marker_install_failed
|
|
else _kseb_terminal_failure post_backup_failed; fi
|
|
return 1
|
|
fi
|
|
printf 'K3S SECRET ENCRYPTION ENABLE SUCCESS\n'
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
PATH="$KSEB_SECURE_PATH"
|
|
export PATH
|
|
apply_k3s_secret_encryption_main "$@"
|
|
fi
|