345 lines
16 KiB
Bash
345 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Read-only k3s Secret-encryption inventory. Privileged execution is limited to
|
|
# bounded invocations of validated, root-owned system binaries; workspace code
|
|
# is never executed or sourced by sudo.
|
|
set -Eeuo pipefail
|
|
|
|
readonly KSE_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin'
|
|
readonly KSE_EXPECTED_VERSION='v1.36.2+k3s1'
|
|
readonly KSE_EXPECTED_NODE='donghyeon-system-product-name'
|
|
readonly KSE_K3S='/usr/local/bin/k3s'
|
|
readonly KSE_TIMEOUT='/usr/bin/timeout'
|
|
readonly KSE_SUDO='/usr/bin/sudo'
|
|
readonly KSE_SYSTEMCTL='/usr/bin/systemctl'
|
|
readonly KSE_STAT='/usr/bin/stat'
|
|
readonly KSE_SHA256SUM='/usr/bin/sha256sum'
|
|
readonly KSE_CAT='/usr/bin/cat'
|
|
readonly KSE_JQ='/usr/bin/jq'
|
|
readonly KSE_FIND='/usr/bin/find'
|
|
readonly KSE_ID='/usr/bin/id'
|
|
readonly KSE_TEST='/usr/bin/test'
|
|
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
|
|
readonly KSE_SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
|
else
|
|
readonly KSE_SCRIPT_DIR='.'
|
|
fi
|
|
readonly KSE_ROOT="$(cd -- "${KSE_SCRIPT_DIR}/../.." && pwd -P)"
|
|
readonly KSE_LIBRARY_PATH="${KSE_ROOT}/scripts/lib/k3s-secret-encryption.sh"
|
|
readonly KSE_CONFIG_PATH='/var/lib/rancher/k3s/server/cred/encryption-config.json'
|
|
readonly KSE_STATE_PATH='/var/lib/rancher/k3s/server/cred/encryption-state.json'
|
|
|
|
kse_fail() {
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
return 1
|
|
}
|
|
|
|
kse_usage() {
|
|
cat <<'USAGE'
|
|
사용법:
|
|
bash scripts/validate/k3s-secret-encryption.sh [기대 상태] [--verified-output-dir DIR]
|
|
|
|
기대 상태(하나만 지정):
|
|
--expect-disabled
|
|
--expect-transition-start
|
|
--expect-enabled
|
|
--expect-reencrypted
|
|
--expect-reencrypted-restore # 격리 복구 host의 다른 node 이름 허용
|
|
USAGE
|
|
}
|
|
|
|
_kse_trusted_executable() {
|
|
local path="$1" metadata uid mode file_type
|
|
[[ -f "$path" && ! -L "$path" && -x "$path" ]] || return 1
|
|
metadata="$($KSE_STAT --format='%u:%a:%F' -- "$path" 2>/dev/null)" || return 1
|
|
IFS=: read -r uid mode file_type <<<"$metadata"
|
|
[[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == 'regular file' ]] || return 1
|
|
(( (8#$mode & 8#022) == 0 ))
|
|
}
|
|
|
|
_kse_validate_runtime_dependencies() {
|
|
local binary
|
|
for binary in \
|
|
"$KSE_TIMEOUT" "$KSE_SUDO" "$KSE_K3S" "$KSE_SYSTEMCTL" \
|
|
"$KSE_STAT" "$KSE_SHA256SUM" "$KSE_CAT" "$KSE_JQ" \
|
|
"$KSE_FIND" "$KSE_ID" "$KSE_TEST" \
|
|
/usr/bin/sed /usr/bin/sort; do
|
|
_kse_trusted_executable "$binary" || return 1
|
|
done
|
|
}
|
|
|
|
_kse_privileged_capture() {
|
|
local binary="$1"
|
|
shift
|
|
"$KSE_TIMEOUT" --signal=TERM --kill-after=1s 9s \
|
|
"$KSE_SUDO" --non-interactive -- "$binary" "$@" 2>/dev/null
|
|
}
|
|
|
|
_kse_validate_sudo_credentials() {
|
|
"$KSE_TIMEOUT" --signal=TERM --kill-after=1s 9s \
|
|
"$KSE_SUDO" --non-interactive --validate >/dev/null 2>&1
|
|
}
|
|
|
|
_kse_privileged_quiet() {
|
|
_kse_privileged_capture "$@" >/dev/null 2>&1
|
|
}
|
|
|
|
_kse_read_version() {
|
|
local raw version
|
|
raw="$(_kse_privileged_capture "$KSE_K3S" --version)" || return 1
|
|
version="$(/usr/bin/sed -n -E 's/^k3s version (v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+)([[:space:]].*)?$/\1/p' <<<"$raw")"
|
|
[[ "$version" == "$KSE_EXPECTED_VERSION" ]] || return 1
|
|
printf '%s\n' "$version"
|
|
}
|
|
|
|
_kse_read_status() {
|
|
_kse_privileged_capture "$KSE_K3S" secrets-encrypt status --output json
|
|
}
|
|
|
|
_kse_read_nodes() {
|
|
_kse_privileged_capture "$KSE_K3S" kubectl get nodes -o json
|
|
}
|
|
|
|
_kse_read_systemctl_show() {
|
|
_kse_privileged_capture "$KSE_SYSTEMCTL" show k3s \
|
|
--property=ExecStart --property=Environment --property=EnvironmentFiles
|
|
}
|
|
|
|
_kse_api_ready() {
|
|
_kse_privileged_quiet "$KSE_K3S" kubectl get --raw=/readyz
|
|
}
|
|
|
|
_kse_allowed_root_text_path() {
|
|
local path="$1"
|
|
[[ "$path" == /etc/rancher/k3s/config.yaml ||
|
|
"$path" == /etc/systemd/system/k3s.service.env ||
|
|
"$path" =~ ^/etc/rancher/k3s/config\.yaml\.d/[A-Za-z0-9._-]+\.yaml$ ]]
|
|
}
|
|
|
|
_kse_read_root_text() {
|
|
local path="$1" metadata uid mode
|
|
_kse_allowed_root_text_path "$path" || return 1
|
|
_kse_privileged_quiet "$KSE_TEST" -f "$path" || return 1
|
|
_kse_privileged_quiet "$KSE_TEST" ! -L "$path" || return 1
|
|
metadata="$(_kse_privileged_capture "$KSE_STAT" --format='%u:%a' -- "$path")" || return 1
|
|
IFS=: read -r uid mode <<<"$metadata"
|
|
[[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ ]] || return 1
|
|
(( (8#$mode & 8#022) == 0 )) || return 1
|
|
_kse_privileged_capture "$KSE_CAT" -- "$path"
|
|
}
|
|
|
|
_kse_root_path_kind() {
|
|
local path="$1"
|
|
if _kse_privileged_quiet "$KSE_TEST" -f "$path"; then
|
|
printf 'file\n'
|
|
elif _kse_privileged_quiet "$KSE_TEST" -d "$path"; then
|
|
printf 'directory\n'
|
|
else
|
|
printf 'absent\n'
|
|
fi
|
|
}
|
|
|
|
_kse_detect_datastore() {
|
|
local service_show="$1"
|
|
_k3s_systemctl_show() { printf '%s\n' "$service_show"; }
|
|
_k3s_systemd_dropin_lines() { return 0; }
|
|
_k3s_read_file() { _kse_read_root_text "$1"; }
|
|
_k3s_list_yaml_files() {
|
|
local config="$1" file listing=''
|
|
[[ "$config" == /etc/rancher/k3s/config.yaml ]] || return 2
|
|
if _kse_privileged_quiet "$KSE_TEST" -f "$config"; then printf '%s\n' "$config"; fi
|
|
if _kse_privileged_quiet "$KSE_TEST" -d "${config}.d"; then
|
|
listing="$(_kse_privileged_capture "$KSE_FIND" "${config}.d" -maxdepth 1 -type f -name '*.yaml' -print)" || return 2
|
|
while IFS= read -r file; do
|
|
[[ -z "$file" ]] || _kse_allowed_root_text_path "$file" || return 2
|
|
done <<<"$listing"
|
|
[[ -z "$listing" ]] || printf '%s\n' "$listing" | /usr/bin/sort
|
|
fi
|
|
}
|
|
_k3s_datastore_local_evidence() {
|
|
local data_dir="$1" state_kind etcd_kind
|
|
[[ "$data_dir" == /var/lib/rancher/k3s ]] || { printf 'ambiguous\n'; return 0; }
|
|
state_kind="$(_kse_root_path_kind "${data_dir}/server/db/state.db")" || return 1
|
|
etcd_kind="$(_kse_root_path_kind "${data_dir}/server/db/etcd")" || return 1
|
|
if [[ "$state_kind" == file && "$etcd_kind" == directory ]]; then printf 'ambiguous\n';
|
|
elif [[ "$state_kind" == file ]]; then printf 'sqlite\n';
|
|
elif [[ "$etcd_kind" == directory ]]; then printf 'embedded-etcd\n';
|
|
else printf 'none\n'; fi
|
|
}
|
|
detect_k3s_datastore
|
|
}
|
|
|
|
_kse_root_metadata() {
|
|
local path="$1" metadata
|
|
metadata="$(_kse_privileged_capture "$KSE_STAT" --format='%u:%a:%F' -- "$path")" || return 1
|
|
[[ "$metadata" == '0:600:regular file' ]] || return 1
|
|
printf '%s\n' "$metadata"
|
|
}
|
|
|
|
_kse_integrity_from_root_evidence() {
|
|
local annotation="$1" stage="$2" config_metadata state_metadata
|
|
local config_hash_line config_hash state_with_sentinel state_value result
|
|
config_metadata="$(_kse_root_metadata "$KSE_CONFIG_PATH")" || { printf 'mismatch\n'; return 1; }
|
|
state_metadata="$(_kse_root_metadata "$KSE_STATE_PATH")" || { printf 'mismatch\n'; return 1; }
|
|
config_hash_line="$(_kse_privileged_capture "$KSE_SHA256SUM" -- "$KSE_CONFIG_PATH")" || { printf 'mismatch\n'; return 1; }
|
|
config_hash="${config_hash_line%% *}"
|
|
[[ "$config_hash_line" == "${config_hash} ${KSE_CONFIG_PATH}" && "$config_hash" =~ ^[0-9a-f]{64}$ ]] || { printf 'mismatch\n'; return 1; }
|
|
state_with_sentinel="$(_kse_privileged_capture "$KSE_CAT" -- "$KSE_STATE_PATH"; printf .)" || { printf 'mismatch\n'; return 1; }
|
|
state_value="${state_with_sentinel%.}"
|
|
result="$(verify_local_encryption_config_integrity_evidence \
|
|
"$config_metadata" "$state_metadata" "$config_hash" "$state_value" \
|
|
"$annotation" "$stage" 2>/dev/null || true)"
|
|
[[ "$result" == match ]] || { printf 'mismatch\n'; return 1; }
|
|
printf 'match\n'
|
|
}
|
|
|
|
_kse_validate_output_dir() {
|
|
local output_dir="$1" physical metadata uid
|
|
[[ "$output_dir" =~ ^/tmp/platform-k3s-encryption\.[A-Za-z0-9]+$ ]] || return 1
|
|
[[ -d /tmp && ! -L /tmp && -d "$output_dir" && ! -L "$output_dir" ]] || return 1
|
|
[[ "$(cd -P -- /tmp && pwd -P)" == /tmp ]] || return 1
|
|
physical="$(cd -P -- "$output_dir" && pwd -P)" || return 1
|
|
[[ "$physical" == "$output_dir" ]] || return 1
|
|
uid="$($KSE_ID -u)" || return 1
|
|
metadata="$($KSE_STAT --format='%u:%a:%F' -- "$output_dir" 2>/dev/null)" || return 1
|
|
[[ "$metadata" == "${uid}:700:directory" ]] || return 1
|
|
[[ -z "$($KSE_FIND "$output_dir" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]
|
|
}
|
|
|
|
_kse_write_verified_output() {
|
|
local output_dir="$1" invoking_uid metadata physical entry
|
|
local -a entries=()
|
|
local directory_fd
|
|
_kse_validate_output_dir "$output_dir" || return 1
|
|
invoking_uid="$($KSE_ID -u)" || return 1
|
|
exec {directory_fd}<"$output_dir" || return 1
|
|
physical="$(cd -P -- "/proc/self/fd/${directory_fd}" && pwd -P)" || { exec {directory_fd}<&-; return 1; }
|
|
[[ "$physical" == "$output_dir" ]] || { exec {directory_fd}<&-; return 1; }
|
|
metadata="$($KSE_STAT -L --format='%u:%a:%F' -- "/proc/self/fd/${directory_fd}" 2>/dev/null)" || { exec {directory_fd}<&-; return 1; }
|
|
[[ "$metadata" == "${invoking_uid}:700:directory" ]] || { exec {directory_fd}<&-; return 1; }
|
|
(
|
|
umask 077
|
|
set -C
|
|
printf 'version=%s\nserver_count=%s\nnode_name=%s\nnode_uid=%s\nnode_ready=%s\ndatastore=%s\nencryption=%s\nrotation=%s\nserver_hashes=%s\nintegrity=%s\napi_readyz=%s\n' \
|
|
"$version" "$server_count" "$node_name" "$node_uid" "$node_ready" "$datastore" "$encryption" "$rotation" "$server_hashes" "$integrity" "$api_readyz" \
|
|
>"/proc/self/fd/${directory_fd}/inventory.env"
|
|
printf '%s\n' "$status_sha256" >"/proc/self/fd/${directory_fd}/status.sha256"
|
|
) || { exec {directory_fd}<&-; return 1; }
|
|
metadata="$($KSE_STAT --format='%u:%a:%F' -- "/proc/self/fd/${directory_fd}/inventory.env" "/proc/self/fd/${directory_fd}/status.sha256" 2>/dev/null)" || { exec {directory_fd}<&-; return 1; }
|
|
[[ "$metadata" == "${invoking_uid}:600:regular file"$'\n'"${invoking_uid}:600:regular file" ]] || { exec {directory_fd}<&-; return 1; }
|
|
while IFS= read -r -d '' entry; do entries+=("${entry##*/}"); done < <(
|
|
"$KSE_FIND" -H "/proc/self/fd/${directory_fd}" -mindepth 1 -maxdepth 1 -print0 2>/dev/null
|
|
)
|
|
exec {directory_fd}<&-
|
|
(( ${#entries[@]} == 2 )) || return 1
|
|
[[ " ${entries[*]} " == *' inventory.env '* && " ${entries[*]} " == *' status.sha256 '* ]]
|
|
}
|
|
|
|
k3s_secret_encryption_main() {
|
|
local expectation='' output_dir='' output_dir_seen=false
|
|
local raw_status canonical_status status_class stage status_hash_line
|
|
local nodes_json service_show annotations='' annotation='' first_annotation=''
|
|
local version server_count node_name node_uid node_ready datastore encryption rotation
|
|
local server_hashes=\mismatch integrity=mismatch api_readyz status_sha256
|
|
|
|
PATH="$KSE_SECURE_PATH"
|
|
export PATH
|
|
unset K3S_CONFIG_DIR K3S_SYSTEMD_DROPIN_DIR
|
|
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--expect-disabled|--expect-transition-start|--expect-enabled|--expect-reencrypted|--expect-reencrypted-restore)
|
|
[[ -z "$expectation" ]] || { kse_usage >&2; return 2; }
|
|
expectation="$1"
|
|
;;
|
|
--verified-output-dir)
|
|
(( $# >= 2 )) || { kse_usage >&2; return 2; }
|
|
! "$output_dir_seen" && [[ -n "$2" ]] || { kse_usage >&2; return 2; }
|
|
output_dir="$2"; output_dir_seen=true; shift
|
|
;;
|
|
--help|-h) kse_usage; return 0 ;;
|
|
*) kse_usage >&2; return 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
_kse_validate_runtime_dependencies || { kse_fail 'trusted runtime dependency validation failed'; return 1; }
|
|
! "$output_dir_seen" || _kse_validate_output_dir "$output_dir" || { kse_fail 'verified output directory is unsafe'; return 1; }
|
|
_kse_validate_sudo_credentials || {
|
|
kse_fail 'noninteractive sudo authorization is unavailable; run sudo -v in the same terminal, then rerun the validator'
|
|
return 1
|
|
}
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$KSE_LIBRARY_PATH"
|
|
version="$(_kse_read_version)" || { kse_fail 'k3s version is not the required production version'; return 1; }
|
|
raw_status="$(_kse_read_status)" || { kse_fail 'read-only encryption status failed'; return 1; }
|
|
canonical_status="$($KSE_JQ -ceS . <<<"$raw_status" 2>/dev/null)" || { kse_fail 'encryption status is invalid'; return 1; }
|
|
status_class="$(classify_encryption_status "$canonical_status")"
|
|
[[ "$status_class" != invalid ]] || { kse_fail 'encryption status is invalid'; return 1; }
|
|
status_hash_line="$(printf '%s' "$canonical_status" | "$KSE_SHA256SUM")" || return 1
|
|
status_sha256="${status_hash_line%% *}"
|
|
[[ "$status_sha256" =~ ^[0-9a-f]{64}$ ]] || return 1
|
|
|
|
encryption=Disabled; rotation=unsafe
|
|
case "$status_class" in
|
|
disabled_no_config) rotation=none ;;
|
|
transition_start) rotation=start ;;
|
|
enabled_stable)
|
|
encryption=Enabled
|
|
stage="$($KSE_JQ -er '.stage' <<<"$canonical_status")" || return 1
|
|
[[ "$stage" == start || "$stage" == reencrypt_finished ]] || return 1
|
|
rotation="$stage"
|
|
;;
|
|
unsafe_transition|hash_mismatch) encryption=Enabled ;;
|
|
*) return 1 ;;
|
|
esac
|
|
|
|
nodes_json="$(_kse_read_nodes)" || { kse_fail 'read-only node inventory failed'; return 1; }
|
|
server_count="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))] | length' <<<"$nodes_json" 2>/dev/null)" || return 1
|
|
[[ "$server_count" == 1 ]] || { kse_fail 'server count is not exactly one'; return 1; }
|
|
node_name="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))][0].metadata.name' <<<"$nodes_json" 2>/dev/null)" || return 1
|
|
node_uid="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))][0].metadata.uid | if type == "string" and test("^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") then . else error("invalid node uid") end' <<<"$nodes_json" 2>/dev/null)" || { kse_fail 'server node UID is invalid'; return 1; }
|
|
if [[ "$expectation" != --expect-reencrypted-restore ]]; then
|
|
[[ "$node_name" == "$KSE_EXPECTED_NODE" ]] || { kse_fail 'server node identity mismatch'; return 1; }
|
|
fi
|
|
node_ready="$($KSE_JQ -er '[.items[] | select(.metadata.name == $name)][0].status.conditions | [ .[] | select(.type == "Ready") ] | if length == 1 and .[0].status == "True" then "Ready" else error("not ready") end' --arg name "$node_name" <<<"$nodes_json" 2>/dev/null)" || { kse_fail 'server node is not Ready'; return 1; }
|
|
|
|
annotations="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null)) | .metadata.annotations["k3s.io/encryption-config-hash"]?] | if length > 0 and all(.[]; type == "string" and test("^[a-z_]+-[0-9a-f]{64}$")) then .[] else empty end' <<<"$nodes_json" 2>/dev/null)" || annotations=''
|
|
if [[ -n "$annotations" ]]; then
|
|
first_annotation="${annotations%%$'\n'*}"; annotation="$first_annotation"
|
|
while IFS= read -r annotation; do [[ "$annotation" == "$first_annotation" ]] || first_annotation=''; done <<<"$annotations"
|
|
[[ -z "$first_annotation" ]] || server_hashes=match
|
|
fi
|
|
|
|
service_show="$(_kse_read_systemctl_show)" || { kse_fail 'read-only systemd inventory failed'; return 1; }
|
|
datastore="$(_kse_detect_datastore "$service_show")" || datastore=ambiguous
|
|
case "$datastore" in sqlite|embedded-etcd|external|ambiguous) ;; *) datastore=ambiguous ;; esac
|
|
|
|
if [[ "$server_hashes" == match && ( "$status_class" == transition_start || "$status_class" == enabled_stable ) ]]; then
|
|
integrity="$(_kse_integrity_from_root_evidence "$first_annotation" "$rotation" 2>/dev/null || true)"
|
|
[[ "$integrity" == match ]] || integrity=mismatch
|
|
fi
|
|
if _kse_api_ready; then api_readyz=pass; else api_readyz=fail; fi
|
|
|
|
printf 'K3s version: %s\nServer count: %s\nNode: %s %s\nDatastore: %s\nEncryption status: %s\nRotation stage: %s\nServer annotation hashes: %s\nLocal config/state/annotation integrity: %s\nAPI readyz: %s\n' \
|
|
"$version" "$server_count" "$node_name" "$node_ready" "$datastore" "$encryption" "$rotation" "$server_hashes" "$integrity" "$api_readyz"
|
|
|
|
[[ "$api_readyz" == pass ]] || return 1
|
|
case "$expectation" in
|
|
'') [[ "$status_class" != unsafe_transition && "$status_class" != hash_mismatch ]] || return 1 ;;
|
|
--expect-disabled) [[ "$status_class" == disabled_no_config ]] || return 1 ;;
|
|
--expect-transition-start) [[ "$status_class" == transition_start ]] || return 1 ;;
|
|
--expect-enabled) [[ "$status_class" == enabled_stable ]] || return 1 ;;
|
|
--expect-reencrypted|--expect-reencrypted-restore)
|
|
[[ "$status_class" == enabled_stable && "$rotation" == reencrypt_finished &&
|
|
"$server_hashes" == match && "$integrity" == match ]] || return 1
|
|
;;
|
|
esac
|
|
! "$output_dir_seen" || _kse_write_verified_output "$output_dir" || { kse_fail 'verified output write failed'; return 1; }
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
k3s_secret_encryption_main "$@"
|
|
fi
|