Add platform infrastructure configuration

This commit is contained in:
donghyeon-ka
2026-08-28 17:35:41 +09:00
parent fa76531e5b
commit 16c337bcc9
302 changed files with 83259 additions and 1 deletions
+266
View File
@@ -0,0 +1,266 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set +x
umask 077
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly PG_PATH="/srv/k3s/ssd/pgadmin"
readonly PG_SECRET="pgadmin-bootstrap"
readonly PG_OIDC_SECRET="pgadmin-keycloak-oidc"
readonly AI_OIDC_SECRET="aistor-keycloak-oidc"
readonly NAMES="admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin"
execute=false
password_file=""
work=""
rollback_armed=false
bootstrap_existed=false
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
사용법:
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/bootstrap/apply-admin-services.sh
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/bootstrap/apply-admin-services.sh \
--execute \
--pgadmin-password-file /home/donghyeon/.secrets/pgadmin/bootstrap-password
인자 없이 실행하면 공식 차트 hash와 manifest만 검증합니다.
--execute는 Local PV, pgAdmin bootstrap Secret, pgAdmin, AIStor OIDC profile을
적용합니다. 비밀번호와 OIDC Secret 값은 출력하지 않습니다.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--execute)
execute=true
shift
;;
--pgadmin-password-file)
(( $# >= 2 )) || fail "--pgadmin-password-file 값이 필요합니다"
password_file="$2"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "지원하지 않는 인자: $1"
;;
esac
done
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"
for cmd in awk curl find findmnt install jq kubectl mktemp od rg seq sha256sum sleep sort stat tail tr wc; do
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
work="$(mktemp -d /tmp/platform-admin-apply.XXXXXX)"
rollback() {
set +e
printf '\nROLLBACK: 관리 UI 트래픽을 내리고 AIStor 이전 spec을 복원합니다.\n' >&2
kubectl -n platform-admin scale deployment/pgadmin --replicas=0 >/dev/null 2>&1 || true
kubectl -n platform-admin delete ingress pgadmin --ignore-not-found >/dev/null 2>&1
kubectl -n object-storage delete ingress minio-aistor-console --ignore-not-found >/dev/null 2>&1
kubectl apply -f "$work/objectstore-before.json" >/dev/null 2>&1 || true
for statefulset_name in $(
kubectl -n object-storage get objectstore minio-aistor \
-o jsonpath='{range .status.pools[*]}{.ssName}{"\n"}{end}' 2>/dev/null
); do
kubectl -n object-storage rollout restart "statefulset/$statefulset_name" >/dev/null 2>&1 || true
kubectl -n object-storage rollout status "statefulset/$statefulset_name" \
--timeout=300s >/dev/null 2>&1 || true
done
if [[ "$bootstrap_existed" == true ]]; then
kubectl apply -f "$work/pgadmin-bootstrap-before.yaml" >/dev/null 2>&1 || true
else
kubectl -n platform-admin delete secret "$PG_SECRET" --ignore-not-found >/dev/null 2>&1
fi
rollback_armed=false
printf 'ROLLBACK complete. pgAdmin PVC/PV와 AIStor 데이터는 삭제하지 않았습니다.\n' >&2
}
cleanup() {
rc=$?
trap - EXIT INT TERM
if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then
rollback
fi
case "$work" in
/tmp/platform-admin-apply.*) rm -rf -- "$work" ;;
esac
exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
unexpected_error() {
rc=$?
line="$1"
trap - ERR
printf 'ERROR: 예상하지 못한 명령 실패(line=%s, exit=%s)\n' "$line" "$rc" >&2
exit "$rc"
}
trap 'unexpected_error "$LINENO"' ERR
if [[ ! -v PLATFORM_HELM_BIN ]]; then
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm
fi
PLATFORM_HELM_BIN="$PLATFORM_HELM_BIN" \
bash "$ROOT/scripts/validate/render-admin-services.sh" --verified-output-dir "$work"
for name in $NAMES; do
file="$work/$name.yaml"
[[ -f "$file" && ! -L "$file" && -O "$file" && -s "$file" ]] || fail "안전하지 않은 handoff: $file"
[[ "$(stat -c '%a' "$file")" == 600 ]] || fail "handoff 권한이 0600이 아닙니다"
sha256sum "$file" | awk '{print $1}' >"$work/$name.sha256"
done
context="$(kubectl config current-context)"
api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
ready="$(kubectl get node "$TARGET_NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')"
[[ "$ready" == True ]] || fail "대상 노드가 Ready가 아닙니다"
printf 'Current context: %s\n' "$context"
printf 'API server: %s\n' "$api"
for name in $NAMES; do
printf 'SHA-256 %-20s %s\n' "$name" "$(cat "$work/$name.sha256")"
done
if [[ "$execute" == false ]]; then
printf 'DRY RUN PASS: --execute를 지정하지 않아 클러스터를 변경하지 않았습니다.\n'
exit 0
fi
[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
[[ "$password_file" == /* && -f "$password_file" && ! -L "$password_file" ]] || \
fail "pgAdmin 비밀번호 파일은 일반 파일인 절대 경로여야 합니다"
[[ "$(stat -c '%a' "$password_file")" == 600 ]] || fail "pgAdmin 비밀번호 파일 권한은 0600이어야 합니다"
password_bytes="$(wc -c <"$password_file" | tr -d '[:space:]')"
(( password_bytes >= 16 && password_bytes <= 256 )) || fail "pgAdmin 비밀번호는 16~256 bytes여야 합니다"
last_byte="$(tail -c 1 "$password_file" | od -An -t x1 | tr -d '[:space:]')"
[[ "$last_byte" != 0a && "$last_byte" != 0d ]] || \
fail "pgAdmin 비밀번호 파일 끝에 개행이 없어야 합니다"
for contract in "object-storage $AI_OIDC_SECRET client-id client-secret" \
"platform-admin $PG_OIDC_SECRET client-id client-secret"; do
set -- $contract
namespace="$1"
secret="$2"
key_a="$3"
key_b="$4"
kubectl -n "$namespace" get "secret/$secret" >/dev/null 2>&1 || \
fail "$namespace/$secret Secret이 없습니다. Keycloak OIDC 구성을 먼저 실행하세요"
keys="$(kubectl -n "$namespace" get "secret/$secret" \
-o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)"
[[ "$keys" == "$key_a"$'\n'"$key_b" ]] || fail "$namespace/$secret key 계약이 다릅니다"
done
core_data="$(kubectl -n kube-system get configmap coredns-custom \
-o jsonpath='{.data.learn-hyeonworks\.server}' 2>/dev/null)" || \
fail "coredns-custom이 없습니다. apply-private-dns.sh를 먼저 실행하세요"
for host in git.learn.hyeonworks.com id.learn.hyeonworks.com \
storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com; do
printf '%s\n' "$core_data" | rg -q -F "$host" || fail "CoreDNS에 $host가 없습니다"
done
printf 'Type APPLY %s to deploy private admin services: ' "$context"
read -r answer
[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다"
[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다"
[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || \
fail "API server가 바뀌었습니다"
for name in $NAMES; do
current_sha="$(sha256sum "$work/$name.yaml" | awk '{print $1}')"
[[ "$current_sha" == "$(cat "$work/$name.sha256")" ]] || fail "$name handoff가 확인 뒤 바뀌었습니다"
done
printf '[1/6] sudo 인증 확인\n'
sudo -v
printf '[2/6] pgAdmin SSD filesystem 경계와 Local PV 경로 확인\n'
root_source="$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target /)"
srv_source="$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target /srv)"
srv_target="$(sudo findmnt --kernel --first-only --noheadings --output TARGET --target /srv)"
[[ "$srv_source" == "$root_source" && "$srv_target" == / ]] || \
fail "/srv가 root SSD 파일시스템 경계가 아닙니다"
if sudo test -L "$PG_PATH"; then
fail "$PG_PATH가 심볼릭 링크입니다"
fi
if sudo test -e "$PG_PATH"; then
sudo test -d "$PG_PATH" || fail "$PG_PATH가 디렉터리가 아닙니다"
else
sudo install -d -o root -g root -m 0750 "$PG_PATH"
fi
[[ "$(sudo findmnt --kernel --first-only --noheadings --output SOURCE --target "$PG_PATH")" == "$root_source" ]] || \
fail "$PG_PATH가 root SSD에 있지 않습니다"
[[ "$(sudo findmnt --kernel --first-only --noheadings --output TARGET --target "$PG_PATH")" == / ]] || \
fail "$PG_PATH 아래에 다른 mount가 있습니다"
printf '[3/6] rollback용 AIStor spec과 기존 pgAdmin Secret 상태 저장\n'
kubectl -n object-storage get objectstore minio-aistor -o json | \
jq '{apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace},spec}' \
>"$work/objectstore-before.json"
if kubectl -n platform-admin get secret "$PG_SECRET" -o yaml >"$work/pgadmin-bootstrap-before.yaml" 2>/dev/null; then
bootstrap_existed=true
fi
rollback_armed=true
printf '[4/6] namespace, Local PV, Secret, AIStor OIDC와 pgAdmin 적용\n'
kubectl apply -f "$work/admin-namespace.yaml"
kubectl apply -f "$work/pgadmin-local-pv.yaml"
kubectl -n platform-admin create secret generic "$PG_SECRET" \
--from-file="password=$password_file" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
kubectl -n platform-admin label secret "$PG_SECRET" \
app.kubernetes.io/name=pgadmin4 \
app.kubernetes.io/component=bootstrap-credential \
app.kubernetes.io/part-of=platform \
app.kubernetes.io/managed-by=bootstrap-script \
--overwrite >/dev/null
kubectl apply -f "$work/aistor-admin-oidc.yaml"
kubectl apply -f "$work/pgadmin.yaml"
printf '[5/6] pgAdmin rollout, EndpointSlice와 AIStor health 대기\n'
kubectl -n platform-admin rollout status deployment/pgadmin --timeout=300s
kubectl -n platform-admin wait --for=jsonpath='{.endpoints[0].conditions.ready}'=true \
endpointslice --selector=kubernetes.io/service-name=pgadmin --timeout=120s
healthy=false
for attempt in $(seq 1 60); do
health="$(kubectl -n object-storage get objectstore minio-aistor -o jsonpath='{.status.healthStatus}' 2>/dev/null)"
if [[ "$health" == green ]]; then
healthy=true
break
fi
if (( attempt == 1 || attempt % 3 == 0 )); then
printf 'AIStor health 대기: status=%s elapsed=%ss/300s\n' \
"${health:-unknown}" "$((attempt * 5))"
fi
sleep 5
done
[[ "$healthy" == true ]] || fail "AIStor ObjectStore가 green으로 복귀하지 않았습니다"
printf '[6/6] Traefik Host routing 확인\n'
pg_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--noproxy '*' -H 'Host: db-admin.learn.hyeonworks.com' http://127.0.0.1:30080/)"
ai_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--noproxy '*' -H 'Host: storage-admin.learn.hyeonworks.com' http://127.0.0.1:30080/)"
[[ "$pg_code" == 200 || "$pg_code" == 302 || "$pg_code" == 303 ]] || fail "pgAdmin Traefik 응답 실패: $pg_code"
[[ "$ai_code" == 200 || "$ai_code" == 302 || "$ai_code" == 303 ]] || fail "AIStor Console Traefik 응답 실패: $ai_code"
rollback_armed=false
printf 'ADMIN SERVICES APPLY SUCCESS\n'
printf 'pgAdmin PVC/PV: Retain, AIStor S3 API: cluster-internal only\n'
+470
View File
@@ -0,0 +1,470 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# 호출자가 bash -x로 실행해도 Secret 경로와 향후 입력이 추적되지 않도록 한다.
set +x
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly EXPECTED_API_SERVICE_IP="10.43.0.1"
readonly EXPECTED_API_ENDPOINT_IP="192.168.0.107"
readonly EXPECTED_AISTOR_DEVICE="/dev/sdb3"
readonly EXPECTED_AISTOR_MOUNT="/srv/k3s/aistor"
readonly EXPECTED_STORAGE_CLASS="aistor-local-xfs-retain"
readonly EXPECTED_PV="aistor-data-local-pv"
readonly EXPECTED_OBJECTSTORE="minio-aistor"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly -a VERIFIED_MANIFEST_NAMES=(
phase2-namespaces
aistor-local-pv
keycloak-operator
platform-postgres-keycloak
keycloak
aistor-operator
minio-aistor
aistor-network-policies
)
readonly -a AISTOR_CRDS=(
customresourcedefinition/adminjobs.aistor.min.io
customresourcedefinition/objectstores.aistor.min.io
customresourcedefinition/policybindings.sts.min.io
)
license_file=""
root_config_file=""
generate_root_config=false
execute_requested=false
mutation_started=false
current_step="preflight"
report_retained_state() {
if [[ "$mutation_started" == true ]]; then
printf '%s\n' \
"SAFE STOP during ${current_step}." \
'No Namespace, Secret, PV, PVC, Operator, ObjectStore, or XFS data was deleted.' \
'The PV reclaim policy remains Retain and PVC protection remains enabled.' \
'Diagnose the failed wait or policy, then rerun this script.' >&2
fi
}
fail() {
printf 'ERROR: %s\n' "$*" >&2
report_retained_state
exit 1
}
on_error() {
local status="$1"
local line="$2"
trap - ERR
set +e
printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2
report_retained_state
exit "$status"
}
on_signal() {
local status="$1"
trap - INT TERM
set +e
printf 'INTERRUPTED: stopping without deleting cluster or XFS state.\n' >&2
report_retained_state
exit "$status"
}
usage() {
cat <<'USAGE'
Usage:
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/bootstrap/apply-aistor.sh \
--license-file /home/donghyeon/.secrets/aistor/minio.license \
--root-config-file /home/donghyeon/.secrets/aistor/root.env \
--generate-root-config \
--execute
Renders, verifies, and applies the internal-only MinIO AIStor path:
aistor and object-storage namespaces
two out-of-Git Secret contracts
one 900Gi Retain Local PV on /srv/k3s/aistor
AIStor Operator 5.10.0 and CRDs
one-server, one-drive ObjectStore 1.0.16
default-deny NetworkPolicies with only required internal paths
It does not configure Host Nginx, Traefik Ingress, NodePort, LoadBalancer,
public DNS, credential rotation, or deletion.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--license-file)
(( $# >= 2 )) || fail "--license-file requires a path"
license_file="$2"
shift 2
;;
--root-config-file)
(( $# >= 2 )) || fail "--root-config-file requires a path"
root_config_file="$2"
shift 2
;;
--generate-root-config)
generate_root_config=true
shift
;;
--execute)
execute_requested=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
[[ "$execute_requested" == true ]] || {
usage >&2
exit 2
}
[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path"
[[ "$root_config_file" == /* ]] || \
fail "--root-config-file must be an absolute path"
for command_name in cmp curl df find findmnt jq kubectl mktemp mountpoint \
rg sed seq sha256sum sleep stat tail tr wc; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then
[[ "$PLATFORM_HELM_BIN" == /* ]] || \
fail "PLATFORM_HELM_BIN must be an absolute path"
[[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}"
readonly HELM_BIN="$PLATFORM_HELM_BIN"
else
HELM_BIN="$(command -v helm 2>/dev/null)" || \
fail "Helm ${EXPECTED_HELM_VERSION} is required"
readonly HELM_BIN
fi
[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \
fail "Helm must be exactly ${EXPECTED_HELM_VERSION}"
kustomize_version="$(
kubectl version --client --output=yaml |
sed -n 's/^kustomizeVersion: //p'
)"
[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}"
umask 077
render_temp_dir="$(mktemp -d /tmp/platform-phase2-apply.XXXXXX)"
cleanup() {
case "$render_temp_dir" in
/tmp/platform-phase2-apply.*)
rm -rf -- "$render_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render directory: %s\n' \
"$render_temp_dir" >&2
;;
esac
}
trap cleanup EXIT
trap 'on_error "$?" "$LINENO"' ERR
trap 'on_signal 130' INT
trap 'on_signal 143' TERM
cd -- "$REPOSITORY_ROOT"
PLATFORM_HELM_BIN="$HELM_BIN" \
bash scripts/validate/render-phase2.sh \
--verified-output-dir "$render_temp_dir"
declare -A verified_manifest_sha256=()
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
manifest_path="${render_temp_dir}/${manifest_name}.yaml"
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest is missing or unsafe: ${manifest_path}"
[[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \
fail "verified manifest must have mode 0600: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}"
done
verified_entry_count="$(
find "$render_temp_dir" -mindepth 1 -maxdepth 1 |
wc -l | tr -d '[:space:]'
)"
[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \
fail "verified handoff must contain exactly eight manifest files"
verify_manifest_unchanged() {
local manifest_name="$1"
local manifest_path="${render_temp_dir}/${manifest_name}.yaml"
local checksum_output
local actual_sha256
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest became missing or unsafe: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
actual_sha256="${checksum_output%% *}"
[[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \
fail "verified manifest changed before apply: ${manifest_name}.yaml"
}
current_context="$(kubectl config current-context)"
api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')"
node_ready="$(
kubectl get node "$TARGET_NODE" \
--output='go-template={{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}true{{end}}{{end}}'
)"
[[ "$node_ready" == "true" ]] || fail "target node is not Ready: ${TARGET_NODE}"
cluster_api_ip="$(
kubectl --namespace default get service kubernetes \
--output=jsonpath='{.spec.clusterIP}'
)"
[[ "$cluster_api_ip" == "$EXPECTED_API_SERVICE_IP" ]] || \
fail "Kubernetes API Service IP changed: expected ${EXPECTED_API_SERVICE_IP}, found ${cluster_api_ip}"
cluster_api_endpoint="$(
kubectl --namespace default get endpointslice \
--selector=kubernetes.io/service-name=kubernetes \
--output=jsonpath='{.items[0].endpoints[0].addresses[0]}'
)"
[[ "$cluster_api_endpoint" == "$EXPECTED_API_ENDPOINT_IP" ]] || \
fail "Kubernetes API endpoint changed: expected ${EXPECTED_API_ENDPOINT_IP}, found ${cluster_api_endpoint}"
mountpoint --quiet "$EXPECTED_AISTOR_MOUNT" || \
fail "${EXPECTED_AISTOR_MOUNT} is not a mountpoint"
mount_source="$(findmnt --noheadings --output SOURCE --target "$EXPECTED_AISTOR_MOUNT" | tr -d '[:space:]')"
mount_fstype="$(findmnt --noheadings --output FSTYPE --target "$EXPECTED_AISTOR_MOUNT" | tr -d '[:space:]')"
mount_options="$(findmnt --noheadings --output OPTIONS --target "$EXPECTED_AISTOR_MOUNT")"
[[ "$mount_source" == "$EXPECTED_AISTOR_DEVICE" ]] || \
fail "AIStor mount source changed: expected ${EXPECTED_AISTOR_DEVICE}, found ${mount_source}"
[[ "$mount_fstype" == "xfs" ]] || \
fail "AIStor mount must be XFS, found ${mount_fstype}"
[[ ",${mount_options}," == *,rw,* ]] || fail "AIStor XFS mount is not writable"
[[ -d "$EXPECTED_AISTOR_MOUNT" && ! -L "$EXPECTED_AISTOR_MOUNT" ]] || \
fail "AIStor mount path must be a non-symlink directory"
available_bytes="$(
df --block-size=1 --output=avail "$EXPECTED_AISTOR_MOUNT" |
tail -n 1 | tr -d '[:space:]'
)"
minimum_bytes=$((900 * 1024 * 1024 * 1024))
(( available_bytes >= minimum_bytes )) || \
fail "AIStor XFS has less than 900Gi available"
if ! kubectl get persistentvolume "$EXPECTED_PV" >/dev/null 2>&1; then
[[ -z "$(find "$EXPECTED_AISTOR_MOUNT" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \
fail "initial AIStor XFS root is not empty; refusing to bind an unknown data directory"
fi
unexpected_consumers="$(
kubectl get persistentvolumeclaim --all-namespaces --output=json |
jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \
'[.items[] | select(.spec.storageClassName == $storage_class)] | length'
)"
if (( unexpected_consumers > 0 )); then
existing_expected_claim="$(
kubectl --namespace object-storage get persistentvolumeclaim \
--ignore-not-found --output=json |
jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \
'[.items[] | select(.spec.storageClassName == $storage_class)] | length'
)"
[[ "$unexpected_consumers" == "1" && "$existing_expected_claim" == "1" ]] || \
fail "the AIStor StorageClass has an unexpected PVC consumer"
fi
if kubectl get customresourcedefinition objectstores.aistor.min.io >/dev/null 2>&1; then
unexpected_objectstores="$(
kubectl get objectstores.aistor.min.io --all-namespaces --output=json |
jq --arg name "$EXPECTED_OBJECTSTORE" \
'[.items[] | select(.metadata.namespace != "object-storage" or .metadata.name != $name)] | length'
)"
[[ "$unexpected_objectstores" == "0" ]] || \
fail "an unexpected AIStor ObjectStore already exists"
fi
[[ -f "$license_file" && ! -L "$license_file" && -O "$license_file" && -s "$license_file" ]] || \
fail "license file must be a non-empty, current-user-owned regular file"
[[ "$(stat --format='%a' -- "$license_file")" == "600" ]] || \
fail "license file must have mode 0600"
printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \
"$current_context" "$api_server" "$TARGET_NODE"
printf 'XFS: %s -> %s (%s, at least 900Gi available)\n' \
"$mount_source" "$EXPECTED_AISTOR_MOUNT" "$mount_fstype"
printf '%s\n' \
'Scope: internal-only AIStor Operator, 900Gi Retain Local PV, one ObjectStore, and NetworkPolicies.' \
'Excluded: Host Nginx, Traefik, NodePort, LoadBalancer, public DNS, rotation, and deletion.' \
'Failure boundary: all applied state and XFS data are retained; rerunning is the recovery path.'
[[ -t 0 ]] || fail "an interactive terminal is required"
printf 'Type APPLY AISTOR %s to start the cluster mutation: ' "$current_context"
read -r confirmation
[[ "$confirmation" == "APPLY AISTOR ${current_context}" ]] || fail "cancelled"
assert_cluster_identity() {
[[ "$(kubectl config current-context)" == "$current_context" ]] || \
fail "kubectl context changed after confirmation"
[[ "$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" == "$api_server" ]] || \
fail "Kubernetes API server changed after confirmation"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node disappeared after confirmation: ${TARGET_NODE}"
mountpoint --quiet "$EXPECTED_AISTOR_MOUNT" || \
fail "AIStor XFS mount disappeared after confirmation"
}
assert_cluster_identity
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
verify_manifest_unchanged "$manifest_name"
done
mutation_started=true
current_step="[1/7] AIStor namespaces"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged phase2-namespaces
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/phase2-namespaces.yaml" >/dev/null
kubectl apply --filename="${render_temp_dir}/phase2-namespaces.yaml"
current_step="[2/7] AIStor Secret contracts"
printf '\n%s\n' "$current_step"
assert_cluster_identity
secret_args=(
--license-file "$license_file"
--root-config-file "$root_config_file"
--execute
)
if [[ "$generate_root_config" == true ]]; then
secret_args+=(--generate-root-config)
fi
bash scripts/bootstrap/create-aistor-secrets.sh "${secret_args[@]}"
current_step="[3/7] 900Gi Retain Local PV"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged aistor-local-pv
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/aistor-local-pv.yaml" >/dev/null
kubectl apply --filename="${render_temp_dir}/aistor-local-pv.yaml"
current_step="[4/7] AIStor Operator and CRDs"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged aistor-operator
kubectl apply --server-side \
--filename="${render_temp_dir}/aistor-operator.yaml"
kubectl wait --for=condition=Established "${AISTOR_CRDS[@]}" --timeout=5m
kubectl --namespace aistor rollout status \
deployment/adminjob-operator --timeout=10m
kubectl --namespace aistor rollout status \
deployment/object-store-operator --timeout=10m
kubectl --namespace aistor rollout status \
deployment/object-store-webhook --timeout=10m
current_step="[5/7] One-node, one-drive AIStor ObjectStore"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged minio-aistor
kubectl apply --server-side --dry-run=server \
--filename="${render_temp_dir}/minio-aistor.yaml" >/dev/null
kubectl apply --server-side \
--filename="${render_temp_dir}/minio-aistor.yaml"
statefulset_name=""
for _ in $(seq 1 180); do
statefulset_names="$(
kubectl --namespace object-storage get statefulset \
--selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \
--output=name
)"
statefulset_count="$(printf '%s\n' "$statefulset_names" | sed '/^$/d' | wc -l | tr -d '[:space:]')"
if [[ "$statefulset_count" == "1" ]]; then
statefulset_name="$statefulset_names"
break
fi
sleep 2
done
[[ -n "$statefulset_name" ]] || \
fail "the ObjectStore Operator did not create exactly one StatefulSet"
kubectl --namespace object-storage rollout status "$statefulset_name" --timeout=15m
kubectl --namespace object-storage wait \
--for=condition=Ready pod \
--selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \
--timeout=10m
current_step="[6/7] AIStor default-deny NetworkPolicies"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged aistor-network-policies
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/aistor-network-policies.yaml" >/dev/null
kubectl apply \
--filename="${render_temp_dir}/aistor-network-policies.yaml"
kubectl --namespace object-storage wait \
--for=condition=Ready pod \
--selector="aistor.min.io/objectStore=${EXPECTED_OBJECTSTORE}" \
--timeout=5m
current_step="[7/7] Storage, service, and exposure acceptance"
printf '\n%s\n' "$current_step"
assert_cluster_identity
pvc_json="$(
kubectl --namespace object-storage get persistentvolumeclaim --output=json |
jq --arg storage_class "$EXPECTED_STORAGE_CLASS" \
'{apiVersion, kind, items: [.items[] | select(.spec.storageClassName == $storage_class)]}'
)"
[[ "$(jq '.items | length' <<<"$pvc_json")" == "1" ]] || \
fail "expected exactly one AIStor PVC"
[[ "$(jq -r '.items[0].status.phase' <<<"$pvc_json")" == "Bound" ]] || \
fail "AIStor PVC is not Bound"
[[ "$(jq -r '.items[0].spec.volumeName' <<<"$pvc_json")" == "$EXPECTED_PV" ]] || \
fail "AIStor PVC did not bind the expected Local PV"
[[ "$(jq -r '.items[0].spec.resources.requests.storage' <<<"$pvc_json")" == "900Gi" ]] || \
fail "AIStor PVC request is not 900Gi"
for service_name in minio minio-aistor-console minio-aistor-hl; do
service_type="$(
kubectl --namespace object-storage get service "$service_name" \
--output=jsonpath='{.spec.type}'
)"
[[ "$service_type" == "ClusterIP" ]] || \
fail "${service_name} must remain ClusterIP"
done
[[ "$(
kubectl --namespace object-storage get service minio \
--output=jsonpath='{.spec.ports[0].port}:{.spec.ports[0].targetPort}'
)" == "80:9000" ]] || fail "S3 Service must map 80/TCP to 9000/TCP"
[[ "$(
kubectl --namespace object-storage get service minio-aistor-console \
--output=jsonpath='{.spec.ports[0].port}:{.spec.ports[0].targetPort}'
)" == "9090:9090" ]] || fail "Console Service must map 9090/TCP to 9090/TCP"
[[ "$(
kubectl --namespace object-storage get service minio-aistor-hl \
--output=jsonpath='{.spec.clusterIP}'
)" == "None" ]] || fail "AIStor headless Service must remain headless"
[[ -z "$(
kubectl --namespace object-storage get service --output=json |
jq -r '.items[].spec.ports[]? | select(.nodePort != null) | .nodePort'
)" ]] || fail "an AIStor service unexpectedly has a NodePort"
[[ -z "$(kubectl --namespace object-storage get ingress --output=name)" ]] || \
fail "AIStor must not have an Ingress"
printf '\nAISTOR APPLY SUCCESS\n'
printf 'ObjectStore: object-storage/%s\n' "$EXPECTED_OBJECTSTORE"
printf 'Storage: %s -> %s (900Gi PVC, Retain)\n' \
"$EXPECTED_AISTOR_DEVICE" "$EXPECTED_AISTOR_MOUNT"
printf '%s\n' \
'Exposure: ClusterIP only; no Host Nginx, Traefik, NodePort, or public DNS.' \
"Root credential file: ${root_config_file}" \
'Next: run the authenticated S3 write/read smoke test.'
+673
View File
@@ -0,0 +1,673 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# Do not inherit xtrace: redirect headers contain an OIDC state value.
set +x
umask 077
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly EXPECTED_NODE_INTERNAL_IP="192.168.0.107"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly GITEA_NAMESPACE="gitea"
readonly GITEA_DEPLOYMENT="gitea"
readonly GITEA_SERVICE="gitea-http"
readonly GITEA_HOST="git.learn.hyeonworks.com"
readonly GITEA_ROOT_URL="https://${GITEA_HOST}"
readonly GITEA_HEALTH_URL="${GITEA_ROOT_URL}/api/healthz"
readonly GITEA_LOGIN_URL="${GITEA_ROOT_URL}/user/login"
readonly GITEA_SIGNUP_URL="${GITEA_ROOT_URL}/user/sign_up"
readonly GITEA_OIDC_START_URL="${GITEA_ROOT_URL}/user/oauth2/keycloak"
readonly GITEA_OIDC_CALLBACK_ENCODED="https%3A%2F%2Fgit.learn.hyeonworks.com%2Fuser%2Foauth2%2Fkeycloak%2Fcallback"
readonly KEYCLOAK_HOST="id.learn.hyeonworks.com"
readonly KEYCLOAK_ISSUER="https://${KEYCLOAK_HOST}/realms/hyeonworks"
readonly KEYCLOAK_DISCOVERY_URL="${KEYCLOAK_ISSUER}/.well-known/openid-configuration"
readonly KEYCLOAK_AUTH_ENDPOINT="${KEYCLOAK_ISSUER}/protocol/openid-connect/auth"
readonly OIDC_SECRET_NAME="gitea-keycloak-oidc"
readonly -a VERIFIED_MANIFEST_NAMES=(
namespaces
ssd-local-pv
cnpg-operator
platform-postgres
gitea
gitea-oidc
)
render_temp_dir=""
runtime_temp_dir=""
gitea_manifest_sha256="not-rendered"
apply_started=0
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
Usage: bash scripts/bootstrap/apply-gitea-oidc.sh --execute
Keycloak 공개 discovery, Gitea OIDC Secret 계약, 현재 Gitea 상태를 먼저
검사합니다. 고정된 Chart SHA를 검증하는 render-phase1.sh의 0600 handoff에서
gitea-oidc.yaml 하나만 적용한 뒤 OIDC, 외부 인증 전용 가입 정책, 브랜딩을 확인합니다.
실패해도 Kubernetes 리소스를 삭제하거나 이전 버전으로 롤백하지 않습니다.
Secret 값, 토큰, OIDC state가 포함된 전체 Location은 출력하지 않습니다.
USAGE
}
cleanup() {
local cleanup_rc=$?
trap - EXIT
set +e
if (( cleanup_rc != 0 )); then
if (( apply_started == 1 )); then
printf '\nRETAINED STATE: Gitea apply가 시작된 뒤 검증에 실패했습니다.\n' >&2
printf '자동 삭제와 롤백은 수행하지 않았으며 현재 클러스터 상태를 보존했습니다.\n' >&2
printf '적용 대상으로 고정했던 gitea-oidc.yaml SHA-256: %s\n' \
"$gitea_manifest_sha256" >&2
printf '확인: kubectl --namespace gitea get deployment,pod,service,endpointslice,ingress\n' >&2
else
printf '\nNO MUTATION: 사전 검사 또는 렌더링 단계에서 중단되어 Gitea를 적용하지 않았습니다.\n' >&2
fi
fi
if [[ -n "$render_temp_dir" ]]; then
case "$render_temp_dir" in
/tmp/platform-phase1-apply.*)
rm -rf -- "$render_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render path: %s\n' \
"$render_temp_dir" >&2
;;
esac
fi
if [[ -n "$runtime_temp_dir" ]]; then
case "$runtime_temp_dir" in
/tmp/gitea-oidc-apply.*)
rm -rf -- "$runtime_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected runtime path: %s\n' \
"$runtime_temp_dir" >&2
;;
esac
fi
exit "$cleanup_rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
[[ -t 0 ]] || fail "an interactive terminal is required"
[[ "$(pwd -P)" == "$REPOSITORY_ROOT" ]] || \
fail "run from ${REPOSITORY_ROOT}"
for command_name in \
kubectl curl jq rg sha256sum stat find wc tr sort mktemp chmod mkdir rm awk; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then
[[ "$PLATFORM_HELM_BIN" == /* ]] || \
fail "PLATFORM_HELM_BIN must be an absolute path"
[[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN is not executable: ${PLATFORM_HELM_BIN}"
readonly HELM_BIN="$PLATFORM_HELM_BIN"
else
HELM_BIN="$(command -v helm 2>/dev/null)" || \
fail "Helm ${EXPECTED_HELM_VERSION} is required"
readonly HELM_BIN
fi
[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \
fail "Helm must be exactly ${EXPECTED_HELM_VERSION}"
readonly CURRENT_CONTEXT="$(kubectl config current-context)"
readonly API_SERVER="$(
kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}'
)"
[[ -n "$CURRENT_CONTEXT" ]] || fail "kubectl current-context is empty"
[[ -n "$API_SERVER" ]] || fail "the selected Kubernetes API server is empty"
https_get() {
local host="$1"
local url="$2"
local output_file="$3"
curl \
--disable \
--silent \
--show-error \
--fail-with-body \
--noproxy '*' \
--resolve "${host}:443:127.0.0.1" \
--connect-timeout 3 \
--max-time 20 \
--header 'Cache-Control: no-cache' \
--output "$output_file" \
"$url"
}
check_target_node() {
local internal_ip
local ready_status
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node is missing: ${TARGET_NODE}"
ready_status="$(
kubectl get node "$TARGET_NODE" \
--output=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
)"
[[ "$ready_status" == "True" ]] || \
fail "target node is not Ready: ${TARGET_NODE}"
internal_ip="$(
kubectl get node "$TARGET_NODE" \
--output=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'
)"
[[ "$internal_ip" == "$EXPECTED_NODE_INTERNAL_IP" ]] || \
fail "target node InternalIP is ${internal_ip:-missing}, expected ${EXPECTED_NODE_INTERNAL_IP}"
}
check_oidc_secret_contract() {
local secret_type
local secret_keys
# These output expressions inspect only the Secret type and data key names.
# They never select, decode, compare, or print either data value.
secret_type="$(
kubectl --namespace "$GITEA_NAMESPACE" get secret "$OIDC_SECRET_NAME" \
--output=jsonpath='{.type}'
)"
secret_keys="$(
kubectl --namespace "$GITEA_NAMESPACE" get secret "$OIDC_SECRET_NAME" \
--output=go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' \
| LC_ALL=C sort
)"
[[ "$secret_type" == "Opaque" ]] || \
fail "${GITEA_NAMESPACE}/${OIDC_SECRET_NAME} type must be Opaque"
[[ "$secret_keys" == $'key\nsecret' ]] || \
fail "${GITEA_NAMESPACE}/${OIDC_SECRET_NAME} must contain exactly key and secret"
}
check_public_discovery() {
local discovery_file="$1"
https_get "$KEYCLOAK_HOST" "$KEYCLOAK_DISCOVERY_URL" "$discovery_file"
jq --exit-status \
--arg issuer "$KEYCLOAK_ISSUER" \
--arg authorization_endpoint "$KEYCLOAK_AUTH_ENDPOINT" \
'type == "object" and
.issuer == $issuer and
.authorization_endpoint == $authorization_endpoint and
(.token_endpoint | type == "string" and startswith($issuer + "/"))' \
"$discovery_file" >/dev/null 2>&1 || \
fail "local-SNI Keycloak discovery is not the expected JSON issuer"
}
check_gitea_health() {
local health_file="$1"
kubectl --namespace "$GITEA_NAMESPACE" wait \
--for=condition=Available "deployment/${GITEA_DEPLOYMENT}" \
--timeout=30s >/dev/null
kubectl --namespace "$GITEA_NAMESPACE" wait \
--for=jsonpath='{.endpoints[0].conditions.ready}'=true \
endpointslice \
--selector="kubernetes.io/service-name=${GITEA_SERVICE}" \
--timeout=30s >/dev/null
https_get "$GITEA_HOST" "$GITEA_HEALTH_URL" "$health_file"
jq --exit-status '.status == "pass"' "$health_file" >/dev/null || \
fail "Gitea public health response is not status=pass JSON"
}
verify_rendered_manifest_unchanged() {
local manifest_path="${render_temp_dir}/gitea-oidc.yaml"
local checksum_output
local actual_sha256
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified Gitea manifest is missing or unsafe"
[[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \
fail "verified Gitea manifest must have mode 0600"
checksum_output="$(sha256sum -- "$manifest_path")"
actual_sha256="${checksum_output%% *}"
[[ "$actual_sha256" == "$gitea_manifest_sha256" ]] || \
fail "verified gitea-oidc.yaml changed after confirmation"
}
check_auth_source() {
local auth_list_file="$1"
local auth_error_file="$2"
local keycloak_count
local active_oauth2_count
if ! kubectl --namespace "$GITEA_NAMESPACE" exec \
"deployment/${GITEA_DEPLOYMENT}" \
--container gitea \
-- gitea admin auth list \
--vertical-bars \
--min-width 1 \
--tab-width 1 \
--padding 0 \
--pad-char ' ' \
>"$auth_list_file" 2>"$auth_error_file"; then
fail "gitea admin auth list failed; its output was retained only in the private temp directory"
fi
read -r keycloak_count active_oauth2_count < <(
awk -F '|' '
function trim(value) {
gsub(/^[[:space:]]+|[[:space:]]+$/, "", value)
return value
}
NF == 4 {
name = trim($2)
type = trim($3)
enabled = trim($4)
if (name == "keycloak") {
keycloak_count++
if (type == "OAuth2" && enabled == "true") {
active_oauth2_count++
}
}
}
END {
print keycloak_count + 0, active_oauth2_count + 0
}
' "$auth_list_file"
)
[[ "$keycloak_count" == "1" && "$active_oauth2_count" == "1" ]] || \
fail "exactly one active OAuth2 auth source named keycloak was not found"
}
check_app_ini_policy() {
local app_ini_error_file="$1"
if ! kubectl --namespace "$GITEA_NAMESPACE" exec \
"deployment/${GITEA_DEPLOYMENT}" \
--container gitea \
-- awk '
function trim(value) {
gsub(/^[[:space:]]+|[[:space:]]+$/, "", value)
return value
}
/^[[:space:]]*\[/ {
section = $0
gsub(/^[[:space:]]*\[|\][[:space:]]*$/, "", section)
section = tolower(section)
next
}
/^[[:space:]]*[#;]/ || /^[[:space:]]*$/ {
next
}
{
split($0, pair, "=")
key = toupper(trim(pair[1]))
value = $0
sub(/^[^=]*=/, "", value)
value = trim(value)
if (section == "service" && key == "DISABLE_REGISTRATION" && tolower(value) == "false") disabled++
if (section == "service" && key == "ALLOW_ONLY_EXTERNAL_REGISTRATION" && tolower(value) == "true") external_only++
if (section == "service" && key == "SHOW_REGISTRATION_BUTTON" && tolower(value) == "false") button_hidden++
if (section == "service" && key == "ENABLE_PASSWORD_SIGNIN_FORM" && tolower(value) == "true") password_signin++
if (section == "oauth2_client" && key == "ENABLE_AUTO_REGISTRATION" && tolower(value) == "true") oidc_jit++
if (section == "oauth2_client" && key == "USERNAME" && value == "preferred_username") username_claim++
if (section == "oauth2_client" && key == "ACCOUNT_LINKING" && value == "login") account_linking++
if (section == "oauth2_client" && key == "OPENID_CONNECT_SCOPES" && value == "profile email") oidc_scopes++
}
END {
exit !(disabled == 1 &&
external_only == 1 &&
button_hidden == 1 &&
password_signin == 1 &&
oidc_jit == 1 &&
username_claim == 1 &&
account_linking == 1 &&
oidc_scopes == 1)
}
' /data/gitea/conf/app.ini >/dev/null 2>"$app_ini_error_file"; then
fail "live app.ini does not satisfy the external-registration-only OIDC policy"
fi
}
check_login_html() {
local login_html_file="$1"
local signup_html_file="$2"
local signup_headers_file="$3"
local signup_status
local header_line
local field_name
local signup_location=""
local signup_location_count=0
https_get "$GITEA_HOST" "$GITEA_LOGIN_URL" "$login_html_file"
rg --quiet --fixed-strings 'href="/user/oauth2/keycloak"' "$login_html_file" || \
fail "Gitea login HTML does not contain the Keycloak OIDC link"
rg --quiet --fixed-strings 'href="/assets/css/hyeonworks.css"' "$login_html_file" || \
fail "Gitea login HTML does not contain the Hyeonworks stylesheet"
rg --quiet --fixed-strings 'name="theme-color" content="#0f172a"' "$login_html_file" || \
fail "Gitea login HTML does not contain the Hyeonworks theme marker"
rg --quiet --fixed-strings 'hw-brand-link' "$login_html_file" || \
fail "Gitea login HTML does not contain the Hyeonworks navigation marker"
if rg --quiet --fixed-strings 'href="/user/sign_up"' "$login_html_file"; then
fail "Gitea login HTML still exposes a local sign-up link"
fi
if ! signup_status="$(
curl \
--disable \
--silent \
--show-error \
--noproxy '*' \
--resolve "${GITEA_HOST}:443:127.0.0.1" \
--connect-timeout 3 \
--max-time 20 \
--header 'Cache-Control: no-cache' \
--output "$signup_html_file" \
--dump-header "$signup_headers_file" \
--write-out '%{http_code}' \
"$GITEA_SIGNUP_URL"
)"; then
fail "Gitea sign-up endpoint transport check failed"
fi
case "$signup_status" in
200)
for field_name in user_name email password retype; do
if rg --quiet --fixed-strings "name=\"${field_name}\"" "$signup_html_file"; then
fail "Gitea sign-up HTML still exposes a local registration input"
fi
done
;;
404)
;;
301|302|303|307|308)
while IFS= read -r header_line; do
header_line="${header_line%$'\r'}"
case "$header_line" in
[Ll][Oo][Cc][Aa][Tt][Ii][Oo][Nn]:*)
signup_location="${header_line#*:}"
signup_location="${signup_location#"${signup_location%%[![:space:]]*}"}"
((signup_location_count += 1))
;;
esac
done <"$signup_headers_file"
[[ "$signup_location_count" == "1" ]] || \
fail "Gitea sign-up redirect must contain exactly one Location header"
case "$signup_location" in
/user/login|"${GITEA_ROOT_URL}/user/login")
;;
*)
fail "Gitea sign-up redirect does not target the same-origin login page"
;;
esac
;;
*)
fail "Gitea sign-up endpoint returned an unexpected HTTP status: ${signup_status}"
;;
esac
}
check_oidc_redirect() {
local headers_file="$1"
local status
local header_line
local location=""
local location_count=0
local query
local parameter
local parameter_name
local parameter_value
local -a query_parameters=()
local client_id_count=0
local response_type_count=0
local redirect_uri_count=0
local state_count=0
status="$(
curl \
--disable \
--silent \
--show-error \
--noproxy '*' \
--resolve "${GITEA_HOST}:443:127.0.0.1" \
--connect-timeout 3 \
--max-time 20 \
--output /dev/null \
--dump-header "$headers_file" \
--write-out '%{http_code}' \
"$GITEA_OIDC_START_URL"
)"
case "$status" in
302|303|307)
;;
*)
fail "Gitea OIDC start returned HTTP ${status}, expected 302, 303, or 307"
;;
esac
while IFS= read -r header_line; do
header_line="${header_line%$'\r'}"
case "$header_line" in
[Ll][Oo][Cc][Aa][Tt][Ii][Oo][Nn]:*)
location="${header_line#*:}"
location="${location#"${location%%[![:space:]]*}"}"
((location_count += 1))
;;
esac
done <"$headers_file"
[[ "$location_count" == "1" && -n "$location" ]] || \
fail "Gitea OIDC start did not return exactly one non-empty Location header"
[[ "$location" != *'#'* ]] || \
fail "Gitea OIDC Location unexpectedly contains a fragment"
case "$location" in
"${KEYCLOAK_AUTH_ENDPOINT}"\?*)
;;
*)
fail "Gitea OIDC Location does not target the expected Keycloak authorization endpoint"
;;
esac
query="${location#*\?}"
IFS='&' read -r -a query_parameters <<<"$query"
for parameter in "${query_parameters[@]}"; do
[[ "$parameter" == *=* ]] || continue
parameter_name="${parameter%%=*}"
parameter_value="${parameter#*=}"
case "$parameter_name" in
client_id)
((client_id_count += 1))
[[ "$parameter_value" == "gitea" ]] || \
fail "OIDC Location client_id is not gitea"
;;
response_type)
((response_type_count += 1))
[[ "$parameter_value" == "code" ]] || \
fail "OIDC Location response_type is not code"
;;
redirect_uri)
((redirect_uri_count += 1))
[[ "$parameter_value" == "$GITEA_OIDC_CALLBACK_ENCODED" ]] || \
fail "OIDC Location callback is not the exact public Gitea callback"
;;
state)
((state_count += 1))
[[ -n "$parameter_value" ]] || fail "OIDC Location state is empty"
;;
esac
done
[[ "$client_id_count" == "1" ]] || fail "OIDC Location must contain one client_id"
[[ "$response_type_count" == "1" ]] || fail "OIDC Location must contain one response_type"
[[ "$redirect_uri_count" == "1" ]] || fail "OIDC Location must contain one redirect_uri"
[[ "$state_count" == "1" ]] || fail "OIDC Location must contain one non-empty state"
# Deliberately do not print $location or any parsed state value.
}
check_branding_hashes() {
local remote_dir="$1"
local index
local source_path
local remote_path
local checksum_output
local local_sha256
local remote_sha256
local -a source_paths=(
"${REPOSITORY_ROOT}/services/gitea/branding/public/assets/css/hyeonworks.css"
"${REPOSITORY_ROOT}/services/gitea/branding/public/assets/img/logo.svg"
"${REPOSITORY_ROOT}/services/gitea/branding/public/assets/img/favicon.svg"
)
local -a public_paths=(
"/assets/css/hyeonworks.css"
"/assets/img/logo.svg"
"/assets/img/favicon.svg"
)
local -a labels=(
"hyeonworks.css"
"logo.svg"
"favicon.svg"
)
for index in "${!source_paths[@]}"; do
source_path="${source_paths[$index]}"
remote_path="${remote_dir}/${labels[$index]}"
[[ -f "$source_path" && ! -L "$source_path" ]] || \
fail "branding source is missing or symlinked: ${source_path}"
https_get \
"$GITEA_HOST" \
"${GITEA_ROOT_URL}${public_paths[$index]}" \
"$remote_path"
checksum_output="$(sha256sum -- "$source_path")"
local_sha256="${checksum_output%% *}"
checksum_output="$(sha256sum -- "$remote_path")"
remote_sha256="${checksum_output%% *}"
[[ "$remote_sha256" == "$local_sha256" ]] || \
fail "public branding hash differs from local source: ${labels[$index]}"
printf '브랜딩 해시 일치: %-16s %s\n' \
"${labels[$index]}" "$local_sha256"
done
}
runtime_temp_dir="$(mktemp -d /tmp/gitea-oidc-apply.XXXXXX)"
chmod 0700 "$runtime_temp_dir"
render_temp_dir="$(mktemp -d /tmp/platform-phase1-apply.XXXXXX)"
chmod 0700 "$render_temp_dir"
readonly DISCOVERY_FILE="${runtime_temp_dir}/keycloak-discovery.json"
readonly HEALTH_FILE="${runtime_temp_dir}/gitea-health.json"
readonly AUTH_LIST_FILE="${runtime_temp_dir}/gitea-auth-list.txt"
readonly AUTH_ERROR_FILE="${runtime_temp_dir}/gitea-auth-list.err"
readonly APP_INI_ERROR_FILE="${runtime_temp_dir}/gitea-app-ini.err"
readonly LOGIN_HTML_FILE="${runtime_temp_dir}/gitea-login.html"
readonly SIGNUP_HTML_FILE="${runtime_temp_dir}/gitea-signup.html"
readonly SIGNUP_HEADERS_FILE="${runtime_temp_dir}/gitea-signup-headers"
readonly OIDC_HEADERS_FILE="${runtime_temp_dir}/gitea-oidc-headers"
readonly BRANDING_REMOTE_DIR="${runtime_temp_dir}/branding-remote"
mkdir -m 0700 -- "$BRANDING_REMOTE_DIR"
printf '[1/8] 현재 context, 노드, Gitea 상태 확인\n'
check_target_node
kubectl --namespace "$GITEA_NAMESPACE" get "deployment/${GITEA_DEPLOYMENT}" >/dev/null
check_gitea_health "$HEALTH_FILE"
printf '[2/8] Host Nginx 로컬 SNI 경로의 Keycloak discovery JSON 확인\n'
check_public_discovery "$DISCOVERY_FILE"
printf '[3/8] Gitea OIDC Secret의 type과 key 이름만 확인\n'
check_oidc_secret_contract
printf '[4/8] 고정 Chart SHA 검증 후 Phase 1 manifest 렌더링\n'
cd -- "$REPOSITORY_ROOT"
PLATFORM_HELM_BIN="$HELM_BIN" \
bash scripts/validate/render-phase1.sh \
--verified-output-dir "$render_temp_dir"
verified_entry_count="$(
find "$render_temp_dir" -mindepth 1 -maxdepth 1 -type f \
-name '*.yaml' | wc -l | tr -d '[:space:]'
)"
[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \
fail "verified handoff must contain exactly ${#VERIFIED_MANIFEST_NAMES[@]} YAML manifests"
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
manifest_path="${render_temp_dir}/${manifest_name}.yaml"
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest is missing or unsafe: ${manifest_name}.yaml"
[[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \
fail "verified manifest must have mode 0600: ${manifest_name}.yaml"
done
checksum_output="$(sha256sum -- "${render_temp_dir}/gitea-oidc.yaml")"
gitea_manifest_sha256="${checksum_output%% *}"
readonly gitea_manifest_sha256
rg --quiet --fixed-strings 'gitea-keycloak-oidc' "${render_temp_dir}/gitea-oidc.yaml" || \
fail "rendered Gitea manifest does not reference the OIDC Secret"
rg --quiet --fixed-strings "$KEYCLOAK_DISCOVERY_URL" "${render_temp_dir}/gitea-oidc.yaml" || \
fail "rendered Gitea manifest does not contain the exact discovery URL"
printf '\nKubernetes context: %s\n' "$CURRENT_CONTEXT"
printf 'API server: %s\n' "$API_SERVER"
printf 'Target node: %s\n' "$TARGET_NODE"
printf 'gitea-oidc.yaml SHA-256: %s\n' "$gitea_manifest_sha256"
printf '적용 범위: 검증된 gitea-oidc.yaml 하나\n'
printf 'Type APPLY %s GITEA-OIDC %s to continue: ' \
"$CURRENT_CONTEXT" "$gitea_manifest_sha256"
read -r confirmation
[[ "$confirmation" == "APPLY ${CURRENT_CONTEXT} GITEA-OIDC ${gitea_manifest_sha256}" ]] || \
fail "cancelled"
[[ "$(kubectl config current-context)" == "$CURRENT_CONTEXT" ]] || \
fail "kubectl context changed after confirmation"
[[ "$(
kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}'
)" == "$API_SERVER" ]] || \
fail "Kubernetes API server changed after confirmation"
check_target_node
check_public_discovery "$DISCOVERY_FILE"
check_oidc_secret_contract
check_gitea_health "$HEALTH_FILE"
verify_rendered_manifest_unchanged
printf '\n[5/8] 검증된 gitea-oidc.yaml 하나만 적용\n'
apply_started=1
kubectl apply --filename="${render_temp_dir}/gitea-oidc.yaml"
verify_rendered_manifest_unchanged
printf '[6/8] Deployment rollout과 ready EndpointSlice 대기\n'
kubectl --namespace "$GITEA_NAMESPACE" rollout status \
"deployment/${GITEA_DEPLOYMENT}" --timeout=10m
kubectl --namespace "$GITEA_NAMESPACE" wait \
--for=jsonpath='{.endpoints[0].conditions.ready}'=true \
endpointslice \
--selector="kubernetes.io/service-name=${GITEA_SERVICE}" \
--timeout=2m
check_gitea_health "$HEALTH_FILE"
printf '[7/8] 활성 OAuth2 source, app.ini 정책, 로그인/OIDC 흐름 확인\n'
check_auth_source "$AUTH_LIST_FILE" "$AUTH_ERROR_FILE"
check_app_ini_policy "$APP_INI_ERROR_FILE"
check_login_html "$LOGIN_HTML_FILE" "$SIGNUP_HTML_FILE" "$SIGNUP_HEADERS_FILE"
check_oidc_redirect "$OIDC_HEADERS_FILE"
check_public_discovery "$DISCOVERY_FILE"
printf '[8/8] 공개 브랜딩 자산과 로컬 소스 SHA-256 비교\n'
check_branding_hashes "$BRANDING_REMOTE_DIR"
printf '\nGITEA OIDC APPLY SUCCESS\n'
printf '적용 manifest SHA-256: %s\n' "$gitea_manifest_sha256"
printf 'Keycloak discovery, 활성 OAuth2 source, 외부 인증 전용 가입 정책, OIDC redirect를 확인했습니다.\n'
printf 'OIDC Secret 값, 토큰, 전체 Location/state는 출력하지 않았습니다.\n'
printf '실제 realm 사용자 login/callback/logout은 별도 수동 수용 시험으로 남습니다.\n'
+332
View File
@@ -0,0 +1,332 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly CANDIDATE="$ROOT/infrastructure/networking/host-nginx/learn-services-admin.conf"
readonly ACTIVE="/etc/nginx/sites-available/learn-services"
readonly CERT_NAME="storage-admin.learn.hyeonworks.com"
readonly CERT_DIR="/etc/letsencrypt/live/$CERT_NAME"
readonly CREDENTIALS="/home/donghyeon/.secrets/certbot/cloudflare.ini"
readonly STORAGE_HOST="storage-admin.learn.hyeonworks.com"
readonly DB_HOST="db-admin.learn.hyeonworks.com"
readonly LAN_IP="192.168.0.107"
readonly TAIL_IP="100.92.240.34"
execute=false
certificate_only=false
certbot_email=""
rollback_armed=false
backup=""
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
사용법:
bash scripts/bootstrap/apply-host-nginx-admin.sh
bash scripts/bootstrap/apply-host-nginx-admin.sh \
--execute --certificate-only --certbot-email you@example.com
bash scripts/bootstrap/apply-host-nginx-admin.sh --execute
dry-run은 공개 DNS, 후보 hash, Ingress endpoint, Cloudflare credential 상태를
검사합니다. --execute는 필요하면 공식 Cloudflare snap plugin과 SAN 인증서를
발급하고 Nginx를 timestamp backup 뒤 교체합니다.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--certificate-only)
certificate_only=true
shift
;;
--execute)
execute=true
shift
;;
--certbot-email)
(( $# >= 2 )) || fail "--certbot-email 값이 필요합니다"
certbot_email="$2"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "지원하지 않는 인자: $1"
;;
esac
done
for cmd in awk curl dig install jq kubectl openssl rg sha256sum sleep stat sudo; do
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "Nginx 후보가 없습니다"
[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "활성 Nginx site가 없습니다"
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"
for name in "$STORAGE_HOST" "$DB_HOST"; do
[[ -z "$(dig +short @1.1.1.1 A "$name" | tr -d '[:space:]')" ]] || fail "$name 공개 A가 있습니다"
[[ -z "$(dig +short @1.1.1.1 AAAA "$name" | tr -d '[:space:]')" ]] || fail "$name 공개 AAAA가 있습니다"
done
kubectl -n object-storage get ingress minio-aistor-console >/dev/null 2>&1 || \
printf 'WARNING: AIStor Console Ingress가 아직 적용되지 않았습니다.\n' >&2
kubectl -n platform-admin get ingress pgadmin >/dev/null 2>&1 || \
printf 'WARNING: pgAdmin Ingress가 아직 적용되지 않았습니다.\n' >&2
candidate_sha="$(sha256sum "$CANDIDATE" | awk '{print $1}')"
active_sha="$(sha256sum "$ACTIVE" | awk '{print $1}')"
printf 'Active SHA-256: %s\n' "$active_sha"
printf 'Candidate SHA-256: %s\n' "$candidate_sha"
printf 'Certificate SAN: %s, %s\n' "$STORAGE_HOST" "$DB_HOST"
# Ubuntu의 sites-enabled/default는 미등록 HTTP Host에 welcome page(200)를
# 반환할 수 있다. HTTP status만으로 관리 도메인의 부분 적용을 판정하지 않고,
# 실제 활성 site에 관리 server_name이 들어갔는지를 먼저 확인한다.
if [[ "$active_sha" != "$candidate_sha" ]] && \
{ rg -q -F "$STORAGE_HOST" "$ACTIVE" || rg -q -F "$DB_HOST" "$ACTIVE"; }; then
fail "활성 Nginx site에 후보와 다른 admin 도메인 설정이 부분 적용돼 있습니다"
fi
check_cloudflare_credentials() {
sudo test -f "$CREDENTIALS" || \
fail "Cloudflare 제한 토큰 파일이 없습니다: $CREDENTIALS"
owner="$(sudo stat -c '%U:%G' "$CREDENTIALS")"
mode="$(sudo stat -c '%a' "$CREDENTIALS")"
size="$(sudo stat -c '%s' "$CREDENTIALS")"
[[ "$owner" == root:root && "$mode" == 600 ]] || \
fail "$CREDENTIALS 소유권/권한은 root:root 0600이어야 합니다"
(( size > 0 )) || fail "$CREDENTIALS 파일이 비어 있습니다"
contract="$(
sudo awk -F= '
BEGIN { count=0; nonempty=0; unexpected=0 }
/^[[:space:]]*($|#)/ { next }
/^[[:space:]]*dns_cloudflare_api_token[[:space:]]*=/ {
count++
value=$0
sub(/^[^=]*=[[:space:]]*/, "", value)
sub(/[[:space:]]+$/, "", value)
if (length(value) >= 20) nonempty++
next
}
{ unexpected++ }
END { printf "%d:%d:%d", count, nonempty, unexpected }
' "$CREDENTIALS"
)"
[[ "$contract" == "1:1:0" ]] || \
fail "$CREDENTIALS에는 비어 있지 않은 dns_cloudflare_api_token 한 개만 있어야 합니다"
printf 'Cloudflare credential: root:root 0600 및 key 형식 확인\n'
}
if [[ "$execute" == false ]]; then
if sudo -n test -f "$CREDENTIALS" 2>/dev/null; then
check_cloudflare_credentials
else
printf 'Cloudflare credential: root 전용이므로 --execute에서 sudo로 확인\n'
fi
printf 'DRY RUN PASS: --execute를 지정하지 않아 인증서와 Nginx를 변경하지 않았습니다.\n'
exit 0
fi
[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
sudo -v
check_cloudflare_credentials
context="$(kubectl config current-context)"
printf 'Type APPLY to issue/renew the admin certificate and replace Host Nginx: '
read -r answer
[[ "$answer" == APPLY ]] || fail "취소했습니다"
if [[ ! -x /snap/bin/certbot ]]; then
fail "지원 대상으로 고정한 snap certbot이 없습니다"
fi
if ! /snap/bin/certbot plugins 2>/dev/null | rg -q 'dns-cloudflare'; then
sudo snap set certbot trust-plugin-with-root=ok
sudo snap install certbot-dns-cloudflare
fi
if ! sudo test -s "$CERT_DIR/fullchain.pem" || ! sudo test -s "$CERT_DIR/privkey.pem"; then
[[ "$certbot_email" == *@* ]] || fail "최초 인증서 발급에는 --certbot-email이 필요합니다"
sudo /snap/bin/certbot certonly \
--dns-cloudflare \
--dns-cloudflare-credentials "$CREDENTIALS" \
--dns-cloudflare-propagation-seconds 60 \
--cert-name "$CERT_NAME" \
--domains "$STORAGE_HOST" \
--domains "$DB_HOST" \
--non-interactive \
--agree-tos \
--email "$certbot_email"
fi
cert_text="$(sudo openssl x509 -in "$CERT_DIR/fullchain.pem" -noout -text)"
printf '%s\n' "$cert_text" | rg -q "DNS:$STORAGE_HOST" || fail "인증서에 storage-admin SAN이 없습니다"
printf '%s\n' "$cert_text" | rg -q "DNS:$DB_HOST" || fail "인증서에 db-admin SAN이 없습니다"
san_count="$(printf '%s\n' "$cert_text" | rg -o 'DNS:[^,[:space:]]+' | sort -u | wc -l | tr -d '[:space:]')"
[[ "$san_count" == 2 ]] || fail "admin 인증서 SAN은 정확히 두 개여야 합니다"
sudo install -d -o root -g root -m 0755 /etc/letsencrypt/renewal-hooks/deploy
sudo install -o root -g root -m 0755 \
"$ROOT/infrastructure/networking/host-nginx/reload-nginx.sh" \
/etc/letsencrypt/renewal-hooks/deploy/reload-nginx
if [[ "$certificate_only" == true ]]; then
printf 'ADMIN CERTIFICATE READY\n'
printf 'Nginx cutover는 서비스 적용 뒤 --certificate-only 없이 다시 실행하세요.\n'
exit 0
fi
for host in "$STORAGE_HOST" "$DB_HOST"; do
code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--noproxy '*' --resolve "$host:80:127.0.0.1" -H "Host: $host" "http://$host/")"
[[ "$code" == 200 || "$code" == 404 || "$code" == 301 ]] || \
fail "기존 HTTP 경계가 예상과 다릅니다: $host=$code"
printf '기존 HTTP fallback: %s=%s\n' "$host" "$code"
route_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--noproxy '*' -H "Host: $host" "http://127.0.0.1:30080/")"
[[ "$route_code" == 200 || "$route_code" == 302 || "$route_code" == 303 ]] || \
fail "Traefik route가 준비되지 않았습니다: $host=$route_code"
done
probe_https_code() {
host="$1"
target_ip="$2"
source_ip="${3:-}"
interface_args=()
if [[ -n "$source_ip" ]]; then
interface_args=(--interface "$source_ip")
fi
curl --disable --silent --output /dev/null --write-out '%{http_code}' \
--connect-timeout 1 --max-time 3 --noproxy '*' \
"${interface_args[@]}" --resolve "$host:443:$target_ip" "https://$host/" \
2>/dev/null || true
}
wait_for_admin_proxy_state() {
consecutive_passes=0
storage_lan=000
db_lan=000
storage_denied=000
db_denied=000
for ((attempt = 1; attempt <= 15; attempt++)); do
storage_lan="$(probe_https_code "$STORAGE_HOST" "$LAN_IP" "$LAN_IP")"
db_lan="$(probe_https_code "$DB_HOST" "$LAN_IP" "$LAN_IP")"
storage_denied="$(probe_https_code "$STORAGE_HOST" 127.0.0.1)"
db_denied="$(probe_https_code "$DB_HOST" 127.0.0.1)"
if [[ "$storage_lan" == 200 || "$storage_lan" == 302 || "$storage_lan" == 303 ]] &&
[[ "$db_lan" == 200 || "$db_lan" == 302 || "$db_lan" == 303 ]] &&
[[ "$storage_denied" == 403 && "$db_denied" == 403 ]]; then
consecutive_passes=$((consecutive_passes + 1))
if (( consecutive_passes >= 2 )); then
printf 'Nginx admin proxy state stabilized after %d probes.\n' "$attempt"
return 0
fi
else
consecutive_passes=0
fi
if (( attempt == 1 || attempt % 5 == 0 )); then
printf 'Nginx admin proxy 대기: probe=%d LAN=%s/%s denied=%s/%s\n' \
"$attempt" "$storage_lan" "$db_lan" "$storage_denied" "$db_denied"
fi
(( attempt < 15 )) && sleep 1
done
printf 'ERROR: Nginx admin proxy가 안정화되지 않았습니다: LAN=%s/%s denied=%s/%s\n' \
"$storage_lan" "$db_lan" "$storage_denied" "$db_denied" >&2
return 1
}
wait_for_admin_tls_rejection() {
consecutive_passes=0
for ((attempt = 1; attempt <= 15; attempt++)); do
if ! curl --disable --insecure --silent --output /dev/null \
--connect-timeout 1 --max-time 3 --noproxy '*' \
--resolve "$STORAGE_HOST:443:127.0.0.1" "https://$STORAGE_HOST/" 2>/dev/null &&
! curl --disable --insecure --silent --output /dev/null \
--connect-timeout 1 --max-time 3 --noproxy '*' \
--resolve "$DB_HOST:443:127.0.0.1" "https://$DB_HOST/" 2>/dev/null; then
consecutive_passes=$((consecutive_passes + 1))
if (( consecutive_passes >= 2 )); then
return 0
fi
else
consecutive_passes=0
fi
(( attempt < 15 )) && sleep 1
done
return 1
}
backup="$ACTIVE.before-admin-$(date +%Y%m%d%H%M%S)"
printf 'Planned backup: %s\n' "$backup"
sudo install -o root -g root -m 0644 "$ACTIVE" "$backup"
rollback() {
set +e
printf '\nROLLBACK: %s 복원\n' "$backup" >&2
sudo install -o root -g root -m 0644 "$backup" "$ACTIVE"
sudo /usr/sbin/nginx -t
sudo systemctl reload nginx
if ! wait_for_admin_tls_rejection; then
printf 'WARNING: rollback 뒤 admin SNI 거부 안정화를 확인하지 못했습니다.\n' >&2
fi
rollback_armed=false
printf 'ROLLBACK complete.\n' >&2
}
finish() {
rc=$?
trap - EXIT INT TERM
if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then
rollback
fi
exit "$rc"
}
trap finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
rollback_armed=true
sudo install -o root -g root -m 0644 "$CANDIDATE" "$ACTIVE"
sudo systemctl daemon-reload
sudo /usr/sbin/nginx -t
sudo systemctl reload nginx
sudo systemctl is-active --quiet nginx
wait_for_admin_proxy_state || fail "LAN HTTPS와 allowlist 상태가 수렴하지 않았습니다"
for host in "$STORAGE_HOST" "$DB_HOST"; do
redirect_result="$(
curl --disable --silent --show-error --output /dev/null \
--write-out $'%{http_code}\n%{redirect_url}' --max-time 5 --noproxy '*' \
--resolve "$host:80:$LAN_IP" "http://$host/"
)"
[[ "$redirect_result" == $'301\nhttps://'"$host/" ]] || \
fail "HTTP redirect 검증 실패: $host=$redirect_result"
done
if curl --insecure --silent --output /dev/null --noproxy '*' \
--resolve unconfigured.invalid:443:127.0.0.1 https://unconfigured.invalid/ 2>/dev/null; then
fail "unknown SNI가 거부되지 않았습니다"
fi
sudo /snap/bin/certbot renew --dry-run
rollback_armed=false
printf 'ADMIN NGINX CUTOVER SUCCESS\n'
printf 'Backup: %s\n' "$backup"
printf 'Candidate SHA-256: %s\n' "$candidate_sha"
+351
View File
@@ -0,0 +1,351 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# Do not expose headers, cookies, or future sensitive values through caller xtrace.
set +x
umask 077
readonly EXPECTED_CANDIDATE_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5"
readonly EXPECTED_PRE_CUTOVER_SHA256="5b5941519ab677f751568827aa4f315193dfbd9eeac5e8a4ad85d724aaefe16f"
readonly EXPECTED_PRE_CUTOVER_HEALTH_SHA256="6b683cb16987ff1f5ded22e9847ac0a45995947927c459b21f427523a41c7484"
readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform"
readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services.conf"
readonly ACTIVE="/etc/nginx/sites-available/learn-services"
readonly ENABLED="/etc/nginx/sites-enabled/learn-services"
readonly CURL_BIN="/usr/bin/curl"
readonly JQ_BIN="/usr/bin/jq"
readonly NGINX_BIN="/usr/sbin/nginx"
readonly SYSTEMCTL_BIN="/usr/bin/systemctl"
readonly INSTALL_BIN="/usr/bin/install"
readonly SHA256SUM_BIN="/usr/bin/sha256sum"
readonly STAT_BIN="/usr/bin/stat"
readonly READLINK_BIN="/usr/bin/readlink"
readonly MKTEMP_BIN="/usr/bin/mktemp"
readonly RM_BIN="/usr/bin/rm"
readonly DATE_BIN="/usr/bin/date"
readonly AWK_BIN="/usr/bin/awk"
readonly SLEEP_BIN="/usr/bin/sleep"
fail() {
printf 'ERROR: %s\n' "$*" >&2
return 1
}
usage() {
printf '%s\n' \
'Usage: sudo bash scripts/bootstrap/apply-host-nginx-gitea.sh --execute' \
'' \
'Backs up the active learn-services site, installs the reviewed Gitea proxy,' \
'tests and reloads Nginx, then runs local acceptance checks. Any failure after' \
'the active file changes triggers an automatic restore and reload.'
}
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
[[ "$EUID" -eq 0 ]] || fail "run this script through sudo"
[[ -t 0 ]] || fail "an interactive terminal is required"
for required_binary in \
"$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \
"$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \
"$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do
[[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}"
done
[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}"
[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}"
[[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}"
[[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || fail "enabled symlink target changed"
[[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \
fail "active file owner or mode changed"
"$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active"
readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-gitea-cutover.XXXXXX)"
readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate"
readonly DIRECT_HEALTH="${TEMP_DIR}/direct-health.json"
readonly NGINX_HEALTH="${TEMP_DIR}/nginx-health.json"
readonly LOGIN_HEADERS="${TEMP_DIR}/login-headers"
rollback_armed=0
backup=""
cleanup() {
case "$TEMP_DIR" in
/tmp/nginx-gitea-cutover.*)
"$RM_BIN" -rf -- "$TEMP_DIR"
;;
*)
printf 'WARNING: refusing to remove unexpected temp path: %s\n' "$TEMP_DIR" >&2
;;
esac
}
rollback() {
local restore_install_rc
local restore_test_rc
local restore_reload_rc
set +e
printf '\nROLLBACK: restoring %s\n' "$backup" >&2
if "$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" &&
[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" ]]; then
restore_install_rc=0
else
restore_install_rc=1
fi
"$NGINX_BIN" -t
restore_test_rc=$?
if (( restore_install_rc == 0 && restore_test_rc == 0 )); then
if "$SYSTEMCTL_BIN" reload nginx &&
"$SYSTEMCTL_BIN" is-active --quiet nginx &&
wait_for_rollback_state; then
restore_reload_rc=0
else
restore_reload_rc=1
fi
else
restore_reload_rc=1
fi
if (( restore_install_rc == 0 && restore_test_rc == 0 && restore_reload_rc == 0 )); then
rollback_armed=0
printf 'ROLLBACK complete. Active config and response were restored.\n' >&2
else
printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' "$backup" >&2
fi
}
on_exit() {
local rc=$?
trap - EXIT INT TERM
if (( rc != 0 && rollback_armed == 1 )); then
rollback
fi
cleanup
exit "$rc"
}
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
"$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT"
readonly CANDIDATE_SHA256="$("$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}')"
readonly ACTIVE_SHA256="$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')"
[[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \
fail "candidate digest changed: ${CANDIDATE_SHA256}"
if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" && "$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then
fail "active config has an unexpected digest: ${ACTIVE_SHA256}"
fi
check_health_json() {
local path="$1"
"$JQ_BIN" -e '
.status == "pass" and
([.checks["database:ping"][], .checks["cache:ping"][]] |
all(.status == "pass"))
' "$path" >/dev/null
}
pre_cutover_check() {
"$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
--header 'Host: git.learn.hyeonworks.com' \
http://127.0.0.1:30080/api/healthz >"$DIRECT_HEALTH"
check_health_json "$DIRECT_HEALTH"
}
wait_for_nginx_health() {
local attempt
local probe_metadata=""
local http_code="curl-error"
local content_type="unavailable"
local size_download="0"
local body_sha256=""
local classification="transport-error"
local consecutive_passes=0
for ((attempt = 1; attempt <= 10; attempt++)); do
if probe_metadata="$(
"$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \
--resolve git.learn.hyeonworks.com:443:127.0.0.1 \
--output "$NGINX_HEALTH" \
--write-out $'%{http_code}\t%{content_type}\t%{size_download}' \
https://git.learn.hyeonworks.com/api/healthz
)"; then
IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata"
body_sha256="$(
"$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}'
)"
if [[ "$http_code" == "200" ]] &&
check_health_json "$NGINX_HEALTH" 2>/dev/null; then
classification="healthy-json"
consecutive_passes=$((consecutive_passes + 1))
if (( consecutive_passes >= 2 )); then
printf 'Nginx proxy health stabilized after %d probes.\n' "$attempt"
return 0
fi
else
consecutive_passes=0
if [[ "$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then
classification="stale-old-generation"
elif [[ "$http_code" =~ ^(502|503|504)$ ]]; then
classification="transient-upstream"
elif "$JQ_BIN" -e '
type == "object" and
has("status") and
(.checks | type == "object")
' "$NGINX_HEALTH" >/dev/null 2>&1; then
classification="unhealthy-health-json"
else
printf 'ERROR: unexpected Nginx health response (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \
"$http_code" "${content_type:-none}" "$size_download" "$body_sha256" >&2
return 1
fi
fi
else
http_code="curl-error"
content_type="unavailable"
size_download="0"
body_sha256=""
classification="transport-error"
consecutive_passes=0
fi
if (( attempt < 10 )); then
"$SLEEP_BIN" 1
fi
done
fail "Nginx health did not converge during the bounded retry window (classification=${classification}, status=${http_code}, content-type=${content_type:-none}, bytes=${size_download})"
}
wait_for_rollback_state() {
local attempt
local probe_metadata=""
local http_code="curl-error"
local content_type="unavailable"
local size_download="0"
local body_sha256=""
for ((attempt = 1; attempt <= 10; attempt++)); do
if probe_metadata="$(
"$CURL_BIN" --noproxy '*' --silent --connect-timeout 1 --max-time 2 \
--resolve git.learn.hyeonworks.com:443:127.0.0.1 \
--output "$NGINX_HEALTH" \
--write-out $'%{http_code}\t%{content_type}\t%{size_download}' \
https://git.learn.hyeonworks.com/api/healthz
)"; then
IFS=$'\t' read -r http_code content_type size_download <<<"$probe_metadata"
body_sha256="$(
"$SHA256SUM_BIN" "$NGINX_HEALTH" | "$AWK_BIN" '{print $1}'
)"
if [[ "$http_code" == "200" &&
"$body_sha256" == "$EXPECTED_PRE_CUTOVER_HEALTH_SHA256" ]]; then
if (( attempt > 1 )); then
printf 'Rollback proxy state stabilized after %d probes.\n' "$attempt" >&2
fi
return 0
fi
else
http_code="curl-error"
content_type="unavailable"
size_download="0"
body_sha256=""
fi
if (( attempt < 10 )); then
"$SLEEP_BIN" 1
fi
done
printf 'CRITICAL: rollback response did not converge (status=%s, content-type=%s, bytes=%s, sha256=%s)\n' \
"$http_code" "${content_type:-none}" "$size_download" "${body_sha256:-none}" >&2
return 1
}
post_cutover_checks() {
local id_body
local redirect_result
wait_for_nginx_health
id_body="$(
"$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
--resolve id.learn.hyeonworks.com:443:127.0.0.1 \
https://id.learn.hyeonworks.com/
)"
[[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] || \
fail "Keycloak hold response changed"
redirect_result="$(
"$CURL_BIN" --noproxy '*' --silent --show-error --max-time 10 \
--resolve git.learn.hyeonworks.com:80:127.0.0.1 \
--output /dev/null --write-out $'%{http_code}\n%{redirect_url}' \
http://git.learn.hyeonworks.com/api/healthz
)"
[[ "$redirect_result" == $'301\nhttps://git.learn.hyeonworks.com/api/healthz' ]] || \
fail "HTTP redirect check failed: ${redirect_result}"
if "$CURL_BIN" --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \
--resolve unconfigured.invalid:443:127.0.0.1 \
https://unconfigured.invalid/ 2>/dev/null; then
fail "unknown TLS hostname was not rejected"
fi
"$CURL_BIN" --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
--resolve git.learn.hyeonworks.com:443:127.0.0.1 \
--dump-header "$LOGIN_HEADERS" --output /dev/null \
https://git.learn.hyeonworks.com/user/login
"$AWK_BIN" '
BEGIN { found = 0; insecure = 0 }
tolower($0) ~ /^set-cookie:/ {
found++
if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1
}
END { exit(found == 0 || insecure) }
' "$LOGIN_HEADERS" || fail "login cookie Secure check failed"
}
pre_cutover_check
"$NGINX_BIN" -t
if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then
post_cutover_checks
printf 'The reviewed Host Nginx configuration is already active and healthy.\n'
exit 0
fi
backup="${ACTIVE}.before-gitea-$("$DATE_BIN" +%Y%m%d%H%M%S)"
[[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}"
printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256"
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
printf 'Planned backup: %s\n' "$backup"
printf 'Type APPLY to replace the Host Nginx site: '
read -r confirmation
[[ "$confirmation" == "APPLY" ]] || fail "cancelled"
"$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup"
[[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == "$ACTIVE_SHA256" ]] || \
fail "backup digest mismatch"
rollback_armed=1
"$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE"
[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == "$CANDIDATE_SHA256" ]] || \
fail "installed digest mismatch"
"$NGINX_BIN" -t
"$SYSTEMCTL_BIN" reload nginx
"$SYSTEMCTL_BIN" is-active --quiet nginx
post_cutover_checks
rollback_armed=0
printf '\nCUTOVER SUCCESS\n'
printf 'Backup: %s\n' "$backup"
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
printf 'Local HTTPS health, redirect, Secure cookie, Keycloak hold, and unknown-host rejection: PASS\n'
printf 'Run the public HTTPS and Git clone/push checks from a separate client next.\n'
+299
View File
@@ -0,0 +1,299 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set +x
umask 077
readonly EXPECTED_CANDIDATE_SHA256="5c5cd74b4992f537fd27c50cf2209573a80a9904e0b19b58c3154717be6ff4a5"
readonly EXPECTED_PRE_CUTOVER_SHA256="de7ebd4f69cd7d2204ee633e074bf6a4370a6f5e3f3fac9067b099d5d75269b5"
readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform"
readonly CANDIDATE="${REPOSITORY_ROOT}/infrastructure/networking/host-nginx/learn-services-keycloak.conf"
readonly ACTIVE="/etc/nginx/sites-available/learn-services"
readonly ENABLED="/etc/nginx/sites-enabled/learn-services"
readonly GITEA_HOST="git.learn.hyeonworks.com"
readonly KEYCLOAK_HOST="id.learn.hyeonworks.com"
readonly KEYCLOAK_ISSUER="https://${KEYCLOAK_HOST}/realms/hyeonworks"
readonly CURL_BIN="/usr/bin/curl"
readonly JQ_BIN="/usr/bin/jq"
readonly NGINX_BIN="/usr/sbin/nginx"
readonly SYSTEMCTL_BIN="/usr/bin/systemctl"
readonly INSTALL_BIN="/usr/bin/install"
readonly SHA256SUM_BIN="/usr/bin/sha256sum"
readonly STAT_BIN="/usr/bin/stat"
readonly READLINK_BIN="/usr/bin/readlink"
readonly MKTEMP_BIN="/usr/bin/mktemp"
readonly RM_BIN="/usr/bin/rm"
readonly DATE_BIN="/usr/bin/date"
readonly AWK_BIN="/usr/bin/awk"
readonly SLEEP_BIN="/usr/bin/sleep"
fail() {
printf 'ERROR: %s\n' "$*" >&2
return 1
}
usage() {
printf '%s\n' \
'Usage: sudo bash scripts/bootstrap/apply-host-nginx-keycloak.sh --execute' \
'' \
'Replaces the static id.learn response with the reviewed Keycloak proxy.' \
'The active site is backed up first; a failed test or health check restores' \
'the prior Gitea-only configuration and reloads Nginx automatically.'
}
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
[[ "$EUID" -eq 0 ]] || fail "run this script through sudo"
[[ -t 0 ]] || fail "an interactive terminal is required"
for required_binary in \
"$CURL_BIN" "$JQ_BIN" "$NGINX_BIN" "$SYSTEMCTL_BIN" "$INSTALL_BIN" \
"$SHA256SUM_BIN" "$STAT_BIN" "$READLINK_BIN" "$MKTEMP_BIN" \
"$RM_BIN" "$DATE_BIN" "$AWK_BIN" "$SLEEP_BIN"; do
[[ -x "$required_binary" ]] || fail "required executable is missing: ${required_binary}"
done
[[ -f "$CANDIDATE" && ! -L "$CANDIDATE" ]] || fail "unsafe candidate: ${CANDIDATE}"
[[ -f "$ACTIVE" && ! -L "$ACTIVE" ]] || fail "unsafe active file: ${ACTIVE}"
[[ -L "$ENABLED" ]] || fail "enabled path is not a symlink: ${ENABLED}"
[[ "$("$READLINK_BIN" -f "$ENABLED")" == "$ACTIVE" ]] || \
fail "enabled symlink target changed"
[[ "$("$STAT_BIN" --format='%U:%G %a' "$ACTIVE")" == "root:root 644" ]] || \
fail "active file owner or mode changed"
"$SYSTEMCTL_BIN" is-active --quiet nginx || fail "nginx is not active"
readonly TEMP_DIR="$("$MKTEMP_BIN" -d /tmp/nginx-keycloak-cutover.XXXXXX)"
readonly CANDIDATE_SNAPSHOT="${TEMP_DIR}/learn-services.candidate"
readonly GITEA_HEALTH="${TEMP_DIR}/gitea-health.json"
readonly KEYCLOAK_DISCOVERY="${TEMP_DIR}/keycloak-discovery.json"
readonly LOGIN_HEADERS="${TEMP_DIR}/gitea-login-headers"
rollback_armed=0
backup=""
cleanup() {
case "$TEMP_DIR" in
/tmp/nginx-keycloak-cutover.*)
"$RM_BIN" -rf -- "$TEMP_DIR"
;;
*)
printf 'WARNING: refusing to remove unexpected temp path: %s\n' \
"$TEMP_DIR" >&2
;;
esac
}
check_gitea_health() {
"$JQ_BIN" -e '
.status == "pass" and
([.checks["database:ping"][], .checks["cache:ping"][]] |
all(.status == "pass"))
' "$GITEA_HEALTH" >/dev/null
}
check_keycloak_discovery() {
"$JQ_BIN" -e \
--arg issuer "$KEYCLOAK_ISSUER" '
.issuer == $issuer and
(.authorization_endpoint | startswith($issuer)) and
(.token_endpoint | startswith($issuer)) and
(.userinfo_endpoint | startswith($issuer)) and
(.jwks_uri | startswith($issuer))
' "$KEYCLOAK_DISCOVERY" >/dev/null
}
pre_cutover_checks() {
"$CURL_BIN" --disable --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
--header "Host: ${GITEA_HOST}" \
--header 'X-Forwarded-Proto: https' \
http://127.0.0.1:30080/api/healthz >"$GITEA_HEALTH"
check_gitea_health
"$CURL_BIN" --disable --noproxy '*' --fail-with-body --silent --show-error --max-time 10 \
--header "Host: ${KEYCLOAK_HOST}" \
--header 'X-Forwarded-Host: id.learn.hyeonworks.com' \
--header 'X-Forwarded-Proto: https' \
--header 'X-Forwarded-Port: 443' \
"http://127.0.0.1:30080/realms/hyeonworks/.well-known/openid-configuration" \
>"$KEYCLOAK_DISCOVERY"
check_keycloak_discovery
}
wait_for_proxy_state() {
local attempt
for ((attempt = 1; attempt <= 12; attempt++)); do
if "$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
--connect-timeout 1 --max-time 3 \
--resolve "${GITEA_HOST}:443:127.0.0.1" \
"https://${GITEA_HOST}/api/healthz" >"$GITEA_HEALTH" 2>/dev/null &&
check_gitea_health 2>/dev/null &&
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
--connect-timeout 1 --max-time 3 \
--resolve "${KEYCLOAK_HOST}:443:127.0.0.1" \
"https://${KEYCLOAK_HOST}/realms/hyeonworks/.well-known/openid-configuration" \
>"$KEYCLOAK_DISCOVERY" 2>/dev/null &&
check_keycloak_discovery 2>/dev/null; then
if (( attempt > 1 )); then
printf 'Nginx proxy state stabilized after %d probes.\n' "$attempt"
fi
return 0
fi
(( attempt < 12 )) && "$SLEEP_BIN" 1
done
return 1
}
wait_for_rollback_state() {
local attempt
local id_body
for ((attempt = 1; attempt <= 12; attempt++)); do
id_body="$(
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
--connect-timeout 1 --max-time 3 \
--resolve "${KEYCLOAK_HOST}:443:127.0.0.1" \
"https://${KEYCLOAK_HOST}/" 2>/dev/null
)" || id_body=""
if [[ "$id_body" == "Keycloak domain reached Nginx successfully" ]] &&
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error \
--connect-timeout 1 --max-time 3 \
--resolve "${GITEA_HOST}:443:127.0.0.1" \
"https://${GITEA_HOST}/api/healthz" >"$GITEA_HEALTH" 2>/dev/null &&
check_gitea_health 2>/dev/null; then
return 0
fi
(( attempt < 12 )) && "$SLEEP_BIN" 1
done
return 1
}
rollback() {
local restore_rc=0
set +e
printf '\nROLLBACK: restoring %s\n' "$backup" >&2
"$INSTALL_BIN" -o root -g root -m 0644 "$backup" "$ACTIVE" || restore_rc=1
"$NGINX_BIN" -t || restore_rc=1
if (( restore_rc == 0 )); then
"$SYSTEMCTL_BIN" reload nginx || restore_rc=1
"$SYSTEMCTL_BIN" is-active --quiet nginx || restore_rc=1
wait_for_rollback_state || restore_rc=1
fi
if (( restore_rc == 0 )); then
rollback_armed=0
printf 'ROLLBACK complete. The Gitea proxy and static Keycloak hold were restored.\n' >&2
else
printf 'CRITICAL: automatic rollback failed; backup remains at %s\n' \
"$backup" >&2
fi
}
on_exit() {
local rc=$?
trap - EXIT INT TERM
if (( rc != 0 && rollback_armed == 1 )); then
rollback
fi
cleanup
exit "$rc"
}
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
"$INSTALL_BIN" -o root -g root -m 0600 "$CANDIDATE" "$CANDIDATE_SNAPSHOT"
readonly CANDIDATE_SHA256="$(
"$SHA256SUM_BIN" "$CANDIDATE_SNAPSHOT" | "$AWK_BIN" '{print $1}'
)"
readonly ACTIVE_SHA256="$(
"$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}'
)"
[[ "$CANDIDATE_SHA256" == "$EXPECTED_CANDIDATE_SHA256" ]] || \
fail "candidate digest changed: ${CANDIDATE_SHA256}"
if [[ "$ACTIVE_SHA256" != "$EXPECTED_PRE_CUTOVER_SHA256" &&
"$ACTIVE_SHA256" != "$CANDIDATE_SHA256" ]]; then
fail "active config has an unexpected digest: ${ACTIVE_SHA256}"
fi
pre_cutover_checks
"$NGINX_BIN" -t
post_cutover_checks() {
local host
local redirect_result
wait_for_proxy_state || fail "Gitea and Keycloak HTTPS proxies did not converge"
for host in "$GITEA_HOST" "$KEYCLOAK_HOST"; do
redirect_result="$(
"$CURL_BIN" --disable --noproxy '*' --silent --show-error --max-time 10 \
--resolve "${host}:80:127.0.0.1" \
--output /dev/null \
--write-out $'%{http_code}\n%{redirect_url}' \
"http://${host}/"
)"
[[ "$redirect_result" == $'301\nhttps://'"${host}/" ]] || \
fail "${host} HTTP redirect check failed: ${redirect_result}"
done
"$CURL_BIN" --disable --noproxy '*' --fail --silent --show-error --max-time 10 \
--resolve "${GITEA_HOST}:443:127.0.0.1" \
--dump-header "$LOGIN_HEADERS" --output /dev/null \
"https://${GITEA_HOST}/user/login"
"$AWK_BIN" '
BEGIN { found = 0; insecure = 0 }
tolower($0) ~ /^set-cookie:/ {
found++
if (tolower($0) !~ /; secure([;[:space:]]|$)/) insecure = 1
}
END { exit(found == 0 || insecure) }
' "$LOGIN_HEADERS" || fail "Gitea login cookie Secure check failed"
if "$CURL_BIN" --disable --noproxy '*' --insecure --silent --output /dev/null --max-time 5 \
--resolve unconfigured.invalid:443:127.0.0.1 \
https://unconfigured.invalid/ 2>/dev/null; then
fail "unknown TLS hostname was not rejected"
fi
}
if [[ "$ACTIVE_SHA256" == "$CANDIDATE_SHA256" ]]; then
post_cutover_checks
printf 'The reviewed Keycloak Host Nginx configuration is already active and healthy.\n'
exit 0
fi
backup="${ACTIVE}.before-keycloak-$("$DATE_BIN" +%Y%m%d%H%M%S)"
[[ ! -e "$backup" && ! -L "$backup" ]] || fail "backup path already exists: ${backup}"
printf '\nActive SHA-256: %s\n' "$ACTIVE_SHA256"
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
printf 'Planned backup: %s\n' "$backup"
printf 'Type APPLY to expose Keycloak through Host Nginx: '
read -r confirmation
[[ "$confirmation" == "APPLY" ]] || fail "cancelled"
"$INSTALL_BIN" -o root -g root -m 0644 "$ACTIVE" "$backup"
[[ "$("$SHA256SUM_BIN" "$backup" | "$AWK_BIN" '{print $1}')" == \
"$ACTIVE_SHA256" ]] || fail "backup digest mismatch"
rollback_armed=1
"$INSTALL_BIN" -o root -g root -m 0644 "$CANDIDATE_SNAPSHOT" "$ACTIVE"
[[ "$("$SHA256SUM_BIN" "$ACTIVE" | "$AWK_BIN" '{print $1}')" == \
"$CANDIDATE_SHA256" ]] || fail "installed digest mismatch"
"$NGINX_BIN" -t
"$SYSTEMCTL_BIN" reload nginx
"$SYSTEMCTL_BIN" is-active --quiet nginx
post_cutover_checks
rollback_armed=0
printf '\nKEYCLOAK CUTOVER SUCCESS\n'
printf 'Backup: %s\n' "$backup"
printf 'Candidate SHA-256: %s\n' "$CANDIDATE_SHA256"
printf 'Gitea health, Keycloak discovery issuer, HTTPS cookies, redirects, and unknown-host rejection: PASS\n'
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+407
View File
@@ -0,0 +1,407 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# 호출자가 bash -x로 실행해도 향후 Secret 입력이 추적 출력에 노출되지 않도록 한다.
set +x
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly -a VERIFIED_MANIFEST_NAMES=(
keycloak-namespace
platform-postgres-keycloak
keycloak-operator
keycloak
)
readonly -a KEYCLOAK_CRDS=(
customresourcedefinition/keycloaks.k8s.keycloak.org
customresourcedefinition/keycloakrealmimports.k8s.keycloak.org
customresourcedefinition/keycloakoidcclients.k8s.keycloak.org
customresourcedefinition/keycloaksamlclients.k8s.keycloak.org
)
mutation_started=false
current_step="preflight"
report_retained_state() {
if [[ "$mutation_started" == true ]]; then
printf '%s\n' \
"SAFE STOP during ${current_step}." \
'No Namespace, Secret, DatabaseRole, Database, Operator, or Keycloak resource was deleted.' \
'Database reclaim policies remain Retain. Diagnose the failed wait or apply, then rerun this script.' >&2
fi
}
fail() {
printf 'ERROR: %s\n' "$*" >&2
report_retained_state
exit 1
}
on_error() {
local status="$1"
local line="$2"
trap - ERR
set +e
printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2
report_retained_state
exit "$status"
}
on_signal() {
local status="$1"
trap - INT TERM
set +e
printf 'INTERRUPTED: stopping without deleting cluster state.\n' >&2
report_retained_state
exit "$status"
}
usage() {
cat <<'USAGE'
Usage: bash scripts/bootstrap/apply-keycloak.sh --execute
Renders, verifies, and applies only the Keycloak path in this order:
Keycloak Namespace
two Keycloak DB Secrets
CloudNativePG DatabaseRole, Database, and NetworkPolicy
Keycloak Operator
Keycloak custom resource and HTTP Ingress
AIStor namespaces, Secrets, storage, Operator, and ObjectStore are not required
or applied. Host Nginx and Gitea OIDC configuration are separate cutovers.
USAGE
}
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
for command_name in kubectl curl find mktemp rg sed sha256sum stat wc; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
kustomize_version="$(
kubectl version --client --output=yaml |
sed -n 's/^kustomizeVersion: //p'
)"
[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}"
umask 077
render_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-keycloak-apply.XXXXXX")"
cleanup() {
case "$render_temp_dir" in
/tmp/platform-keycloak-apply.*|"${TMPDIR:-/tmp}"/platform-keycloak-apply.*)
rm -rf -- "$render_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render directory: %s\n' \
"$render_temp_dir" >&2
;;
esac
}
trap cleanup EXIT
trap 'on_error "$?" "$LINENO"' ERR
trap 'on_signal 130' INT
trap 'on_signal 143' TERM
assert_regex_count() {
local file="$1"
local pattern="$2"
local expected="$3"
local description="$4"
local actual
actual="$(rg --count --no-filename -- "$pattern" "$file" || true)"
actual="${actual:-0}"
[[ "$actual" == "$expected" ]] || \
fail "${description}: expected ${expected}, found ${actual}"
}
render_kustomization() {
local name="$1"
local relative_path="$2"
local output="${render_temp_dir}/${name}.yaml"
kubectl kustomize "${REPOSITORY_ROOT}/${relative_path}" >"$output"
[[ -s "$output" ]] || fail "${name} rendered an empty manifest"
printf 'Rendered %-30s %8s bytes\n' \
"$name" "$(wc -c <"$output" | tr -d '[:space:]')"
}
cd -- "$REPOSITORY_ROOT"
kubectl create \
--dry-run=client \
--filename=infrastructure/namespaces/phase2/keycloak.yaml \
--output=yaml >"${render_temp_dir}/keycloak-namespace.yaml"
render_kustomization \
platform-postgres-keycloak services/platform-postgres-keycloak
render_kustomization \
keycloak-operator infrastructure/controllers/keycloak-operator
render_kustomization keycloak services/keycloak
if rg --line-number \
'^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \
"${render_temp_dir}"/*.yaml; then
fail "a rendered Keycloak manifest unexpectedly contains a Secret"
fi
assert_regex_count \
"${render_temp_dir}/keycloak-namespace.yaml" \
'^kind:[[:space:]]Namespace$' 1 \
"Keycloak namespace resource count"
assert_regex_count \
"${render_temp_dir}/keycloak-namespace.yaml" \
'^[[:space:]]*name:[[:space:]]keycloak$' 1 \
"Keycloak namespace name"
assert_regex_count \
"${render_temp_dir}/platform-postgres-keycloak.yaml" \
'^kind:[[:space:]]DatabaseRole$' 1 \
"Keycloak DatabaseRole count"
assert_regex_count \
"${render_temp_dir}/platform-postgres-keycloak.yaml" \
'^kind:[[:space:]]Database$' 1 \
"Keycloak Database count"
assert_regex_count \
"${render_temp_dir}/platform-postgres-keycloak.yaml" \
'^kind:[[:space:]]NetworkPolicy$' 1 \
"Keycloak PostgreSQL NetworkPolicy count"
assert_regex_count \
"${render_temp_dir}/platform-postgres-keycloak.yaml" \
'^[[:space:]]*namespace:[[:space:]]platform-data$' 3 \
"Keycloak PostgreSQL resource namespace count"
assert_regex_count \
"${render_temp_dir}/platform-postgres-keycloak.yaml" \
'^[[:space:]]*(databaseRoleReclaimPolicy|databaseReclaimPolicy):[[:space:]]retain$' 2 \
"Keycloak database Retain policy count"
assert_regex_count \
"${render_temp_dir}/platform-postgres-keycloak.yaml" \
'^[[:space:]]*name:[[:space:]]keycloak-db-credentials$' 1 \
"Keycloak DatabaseRole Secret reference"
assert_regex_count \
"${REPOSITORY_ROOT}/infrastructure/controllers/keycloak-operator/kustomization.yaml" \
'github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=26\.7\.0' 1 \
"Keycloak Operator 26.7.0 source pin"
assert_regex_count \
"${render_temp_dir}/keycloak-operator.yaml" \
'^kind:[[:space:]]CustomResourceDefinition$' 4 \
"Keycloak Operator CRD count"
assert_regex_count \
"${render_temp_dir}/keycloak-operator.yaml" \
'^kind:[[:space:]]Deployment$' 1 \
"Keycloak Operator Deployment count"
assert_regex_count \
"${render_temp_dir}/keycloak-operator.yaml" \
'^[[:space:]]*image:[[:space:]]quay\.io/keycloak/keycloak-operator:26\.7\.0$' 1 \
"Keycloak Operator image"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^kind:[[:space:]]Keycloak$' 1 \
"Keycloak custom resource count"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^kind:[[:space:]]Ingress$' 1 \
"Keycloak Ingress count"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*namespace:[[:space:]]keycloak$' 2 \
"Keycloak service resource namespace count"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*instances:[[:space:]]1$' 1 \
"Keycloak instance count"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*hostname:[[:space:]]https://id\.learn\.hyeonworks\.com$' 1 \
"Keycloak external hostname"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*headers:[[:space:]]xforwarded$' 1 \
"Keycloak forwarded header mode"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*ingressClassName:[[:space:]]traefik$' 1 \
"Keycloak Ingress class"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*name:[[:space:]]keycloak-service$' 1 \
"Keycloak Ingress backend"
assert_regex_count \
"${render_temp_dir}/keycloak.yaml" \
'^[[:space:]]*number:[[:space:]]8080$' 1 \
"Keycloak Ingress backend port"
if rg --quiet \
'^[[:space:]]*namespace:[[:space:]](aistor|object-storage)[[:space:]]*$' \
"${render_temp_dir}"/*.yaml; then
fail "the Keycloak-only render contains an AIStor namespace"
fi
if rg --quiet \
'^[[:space:]]*(serviceType|type):[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \
"${render_temp_dir}/keycloak.yaml"; then
fail "Keycloak must not render NodePort or LoadBalancer exposure"
fi
declare -A verified_manifest_sha256=()
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
manifest_path="${render_temp_dir}/${manifest_name}.yaml"
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest is missing or unsafe: ${manifest_path}"
[[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \
fail "verified manifest must have mode 0600: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}"
done
verified_entry_count="$(
find "$render_temp_dir" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]'
)"
[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \
fail "verified handoff must contain exactly four manifest files"
verify_manifest_unchanged() {
local manifest_name="$1"
local manifest_path="${render_temp_dir}/${manifest_name}.yaml"
local checksum_output
local actual_sha256
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest became missing or unsafe: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
actual_sha256="${checksum_output%% *}"
[[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \
fail "verified manifest changed before apply: ${manifest_name}.yaml"
}
current_context="$(kubectl config current-context)"
api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node is missing from the selected cluster: ${TARGET_NODE}"
kubectl get namespace platform-data >/dev/null 2>&1 || \
fail "Phase 1 namespace platform-data is missing"
kubectl get customresourcedefinition \
clusters.postgresql.cnpg.io \
databaseroles.postgresql.cnpg.io \
databases.postgresql.cnpg.io >/dev/null
postgres_ready="$(
kubectl --namespace platform-data get cluster platform-postgres \
--output='go-template={{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}true{{end}}{{end}}'
)"
[[ "$postgres_ready" == "true" ]] || \
fail "platform-data/platform-postgres is not Ready"
printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \
"$current_context" "$api_server" "$TARGET_NODE"
printf '%s\n' \
'Scope: Keycloak namespace, two DB Secrets, DB Role/Database/NetworkPolicy, Operator, Keycloak, and Ingress.' \
'Excluded: AIStor resources, Host Nginx, Gitea OIDC source, and all data deletion.' \
'Rollback boundary: applied state is retained on failure; rerunning is the recovery path.'
[[ -t 0 ]] || fail "an interactive terminal is required"
printf 'Type APPLY to start the Keycloak cluster mutation: '
read -r confirmation
[[ "$confirmation" == "APPLY" ]] || fail "cancelled"
assert_cluster_identity() {
[[ "$(kubectl config current-context)" == "$current_context" ]] || \
fail "kubectl context changed after confirmation"
[[ "$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')" == "$api_server" ]] || \
fail "Kubernetes API server changed after confirmation"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node disappeared after confirmation: ${TARGET_NODE}"
}
assert_cluster_identity
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
verify_manifest_unchanged "$manifest_name"
done
mutation_started=true
current_step="[1/6] Keycloak namespace"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged keycloak-namespace
kubectl apply --dry-run=server \
--filename="${render_temp_dir}/keycloak-namespace.yaml" >/dev/null
kubectl apply --filename="${render_temp_dir}/keycloak-namespace.yaml"
current_step="[2/6] Keycloak database Secret contracts"
printf '\n%s\n' "$current_step"
assert_cluster_identity
bash scripts/bootstrap/create-keycloak-secrets.sh --execute
current_step="[3/6] Keycloak DatabaseRole, Database, and NetworkPolicy"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged platform-postgres-keycloak
kubectl apply --server-side --dry-run=server \
--filename="${render_temp_dir}/platform-postgres-keycloak.yaml" >/dev/null
kubectl apply --server-side \
--filename="${render_temp_dir}/platform-postgres-keycloak.yaml"
kubectl --namespace platform-data wait \
--for=jsonpath='{.status.applied}'=true \
databaserole/platform-postgres-keycloak \
database/platform-postgres-keycloak \
--timeout=3m
current_step="[4/6] Keycloak Operator and CRDs"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged keycloak-operator
kubectl apply --server-side --dry-run=server \
--filename="${render_temp_dir}/keycloak-operator.yaml" >/dev/null
kubectl apply --server-side \
--filename="${render_temp_dir}/keycloak-operator.yaml"
kubectl wait --for=condition=Established \
"${KEYCLOAK_CRDS[@]}" \
--timeout=3m
kubectl --namespace keycloak rollout status \
deployment/keycloak-operator --timeout=5m
current_step="[5/6] Keycloak instance and HTTP Ingress"
printf '\n%s\n' "$current_step"
assert_cluster_identity
verify_manifest_unchanged keycloak
kubectl apply --server-side --dry-run=server \
--filename="${render_temp_dir}/keycloak.yaml" >/dev/null
kubectl apply --server-side \
--filename="${render_temp_dir}/keycloak.yaml"
kubectl --namespace keycloak wait \
--for=condition=Ready keycloak/keycloak --timeout=15m
kubectl --namespace keycloak wait \
--for=jsonpath='{.endpoints[0].conditions.ready}'=true \
endpointslice \
--selector=kubernetes.io/service-name=keycloak-service \
--timeout=2m
current_step="[6/6] OIDC discovery through Traefik HTTP NodePort"
printf '\n%s\n' "$current_step"
discovery_file="${render_temp_dir}/keycloak-discovery.json"
curl --fail --silent --show-error \
--retry 24 \
--retry-all-errors \
--retry-connrefused \
--retry-delay 5 \
--retry-max-time 120 \
--max-time 10 \
--header 'Host: id.learn.hyeonworks.com' \
--output "$discovery_file" \
http://127.0.0.1:30080/realms/master/.well-known/openid-configuration
rg --quiet \
'"issuer"[[:space:]]*:[[:space:]]*"https://id\.learn\.hyeonworks\.com/realms/master"' \
"$discovery_file" || \
fail "OIDC discovery issuer does not match the public Keycloak hostname"
mutation_started=false
printf '\nKeycloak cluster resources and internal OIDC discovery are ready.\n'
printf 'No AIStor resource, Host Nginx configuration, or Gitea OIDC source was changed.\n'
File diff suppressed because it is too large Load Diff
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# Do not allow a caller's `bash -x` setting to expose prompts or future secrets.
set +x
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly -a VERIFIED_MANIFEST_NAMES=(
namespaces
ssd-local-pv
cnpg-operator
platform-postgres
gitea
gitea-oidc
)
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
check_create_only_state() {
if kubectl --namespace gitea get deployment/gitea >/dev/null 2>&1; then
fail "Phase 1 is create-only and gitea/gitea already exists; use a dedicated Gitea lifecycle script"
fi
if kubectl --namespace gitea get secret/gitea-keycloak-oidc >/dev/null 2>&1; then
fail "Phase 1 baseline is blocked because gitea/gitea-keycloak-oidc already exists"
fi
}
usage() {
cat <<'USAGE'
Usage: bash scripts/bootstrap/apply-phase1-gitea.sh --execute
Applies the create-only Gitea baseline in dependency order. It does not modify
Host Nginx, configure Keycloak OIDC, or install Argo CD. If a Gitea Deployment
or OIDC Secret already exists, use the dedicated lifecycle scripts instead.
USAGE
}
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
for command_name in kubectl curl rg sha256sum stat; do
command -v "$command_name" >/dev/null 2>&1 || fail "${command_name} is required"
done
if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then
[[ "$PLATFORM_HELM_BIN" == /* ]] || fail "PLATFORM_HELM_BIN must be an absolute path"
[[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}"
readonly HELM_BIN="$PLATFORM_HELM_BIN"
else
HELM_BIN="$(command -v helm 2>/dev/null)" || \
fail "Helm ${EXPECTED_HELM_VERSION} is required"
readonly HELM_BIN
fi
[[ "$("$HELM_BIN" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \
fail "Helm must be exactly ${EXPECTED_HELM_VERSION}"
render_temp_dir="$(mktemp -d /tmp/platform-phase1-apply.XXXXXX)"
cleanup() {
case "$render_temp_dir" in
/tmp/platform-phase1-apply.*)
rm -rf -- "$render_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render directory: %s\n' \
"$render_temp_dir" >&2
;;
esac
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
cd -- "$REPOSITORY_ROOT"
PLATFORM_HELM_BIN="$HELM_BIN" bash scripts/validate/render-phase1.sh \
--verified-output-dir "$render_temp_dir"
declare -A verified_manifest_sha256=()
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
manifest_path="${render_temp_dir}/${manifest_name}.yaml"
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest is missing or unsafe: ${manifest_path}"
[[ "$(stat --format='%a' -- "$manifest_path")" == "600" ]] || \
fail "verified manifest must have mode 0600: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
verified_manifest_sha256["$manifest_name"]="${checksum_output%% *}"
done
verified_entry_count="$(find "$render_temp_dir" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]')"
[[ "$verified_entry_count" == "${#VERIFIED_MANIFEST_NAMES[@]}" ]] || \
fail "verified handoff must contain exactly ${#VERIFIED_MANIFEST_NAMES[@]} manifest files"
verify_manifest_unchanged() {
local manifest_name="$1"
local manifest_path="${render_temp_dir}/${manifest_name}.yaml"
local checksum_output
local actual_sha256
[[ -f "$manifest_path" && ! -L "$manifest_path" && -O "$manifest_path" && -s "$manifest_path" ]] || \
fail "verified manifest became missing or unsafe: ${manifest_path}"
checksum_output="$(sha256sum -- "$manifest_path")"
actual_sha256="${checksum_output%% *}"
[[ "$actual_sha256" == "${verified_manifest_sha256[$manifest_name]}" ]] || \
fail "verified manifest changed before apply: ${manifest_name}.yaml"
}
current_context="$(kubectl config current-context)"
api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node is missing from the selected cluster: ${TARGET_NODE}"
check_create_only_state
printf '\nKubernetes context: %s\nAPI server: %s\nTarget node: %s\n' \
"$current_context" "$api_server" "$TARGET_NODE"
printf 'Type APPLY %s to start the cluster mutation: ' "$current_context"
read -r confirmation
[[ "$confirmation" == "APPLY ${current_context}" ]] || fail "cancelled"
[[ "$(kubectl config current-context)" == "$current_context" ]] || \
fail "kubectl context changed after confirmation"
confirmed_api_server="$(kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')"
[[ "$confirmed_api_server" == "$api_server" ]] || \
fail "Kubernetes API server changed after confirmation"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node disappeared after confirmation: ${TARGET_NODE}"
check_create_only_state
for manifest_name in "${VERIFIED_MANIFEST_NAMES[@]}"; do
verify_manifest_unchanged "$manifest_name"
done
printf '\n[1/8] Preparing exact SSD Local PV directories\n'
bash scripts/bootstrap/prepare-ssd-local-paths.sh
printf '\n[2/8] Applying Phase 1 namespaces\n'
verify_manifest_unchanged namespaces
kubectl apply --filename="${render_temp_dir}/namespaces.yaml"
printf '\n[3/8] Applying static SSD StorageClasses and Local PVs\n'
verify_manifest_unchanged ssd-local-pv
kubectl apply --filename="${render_temp_dir}/ssd-local-pv.yaml"
printf '\n[4/8] Installing CloudNativePG CRDs and operator\n'
verify_manifest_unchanged cnpg-operator
kubectl apply --server-side --filename="${render_temp_dir}/cnpg-operator.yaml"
kubectl wait --for=condition=Established \
customresourcedefinition/clusters.postgresql.cnpg.io \
customresourcedefinition/databaseroles.postgresql.cnpg.io \
customresourcedefinition/databases.postgresql.cnpg.io \
--timeout=3m
kubectl --namespace cnpg-system wait --for=condition=Available deployment \
--selector=app.kubernetes.io/name=cloudnative-pg --timeout=5m
printf '\n[5/8] Ensuring or reusing the three Secret contracts\n'
bash scripts/bootstrap/create-phase1-secrets.sh --execute
printf '\n[6/8] Applying the shared platform PostgreSQL resources\n'
verify_manifest_unchanged platform-postgres
kubectl apply --server-side --filename="${render_temp_dir}/platform-postgres.yaml"
kubectl --namespace platform-data wait --for=condition=Ready \
cluster/platform-postgres --timeout=10m
kubectl --namespace platform-data wait \
--for=jsonpath='{.status.applied}'=true \
database/platform-postgres-gitea --timeout=3m
printf '\n[7/8] Applying the Gitea baseline, PVC, policies, and HTTP Ingress\n'
verify_manifest_unchanged gitea
kubectl apply --filename="${render_temp_dir}/gitea.yaml"
kubectl --namespace gitea rollout status deployment/gitea --timeout=10m
kubectl --namespace gitea wait \
--for=jsonpath='{.endpoints[0].conditions.ready}'=true \
endpointslice \
--selector=kubernetes.io/service-name=gitea-http \
--timeout=2m
printf '\n[8/8] Checking Host-based routing through Traefik HTTP NodePort\n'
curl --fail-with-body --show-error \
--retry 24 \
--retry-all-errors \
--retry-connrefused \
--retry-delay 5 \
--retry-max-time 120 \
--max-time 10 \
--header 'Host: git.learn.hyeonworks.com' \
http://127.0.0.1:30080/api/healthz
printf '\nPhase 1 cluster resources are ready. Host Nginx was not changed.\n'
printf 'Review infrastructure/networking/host-nginx/README.md for the final cutover.\n'
+287
View File
@@ -0,0 +1,287 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly HOST_SOURCE="$ROOT/infrastructure/networking/private-dns/host"
readonly K8S_SOURCE="$ROOT/infrastructure/networking/private-dns/kubernetes"
readonly LAN_IP="192.168.0.107"
readonly TAIL_IP="100.92.240.34"
readonly TARGET_NODE="donghyeon-system-product-name"
readonly BUSYBOX_IMAGE="docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0"
readonly -a PRIVATE_HOSTS=(git.learn.hyeonworks.com id.learn.hyeonworks.com storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com)
readonly -a PUBLIC_PRIVATE_HOSTS=(storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com)
execute=false
mutation=false
rollback_armed=false
temp=""
lan_was_active=false
tail_was_active=false
lan_was_enabled=false
tail_was_enabled=false
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
사용법:
bash scripts/bootstrap/apply-private-dns.sh
bash scripts/bootstrap/apply-private-dns.sh --execute
인자 없이 실행하면 설정, 주소, 공개 DNS, manifest hash만 검사합니다.
--execute는 두 dnsmasq 인스턴스와 coredns-custom을 적용합니다.
공유기 DHCP DNS와 Tailscale 관리 화면은 변경하지 않습니다.
USAGE
}
if (( $# == 0 )); then
:
elif (( $# == 1 )) && [[ "$1" == "-h" || "$1" == "--help" ]]; then
usage
exit 0
elif (( $# == 1 )) && [[ "$1" == "--execute" ]]; then
execute=true
else
usage >&2
exit 2
fi
for cmd in awk dig find ip install journalctl kubectl rg sha256sum sleep ss stat systemctl systemd-analyze; do
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
[[ -x /usr/sbin/dnsmasq ]] || fail "/usr/sbin/dnsmasq가 없습니다"
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"
for file in dnsmasq-lan.conf dnsmasq-tailscale.conf \
hyeonworks-dnsmasq-lan.service hyeonworks-dnsmasq-tailscale.service; do
[[ -f "$HOST_SOURCE/$file" && ! -L "$HOST_SOURCE/$file" ]] || fail "후보 파일이 없습니다: $file"
done
/usr/sbin/dnsmasq --test --conf-file="$HOST_SOURCE/dnsmasq-lan.conf"
/usr/sbin/dnsmasq --test --conf-file="$HOST_SOURCE/dnsmasq-tailscale.conf"
systemd-analyze verify "$HOST_SOURCE/hyeonworks-dnsmasq-lan.service" \
"$HOST_SOURCE/hyeonworks-dnsmasq-tailscale.service"
kubectl kustomize "$K8S_SOURCE" >"/tmp/private-dns-render.$$"
trap 'rm -f -- "/tmp/private-dns-render.$$"' EXIT
manifest_sha="$(sha256sum "/tmp/private-dns-render.$$" | awk '{print $1}')"
ip -4 address show | rg -q -F "$LAN_IP/" || fail "LAN 주소 $LAN_IP가 호스트에 없습니다"
ip -4 address show | rg -q -F "$TAIL_IP/" || fail "Tailscale 주소 $TAIL_IP가 호스트에 없습니다"
[[ "$(kubectl get node "$TARGET_NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" == True ]] || \
fail "대상 노드가 Ready가 아닙니다"
for name in "${PUBLIC_PRIVATE_HOSTS[@]}"; do
public_a="$(dig +short @1.1.1.1 A "$name" | tr -d '[:space:]')"
public_aaaa="$(dig +short @1.1.1.1 AAAA "$name" | tr -d '[:space:]')"
[[ -z "$public_a" && -z "$public_aaaa" ]] || fail "$name 공개 A/AAAA가 존재합니다"
done
printf 'Current context: %s\n' "$(kubectl config current-context)"
printf 'LAN listener: %s:53\n' "$LAN_IP"
printf 'Tail listener: %s:53\n' "$TAIL_IP"
printf 'CoreDNS SHA-256: %s\n' "$manifest_sha"
printf '공개 private-service A/AAAA: 없음\n'
if [[ "$execute" == false ]]; then
printf 'DRY RUN PASS: --execute를 지정하지 않아 변경하지 않았습니다.\n'
exit 0
fi
[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
context="$(kubectl config current-context)"
api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
printf 'Type APPLY %s to install private DNS: ' "$context"
read -r answer
[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다"
[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다"
[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || \
fail "API server가 바뀌었습니다"
sudo -v
managed_listeners_are_exact() {
local lan_pid tail_pid line protocol local_address
local lan_tcp=0 lan_udp=0 tail_tcp=0 tail_udp=0
local -a listener_lines=()
sudo systemctl is-active --quiet hyeonworks-dnsmasq-lan.service || return 1
sudo systemctl is-active --quiet hyeonworks-dnsmasq-tailscale.service || return 1
lan_pid="$(sudo systemctl show hyeonworks-dnsmasq-lan.service --property MainPID --value)" || return 1
tail_pid="$(sudo systemctl show hyeonworks-dnsmasq-tailscale.service --property MainPID --value)" || return 1
[[ "$lan_pid" =~ ^[1-9][0-9]*$ && "$tail_pid" =~ ^[1-9][0-9]*$ ]] || return 1
sudo cmp -s "$HOST_SOURCE/dnsmasq-lan.conf" /etc/dnsmasq-hyeonworks/lan.conf || return 1
sudo cmp -s "$HOST_SOURCE/dnsmasq-tailscale.conf" /etc/dnsmasq-hyeonworks/tailscale.conf || return 1
sudo cmp -s "$HOST_SOURCE/hyeonworks-dnsmasq-lan.service" \
/etc/systemd/system/hyeonworks-dnsmasq-lan.service || return 1
sudo cmp -s "$HOST_SOURCE/hyeonworks-dnsmasq-tailscale.service" \
/etc/systemd/system/hyeonworks-dnsmasq-tailscale.service || return 1
mapfile -t listener_lines < <(sudo ss -H -lnupt '( sport = :53 )') || return 1
for line in "${listener_lines[@]}"; do
read -r protocol _ _ _ local_address _ <<<"$line"
case "$local_address" in
"$LAN_IP:53")
[[ "$line" == *"pid=$lan_pid,"* ]] || return 1
[[ "$protocol" == tcp ]] && lan_tcp=$((lan_tcp + 1))
[[ "$protocol" == udp ]] && lan_udp=$((lan_udp + 1))
;;
"$TAIL_IP:53")
[[ "$line" == *"pid=$tail_pid,"* ]] || return 1
[[ "$protocol" == tcp ]] && tail_tcp=$((tail_tcp + 1))
[[ "$protocol" == udp ]] && tail_udp=$((tail_udp + 1))
;;
esac
done
(( lan_tcp == 1 && lan_udp == 1 && tail_tcp == 1 && tail_udp == 1 ))
}
if sudo ss -H -lntu '( sport = :53 )' | awk -v lan="$LAN_IP:53" -v tail="$TAIL_IP:53" \
'$5 == lan || $5 == tail {found=1} END {exit found ? 0 : 1}'; then
if managed_listeners_are_exact; then
printf '기존 exact managed DNS listener를 안전한 재적용 대상으로 확인했습니다.\n'
else
fail "대상 LAN/Tailscale 주소의 53번 포트를 exact managed listener가 아닌 프로세스가 사용 중입니다"
fi
fi
umask 077
temp="$(mktemp -d /tmp/platform-private-dns.XXXXXX)"
cm_existed=false
sudo systemctl is-active --quiet hyeonworks-dnsmasq-lan.service && lan_was_active=true
sudo systemctl is-active --quiet hyeonworks-dnsmasq-tailscale.service && tail_was_active=true
[[ "$(sudo systemctl is-enabled hyeonworks-dnsmasq-lan.service 2>/dev/null || true)" == enabled ]] && lan_was_enabled=true
[[ "$(sudo systemctl is-enabled hyeonworks-dnsmasq-tailscale.service 2>/dev/null || true)" == enabled ]] && tail_was_enabled=true
[[ -e /etc/dnsmasq-hyeonworks ]] && sudo cp -a /etc/dnsmasq-hyeonworks "$temp/etc-dnsmasq"
[[ -e /etc/systemd/system/hyeonworks-dnsmasq-lan.service ]] && \
sudo cp -a /etc/systemd/system/hyeonworks-dnsmasq-lan.service "$temp/lan.service"
[[ -e /etc/systemd/system/hyeonworks-dnsmasq-tailscale.service ]] && \
sudo cp -a /etc/systemd/system/hyeonworks-dnsmasq-tailscale.service "$temp/tailscale.service"
if kubectl -n kube-system get configmap coredns-custom -o yaml >"$temp/coredns-custom.yaml" 2>/dev/null; then
cm_existed=true
fi
restore_service_state() {
local unit=$1 was_active=$2 was_enabled=$3
if [[ "$was_enabled" == true ]]; then
sudo systemctl enable "$unit" >/dev/null 2>&1
else
sudo systemctl disable "$unit" >/dev/null 2>&1 || true
fi
if [[ "$was_active" == true ]]; then
sudo systemctl start "$unit" >/dev/null 2>&1
else
sudo systemctl stop "$unit" >/dev/null 2>&1 || true
fi
}
rollback() {
set +e
printf '\nROLLBACK: private DNS 이전 상태를 복원합니다.\n' >&2
sudo systemctl disable --now hyeonworks-dnsmasq-lan.service \
hyeonworks-dnsmasq-tailscale.service >/dev/null 2>&1
if [[ -d "$temp/etc-dnsmasq" ]]; then
sudo rm -rf -- /etc/dnsmasq-hyeonworks
sudo cp -a "$temp/etc-dnsmasq" /etc/dnsmasq-hyeonworks
else
sudo rm -rf -- /etc/dnsmasq-hyeonworks
fi
for unit in lan tailscale; do
target="/etc/systemd/system/hyeonworks-dnsmasq-$unit.service"
if [[ -f "$temp/$unit.service" ]]; then
sudo install -o root -g root -m 0644 "$temp/$unit.service" "$target"
else
sudo rm -f -- "$target"
fi
done
sudo systemctl daemon-reload
restore_service_state hyeonworks-dnsmasq-lan.service "$lan_was_active" "$lan_was_enabled"
restore_service_state hyeonworks-dnsmasq-tailscale.service "$tail_was_active" "$tail_was_enabled"
if [[ "$cm_existed" == true ]]; then
kubectl apply -f "$temp/coredns-custom.yaml" >/dev/null
else
kubectl -n kube-system delete configmap coredns-custom --ignore-not-found >/dev/null
fi
kubectl -n kube-system rollout restart deployment/coredns >/dev/null
kubectl -n kube-system rollout status deployment/coredns --timeout=90s >/dev/null
rollback_armed=false
printf 'ROLLBACK complete.\n' >&2
}
finish() {
rc=$?
trap - EXIT INT TERM
if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then
rollback
fi
if [[ -n "$temp" ]]; then
case "$temp" in
/tmp/platform-private-dns.*) rm -rf -- "$temp" ;;
esac
fi
rm -f -- "/tmp/private-dns-render.$$"
exit "$rc"
}
trap finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
rollback_armed=true
mutation=true
sudo install -d -o root -g root -m 0755 /etc/dnsmasq-hyeonworks
sudo install -o root -g root -m 0644 "$HOST_SOURCE/dnsmasq-lan.conf" /etc/dnsmasq-hyeonworks/lan.conf
sudo install -o root -g root -m 0644 "$HOST_SOURCE/dnsmasq-tailscale.conf" /etc/dnsmasq-hyeonworks/tailscale.conf
sudo install -o root -g root -m 0644 "$HOST_SOURCE/hyeonworks-dnsmasq-lan.service" \
/etc/systemd/system/hyeonworks-dnsmasq-lan.service
sudo install -o root -g root -m 0644 "$HOST_SOURCE/hyeonworks-dnsmasq-tailscale.service" \
/etc/systemd/system/hyeonworks-dnsmasq-tailscale.service
sudo systemctl daemon-reload
sudo systemctl enable --now hyeonworks-dnsmasq-lan.service hyeonworks-dnsmasq-tailscale.service
stable=0
for (( attempt=1; attempt<=20; attempt++ )); do
if sudo systemctl is-active --quiet hyeonworks-dnsmasq-lan.service \
hyeonworks-dnsmasq-tailscale.service \
&& [[ "$(dig +time=1 +tries=1 +short "@$LAN_IP" git.learn.hyeonworks.com A | tail -n1)" == "$LAN_IP" ]] \
&& [[ "$(dig +time=1 +tries=1 +short "@$TAIL_IP" git.learn.hyeonworks.com A | tail -n1)" == "$TAIL_IP" ]]; then
stable=$((stable + 1))
if (( stable >= 3 )); then
break
fi
else
stable=0
fi
sleep 0.5
done
if (( stable < 3 )); then
for unit in hyeonworks-dnsmasq-lan.service hyeonworks-dnsmasq-tailscale.service; do
sudo systemctl status "$unit" --no-pager -n 20 >&2 || true
sudo journalctl -u "$unit" --no-pager -n 20 >&2 || true
done
fail "private DNS listener가 안정화되지 않았습니다"
fi
kubectl apply -f "/tmp/private-dns-render.$$"
kubectl -n kube-system rollout restart deployment/coredns
kubectl -n kube-system rollout status deployment/coredns --timeout=90s
for resolver in "$LAN_IP" "$TAIL_IP"; do
for name in "${PRIVATE_HOSTS[@]}"; do
[[ "$(dig +short "@$resolver" A "$name" | tail -n1)" == "$resolver" ]] ||
fail "$name private DNS 검증 실패"
done
done
pod="private-dns-smoke-$(date +%H%M%S)"
kubectl -n default run "$pod" --restart=Never --image="$BUSYBOX_IMAGE" \
--labels=platform.hyeonworks.com/transient=true \
--command -- sh -c 'nslookup git.learn.hyeonworks.com >/dev/null && nslookup id.learn.hyeonworks.com >/dev/null && nslookup storage-admin.learn.hyeonworks.com >/dev/null && nslookup db-admin.learn.hyeonworks.com >/dev/null && nslookup grafana.learn.hyeonworks.com >/dev/null'
kubectl -n default wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$pod" --timeout=60s
kubectl -n default delete "pod/$pod" --wait=true >/dev/null
rollback_armed=false
printf 'PRIVATE DNS APPLY SUCCESS\n'
printf '공유기 DHCP DNS와 Tailscale split DNS는 문서에 따라 별도로 등록하세요.\n'
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/bash
set -Eeuo pipefail
set +x
umask 077
ulimit -c 0
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
_swr_wrapper_directory="${BASH_SOURCE[0]%/*}"
else
_swr_wrapper_directory='.'
fi
readonly SWR_REPOSITORY_ROOT="$(cd -- "${_swr_wrapper_directory}/../.." && pwd -P)"
readonly SWR_RECOVERY_CONTRACT="${SWR_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env"
unset _swr_wrapper_directory
# shellcheck source=/dev/null
source "${SWR_REPOSITORY_ROOT}/scripts/lib/slack-webhook-recovery.sh"
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_swr_main "$@"
fi
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
PATH='/usr/sbin:/usr/bin:/sbin:/bin'
export PATH
LC_ALL=C
export LC_ALL
umask 077
_k3slr_wrapper_initial_guard() {
local effective_uid="${1-}" shell_options="${2-}"
[[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1
[[ "$shell_options" != *x* ]]
}
if ! _k3slr_wrapper_initial_guard "${EUID:-}" "$-"; then
printf 'ERROR: lifecycle wrapper refuses root or xtrace execution\n' >&2
return 1 2>/dev/null || exit 1
fi
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
K3SLR_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}"
else
K3SLR_WRAPPER_DIRECTORY='.'
fi
K3SLR_WRAPPER_ROOT="$(cd -- "${K3SLR_WRAPPER_DIRECTORY}/../.." && pwd -P)" || {
return 1 2>/dev/null || exit 1
}
K3SLR_WRAPPER_CONTRACT="${K3SLR_WRAPPER_ROOT}/infrastructure/security/k3s/local-recovery.env"
# shellcheck source=/dev/null
source "${K3SLR_WRAPPER_ROOT}/scripts/lib/k3s-local-recovery.sh" || {
return 1 2>/dev/null || exit 1
}
_k3slr_close_usage() {
printf 'Usage: bash scripts/bootstrap/close-k3s-local-recovery.sh [--execute]\n' >&2
}
_k3slr_close_main() {
local execution_mode
if ! _k3slr_parse_lifecycle_cli execution_mode "$@"; then
_k3slr_close_usage
return 2
fi
_k3slr_lifecycle_main close "$execution_mode"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_k3slr_close_main "$@"
fi
+400
View File
@@ -0,0 +1,400 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set +x
umask 077
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly KC_NS=keycloak
readonly KC_NAME=keycloak
readonly KC_SERVICE=keycloak-service
readonly KC_HOST=id.learn.hyeonworks.com
readonly REALM=hyeonworks
readonly PORT=18081
readonly BASE=http://127.0.0.1:$PORT
execute=false
object_admin=""
db_admin=""
temp=""
pf=""
rollback_armed=false
ai_client_created=false
pg_client_created=false
ai_group_created=false
pg_group_created=false
ai_member_added=false
pg_member_added=false
ai_client_uuid=""
pg_client_uuid=""
ai_group_uuid=""
pg_group_uuid=""
ai_user_uuid=""
pg_user_uuid=""
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
사용법:
bash scripts/bootstrap/configure-keycloak-admin-oidc.sh
bash scripts/bootstrap/configure-keycloak-admin-oidc.sh --execute \
--object-admin donghyeon.kang --db-admin donghyeon.kang
두 confidential client, 두 관리자 그룹, claim mapper와 다음 Secret을 만듭니다.
- object-storage/aistor-keycloak-oidc: client-id, client-secret
- platform-admin/pgadmin-keycloak-oidc: client-id, client-secret
USAGE
}
while (( $# > 0 )); do
case "$1" in
--execute) execute=true; shift ;;
--object-admin)
(( $# >= 2 )) || fail "--object-admin 값이 필요합니다"
object_admin="$2"; shift 2 ;;
--db-admin)
(( $# >= 2 )) || fail "--db-admin 값이 필요합니다"
db_admin="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) usage >&2; fail "지원하지 않는 인자: $1" ;;
esac
done
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"
for cmd in base64 chmod curl jq kill kubectl mktemp rg rm seq sleep sort tr; do
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
context="$(kubectl config current-context)"
api="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
kubectl -n "$KC_NS" wait --for=condition=Ready "keycloak.k8s.keycloak.org/$KC_NAME" --timeout=30s >/dev/null
kubectl -n "$KC_NS" get "secret/$KC_NAME-initial-admin" >/dev/null
printf 'Current context: %s\nRealm: %s\n' "$context" "$REALM"
printf 'Clients: aistor-console, pgadmin\n'
printf 'Groups: /platform-object-admins, /platform-db-admins\n'
[[ -n "$object_admin" ]] && printf 'Object admin: %s\n' "$object_admin"
[[ -n "$db_admin" ]] && printf 'DB admin: %s\n' "$db_admin"
if [[ "$execute" == false ]]; then
printf 'DRY RUN PASS: --execute를 지정하지 않아 변경하지 않았습니다.\n'
exit 0
fi
[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
printf 'Type APPLY %s to configure admin OIDC: ' "$context"
read -r answer
[[ "$answer" == "APPLY $context" ]] || fail "취소했습니다"
[[ "$(kubectl config current-context)" == "$context" ]] || fail "context가 바뀌었습니다"
[[ "$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" == "$api" ]] || fail "API server가 바뀌었습니다"
kubectl apply -k "$ROOT/infrastructure/namespaces/admin-tools" >/dev/null
temp="$(mktemp -d /tmp/keycloak-admin-oidc.XXXXXX)"
admin_user="$temp/admin-user"
admin_password="$temp/admin-password"
token_json="$temp/token.json"
token="$temp/token"
auth="$temp/auth.conf"
response="$temp/response.json"
early_cleanup() {
rc=$?
trap - EXIT INT TERM
if [[ -n "$pf" ]] && kill -0 "$pf" 2>/dev/null; then
kill "$pf" 2>/dev/null || true
wait "$pf" 2>/dev/null || true
fi
case "$temp" in /tmp/keycloak-admin-oidc.*) rm -rf -- "$temp" ;; esac
exit "$rc"
}
trap early_cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
kubectl -n "$KC_NS" get "secret/$KC_NAME-initial-admin" \
-o jsonpath='{.data.username}' | base64 -d >"$admin_user"
kubectl -n "$KC_NS" get "secret/$KC_NAME-initial-admin" \
-o jsonpath='{.data.password}' | base64 -d >"$admin_password"
chmod 0600 "$admin_user" "$admin_password"
kubectl -n "$KC_NS" port-forward --address=127.0.0.1 "service/$KC_SERVICE" \
"$PORT:8080" >"$temp/port-forward.log" 2>&1 &
pf=$!
ready=false
for _ in $(seq 1 30); do
kill -0 "$pf" 2>/dev/null || fail "Keycloak port-forward가 종료됐습니다"
if rg -q -F "Forwarding from 127.0.0.1:$PORT -> 8080" "$temp/port-forward.log"; then
ready=true
break
fi
sleep 1
done
[[ "$ready" == true ]] || fail "Keycloak port-forward가 준비되지 않았습니다"
kc_curl() {
curl --disable --silent --show-error --noproxy '*' \
--connect-timeout 3 --max-time 20 \
--header "Host: $KC_HOST" \
--header 'X-Forwarded-Proto: https' \
--header 'X-Forwarded-Port: 443' "$@"
}
kc_curl --fail-with-body --output "$token_json" \
--data-urlencode grant_type=password --data-urlencode client_id=admin-cli \
--data-urlencode "username@$admin_user" --data-urlencode "password@$admin_password" \
"$BASE/realms/master/protocol/openid-connect/token" >/dev/null
jq -er '.access_token | select(type == "string" and length > 0)' "$token_json" >"$token"
{
printf 'header = "Authorization: Bearer '
tr -d '\r\n' <"$token"
printf '"\n'
} >"$auth"
chmod 0600 "$token" "$auth"
request() {
kc_curl --config "$auth" "$@"
}
backup_secret() {
if kubectl -n "$1" get "secret/$2" -o yaml >"$3" 2>/dev/null; then
printf true
else
printf false
fi
}
ai_secret_existed="$(backup_secret object-storage aistor-keycloak-oidc "$temp/ai-secret.yaml")"
pg_secret_existed="$(backup_secret platform-admin pgadmin-keycloak-oidc "$temp/pg-secret.yaml")"
upsert_group() {
group_name="$1"
policy="$2"
before="$3"
body="$temp/group.json"
request --get --data-urlencode "search=$group_name" --data-urlencode exact=true \
-o "$response" "$BASE/admin/realms/$REALM/groups"
group_id="$(jq -r --arg path "/$group_name" '[.[] | select(.path == $path)][0].id // empty' "$response")"
if [[ -n "$policy" ]]; then
jq -n --arg name "$group_name" --arg policy "$policy" \
'{name:$name,attributes:{policy:[$policy]}}' >"$body"
else
jq -n --arg name "$group_name" '{name:$name}' >"$body"
fi
if [[ -z "$group_id" ]]; then
code="$(request -X POST -H 'Content-Type: application/json' --data-binary "@$body" \
-o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/groups")"
[[ "$code" == 201 ]] || fail "그룹 생성 실패: $group_name HTTP $code"
request --get --data-urlencode "search=$group_name" --data-urlencode exact=true \
-o "$response" "$BASE/admin/realms/$REALM/groups"
group_id="$(jq -er --arg path "/$group_name" '[.[] | select(.path == $path)][0].id' "$response")"
last_created=true
else
request -o "$before" "$BASE/admin/realms/$REALM/groups/$group_id"
code="$(request -X PUT -H 'Content-Type: application/json' --data-binary "@$body" \
-o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/groups/$group_id")"
[[ "$code" == 204 ]] || fail "그룹 갱신 실패: $group_name HTTP $code"
last_created=false
fi
last_id="$group_id"
}
upsert_client() {
client="$1"
display="$2"
root_url="$3"
redirect="$4"
before="$5"
body="$temp/client.json"
jq -n --arg client "$client" --arg display "$display" --arg root "$root_url" \
--arg redirect "$redirect" '{
clientId:$client,name:$display,enabled:true,protocol:"openid-connect",
clientAuthenticatorType:"client-secret",publicClient:false,
standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,
serviceAccountsEnabled:false,authorizationServicesEnabled:false,
consentRequired:false,fullScopeAllowed:false,rootUrl:$root,baseUrl:($root+"/"),
redirectUris:[$redirect],webOrigins:[$root],
attributes:{"oauth2.device.authorization.grant.enabled":"false",
"oidc.ciba.grant.enabled":"false","post.logout.redirect.uris":($root+"/*")}
}' >"$body"
request --get --data-urlencode "clientId=$client" --data-urlencode max=2 \
-o "$response" "$BASE/admin/realms/$REALM/clients"
count="$(jq 'length' "$response")"
[[ "$count" == 0 || "$count" == 1 ]] || fail "$client clientId가 중복됐습니다"
if [[ "$count" == 0 ]]; then
code="$(request -X POST -H 'Content-Type: application/json' --data-binary "@$body" \
-o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/clients")"
[[ "$code" == 201 ]] || fail "client 생성 실패: $client HTTP $code"
last_created=true
else
client_id="$(jq -er '.[0].id' "$response")"
request -o "$before" "$BASE/admin/realms/$REALM/clients/$client_id"
code="$(request -X PUT -H 'Content-Type: application/json' --data-binary "@$body" \
-o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/clients/$client_id")"
[[ "$code" == 204 ]] || fail "client 갱신 실패: $client HTTP $code"
last_created=false
fi
request --get --data-urlencode "clientId=$client" --data-urlencode max=2 \
-o "$response" "$BASE/admin/realms/$REALM/clients"
last_id="$(jq -er 'select(length == 1) | .[0].id' "$response")"
}
upsert_mapper() {
client_id="$1"
mapper_name="$2"
mapper_type="$3"
body="$temp/mapper.json"
if [[ "$mapper_type" == group ]]; then
jq -n --arg name "$mapper_name" '{
name:$name,protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper",
consentRequired:false,config:{"claim.name":"groups","full.path":"true",
"id.token.claim":"true","access.token.claim":"true",
"userinfo.token.claim":"true","introspection.token.claim":"true"}
}' >"$body"
else
jq -n --arg name "$mapper_name" '{
name:$name,protocol:"openid-connect",protocolMapper:"oidc-usermodel-attribute-mapper",
consentRequired:false,config:{"user.attribute":"policy","claim.name":"policy",
"jsonType.label":"String","multivalued":"true","aggregate.attrs":"true",
"id.token.claim":"true","access.token.claim":"true",
"userinfo.token.claim":"true","introspection.token.claim":"true"}
}' >"$body"
fi
request -o "$response" "$BASE/admin/realms/$REALM/clients/$client_id/protocol-mappers/models"
mapper_id="$(jq -r --arg name "$mapper_name" '[.[] | select(.name == $name)][0].id // empty' "$response")"
if [[ -z "$mapper_id" ]]; then
code="$(request -X POST -H 'Content-Type: application/json' --data-binary "@$body" \
-o "$response" -w '%{http_code}' "$BASE/admin/realms/$REALM/clients/$client_id/protocol-mappers/models")"
[[ "$code" == 201 ]] || fail "mapper 생성 실패: $mapper_name HTTP $code"
else
code="$(request -X PUT -H 'Content-Type: application/json' --data-binary "@$body" \
-o "$response" -w '%{http_code}' \
"$BASE/admin/realms/$REALM/clients/$client_id/protocol-mappers/models/$mapper_id")"
[[ "$code" == 204 ]] || fail "mapper 갱신 실패: $mapper_name HTTP $code"
fi
}
assign_member() {
username="$1"
group_id="$2"
request --get --data-urlencode "username=$username" --data-urlencode exact=true \
-o "$response" "$BASE/admin/realms/$REALM/users"
[[ "$(jq 'length' "$response")" == 1 ]] || fail "사용자를 정확히 찾지 못했습니다: $username"
user_id="$(jq -er '.[0].id' "$response")"
request -o "$response" "$BASE/admin/realms/$REALM/users/$user_id/groups"
if jq -e --arg id "$group_id" 'any(.[]; .id == $id)' "$response" >/dev/null; then
last_member_added=false
else
code="$(request -X PUT -o "$response" -w '%{http_code}' \
"$BASE/admin/realms/$REALM/users/$user_id/groups/$group_id")"
[[ "$code" == 204 ]] || fail "그룹 구성원 추가 실패: $username HTTP $code"
last_member_added=true
fi
last_user_id="$user_id"
}
restore_secret() {
if [[ "$1" == true ]]; then
kubectl apply -f "$2" >/dev/null 2>&1 || true
else
kubectl -n "$3" delete "secret/$4" --ignore-not-found >/dev/null 2>&1
fi
}
rollback() {
set +e
printf '\nROLLBACK: Keycloak 및 OIDC Secret 이전 상태 복원\n' >&2
[[ "$ai_member_added" == true ]] && \
request -X DELETE "$BASE/admin/realms/$REALM/users/$ai_user_uuid/groups/$ai_group_uuid" >/dev/null
[[ "$pg_member_added" == true ]] && \
request -X DELETE "$BASE/admin/realms/$REALM/users/$pg_user_uuid/groups/$pg_group_uuid" >/dev/null
for item in "ai $ai_client_uuid $ai_client_created" "pg $pg_client_uuid $pg_client_created"; do
set -- $item
if [[ "$3" == true ]]; then
request -X DELETE "$BASE/admin/realms/$REALM/clients/$2" >/dev/null
elif [[ -n "$2" ]]; then
request -X PUT -H 'Content-Type: application/json' \
--data-binary "@$temp/$1-client-before.json" \
"$BASE/admin/realms/$REALM/clients/$2" >/dev/null
fi
done
for item in "ai $ai_group_uuid $ai_group_created" "pg $pg_group_uuid $pg_group_created"; do
set -- $item
if [[ "$3" == true ]]; then
request -X DELETE "$BASE/admin/realms/$REALM/groups/$2" >/dev/null
elif [[ -n "$2" ]]; then
request -X PUT -H 'Content-Type: application/json' \
--data-binary "@$temp/$1-group-before.json" \
"$BASE/admin/realms/$REALM/groups/$2" >/dev/null
fi
done
restore_secret "$ai_secret_existed" "$temp/ai-secret.yaml" object-storage aistor-keycloak-oidc
restore_secret "$pg_secret_existed" "$temp/pg-secret.yaml" platform-admin pgadmin-keycloak-oidc
rollback_armed=false
printf 'ROLLBACK complete.\n' >&2
}
cleanup() {
rc=$?
trap - EXIT INT TERM
if (( rc != 0 )) && [[ "$rollback_armed" == true ]]; then rollback; fi
if [[ -n "$pf" ]] && kill -0 "$pf" 2>/dev/null; then
kill "$pf" 2>/dev/null || true
wait "$pf" 2>/dev/null || true
fi
case "$temp" in /tmp/keycloak-admin-oidc.*) rm -rf -- "$temp" ;; esac
exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
rollback_armed=true
upsert_group platform-object-admins consoleAdmin "$temp/ai-group-before.json"
ai_group_uuid="$last_id"; ai_group_created="$last_created"
upsert_group platform-db-admins "" "$temp/pg-group-before.json"
pg_group_uuid="$last_id"; pg_group_created="$last_created"
upsert_client aistor-console "AIStor Console" https://storage-admin.learn.hyeonworks.com \
https://storage-admin.learn.hyeonworks.com/oauth_callback "$temp/ai-client-before.json"
ai_client_uuid="$last_id"; ai_client_created="$last_created"
upsert_mapper "$ai_client_uuid" aistor-policy-claim policy
upsert_client pgadmin pgAdmin https://db-admin.learn.hyeonworks.com \
https://db-admin.learn.hyeonworks.com/oauth2/authorize "$temp/pg-client-before.json"
pg_client_uuid="$last_id"; pg_client_created="$last_created"
upsert_mapper "$pg_client_uuid" pgadmin-groups group
for item in "object-storage aistor-keycloak-oidc aistor-console $ai_client_uuid ai" \
"platform-admin pgadmin-keycloak-oidc pgadmin $pg_client_uuid pg"; do
set -- $item
secret_file="$temp/$5-client-secret"
request -o "$response" "$BASE/admin/realms/$REALM/clients/$4/client-secret"
jq --exit-status --join-output --raw-output '.value | select(type == "string" and length >= 16)' "$response" >"$secret_file"
chmod 0600 "$secret_file"
kubectl -n "$1" create secret generic "$2" --from-literal="client-id=$3" \
--from-file="client-secret=$secret_file" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
kubectl -n "$1" label "secret/$2" app.kubernetes.io/component=oidc-client \
app.kubernetes.io/part-of=platform app.kubernetes.io/managed-by=bootstrap-script \
--overwrite >/dev/null
done
if [[ -n "$object_admin" ]]; then
assign_member "$object_admin" "$ai_group_uuid"
ai_member_added="$last_member_added"; ai_user_uuid="$last_user_id"
fi
if [[ -n "$db_admin" ]]; then
assign_member "$db_admin" "$pg_group_uuid"
pg_member_added="$last_member_added"; pg_user_uuid="$last_user_id"
fi
for contract in "object-storage aistor-keycloak-oidc" "platform-admin pgadmin-keycloak-oidc"; do
set -- $contract
keys="$(kubectl -n "$1" get "secret/$2" \
-o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)"
[[ "$keys" == $'client-id\nclient-secret' ]] || fail "$1/$2 Secret 계약이 다릅니다"
done
request -o "$response" "$BASE/realms/$REALM/.well-known/openid-configuration"
jq -e --arg issuer "https://$KC_HOST/realms/$REALM" '.issuer == $issuer' "$response" >/dev/null || \
fail "OIDC discovery issuer가 다릅니다"
rollback_armed=false
printf 'KEYCLOAK ADMIN OIDC CONFIG SUCCESS\n'
printf 'Secret payload, 관리자 암호, token은 출력하지 않았습니다.\n'
+445
View File
@@ -0,0 +1,445 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# Never inherit caller xtrace: this script handles bootstrap credentials,
# bearer tokens, and the Gitea OIDC client secret.
set +x
umask 077
readonly REPOSITORY_ROOT="/home/donghyeon/workspace/platform"
readonly KEYCLOAK_NAMESPACE="keycloak"
readonly KEYCLOAK_NAME="keycloak"
readonly KEYCLOAK_SERVICE="keycloak-service"
readonly KEYCLOAK_HOST="id.learn.hyeonworks.com"
readonly KEYCLOAK_REALM="hyeonworks"
readonly GITEA_NAMESPACE="gitea"
readonly GITEA_OIDC_SECRET="gitea-keycloak-oidc"
readonly GITEA_CLIENT_ID="gitea"
readonly GITEA_ROOT_URL="https://git.learn.hyeonworks.com"
readonly GITEA_REDIRECT_URI="${GITEA_ROOT_URL}/user/oauth2/keycloak/callback"
readonly LOCAL_PORT="${KEYCLOAK_LOCAL_PORT:-18080}"
readonly LOCAL_BASE_URL="http://127.0.0.1:${LOCAL_PORT}"
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
printf '%s\n' \
'Usage: bash scripts/bootstrap/configure-keycloak-gitea-oidc.sh --execute' \
'' \
'Uses the Operator-generated temporary Keycloak administrator through a' \
'loopback-only kubectl port-forward. It creates or updates the hyeonworks' \
'realm and confidential Gitea client, then writes only the generated client' \
'credential to gitea/gitea-keycloak-oidc.' \
'' \
'No credential, token, or Secret payload is printed or written to Git.'
}
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
[[ "$LOCAL_PORT" =~ ^[0-9]+$ ]] || fail "KEYCLOAK_LOCAL_PORT must be numeric"
(( LOCAL_PORT >= 1024 && LOCAL_PORT <= 65535 )) || \
fail "KEYCLOAK_LOCAL_PORT must be between 1024 and 65535"
[[ "$PWD" == "$REPOSITORY_ROOT" ]] || \
fail "run from ${REPOSITORY_ROOT}"
[[ -t 0 ]] || fail "an interactive terminal is required"
for required_binary in \
kubectl curl jq base64 mktemp chmod kill sleep seq rg sed tr sort rm; do
command -v "$required_binary" >/dev/null 2>&1 || \
fail "${required_binary} is required"
done
readonly CURRENT_CONTEXT="$(kubectl config current-context)"
readonly API_SERVER="$(
kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}'
)"
readonly TARGET_NODE="$(
kubectl get nodes \
--selector='node-role.kubernetes.io/control-plane' \
--output=jsonpath='{.items[0].metadata.name}'
)"
[[ -n "$CURRENT_CONTEXT" ]] || fail "kubectl current-context is empty"
[[ -n "$API_SERVER" ]] || fail "the selected Kubernetes API server is empty"
[[ -n "$TARGET_NODE" ]] || fail "the control-plane node was not found"
kubectl get namespace "$KEYCLOAK_NAMESPACE" >/dev/null
kubectl get namespace "$GITEA_NAMESPACE" >/dev/null
kubectl --namespace "$KEYCLOAK_NAMESPACE" get \
"keycloak.k8s.keycloak.org/${KEYCLOAK_NAME}" >/dev/null
kubectl --namespace "$KEYCLOAK_NAMESPACE" wait \
--for=condition=Ready \
"keycloak.k8s.keycloak.org/${KEYCLOAK_NAME}" \
--timeout=30s >/dev/null
kubectl --namespace "$KEYCLOAK_NAMESPACE" get \
"service/${KEYCLOAK_SERVICE}" >/dev/null
kubectl --namespace "$KEYCLOAK_NAMESPACE" get \
"secret/${KEYCLOAK_NAME}-initial-admin" >/dev/null
printf 'Current context: %s\n' "$CURRENT_CONTEXT"
printf 'API server: %s\n' "$API_SERVER"
printf 'Target node: %s\n' "$TARGET_NODE"
printf 'Realm: %s\n' "$KEYCLOAK_REALM"
printf 'OIDC client: %s\n' "$GITEA_CLIENT_ID"
printf 'Redirect URI: %s\n' "$GITEA_REDIRECT_URI"
printf 'Type APPLY %s to configure Keycloak and create the Gitea OIDC Secret: ' \
"$CURRENT_CONTEXT"
read -r confirmation
[[ "$confirmation" == "APPLY ${CURRENT_CONTEXT}" ]] || fail "cancelled"
[[ "$(kubectl config current-context)" == "$CURRENT_CONTEXT" ]] || \
fail "kubectl context changed after confirmation"
[[ "$(
kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}'
)" == "$API_SERVER" ]] || fail "Kubernetes API server changed after confirmation"
[[ "$(
kubectl get nodes \
--selector='node-role.kubernetes.io/control-plane' \
--output=jsonpath='{.items[0].metadata.name}'
)" == "$TARGET_NODE" ]] || fail "target control-plane node changed after confirmation"
kubectl get node "$TARGET_NODE" >/dev/null 2>&1 || \
fail "target node disappeared after confirmation: ${TARGET_NODE}"
readonly TEMP_DIR="$(mktemp -d /tmp/keycloak-gitea-oidc.XXXXXX)"
readonly PORT_FORWARD_LOG="${TEMP_DIR}/port-forward.log"
readonly ADMIN_USERNAME_FILE="${TEMP_DIR}/admin-username"
readonly ADMIN_PASSWORD_FILE="${TEMP_DIR}/admin-password"
readonly TOKEN_RESPONSE_FILE="${TEMP_DIR}/token-response.json"
readonly ACCESS_TOKEN_FILE="${TEMP_DIR}/access-token"
readonly AUTH_CONFIG_FILE="${TEMP_DIR}/curl-auth.conf"
readonly REALM_FILE="${TEMP_DIR}/realm.json"
readonly CLIENT_FILE="${TEMP_DIR}/client.json"
readonly RESPONSE_FILE="${TEMP_DIR}/response.json"
readonly CLIENT_SECRET_RESPONSE_FILE="${TEMP_DIR}/client-secret.json"
readonly CLIENT_SECRET_FILE="${TEMP_DIR}/client-secret"
port_forward_pid=""
cleanup() {
local cleanup_rc=$?
if [[ -n "$port_forward_pid" ]] && kill -0 "$port_forward_pid" 2>/dev/null; then
kill "$port_forward_pid" 2>/dev/null || true
wait "$port_forward_pid" 2>/dev/null || true
fi
case "$TEMP_DIR" in
/tmp/keycloak-gitea-oidc.*)
rm -rf -- "$TEMP_DIR"
;;
*)
printf 'WARNING: refusing to remove unexpected temp path: %s\n' \
"$TEMP_DIR" >&2
;;
esac
exit "$cleanup_rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
kubectl --namespace "$KEYCLOAK_NAMESPACE" get \
"secret/${KEYCLOAK_NAME}-initial-admin" \
--output=jsonpath='{.data.username}' \
| base64 --decode >"$ADMIN_USERNAME_FILE"
kubectl --namespace "$KEYCLOAK_NAMESPACE" get \
"secret/${KEYCLOAK_NAME}-initial-admin" \
--output=jsonpath='{.data.password}' \
| base64 --decode >"$ADMIN_PASSWORD_FILE"
chmod 0600 "$ADMIN_USERNAME_FILE" "$ADMIN_PASSWORD_FILE"
[[ -s "$ADMIN_USERNAME_FILE" ]] || fail "temporary admin username is empty"
[[ -s "$ADMIN_PASSWORD_FILE" ]] || fail "temporary admin password is empty"
kubectl --namespace "$KEYCLOAK_NAMESPACE" port-forward \
--address=127.0.0.1 \
"service/${KEYCLOAK_SERVICE}" \
"${LOCAL_PORT}:8080" >"$PORT_FORWARD_LOG" 2>&1 &
port_forward_pid=$!
port_forward_ready=0
for _ in $(seq 1 30); do
kill -0 "$port_forward_pid" 2>/dev/null || {
printf 'Port-forward failed; non-sensitive log follows:\n' >&2
sed -n '1,20p' "$PORT_FORWARD_LOG" >&2
fail "Keycloak port-forward exited"
}
if rg --quiet --fixed-strings \
"Forwarding from 127.0.0.1:${LOCAL_PORT} -> 8080" \
"$PORT_FORWARD_LOG"; then
port_forward_ready=1
break
fi
sleep 1
done
(( port_forward_ready == 1 )) || fail "Keycloak port-forward did not become ready"
curl_common=(
--disable
--silent
--show-error
--noproxy '*'
--connect-timeout 3
--max-time 20
--header "Host: ${KEYCLOAK_HOST}"
--header 'X-Forwarded-Proto: https'
--header 'X-Forwarded-Port: 443'
)
curl "${curl_common[@]}" \
--fail-with-body \
--output "$TOKEN_RESPONSE_FILE" \
--data-urlencode 'grant_type=password' \
--data-urlencode 'client_id=admin-cli' \
--data-urlencode "username@${ADMIN_USERNAME_FILE}" \
--data-urlencode "password@${ADMIN_PASSWORD_FILE}" \
"${LOCAL_BASE_URL}/realms/master/protocol/openid-connect/token" \
>/dev/null || fail "temporary Keycloak administrator authentication failed"
jq --exit-status --raw-output \
'.access_token | select(type == "string" and length > 0)' \
"$TOKEN_RESPONSE_FILE" >"$ACCESS_TOKEN_FILE" || \
fail "Keycloak token response did not contain an access token"
chmod 0600 "$ACCESS_TOKEN_FILE"
{
printf 'header = "Authorization: Bearer '
tr -d '\r\n' <"$ACCESS_TOKEN_FILE"
printf '"\n'
} >"$AUTH_CONFIG_FILE"
chmod 0600 "$AUTH_CONFIG_FILE"
cat >"$REALM_FILE" <<'JSON'
{
"realm": "hyeonworks",
"displayName": "Hyeonworks",
"enabled": true,
"sslRequired": "external",
"registrationAllowed": false,
"registrationEmailAsUsername": false,
"rememberMe": true,
"verifyEmail": false,
"loginWithEmailAllowed": true,
"duplicateEmailsAllowed": false,
"resetPasswordAllowed": true,
"editUsernameAllowed": false,
"bruteForceProtected": true,
"permanentLockout": false,
"maxFailureWaitSeconds": 900,
"minimumQuickLoginWaitSeconds": 60,
"waitIncrementSeconds": 60,
"quickLoginCheckMilliSeconds": 1000,
"maxDeltaTimeSeconds": 43200,
"failureFactor": 5,
"internationalizationEnabled": true,
"supportedLocales": ["ko", "en"],
"defaultLocale": "ko"
}
JSON
cat >"$CLIENT_FILE" <<'JSON'
{
"clientId": "gitea",
"name": "Hyeonworks Gitea",
"description": "Gitea confidential OIDC client managed by the platform bootstrap",
"enabled": true,
"protocol": "openid-connect",
"clientAuthenticatorType": "client-secret",
"publicClient": false,
"standardFlowEnabled": true,
"implicitFlowEnabled": false,
"directAccessGrantsEnabled": false,
"serviceAccountsEnabled": false,
"authorizationServicesEnabled": false,
"consentRequired": false,
"fullScopeAllowed": false,
"rootUrl": "https://git.learn.hyeonworks.com",
"baseUrl": "https://git.learn.hyeonworks.com/",
"redirectUris": [
"https://git.learn.hyeonworks.com/user/oauth2/keycloak/callback"
],
"webOrigins": [
"https://git.learn.hyeonworks.com"
],
"attributes": {
"post.logout.redirect.uris": "https://git.learn.hyeonworks.com/*",
"oauth2.device.authorization.grant.enabled": "false",
"oidc.ciba.grant.enabled": "false"
}
}
JSON
admin_request() {
curl "${curl_common[@]}" \
--config "$AUTH_CONFIG_FILE" \
"$@"
}
realm_status="$(
admin_request \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}"
)"
case "$realm_status" in
200)
update_status="$(
admin_request \
--request PUT \
--header 'Content-Type: application/json' \
--data-binary "@${REALM_FILE}" \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}"
)"
[[ "$update_status" == "204" ]] || \
fail "Keycloak realm update failed with HTTP ${update_status}"
printf 'Updated Keycloak realm %s.\n' "$KEYCLOAK_REALM"
;;
404)
create_status="$(
admin_request \
--request POST \
--header 'Content-Type: application/json' \
--data-binary "@${REALM_FILE}" \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms"
)"
[[ "$create_status" == "201" ]] || \
fail "Keycloak realm creation failed with HTTP ${create_status}"
printf 'Created Keycloak realm %s.\n' "$KEYCLOAK_REALM"
;;
*)
fail "Keycloak realm lookup failed with HTTP ${realm_status}"
;;
esac
client_lookup_status="$(
admin_request \
--get \
--data-urlencode "clientId=${GITEA_CLIENT_ID}" \
--data-urlencode 'max=2' \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients"
)"
[[ "$client_lookup_status" == "200" ]] || \
fail "Keycloak client lookup failed with HTTP ${client_lookup_status}"
client_count="$(jq 'length' "$RESPONSE_FILE")"
[[ "$client_count" == "0" || "$client_count" == "1" ]] || \
fail "more than one Keycloak client uses clientId=${GITEA_CLIENT_ID}"
if [[ "$client_count" == "0" ]]; then
client_create_status="$(
admin_request \
--request POST \
--header 'Content-Type: application/json' \
--data-binary "@${CLIENT_FILE}" \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients"
)"
[[ "$client_create_status" == "201" ]] || \
fail "Keycloak Gitea client creation failed with HTTP ${client_create_status}"
printf 'Created confidential Keycloak client %s.\n' "$GITEA_CLIENT_ID"
else
client_uuid="$(jq --exit-status --raw-output '.[0].id' "$RESPONSE_FILE")"
[[ -n "$client_uuid" ]] || fail "existing Gitea client has no internal id"
client_update_status="$(
admin_request \
--request PUT \
--header 'Content-Type: application/json' \
--data-binary "@${CLIENT_FILE}" \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients/${client_uuid}"
)"
[[ "$client_update_status" == "204" ]] || \
fail "Keycloak Gitea client update failed with HTTP ${client_update_status}"
printf 'Updated confidential Keycloak client %s.\n' "$GITEA_CLIENT_ID"
fi
client_lookup_status="$(
admin_request \
--get \
--data-urlencode "clientId=${GITEA_CLIENT_ID}" \
--data-urlencode 'max=2' \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients"
)"
[[ "$client_lookup_status" == "200" ]] || \
fail "post-update Keycloak client lookup failed with HTTP ${client_lookup_status}"
[[ "$(jq 'length' "$RESPONSE_FILE")" == "1" ]] || \
fail "post-update Gitea client lookup did not return exactly one client"
client_uuid="$(jq --exit-status --raw-output '.[0].id' "$RESPONSE_FILE")"
client_secret_status="$(
admin_request \
--output "$CLIENT_SECRET_RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/admin/realms/${KEYCLOAK_REALM}/clients/${client_uuid}/client-secret"
)"
[[ "$client_secret_status" == "200" ]] || \
fail "Keycloak client-secret retrieval failed with HTTP ${client_secret_status}"
jq --exit-status --raw-output --join-output \
'.value | select(type == "string" and length >= 16)' \
"$CLIENT_SECRET_RESPONSE_FILE" >"$CLIENT_SECRET_FILE" || \
fail "Keycloak returned an invalid Gitea client secret"
chmod 0600 "$CLIENT_SECRET_FILE"
kubectl --namespace "$GITEA_NAMESPACE" create secret generic \
"$GITEA_OIDC_SECRET" \
--from-literal="key=${GITEA_CLIENT_ID}" \
--from-file="secret=${CLIENT_SECRET_FILE}" \
--dry-run=client \
--output=yaml \
| kubectl apply --filename=- >/dev/null
kubectl --namespace "$GITEA_NAMESPACE" label secret "$GITEA_OIDC_SECRET" \
app.kubernetes.io/name=gitea \
app.kubernetes.io/instance=gitea \
app.kubernetes.io/component=oidc-client \
app.kubernetes.io/part-of=platform \
app.kubernetes.io/managed-by=bootstrap-script \
--overwrite >/dev/null
secret_type="$(
kubectl --namespace "$GITEA_NAMESPACE" get secret "$GITEA_OIDC_SECRET" \
--output=jsonpath='{.type}'
)"
secret_keys="$(
kubectl --namespace "$GITEA_NAMESPACE" get secret "$GITEA_OIDC_SECRET" \
--output=go-template='{{range $key, $value := .data}}{{$key}}{{"\n"}}{{end}}' \
| LC_ALL=C sort
)"
[[ "$secret_type" == "Opaque" ]] || fail "Gitea OIDC Secret type is not Opaque"
[[ "$secret_keys" == $'key\nsecret' ]] || \
fail "Gitea OIDC Secret key contract is invalid"
discovery_status="$(
curl "${curl_common[@]}" \
--output "$RESPONSE_FILE" \
--write-out '%{http_code}' \
"${LOCAL_BASE_URL}/realms/${KEYCLOAK_REALM}/.well-known/openid-configuration"
)"
[[ "$discovery_status" == "200" ]] || \
fail "Keycloak OIDC discovery failed with HTTP ${discovery_status}"
jq --exit-status \
--arg issuer "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}" \
'.issuer == $issuer and
(.authorization_endpoint | startswith($issuer)) and
(.token_endpoint | startswith($issuer))' \
"$RESPONSE_FILE" >/dev/null || \
fail "Keycloak discovery metadata contains an unexpected public issuer"
printf '\nKEYCLOAK OIDC BOOTSTRAP SUCCESS\n'
printf 'Realm: %s\n' "$KEYCLOAK_REALM"
printf 'Client: %s (confidential, authorization code flow)\n' "$GITEA_CLIENT_ID"
printf 'Gitea Secret: %s/%s\n' "$GITEA_NAMESPACE" "$GITEA_OIDC_SECRET"
printf 'Secret and token payloads were not printed and the temporary files were removed.\n'
printf 'Next: run scripts/bootstrap/apply-host-nginx-keycloak.sh, then apply-gitea-oidc.sh.\n'
printf 'Security follow-up: replace the temporary Keycloak administrator with a named administrator and MFA.\n'
File diff suppressed because it is too large Load Diff
+324
View File
@@ -0,0 +1,324 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# 비밀값이 명령 추적, argv 또는 표준 출력에 노출되지 않도록 한다.
set +x
readonly REQUIRED_CONFIRMATION="APPLY AISTOR SECRETS"
readonly -a SECRET_CONTRACTS=(
"aistor/minio-license"
"object-storage/aistor-root-configuration"
)
license_file=""
root_config_file=""
generate_root_config=false
execute_requested=false
license_created=false
root_config_created=false
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/bootstrap/create-aistor-secrets.sh \
--license-file /home/donghyeon/.secrets/aistor/minio.license \
--root-config-file /home/donghyeon/.secrets/aistor/root.env \
--generate-root-config \
--execute
Creates exactly these two Secrets only when both are absent:
aistor/minio-license
object-storage/aistor-root-configuration
When both already exist, validates and reuses them unchanged. A partial state
is refused. --generate-root-config creates the local 0600 credential file only
when it is absent; it never overwrites or rotates an existing credential.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--license-file)
(( $# >= 2 )) || fail "--license-file requires a path"
license_file="$2"
shift 2
;;
--root-config-file)
(( $# >= 2 )) || fail "--root-config-file requires a path"
root_config_file="$2"
shift 2
;;
--generate-root-config)
generate_root_config=true
shift
;;
--execute)
execute_requested=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
[[ "$execute_requested" == true ]] || {
usage >&2
exit 2
}
[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path"
[[ "$root_config_file" == /* ]] || \
fail "--root-config-file must be an absolute path"
for command_name in awk base64 chmod cmp find install jq kubectl mktemp \
openssl sort stat wc; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
validate_private_file() {
local file="$1"
local description="$2"
[[ -f "$file" && ! -L "$file" && -O "$file" && -r "$file" && -s "$file" ]] || \
fail "${description} must be a readable, non-empty, current-user-owned regular file"
[[ "$(stat --format='%a' -- "$file")" == "600" ]] || \
fail "${description} must have mode 0600: ${file}"
}
validate_license_file() {
local payload
validate_private_file "$license_file" "license file"
payload="$(<"$license_file")"
payload="${payload%$'\r'}"
[[ "$payload" =~ ^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]] || \
fail "license file must contain one JWT value beginning with eyJ"
unset payload
}
validate_root_config_file() {
local file="$1"
validate_private_file "$file" "AIStor root configuration file"
awk '
BEGIN {
user_prefix = "export MINIO_ROOT_USER="
password_prefix = "export MINIO_ROOT_PASSWORD="
}
NR == 1 && index($0, user_prefix) == 1 {
value = substr($0, length(user_prefix) + 1)
if (length(value) >= 3 && substr(value, 1, 1) == "\042" && substr(value, length(value), 1) == "\042") {
value = substr(value, 2, length(value) - 2)
users++
user = value
if (length(value) < 8 || value == "minioadmin") bad = 1
next
}
}
NR == 2 && index($0, password_prefix) == 1 {
value = substr($0, length(password_prefix) + 1)
if (length(value) >= 3 && substr(value, 1, 1) == "\042" && substr(value, length(value), 1) == "\042") {
value = substr(value, 2, length(value) - 2)
passwords++
password = value
if (length(value) < 16 || value == "minioadmin") bad = 1
next
}
}
{ bad = 1 }
END {
if (NR != 2 || users != 1 || passwords != 1 ||
user == password || bad) exit 1
}
' "$file" || \
fail "root configuration must contain exactly valid MINIO_ROOT_USER and MINIO_ROOT_PASSWORD exports"
}
generate_root_configuration() {
local parent_dir="${root_config_file%/*}"
local root_user
local root_password
[[ "$parent_dir" != "$root_config_file" ]] || \
fail "root configuration path has no parent directory"
[[ ! -e "$root_config_file" && ! -L "$root_config_file" ]] || \
fail "refusing to overwrite existing root configuration: ${root_config_file}"
install -d -m 0700 -- "$parent_dir"
[[ -d "$parent_dir" && ! -L "$parent_dir" && -O "$parent_dir" ]] || \
fail "root configuration parent must be a current-user-owned directory"
[[ "$(stat --format='%a' -- "$parent_dir")" == "700" ]] || \
fail "root configuration parent must have mode 0700: ${parent_dir}"
root_user="hyeonworks-aistor-$(openssl rand -hex 4)"
root_password="$(openssl rand -hex 24)"
umask 077
{
printf 'export MINIO_ROOT_USER="%s"\n' "$root_user"
printf 'export MINIO_ROOT_PASSWORD="%s"\n' "$root_password"
} >"$root_config_file"
chmod 0600 -- "$root_config_file"
unset root_user root_password
validate_root_config_file "$root_config_file"
printf 'Generated a local AIStor root configuration with mode 0600: %s\n' \
"$root_config_file"
}
validate_license_file
for namespace in aistor object-storage; do
kubectl get namespace "$namespace" >/dev/null 2>&1 || \
fail "namespace ${namespace} does not exist; apply AIStor namespaces first"
done
existing_secret_count=0
for contract in "${SECRET_CONTRACTS[@]}"; do
namespace="${contract%%/*}"
name="${contract#*/}"
if kubectl --namespace "$namespace" get secret "$name" >/dev/null 2>&1; then
(( existing_secret_count += 1 ))
fi
done
if (( existing_secret_count > 0 && existing_secret_count < ${#SECRET_CONTRACTS[@]} )); then
fail "partial AIStor Secret state detected; no Secret was created or rotated"
fi
umask 077
secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-aistor-secrets.XXXXXX")"
existing_license_file="${secret_temp_dir}/existing-minio-license"
existing_root_config_file="${secret_temp_dir}/existing-root-config"
cleanup() {
case "$secret_temp_dir" in
/tmp/platform-aistor-secrets.*|"${TMPDIR:-/tmp}"/platform-aistor-secrets.*)
rm -rf -- "$secret_temp_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected temporary directory: %s\n' \
"$secret_temp_dir" >&2
;;
esac
}
trap cleanup EXIT
validate_secret_contract() {
local namespace="$1"
local name="$2"
local expected_key="$3"
local actual_type
local actual_keys
actual_type="$(
kubectl --namespace "$namespace" get secret "$name" \
--output=jsonpath='{.type}'
)"
[[ "$actual_type" == "Opaque" ]] || \
fail "${namespace}/${name} must have type Opaque"
actual_keys="$(
kubectl --namespace "$namespace" get secret "$name" --output=json |
jq -r '.data | keys[]' | sort
)"
[[ "$actual_keys" == "$expected_key" ]] || \
fail "${namespace}/${name} must contain only the ${expected_key} key"
}
if (( existing_secret_count == ${#SECRET_CONTRACTS[@]} )); then
validate_secret_contract aistor minio-license minio.license
validate_secret_contract \
object-storage aistor-root-configuration config.env
kubectl --namespace aistor get secret minio-license \
--output=jsonpath='{.data.minio\.license}' |
base64 --decode >"$existing_license_file"
cmp --silent -- "$license_file" "$existing_license_file" || \
fail "the supplied license differs from the existing Secret; rotation was not performed"
kubectl --namespace object-storage get secret aistor-root-configuration \
--output=jsonpath='{.data.config\.env}' |
base64 --decode >"$existing_root_config_file"
chmod 0600 -- "$existing_root_config_file"
validate_root_config_file "$existing_root_config_file"
if [[ -e "$root_config_file" || -L "$root_config_file" ]]; then
validate_root_config_file "$root_config_file"
cmp --silent -- "$root_config_file" "$existing_root_config_file" || \
fail "local root configuration differs from the existing Secret; rotation was not performed"
else
fail "existing Secret is valid, but the local root configuration file is missing"
fi
printf 'Existing AIStor Secret contracts are valid and were reused unchanged.\n'
printf 'No credential or license rotation was performed.\n'
exit 0
fi
if [[ ! -e "$root_config_file" && ! -L "$root_config_file" ]]; then
[[ "$generate_root_config" == true ]] || \
fail "root configuration is absent; pass --generate-root-config to create it"
generate_root_configuration
else
validate_root_config_file "$root_config_file"
fi
[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation"
printf '\nThis will create exactly two AIStor Secrets. Type %s to continue: ' \
"$REQUIRED_CONFIRMATION"
read -r confirmation
[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled"
rollback_new_secrets() {
set +e
if [[ "$root_config_created" == true ]]; then
kubectl --namespace object-storage delete secret \
aistor-root-configuration --ignore-not-found >/dev/null
fi
if [[ "$license_created" == true ]]; then
kubectl --namespace aistor delete secret \
minio-license --ignore-not-found >/dev/null
fi
if [[ "$root_config_created" == true || "$license_created" == true ]]; then
printf 'ROLLBACK: removed only Secrets created by this failed invocation.\n' >&2
fi
}
on_error() {
local status="$1"
local line="$2"
trap - ERR
rollback_new_secrets
printf 'ERROR: Secret creation failed at line %s (exit %s).\n' \
"$line" "$status" >&2
exit "$status"
}
trap 'on_error "$?" "$LINENO"' ERR
kubectl --namespace aistor create secret generic minio-license \
--type=Opaque \
--from-file="minio.license=${license_file}"
license_created=true
kubectl --namespace object-storage create secret generic \
aistor-root-configuration \
--type=Opaque \
--from-file="config.env=${root_config_file}"
root_config_created=true
validate_secret_contract aistor minio-license minio.license
validate_secret_contract object-storage aistor-root-configuration config.env
printf 'Created both AIStor Secret contracts without printing payloads.\n'
printf 'Local root credentials remain only in: %s\n' "$root_config_file"
printf 'This script does not perform credential or license rotation.\n'
+298
View File
@@ -0,0 +1,298 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# 호출자가 bash -x로 실행해도 비밀번호가 추적 출력에 노출되지 않도록 한다.
set +x
readonly REQUIRED_CONFIRMATION="APPLY KEYCLOAK SECRETS"
readonly DB_USERNAME="keycloak"
readonly TOTAL_SECRET_CONTRACTS=2
readonly -a SECRET_CONTRACTS=(
"platform-data/keycloak-db-credentials"
"keycloak/keycloak-db-credentials"
)
mutation_started=false
secret_temp_dir=""
report_retained_state() {
if [[ "$mutation_started" == true ]]; then
printf '%s\n' \
'SAFE STOP: no Secret or database data was deleted or rolled back.' \
'Any Secret created before the failure remains in the cluster.' \
'Inspect Secret names and events without printing Secret data, then rerun after resolving the cause.' >&2
fi
}
fail() {
printf 'ERROR: %s\n' "$*" >&2
report_retained_state
exit 1
}
on_error() {
local status="$1"
local line="$2"
trap - ERR
set +e
printf 'ERROR: command failed at line %s (exit %s).\n' "$line" "$status" >&2
report_retained_state
exit "$status"
}
on_signal() {
local status="$1"
trap - INT TERM
set +e
printf 'INTERRUPTED: stopping without deleting cluster state.\n' >&2
report_retained_state
exit "$status"
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/bootstrap/create-keycloak-secrets.sh --execute
bash scripts/bootstrap/create-keycloak-secrets.sh --generate --execute
Creates exactly these two Secrets only when both are absent:
platform-data/keycloak-db-credentials
keycloak/keycloak-db-credentials
When both already exist, validates and reuses their data unchanged. A partial
state is refused. This script does not require or modify an AIStor license,
does not rotate credentials, and never prints Secret payloads.
--execute prompts twice for a database password and requires the exact
confirmation text. --generate --execute is an explicit non-interactive mode:
it generates 32 random bytes with OpenSSL in a private 0600 temporary file.
USAGE
}
execute_requested=false
generate_requested=false
while (( $# > 0 )); do
case "$1" in
--execute)
execute_requested=true
shift
;;
--generate)
generate_requested=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
[[ "$execute_requested" == true ]] || {
usage >&2
exit 2
}
for command_name in kubectl base64 cmp mktemp sort stat tr wc; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
if [[ "$generate_requested" == true ]]; then
command -v openssl >/dev/null 2>&1 || fail "openssl is required for --generate"
fi
for namespace in platform-data keycloak; do
kubectl get namespace "$namespace" >/dev/null 2>&1 || \
fail "namespace ${namespace} does not exist"
done
umask 077
secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-keycloak-secrets.XXXXXX")"
db_user_file="${secret_temp_dir}/expected-username"
db_password_file="${secret_temp_dir}/new-password"
platform_username_file="${secret_temp_dir}/platform-username"
keycloak_username_file="${secret_temp_dir}/keycloak-username"
platform_password_file="${secret_temp_dir}/platform-password"
keycloak_password_file="${secret_temp_dir}/keycloak-password"
cleanup() {
unset db_password
if [[ -n "$secret_temp_dir" ]]; then
rm -f -- \
"$db_user_file" \
"$db_password_file" \
"$platform_username_file" \
"$keycloak_username_file" \
"$platform_password_file" \
"$keycloak_password_file"
rmdir -- "$secret_temp_dir" 2>/dev/null || true
fi
}
trap cleanup EXIT
trap 'on_error "$?" "$LINENO"' ERR
trap 'on_signal 130' INT
trap 'on_signal 143' TERM
printf '%s' "$DB_USERNAME" >"$db_user_file"
secret_data_b64() {
local namespace="$1"
local name="$2"
local key="$3"
kubectl --namespace "$namespace" get secret "$name" \
--output="go-template={{ index .data \"${key}\" }}"
}
validate_secret_type_and_keys() {
local namespace="$1"
local name="$2"
local actual_type
local actual_keys
local actual_keys_sorted
actual_type="$(
kubectl --namespace "$namespace" get secret "$name" \
--output='jsonpath={.type}'
)"
[[ "$actual_type" == "kubernetes.io/basic-auth" ]] || \
fail "${namespace}/${name} must have type kubernetes.io/basic-auth"
actual_keys="$(
kubectl --namespace "$namespace" get secret "$name" \
--output='go-template={{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}'
)"
actual_keys_sorted="$(printf '%s\n' "$actual_keys" | LC_ALL=C sort)"
[[ "$actual_keys_sorted" == $'password\nusername' ]] || \
fail "${namespace}/${name} has an unexpected data key set"
}
validate_existing_contracts() {
validate_secret_type_and_keys platform-data keycloak-db-credentials
validate_secret_type_and_keys keycloak keycloak-db-credentials
secret_data_b64 platform-data keycloak-db-credentials username \
| base64 --decode >"$platform_username_file"
secret_data_b64 keycloak keycloak-db-credentials username \
| base64 --decode >"$keycloak_username_file"
cmp --silent -- "$db_user_file" "$platform_username_file" || \
fail "platform-data/keycloak-db-credentials username must be keycloak"
cmp --silent -- "$db_user_file" "$keycloak_username_file" || \
fail "keycloak/keycloak-db-credentials username must be keycloak"
secret_data_b64 platform-data keycloak-db-credentials password \
| base64 --decode >"$platform_password_file"
secret_data_b64 keycloak keycloak-db-credentials password \
| base64 --decode >"$keycloak_password_file"
(( $(wc -c <"$platform_password_file") >= 16 )) || \
fail "platform-data/keycloak-db-credentials password must contain at least 16 bytes"
(( $(wc -c <"$keycloak_password_file") >= 16 )) || \
fail "keycloak/keycloak-db-credentials password must contain at least 16 bytes"
cmp --silent -- "$platform_password_file" "$keycloak_password_file" || \
fail "the two Keycloak DB Secret passwords do not match"
}
existing_secret_count=0
for contract in "${SECRET_CONTRACTS[@]}"; do
namespace="${contract%%/*}"
name="${contract#*/}"
existing_resource="$(
kubectl --namespace "$namespace" get secret "$name" \
--ignore-not-found --output=name
)"
if [[ -n "$existing_resource" ]]; then
((existing_secret_count += 1))
fi
done
if (( existing_secret_count > 0 && existing_secret_count < TOTAL_SECRET_CONTRACTS )); then
fail "partial Keycloak Secret state detected (${existing_secret_count}/${TOTAL_SECRET_CONTRACTS}); refusing creation or rotation"
fi
if (( existing_secret_count == TOTAL_SECRET_CONTRACTS )); then
validate_existing_contracts
reload_label="$(
kubectl --namespace platform-data get secret keycloak-db-credentials \
--output='jsonpath={.metadata.labels.cnpg\.io/reload}'
)"
if [[ "$reload_label" != "true" ]]; then
mutation_started=true
kubectl --namespace platform-data label secret keycloak-db-credentials \
cnpg.io/reload=true --overwrite
mutation_started=false
printf 'Repaired cnpg.io/reload=true without changing Secret data.\n'
fi
printf 'Existing Keycloak Secret contracts are valid and were reused unchanged.\n'
printf 'No credential rotation was performed.\n'
exit 0
fi
read_secret_twice() {
local prompt="$1"
local first
local second
read -r -s -p "${prompt}: " first
printf '\n' >&2
read -r -s -p "Confirm ${prompt}: " second
printf '\n' >&2
[[ "$first" == "$second" ]] || fail "the two values do not match"
(( ${#first} >= 16 )) || fail "${prompt} must contain at least 16 characters"
[[ "$first" != *$'\n'* && "$first" != *$'\r'* ]] || \
fail "${prompt} contains an unsupported line break"
printf '%s' "$first"
}
if [[ "$generate_requested" == true ]]; then
openssl rand -hex 32 | tr -d '\n' >"$db_password_file"
[[ "$(stat --format='%a' -- "$db_password_file")" == "600" ]] || \
fail "generated password file must have mode 0600"
[[ "$(wc -c <"$db_password_file" | tr -d '[:space:]')" == "64" ]] || \
fail "OpenSSL did not generate the expected 32-byte password"
printf '%s\n' \
'Authorized by explicit --generate --execute flags.' \
'A 32-byte random database password was generated without printing it.'
else
[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation"
db_password="$(read_secret_twice 'Keycloak database password')"
printf '%s' "$db_password" >"$db_password_file"
unset db_password
printf '\nThe script will create exactly two Keycloak DB Secrets.\n'
printf 'It will not read or modify any AIStor Secret.\n'
printf 'Type %s to continue: ' "$REQUIRED_CONFIRMATION"
read -r confirmation
[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled"
fi
create_basic_auth_secret() {
local namespace="$1"
kubectl --namespace "$namespace" create secret generic keycloak-db-credentials \
--type=kubernetes.io/basic-auth \
--from-file="username=${db_user_file}" \
--from-file="password=${db_password_file}" \
--dry-run=client \
--output=yaml \
| kubectl create --filename=-
}
mutation_started=true
create_basic_auth_secret platform-data
create_basic_auth_secret keycloak
kubectl --namespace platform-data label secret keycloak-db-credentials \
cnpg.io/reload=true --overwrite
validate_existing_contracts
mutation_started=false
printf 'Initial Keycloak Secret contracts were created without printing payloads.\n'
printf 'This script does not perform credential rotation.\n'
File diff suppressed because it is too large Load Diff
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# 비밀값이 명령 추적 출력에 노출되지 않도록 호출자가 활성화한 xtrace도 끈다.
set +x
readonly REQUIRED_CONFIRMATION="APPLY"
readonly DB_USERNAME="gitea"
readonly TOTAL_SECRET_CONTRACTS=3
readonly -a SECRET_CONTRACTS=(
"platform-data/gitea-db-credentials"
"gitea/gitea-db-credentials"
"gitea/gitea-admin"
)
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
Usage: bash scripts/bootstrap/create-phase1-secrets.sh --execute
Creates these Secrets only when all three are absent:
platform-data/gitea-db-credentials
gitea/gitea-db-credentials
gitea/gitea-admin
When all three already exist, validates and reuses their data unchanged. The
script refuses a partial state and does not perform credential rotation. It may
repair only the cnpg.io/reload=true label on the platform-data DB Secret.
USAGE
}
[[ "${1:-}" == "--execute" && "$#" -eq 1 ]] || {
usage
exit 2
}
command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"
command -v base64 >/dev/null 2>&1 || fail "base64 is required"
for namespace in platform-data gitea; do
kubectl get namespace "$namespace" >/dev/null 2>&1 || \
fail "namespace ${namespace} does not exist; apply namespaces first"
done
existing_secret_count=0
for contract in "${SECRET_CONTRACTS[@]}"; do
namespace="${contract%%/*}"
name="${contract#*/}"
existing_resource="$(
kubectl --namespace "$namespace" get secret "$name" \
--ignore-not-found --output=name
)"
if [[ -n "$existing_resource" ]]; then
((existing_secret_count += 1))
fi
done
if (( existing_secret_count > 0 && existing_secret_count < TOTAL_SECRET_CONTRACTS )); then
fail "partial Phase 1 Secret state detected (${existing_secret_count}/${TOTAL_SECRET_CONTRACTS}); refusing creation or rotation"
fi
secret_data_b64() {
local namespace="$1"
local name="$2"
local key="$3"
kubectl --namespace "$namespace" get secret "$name" \
--output="jsonpath={.data.${key}}"
}
validate_basic_auth_secret() {
local namespace="$1"
local name="$2"
local secret_type
local username_b64
local password_b64
secret_type="$(
kubectl --namespace "$namespace" get secret "$name" \
--output='jsonpath={.type}'
)"
[[ "$secret_type" == "kubernetes.io/basic-auth" ]] || \
fail "${namespace}/${name} must have type kubernetes.io/basic-auth"
username_b64="$(secret_data_b64 "$namespace" "$name" username)"
password_b64="$(secret_data_b64 "$namespace" "$name" password)"
[[ -n "$username_b64" ]] || fail "${namespace}/${name} is missing non-empty data.username"
[[ -n "$password_b64" ]] || fail "${namespace}/${name} is missing non-empty data.password"
}
if (( existing_secret_count == TOTAL_SECRET_CONTRACTS )); then
validate_basic_auth_secret platform-data gitea-db-credentials
validate_basic_auth_secret gitea gitea-db-credentials
validate_basic_auth_secret gitea gitea-admin
platform_db_username_b64="$(
secret_data_b64 platform-data gitea-db-credentials username
)"
platform_db_password_b64="$(
secret_data_b64 platform-data gitea-db-credentials password
)"
gitea_db_username_b64="$(
secret_data_b64 gitea gitea-db-credentials username
)"
gitea_db_password_b64="$(
secret_data_b64 gitea gitea-db-credentials password
)"
expected_db_username_b64="$(printf '%s' "$DB_USERNAME" | base64)"
[[ "$platform_db_username_b64" == "$expected_db_username_b64" ]] || \
fail "platform-data/gitea-db-credentials username must be gitea"
[[ "$gitea_db_username_b64" == "$expected_db_username_b64" ]] || \
fail "gitea/gitea-db-credentials username must be gitea"
[[ "$platform_db_username_b64" == "$gitea_db_username_b64" ]] || \
fail "the two DB Secret usernames do not match"
[[ "$platform_db_password_b64" == "$gitea_db_password_b64" ]] || \
fail "the two DB Secret passwords do not match"
reload_label="$(
kubectl --namespace platform-data get secret gitea-db-credentials \
--output='jsonpath={.metadata.labels.cnpg\.io/reload}'
)"
if [[ "$reload_label" != "true" ]]; then
kubectl --namespace platform-data label secret gitea-db-credentials \
cnpg.io/reload=true --overwrite
printf 'Repaired cnpg.io/reload=true without changing Secret data.\n'
fi
unset platform_db_username_b64 platform_db_password_b64
unset gitea_db_username_b64 gitea_db_password_b64 expected_db_username_b64
printf 'Existing Phase 1 Secret contracts are valid and were reused unchanged.\n'
printf 'No credential rotation was performed.\n'
exit 0
fi
[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation"
read_secret_twice() {
local prompt="$1"
local first
local second
read -r -s -p "${prompt}: " first
printf '\n' >&2
read -r -s -p "Confirm ${prompt}: " second
printf '\n' >&2
[[ "$first" == "$second" ]] || fail "the two values do not match"
(( ${#first} >= 16 )) || fail "use at least 16 characters"
printf '%s' "$first"
}
read -r -p 'Gitea administrator username: ' admin_username
[[ "$admin_username" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || \
fail "administrator username contains unsupported characters"
db_password="$(read_secret_twice 'Gitea database password')"
admin_password="$(read_secret_twice 'Gitea administrator password')"
secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase1-secrets.XXXXXX")"
db_user_file="${secret_temp_dir}/db-username"
db_password_file="${secret_temp_dir}/db-password"
admin_user_file="${secret_temp_dir}/admin-username"
admin_password_file="${secret_temp_dir}/admin-password"
cleanup() {
unset db_password admin_password
rm -f -- "$db_user_file" "$db_password_file" "$admin_user_file" "$admin_password_file"
rmdir -- "$secret_temp_dir" 2>/dev/null || true
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
umask 077
printf '%s' "$DB_USERNAME" >"$db_user_file"
printf '%s' "$db_password" >"$db_password_file"
printf '%s' "$admin_username" >"$admin_user_file"
printf '%s' "$admin_password" >"$admin_password_file"
unset db_password admin_password
printf '\nThe script will create exactly three Secrets. Type %s to continue: ' "$REQUIRED_CONFIRMATION"
read -r confirmation
[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled"
apply_basic_auth_secret() {
local namespace="$1"
local name="$2"
local username_file="$3"
local password_file="$4"
kubectl --namespace "$namespace" create secret generic "$name" \
--type=kubernetes.io/basic-auth \
--from-file="username=${username_file}" \
--from-file="password=${password_file}" \
--dry-run=client \
--output=yaml \
| kubectl apply --filename=-
}
apply_basic_auth_secret \
platform-data gitea-db-credentials "$db_user_file" "$db_password_file"
kubectl --namespace platform-data label secret gitea-db-credentials \
cnpg.io/reload=true --overwrite
apply_basic_auth_secret \
gitea gitea-db-credentials "$db_user_file" "$db_password_file"
apply_basic_auth_secret \
gitea gitea-admin "$admin_user_file" "$admin_password_file"
printf 'Initial Phase 1 Secret contracts were created. No values were written to the repository.\n'
printf 'This script does not perform credential rotation.\n'
+323
View File
@@ -0,0 +1,323 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# 비밀값이 명령 추적, argv 또는 표준 출력에 노출되지 않도록 한다.
set +x
readonly REQUIRED_CONFIRMATION="APPLY PHASE 2 SECRETS"
readonly DB_USERNAME="keycloak"
readonly TOTAL_SECRET_CONTRACTS=4
readonly -a SECRET_CONTRACTS=(
"platform-data/keycloak-db-credentials"
"keycloak/keycloak-db-credentials"
"aistor/minio-license"
"object-storage/aistor-root-configuration"
)
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/bootstrap/create-phase2-secrets.sh \
--license-file /absolute/path/to/minio.license \
--execute
Creates all four Phase 2 Secrets only when all four are absent. When all four
already exist, validates and reuses them unchanged. A partial state is refused.
This script never rotates credentials or the AIStor license.
USAGE
}
license_file=""
execute_requested=false
while (( $# > 0 )); do
case "$1" in
--license-file)
(( $# >= 2 )) || fail "--license-file requires a path"
license_file="$2"
shift 2
;;
--execute)
execute_requested=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
[[ "$execute_requested" == true ]] || {
usage >&2
exit 2
}
[[ -n "$license_file" ]] || fail "--license-file is required"
[[ "$license_file" == /* ]] || fail "--license-file must be an absolute path"
[[ -f "$license_file" && -r "$license_file" && -s "$license_file" ]] || \
fail "license file must be a readable, non-empty regular file"
license_payload="$(<"$license_file")"
license_payload="${license_payload%$'\r'}"
[[ "$license_payload" =~ ^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]] || \
fail "license file must contain one decoded JWT value beginning with eyJ"
unset license_payload
command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"
command -v base64 >/dev/null 2>&1 || fail "base64 is required"
command -v cmp >/dev/null 2>&1 || fail "cmp is required"
command -v wc >/dev/null 2>&1 || fail "wc is required"
for namespace in platform-data keycloak aistor object-storage; do
kubectl get namespace "$namespace" >/dev/null 2>&1 || \
fail "namespace ${namespace} does not exist; apply Phase 2 namespaces first"
done
existing_secret_count=0
for contract in "${SECRET_CONTRACTS[@]}"; do
namespace="${contract%%/*}"
name="${contract#*/}"
existing_resource="$(
kubectl --namespace "$namespace" get secret "$name" \
--ignore-not-found --output=name
)"
if [[ -n "$existing_resource" ]]; then
((existing_secret_count += 1))
fi
done
if (( existing_secret_count > 0 && existing_secret_count < TOTAL_SECRET_CONTRACTS )); then
fail "partial Phase 2 Secret state detected (${existing_secret_count}/${TOTAL_SECRET_CONTRACTS}); refusing creation or rotation"
fi
umask 077
secret_temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase2-secrets.XXXXXX")"
db_user_file="${secret_temp_dir}/db-username"
db_password_file="${secret_temp_dir}/db-password"
root_config_file="${secret_temp_dir}/config.env"
existing_license_file="${secret_temp_dir}/existing-minio-license"
existing_root_config_file="${secret_temp_dir}/existing-config.env"
existing_platform_db_password_file="${secret_temp_dir}/existing-platform-db-password"
existing_keycloak_db_password_file="${secret_temp_dir}/existing-keycloak-db-password"
cleanup() {
unset db_password root_user root_password
rm -f -- \
"$db_user_file" "$db_password_file" "$root_config_file" \
"$existing_license_file" "$existing_root_config_file" \
"$existing_platform_db_password_file" \
"$existing_keycloak_db_password_file"
rmdir -- "$secret_temp_dir" 2>/dev/null || true
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
secret_data_b64() {
local namespace="$1"
local name="$2"
local key="$3"
kubectl --namespace "$namespace" get secret "$name" \
--output="go-template={{ index .data \"${key}\" }}"
}
validate_secret_type_and_keys() {
local namespace="$1"
local name="$2"
local expected_type="$3"
local expected_keys="$4"
local actual_type
local actual_keys
local actual_keys_sorted
local expected_keys_sorted
actual_type="$(
kubectl --namespace "$namespace" get secret "$name" \
--output='jsonpath={.type}'
)"
[[ "$actual_type" == "$expected_type" ]] || \
fail "${namespace}/${name} must have type ${expected_type}"
actual_keys="$(
kubectl --namespace "$namespace" get secret "$name" \
--output='go-template={{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}'
)"
actual_keys_sorted="$(printf '%s\n' "$actual_keys" | LC_ALL=C sort)"
expected_keys_sorted="$(printf '%s\n' "$expected_keys" | LC_ALL=C sort)"
[[ "$actual_keys_sorted" == "$expected_keys_sorted" ]] || \
fail "${namespace}/${name} has an unexpected data key set"
}
validate_root_config_file() {
local file="$1"
awk '
BEGIN {
user_prefix = "export MINIO_ROOT_USER=\042"
password_prefix = "export MINIO_ROOT_PASSWORD=\042"
}
NR == 1 && index($0, user_prefix) == 1 &&
substr($0, length($0), 1) == "\042" {
value = substr($0, length(user_prefix) + 1,
length($0) - length(user_prefix) - 1)
users++
user = value
if (length(value) < 8 || value == "minioadmin" || index(value, "\042") > 0) bad = 1
next
}
NR == 2 && index($0, password_prefix) == 1 &&
substr($0, length($0), 1) == "\042" {
value = substr($0, length(password_prefix) + 1,
length($0) - length(password_prefix) - 1)
passwords++
password = value
if (length(value) < 16 || value == "minioadmin" || index(value, "\042") > 0) bad = 1
next
}
{ bad = 1 }
END {
if (NR != 2 || users != 1 || passwords != 1 ||
user == password || bad) exit 1
}
' "$file" || fail "object-storage/aistor-root-configuration has an invalid config.env contract"
}
if (( existing_secret_count == TOTAL_SECRET_CONTRACTS )); then
validate_secret_type_and_keys \
platform-data keycloak-db-credentials kubernetes.io/basic-auth $'password\nusername'
validate_secret_type_and_keys \
keycloak keycloak-db-credentials kubernetes.io/basic-auth $'password\nusername'
validate_secret_type_and_keys aistor minio-license Opaque 'minio.license'
validate_secret_type_and_keys \
object-storage aistor-root-configuration Opaque 'config.env'
platform_username_b64="$(secret_data_b64 platform-data keycloak-db-credentials username)"
keycloak_username_b64="$(secret_data_b64 keycloak keycloak-db-credentials username)"
expected_username_b64="$(printf '%s' "$DB_USERNAME" | base64)"
[[ "$platform_username_b64" == "$expected_username_b64" ]] || \
fail "platform-data/keycloak-db-credentials username must be keycloak"
[[ "$keycloak_username_b64" == "$expected_username_b64" ]] || \
fail "keycloak/keycloak-db-credentials username must be keycloak"
[[ "$platform_username_b64" == "$keycloak_username_b64" ]] || \
fail "the two Keycloak DB Secret usernames do not match"
secret_data_b64 platform-data keycloak-db-credentials password \
| base64 --decode >"$existing_platform_db_password_file"
secret_data_b64 keycloak keycloak-db-credentials password \
| base64 --decode >"$existing_keycloak_db_password_file"
(( $(wc -c <"$existing_platform_db_password_file") >= 16 )) || \
fail "platform-data/keycloak-db-credentials password must contain at least 16 bytes"
(( $(wc -c <"$existing_keycloak_db_password_file") >= 16 )) || \
fail "keycloak/keycloak-db-credentials password must contain at least 16 bytes"
cmp --silent -- "$existing_platform_db_password_file" \
"$existing_keycloak_db_password_file" || \
fail "the two Keycloak DB Secret passwords do not match"
reload_label="$(
kubectl --namespace platform-data get secret keycloak-db-credentials \
--output='jsonpath={.metadata.labels.cnpg\.io/reload}'
)"
[[ "$reload_label" == "true" ]] || \
fail "platform-data/keycloak-db-credentials must have cnpg.io/reload=true"
secret_data_b64 aistor minio-license minio.license \
| base64 --decode >"$existing_license_file"
[[ -s "$existing_license_file" ]] || \
fail "aistor/minio-license has an empty minio.license payload"
cmp --silent -- "$license_file" "$existing_license_file" || \
fail "the supplied license file differs from the existing Secret; rotation was not performed"
secret_data_b64 object-storage aistor-root-configuration config.env \
| base64 --decode >"$existing_root_config_file"
[[ -s "$existing_root_config_file" ]] || \
fail "object-storage/aistor-root-configuration has an empty config.env payload"
validate_root_config_file "$existing_root_config_file"
unset platform_username_b64 keycloak_username_b64 expected_username_b64
printf 'Existing Phase 2 Secret contracts are valid and were reused unchanged.\n'
printf 'No credential or license rotation was performed.\n'
exit 0
fi
[[ -t 0 ]] || fail "an interactive terminal is required for initial Secret creation"
read_secret_twice() {
local prompt="$1"
local minimum_length="$2"
local first
local second
read -r -s -p "${prompt}: " first
printf '\n' >&2
read -r -s -p "Confirm ${prompt}: " second
printf '\n' >&2
[[ "$first" == "$second" ]] || fail "the two values do not match"
(( ${#first} >= minimum_length )) || \
fail "${prompt} must contain at least ${minimum_length} characters"
[[ "$first" != *$'\n'* && "$first" != *$'\r'* && "$first" != *"'"* ]] || \
fail "${prompt} contains a character unsupported by config.env"
printf '%s' "$first"
}
db_password="$(read_secret_twice 'Keycloak database password' 16)"
root_user="$(read_secret_twice 'AIStor root username' 8)"
root_password="$(read_secret_twice 'AIStor root password' 16)"
[[ "$root_user" != "minioadmin" ]] || fail "do not use the default root username"
[[ "$root_password" != "minioadmin" ]] || fail "do not use the default root password"
[[ "$root_user" != "$root_password" ]] || fail "root username and password must differ"
printf '%s' "$DB_USERNAME" >"$db_user_file"
printf '%s' "$db_password" >"$db_password_file"
printf 'export MINIO_ROOT_USER="%s"\n' "$root_user" >"$root_config_file"
printf 'export MINIO_ROOT_PASSWORD="%s"\n' "$root_password" >>"$root_config_file"
validate_root_config_file "$root_config_file"
unset db_password root_user root_password
printf '\nThis will create exactly four Phase 2 Secrets. Type %s to continue: ' \
"$REQUIRED_CONFIRMATION"
read -r confirmation
[[ "$confirmation" == "$REQUIRED_CONFIRMATION" ]] || fail "cancelled"
create_secret_from_files() {
local namespace="$1"
local name="$2"
local type="$3"
shift 3
kubectl --namespace "$namespace" create secret generic "$name" \
--type="$type" "$@" \
--dry-run=client --output=yaml \
| kubectl create --filename=-
}
create_secret_from_files \
platform-data keycloak-db-credentials kubernetes.io/basic-auth \
--from-file="username=${db_user_file}" \
--from-file="password=${db_password_file}"
kubectl --namespace platform-data label secret keycloak-db-credentials \
cnpg.io/reload=true --overwrite
create_secret_from_files \
keycloak keycloak-db-credentials kubernetes.io/basic-auth \
--from-file="username=${db_user_file}" \
--from-file="password=${db_password_file}"
create_secret_from_files \
aistor minio-license Opaque \
--from-file="minio.license=${license_file}"
create_secret_from_files \
object-storage aistor-root-configuration Opaque \
--from-file="config.env=${root_config_file}"
printf 'Initial Phase 2 Secret contracts were created without printing payloads.\n'
printf 'This script does not perform credential or license rotation.\n'
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
PATH='/usr/sbin:/usr/bin:/sbin:/bin'
export PATH
LC_ALL=C
export LC_ALL
umask 077
_k3slr_wrapper_initial_guard() {
local effective_uid="${1-}" shell_options="${2-}"
[[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1
[[ "$shell_options" != *x* ]]
}
if ! _k3slr_wrapper_initial_guard "${EUID:-}" "$-"; then
printf 'ERROR: lifecycle wrapper refuses root or xtrace execution\n' >&2
return 1 2>/dev/null || exit 1
fi
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
K3SLR_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}"
else
K3SLR_WRAPPER_DIRECTORY='.'
fi
K3SLR_WRAPPER_ROOT="$(cd -- "${K3SLR_WRAPPER_DIRECTORY}/../.." && pwd -P)" || {
return 1 2>/dev/null || exit 1
}
K3SLR_WRAPPER_CONTRACT="${K3SLR_WRAPPER_ROOT}/infrastructure/security/k3s/local-recovery.env"
# shellcheck source=/dev/null
source "${K3SLR_WRAPPER_ROOT}/scripts/lib/k3s-local-recovery.sh" || {
return 1 2>/dev/null || exit 1
}
_k3slr_open_usage() {
printf 'Usage: bash scripts/bootstrap/open-k3s-local-recovery.sh [--execute]\n' >&2
}
_k3slr_open_main() {
local execution_mode
if ! _k3slr_parse_lifecycle_cli execution_mode "$@"; then
_k3slr_open_usage
return 2
fi
_k3slr_lifecycle_main open "$execution_mode"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_k3slr_open_main "$@"
fi
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
PATH='/usr/sbin:/usr/bin:/sbin:/bin'
export PATH
LC_ALL=C
export LC_ALL
umask 077
_k3slr_wrapper_initial_guard() {
local effective_uid="${1-}" shell_options="${2-}"
[[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1
[[ "$shell_options" != *x* ]]
}
if ! _k3slr_wrapper_initial_guard "${EUID:-}" "$-"; then
printf 'ERROR: lifecycle wrapper refuses root or xtrace execution\n' >&2
return 1 2>/dev/null || exit 1
fi
unset -f _k3slr_prove_header_restore 2>/dev/null || :
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
K3SLR_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}"
else
K3SLR_WRAPPER_DIRECTORY='.'
fi
K3SLR_WRAPPER_ROOT="$(cd -- "${K3SLR_WRAPPER_DIRECTORY}/../.." && pwd -P)" || {
return 1 2>/dev/null || exit 1
}
K3SLR_WRAPPER_CONTRACT="${K3SLR_WRAPPER_ROOT}/infrastructure/security/k3s/local-recovery.env"
# shellcheck source=/dev/null
source "${K3SLR_WRAPPER_ROOT}/scripts/lib/k3s-local-recovery.sh" || {
return 1 2>/dev/null || exit 1
}
_k3slr_prepare_usage() {
printf 'Usage: bash scripts/bootstrap/prepare-k3s-local-recovery.sh [--execute]\n' >&2
}
_k3slr_prepare_main() {
local execution_mode
if ! _k3slr_parse_lifecycle_cli execution_mode "$@"; then
_k3slr_prepare_usage
return 2
fi
_k3slr_lifecycle_main prepare "$execution_mode"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_k3slr_prepare_main "$@"
fi
@@ -0,0 +1,564 @@
#!/usr/bin/env bash
# Intentionally sourceable. Privileged filesystem work crosses exactly one
# helper seam and runs descriptor-relative inside one process.
_olp_error() {
printf 'ERROR: %s\n' "$*" >&2
return 1
}
_olp_target_rows() {
local base="$1"
printf '%s\t%s\t%s\n' \
"$base/observability/prometheus" 1000 2000 \
"$base/observability/grafana" 472 472 \
"$base/observability/alertmanager" 1000 2000 \
"$base/observability/alloy" 473 473 \
"$base/observability/loki" 10001 10001 \
"$base/observability/tempo" 10001 10001
}
_olp_helper_program() {
/usr/bin/cat <<'PY'
import errno
import os
import signal
import stat
import sys
TARGETS = (
("prometheus", 1000, 2000),
("grafana", 472, 472),
("alertmanager", 1000, 2000),
("alloy", 473, 473),
("loki", 10001, 10001),
("tempo", 10001, 10001),
)
MIN_AVAILABLE = 50 * 1024 * 1024 * 1024
MAX_USE = 1024 * 1024 * 1024
MAX_INTEGER = (1 << 63) - 1
OPEN_DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW
SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM)
class Rejected(Exception):
pass
class Interrupted(Exception):
def __init__(self, signum):
self.signum = signum
def reject(message):
raise Rejected(message)
if len(sys.argv) != 5 or sys.argv[1] not in {"preflight", "apply"}:
print("REJECT: helper argv differs", file=sys.stderr)
raise SystemExit(97)
action, root_path, ssd_path, aistor_path = sys.argv[1:]
def normalized_absolute(path):
return os.path.isabs(path) and os.path.normpath(path) == path and "//" not in path
for candidate in (root_path, ssd_path, aistor_path):
if not normalized_absolute(candidate):
reject("path is not normalized absolute")
fixture_root = os.environ.get("OLP_HELPER_FIXTURE_ROOT", "")
fixture = bool(fixture_root) and os.geteuid() != 0
if fixture:
if not normalized_absolute(fixture_root):
reject("fixture root is invalid")
for candidate in (root_path, ssd_path, aistor_path):
if os.path.commonpath((fixture_root, candidate)) != fixture_root:
reject("fixture path escaped fixture root")
elif (root_path, ssd_path, aistor_path) != ("/", "/srv/k3s/ssd", "/srv/k3s/aistor"):
reject("production helper paths differ from exact contract")
def fixture_value(name, default=""):
return os.environ.get(name, default) if fixture else default
audit_path = fixture_value("OLP_HELPER_FIXTURE_AUDIT")
if audit_path and os.path.commonpath((fixture_root, audit_path)) != fixture_root:
reject("fixture audit escaped fixture root")
def audit(message):
if audit_path:
with open(audit_path, "a", encoding="utf-8") as stream:
stream.write(message + "\n")
def open_physical_absolute(path):
descriptor = os.open("/", OPEN_DIR)
try:
for component in path.split("/")[1:]:
if not component:
continue
child = os.open(component, OPEN_DIR, dir_fd=descriptor)
os.close(descriptor)
descriptor = child
return descriptor
except BaseException:
os.close(descriptor)
raise
def open_child(parent, name, missing_ok=False):
try:
return os.open(name, OPEN_DIR, dir_fd=parent)
except FileNotFoundError:
if missing_ok:
return None
raise
def identity(descriptor):
metadata = os.fstat(descriptor)
return metadata.st_dev, metadata.st_ino
def entry_identity(parent, name):
metadata = os.stat(name, dir_fd=parent, follow_symlinks=False)
if not stat.S_ISDIR(metadata.st_mode):
reject("entry is not a physical directory")
return metadata.st_dev, metadata.st_ino
def directory_use(descriptor):
total = os.fstat(descriptor).st_blocks * 512
for name in os.listdir(descriptor):
metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False)
if stat.S_ISLNK(metadata.st_mode):
reject("symbolic link below target is forbidden")
if stat.S_ISDIR(metadata.st_mode):
child = os.open(name, OPEN_DIR, dir_fd=descriptor)
try:
total += directory_use(child)
finally:
os.close(child)
else:
total += metadata.st_blocks * 512
if total > MAX_USE:
return total
return total
def filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd):
root_dev = os.fstat(root_fd).st_dev
if os.fstat(ssd_fd).st_dev != root_dev:
reject("SSD base is not on root filesystem")
aistor_dev = os.fstat(aistor_fd).st_dev
if fixture_value("OLP_HELPER_FIXTURE_VIRTUAL_AISTOR") == "1":
aistor_dev = root_dev + 1
if aistor_dev == root_dev:
reject("AIStor must use another filesystem")
fs = os.fstatvfs(root_fd)
available = fs.f_bavail * fs.f_frsize
if available < 0 or available > MAX_INTEGER or available < MIN_AVAILABLE:
reject("root available bytes fail boundary")
uses = []
if obs_fd is None:
uses = [0] * len(TARGETS)
else:
for name, _, _ in TARGETS:
target_fd = open_child(obs_fd, name, True)
if target_fd is None:
uses.append(0)
continue
try:
value = directory_use(target_fd)
if value < 0 or value > MAX_USE:
reject("individual target use exceeds 1GiB")
uses.append(value)
if os.listdir(target_fd):
reject("existing target is non-empty")
finally:
os.close(target_fd)
total = 0
for value in uses:
if value < 0 or value > MAX_USE:
reject("individual target use exceeds 1GiB")
total += value
if total > MAX_USE:
reject("aggregate target use exceeds 1GiB")
return available, uses
root_fd = ssd_fd = aistor_fd = obs_fd = None
created = []
restored = []
active_created = None
def close_all():
seen = set()
values = [record[2] for record in created]
values.extend(record[1] for record in restored)
values.extend((obs_fd, aistor_fd, ssd_fd, root_fd))
for descriptor in values:
if descriptor is None or descriptor in seen:
continue
seen.add(descriptor)
try:
os.close(descriptor)
except OSError:
pass
def rollback():
global active_created
failed = False
if active_created is not None:
parent_fd, name, expected = active_created
try:
if entry_identity(parent_fd, name) == expected:
descriptor = open_child(parent_fd, name)
try:
if os.listdir(descriptor):
failed = True
else:
os.rmdir(name, dir_fd=parent_fd)
audit(f"remove-active-created {name}")
finally:
os.close(descriptor)
else:
failed = True
except OSError:
failed = True
active_created = None
for parent_fd, name, target_fd, expected in reversed(created):
try:
if entry_identity(parent_fd, name) != expected:
failed = True
continue
if os.listdir(target_fd):
failed = True
continue
os.rmdir(name, dir_fd=parent_fd)
audit(f"remove-created {name}")
except OSError:
failed = True
for name, target_fd, uid, gid, mode, _expected in reversed(restored):
try:
os.fchown(target_fd, uid, gid)
audit(f"restore-chown {name} {uid} {gid}")
os.fchmod(target_fd, mode)
audit(f"restore-chmod {name} {mode:04o}")
except OSError:
failed = True
return not failed
def signal_handler(signum, _frame):
raise Interrupted(signum)
def block_signals():
return signal.pthread_sigmask(signal.SIG_BLOCK, SIGNALS)
def restore_signal_mask(previous):
signal.pthread_sigmask(signal.SIG_SETMASK, previous)
def requested_owner(uid, gid):
if fixture_value("OLP_HELPER_FIXTURE_OWNERS") == "1":
return os.getuid(), os.getgid()
return uid, gid
def maybe_fixture_race(kind, name=""):
race = fixture_value("OLP_HELPER_FIXTURE_RACE")
external = fixture_value("OLP_HELPER_FIXTURE_EXTERNAL")
if not race:
return
if not external or os.path.commonpath((fixture_root, external)) != fixture_root:
reject("fixture race external path is invalid")
if kind == "ancestor" and race == "ancestor":
os.rename("observability", "observability.pinned", src_dir_fd=ssd_fd, dst_dir_fd=ssd_fd)
os.symlink(external, "observability", dir_fd=ssd_fd)
audit("fixture-race ancestor")
elif kind == "target" and race == f"target:{name}":
os.rename(name, f"{name}.pinned", src_dir_fd=obs_fd, dst_dir_fd=obs_fd)
os.symlink(external, name, dir_fd=obs_fd)
audit(f"fixture-race target {name}")
def maybe_fixture_signal(name):
configured = fixture_value("OLP_HELPER_FIXTURE_SIGNAL")
target = fixture_value("OLP_HELPER_FIXTURE_SIGNAL_TARGET")
if configured and target == name:
signum = {"HUP": signal.SIGHUP, "INT": signal.SIGINT, "TERM": signal.SIGTERM}.get(configured)
if signum is None:
reject("fixture signal is invalid")
os.kill(os.getpid(), signum)
exit_status = 0
try:
root_fd = open_physical_absolute(root_path)
ssd_fd = open_physical_absolute(ssd_path)
aistor_fd = open_physical_absolute(aistor_path)
obs_fd = open_child(ssd_fd, "observability", True)
available, uses = filesystem_snapshot(root_fd, ssd_fd, aistor_fd, obs_fd)
if action == "preflight":
print("\t".join(["PREFLIGHT", str(available), *(str(value) for value in uses)]))
else:
for signum in SIGNALS:
signal.signal(signum, signal_handler)
if obs_fd is None:
previous = block_signals()
try:
os.mkdir("observability", 0o770, dir_fd=ssd_fd)
active_created = (ssd_fd, "observability", entry_identity(ssd_fd, "observability"))
obs_fd = open_child(ssd_fd, "observability")
if identity(obs_fd) != active_created[2]:
reject("new observability ancestor identity changed before open")
created.append((ssd_fd, "observability", obs_fd, identity(obs_fd)))
active_created = None
finally:
restore_signal_mask(previous)
obs_expected = identity(obs_fd)
maybe_fixture_race("ancestor")
for name, requested_uid, requested_gid in TARGETS:
target_fd = open_child(obs_fd, name, True)
if target_fd is None:
previous = block_signals()
try:
os.mkdir(name, 0o770, dir_fd=obs_fd)
active_created = (obs_fd, name, entry_identity(obs_fd, name))
target_fd = open_child(obs_fd, name)
if identity(target_fd) != active_created[2]:
reject("new target identity changed before open")
created.append((obs_fd, name, target_fd, identity(target_fd)))
active_created = None
actual_uid, actual_gid = requested_owner(requested_uid, requested_gid)
os.fchown(target_fd, actual_uid, actual_gid)
os.fchmod(target_fd, 0o770)
audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770")
finally:
restore_signal_mask(previous)
maybe_fixture_signal(name)
else:
metadata = os.fstat(target_fd)
if os.listdir(target_fd):
reject("existing target is non-empty")
restored.append((
name,
target_fd,
metadata.st_uid,
metadata.st_gid,
stat.S_IMODE(metadata.st_mode),
identity(target_fd),
))
maybe_fixture_race("target", name)
actual_uid, actual_gid = requested_owner(requested_uid, requested_gid)
os.fchown(target_fd, actual_uid, actual_gid)
os.fchmod(target_fd, 0o770)
audit(f"set-owner-mode {name} {requested_uid} {requested_gid} 0770")
metadata = os.fstat(target_fd)
actual_uid, actual_gid = requested_owner(requested_uid, requested_gid)
if (metadata.st_uid, metadata.st_gid, stat.S_IMODE(metadata.st_mode)) != (actual_uid, actual_gid, 0o770):
reject("post-create owner or mode differs")
for parent_fd, name, target_fd, expected in created:
if entry_identity(parent_fd, name) != expected:
reject("created entry identity changed")
for name, _target_fd, _uid, _gid, _mode, expected in restored:
if entry_identity(obs_fd, name) != expected:
reject("pre-existing target entry identity changed")
if entry_identity(ssd_fd, "observability") != obs_expected:
reject("observability ancestor identity changed")
print("APPLIED")
except Interrupted as error:
for signum in SIGNALS:
signal.signal(signum, signal.SIG_IGN)
rollback()
print(f"REJECT: interrupted by signal {error.signum}", file=sys.stderr)
exit_status = 128 + error.signum
except (Rejected, OSError) as error:
for signum in SIGNALS:
signal.signal(signum, signal.SIG_IGN)
rollback()
message = str(error) if str(error) else "privileged helper failed"
print(f"REJECT: {message}", file=sys.stderr)
exit_status = 23
finally:
close_all()
raise SystemExit(exit_status)
PY
}
_olp_privileged_helper() {
if (( $# != 4 )) || [[ "$1" != preflight && "$1" != apply ]]; then
return 97
fi
[[ "$2" == / && "$3" == /srv/k3s/ssd && "$4" == /srv/k3s/aistor ]] || \
return 97
local program
program="$(_olp_helper_program)" || return
/usr/bin/printf '%s\n' "$program" | \
/usr/bin/sudo -- /usr/bin/python3 - "$@"
}
_olp_root_facts() {
/usr/bin/findmnt --kernel --first-only --noheadings --output SOURCE,FSTYPE \
--target "$1" | /usr/bin/awk 'NF == 2 { print $1 "\t" $2 }'
}
_olp_available_bytes() {
/usr/bin/df --block-size=1 --output=avail "$1" | /usr/bin/tail -n 1 | \
/usr/bin/tr -d '[:space:]'
}
_olp_current_context() {
local kubectl_bin
kubectl_bin="$(command -v kubectl)" || return 1
"$kubectl_bin" config current-context
}
_olp_node_names() {
local kubectl_bin
kubectl_bin="$(command -v kubectl)" || return 1
"$kubectl_bin" get nodes \
-o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}'
}
_olp_read_confirmation() {
local value
printf 'Type APPLY default to prepare the six Local PV paths: ' >&2
IFS= read -r value </dev/tty || return 1
printf '%s\n' "$value"
}
_olp_is_normalized_absolute() {
local path="$1"
[[ "$path" == /* && "$path" != *'//' && "$path" != */. && \
"$path" != */.. && "$path" != *'/./'* && "$path" != *'/../'* ]]
}
_olp_is_canonical_decimal() {
[[ "$1" =~ ^(0|[1-9][0-9]*)$ ]]
}
_olp_decimal_le() {
local left="$1" right="$2" LC_ALL=C
(( ${#left} < ${#right} )) || \
{ (( ${#left} == ${#right} )) && [[ "$left" == "$right" || "$left" < "$right" ]]; }
}
_olp_validate_preflight_reply() {
local reply="$1" tag available value total=0
local -a fields=()
[[ "$reply" != *$'\n'* && "$reply" != *$'\r'* ]] || return 1
IFS=$'\t' read -r -a fields <<<"$reply"
(( ${#fields[@]} == 8 )) || return 1
tag="${fields[0]}"
available="${fields[1]}"
[[ "$tag" == PREFLIGHT ]] || return 1
_olp_is_canonical_decimal "$available" || return 1
_olp_decimal_le "$available" 9223372036854775807 || return 1
_olp_decimal_le 53687091200 "$available" || return 1
for value in "${fields[@]:2}"; do
_olp_is_canonical_decimal "$value" || return 1
_olp_decimal_le "$value" 9223372036854775807 || return 1
_olp_decimal_le "$value" 1073741824 || return 1
total=$((total + 10#$value))
(( total <= 1073741824 )) || return 1
done
}
observability_local_paths_dry_run() {
local root_path="$1" ssd_base="$2" aistor_base="$3"
local facts device filesystem available target uid gid
_olp_is_normalized_absolute "$root_path" || \
_olp_error "root path is not normalized" || return
_olp_is_normalized_absolute "$ssd_base" || \
_olp_error "SSD base path is not normalized" || return
_olp_is_normalized_absolute "$aistor_base" || \
_olp_error "AIStor path is not normalized" || return
facts="$(_olp_root_facts "$root_path")" || \
_olp_error "could not read root SSD device/filesystem facts" || return
IFS=$'\t' read -r device filesystem <<<"$facts"
[[ "$device" =~ ^[A-Za-z0-9._/+:=-]+$ && \
"$filesystem" =~ ^[A-Za-z0-9._+-]+$ ]] || \
_olp_error "root SSD facts are not safely printable" || return
available="$(_olp_available_bytes "$root_path")" || \
_olp_error "could not read root SSD available bytes" || return
_olp_is_canonical_decimal "$available" || \
_olp_error "root SSD available bytes are invalid" || return
_olp_decimal_le "$available" 9223372036854775807 || \
_olp_error "root SSD available bytes are invalid" || return
printf 'Root SSD device: %s\n' "$device"
printf 'Root SSD filesystem: %s\n' "$filesystem"
printf 'Root SSD available bytes: %s\n' "$available"
while IFS=$'\t' read -r target uid gid; do
printf 'Planned path: %s\n' "$target"
done < <(_olp_target_rows "$ssd_base")
}
_olp_execute_preflight() {
local root_path="$1" ssd_base="$2" aistor_base="$3"
local reply context nodes
reply="$(_olp_privileged_helper preflight "$root_path" "$ssd_base" "$aistor_base" \
2>/dev/null)" || _olp_error "privileged filesystem preflight failed" || return
_olp_validate_preflight_reply "$reply" || \
_olp_error "privileged preflight protocol or capacity boundary failed" || return
context="$(_olp_current_context)" || \
_olp_error "could not read Kubernetes context" || return
[[ "$context" == default ]] || \
_olp_error "Kubernetes context must be exactly default" || return
nodes="$(_olp_node_names)" || \
_olp_error "could not read Kubernetes node names" || return
[[ "$nodes" == donghyeon-system-product-name ]] || \
_olp_error "Kubernetes node must be exactly donghyeon-system-product-name" || return
}
observability_local_paths_execute() {
local root_path="$1" ssd_base="$2" aistor_base="$3" confirmation reply
_olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return
confirmation="$(_olp_read_confirmation)" || \
_olp_error "TTY confirmation was not read" || return
[[ "$confirmation" == 'APPLY default' ]] || \
_olp_error "confirmation did not match APPLY default" || return
_olp_execute_preflight "$root_path" "$ssd_base" "$aistor_base" || return
reply="$(_olp_privileged_helper apply "$root_path" "$ssd_base" "$aistor_base")" || \
_olp_error "descriptor-relative path preparation failed" || return
[[ "$reply" == APPLIED ]] || \
_olp_error "privileged apply protocol differed" || return
printf 'Prepared six observability Local PV paths on the root filesystem.\n'
printf 'No Kubernetes resources were applied.\n'
}
_olp_usage() {
printf 'Usage: bash scripts/bootstrap/prepare-observability-local-paths.sh [--execute]\n' >&2
}
_olp_main() {
if (( $# == 0 )); then
observability_local_paths_dry_run / /srv/k3s/ssd /srv/k3s/aistor
elif (( $# == 1 )) && [[ "$1" == --execute ]]; then
observability_local_paths_execute / /srv/k3s/ssd /srv/k3s/aistor
else
_olp_usage
return 2
fi
}
_olp_entry() (
set -Eeuo pipefail
_olp_main "$@"
)
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_olp_entry "$@"
fi
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly SSD_BASE_PATH="/srv/k3s/ssd"
readonly POSTGRES_PATH="${SSD_BASE_PATH}/platform-postgres"
readonly GITEA_PATH="${SSD_BASE_PATH}/gitea"
readonly DECLARED_CAPACITY_BYTES=$((70 * 1024 * 1024 * 1024))
readonly SUDO_BIN="/usr/bin/sudo"
readonly FINDMNT_BIN="/usr/bin/findmnt"
readonly INSTALL_BIN="/usr/bin/install"
readonly TEST_BIN="/usr/bin/test"
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
mount_source_for() {
# The Local PV directories are intentionally root:root 0750. An unprivileged
# findmnt cannot canonicalize a child below /srv/k3s/ssd after that directory
# has been created, so perform the mount lookup with the same privilege that
# creates and owns the paths.
"$SUDO_BIN" -- "$FINDMNT_BIN" --kernel --first-only \
--noheadings --output SOURCE --target "$1"
}
mount_target_for() {
"$SUDO_BIN" -- "$FINDMNT_BIN" --kernel --first-only \
--noheadings --output TARGET --target "$1"
}
sudo_test() {
"$SUDO_BIN" -- "$TEST_BIN" "$@"
}
validate_existing_path_kind() {
local target="$1"
if sudo_test -L "$target"; then
fail "refusing symbolic-link target: ${target}"
fi
if sudo_test -e "$target"; then
sudo_test -d "$target" || \
fail "target exists but is not a directory: ${target}"
return 0
fi
return 1
}
ensure_plain_directory() {
local target="$1"
if validate_existing_path_kind "$target"; then
printf 'Already present; left ownership and mode unchanged: %s\n' "$target"
return 0
fi
"$SUDO_BIN" -- "$INSTALL_BIN" -d -o root -g root -m 0750 -- "$target"
validate_existing_path_kind "$target" || \
fail "post-create target is missing: ${target}"
printf 'Created %s\n' "$target"
}
for required_binary in "$SUDO_BIN" "$FINDMNT_BIN" "$INSTALL_BIN" "$TEST_BIN"; do
[[ -x "$required_binary" ]] || \
fail "required executable is missing: ${required_binary}"
done
for command_name in df tail tr; do
command -v "$command_name" >/dev/null 2>&1 || fail "${command_name} is required"
done
[[ -d /srv ]] || fail "/srv does not exist"
# Authenticate before command substitutions call sudo so a failed or cancelled
# prompt cannot be confused with an empty mount-source result.
printf 'Validating sudo access for SSD path checks...\n'
"$SUDO_BIN" -v || fail "sudo authentication failed"
readonly ROOT_SOURCE="$(mount_source_for /)"
readonly SRV_SOURCE="$(mount_source_for /srv)"
readonly SRV_MOUNT_TARGET="$(mount_target_for /srv)"
[[ -n "$ROOT_SOURCE" ]] || fail "could not identify the root filesystem source"
[[ "$SRV_SOURCE" == "$ROOT_SOURCE" ]] || fail "/srv is not on the root filesystem (${SRV_SOURCE} != ${ROOT_SOURCE})"
[[ "$SRV_MOUNT_TARGET" == "/" ]] || fail "/srv is covered by a separate mount (${SRV_MOUNT_TARGET})"
for target in "$SSD_BASE_PATH" "$POSTGRES_PATH" "$GITEA_PATH"; do
if validate_existing_path_kind "$target"; then
[[ "$(mount_source_for "$target")" == "$ROOT_SOURCE" ]] || fail "${target} is not on the root filesystem"
[[ "$(mount_target_for "$target")" == "/" ]] || fail "${target} is covered by a separate mount"
fi
done
readonly ROOT_AVAILABLE_BYTES="$(df --block-size=1 --output=avail / | tail -n 1 | tr -d '[:space:]')"
if [[ "$ROOT_AVAILABLE_BYTES" =~ ^[0-9]+$ ]] && (( ROOT_AVAILABLE_BYTES < DECLARED_CAPACITY_BYTES )); then
printf 'WARNING: root filesystem has less than 70 GiB free; Local PV capacity is not a quota.\n' >&2
fi
ensure_plain_directory "$SSD_BASE_PATH"
ensure_plain_directory "$POSTGRES_PATH"
ensure_plain_directory "$GITEA_PATH"
for target in "$SSD_BASE_PATH" "$POSTGRES_PATH" "$GITEA_PATH"; do
validate_existing_path_kind "$target" || fail "post-create target is missing: ${target}"
[[ "$(mount_source_for "$target")" == "$ROOT_SOURCE" ]] || fail "post-create filesystem check failed for ${target}"
[[ "$(mount_target_for "$target")" == "/" ]] || fail "post-create mount check failed for ${target}"
done
printf 'SSD Local PV paths are ready on root source %s.\n' "$ROOT_SOURCE"
printf 'No Kubernetes resources were applied.\n'
@@ -0,0 +1,70 @@
#!/usr/bin/env bash
PATH='/usr/sbin:/usr/bin:/sbin:/bin'
export PATH
LC_ALL=C
export LC_ALL
umask 077
_k3slrhp_inherited_namespace_is_clean() {
if compgen -A function _k3slrh_ >/dev/null; then return 1; fi
if compgen -A variable K3SLRH_ >/dev/null; then return 1; fi
if compgen -A variable _K3SLRH_ >/dev/null; then return 1; fi
if compgen -A variable K3SLR_ >/dev/null; then return 1; fi
}
if ! _k3slrhp_inherited_namespace_is_clean; then
return 1 2>/dev/null || exit 1
fi
_k3slrh_wrapper_initial_guard() {
local effective_uid="${1-}" shell_options="${2-}"
(( $# == 2 )) || return 1
[[ "$effective_uid" =~ ^[0-9]+$ && "$effective_uid" != 0 ]] || return 1
[[ "$shell_options" != *x* ]]
}
if ! _k3slrh_wrapper_initial_guard "${EUID:-}" "$-"; then
return 1 2>/dev/null || exit 1
fi
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
K3SLRH_WRAPPER_DIRECTORY="${BASH_SOURCE[0]%/*}"
else
K3SLRH_WRAPPER_DIRECTORY='.'
fi
K3SLRH_REPOSITORY_ROOT="$(cd -- "${K3SLRH_WRAPPER_DIRECTORY}/../.." && pwd -P)" || {
return 1 2>/dev/null || exit 1
}
K3SLRH_CONTRACT_PATH="${K3SLRH_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env"
readonly K3SLRH_WRAPPER_DIRECTORY K3SLRH_REPOSITORY_ROOT K3SLRH_CONTRACT_PATH
# shellcheck source=/dev/null
builtin source "${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh" || {
return 1 2>/dev/null || exit 1
}
# shellcheck source=/dev/null
builtin source "${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-a1.sh" || {
return 1 2>/dev/null || exit 1
}
# shellcheck source=/dev/null
builtin source "${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-header-proof.sh" || {
return 1 2>/dev/null || exit 1
}
_k3slr_load_contract "$K3SLRH_CONTRACT_PATH" || {
return 1 2>/dev/null || exit 1
}
readonly K3SLR_SCHEMA_VERSION K3SLR_RECOVERY_DISK_BY_ID K3SLR_RECOVERY_PARTITION_BY_ID
readonly K3SLR_RECOVERY_FS_UUID K3SLR_RECOVERY_PARTUUID K3SLR_RECOVERY_MODEL
readonly K3SLR_RECOVERY_SERIAL K3SLR_RECOVERY_WWN K3SLR_K3S_DISK_BY_ID
readonly K3SLR_K3S_PARTITION_BY_ID K3SLR_K3S_FS_UUID K3SLR_K3S_PARTUUID
readonly K3SLR_K3S_MODEL K3SLR_K3S_SERIAL K3SLR_K3S_WWN K3SLR_OWNER_UID
readonly K3SLR_OWNER_GID K3SLR_OUTER_MOUNT K3SLR_INNER_MOUNT K3SLR_ROOT_RELATIVE
readonly K3SLR_DATABASE_RELATIVE K3SLR_CONTAINER_RELATIVE K3SLR_RUNTIME_METADATA_RELATIVE
readonly K3SLR_MAPPING_NAME K3SLR_PROOF_MAPPING_NAME K3SLR_INNER_LABEL
readonly K3SLR_CONTAINER_SIZE_BYTES K3SLR_MINIMUM_FREE_BYTES K3SLR_OUTER_MIN_REMAINING_PERCENT
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_k3slrh_header_proof_main "$@"
fi
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,344 @@
#!/usr/bin/env bash
# Source-only foundation for the standalone K3s LUKS header proof.
readonly K3SLRH_PROOF_RELATIVE='HyeonworksRecovery/containers/k3s-recovery.header-proof.luks'
readonly K3SLRH_PRE_KDBX_PREFIX='hyeonworks-recovery.header-proof-pre.'
readonly K3SLRH_POST_KDBX_PREFIX='hyeonworks-recovery.header-proof-post.'
readonly K3SLRH_HEADER_BACKUP_NAME='header-original.bin'
readonly K3SLRH_HEADER_EXPORT_NAME='header-exported.bin'
readonly K3SLRH_SQLITE_PACKAGE_VERSION='3.45.1-1ubuntu2.7'
readonly K3SLRH_SHARED_LIBRARY_SHA256='a206d81539e6c37fa085457ba01cff9de57de50f7235e3a4085989780e99585d'
readonly K3SLRH_A1_LIBRARY_SHA256='8b39fc46229ebd4c77ff20040f942c4ef7b756d564d344a8ecdd11a7428e2968'
readonly K3SLRH_A1_HELPER_SHA256='06831fa35943a328ecd183d3c6f721b24c4607b2f349abf6201a12e690d08ebe'
readonly K3SLRH_CONTRACT_SHA256='c045f42c2dc3eda09c37a123bcc05d88d94f275e61504a2d5816948237e516df'
readonly K3SLRH_VALIDATOR_SHA256='6b869032fc325759ee25b219fad4ea6190a3389a561c496f0664a51b802f73e6'
_k3slrh_command() {
"$@"
}
_k3slrh_output_name_is_safe() {
local _k3slrh_output_candidate="${1-}"
(( $# == 1 )) || return 1
[[ "$_k3slrh_output_candidate" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 1
[[ "$_k3slrh_output_candidate" != _k3slrh_* ]]
}
_k3slrh_stderr_marker() {
local _k3slrh_stderr_chunk='' _k3slrh_stderr_rc=0
if IFS= read -r -d '' -n 1 _k3slrh_stderr_chunk; then
_k3slrh_stderr_rc=0
else
_k3slrh_stderr_rc=$?
fi
if (( _k3slrh_stderr_rc == 0 )); then
printf '\0STDERR\0'
while IFS= read -r -d '' -n 4096 _k3slrh_stderr_chunk; do
:
done
elif [[ -n "$_k3slrh_stderr_chunk" ]]; then
printf '\0STDERR\0'
fi
}
_k3slrh_command_packet() {
local _k3slrh_packet_rc=0
(( $# > 0 )) || return 1
if _k3slrh_command "$@" 2> >(_k3slrh_stderr_marker); then
_k3slrh_packet_rc=0
else
_k3slrh_packet_rc=$?
fi
printf '\0RC=%s\0' "$_k3slrh_packet_rc"
}
_k3slrh_capture_command() {
local _k3slrh_capture_output="${1-}" _k3slrh_capture_expected_rc="${2-}"
local -a _k3slrh_capture_records=()
shift 2 || return 1
(( $# > 0 )) || return 1
_k3slrh_output_name_is_safe "$_k3slrh_capture_output" || return 1
[[ "$_k3slrh_capture_expected_rc" =~ ^(0|[1-9][0-9]{0,2})$ ]] || return 1
(( 10#$_k3slrh_capture_expected_rc <= 255 )) || return 1
mapfile -d '' -t _k3slrh_capture_records < <(_k3slrh_command_packet "$@")
(( ${#_k3slrh_capture_records[@]} == 2 )) || return 1
[[ "${_k3slrh_capture_records[1]}" == "RC=${_k3slrh_capture_expected_rc}" ]] || return 1
printf -v "$_k3slrh_capture_output" '%s' "${_k3slrh_capture_records[0]}"
}
_k3slrh_capture_one_line() {
local _k3slrh_line_output="${1-}" _k3slrh_line_value=''
shift || return 1
_k3slrh_output_name_is_safe "$_k3slrh_line_output" || return 1
_k3slrh_capture_command _k3slrh_line_value 0 "$@" || return 1
[[ "$_k3slrh_line_value" == *$'\n' ]] || return 1
[[ "$_k3slrh_line_value" != *$'\r'* ]] || return 1
_k3slrh_line_value="${_k3slrh_line_value%$'\n'}"
[[ -n "$_k3slrh_line_value" && "$_k3slrh_line_value" != *$'\n'* ]] || return 1
printf -v "$_k3slrh_line_output" '%s' "$_k3slrh_line_value"
}
_k3slrh_parse_cli() {
local _k3slrh_cli_output="${1-}" _k3slrh_cli_mode=''
shift || return 1
_k3slrh_output_name_is_safe "$_k3slrh_cli_output" || return 1
case "$#" in
0) _k3slrh_cli_mode=dry-run ;;
1)
[[ "$1" == --execute ]] || return 1
_k3slrh_cli_mode=execute
;;
*) return 1 ;;
esac
printf -v "$_k3slrh_cli_output" '%s' "$_k3slrh_cli_mode"
}
_k3slrh_target_state_is_consistent() {
local _k3slrh_target_utc="${1-}" _k3slrh_target_pre="${2-}"
local _k3slrh_target_post="${3-}" _k3slrh_target_expected_pre=''
local _k3slrh_target_expected_post=''
(( $# == 3 )) || return 1
[[ "$_k3slrh_target_utc" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1
_k3slrh_target_expected_pre="HyeonworksRecovery/vault/backups/${K3SLRH_PRE_KDBX_PREFIX}${_k3slrh_target_utc}.kdbx"
_k3slrh_target_expected_post="HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}${_k3slrh_target_utc}.kdbx"
[[ "$_k3slrh_target_pre" == "$_k3slrh_target_expected_pre" ]] || return 1
[[ "$_k3slrh_target_post" == "$_k3slrh_target_expected_post" ]] || return 1
[[ "$_k3slrh_target_pre" != "$_k3slrh_target_post" ]]
}
_k3slrh_derive_targets() {
local _k3slrh_derive_utc="${1-}" _k3slrh_derive_pre='' _k3slrh_derive_post=''
(( $# == 1 )) || return 1
[[ "$_k3slrh_derive_utc" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1
[[ "${K3SLRH_RUN_UTC+x}" != x && "${K3SLRH_PRE_KDBX_RELATIVE+x}" != x &&
"${K3SLRH_POST_KDBX_RELATIVE+x}" != x ]] || return 1
_k3slrh_derive_pre="HyeonworksRecovery/vault/backups/${K3SLRH_PRE_KDBX_PREFIX}${_k3slrh_derive_utc}.kdbx"
_k3slrh_derive_post="HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}${_k3slrh_derive_utc}.kdbx"
_k3slrh_target_state_is_consistent "$_k3slrh_derive_utc" "$_k3slrh_derive_pre" "$_k3slrh_derive_post" || return 1
declare -gr K3SLRH_RUN_UTC="$_k3slrh_derive_utc"
declare -gr K3SLRH_PRE_KDBX_RELATIVE="$_k3slrh_derive_pre"
declare -gr K3SLRH_POST_KDBX_RELATIVE="$_k3slrh_derive_post"
}
_k3slrh_source_file_is_exact() {
local _k3slrh_source_path="${1-}" _k3slrh_source_hash="${2-}"
local _k3slrh_source_canonical='' _k3slrh_source_metadata='' _k3slrh_source_digest=''
(( $# == 2 )) || return 1
[[ "$_k3slrh_source_path" == /* && "$_k3slrh_source_hash" =~ ^[0-9a-f]{64}$ ]] || return 1
_k3slrh_command /usr/bin/test -f "$_k3slrh_source_path" >/dev/null 2>&1 || return 1
_k3slrh_command /usr/bin/test ! -L "$_k3slrh_source_path" >/dev/null 2>&1 || return 1
_k3slrh_capture_one_line _k3slrh_source_canonical /usr/bin/readlink -e -- "$_k3slrh_source_path" || return 1
[[ "$_k3slrh_source_canonical" == "$_k3slrh_source_path" ]] || return 1
_k3slrh_capture_one_line _k3slrh_source_metadata /usr/bin/stat '--format=%F|%u|%g|%a' -- "$_k3slrh_source_path" || return 1
[[ "$_k3slrh_source_metadata" == 'regular file|'* || "$_k3slrh_source_metadata" == 'regular empty file|'* ]] || return 1
_k3slrh_capture_one_line _k3slrh_source_digest /usr/bin/sha256sum -- "$_k3slrh_source_path" || return 1
[[ "$_k3slrh_source_digest" == "${_k3slrh_source_hash} ${_k3slrh_source_path}" ]]
}
_k3slrh_required_sources_are_exact() {
local _k3slrh_source_root="${K3SLRH_REPOSITORY_ROOT:-}"
(( $# == 0 )) || return 1
[[ "$_k3slrh_source_root" == /* ]] || return 1
_k3slrh_source_file_is_exact "${_k3slrh_source_root}/scripts/lib/k3s-local-recovery.sh" "$K3SLRH_SHARED_LIBRARY_SHA256" || return 1
_k3slrh_source_file_is_exact "${_k3slrh_source_root}/scripts/lib/k3s-local-recovery-a1.sh" "$K3SLRH_A1_LIBRARY_SHA256" || return 1
_k3slrh_source_file_is_exact "${_k3slrh_source_root}/scripts/libexec/k3s-local-recovery-a1-check.sh" "$K3SLRH_A1_HELPER_SHA256" || return 1
_k3slrh_source_file_is_exact "${_k3slrh_source_root}/infrastructure/security/k3s/local-recovery.env" "$K3SLRH_CONTRACT_SHA256"
}
_k3slrh_required_binaries_are_trusted() {
local _k3slrh_binary=''
local -a _k3slrh_binaries=(
/usr/bin/bash
/usr/bin/date
/usr/bin/dpkg-query
/usr/bin/env
/usr/bin/find
/usr/bin/findmnt
/usr/bin/keepassxc-cli
/usr/bin/readlink
/usr/bin/sha256sum
/usr/bin/sqlite3
/usr/bin/stat
/usr/bin/sudo
/usr/bin/systemctl
/usr/bin/test
/usr/bin/timeout
/usr/local/bin/k3s
/usr/sbin/cryptsetup
/usr/sbin/losetup
)
(( $# == 0 )) || return 1
for _k3slrh_binary in "${_k3slrh_binaries[@]}"; do
_k3slr_trusted_root_executable "$_k3slrh_binary" || return 1
done
}
_k3slrh_package_is_exact() {
local _k3slrh_package_name="${1-}" _k3slrh_package_expected="${2-}"
local _k3slrh_package_record=''
(( $# == 2 )) || return 1
case "$_k3slrh_package_name|$_k3slrh_package_expected" in
'keepassxc|2.7.6+dfsg.1-1build3'|'cryptsetup-bin|2:2.7.0-1ubuntu4.2'|"sqlite3|${K3SLRH_SQLITE_PACKAGE_VERSION}") ;;
*) return 1 ;;
esac
_k3slrh_capture_command _k3slrh_package_record 0 /usr/bin/dpkg-query --show '--showformat=${Status}|${Version}\n' "$_k3slrh_package_name" || return 1
[[ "$_k3slrh_package_record" == "install ok installed|${_k3slrh_package_expected}"$'\n' ]]
}
_k3slrh_packages_are_exact() {
(( $# == 0 )) || return 1
_k3slrh_package_is_exact keepassxc '2.7.6+dfsg.1-1build3' || return 1
_k3slrh_package_is_exact cryptsetup-bin '2:2.7.0-1ubuntu4.2' || return 1
_k3slrh_package_is_exact sqlite3 "$K3SLRH_SQLITE_PACKAGE_VERSION"
}
_k3slrh_cached_sudo_is_available() {
(( $# == 0 )) || return 1
_k3slrh_command /usr/bin/sudo --non-interactive --validate >/dev/null 2>&1
}
_k3slrh_k3s_health() {
local _k3slrh_health_systemctl='' _k3slrh_health_readyz=''
(( $# == 0 )) || return 1
_k3slrh_capture_command _k3slrh_health_systemctl 0 /usr/bin/systemctl is-active --quiet k3s || return 1
[[ -z "$_k3slrh_health_systemctl" ]] || return 1
_k3slrh_capture_command _k3slrh_health_readyz 0 \
/usr/bin/sudo --non-interactive -- \
/usr/bin/env -i 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' LC_ALL=C \
/usr/bin/timeout --signal=TERM --kill-after=1s 30s \
/usr/local/bin/k3s kubectl get --raw=/readyz || return 1
[[ "$_k3slrh_health_readyz" == $'ok\n' ]]
}
_k3slrh_run_closed_validator() {
local _k3slrh_validator_path='' _k3slrh_validator_record=''
local _k3slrh_validator_command_rc=0 _k3slrh_validator_post_rc=0
(( $# == 0 )) || return 1
[[ "${K3SLRH_REPOSITORY_ROOT:-}" == /* ]] || return 1
_k3slrh_validator_path="${K3SLRH_REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh"
_k3slrh_source_file_is_exact "$_k3slrh_validator_path" "$K3SLRH_VALIDATOR_SHA256" || return 1
_k3slrh_capture_command _k3slrh_validator_record 0 \
/usr/bin/env -i 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' LC_ALL=C \
/usr/bin/bash --noprofile --norc -- "$_k3slrh_validator_path" --expect-closed ||
_k3slrh_validator_command_rc=1
_k3slrh_source_file_is_exact "$_k3slrh_validator_path" "$K3SLRH_VALIDATOR_SHA256" ||
_k3slrh_validator_post_rc=1
(( _k3slrh_validator_command_rc == 0 && _k3slrh_validator_post_rc == 0 )) || return 1
[[ "$_k3slrh_validator_record" == $'Recovery device: match\nRecovery state: closed\nLineage: match\nLatest bundle: not_checked\n' ]]
}
_k3slrh_mount_is_absent() {
local _k3slrh_mount_path="${1-}" _k3slrh_mount_record=''
(( $# == 1 )) || return 1
[[ "$_k3slrh_mount_path" == /* ]] || return 1
_k3slrh_capture_command _k3slrh_mount_record 1 /usr/bin/findmnt -rn -M "$_k3slrh_mount_path" -o TARGET || return 1
[[ -z "$_k3slrh_mount_record" ]]
}
_k3slrh_mapping_is_absent() {
local _k3slrh_mapping_path="${1-}"
(( $# == 1 )) || return 1
case "$_k3slrh_mapping_path" in
"/dev/mapper/${K3SLR_MAPPING_NAME}"|"/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}") ;;
*) return 1 ;;
esac
_k3slrh_command /usr/bin/test ! -e "$_k3slrh_mapping_path" >/dev/null 2>&1 || return 1
_k3slrh_command /usr/bin/test ! -L "$_k3slrh_mapping_path" >/dev/null 2>&1
}
_k3slrh_related_loops_are_absent() {
local _k3slrh_loop_record='' _k3slrh_loop_line=''
local _k3slrh_loop_original="${K3SLR_OUTER_MOUNT}/${K3SLR_CONTAINER_RELATIVE}"
local _k3slrh_loop_proof="${K3SLR_OUTER_MOUNT}/${K3SLRH_PROOF_RELATIVE}"
(( $# == 0 )) || return 1
_k3slrh_capture_command _k3slrh_loop_record 0 /usr/sbin/losetup --list --noheadings --output NAME,BACK-FILE || return 1
while IFS= read -r _k3slrh_loop_line; do
[[ -n "$_k3slrh_loop_line" ]] || continue
case "$_k3slrh_loop_line" in
*" ${_k3slrh_loop_original}"|*" ${_k3slrh_loop_proof}") return 1 ;;
esac
done <<<"$_k3slrh_loop_record"
}
_k3slrh_outer_mountpoint_is_clean() {
local _k3slrh_outer_canonical='' _k3slrh_outer_metadata='' _k3slrh_outer_children=''
(( $# == 0 )) || return 1
_k3slrh_command /usr/bin/test -d "$K3SLR_OUTER_MOUNT" >/dev/null 2>&1 || return 1
_k3slrh_command /usr/bin/test ! -L "$K3SLR_OUTER_MOUNT" >/dev/null 2>&1 || return 1
_k3slrh_capture_one_line _k3slrh_outer_canonical /usr/bin/readlink -e -- "$K3SLR_OUTER_MOUNT" || return 1
[[ "$_k3slrh_outer_canonical" == "$K3SLR_OUTER_MOUNT" ]] || return 1
_k3slrh_capture_one_line _k3slrh_outer_metadata /usr/bin/stat '--format=%F|%u|%g|%a' -- "$K3SLR_OUTER_MOUNT" || return 1
[[ "$_k3slrh_outer_metadata" == 'directory|0|0|755' ]] || return 1
_k3slrh_capture_command _k3slrh_outer_children 0 /usr/bin/find "$K3SLR_OUTER_MOUNT" -mindepth 1 -maxdepth 1 -print -quit || return 1
[[ -z "$_k3slrh_outer_children" ]]
}
_k3slrh_runtime_residue_is_absent() {
local _k3slrh_runtime_record=''
(( $# == 0 )) || return 1
_k3slrh_capture_command _k3slrh_runtime_record 0 /usr/bin/find /run -mindepth 1 -maxdepth 1 -name 'hyeonworks-k3s-recovery.*' -print -quit || return 1
[[ -z "$_k3slrh_runtime_record" ]]
}
_k3slrh_closed_resources_are_absent() {
(( $# == 0 )) || return 1
_k3slrh_mount_is_absent "$K3SLR_OUTER_MOUNT" || return 1
_k3slrh_mount_is_absent "$K3SLR_INNER_MOUNT" || return 1
_k3slrh_mapping_is_absent "/dev/mapper/${K3SLR_MAPPING_NAME}" || return 1
_k3slrh_mapping_is_absent "/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}" || return 1
_k3slrh_related_loops_are_absent || return 1
_k3slrh_outer_mountpoint_is_clean || return 1
_k3slrh_runtime_residue_is_absent
}
_k3slrh_pre_mount_checks() {
(( $# == 0 )) || return 1
_k3slrh_required_sources_are_exact || return 1
_k3slrh_required_binaries_are_trusted || return 1
_k3slrh_packages_are_exact || return 1
_k3slrh_cached_sudo_is_available || return 1
_k3slrh_k3s_health || return 1
_k3slrh_run_closed_validator || return 1
_k3slrh_closed_resources_are_absent
}
_k3slrh_pre_mount_capability() {
(( $# == 0 )) || return 1
_k3slrh_pre_mount_checks || return 1
printf 'Header proof capability: pass\n'
}
_k3slrh_current_utc() {
local _k3slrh_utc_output="${1-}" _k3slrh_utc_value=''
(( $# == 1 )) || return 1
_k3slrh_output_name_is_safe "$_k3slrh_utc_output" || return 1
_k3slrh_capture_one_line _k3slrh_utc_value /usr/bin/date --utc +%Y%m%dT%H%M%SZ || return 1
[[ "$_k3slrh_utc_value" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1
printf -v "$_k3slrh_utc_output" '%s' "$_k3slrh_utc_value"
}
_k3slrh_header_proof_main() {
local _k3slrh_main_mode='' _k3slrh_main_utc='' _k3slrh_main_parse_rc=0
if _k3slrh_parse_cli _k3slrh_main_mode "$@"; then
_k3slrh_main_parse_rc=0
else
_k3slrh_main_parse_rc=$?
fi
(( _k3slrh_main_parse_rc == 0 )) || return 2
[[ "${EUID:-}" =~ ^[0-9]+$ && "${EUID:-}" != 0 ]] || return 1
_k3slr_xtrace_is_disabled || return 1
if [[ "$_k3slrh_main_mode" == execute ]]; then
_k3slr_require_execute_tty || return 1
declare -F _k3slrh_prove_header_restore_a1 >/dev/null || return 1
return 1
fi
_k3slrh_current_utc _k3slrh_main_utc || return 1
_k3slrh_derive_targets "$_k3slrh_main_utc" || return 1
_k3slrh_pre_mount_checks || return 1
printf '%s\n' \
'Header proof state: closed' \
'Header proof capability: pass' \
'Execute-time checks: pending' \
'DRY RUN PASS: no changes'
}
File diff suppressed because it is too large Load Diff
+569
View File
@@ -0,0 +1,569 @@
#!/usr/bin/env bash
# Source-only helpers for k3s Secret-encryption validation. They deliberately
# return classifications instead of printing service, config, or key material.
_k3s_status_schema_valid() {
jq -e -s '
length == 1 and
(.[0] | type == "object" and
(.stage | type == "string") and
(.activekey | type == "string") and
((has("enable") | not) or (.enable | type == "boolean")) and
((has("hashmatch") | not) or (.hashmatch | type == "boolean")) and
((has("hasherror") | not) or (.hasherror | type == "string")) and
((has("inactivekeys") | not) or (.inactivekeys | type == "array")))
' >/dev/null 2>&1 <<<"$1"
}
classify_encryption_status() {
local status_json="$1"
if ! _k3s_status_schema_valid "$status_json"; then
printf 'invalid\n'
return 0
fi
if jq -e '
(has("enable") | not) and .stage == "" and .activekey == "" and
(has("hasherror") | not)
' >/dev/null <<<"$status_json"; then
printf 'disabled_no_config\n'
return 0
fi
if jq -e '
has("enable") and
((.hashmatch != true) or (has("hasherror") and .hasherror != ""))
' >/dev/null <<<"$status_json"; then
printf 'hash_mismatch\n'
return 0
fi
if jq -e '
.enable == false and .stage == "start" and .activekey == "" and
.hashmatch == true and ((has("inactivekeys") | not) or (.inactivekeys | length == 0))
' >/dev/null <<<"$status_json"; then
printf 'transition_start\n'
return 0
fi
if jq -e '
.enable == true and (.activekey | length > 0) and .hashmatch == true and
(.stage == "start" or .stage == "reencrypt_finished")
' >/dev/null <<<"$status_json"; then
printf 'enabled_stable\n'
return 0
fi
if jq -e '.enable == true and .hashmatch == true' >/dev/null <<<"$status_json"; then
printf 'unsafe_transition\n'
else
printf 'invalid\n'
fi
}
classify_encryption_provider() {
local status_json="$1"
local active_key
if ! _k3s_status_schema_valid "$status_json"; then
printf 'invalid\n'
return 0
fi
active_key="$(jq -r '.activekey' <<<"$status_json")"
case "$active_key" in
'AES-CBC '*)
[[ "${active_key#AES-CBC }" != "$active_key" && -n "${active_key#AES-CBC }" ]] && printf 'aescbc\n' || printf 'invalid\n'
;;
'XSalsa20-POLY1305 '*)
[[ "${active_key#XSalsa20-POLY1305 }" != "$active_key" && -n "${active_key#XSalsa20-POLY1305 }" ]] && printf 'secretbox\n' || printf 'invalid\n'
;;
*)
printf 'invalid\n'
;;
esac
}
version_supports_late_enable() {
local version="$1"
local minor patch
[[ "$version" =~ ^v1\.([0-9]+)\.([0-9]+)\+k3s[0-9]+$ ]] || return 1
minor="${BASH_REMATCH[1]}"
patch="${BASH_REMATCH[2]}"
case "$minor" in
33) (( patch >= 10 )) ;;
34) (( patch >= 6 )) ;;
35) (( patch >= 3 )) ;;
*) (( minor >= 36 )) ;;
esac
}
require_exact_encryption_state() {
local expected="$1"
local status_json="$2"
[[ "$(classify_encryption_status "$status_json")" == "$expected" ]]
}
_k3s_secure_regular_file() {
local path="$1"
local metadata
[[ -f "$path" && ! -L "$path" ]] || return 1
metadata="$(stat --format='%u:%a:%F' -- "$path" 2>/dev/null)" || return 1
[[ "$metadata" == '0:600:regular file' ]]
}
_k3s_read_file() {
local path="$1"
[[ -f "$path" && ! -L "$path" ]] || return 1
command cat -- "$path"
}
_k3s_list_yaml_files() {
local config="$1" file
[[ -f "$config" && ! -L "$config" ]] && printf '%s\n' "$config"
if [[ -d "${config}.d" && ! -L "${config}.d" ]]; then
while IFS= read -r file; do printf '%s\n' "$file"; done < <(
find "${config}.d" -maxdepth 1 -type f -name '*.yaml' -print | sort
)
fi
}
verify_local_encryption_config_integrity() {
local config_path="$1"
local state_path="$2"
local annotation_value="$3"
local status_stage="$4"
local state_value config_hash config_metadata state_metadata
if ! _k3s_secure_regular_file "$config_path" || ! _k3s_secure_regular_file "$state_path"; then
printf 'mismatch\n'
return 1
fi
[[ "$status_stage" =~ ^[a-z_]+$ ]] || {
printf 'mismatch\n'
return 1
}
state_value="$(<"$state_path")"
[[ "$(wc -c <"$state_path" | tr -d '[:space:]')" == "${#state_value}" ]] || {
printf 'mismatch\n'
return 1
}
[[ "$state_value" =~ ^([a-z_]+)-([0-9a-f]{64})$ ]] || {
printf 'mismatch\n'
return 1
}
config_hash="$(sha256sum -- "$config_path" | awk '{print $1}')" || {
printf 'mismatch\n'
return 1
}
config_metadata="$(stat --format='%u:%a:%F' -- "$config_path" 2>/dev/null)" || config_metadata=''
state_metadata="$(stat --format='%u:%a:%F' -- "$state_path" 2>/dev/null)" || state_metadata=''
verify_local_encryption_config_integrity_evidence \
"$config_metadata" "$state_metadata" "$config_hash" "$state_value" \
"$annotation_value" "$status_stage"
}
verify_local_encryption_config_integrity_evidence() {
local config_metadata="$1" state_metadata="$2" config_hash="$3"
local state_value="$4" annotation_value="$5" status_stage="$6" expected
if [[ "$config_metadata" != '0:600:regular file' ||
"$state_metadata" != '0:600:regular file' ||
! "$config_hash" =~ ^[0-9a-f]{64}$ ||
! "$status_stage" =~ ^[a-z_]+$ ||
! "$state_value" =~ ^[a-z_]+-[0-9a-f]{64}$ ]]; then
printf 'mismatch\n'
return 1
fi
expected="${status_stage}-${config_hash}"
if [[ "$state_value" == "$expected" && "$annotation_value" == "$expected" ]]; then
printf 'match\n'
return 0
fi
printf 'mismatch\n'
return 1
}
_k3s_systemctl_show() {
systemctl show k3s \
--property=ExecStart \
--property=Environment \
--property=EnvironmentFiles 2>/dev/null
}
_k3s_systemd_dropin_lines() {
local dropin_dir="${K3S_SYSTEMD_DROPIN_DIR:-/etc/systemd/system/k3s.service.d}"
local file
[[ -e "$dropin_dir" ]] || return 0
[[ -d "$dropin_dir" && ! -L "$dropin_dir" ]] || return 1
while IFS= read -r file; do
[[ -r "$file" && ! -L "$file" ]] || return 1
sed -n -E '/^[[:space:]]*(Environment|EnvironmentFile)=/p' "$file"
done < <(find "$dropin_dir" -maxdepth 1 -type f -name '*.conf' -print | sort)
}
_k3s_is_dynamic_value() {
local value="$1"
[[ "$value" == *'$'* || "$value" == *'`'* || "$value" == *'\\'* ]]
}
# Strict, non-evaluating readers for systemd's already-resolved show output and
# EnvironmentFile syntax. Unsupported quoting, resets, or expansions fail
# closed instead of being interpreted by a shell.
_k3s_plain_value() {
local value="$1" quote="'"
[[ -n "$value" && "$value" != *' '* && "$value" != *'"'* && "$value" != *"$quote"* && "$value" != *'$'* && "$value" != *'`'* && "$value" != *'\\'* ]]
}
_k3s_safe_payload() {
local value="$1" quote="'"
[[ -n "$value" && "$value" != *'"'* && "$value" != *"$quote"* && "$value" != *'$'* && "$value" != *'`'* && "$value" != *'\\'* ]]
}
_k3s_effective_service_show() {
local show dropins line has_environment=false
show="$(_k3s_systemctl_show)" || return 2
while IFS= read -r line; do
[[ "$line" == Environment=* || "$line" == EnvironmentFiles=* ]] || continue
[[ -n "${line#*=}" ]] && has_environment=true
done <<<"$show"
if ! "$has_environment"; then
dropins="$(_k3s_systemd_dropin_lines)" || return 2
[[ -z "$dropins" ]] || show+=$'\n'"$dropins"
fi
printf '%s\n' "$show"
}
_k3s_envfile_values() {
local file="$1" key="$2" line trimmed name value text char quote='' content
local in_record=false target_record=false continued=false
local index length
content="$(_k3s_read_file "$file")" || return 2
while IFS= read -r line || [[ -n "$line" ]]; do
if ! "$in_record"; then
trimmed="${line#"${line%%[![:space:]]*}"}"
[[ -z "$trimmed" || "$trimmed" == \#* || "$trimmed" == \;* ]] && continue
[[ "$trimmed" == *=* ]] || { [[ "$trimmed" == "$key" ]] && return 2; continue; }
name="${trimmed%%=*}"
value="${trimmed#*=}"
target_record=false
[[ "$name" == "$key" ]] && target_record=true
text="$value"
quote=''
else
text="$line"
fi
continued=false
length=${#text}
for ((index=0; index<length; index++)); do
char="${text:index:1}"
case "$quote" in
"'")
[[ "$char" == "'" ]] && quote=''
;;
'"')
if [[ "$char" == "\\" ]]; then
if (( index + 1 < length )); then ((index++)); else continued=true; fi
elif [[ "$char" == '"' ]]; then
quote=''
fi
;;
'')
if [[ "$char" == "\\" ]]; then
if (( index + 1 < length )); then ((index++)); else continued=true; fi
elif [[ "$char" == "'" || "$char" == '"' ]]; then
quote="$char"
fi
;;
esac
done
if [[ -n "$quote" ]] || "$continued"; then
in_record=true
continue
fi
if "$target_record"; then
_k3s_plain_value "$value" || return 2
printf '%s\n' "$value"
fi
in_record=false
target_record=false
done <<<"$content"
"$in_record" && return 2
return 0
}
_k3s_service_env_values() {
local show="$1" key="$2" line payload token name value candidate file result policy
local index
local -a words=()
candidate=''
while IFS= read -r line; do
[[ "$line" == Environment=* ]] || continue
payload="${line#Environment=}"
[[ -n "$payload" ]] || continue
_k3s_safe_payload "$payload" || return 2
for token in $payload; do
[[ "$token" == *=* ]] || return 2
name="${token%%=*}"; value="${token#*=}"
[[ "$name" == "$key" ]] || continue
[[ -n "$value" ]] || return 2
candidate="$value"
done
done <<<"$show"
while IFS= read -r line; do
[[ "$line" == EnvironmentFiles=* ]] || continue
payload="${line#EnvironmentFiles=}"
[[ -n "$payload" ]] || continue
_k3s_safe_payload "$payload" || return 2
read -r -a words <<<"$payload"
for ((index=0; index<${#words[@]};)); do
file="${words[index]}"; ((index++))
policy=no
if (( index < ${#words[@]} )) && [[ "${words[index]}" =~ ^\(ignore_errors=(yes|no)\)$ ]]; then
policy="${BASH_REMATCH[1]}"; ((index++))
fi
[[ "$file" != \(* ]] || return 2
file="${file#-}"
if [[ ! -r "$file" || -L "$file" ]]; then
[[ "$policy" == yes ]] && continue
return 2
fi
result="$(_k3s_envfile_values "$file" "$key")" || return $?
while IFS= read -r token; do [[ -n "$token" ]] && candidate="$token"; done <<<"$result"
done
done <<<"$show"
[[ -n "$candidate" ]] && printf '%s\n' "$candidate"
return 0
}
_k3s_cli_values() {
local show="$1" option="$2" line payload option_value exec_count=0
local serialized_path serialized_argv serialized_body serialized_tail
local serialized_prefix='{ path=' serialized_separator=' ; argv[]='
local serialized_short_suffix=' ; ignore_errors=no ; }'
local serialized_full_marker=' ; ignore_errors=no ; start_time='
local serialized_full_pattern='^ ; ignore_errors=no ; start_time=\[[^][;={}]+\] ; stop_time=\[[^][;={}]+\] ; pid=[0-9]+ ; code=(\(null\)|exited|killed|dumped) ; status=[0-9]+/[0-9]+ \}$'
local -a words=()
local index token
while IFS= read -r line; do
[[ "$line" == ExecStart=* ]] || continue
((exec_count+=1))
payload="${line#ExecStart=}"
[[ -n "$payload" ]] || return 2
_k3s_safe_payload "$payload" || return 2
if [[ "$payload" == "$serialized_prefix"* ]]; then
serialized_body="${payload#"$serialized_prefix"}"
[[ "$serialized_body" == *"$serialized_separator"* ]] || return 2
serialized_path="${serialized_body%%"$serialized_separator"*}"
serialized_body="${serialized_body#"${serialized_path}${serialized_separator}"}"
if [[ "$serialized_body" == *"$serialized_short_suffix" ]]; then
serialized_argv="${serialized_body%"$serialized_short_suffix"}"
serialized_tail="$serialized_short_suffix"
elif [[ "$serialized_body" == *"$serialized_full_marker"* ]]; then
serialized_argv="${serialized_body%%"$serialized_full_marker"*}"
serialized_tail="${serialized_body#"$serialized_argv"}"
[[ "$serialized_tail" =~ $serialized_full_pattern ]] || return 2
else
return 2
fi
[[ "$payload" == "${serialized_prefix}${serialized_path}${serialized_separator}${serialized_argv}${serialized_tail}" ]] || return 2
[[ "$serialized_path" == /usr/local/bin/k3s ]] || return 2
[[ -n "$serialized_argv" && "$serialized_argv" != *'argv[]='* && "$serialized_argv" != *' ; '* && "$serialized_argv" != *'{'* && "$serialized_argv" != *'}'* ]] || return 2
payload="$serialized_argv"
elif [[ "$payload" == *'argv[]='* || "$payload" == *'path='* || "$payload" == *'ignore_errors='* || "$payload" == *'{'* || "$payload" == *'}'* || "$payload" == *';'* ]]; then
return 2
fi
read -r -a words <<<"$payload"
(( ${#words[@]} >= 2 )) || return 2
[[ "${words[0]##*/}" == k3s && "${words[1]}" == server ]] || return 2
for ((index=0; index<${#words[@]}; index++)); do
token="${words[index]}"
case "$token" in
"${option}" )
if [[ "$option" == '--secrets-encryption' ]]; then printf 'true\n';
elif (( index + 1 < ${#words[@]} )); then option_value="${words[index + 1]}"; _k3s_plain_value "$option_value" || return 2; printf '%s\n' "$option_value"; else return 2; fi
;;
"${option}"=*) option_value="${token#*=}"; _k3s_plain_value "$option_value" || return 2; printf '%s\n' "$option_value" ;;
esac
done
done <<<"$show"
(( exec_count == 1 )) || return 2
return 0
}
_k3s_unique_value() {
local values="$1" value count=0 chosen=''
while IFS= read -r value; do
[[ -n "$value" ]] || continue
((count++)); chosen="$value"
done <<<"$values"
(( count <= 1 )) || return 2
[[ -n "$chosen" ]] && printf '%s\n' "$chosen"
return 0
}
_k3s_selected_config() {
local show="$1" cli env default_root raw
raw="$(_k3s_cli_values "$show" --config)" || return 2
cli="$(_k3s_unique_value "$raw")" || return 2
raw="$(_k3s_service_env_values "$show" K3S_CONFIG_FILE)" || return 2
env="$(_k3s_unique_value "$raw")" || return 2
if _k3s_is_dynamic_value "$cli" || _k3s_is_dynamic_value "$env"; then
return 2
fi
default_root="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
if [[ -n "$cli" ]]; then printf '%s\n' "$cli"; elif [[ -n "$env" ]]; then printf '%s\n' "$env"; else printf '%s\n' "${default_root}/config.yaml"; fi
}
_k3s_yaml_value() {
local config="$1" key="$2" file line raw value selected='' selected_file='' content files
files="$(_k3s_list_yaml_files "$config")" || return 2
while IFS= read -r file; do
[[ -n "$file" ]] || continue
content="$(_k3s_read_file "$file")" || return 2
raw=''
while IFS= read -r line || [[ -n "$line" ]]; do
line="${line#"${line%%[![:space:]]*}"}"
[[ "$line" == "${key}:"* ]] || continue
value="${line#"${key}:"}"
value="$(sed -E 's/[[:space:]]+#.*$//; s/^[[:space:]]+//; s/[[:space:]]+$//;' <<<"$value")"
_k3s_plain_value "$value" || return 2
[[ -z "$raw" || "$raw" == "$value" ]] || return 2
raw="$value"
done <<<"$content"
if [[ -n "$raw" ]]; then selected="$raw"; selected_file="$file"; fi
done <<<"$files"
[[ -n "$selected" ]] && printf '%s|%s\n' "$selected_file" "$selected"
return 0
}
_k3s_resolve_key() {
local show="$1" cli_option="$2" env_key="$3" yaml_key="$4" cli env config yaml raw
raw="$(_k3s_cli_values "$show" "$cli_option")" || return 2
cli="$(_k3s_unique_value "$raw")" || return 2
[[ -z "$cli" ]] || { printf 'command-line|%s\n' "$cli"; return 0; }
raw="$(_k3s_service_env_values "$show" "$env_key")" || return 2
env="$(_k3s_unique_value "$raw")" || return 2
[[ -z "$env" ]] || { printf 'environment|%s\n' "$env"; return 0; }
config="$(_k3s_selected_config "$show")" || return 2
yaml="$(_k3s_yaml_value "$config" "$yaml_key")" || return 2
[[ -z "$yaml" ]] || printf '%s\n' "$yaml"
}
detect_effective_encryption_config_owner() {
local show enable provider enable_source enable_value provider_source provider_value
show="$(_k3s_effective_service_show)" || { printf 'ambiguous\n'; return 0; }
enable="$(_k3s_resolve_key "$show" --secrets-encryption K3S_SECRETS_ENCRYPTION secrets-encryption)" || { printf 'ambiguous\n'; return 0; }
provider="$(_k3s_resolve_key "$show" --secrets-encryption-provider K3S_SECRETS_ENCRYPTION_PROVIDER secrets-encryption-provider)" || { printf 'ambiguous\n'; return 0; }
[[ -n "$enable" ]] || { printf 'ambiguous\n'; return 0; }
IFS='|' read -r enable_source enable_value <<<"$enable"
[[ "$enable_value" == true ]] || { printf 'ambiguous\n'; return 0; }
if [[ -n "$provider" ]]; then IFS='|' read -r provider_source provider_value <<<"$provider"; [[ "$provider_value" == aescbc || "$provider_value" == secretbox ]] || { printf 'ambiguous\n'; return 0; }; printf '%s/%s\n' "$provider_value" "$provider_source"; else printf 'aescbc/implicit-default\n'; fi
}
_k3s_datastore_local_evidence() {
local data_dir="$1"
local state_db="${data_dir}/server/db/state.db"
local etcd_dir="${data_dir}/server/db/etcd"
if [[ -f "$state_db" && -d "$etcd_dir" ]]; then printf 'ambiguous\n';
elif [[ -f "$state_db" ]]; then printf 'sqlite\n';
elif [[ -d "$etcd_dir" ]]; then printf 'embedded-etcd\n';
else printf 'none\n'; fi
}
detect_k3s_datastore() {
local show data endpoint source value evidence
show="$(_k3s_effective_service_show)" || { printf 'ambiguous\n'; return 0; }
data="$(_k3s_resolve_key "$show" --data-dir K3S_DATA_DIR data-dir)" || { printf 'ambiguous\n'; return 0; }
endpoint="$(_k3s_resolve_key "$show" --datastore-endpoint K3S_DATASTORE_ENDPOINT datastore-endpoint)" || { printf 'ambiguous\n'; return 0; }
if [[ -n "$data" ]]; then IFS='|' read -r source value <<<"$data"; data="$value"; else source=default; data=/var/lib/rancher/k3s; fi
_k3s_is_dynamic_value "$data" || [[ "$data" == /* ]] || { printf 'ambiguous\n'; return 0; }
[[ "$source" == default || "$data" == /var/lib/rancher/k3s ]] || { printf 'ambiguous\n'; return 0; }
evidence="$(_k3s_datastore_local_evidence "$data")" || { printf 'ambiguous\n'; return 0; }
if [[ -n "$endpoint" ]]; then [[ "$evidence" == none ]] && printf 'external\n' || printf 'ambiguous\n';
elif [[ "$evidence" == sqlite || "$evidence" == embedded-etcd ]]; then printf '%s\n' "$evidence";
else printf 'ambiguous\n'; fi
}
_k3s_now_seconds() {
printf '%s\n' "$SECONDS"
}
_k3s_timeout_before_deadline() {
local deadline="$1" now remaining command_timeout=9
now="$(_k3s_now_seconds)" || return 1
[[ "$now" =~ ^[0-9]+$ ]] || return 1
remaining=$((deadline - now))
(( remaining > 1 )) || return 1
(( command_timeout < remaining )) || command_timeout=$((remaining - 1))
printf '%s\n' "$command_timeout"
}
_k3s_sleep_until_poll() {
local deadline="$1" next_poll="$2" now delay remaining
now="$(_k3s_now_seconds)" || return 1
[[ "$now" =~ ^[0-9]+$ ]] || return 1
(( now < deadline )) || return 1
delay=$((next_poll - now))
(( delay > 0 )) || return 0
remaining=$((deadline - now))
(( delay < remaining )) || delay="$remaining"
sleep "$delay"
}
_k3s_read_encryption_status() {
local command_timeout="$1"
[[ "$command_timeout" =~ ^[1-9]$ ]] || return 1
timeout --signal=TERM --kill-after=1s "${command_timeout}s" \
sudo --non-interactive k3s secrets-encrypt status --output json 2>/dev/null
}
wait_for_k3s_api() {
local attempt started_at deadline next_poll command_timeout
started_at="$(_k3s_now_seconds)" || return 1
[[ "$started_at" =~ ^[0-9]+$ ]] || return 1
deadline=$((started_at + 600))
for ((attempt=1; attempt<=60; attempt++)); do
command_timeout="$(_k3s_timeout_before_deadline "$deadline")" || return 1
if timeout --signal=TERM --kill-after=1s "${command_timeout}s" \
sudo --non-interactive k3s kubectl get --raw=/readyz >/dev/null 2>&1; then
return 0
fi
(( attempt < 60 )) || break
next_poll=$((started_at + attempt * 10))
_k3s_sleep_until_poll "$deadline" "$next_poll" || return 1
done
return 1
}
wait_for_reencrypt_finished() {
local rotate_rc="$1"
local attempt status_json status_class stage started_at deadline next_poll command_timeout
# rotate-keys can return non-zero after the server accepted the operation;
# a later authoritative finished status is therefore allowed to succeed.
[[ "$rotate_rc" =~ ^[0-9]+$ ]] || return 1
started_at="$(_k3s_now_seconds)" || return 1
[[ "$started_at" =~ ^[0-9]+$ ]] || return 1
deadline=$((started_at + 600))
for ((attempt=1; attempt<=60; attempt++)); do
command_timeout="$(_k3s_timeout_before_deadline "$deadline")" || return 1
status_json="$(_k3s_read_encryption_status "$command_timeout")" || return 1
status_class="$(classify_encryption_status "$status_json")"
[[ "$status_class" != hash_mismatch && "$status_class" != invalid ]] || return 1
stage="$(jq -r '.stage' <<<"$status_json" 2>/dev/null)" || return 1
[[ "$stage" != start ]] || return 1
[[ "$stage" == reencrypt_finished ]] && return 0
[[ "$stage" == reencrypt_active ]] || return 1
(( attempt < 60 )) || break
next_poll=$((started_at + attempt * 10))
_k3s_sleep_until_poll "$deadline" "$next_poll" || return 1
done
return 1
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,243 @@
#!/usr/bin/env bash
# Direct-exec root helper for one staged A1 transaction.
_k3slra1h_command() {
"$@"
}
_k3slra1h_euid_is_root() {
(( EUID == 0 ))
}
_k3slra1h_output_name_is_safe() {
local output_name="${1-}"
(( $# == 1 )) || return 1
[[ "$output_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || return 1
[[ "$output_name" != _k3slra1h_* ]]
}
_k3slra1h_capture_metadata() {
local target="${1-}" output_name="${2-}" line=''
(( $# == 2 )) || return 1
_k3slra1h_output_name_is_safe "$output_name" || return 1
case "$output_name" in target|output_name|line|record) return 1 ;; esac
_k3slra1h_packet_line metadata "$target" line || return 1
printf -v "$output_name" '%s' "$line"
}
_k3slra1h_capture_canonical() {
local target="${1-}" output_name="${2-}" line=''
(( $# == 2 )) || return 1
_k3slra1h_output_name_is_safe "$output_name" || return 1
case "$output_name" in target|output_name|line) return 1 ;; esac
_k3slra1h_packet_line readlink "$target" line || return 1
[[ "$line" == "$target" ]] || return 1
printf -v "$output_name" '%s' "$line"
}
_k3slra1h_stderr_marker() {
local chunk='' read_rc=0
if IFS= read -r -d '' -n 1 chunk; then
read_rc=0
else
read_rc=$?
fi
if (( read_rc == 0 )); then
printf '\0STDERR\0'
while IFS= read -r -d '' -n 4096 chunk; do
:
done
else
if [[ -n "$chunk" ]]; then
printf '\0STDERR\0'
fi
fi
}
_k3slra1h_metadata_packet() {
local target="${1-}" rc=0
(( $# == 1 )) || return 1
if _k3slra1h_command /usr/bin/stat --format='%d:%i|%h|%s|%y|%F|%u:%g|%a' -- "$target" 2> >(_k3slra1h_stderr_marker); then
rc=0
else
rc=$?
fi
printf '\0RC=%s\0' "$rc"
}
_k3slra1h_readlink_packet() {
local target="${1-}" rc=0
(( $# == 1 )) || return 1
if _k3slra1h_command /usr/bin/readlink --canonicalize-existing -- "$target" 2> >(_k3slra1h_stderr_marker); then
rc=0
else
rc=$?
fi
printf '\0RC=%s\0' "$rc"
}
_k3slra1h_packet_value() {
local producer="${1-}" argument="${2-}" output_name="${3-}"
local -a records=()
(( $# == 3 )) || return 1
_k3slra1h_output_name_is_safe "$output_name" || return 1
case "$output_name" in producer|argument|output_name|records) return 1 ;; esac
case "$producer" in
metadata) mapfile -d '' -t records < <(_k3slra1h_metadata_packet "$argument") ;;
readlink) mapfile -d '' -t records < <(_k3slra1h_readlink_packet "$argument") ;;
*) return 1 ;;
esac
(( ${#records[@]} == 2 )) || return 1
[[ "${records[1]}" == RC=0 ]] || return 1
printf -v "$output_name" '%s' "${records[0]}"
}
_k3slra1h_packet_line() {
local producer="${1-}" argument="${2-}" output_name="${3-}" value=''
(( $# == 3 )) || return 1
_k3slra1h_output_name_is_safe "$output_name" || return 1
case "$output_name" in producer|argument|output_name|value) return 1 ;; esac
_k3slra1h_packet_value "$producer" "$argument" value || return 1
[[ "$value" == *$'\n' ]] || return 1
[[ "$value" != *$'\r'* ]] || return 1
value="${value%$'\n'}"
[[ -n "$value" ]] || return 1
[[ "$value" != *$'\n'* ]] || return 1
printf -v "$output_name" '%s' "$value"
}
_k3slra1h_output_names_are_distinct() {
local name
local -A seen=()
(( $# >= 1 )) || return 1
for name in "$@"; do
_k3slra1h_output_name_is_safe "$name" || return 1
[[ "${seen[$name]+set}" != set ]] || return 1
printf -v 'seen[$name]' '%s' 1
done
}
_k3slra1h_metadata_fields() {
local target="${1-}" out_identity="${2-}" out_nlink="${3-}" out_size="${4-}"
local out_mtime="${5-}" out_type="${6-}" out_owner="${7-}" out_mode="${8-}"
local metadata='' normalized_type=''
(( $# == 8 )) || return 1
_k3slra1h_output_names_are_distinct "$out_identity" "$out_nlink" "$out_size" "$out_mtime" "$out_type" "$out_owner" "$out_mode" || return 1
case "$out_identity|$out_nlink|$out_size|$out_mtime|$out_type|$out_owner|$out_mode" in
*'|target|'*|target\|*|*\|target|*'|out_identity|'*|out_identity\|*|*\|out_identity|*'|out_nlink|'*|out_nlink\|*|*\|out_nlink|*'|out_size|'*|out_size\|*|*\|out_size|*'|out_mtime|'*|out_mtime\|*|*\|out_mtime|*'|out_type|'*|out_type\|*|*\|out_type|*'|out_owner|'*|out_owner\|*|*\|out_owner|*'|out_mode|'*|out_mode\|*|*\|out_mode|*'|metadata|'*|metadata\|*|*\|metadata|*'|normalized_type|'*|normalized_type\|*|*\|normalized_type) return 1 ;;
esac
_k3slra1h_capture_metadata "$target" metadata || return 1
[[ "$metadata" =~ ^([0-9]+:[0-9]+)\|([0-9]+)\|([0-9]+)\|([^|]+)\|(directory|regular[[:space:]]file|regular[[:space:]]empty[[:space:]]file|symbolic[[:space:]]link)\|([0-9]+:[0-9]+)\|([0-9]+)$ ]] || return 1
normalized_type="${BASH_REMATCH[5]}"
if [[ "$normalized_type" == 'regular empty file' ]]; then
printf -v normalized_type '%s' 'regular file'
fi
printf -v "$out_identity" '%s' "${BASH_REMATCH[1]}"
printf -v "$out_nlink" '%s' "${BASH_REMATCH[2]}"
printf -v "$out_size" '%s' "${BASH_REMATCH[3]}"
printf -v "$out_mtime" '%s' "${BASH_REMATCH[4]}"
printf -v "$out_type" '%s' "$normalized_type"
printf -v "$out_owner" '%s' "${BASH_REMATCH[6]}"
printf -v "$out_mode" '%s' "${BASH_REMATCH[7]}"
}
_k3slra1h_regular_root_file_is_safe() {
local target="${1-}" expected_device="${2-}" canonical=''
local identity='' nlink='' size='' mtime='' type='' owner='' mode=''
(( $# == 2 )) || return 1
_k3slra1h_capture_canonical "$target" canonical || return 1
[[ "$canonical" == "$target" ]] || return 1
_k3slra1h_metadata_fields "$target" identity nlink size mtime type owner mode || return 1
[[ "${identity%%:*}" == "$expected_device" ]] || return 1
[[ "$nlink" == 1 ]] || return 1
[[ "$type" == 'regular file' ]] || return 1
[[ "$owner" == 0:0 ]] || return 1
[[ "$mode" == 600 ]]
}
_k3slra1h_root_directory_is_safe() {
local target="${1-}" expected_device="${2-}" canonical=''
local identity='' nlink='' size='' mtime='' type='' owner='' mode=''
(( $# == 2 )) || return 1
_k3slra1h_capture_canonical "$target" canonical || return 1
[[ "$canonical" == "$target" ]] || return 1
_k3slra1h_metadata_fields "$target" identity nlink size mtime type owner mode || return 1
[[ "${identity%%:*}" == "$expected_device" ]] || return 1
[[ "$type" == directory ]] || return 1
[[ "$owner" == 0:0 ]] || return 1
[[ "$mode" == 700 ]]
}
_k3slra1h_staged_root_is_safe() {
local capture_root="${1-}" canonical='' device=''
local identity='' nlink='' size='' mtime='' type='' owner='' mode=''
(( $# == 1 )) || return 1
[[ "$capture_root" == /* ]] || return 1
[[ "$capture_root" != */ ]] || return 1
[[ "$capture_root" != *//* ]] || return 1
_k3slra1h_capture_canonical "$capture_root" canonical || return 1
[[ "$canonical" == "$capture_root" ]] || return 1
_k3slra1h_metadata_fields "$capture_root" identity nlink size mtime type owner mode || return 1
[[ "$type" == directory ]] || return 1
[[ "$owner" == 0:0 ]] || return 1
[[ "$mode" == 700 ]] || return 1
device="${identity%%:*}"
[[ "$device" =~ ^[0-9]+$ ]] || return 1
_k3slra1h_root_directory_is_safe "${capture_root}/lib" "$device" || return 1
_k3slra1h_root_directory_is_safe "${capture_root}/libexec" "$device" || return 1
_k3slra1h_root_directory_is_safe "${capture_root}/contract" "$device" || return 1
_k3slra1h_regular_root_file_is_safe "${capture_root}/lib/k3s-local-recovery.sh" "$device" || return 1
_k3slra1h_regular_root_file_is_safe "${capture_root}/lib/k3s-local-recovery-a1.sh" "$device" || return 1
_k3slra1h_regular_root_file_is_safe "${capture_root}/contract/local-recovery.env" "$device" || return 1
_k3slra1h_regular_root_file_is_safe "${capture_root}/libexec/k3s-local-recovery-a1-check.sh" "$device"
}
_k3slra1h_load_staged_dependencies() {
local capture_root="${1-}" shared_library='' a1_library='' contract_file=''
(( $# == 1 )) || return 1
_k3slra1h_staged_root_is_safe "$capture_root" || return 1
shared_library="${capture_root}/lib/k3s-local-recovery.sh"
a1_library="${capture_root}/lib/k3s-local-recovery-a1.sh"
contract_file="${capture_root}/contract/local-recovery.env"
source "$shared_library" || return 1
source "$a1_library" || return 1
_k3slr_load_contract "$contract_file" || return 1
[[ "${K3SLR_MAPPING_NAME-}" == "$K3SLRA1_MAIN_MAPPING_NAME" ]] || return 1
[[ "${K3SLR_PROOF_MAPPING_NAME-}" == k3s-recovery-proof ]]
}
_k3slra1h_check_main_impl() {
local mount_root="${1-}" role="${2-}" capture_root="${3-}"
(( $# == 3 )) || return 1
_k3slra1h_euid_is_root || return 1
[[ "$-" != *x* ]] || return 1
case "$role" in
original-baseline) [[ "$mount_root" == "${capture_root}/original-baseline-mount" ]] || return 1 ;;
proof) [[ "$mount_root" == "${capture_root}/proof-mount" ]] || return 1 ;;
original-final) [[ "$mount_root" == "${capture_root}/original-final-mount" ]] || return 1 ;;
*) return 1 ;;
esac
_k3slra1h_load_staged_dependencies "$capture_root" || return 1
_k3slra1_verify_payload_root "$mount_root" "$role" "$capture_root"
}
k3slra1_check_main() {
_k3slra1h_check_main_impl "$@" >/dev/null 2>&1
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
_k3slra1h_direct_signal_rc=0
_k3slra1h_direct_rc=0
trap '_k3slra1h_direct_signal_rc=130; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=130; fi' INT
trap '_k3slra1h_direct_signal_rc=143; if (( _k3slra1h_direct_rc == 0 )); then _k3slra1h_direct_rc=143; fi' TERM
if k3slra1_check_main "$@"; then
_k3slra1h_direct_rc=0
else
_k3slra1h_direct_rc=$?
fi
if (( _k3slra1h_direct_signal_rc != 0 && _k3slra1h_direct_rc == 0 )); then
_k3slra1h_direct_rc=$_k3slra1h_direct_signal_rc
fi
exit "$_k3slra1h_direct_rc"
fi
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,947 @@
#!/usr/bin/python3
from __future__ import annotations
import hmac
import ctypes
import os
import re
import signal
import socket
import stat
import struct
import subprocess
import sys
import termios
from typing import Callable, Sequence
KEEPASSXC_CLI = "/usr/bin/keepassxc-cli"
PYTHON3 = "/usr/bin/python3"
SOCKET_NAME = "vault.sock"
BASELINE_NAME = "baseline.kdbx"
CANDIDATE_NAME = "candidate.kdbx"
ENTRY_GROUP = "/Platform/Observability/Slack"
ENTRY_TITLE = "Alertmanager webhook - desktop-infra-전체"
ENTRY_PATH = ENTRY_GROUP + "/" + ENTRY_TITLE
ENTRY_USERNAME = "desktop-infra"
ENTRY_URL = ""
MAX_WEBHOOK_BYTES = 4096
MAX_INPUT_BYTES = 4096
MAX_PROTOCOL_BYTES = 4096
MAX_CHILD_OUTPUT_BYTES = 64 * 1024
MAX_CHILD_STDERR_BYTES = 8192
DEFAULT_ACCEPT_TIMEOUT = 900.0
DEFAULT_IO_TIMEOUT = 15.0
DEFAULT_CHILD_TIMEOUT = 30.0
WEBHOOK_RE = re.compile(
rb"https://hooks[.]slack[.]com/services/"
rb"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+/[A-Za-z0-9_-]+"
)
class VaultContractError(RuntimeError):
pass
class VaultProtocolError(VaultContractError):
pass
class VaultSignal(BaseException):
def __init__(self, signum: int) -> None:
super().__init__(signum)
self.signum = signum
HANDLED_SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM)
def install_process_signal_handlers() -> None:
def raise_first_signal(signum: int, _frame: object) -> None:
for handled in HANDLED_SIGNALS:
signal.signal(handled, signal.SIG_IGN)
raise VaultSignal(signum)
for signum in HANDLED_SIGNALS:
signal.signal(signum, raise_first_signal)
class KeepassResult:
def __init__(self, returncode: int, stdout: bytes, stderr: bytes) -> None:
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
def wipe_mutable_buffer(buffer: bytearray) -> None:
for index in range(len(buffer)):
buffer[index] = 0
def _regular_metadata(fd: int, *, allow_unlinked: bool = False) -> tuple[int, ...]:
info = os.fstat(fd)
if not stat.S_ISREG(info.st_mode):
raise VaultContractError("expected regular file")
if info.st_uid != os.getuid():
raise VaultContractError("file owner mismatch")
if stat.S_IMODE(info.st_mode) != 0o600:
raise VaultContractError("file mode mismatch")
if info.st_nlink != 1 and not (allow_unlinked and info.st_nlink == 0):
raise VaultContractError("file link count mismatch")
return (
info.st_dev,
info.st_ino,
info.st_uid,
stat.S_IMODE(info.st_mode),
info.st_nlink,
info.st_size,
)
class RetainedWebhook:
def __init__(self, fd: int, metadata: tuple[int, ...], content: bytearray) -> None:
self._fd = fd
self._metadata = metadata
self._content = bytearray(content)
self._closed = False
@property
def fd(self) -> int:
if self._closed:
raise VaultContractError("webhook descriptor is closed")
return self._fd
def read_bytes(self) -> bytes:
if self._closed:
raise VaultContractError("webhook descriptor is closed")
before = _regular_metadata(self._fd)
if before != self._metadata:
raise VaultContractError("webhook metadata drift")
collected = bytearray()
confirmation = bytearray()
try:
for target in (collected, confirmation):
os.lseek(self._fd, 0, os.SEEK_SET)
while len(target) <= MAX_WEBHOOK_BYTES:
chunk = os.read(
self._fd,
min(1024, MAX_WEBHOOK_BYTES + 1 - len(target)),
)
if not chunk:
break
target.extend(chunk)
after = _regular_metadata(self._fd)
if after != before:
raise VaultContractError("webhook metadata drift")
if len(collected) > MAX_WEBHOOK_BYTES or len(confirmation) > MAX_WEBHOOK_BYTES:
raise VaultContractError("webhook content is oversized")
if not hmac.compare_digest(bytes(collected), bytes(confirmation)) or not hmac.compare_digest(
bytes(confirmation), self._content
):
raise VaultContractError("webhook content drift")
return bytes(collected)
finally:
wipe_mutable_buffer(collected)
wipe_mutable_buffer(confirmation)
def close(self) -> None:
if not self._closed:
try:
os.close(self._fd)
finally:
wipe_mutable_buffer(self._content)
self._closed = True
def _validate_webhook_content(content: bytes) -> None:
if not content or len(content) > MAX_WEBHOOK_BYTES:
raise VaultContractError("invalid webhook content")
if b"\r" in content or b"\n" in content or b"\0" in content:
raise VaultContractError("invalid webhook content")
if content.strip() != content:
raise VaultContractError("invalid webhook content")
if WEBHOOK_RE.fullmatch(content) is None:
raise VaultContractError("invalid webhook content")
def open_validated_webhook(path: str) -> RetainedWebhook:
if not os.path.isabs(path):
raise VaultContractError("absolute webhook path required")
flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC
fd = os.open(path, flags)
try:
before = _regular_metadata(fd)
content_buffer = bytearray()
confirmation_buffer = bytearray()
try:
for target in (content_buffer, confirmation_buffer):
if _regular_metadata(fd) != before:
raise VaultContractError("webhook metadata drift")
os.lseek(fd, 0, os.SEEK_SET)
while len(target) <= MAX_WEBHOOK_BYTES:
chunk = os.read(
fd,
min(1024, MAX_WEBHOOK_BYTES + 1 - len(target)),
)
if not chunk:
break
target.extend(chunk)
if _regular_metadata(fd) != before:
raise VaultContractError("webhook metadata drift")
after = _regular_metadata(fd)
if before != after:
raise VaultContractError("webhook metadata drift")
if not hmac.compare_digest(
bytes(content_buffer), bytes(confirmation_buffer)
):
raise VaultContractError("webhook content drift")
_validate_webhook_content(confirmation_buffer)
return RetainedWebhook(fd, before, confirmation_buffer)
finally:
wipe_mutable_buffer(content_buffer)
wipe_mutable_buffer(confirmation_buffer)
except BaseException:
os.close(fd)
raise
def _read_line_fd(fd: int, limit: int) -> bytearray:
value = bytearray()
try:
while len(value) <= limit:
chunk = os.read(fd, 1)
if not chunk:
raise VaultContractError("operator input ended early")
if chunk == b"\n":
return value
if chunk in {b"\r", b"\0"}:
raise VaultContractError("invalid operator input")
value.extend(chunk)
raise VaultContractError("operator input is oversized")
except BaseException:
wipe_mutable_buffer(value)
raise
def _validate_app_name(raw: bytes) -> str:
try:
app = raw.decode("utf-8", "strict")
except UnicodeDecodeError as exc:
raise VaultContractError("invalid app name") from exc
if not 1 <= len(app) <= 80 or app.strip() != app:
raise VaultContractError("invalid app name")
if any(character in ";=\\\r\n" or not character.isprintable() for character in app):
raise VaultContractError("invalid app name")
return app
def read_operator_secrets(
*, input_fd: int | None = None, output_fd: int | None = None
) -> tuple[str, bytearray]:
owned_fd = False
if input_fd is None:
input_fd = os.open("/dev/tty", os.O_RDWR | os.O_CLOEXEC)
output_fd = input_fd
owned_fd = True
if output_fd is None:
raise VaultContractError("operator output descriptor required")
master = bytearray()
old_termios = None
try:
os.write(output_fd, b"Slack app name: ")
raw_app = _read_line_fd(input_fd, MAX_INPUT_BYTES)
try:
app = _validate_app_name(bytes(raw_app))
finally:
wipe_mutable_buffer(raw_app)
if os.isatty(input_fd):
previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
try:
old_termios = termios.tcgetattr(input_fd)
new_termios = list(old_termios)
new_termios[3] &= ~termios.ECHO
termios.tcsetattr(input_fd, termios.TCSAFLUSH, new_termios)
os.write(output_fd, b"KeePassXC master password: ")
finally:
signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask)
else:
os.write(output_fd, b"KeePassXC master password: ")
master = _read_line_fd(input_fd, MAX_INPUT_BYTES)
if not master:
raise VaultContractError("empty master password")
if old_termios is not None:
previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
try:
termios.tcsetattr(input_fd, termios.TCSAFLUSH, old_termios)
old_termios = None
os.write(output_fd, b"\n")
finally:
signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask)
return app, master
except BaseException:
wipe_mutable_buffer(master)
raise
finally:
if old_termios is not None:
previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
try:
termios.tcsetattr(input_fd, termios.TCSAFLUSH, old_termios)
os.write(output_fd, b"\n")
finally:
signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask)
if owned_fd:
os.close(input_fd)
def _validate_child_stderr(stderr: bytes) -> None:
if len(stderr) > MAX_CHILD_STDERR_BYTES or b"\0" in stderr or b"\r" in stderr:
raise VaultContractError("invalid KeePass diagnostic")
try:
text = stderr.decode("utf-8", "strict")
except UnicodeDecodeError as exc:
raise VaultContractError("invalid KeePass diagnostic") from exc
if any(character not in "\t\n" and not character.isprintable() for character in text):
raise VaultContractError("invalid KeePass diagnostic")
def run_keepass(
arguments: Sequence[str],
master_buffer: bytearray,
private_home: str,
*,
webhook: RetainedWebhook | None = None,
timeout: float = DEFAULT_CHILD_TIMEOUT,
check: bool = True,
) -> KeepassResult:
if not arguments or any(not isinstance(value, str) or "\0" in value for value in arguments):
raise VaultContractError("invalid KeePass arguments")
if not os.path.isabs(private_home):
raise VaultContractError("private home must be absolute")
child_input = bytearray(master_buffer)
child_input.extend(b"\n")
if webhook is not None:
protected = bytearray(webhook.read_bytes())
try:
child_input.extend(protected)
child_input.extend(b"\n")
finally:
wipe_mutable_buffer(protected)
environment = {
"HOME": os.devnull,
"XDG_CONFIG_HOME": os.devnull,
"LC_ALL": "C.UTF-8",
"PATH": "/usr/bin:/bin",
}
expected_parent_pid = os.getpid()
def prepare_child() -> None:
os.umask(0o077)
arm_parent_death_signal(expected_parent_pid)
process = subprocess.Popen(
[KEEPASSXC_CLI, *arguments],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=environment,
close_fds=True,
start_new_session=True,
preexec_fn=prepare_child,
)
try:
try:
stdout, stderr = process.communicate(bytes(child_input), timeout=timeout)
except subprocess.TimeoutExpired as exc:
try:
os.killpg(process.pid, 15)
except ProcessLookupError:
pass
try:
process.communicate(timeout=1.0)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, 9)
except ProcessLookupError:
pass
process.communicate()
raise VaultContractError("KeePass child timeout") from exc
if len(stdout) > MAX_CHILD_OUTPUT_BYTES:
raise VaultContractError("KeePass output is oversized")
_validate_child_stderr(stderr)
result = KeepassResult(process.returncode, stdout, stderr)
if check and process.returncode != 0:
raise VaultContractError("KeePass command failed")
return result
finally:
wipe_mutable_buffer(child_input)
def arm_parent_death_signal(expected_parent_pid: int) -> None:
"""Ensure a vault death cannot orphan a secret-bearing KeePass child."""
if expected_parent_pid <= 1:
raise VaultContractError("invalid expected parent PID")
libc = ctypes.CDLL(None, use_errno=True)
prctl = libc.prctl
rc = prctl(1, 9, 0, 0, 0) # PR_SET_PDEATHSIG, SIGKILL
if rc != 0:
err = ctypes.get_errno()
raise OSError(err, os.strerror(err))
if os.getppid() != expected_parent_pid:
os.kill(os.getpid(), 9)
def expected_notes(app_name: str) -> str:
_validate_app_name(app_name.encode("utf-8"))
return (
"channel=desktop-infra-전체;"
f"app={app_name};"
"recovery=revoke-and-reissue-in-slack"
)
def _entry_count(
database: str,
master: bytearray,
private_home: str,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> tuple[int, set[str]]:
result = run_keepass(
["ls", "-q", "-R", "-f", database], master, private_home,
check=False, timeout=child_timeout,
)
if result.returncode != 0:
raise VaultContractError("KeePass database unlock failed")
try:
text = result.stdout.decode("utf-8", "strict")
except UnicodeDecodeError as exc:
raise VaultContractError("invalid KeePass listing") from exc
lines = text.splitlines()
exact = ENTRY_PATH.lstrip("/")
return sum(line == exact for line in lines), set(lines)
def _shown_attributes(
database: str,
master: bytearray,
private_home: str,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> tuple[str, str, str, str]:
result = run_keepass(
[
"show", "-q", "-a", "Title", "-a", "UserName", "-a", "URL",
"-a", "Notes", database, ENTRY_PATH,
],
master,
private_home,
timeout=child_timeout,
)
try:
text = result.stdout.decode("utf-8", "strict")
except UnicodeDecodeError as exc:
raise VaultContractError("invalid KeePass attributes") from exc
lines = text.split("\n")
if len(lines) != 5 or lines[-1] != "" or any("\r" in line for line in lines[:-1]):
raise VaultContractError("invalid KeePass attribute framing")
return lines[0], lines[1], lines[2], lines[3]
def protected_value_matches(producer_rc: int, shown: bytes, expected: bytes) -> bool:
comparison = bytearray(expected)
comparison.extend(b"\n")
try:
return producer_rc == 0 and hmac.compare_digest(shown, bytes(comparison))
finally:
wipe_mutable_buffer(comparison)
def _password_matches(
database: str,
master: bytearray,
webhook: RetainedWebhook,
private_home: str,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> bool:
result = run_keepass(
["show", "-q", "-s", "-a", "Password", database, ENTRY_PATH],
master,
private_home,
check=False,
timeout=child_timeout,
)
expected = bytearray(webhook.read_bytes())
shown = bytearray(result.stdout)
try:
return protected_value_matches(result.returncode, bytes(shown), bytes(expected))
finally:
wipe_mutable_buffer(expected)
wipe_mutable_buffer(shown)
def classify_database(
database: str,
master_buffer: bytearray,
webhook: RetainedWebhook,
app_name: str,
private_home: str,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> str:
try:
count, _ = _entry_count(database, master_buffer, private_home, child_timeout)
if count == 0:
return "absent"
if count > 1:
return "ambiguous"
expected = (ENTRY_TITLE, ENTRY_USERNAME, ENTRY_URL, expected_notes(app_name))
if _shown_attributes(database, master_buffer, private_home, child_timeout) != expected:
return "mismatch"
if not _password_matches(
database, master_buffer, webhook, private_home, child_timeout
):
return "mismatch"
return "exact-noop"
except (OSError, VaultContractError):
return "failed"
def _copy_private_database(source: str, destination: str) -> None:
source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
destination_fd = -1
try:
_regular_metadata(source_fd)
destination_fd = os.open(
destination,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
)
while True:
chunk = os.read(source_fd, 1024 * 1024)
if not chunk:
break
view = memoryview(chunk)
while view:
written = os.write(destination_fd, view)
view = view[written:]
os.fsync(destination_fd)
_regular_metadata(destination_fd)
finally:
if destination_fd >= 0:
os.close(destination_fd)
os.close(source_fd)
def _ensure_groups(
database: str,
master: bytearray,
private_home: str,
existing_lines: set[str],
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> None:
for group in ("/Platform", "/Platform/Observability", ENTRY_GROUP):
flattened = group.lstrip("/") + "/"
if flattened not in existing_lines:
run_keepass(
["mkdir", "-q", database, group], master, private_home,
timeout=child_timeout,
)
existing_lines.add(flattened)
def build_candidate(
baseline: str,
candidate: str,
mode: str,
master_buffer: bytearray,
webhook: RetainedWebhook,
app_name: str,
private_home: str,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> None:
if mode not in {"ADD", "UPDATE"}:
raise VaultContractError("invalid candidate mode")
classification = classify_database(
baseline, master_buffer, webhook, app_name, private_home, child_timeout
)
if (mode == "ADD" and classification != "absent") or (
mode == "UPDATE" and classification != "mismatch"
):
raise VaultContractError("candidate mode does not match classification")
_copy_private_database(baseline, candidate)
try:
_, existing_lines = _entry_count(
candidate, master_buffer, private_home, child_timeout
)
notes = expected_notes(app_name)
if mode == "ADD":
_ensure_groups(
candidate, master_buffer, private_home, existing_lines, child_timeout
)
arguments = [
"add", "-q", "-p", "-u", ENTRY_USERNAME, "--url", ENTRY_URL,
"--notes", notes, candidate, ENTRY_PATH,
]
else:
arguments = [
"edit", "-q", "-p", "-t", ENTRY_TITLE, "-u", ENTRY_USERNAME,
"--url", ENTRY_URL, "--notes", notes, candidate, ENTRY_PATH,
]
run_keepass(
arguments, master_buffer, private_home, webhook=webhook,
timeout=child_timeout,
)
if not verify_database(
candidate, master_buffer, webhook, app_name, private_home, child_timeout
):
raise VaultContractError("candidate verification failed")
except BaseException:
try:
os.unlink(candidate)
except OSError:
pass
raise
def verify_database(
database: str,
master_buffer: bytearray,
webhook: RetainedWebhook,
app_name: str,
private_home: str,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
) -> bool:
return classify_database(
database, master_buffer, webhook, app_name, private_home, child_timeout
) == "exact-noop"
def _attest_runtime_root(runtime_root: str) -> str:
if not os.path.isabs(runtime_root):
raise VaultContractError("absolute runtime root required")
canonical = os.path.realpath(runtime_root)
if canonical != os.path.abspath(runtime_root):
raise VaultContractError("runtime root must be canonical")
expected_parent = f"/run/user/{os.getuid()}"
if os.path.commonpath((canonical, expected_parent)) != expected_parent or canonical == expected_parent:
raise VaultContractError("runtime root is outside the private runtime")
fd = os.open(canonical, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
try:
info = os.fstat(fd)
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
raise VaultContractError("runtime root metadata mismatch")
finally:
os.close(fd)
mount_type = None
best_length = -1
with open("/proc/self/mountinfo", "r", encoding="utf-8") as stream:
for line in stream:
left, right = line.rstrip("\n").split(" - ", 1)
mount_point = left.split()[4].replace("\\040", " ")
if canonical == mount_point or canonical.startswith(mount_point.rstrip("/") + "/"):
if len(mount_point) > best_length:
mount_type = right.split()[0]
best_length = len(mount_point)
if mount_type != "tmpfs":
raise VaultContractError("runtime root is not on tmpfs")
return canonical
def _retained_runtime_path(runtime_fd: int) -> str:
info = os.fstat(runtime_fd)
if not stat.S_ISDIR(info.st_mode):
raise VaultContractError("retained runtime is not a directory")
proc_path = f"/proc/{os.getpid()}/fd/{runtime_fd}"
link_info = os.lstat(proc_path)
if not stat.S_ISLNK(link_info.st_mode):
raise VaultContractError("retained runtime proc anchor is unavailable")
resolved = os.stat(proc_path)
if (resolved.st_dev, resolved.st_ino) != (info.st_dev, info.st_ino):
raise VaultContractError("retained runtime proc anchor mismatch")
return proc_path
def _open_snapshot_below_runtime(
path: str, runtime_root: str, runtime_fd: int
) -> int:
if not os.path.isabs(path) or os.path.normpath(path) != path:
raise VaultProtocolError("snapshot must be canonical")
relative = os.path.relpath(path, runtime_root)
parts = relative.split(os.sep)
if not parts or any(
not part or part in {".", ".."} or os.sep in part for part in parts
):
raise VaultProtocolError("snapshot is outside runtime root")
current_fd = os.dup(runtime_fd)
try:
for part in parts[:-1]:
next_fd = os.open(
part,
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC,
dir_fd=current_fd,
)
os.close(current_fd)
current_fd = next_fd
directory_info = os.fstat(current_fd)
if (
directory_info.st_uid != os.getuid()
or stat.S_IMODE(directory_info.st_mode) != 0o700
):
raise VaultProtocolError("snapshot directory metadata mismatch")
return os.open(
parts[-1],
os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC,
dir_fd=current_fd,
)
finally:
os.close(current_fd)
def _attest_snapshot(
path: str, runtime_root: str, runtime_fd: int | None = None
) -> None:
if runtime_fd is not None:
fd = _open_attested_snapshot(path, runtime_root, runtime_fd)
try:
return
finally:
os.close(fd)
if not os.path.isabs(path) or os.path.realpath(path) != os.path.abspath(path):
raise VaultProtocolError("snapshot must be canonical")
if os.path.commonpath((path, runtime_root)) != runtime_root or path == runtime_root:
raise VaultProtocolError("snapshot is outside runtime root")
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
try:
_regular_metadata(fd)
finally:
os.close(fd)
def _open_attested_snapshot(path: str, runtime_root: str, runtime_fd: int) -> int:
fd = _open_snapshot_below_runtime(path, runtime_root, runtime_fd)
try:
_regular_metadata(fd)
return fd
except BaseException:
os.close(fd)
raise
def read_protocol_line(connection: socket.socket, *, timeout: float) -> str:
connection.settimeout(timeout)
data = bytearray()
try:
while len(data) <= MAX_PROTOCOL_BYTES:
chunk = connection.recv(MAX_PROTOCOL_BYTES + 2 - len(data))
if not chunk:
raise VaultProtocolError("partial protocol EOF")
data.extend(chunk)
if b"\n" in data:
break
if len(data) > MAX_PROTOCOL_BYTES + 1:
raise VaultProtocolError("protocol request is oversized")
if not data.endswith(b"\n") or data.count(b"\n") != 1:
raise VaultProtocolError("invalid protocol framing")
line = data[:-1]
if b"\r" in line or b"\0" in line:
raise VaultProtocolError("invalid protocol bytes")
try:
return line.decode("utf-8", "strict")
except UnicodeDecodeError as exc:
raise VaultProtocolError("invalid protocol UTF-8") from exc
finally:
wipe_mutable_buffer(data)
def parse_protocol_command(line: str) -> tuple[str, str | None]:
if line in {"PREPARE", "VERIFY_CANDIDATE", "SHUTDOWN"}:
return line, None
if line in {"BUILD ADD", "BUILD UPDATE"}:
return "BUILD", line.split(" ", 1)[1]
prefix = "VERIFY_COMMITTED "
if line.startswith(prefix):
path = line[len(prefix):]
if not path or " " in path or not os.path.isabs(path):
raise VaultProtocolError("invalid committed snapshot command")
return "VERIFY_COMMITTED", path
raise VaultProtocolError("unknown protocol command")
def peer_uid(connection: socket.socket) -> int:
credentials = connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)
_, uid, _ = struct.unpack("3i", credentials)
return uid
def _write_reply(connection: socket.socket, reply: str, timeout: float) -> None:
payload = reply.encode("ascii") + b"\n"
connection.settimeout(timeout)
connection.sendall(payload)
def serve_private_socket(
runtime_root: str,
webhook_file: str,
*,
input_fd: int | None = None,
output_fd: int | None = None,
accept_timeout: float = DEFAULT_ACCEPT_TIMEOUT,
io_timeout: float = DEFAULT_IO_TIMEOUT,
child_timeout: float = DEFAULT_CHILD_TIMEOUT,
wipe_hook: Callable[[bytearray], None] | None = None,
) -> None:
if child_timeout <= 0:
raise VaultContractError("invalid child timeout")
runtime = ""
runtime_fd = -1
runtime_access = ""
webhook = None
master = bytearray()
listener = None
socket_identity = None
try:
runtime = _attest_runtime_root(runtime_root)
runtime_fd = os.open(
runtime, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
)
runtime_access = _retained_runtime_path(runtime_fd)
webhook = open_validated_webhook(webhook_file)
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
socket_path = os.path.join(runtime_access, SOCKET_NAME)
private_home = os.devnull
baseline = os.path.join(runtime_access, BASELINE_NAME)
candidate = os.path.join(runtime_access, CANDIDATE_NAME)
app_name, master = read_operator_secrets(input_fd=input_fd, output_fd=output_fd)
previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS)
try:
listener.bind(socket_path)
os.chmod(SOCKET_NAME, 0o600, dir_fd=runtime_fd, follow_symlinks=False)
socket_info = os.stat(
SOCKET_NAME, dir_fd=runtime_fd, follow_symlinks=False
)
socket_identity = (socket_info.st_dev, socket_info.st_ino)
listener.listen(1)
finally:
signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask)
listener.settimeout(accept_timeout)
connection, _ = listener.accept()
listener.close()
if peer_uid(connection) != os.getuid():
connection.close()
raise VaultContractError("socket peer UID mismatch")
state = "initial"
classification = None
with connection:
while True:
try:
command, argument = parse_protocol_command(
read_protocol_line(connection, timeout=io_timeout)
)
if command == "PREPARE":
if state != "initial":
raise VaultProtocolError("PREPARE out of state")
classification = classify_database(
baseline, master, webhook, app_name, private_home,
child_timeout,
)
state = "prepared"
_write_reply(connection, classification, io_timeout)
elif command == "BUILD":
if state != "prepared" or (
argument == "ADD" and classification != "absent"
) or (
argument == "UPDATE" and classification != "mismatch"
):
raise VaultProtocolError("BUILD out of state")
try:
build_candidate(
baseline, candidate, argument or "", master,
webhook, app_name, private_home, child_timeout,
)
except (OSError, VaultContractError):
_write_reply(connection, "failed", io_timeout)
continue
state = "candidate-ready"
_write_reply(connection, "candidate-ready", io_timeout)
elif command == "VERIFY_CANDIDATE":
if state != "candidate-ready":
raise VaultProtocolError("VERIFY_CANDIDATE out of state")
verified = verify_database(
candidate, master, webhook, app_name, private_home,
child_timeout,
)
if verified:
state = "candidate-verified"
_write_reply(connection, "verified" if verified else "failed", io_timeout)
elif command == "VERIFY_COMMITTED":
if state != "candidate-verified" or argument is None:
raise VaultProtocolError("VERIFY_COMMITTED out of state")
snapshot_fd = _open_attested_snapshot(
argument, runtime, runtime_fd
)
try:
verified = verify_database(
f"/proc/{os.getpid()}/fd/{snapshot_fd}",
master,
webhook,
app_name,
private_home,
child_timeout,
)
finally:
os.close(snapshot_fd)
_write_reply(connection, "verified" if verified else "failed", io_timeout)
elif command == "SHUTDOWN":
_write_reply(connection, "stopped", io_timeout)
break
except VaultProtocolError:
try:
_write_reply(connection, "failed", io_timeout)
except OSError:
pass
raise
finally:
try:
if listener is not None:
listener.close()
finally:
try:
if webhook is not None:
webhook.close()
finally:
try:
wipe_mutable_buffer(master)
if wipe_hook is not None:
wipe_hook(master)
finally:
try:
if socket_identity is not None:
try:
current = os.stat(
SOCKET_NAME, dir_fd=runtime_fd,
follow_symlinks=False,
)
if (current.st_dev, current.st_ino) == socket_identity:
os.unlink(SOCKET_NAME, dir_fd=runtime_fd)
except FileNotFoundError:
pass
finally:
if runtime_fd >= 0:
os.close(runtime_fd)
def _main(argv: Sequence[str]) -> int:
if len(argv) != 5 or argv[0] != "--serve" or argv[1] != "--runtime-root" or argv[3] != "--webhook-file":
return 2
runtime_root, webhook_file = argv[2], argv[4]
if not os.path.isabs(runtime_root) or not os.path.isabs(webhook_file):
return 2
install_process_signal_handlers()
try:
serve_private_socket(runtime_root, webhook_file)
return 0
except VaultSignal as exc:
return 128 + exc.signum
except (OSError, VaultContractError):
os.write(2, b"vault helper failed\n")
return 1
if __name__ == "__main__":
raise SystemExit(_main(sys.argv[1:]))
+561
View File
@@ -0,0 +1,561 @@
# Phase 1·2 검증
`render-phase1.sh`와 `render-phase2.sh`는 클러스터를 변경하지 않고 각 단계의
구성을 렌더링한 뒤 핵심 불변 조건을 확인한다. Phase 2 검증은 먼저 Phase 1
검증을 재실행하므로 두 단계의 경계도 함께 확인한다.
```sh
bash scripts/validate/render-phase1.sh
bash scripts/validate/render-phase2.sh
```
`helm`이 `PATH`에 없거나 검증용 바이너리를 별도로 내려받았다면 절대 경로를
지정할 수 있다.
```sh
PLATFORM_HELM_BIN=/tmp/helm-v3.19.4/helm \
bash scripts/validate/render-phase2.sh
```
`PLATFORM_HELM_BIN`은 실행 가능한 일반 파일의 절대 경로여야 하며, 지정하지
않으면 `PATH`에서 `helm`을 찾는다. 어느 경로를 사용하든 버전은 정확히
`v3.19.4`여야 한다.
`render-phase1.sh --verified-output-dir`는 일반 사용자가 직접 호출하는 출력 옵션이
아니라 `apply-phase1-gitea.sh`와 `apply-gitea-oidc.sh`가 함께 사용하는 내부
handoff 전용이다. 기존의 비어 있는 `/tmp/platform-phase1-apply.*` 디렉터리만
허용하며, 경로가 심볼릭 링크이거나 현재 사용자 소유가 아니거나 mode `0700`이
아니면 중단한다. 검증을 모두 통과한 경우에만 다음 여섯 manifest를 mode
`0600`으로 복사하고 원본과 `cmp`로 다시 비교한다.
- `namespaces.yaml`
- `ssd-local-pv.yaml`
- `cnpg-operator.yaml`
- `platform-postgres.yaml`
- `gitea.yaml`
- `gitea-oidc.yaml`
`apply-phase1-gitea.sh`는 여섯 산출물의 SHA-256을 메모리에 고정하고 확인 후,
실제 적용 대상인 `namespaces.yaml`부터 `gitea.yaml`까지 다섯 manifest를 각각
apply하기 직전에 다시 검사한다. `gitea-oidc.yaml`은 적용하지 않는다.
`apply-gitea-oidc.sh`는 같은 handoff의 파일 수·권한·소유권을 확인하되,
`gitea-oidc.yaml`의 SHA-256만 적용 대상으로 고정하고 재확인한 뒤 그 하나만
적용한다. 두 경로 모두 검증 뒤 Chart를 다시 내려받거나 Kustomize를 다시 실행하지
않는다. 취소·오류·`INT`·`TERM` 종료 시 각 적용 스크립트가 자신의 handoff
디렉터리 전체를 삭제한다.
공통 검증 기준은 다음과 같다.
- 로컬 `kubectl`의 Kustomize가 Argo CD 3.4.2와 같은 `v5.8.1`인지 확인
- Helm이 Argo CD 3.4.2와 같은 `v3.19.4`인지 확인
- `sha256sum`과 `tar`가 설치되어 있는지 확인
- `helm pull`로 고정 버전 패키지를 공식 저장소에서 임시 디렉터리로 받은 뒤
SHA-256으로 바이트 단위 무결성 확인
- CloudNativePG Chart `0.29.0`의 SHA-256이
`668e065ff53508d58238788fd35b355a925060843629a951df0e6a9362e6d32f`인지 확인
- Gitea Chart `12.7.0`의 SHA-256이
`5881ef9c59400bee2d5547e77c4cd0efb925143c2f5d93fb4f38446db76b0167`인지 확인
- Helm Chart는 반드시 `--enable-helm`과 기본 `LoadRestrictionsRootOnly`로 렌더링
Phase 1은 CloudNativePG CRD, Gitea용 `Cluster`·`DatabaseRole`·`Database`, Gitea
PVC·Ingress를 확인한다. Ingress의 `git.learn.hyeonworks.com` host, 내부 TLS 부재,
외부 HTTPS `ROOT_URL`, 애플리케이션 `NodePort`·`LoadBalancer`와 Gitea SSH Service
부재도 검증한다. Phase 1 PostgreSQL 빌드 루트에는 Keycloak 리소스·Secret 참조·
namespace ingress 허용이 없어야 한다.
Gitea는 두 프로필을 모두 렌더하되 적용 경계를 분리해 검증한다.
- `services/gitea` → `gitea.yaml`: 신규 설치용 baseline. OIDC Secret·ID host
alias·Keycloak egress·브랜딩이 없어야 하고 자체 가입을 닫은 로컬 비상 로그인만
유지한다.
- `services/gitea/profiles/oidc` → `gitea-oidc.yaml`: 공통
`values/baseline.yaml`에 `values/oidc.yaml`을 병합한다. 정확한 OIDC Secret 참조,
public issuer, 외부 인증 전용 가입, CoreDNS 기반 public issuer 접근, `/32` egress와 해시된 브랜딩
ConfigMap·read-only mount를 확인한다.
공유 Cluster의 `pg_hba`도 네 규칙의 순서와 값을 검증한다. `gitea` Role은 `gitea`
DB, `keycloak` Role은 `keycloak` DB에만 SCRAM으로 접속을 허용하고, 각 Role이 다른
DB에 접속하는 경우에는 바로 다음 규칙에서 `reject`해야 한다.
Phase 2는 다음을 추가로 검증한다.
- 공식 Keycloak `26.7.0` 원격 리소스와 Keycloak 단일 인스턴스·외부 HTTPS
hostname·내부 HTTP·Traefik Ingress
- Keycloak PostgreSQL 증분 루트에 `DatabaseRole`·`Database`·추가형
NetworkPolicy만 있고 `Cluster` 사본은 없는지 확인
- AIStor Operator Chart `5.10.0` SHA-256 `e5534f5ae4f6f12a3528a8cda5954d73280cba1a8e979e1628fbf3aac76babd1`
- AIStor ObjectStore Chart `1.0.16` SHA-256 `50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e`
- AIStor XFS Local PV의 `Retain`·`WaitForFirstConsumer`·`900Gi`·node affinity
- ObjectStore 서버 1개·볼륨 1개, PVC 보호, S3·Console `ClusterIP`, 외부
Ingress·NodePort·LoadBalancer 부재
- Phase 2 소스와 렌더 결과의 자격 증명·라이선스 `Secret` 부재
Secret 검사는 관리 대상 YAML 전체에 적용하되, 렌더 과정에서 생성되고 Git에서
무시되는 업스트림 Chart 캐시인 `**/.helm/**`과 `**/charts/**`는 제외한다.
스크립트는 `/tmp/platform-phase1-render.*` 아래에만 중간 산출물을 만들고 종료 시
삭제한다. 따라서 `helm pull`로 받은 임시 `.tgz`도 종료 시 제거된다. 각 Helm
렌더 직전에는 다이제스트를 통과한 패키지만 아래의 정확한 생성 캐시 버전
디렉터리에 압축 해제한다.
- `infrastructure/controllers/cloudnative-pg/.helm/charts/cloudnative-pg-0.29.0/cloudnative-pg/Chart.yaml`
- `services/gitea/profiles/oidc/.helm/charts/gitea-12.7.0/gitea/Chart.yaml`
Phase 2도 같은 방식으로 AIStor Operator와 ObjectStore의 정확한 버전 캐시만
일시 생성한다. 스크립트는 해당 버전 디렉터리가 이미 존재하거나 캐시 상위 디렉터리가
심볼릭 링크면 이를 임의 삭제·덮어쓰지 않고 중단한다. 패키지 다이제스트가 일치한 뒤에만
정확한 디렉터리를 생성하며, 자신이 생성한 버전 디렉터리는 `EXIT` 종료 정리에서
제거한다. `.helm/charts`와 `.helm`도 이번 실행에서 생성했고 비어 있을 때만
`rmdir`로 정리한다. 기존의 다른 캐시는 삭제하지 않는다.
정상 종료와 처리되는 `INT`·`TERM`에서는 생성한 압축 해제 캐시와 `/tmp` 렌더
산출물이 제거된다. 강제 종료 등으로 정확한 캐시가 남으면 다음 실행은 안전하게
거부하므로 운영자가 경로와 내용을 확인한 뒤에만 수동 정리한다. 패키지가 없거나
다이제스트가 다르거나 압축 해제 후 `Chart.yaml`이 없으면 렌더 전에 즉시 실패한다.
실제 적용, namespace 생성, Secret 변경, Nginx 변경은 수행하지 않는다.
내부 handoff 모드를 사용해도 검증 스크립트 자신의 Chart package, generated cache와
작업용 render 디렉터리는 동일하게 정리된다. 보존되는 것은 적용 스크립트가 만든
제한된 임시 디렉터리의 검증 완료 manifest 여섯 개뿐이며, 적용 스크립트 종료 시
함께 제거된다.
## 관측성 코어 정적 render 검증
`render-observability-core.sh`는 Helm `v3.19.4`, Kustomize `v5.8.1`, Kubernetes
render target `1.36.2`와 계획에 기록된 여섯 chart archive/name/version/appVersion,
열두 image digest를 고정한다. 기본 실행은 정적 render만 수행하며 Kubernetes
resource를 적용하지 않는다. live 관측성 mutation gate가 닫힌 동안에는 fixture 시험만
실행한다.
```sh
bash -n scripts/validate/render-observability-core.sh
bash -n scripts/validate/test-render-observability-core.sh
bash scripts/validate/test-render-observability-core.sh
```
fixture 시험은 chart를 다운로드하지 않고 source된 assertion 함수에 변형 YAML을
전달한다. 성공 표식은 `OBSERVABILITY CORE RENDER ASSERTION TEST PASS`다. 전체 renderer는
Task 2 이후 child root가 존재할 때 다음과 같이 사용한다.
```sh
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/validate/render-observability-core.sh
```
내부 `--verified-output-dir`는 비어 있는 current-user 소유 mode `0700`의
`/tmp/platform-observability-core-apply.*`만 허용한다. 모든 검증과 child/aggregate
canonical resource equivalence가 끝난 뒤에만 manifest 열네 개와
`resource-index.tsv`를 regular mode `0600` 파일로 넘긴다. Grafana와 Blackbox child
manifest도 각각 `grafana.yaml`, `blackbox.yaml`로 포함한다. handoff directory의
device/inode를 고정하고 쓰기 직전에 identity와 empty 상태를 다시 확인하며, 각 파일은
완성된 mode `0600` staging inode를 exact name에 no-clobber link하는 방식으로 공개한다.
최종 entry set은 열일곱 exact name뿐이어야 한다. 각 chart cache는 모든 render root의
physical lineage를 먼저 확인한 뒤 directory FD 기준 exclusive mkdir로 이번 실행의 정확한
version 경로만 만들고, 기존 경로를 덮어쓰지 않으며, 종료 시 실제로 생성·추적한 경로만
제거한다. 실행 전부터 존재한 빈 `.helm`이나 `.helm/charts`는 제거하지 않는다.
cache cleanup은 tracked directory를 random no-replace quarantine name으로 옮긴 뒤 identity를
다시 확인하고, 모든 child를 directory FD 기준으로 같은 방식으로 격리해 제거한다. cleanup
도중 원래 path에 replacement가 생겨도 이를 순회하거나 삭제하지 않으며 identity mismatch나
unexpected entry는 성공 종료로 숨기지 않는다.
Alertmanager Slack source 계약은 Secret payload를 읽지 않는 focused validator로
검사한다. 기본 모드는 source와 Kustomize/KPS wiring만 검사하며, 선택적 server dry-run도
두 non-Secret source만 admission에 제출하고 live object를 변경하지 않는다.
```sh
bash scripts/validate/test-observability-alerting.sh
bash scripts/validate/test-observability-alerting.sh --server-dry-run
```
`render-observability-access.sh`는 core renderer의 검증된 Grafana·Blackbox·targets·
dashboard·rule·Alertmanager bytes와 private DNS를 access 단계의 일곱 manifest로
분리한다. rules-alerts와 complete mode는 두 metric inventory phase가 있는 direct
`/tmp/platform-observability-metrics.XXXXXX` mode `0700` 디렉터리만 허용한다. 두
inventory directory는 mode `0700`, JSON/checksum은 owner mode `0600`, link count 1과
exact checksum이어야 한다. 게시 전 모든 destination absence와 root inode를 고정하고,
모든 source를 mode `0600` staging inode에 동기화한 뒤 no-clobber link한다. 한 파일이라도
실패하면 이번 실행이 생성한 파일을 전부 제거해 부분 handoff를 남기지 않는다.
```sh
bash -n scripts/validate/render-observability-access.sh
bash scripts/validate/test-render-observability-access.sh
bash scripts/validate/render-observability-access.sh \
--component complete \
--verified-output-dir /tmp/platform-observability-metrics.XXXXXX
```
출력에는 Secret kind나 credential payload가 없으며, 이미 게시된 output root를 재사용하면
no-clobber로 즉시 거부한다. live 적용은 별도 apply transaction과 Slack Secret,
HTTPS runbook URL, source-proof/acceptance evidence가 모두 준비된 뒤에만 수행한다.
## Slack webhook KeePass 복구 사본
`backup-slack-webhook-recovery.sh`는 Alertmanager Slack webhook의 재해 복구용 암호화
사본을 같은 host의 별도 내장 Windows SSD에 있는 기존 KDBX에 보관한다. 이 사본은 일반
K3s 재시작이나 host reboot에 필요하지 않으며, datastore·Secret·bootstrap state를 잃었을
때를 위한 것이다. 기본 no-argument 실행은 고정 contract만 출력하고 SSD, KDBX, webhook,
sudo에 접근하지 않는다. 지원되는 public interface는 다음 두 형식뿐이다.
```sh
bash scripts/bootstrap/backup-slack-webhook-recovery.sh
bash scripts/bootstrap/backup-slack-webhook-recovery.sh \
--execute \
--slack-webhook-file /home/donghyeon/.secrets/alertmanager/slack-webhook
```
execute는 전용 TTY에서 Slack app name과 KeePassXC master password를 받고, write 또는
mismatch update 전에 정확한 확인 문자열을 요구한다. webhook payload, master password,
그 hash·encoding·size·URL component, KDBX protected output은 argv, environment, stdout,
log 또는 plaintext 파일에 출력하거나 기록하지 않는다.
검증된 read-only exact match는 `SLACK_KEEPASS_RECOVERY=NOOP`, 검증된 변경은
`SLACK_KEEPASS_RECOVERY=COMMITTED`를 출력한다. 두 성공 분기 모두 SSD source가 실제로
unmount된 뒤 `WINDOWS_SSD_UNMOUNTED=PASS`와
`OFF_HOST_RECOVERY_SATISFIED=NO`를 출력한다. 변경 분기는 durable non-clobbering
pre-change backup을 만든 뒤 `KDBX_PRECHANGE_BACKUP=CREATED`도 출력한다. rename 시도 뒤
응답 손실, post-commit 검증 실패, 또는 cleanup/unmount 불명은 재시도하지 않고
`SLACK_KEEPASS_RECOVERY=MANUAL_RECOVERY_REQUIRED`로 중단하며 main과 backup을 보존한다.
private tmpfs work root, socket, helper process와 mount source의 제거·부재 증명 전에는 성공
token을 출력하지 않는다.
이 same-host encrypted copy는 off-host escrow가 아니며 Phase 4의 off-host recovery gate를
충족하거나 승인하지 않는다.
## 관측성 Secret create-only bootstrap
`create-observability-secrets.sh`는 기본 실행에서 고정 contract만 출력하며 cluster, sudo,
payload file에 접근하지 않는다. execute는 선택한 `grafana-admin`과
`alertmanager-slack-webhook`의 exact `Opaque` key set과 기존 payload 일치를 확인하며,
기존 payload가 다르면 rotation 없이 중단한다. 둘을 함께 선택하면 한 create-only
transaction으로 처리한다.
```sh
bash scripts/bootstrap/create-observability-secrets.sh
bash scripts/bootstrap/create-observability-secrets.sh \
--execute --grafana-admin \
--grafana-admin-user-file /absolute/current-user-0600/admin-user \
--grafana-admin-password-file /absolute/current-user-0600/admin-password
bash scripts/bootstrap/create-observability-secrets.sh \
--execute --slack-webhook \
--slack-webhook-file /absolute/current-user-0600/slack-webhook
bash scripts/bootstrap/create-observability-secrets.sh \
--check-grafana-recovery-evidence
bash scripts/bootstrap/create-observability-secrets.sh \
--check-slack-recovery-evidence
```
execute는 confirmation 전과 create 직전에 k3s encryption/restore validator를 각각 새
process로 실행한다. 입력은 no-follow private snapshot으로만 create에 전달하고 값이나 hash를
출력하지 않는다. transaction 실패 시 이번 호출이 생성하고 UID를 캡처한 Secret만 Kubernetes
API UID precondition으로 삭제한다. create 결과 또는 ownership이 모호하면 해당 object는
삭제하지 않고 `MANUAL_RECOVERY_REQUIRED=YES`로 중단한다. recovery marker는 root-owned
mode `0600` 네 필드만 허용하며 standalone check는 current-user kube context와 30일 age를
확인하고 marker stat/read에만 좁은 sudo를 사용한다.
fixture 회귀는 실제 production 함수와 private Unix-socket API precondition 경계를 실행하되
live Secret이나 실제 payload에는 접근하지 않는다.
```sh
bash -n scripts/bootstrap/create-observability-secrets.sh
bash scripts/validate/test-create-observability-secrets.sh
```
## Grafana–Keycloak OIDC bootstrap 검증
`configure-keycloak-grafana-oidc.sh`는 인자 없이 실행하면 고정 client/group/mapper/Secret
계획만 출력하며 Kubernetes, Keycloak, sudo와 payload에 접근하지 않는다. execute는 정확히
`default` context, `https://127.0.0.1:6443` API, Ready
`donghyeon-system-product-name` node와 bounded authorization을 다시 고정한 뒤 loopback-only
Keycloak Admin API와 private Unix-socket Kubernetes Secret API만 사용한다.
```sh
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh --execute
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \
--execute --admin OBS_ADMIN_USER --viewer OBS_VIEWER_USER
bash scripts/bootstrap/configure-keycloak-grafana-oidc.sh \
--check-recovery-evidence
```
`--admin`과 `--viewer`는 정확히 한 명의 기존 Keycloak user만 각 고정 group에 추가한다.
옵션이 없으면 임의 사용자를 만들거나 membership을 바꾸지 않는다. 기존 exact client,
full-path `groups` mapper, 두 group과 matching `observability/grafana-keycloak-oidc` Secret은
credential rotation과 client/Secret rewrite 없이 재사용한다. declarative update가 필요한
기존 client는 현재 credential을 private file로 보존해 PUT과 rollback에 명시적으로 넣고,
전후 client-secret 관계를 payload 출력 없이 비교한다.
execute는 confirmation 전과 첫 mutation 직전에 encryption/restore validator를 각각 fresh
`/usr/bin/env -i` process로 실행한다. `APPLY default`와
`RECOVERY KEYCLOAK default` 뒤에만 mutation을 시작한다. 실패나 `HUP`/`INT`/`TERM`은 exact
snapshot과 transaction ledger로 이번 실행이 추가한 membership, mapper/client/group/Secret만
역순 복구한다. 생성 Secret 삭제는 UID와 resourceVersion precondition을 모두 사용한다.
ownership 또는 response 결과가 모호하면 대상 삭제를 시도하지 않고
`MANUAL_RECOVERY_REQUIRED=YES`로 중단한다.
성공 시 `/etc/hyeonworks/platform/recovery-evidence/keycloak.env`에는 schema, context,
resource, checked-at UTC 네 non-secret field만 `root:root 0600`으로 atomic 기록한다.
standalone check는 current-user kube context와 exact marker schema, no-follow/link metadata와
30일 age를 검사하고 marker 접근에만 좁은 sudo를 사용한다.
focused fixture는 stateful fake Keycloak Admin API와 raw Kubernetes Secret API를 통해 실제
production state machine을 실행한다. create/update/no-op, duplicate cardinality, membership
ownership, response loss, conflict, timeout, rollback과 signals를 검증하되 live cluster, sudo,
실제 Secret을 읽거나 변경하지 않는다.
```sh
bash -n scripts/bootstrap/configure-keycloak-grafana-oidc.sh
bash -n scripts/validate/test-configure-keycloak-grafana-oidc.sh
bash scripts/validate/test-configure-keycloak-grafana-oidc.sh
```
## Phase 3 비공개 관리 UI 검증
`render-admin-services.sh`는 pgAdmin OCI Chart `9.16.0`과 AIStor ObjectStore
Chart `1.0.16` archive SHA-256을 다시 검증한다. pgAdmin main과 두 init image가
모두 `9.16` amd64 digest로 바뀌었는지, 렌더 결과에 Secret kind가 없는지,
Recreate·2Gi Retain Local PV·두 admin Ingress와 namespace 간 NetworkPolicy가
유지되는지 확인한다.
```sh
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/validate/render-admin-services.sh
```
실제 적용 뒤 `admin-ui-smoke.sh --execute --run-s3`로 공개 admin DNS 부재,
LAN·Tailscale·Pod DNS, 인증서 SAN, 외부 403, loopback NodePort, pgAdmin 비밀번호
저장 차단과 AIStor S3 회귀를 확인한다. 허용·비허용 Keycloak 사용자 브라우저
시험은 수동 수용 시험으로 남긴다.
임시 network smoke Pod는 BusyBox 1.37.0의 amd64 manifest digest를 고정하고
non-root UID, RuntimeDefault seccomp, 전체 capability drop,
allowPrivilegeEscalation=false, read-only root filesystem과 service account token
미마운트를 적용한다. `platform-admin`의 Pod Security restricted 정책을
우회하지 않는다.
## 로컬 K3s recovery 저장소 읽기 전용 검증
`k3s-local-recovery.sh`는 recovery SSD의 고정 hardware identity, SMART/NTFS 상태와
mount → container → loop → LUKS2 → ext4 lineage를 읽기 전용으로 판정한다. 실행
전에 같은 terminal에서 sudo credential을 미리 준비해야 하며, validator 자체가 암호를
요구하거나 package를 설치하지 않는다.
```sh
sudo -v
bash scripts/validate/k3s-local-recovery.sh --expect-device-ready
bash scripts/validate/k3s-local-recovery.sh --expect-closed
bash scripts/validate/k3s-local-recovery.sh --expect-open
bash scripts/validate/k3s-local-recovery.sh --expect-open --check-latest-bundle
```
기대 상태는 정확히 하나만 지정한다. `--expect-device-ready`와 `--expect-closed`는 두
platform mount, 운영·proof mapping과 container 관련 loop가 모두 없는 상태만 허용한다.
device-ready 검사는 canonical recovery partition이 다른 mountpoint에도 source로 쓰이지
않고 `ntfs-3g.probe --readwrite`가 성공해야 한다. `--expect-open`은 그 partition이 승인된
outer mount 한 곳에만 연결되고, `ntfs3`의 `nodev,nosuid,noexec`와 안전한
`umask=077` 또는 동등한 `dmask=0077,fmask=0077`을 가져야 한다. 또한 canonical parent
chain과 container inode, loop backing inode/device, zero offset/size limit, LUKS2 mapper와
두 mount의 major:minor가 하나의 lineage여야 하며 마지막에 같은 snapshot을 다시 확인한다.
완전 할당된 고정 크기 container, ext4 label과 `root:root 0700` inner root도 모두
필수다. dirty/hibernated NTFS를 고치거나 force mount하는 동작은 없다. root EUID나
상속된 xtrace 상태에서는 workspace path/config를 읽기 전에 거부한다.
`--check-latest-bundle`은 open 상태에서만 쓸 수 있다. inner root의 root-owned mode
`0600` `.latest-post-bundle.env`는 다음 세 key를 정확히 한 번씩 가져야 한다.
```text
schema=k3slr-latest-post-bundle-v1
relative_path=k3s-secrets-encryption-YYYYMMDDTHHMMSSZ/post
directory_identity=DEVICE:INODE
```
validator는 이 제한된 relative path와 directory identity를 전후로 재확인하고,
root-owned post `bundle.env` schema와 안전한 relative-name manifest를 검사한다. manifest의
각 parent와 leaf는 symlink가 아닌 pinned directory 내부 object여야 하며, regular-file
device/inode를 hash 전후에 확인하면서 각 target을 개별 `sha256sum`으로 검증한다. 성공
출력은 다음 네 분류뿐이며 stable ID, UUID, serial, WWN, loop/KDBX/bundle path와 payload를
표시하지 않는다.
```text
Recovery device: match
Recovery state: device_ready|closed|open
Lineage: match
Latest bundle: verified|not_checked
```
fixture 회귀는 system command 경계만 argv log를 남기는 fake로 바꾸고 실제 collector와
parser를 호출한다. latest verifier는 임시 일반 directory의 valid, malformed, duplicate,
symlink escape, hash mismatch bundle을 직접 검사하며 live mount나 block device를 만들지
않는다.
```sh
bash scripts/validate/test-k3s-local-recovery.sh
```
### KeePassXC → cryptsetup anonymous-pipe feasibility
`k3s-local-recovery-feasibility.sh`는 recovery 저장소를 만들기 전에 고정 package와
KeePassXC CLI의 synthetic KDBX 동작을 확인한다. 요구 version은 `keepassxc`
`2.7.6+dfsg.1-1build3`, `cryptsetup-bin` `2:2.7.0-1ubuntu4.2`이며 executable도
root-owned regular non-symlink, group/other non-writable 조건을 만족해야 한다.
```sh
bash scripts/validate/k3s-local-recovery-feasibility.sh
sudo -v
bash scripts/validate/k3s-local-recovery-feasibility.sh --execute
```
기본 실행은 package/executable prerequisite만 판정한다. `--execute`는 interactive stdin과
같은 terminal의 cached sudo credential을 요구하며, inherited xtrace나 root EUID에서는
workspace library를 읽기 전에 거부한다. 이 mode가 만드는 것은 current-user `0700`
`/tmp/k3slr-feasibility.*` 아래 synthetic KDBX와 attachment round-trip fixture뿐이다.
실제 KeePass DB, LUKS file, loop device, mapping, mount와 K3s 상태는 읽거나 변경하지 않는다.
성공·실패·`INT`·`TERM` 모두 exact fixture directory를 정리한다.
실제 recovery open/format API는 고정 entry의 Password attribute를 shell 변수, command
substitution, argv, environment, file, log 또는 `tee`에 넣지 않는다. producer stdout은
anonymous kernel pipe FD로만 전달된다. 검증된 caller TTY FD를 producer stdin에 명시적으로
연결하고 producer 내부에서도 TTY를 재검증한다. producer output은 streaming `od`와 bounded
`mawk` validator가 전부 drain하며 non-shell process 안에 최대 40 byte만 유지한다. producer,
EOF, exact 40자+LF/class 검증과 KDBX 전체 parent lineage 재검증이 모두 성공한 뒤에만 정확한
41 byte가 parent pipe로 전달되고 다음 두 고정 consumer 중 하나가 읽기 시작한다.
producer supervisor는 시작 직후 STOP handshake를 수행하고 caller와 다른 실제 PGID를 trusted
`ps`로 확인한 뒤에만 pipeline을 시작한다. 이때 `/proc/PID/stat`의 start time, direct parent,
observed PGID를 함께 고정하며, negative group signal 직전마다 동일 identity와 caller PGID를
다시 확인한다. `INT`/`TERM` trap은 pending state만 기록하고 lifecycle checkpoint가 TTY/pipe
FD를 닫은 뒤 bounded TERM, 필요 시 KILL, direct-child wait/reap과 group 소멸 확인을 수행한다.
`/proc` reader는 정확히 LF 하나로 끝나는 단일 record와 EOF를 요구하고 CR, 추가 record,
non-canonical PID/PPID/PGID/starttime을 거부한다. PID publication 직후 STOP query보다 먼저
direct-child identity를 pin하며, 이 최초 pin 자체가 실패하면 worker를 CONT하지 않고 freshly
published direct PID에만 positive KILL한 뒤 wait한다.
same-caller/query-failure에서는 group signal 없이 STOP된 exact direct child만 positive
TERM/CONT/KILL한다. wait/reap과 PID/PGID/identity clear는 pending signal이 stale group state를
소비할 수 없는 하나의 transition으로 처리한다. coprocess PID가 PGID라고 가정하지 않는다.
supervisor는 initial STOP 전에 child-side INT/TERM cancellation deferral을 설치한다. cleanup
TERM을 받으면 worker를 시작하거나 종료하지 않고 ownership anchor로 남아, TERM-ignore
descendant가 있더라도 fresh authority로 group KILL과 direct wait를 완료할 수 있게 한다.
```text
/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup luksFormat --batch-mode --type luks2 --key-file=- LOOP
/usr/bin/sudo --non-interactive -- /usr/sbin/cryptsetup open --type luks2 --key-file=- LOOP ALLOWED_MAPPING
```
production helper는 outer mount부터 exact current-user `0600` regular non-symlink KDBX까지
모든 component가 canonical non-symlink인 것, library-owned entry, canonical loop block
device, allowlisted mapping과 cached sudo를 요구한다. fixture suite는 partial/zero producer
failure의 consumer dispatch 0, xtrace 선차단, non-TTY, exact KeePassXC argv, process
argv/environment와 stdout/stderr/tmp/run 누출, real blocking-child signal reap, exclusive
synthetic lifecycle과 cleanup failure 전파를 low command-boundary fake로 검사한다.
signal fixture는 launch, PID publish, STOP/query, CONT 전후, wait/reap-clear, consumer 직전
경계에서 `INT`/`TERM`을 결정적으로 주입하고 supervisor, process-substitution shell,
KeePassXC, `od`, `mawk`와 feasibility nested roles의 종료, caller/unrelated sentinel 생존,
consumer dispatch 0과 반복-run leak 0을 확인한다.
synthetic KDBX의 `add`와 `attachment-import`는 atomic-save inode 교체를 허용하되, 각 mutation
직후 exact path/owner/mode/non-symlink를 다시 확인하여 secure 새 inode를 baseline으로 삼는다.
read-only 단계에서는 baseline과 fixture parent identity가 바뀌면 즉시 실패한다.
고정 package가 없는 host에서는 `--execute`를 실행하거나 임의 version을 설치하지 않고
명시적으로 SKIP한다.
## k3s Secret 암호화 live 읽기 전용 검증
`k3s-secret-encryption.sh`는 k3s 서비스나 Kubernetes 리소스를 변경하지 않고 현재
Secret 암호화 상태, 서버 annotation 일치 여부, local config 무결성과 API readiness를
분류한다. 일반 사용자 shell에서 실행하면 제한 시간과 `sudo --non-interactive`가 적용된
개별 read-only 명령만 root 전용 systemd 환경, datastore 증거와 credential 파일을 읽는다.
workspace의 validator나 library 자체를 root shell에서 실행하거나 source하지 않는다.
`/usr/local/bin/k3s`, `/usr/bin/systemctl`, `/usr/bin/stat` 등 허용된 절대경로의
root-owned·non-group/other-writable system binary만 검증한 뒤 실행한다. raw status, 환경 변수
원문, annotation, hash, active key, token, password 또는 config 내용은 사용자 terminal,
명령 인자나 handoff에 출력하지 않고 현재 사용자 프로세스의 메모리에서만 분류한다.
validator는 비대화형 sudo만 사용하므로 실행 직전에 **같은 terminal**에서 credential을
먼저 갱신해야 한다. 다른 terminal에서 실행한 `sudo -v`는 이 실행의 prerequisite를
충족한다고 가정하지 않는다.
```sh
sudo -v
```
credential validation이 실패하면 validator는 root-only evidence를 하나도 읽지 않고
`sudo -v`를 같은 terminal에서 실행한 뒤 다시 시도하라는 오류로 즉시 종료한다.
```sh
bash scripts/validate/k3s-secret-encryption.sh
bash scripts/validate/k3s-secret-encryption.sh --expect-disabled
bash scripts/validate/k3s-secret-encryption.sh --expect-transition-start
bash scripts/validate/k3s-secret-encryption.sh --expect-enabled
bash scripts/validate/k3s-secret-encryption.sh --expect-reencrypted
```
기대 상태 옵션은 최대 하나만 지정한다. 옵션이 없으면 inventory만 수행하며 status가
`unsafe_transition`, `hash_mismatch`, `invalid`로 분류되면 실패한다.
모든 성공 경로는 version이 정확히 `v1.36.2+k3s1`, server가 정확히 1개, server
node가 정확히 `donghyeon-system-product-name`이고 Ready인 것을 요구한다.
`--expect-enabled`는 운영 진단용으로 `Enabled/start`와
`Enabled/reencrypt_finished`를 모두 허용한다. Phase 4 완료 gate인
`--expect-reencrypted`는 `Enabled/reencrypt_finished`이면서 local config, state,
server annotation hash가 모두 일치할 때만 성공한다. `--expect-transition-start`도
status의 hash match가 확인된 exact transition만 허용한다. 기대 상태 불일치와 분류할 수
없는 결과, API readyz 실패는 non-zero로 종료한다.
검증된 비민감 handoff가 필요하면 caller가 먼저 제한된 임시 디렉터리를 만든다.
```sh
VERIFY_DIR="$(mktemp -d /tmp/platform-k3s-encryption.XXXXXX)"
chmod 0700 "$VERIFY_DIR"
bash scripts/validate/k3s-secret-encryption.sh --expect-enabled \
--verified-output-dir "$VERIFY_DIR"
```
validator는 handoff 디렉터리를 만들지 않는다. slash가 없는 nonempty suffix를 가진
`/tmp/platform-k3s-encryption.<suffix>` physical direct child 중 기존 empty,
non-symlink, 현재 사용자 소유 mode `0700` 디렉터리만 허용한다. 검증과 write는 같은
열린 directory FD에 묶는다.
inventory와 기대 상태 검증이 성공한 뒤에만 noclobber와 `umask 077`로 정확히
`inventory.env`, `status.sha256` 두 파일을 만들며, 두 파일은 현재 사용자 소유 regular
file mode `0600`이어야 한다. status 원문과 active key 이름은 어느 handoff에도 쓰지
않는다. `status.sha256`은 status stdout의 공백이나 trailing newline이 아니라 `jq -cS`로
검증·정렬·압축한 single JSON value의 UTF-8 bytes(끝 newline 없음)에 대한 SHA-256이다.
기존 파일, 추가 파일, nested/symlink parent, 다른 owner 또는 group/other 권한이 있으면
실패한다.
## k3s Secret 복구 증거 검증
`k3s-secret-encryption-restore-evidence.sh`는 다음 세 mode를 제공한다.
```sh
bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \
--emit-result --bundle-metadata BUNDLE_METADATA_FILE --output RESULT_FILE
bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \
--record --bundle-metadata BUNDLE_METADATA_FILE --result-file RESULT_FILE
bash scripts/validate/k3s-secret-encryption-restore-evidence.sh --check
```
`--emit-result`는 격리 복구 host에서 current-user kube context와 live API를 확인하고
`Enabled/reencrypt_finished`, server hash, local integrity, node Ready, version/backend와
복구 전후 Secret object count가 모두 맞을 때만 non-sensitive result를 만든다. operator가
default route, upstream DNS, 운영 API/datastore route와 모든 egress 차단 시험까지 완료했다고
현재 context로 확인해야 한다. 결과에는 `destroyed`가 없다.
`--record`는 외부 post metadata/result, 24시간 age와 운영 host의 권위 post metadata
일곱 field 전체를 비교한다. 일회용 복구 환경과 bundle 복사본 파기 후 정확한 확인 입력을
받아야 하며 기존 evidence를 덮어쓰지 않는다. `--check`는 현재 권위 bundle-id,
version/backend, 30일 age, `destroyed=confirmed`, live re-encryption과 local integrity를
읽기 전용으로 다시 검사한다. 2026-08-09 live 전환은 `Enabled/reencrypt_finished`,
hash·integrity·API·node 검사, post bundle 기록과 최신 marker 검증까지 통과했다. 다만
격리 restore·파기 evidence는 아직 없으므로 `--check` 성공을 기록하거나 관측성 Phase 4
gate를 열지 않는다.
외부 입력은 현재 사용자 소유 regular non-symlink, mode `0600`, non-empty여야 한다. exact
allowlist parser가 duplicate/unknown/empty key, control character, malformed/trailing data,
`$(`와 backtick을 sudo 전에 거부한다. 전체 script를 sudo로 실행하지 않고 root 전용 파일의
stat/read/install만 좁게 승격한다. result/evidence target은 같은 directory의 `0600`
temporary file을 검증한 뒤 기존 파일을 덮어쓰지 않는 atomic install로 만든다.
전체 수동 절차와 격리 checklist는
[k3s Secret 복구 drill](../../bootstrap/manual/k3s-secret-encryption-restore-drill.md)을 따른다.
parser와 세 mode의 fixture 회귀는 live cluster에 연결하지 않고 다음 명령에 포함된다.
```sh
bash scripts/validate/test-k3s-secret-encryption-status.sh
```
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set +x
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly LAN_IP=192.168.0.107
readonly TAIL_IP=100.92.240.34
readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com
readonly DB_HOST=db-admin.learn.hyeonworks.com
readonly SMOKE_IMAGE="busybox@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0"
execute=false
run_s3=false
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
사용법:
bash scripts/validate/admin-ui-smoke.sh
bash scripts/validate/admin-ui-smoke.sh --execute [--run-s3]
기본 실행은 공개 DNS, 인증서, 리소스, Secret key 이름만 읽습니다.
--execute는 LAN/Tailscale/Pod DNS와 실제 HTTPS·NetworkPolicy를 검사하는
임시 Pod를 만들었다가 삭제합니다. --run-s3는 기존 인증 S3 회귀 검사도
연결하며 그 스크립트의 APPLY 확인을 한 번 더 요구합니다.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--execute) execute=true; shift ;;
--run-s3) run_s3=true; shift ;;
-h|--help) usage; exit 0 ;;
*) usage >&2; fail "지원하지 않는 인자: $1" ;;
esac
done
[[ "$run_s3" == false || "$execute" == true ]] || fail "--run-s3에는 --execute가 필요합니다"
[[ "$(pwd -P)" == "$ROOT" ]] || fail "$ROOT에서 실행하세요"
for cmd in curl dig jq kubectl nc openssl rg sort tr; do
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
for host in "$STORAGE_HOST" "$DB_HOST"; do
[[ -z "$(dig +short @1.1.1.1 A "$host" | tr -d '[:space:]')" ]] || fail "$host 공개 A가 있습니다"
[[ -z "$(dig +short @1.1.1.1 AAAA "$host" | tr -d '[:space:]')" ]] || fail "$host 공개 AAAA가 있습니다"
done
kubectl -n object-storage get objectstore minio-aistor -o json | \
jq -e '.status.healthStatus == "green"' >/dev/null || fail "AIStor가 green이 아닙니다"
kubectl -n platform-admin get deployment pgadmin -o json | \
jq -e '.status.availableReplicas == 1 and .spec.strategy.type == "Recreate"' >/dev/null || \
fail "pgAdmin Deployment가 available/Recreate 상태가 아닙니다"
kubectl -n platform-admin get pvc pgadmin -o json | \
jq -e '.status.phase == "Bound" and .spec.resources.requests.storage == "2Gi"' >/dev/null || \
fail "pgAdmin PVC가 2Gi Bound가 아닙니다"
kubectl get pv pgadmin-data-local-pv -o json | \
jq -e '.spec.persistentVolumeReclaimPolicy == "Retain"' >/dev/null || fail "pgAdmin PV가 Retain이 아닙니다"
for contract in "object-storage aistor-keycloak-oidc" "platform-admin pgadmin-keycloak-oidc"; do
set -- $contract
keys="$(kubectl -n "$1" get "secret/$2" \
-o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}' | LC_ALL=C sort)"
[[ "$keys" == $'client-id\nclient-secret' ]] || fail "$1/$2 Secret 계약이 다릅니다"
done
keys="$(kubectl -n platform-admin get secret pgadmin-bootstrap \
-o go-template='{{range $key, $_ := .data}}{{$key}}{{"\n"}}{{end}}')"
[[ "$keys" == password ]] || fail "pgAdmin bootstrap Secret 계약이 다릅니다"
config="$(kubectl -n platform-admin get configmap pgadmin-config -o jsonpath='{.data.config_local\.py}')"
printf '%s\n' "$config" | rg -q -F 'ALLOW_SAVE_PASSWORD = False' || fail "비밀번호 저장 차단이 없습니다"
printf '%s\n' "$config" | rg -q -F "'groups': ['/platform-db-admins']" || fail "pgAdmin 그룹 gate가 없습니다"
kubectl -n platform-data exec platform-postgres-1 -c postgres -- \
psql -Atqc "SELECT count(*) FROM pg_roles WHERE rolname IN ('pgadmin','platform-dba') OR (rolsuper AND rolname NOT IN ('postgres','streaming_replica'));" \
| rg -q '^0$' || fail "예상하지 못한 pgAdmin/DBA/superuser 역할이 있습니다"
cert="$(openssl s_client -connect 127.0.0.1:443 -servername "$STORAGE_HOST" </dev/null 2>/dev/null | \
openssl x509 -noout -text)"
printf '%s\n' "$cert" | rg -q "DNS:$STORAGE_HOST" || fail "admin 인증서 SAN이 없습니다"
printf '%s\n' "$cert" | rg -q "DNS:$DB_HOST" || fail "admin 인증서 SAN이 없습니다"
printf 'READ-ONLY CHECK PASS: 공개 admin DNS 없음, 리소스·Secret 계약·인증서 정상\n'
if [[ "$execute" == false ]]; then
printf '%s\n' '--execute를 지정하지 않아 임시 Pod와 네트워크 요청은 만들지 않았습니다.'
exit 0
fi
[[ -t 0 ]] || fail "--execute는 대화형 터미널이 필요합니다"
printf 'Type APPLY to run transient DNS and network smoke tests: '
read -r answer
[[ "$answer" == APPLY ]] || fail "취소했습니다"
for pair in "$LAN_IP $STORAGE_HOST" "$LAN_IP $DB_HOST" "$TAIL_IP $STORAGE_HOST" "$TAIL_IP $DB_HOST"; do
set -- $pair
[[ "$(dig +short "@$1" A "$2" | tail -n1)" == "$1" ]] || fail "$2 private DNS 실패"
done
for host in "$STORAGE_HOST" "$DB_HOST"; do
code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--noproxy '*' --interface "$LAN_IP" --resolve "$host:443:$LAN_IP" "https://$host/")"
[[ "$code" == 200 || "$code" == 302 || "$code" == 303 ]] || fail "$host LAN HTTPS 실패: $code"
denied="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--noproxy '*' --resolve "$host:443:127.0.0.1" "https://$host/")"
[[ "$denied" == 403 ]] || fail "$host 허용 목록 밖 요청이 403이 아닙니다"
done
if nc -z -w 2 "$LAN_IP" 30080 || nc -z -w 2 "$LAN_IP" 30443; then
fail "Traefik NodePort가 LAN 주소에 노출됐습니다"
fi
nc -z -w 2 127.0.0.1 30080 || fail "Traefik HTTP NodePort loopback이 닫혔습니다"
pod="admin-ui-smoke-$(date +%H%M%S)"
cleanup_pod() {
kubectl -n platform-admin delete "pod/$pod" --ignore-not-found --wait=false >/dev/null 2>&1 || true
}
trap cleanup_pod EXIT INT TERM
kubectl apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
name: $pod
namespace: platform-admin
labels:
app.kubernetes.io/name: pgadmin4
app.kubernetes.io/instance: pgadmin
app.kubernetes.io/component: network-smoke
app.kubernetes.io/part-of: platform
spec:
automountServiceAccountToken: false
restartPolicy: Never
terminationGracePeriodSeconds: 1
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: network-smoke
image: $SMOKE_IMAGE
imagePullPolicy: IfNotPresent
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
command:
- sh
- -ec
- |
test "\$(nslookup id.learn.hyeonworks.com | awk '/^Address: /{print \$2}' | tail -n1)" = 192.168.0.107
test "\$(nslookup storage-admin.learn.hyeonworks.com | awk '/^Address: /{print \$2}' | tail -n1)" = 192.168.0.107
nc -z -w 5 platform-postgres-rw.platform-data.svc.cluster.local 5432
EOF
kubectl -n platform-admin wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$pod" --timeout=90s
cleanup_pod
trap - EXIT INT TERM
if [[ "$run_s3" == true ]]; then
bash "$ROOT/scripts/validate/aistor-s3-smoke.sh" --execute
fi
printf 'ADMIN UI SMOKE PASS\n'
printf '남은 수동 검증: 허용/비허용 Keycloak 그룹 계정으로 두 UI 로그인\n'
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env bash
set -Eeuo pipefail
# Secret이 셸 추적에 노출되지 않도록 호출자가 bash -x를 사용해도 끈다.
set +x
readonly NAMESPACE="object-storage"
readonly OBJECTSTORE="minio-aistor"
readonly ROOT_SECRET="aistor-root-configuration"
readonly S3_ENDPOINT="http://minio.object-storage.svc.cluster.local"
readonly MC_IMAGE="quay.io/minio/aistor/mc@sha256:c5ec777c080fd6292b7529309a0f311c58c3a92e268c9c57c0901da86881949b"
readonly PASS_LINE="AISTOR_S3_SMOKE_PASS bucket-create object-write object-read object-delete bucket-delete"
execute_requested=false
pod_name=""
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
cleanup() {
if [[ -n "$pod_name" ]]; then
kubectl --namespace "$NAMESPACE" delete pod "$pod_name" \
--ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true
fi
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/validate/aistor-s3-smoke.sh --execute
Creates one restricted, short-lived AIStor Client Pod. It authenticates with
object-storage/aistor-root-configuration and performs:
bucket create -> object write -> SHA-256 read check
-> object delete -> bucket delete
The script never prints credentials. It removes the temporary bucket and Pod.
Type APPLY only when prompted because this performs temporary S3 writes.
USAGE
}
while (( $# > 0 )); do
case "$1" in
--execute)
execute_requested=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
usage >&2
fail "unsupported argument: $1"
;;
esac
done
[[ "$execute_requested" == true ]] || {
usage
fail "--execute is required"
}
for command_name in kubectl grep jq; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
[[ "$(
kubectl --namespace "$NAMESPACE" get objectstore "$OBJECTSTORE" \
--output=jsonpath='{.status.currentState}'
)" == "Initialized" ]] || fail "ObjectStore is not Initialized"
[[ "$(
kubectl --namespace "$NAMESPACE" get objectstore "$OBJECTSTORE" \
--output=jsonpath='{.status.healthStatus}'
)" == "green" ]] || fail "ObjectStore health is not green"
kubectl --namespace "$NAMESPACE" get secret "$ROOT_SECRET" >/dev/null
printf '%s\n' \
"Target: ${NAMESPACE}/${OBJECTSTORE}" \
"Endpoint: ${S3_ENDPOINT}" \
'Temporary resources: one bucket, one object, and one restricted client Pod' \
'No credential value will be printed.'
printf 'Type APPLY to run the authenticated S3 smoke test: '
IFS= read -r confirmation
[[ "$confirmation" == "APPLY" ]] || fail "confirmation did not match APPLY"
pod_name="aistor-s3-smoke-$(date +%H%M%S)-$$"
trap cleanup EXIT INT TERM
smoke_command='set -eu
. /run/aistor-root/config.env
bucket="platform-smoke-$(date +%s)"
cleanup_bucket() { mc rb --force "local/${bucket}" >/dev/null 2>&1 || true; }
trap cleanup_bucket EXIT
mc alias set local "'"$S3_ENDPOINT"'" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" --api S3v4 >/dev/null
payload="aistor-authenticated-smoke-$(date +%s)"
expected="$(printf "%s" "$payload" | sha256sum | cut -d " " -f 1)"
mc mb "local/${bucket}" >/dev/null
printf "%s" "$payload" | mc pipe "local/${bucket}/probe.txt" >/dev/null
actual="$(mc cat "local/${bucket}/probe.txt" | sha256sum | cut -d " " -f 1)"
[ "$expected" = "$actual" ]
mc stat "local/${bucket}/probe.txt" >/dev/null
mc rm "local/${bucket}/probe.txt" >/dev/null
mc rb "local/${bucket}" >/dev/null
trap - EXIT
printf "%s\n" "'"$PASS_LINE"'"'
overrides="$(
jq -nc --arg command "$smoke_command" --arg image "$MC_IMAGE" '{
spec: {
automountServiceAccountToken: false,
securityContext: {
runAsNonRoot: true,
runAsUser: 1000,
runAsGroup: 1000,
fsGroup: 1000,
seccompProfile: {type: "RuntimeDefault"}
},
containers: [{
name: "aistor-s3-smoke",
image: $image,
imagePullPolicy: "IfNotPresent",
command: ["/bin/sh", "-ec"],
args: [$command],
securityContext: {
allowPrivilegeEscalation: false,
capabilities: {drop: ["ALL"]},
readOnlyRootFilesystem: true
},
env: [{name: "MC_CONFIG_DIR", value: "/tmp/mc"}],
volumeMounts: [
{name: "root-config", mountPath: "/run/aistor-root", readOnly: true},
{name: "tmp", mountPath: "/tmp"}
]
}],
volumes: [
{
name: "root-config",
secret: {secretName: "'"$ROOT_SECRET"'", defaultMode: 256}
},
{name: "tmp", emptyDir: {}}
]
}
}'
)"
kubectl --namespace "$NAMESPACE" run "$pod_name" \
--image="$MC_IMAGE" \
--restart=Never \
--labels='platform.hyeonworks.com/aistor-client=true,app.kubernetes.io/name=aistor-s3-smoke' \
--overrides="$overrides"
succeeded=false
for _ in $(seq 1 90); do
phase="$(
kubectl --namespace "$NAMESPACE" get pod "$pod_name" \
--output=jsonpath='{.status.phase}'
)"
if [[ "$phase" == "Succeeded" ]]; then
succeeded=true
break
fi
[[ "$phase" != "Failed" ]] || break
waiting_reason="$(
kubectl --namespace "$NAMESPACE" get pod "$pod_name" \
--output=jsonpath='{.status.containerStatuses[0].state.waiting.reason}' \
2>/dev/null || true
)"
case "$waiting_reason" in
CreateContainerConfigError|CrashLoopBackOff|ErrImagePull|ImagePullBackOff)
break
;;
esac
sleep 2
done
if [[ "$succeeded" != true ]]; then
kubectl --namespace "$NAMESPACE" logs "$pod_name" || true
kubectl --namespace "$NAMESPACE" describe pod "$pod_name" | tail -n 60
fail "authenticated S3 smoke test failed"
fi
log_output="$(kubectl --namespace "$NAMESPACE" logs "$pod_name")"
grep --fixed-strings --line-regexp --quiet "$PASS_LINE" <<<"$log_output" || \
fail "smoke Pod did not emit the exact success contract"
printf '%s\n' "$PASS_LINE"
printf '%s\n' \
"Client image: ${MC_IMAGE}" \
'Temporary bucket, object, and client Pod cleanup: complete'
@@ -0,0 +1,812 @@
#!/usr/bin/env bash
# Repository-owned, read-only Prometheus metric inventory capture. This file is
# sourceable so its parsers can be exercised without contacting a cluster.
readonly INVENTORY_REJECTION_STATUS=23
readonly INVENTORY_SCHEMA='platform-observability-metric-inventory/v1'
readonly INVENTORY_CONTEXT='default'
readonly INVENTORY_API_SERVER='https://127.0.0.1:6443'
readonly INVENTORY_PROMETHEUS_NAMESPACE='observability'
readonly INVENTORY_PROMETHEUS_NAME='observability-core-kube-pr-prometheus'
readonly INVENTORY_PROMETHEUS_SERVICE='observability-core-kube-pr-prometheus'
readonly INVENTORY_MAX_RESPONSE_BYTES=8388608
inventory_reject() {
printf 'REJECT: target-initial inventory precondition failed\n' >&2
return "$INVENTORY_REJECTION_STATUS"
}
inventory_expected_pools() {
local phase="${1-target-initial}"
[[ "$phase" == target-initial || "$phase" == post-substrate ]] || {
inventory_reject
return
}
cat <<'POOLS'
podMonitor/platform-data/platform-postgres/0
serviceMonitor/gitea/gitea/0
serviceMonitor/keycloak/keycloak/0
serviceMonitor/kube-system/traefik/0
serviceMonitor/object-storage/aistor-bucket-usage/0
serviceMonitor/observability-agent/alloy/0
serviceMonitor/observability-agent/node-exporter/0
serviceMonitor/observability/grafana/0
serviceMonitor/observability/loki/0
serviceMonitor/observability/observability-core-kube-pr-alertmanager/0
serviceMonitor/observability/observability-core-kube-pr-alertmanager/1
serviceMonitor/observability/observability-core-kube-pr-apiserver/0
serviceMonitor/observability/observability-core-kube-pr-coredns/0
serviceMonitor/observability/observability-core-kube-pr-kubelet/0
serviceMonitor/observability/observability-core-kube-pr-kubelet/1
serviceMonitor/observability/observability-core-kube-pr-kubelet/2
serviceMonitor/observability/observability-core-kube-pr-operator/0
serviceMonitor/observability/observability-core-kube-pr-prometheus/0
serviceMonitor/observability/observability-core-kube-pr-prometheus/1
serviceMonitor/observability/observability-core-kube-state-metrics/0
serviceMonitor/observability/tempo/0
POOLS
if [[ "$phase" == post-substrate ]]; then
cat <<'POOLS'
probe/observability/platform-private-edge
probe/observability/platform-private-edge
probe/observability/platform-private-edge
probe/observability/platform-private-internal
probe/observability/platform-private-internal
probe/observability/platform-private-internal
probe/observability/platform-public-edge
probe/observability/platform-public-edge
serviceMonitor/observability/blackbox-exporter/0
POOLS
fi
}
post_substrate_expected_probe_targets() {
cat <<'JSON'
[
{"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fdb-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://db-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}},
{"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fgrafana.learn.hyeonworks.com%2F","labels":{"instance":"https://grafana.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}},
{"scrapePool":"probe/observability/platform-private-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_private_edge_403&target=https%3A%2F%2Fstorage-admin.learn.hyeonworks.com%2F","labels":{"instance":"https://storage-admin.learn.hyeonworks.com/","job":"blackbox-private-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"private-edge"}},
{"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=db-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fpgadmin.platform-admin.svc.cluster.local%2Fmisc%2Fping","labels":{"instance":"http://pgadmin.platform-admin.svc.cluster.local/misc/ping","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}},
{"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=grafana.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fgrafana.observability.svc.cluster.local%2Fapi%2Fhealth","labels":{"instance":"http://grafana.observability.svc.cluster.local/api/health","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}},
{"scrapePool":"probe/observability/platform-private-internal","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?hostname=storage-admin.learn.hyeonworks.com&module=http_private_internal_200&target=http%3A%2F%2Fminio-aistor-console.object-storage.svc.cluster.local%3A9090%2F","labels":{"instance":"http://minio-aistor-console.object-storage.svc.cluster.local:9090/","job":"blackbox-private-internal","namespace":"observability","observability.hyeonworks.com/probe-group":"private-internal"}},
{"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fgit.learn.hyeonworks.com%2Fapi%2Fhealthz","labels":{"instance":"https://git.learn.hyeonworks.com/api/healthz","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}},
{"scrapePool":"probe/observability/platform-public-edge","scrapeUrl":"http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fid.learn.hyeonworks.com%2Frealms%2Fhyeonworks%2F.well-known%2Fopenid-configuration","labels":{"instance":"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}}
]
JSON
}
sanitize_scrape_url() {
local url="${1-}" mode="${2-strict}" remainder authority
if [[ -z "$url" || "$url" == *[$' \t\r\n']* || "$url" == *'#'* ]]; then
inventory_reject
return
fi
if [[ "$mode" == strip-query ]]; then
[[ "$url" == *'?'* ]] || { inventory_reject; return; }
url="${url%%\?*}"
elif [[ "$mode" != strict || "$url" == *'?'* ]]; then
inventory_reject
return
fi
case "$url" in
http://*|https://*) ;;
*) inventory_reject; return ;;
esac
remainder="${url#*://}"
authority="${remainder%%/*}"
if [[ -z "$authority" || "$authority" == *'@'* ]]; then
inventory_reject
return
fi
printf '%s\n' "$url"
}
validate_targets_json() {
local path="${1--}" phase="${2-target-initial}" expected_json post_probes='[]'
expected_json="$(inventory_expected_pools "$phase" | jq -Rsc 'split("\n") | map(select(length > 0)) | sort')" || {
inventory_reject
return
}
if [[ "$phase" == post-substrate ]]; then
post_probes="$(post_substrate_expected_probe_targets | jq -cS 'sort_by(.scrapePool,.scrapeUrl)')" || {
inventory_reject
return
}
fi
if ! jq -e --arg phase "$phase" --argjson expected "$expected_json" --argjson postProbes "$post_probes" '
type == "object" and .status == "success" and
(.data | type == "object") and
(.data.activeTargets | type == "array") and
(.data.droppedTargets | type == "array") and
((.data.activeTargets | map(.scrapePool) | sort) == $expected) and
all(.data.activeTargets[];
type == "object" and (.scrapePool | type == "string") and
(.scrapeUrl | type == "string") and (.labels | type == "object") and
(.scrapeUrl | test("^https?://[^/?#@[:space:]]+(?:/[^#[:space:]]*)?$")) and
($phase == "post-substrate" or
(.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not)) and
(.health == "up") and (.lastError == "")) and
all(.data.droppedTargets[]?;
(.scrapePool | type == "string") and
($phase == "post-substrate" or
(.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not))) and
([.data.activeTargets[] |
select(.scrapePool == "serviceMonitor/object-storage/aistor-bucket-usage/0") |
select(.labels.job == "minio-aistor" and
.labels.namespace == "object-storage" and
.labels.service == "minio" and
.labels.endpoint == "http-minio" and
.labels.container == "minio")] | length == 1) and
($phase == "target-initial" or (
([.data.activeTargets[] |
select(.scrapePool | startswith("probe/observability/")) |
{scrapePool,scrapeUrl,labels}] | sort_by(.scrapePool,.scrapeUrl)) == $postProbes and
([.data.activeTargets[] |
select(.scrapePool == "serviceMonitor/observability/blackbox-exporter/0") |
select(.labels.job == "blackbox-exporter" and
.labels.namespace == "observability" and
.labels.service == "blackbox-exporter" and
.labels.endpoint == "http" and
.labels.container == "blackbox-exporter" and
(.labels.pod | test("^blackbox-exporter-[a-z0-9]+-[a-z0-9]+$")) and
(.labels.instance | test("^[0-9.]+:9115$")) and
(.scrapeUrl | test("^http://[0-9.]+:9115/metrics$")))] | length == 1)
))
' "$path" >/dev/null 2>&1; then
inventory_reject
return
fi
}
validate_dropped_targets_json() {
local path="${1--}" phase="${2-target-initial}"
if ! jq -e --arg phase "$phase" '
type == "object" and .status == "success" and
(.data | type == "object") and
(.data.activeTargets | type == "array") and
(.data.droppedTargets | type == "array") and
all(.data.droppedTargets[]?;
(.scrapePool | type == "string") and
(if $phase == "target-initial"
then (.scrapePool | test("(?:blackbox|(?:^|/)probe(?:/|$))"; "i") | not)
else (.scrapePool | test("(?:^|/)probe(?:/|$)"; "i") | not)
end))
' "$path" >/dev/null 2>&1; then
inventory_reject
return
fi
}
validate_probe_series_phase() {
local path="${1--}" phase="${2-target-initial}"
if ! jq -e --arg phase "$phase" '
type == "object" and .status == "success" and (.data | type == "array") and
(if $phase == "target-initial" then (.data | length) == 0
elif $phase == "post-substrate" then
(.data | length) > 0 and
all(.data[];
type == "object" and (."__name__" | type == "string") and
(."__name__" | startswith("probe_")) and
.namespace == "observability")
else false end)
' "$path" >/dev/null 2>&1; then
inventory_reject
return
fi
}
extract_metric_names() {
local path="${1--}" names
if ! names="$(jq -er '
if type == "object" and .status == "success" then
if (.data | type == "object") and (.data | length > 0) and
all(.data | keys[]; test("^[a-zA-Z_:][a-zA-Z0-9_:]*$"))
then .data | keys | sort | .[]
elif (.data | type == "array") and (.data | length > 0) and
all(.data[];
type == "object" and (.target | type == "object") and
(.metric | type == "string") and
(.metric | test("^[a-zA-Z_:][a-zA-Z0-9_:]*$")) and
(.type | type == "string") and (.help | type == "string") and
(.unit | type == "string"))
then [.data[].metric] | unique | sort | .[]
else error("invalid metadata") end
else error("invalid metadata") end
' "$path" 2>/dev/null)"; then
inventory_reject
return
fi
printf '%s\n' "$names"
}
extract_recording_rule_names() {
local path="${1--}" names
if ! names="$(jq -er '
if type == "object" and .status == "success" and
(.data | type == "object") and (.data.groups | type == "array") and
all(.data.groups[]; (.rules | type == "array"))
then [.data.groups[].rules[] | select(.type == "recording") | .name] |
if all(.[]; type == "string" and test("^[a-zA-Z_:][a-zA-Z0-9_:]*$"))
then unique | sort | join("\n") else error("invalid recording rule") end
else error("invalid rules") end
' "$path" 2>/dev/null)"; then
inventory_reject
return
fi
[[ -z "$names" ]] || printf '%s\n' "$names"
}
reject_recording_rule_collisions() {
local rules_path="${1-}" metadata_path="${2-}" rules metrics name
if ! rules="$(extract_recording_rule_names "$rules_path")" ||
! metrics="$(extract_metric_names "$metadata_path")"; then
inventory_reject
return
fi
reject_metric_name_collisions "$rules" "$metrics"
}
reject_metric_name_collisions() {
local rules="${1-}" metrics="${2-}" name
while IFS= read -r name; do
[[ -z "$name" ]] && continue
if grep -Fqx -- "$name" <<<"$rules"; then
inventory_reject
return
fi
done <<<"$metrics"
}
validate_metric_names_for_pool() {
local metrics="${1-}" pool="${2-}" phase="${3-target-initial}"
if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then
grep -Fqx probe_success <<<"$metrics" || { inventory_reject; return; }
elif grep -Eq '^probe_' <<<"$metrics"; then
inventory_reject
return
fi
}
extract_series_label_names() {
local path="${1--}" result
if ! result="$(jq -ceS '
if type == "object" and .status == "success" and
(.data | type == "array") and (.data | length > 0) and
all(.data[]; type == "object") and
all([.data[] | keys[]][]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$"))
then [.data[] | keys[]] | unique | sort
else error("invalid series") end
' "$path" 2>/dev/null)"; then
inventory_reject
return
fi
printf '%s\n' "$result"
}
prometheus_selector_for_labels() {
local labels_json="${1-}"
jq -cer '
if type == "object" and length > 0 and
all(keys[]; test("^[a-zA-Z_][a-zA-Z0-9_./-]*$")) and
all(.[]; type == "string" and (explode | all(. >= 32 and . != 127)))
then "{" + (to_entries | sort_by(.key) |
map((if (.key | test("^[a-zA-Z_][a-zA-Z0-9_]*$"))
then .key else (.key | @json) end) + "=" + (.value | @json)) | join(",")) + "}"
else error("invalid target labels") end
' <<<"$labels_json" 2>/dev/null || inventory_reject
}
inventory_provenance() {
case "${1-}" in
podMonitor/platform-data/platform-postgres/0|\
serviceMonitor/gitea/gitea/0|\
serviceMonitor/keycloak/keycloak/0|\
serviceMonitor/kube-system/traefik/0|\
serviceMonitor/object-storage/aistor-bucket-usage/0)
printf '%s\n' 'task6-owned'
;;
serviceMonitor/observability/grafana/0)
printf '%s\n' 'preexisting-verify-only'
;;
serviceMonitor/observability/blackbox-exporter/0|probe/observability/platform-*)
printf '%s\n' 'substrate-owned'
;;
*)
printf '%s\n' 'core'
;;
esac
}
validate_output_directory() {
local directory="${1-}" canonical uid mode current component
if [[ -z "$directory" || "$directory" != /* || "$directory" == *[$'\t\r\n']* ||
! -d "$directory" || -L "$directory" ]]; then
inventory_reject
return
fi
if ! canonical="$(realpath -e -- "$directory" 2>/dev/null)" || [[ "$canonical" != "$directory" ]]; then
inventory_reject
return
fi
current=''
IFS='/' read -r -a components <<<"${directory#/}"
for component in "${components[@]}"; do
[[ -n "$component" && "$component" != '.' && "$component" != '..' ]] || {
inventory_reject
return
}
current="$current/$component"
if [[ -L "$current" ]]; then
inventory_reject
return
fi
done
if ! uid="$(stat -c %u -- "$directory" 2>/dev/null)" ||
! mode="$(stat -c %a -- "$directory" 2>/dev/null)" ||
[[ "$uid" != "$(id -u)" || "$mode" != 700 ]]; then
inventory_reject
return
fi
if [[ -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then
inventory_reject
return
fi
}
inventory_kubectl() {
case "${1-}" in
config)
case "${2-}" in current-context|view) ;; *) inventory_reject; return ;; esac
;;
get) ;;
auth)
[[ "${2-}" == can-i ]] || { inventory_reject; return; }
;;
*)
inventory_reject
return
;;
esac
timeout --signal=TERM --kill-after=2s 15s \
kubectl --request-timeout=10s "$@" 2>/dev/null
}
inventory_process_snapshot() {
local pid="${1-}" stat_line remainder state start_time
[[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] || return 1
IFS= read -r stat_line <"/proc/$pid/stat" || return 1
remainder="${stat_line##*) }"
state="${remainder%% *}"
start_time="$(awk '{print $20}' <<<"$remainder")" || return 1
[[ "$state" =~ ^[A-Z]$ && "$start_time" =~ ^[0-9]+$ ]] || return 1
printf '%s:%s\n' "$state" "$start_time"
}
inventory_port_forward_is_owned() {
local pid="${INVENTORY_PORT_FORWARD_PID:-}" expected="${INVENTORY_PORT_FORWARD_START:-}" snapshot
[[ "$pid" =~ ^[1-9][0-9]*$ && "$expected" =~ ^[0-9]+$ ]] || return 1
snapshot="$(inventory_process_snapshot "$pid")" || return 1
[[ "${snapshot#*:}" == "$expected" ]]
}
inventory_stop_port_forward() {
local pid="${INVENTORY_PORT_FORWARD_PID:-}" snapshot state attempt
[[ -n "$pid" ]] || return 0
if ! inventory_port_forward_is_owned; then
INVENTORY_PORT_FORWARD_PID=''
INVENTORY_PORT_FORWARD_START=''
return 0
fi
kill -TERM "$pid" 2>/dev/null || true
for ((attempt = 0; attempt < 40; attempt++)); do
if ! inventory_port_forward_is_owned; then break; fi
snapshot="$(inventory_process_snapshot "$pid")" || break
state="${snapshot%%:*}"
[[ "$state" == Z || "$state" == X ]] && break
sleep 0.025
done
if inventory_port_forward_is_owned; then
snapshot="$(inventory_process_snapshot "$pid")" || snapshot=''
state="${snapshot%%:*}"
if [[ "$state" != Z && "$state" != X ]]; then
kill -KILL "$pid" 2>/dev/null || true
fi
fi
for ((attempt = 0; attempt < 40; attempt++)); do
if ! inventory_port_forward_is_owned; then break; fi
snapshot="$(inventory_process_snapshot "$pid")" || break
state="${snapshot%%:*}"
[[ "$state" == Z || "$state" == X ]] && break
sleep 0.025
done
if inventory_port_forward_is_owned; then
snapshot="$(inventory_process_snapshot "$pid")" || snapshot=''
state="${snapshot%%:*}"
if [[ "$state" != Z && "$state" != X ]]; then return 1; fi
fi
INVENTORY_PORT_FORWARD_PID=''
INVENTORY_PORT_FORWARD_START=''
}
build_inventory_json() {
local target_rows="${1-}" phase="${2-}" timestamp="${3-}"
[[ -n "$target_rows" && -f "$target_rows" && ! -L "$target_rows" &&
-n "$phase" && -n "$timestamp" ]] || { inventory_reject; return; }
jq -cnS --slurpfile targets "$target_rows" \
--arg schema "$INVENTORY_SCHEMA" --arg phase "$phase" --arg captured_at_utc "$timestamp" \
--arg context "$INVENTORY_CONTEXT" --arg api_server "$INVENTORY_API_SERVER" \
--arg prometheus "$INVENTORY_PROMETHEUS_NAMESPACE/$INVENTORY_PROMETHEUS_NAME" '
{schema:$schema,phase:$phase,captured_at_utc:$captured_at_utc,context:$context,
api_server:$api_server,prometheus:$prometheus,preexisting_verify_only:["grafana"],
targets:($targets | sort_by(.scrape_pool))}
'
}
validate_post_substrate_query_body() {
local path="${1--}" kind="${2-}" expected expected_tls now
expected="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels] | sort_by(.job,.instance)')" || {
inventory_reject
return
}
expected_tls="$(post_substrate_expected_probe_targets | jq -cS '[.[].labels | select(.instance | startswith("https://"))] | sort_by(.job,.instance)')" || {
inventory_reject
return
}
now="$(date -u +%s)" || { inventory_reject; return; }
if ! jq -e --arg kind "$kind" --argjson expected "$expected" \
--argjson expectedTls "$expected_tls" --argjson now "$now" '
type == "object" and .status == "success" and .data.resultType == "vector" and
(.data.result | type == "array") and
(if $kind == "tls" then
([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expectedTls and
all(.data.result[]; (.value[1] | tonumber) > $now)
else
([.data.result[].metric | del(.__name__)] | sort_by(.job,.instance)) == $expected and
(if $kind == "success" then all(.data.result[]; .value[1] == "1")
elif $kind == "duration" then all(.data.result[]; (.value[1] | tonumber) >= 0)
elif $kind == "status" then
all(.data.result[];
if .metric.job == "blackbox-private-edge" then .value[1] == "403"
else .value[1] == "200" end)
else false end)
end)
' "$path" >/dev/null 2>&1; then
inventory_reject
return
fi
}
inventory_validate_post_substrate_metrics() {
local body spec
for spec in \
'success:probe_success{namespace="observability"}' \
'status:probe_http_status_code{namespace="observability"}' \
'duration:probe_duration_seconds{namespace="observability"}' \
'tls:probe_ssl_earliest_cert_expiry{namespace="observability"}'; do
if ! inventory_prometheus_json body '/api/v1/query' --get \
--data-urlencode "query=${spec#*:}" ||
! printf '%s' "$body" | validate_post_substrate_query_body - "${spec%%:*}"; then
inventory_reject
return
fi
body=''
done
}
inventory_capture_main() {
local output_dir phase context api prometheus_json pods_json permission
local port='' attempt start_attempt forward_ready=0 timestamp snapshot
local targets_raw='' dropped_raw='' rules_raw='' probe_raw='' metadata_raw='' series_raw='' recording_names=''
local row pool scrape_url labels selector metric_names metric
local metric_labels series_selector series_count metrics_path metrics_json provenance
local target_rows counter=0 inventory hash actual_files
if (($# != 5)) || [[ "$1" != --execute || "$2" != --phase ||
("$3" != target-initial && "$3" != post-substrate) ||
"$4" != --output-dir || -z "$5" ]]; then
inventory_reject
return
fi
phase="$3"
output_dir="$5"
for permission in jq kubectl curl timeout realpath stat find mktemp sha256sum sync grep head; do
command -v "$permission" >/dev/null 2>&1 || { inventory_reject; return; }
done
validate_output_directory "$output_dir" || return
INVENTORY_WORK_DIR="$(mktemp -d /tmp/platform-observability-inventory-capture.XXXXXX)" || {
inventory_reject
return
}
chmod 0700 "$INVENTORY_WORK_DIR" || { inventory_reject; return; }
INVENTORY_STAGE_JSON=''
INVENTORY_STAGE_SHA=''
INVENTORY_OUTPUT_DIR="$output_dir"
INVENTORY_CREATED_JSON=0
INVENTORY_CREATED_SHA=0
INVENTORY_JSON_ID=''
INVENTORY_SHA_ID=''
INVENTORY_SUCCESS=0
INVENTORY_PORT_FORWARD_PID=''
INVENTORY_PORT_FORWARD_START=''
trap inventory_capture_cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
if ! context="$(inventory_kubectl config current-context)" || [[ "$context" != "$INVENTORY_CONTEXT" ]]; then
inventory_reject
return
fi
if ! api="$(inventory_kubectl config view --minify -o json | jq -er '.clusters | if length == 1 then .[0].cluster.server else error("cluster count") end' 2>/dev/null)" ||
[[ "$api" != "$INVENTORY_API_SERVER" ]]; then
inventory_reject
return
fi
if ! inventory_kubectl get --raw=/readyz >/dev/null; then inventory_reject; return; fi
for permission in \
'get pods' \
'get services' \
'get prometheuses.monitoring.coreos.com' \
'create pods/portforward'; do
if ! inventory_kubectl auth can-i $permission --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" | grep -Fxq yes; then
inventory_reject
return
fi
done
if ! prometheus_json="$(inventory_kubectl get prometheus "$INVENTORY_PROMETHEUS_NAME" \
--namespace "$INVENTORY_PROMETHEUS_NAMESPACE" -o json)" ||
! jq -e --arg namespace "$INVENTORY_PROMETHEUS_NAMESPACE" --arg name "$INVENTORY_PROMETHEUS_NAME" '
.kind == "Prometheus" and .metadata.namespace == $namespace and .metadata.name == $name
' <<<"$prometheus_json" >/dev/null 2>&1; then
inventory_reject
return
fi
if ! pods_json="$(inventory_kubectl get pods --namespace "$INVENTORY_PROMETHEUS_NAMESPACE" \
-l "prometheus=$INVENTORY_PROMETHEUS_NAME" -o json)" ||
! jq -e '
(.items | length == 1) and .items[0].status.phase == "Running" and
any(.items[0].status.conditions[]; .type == "Ready" and .status == "True") and
any(.items[0].status.containerStatuses[]; .name == "prometheus" and .ready == true)
' <<<"$pods_json" >/dev/null 2>&1; then
inventory_reject
return
fi
: >"$INVENTORY_WORK_DIR/port-forward.log"
for ((start_attempt = 0; start_attempt < 10; start_attempt++)); do
port=$((20000 + ((RANDOM + $$ + start_attempt * 7919) % 40000)))
kubectl --request-timeout=10s -n "$INVENTORY_PROMETHEUS_NAMESPACE" \
port-forward --address 127.0.0.1 \
"service/$INVENTORY_PROMETHEUS_SERVICE" "$port:9090" \
>/dev/null 2>"$INVENTORY_WORK_DIR/port-forward.log" &
INVENTORY_PORT_FORWARD_PID=$!
snapshot="$(inventory_process_snapshot "$INVENTORY_PORT_FORWARD_PID")" || {
inventory_reject
return
}
INVENTORY_PORT_FORWARD_START="${snapshot#*:}"
INVENTORY_PROMETHEUS_BASE="http://127.0.0.1:$port"
for ((attempt = 0; attempt < 40; attempt++)); do
if ! kill -0 "$INVENTORY_PORT_FORWARD_PID" 2>/dev/null; then break; fi
if inventory_prometheus_ready; then
forward_ready=1
break 2
fi
sleep 0.05
done
inventory_stop_port_forward || { inventory_reject; return; }
done
if [[ "$forward_ready" != 1 || -z "$INVENTORY_PORT_FORWARD_PID" ]]; then
inventory_reject
return
fi
if ! inventory_prometheus_json targets_raw '/api/v1/targets' --get --data 'state=active' ||
! printf '%s' "$targets_raw" | validate_targets_json - "$phase" ||
! inventory_prometheus_json dropped_raw '/api/v1/targets' --get --data 'state=dropped' ||
! printf '%s' "$dropped_raw" | validate_dropped_targets_json - "$phase" ||
! inventory_prometheus_json rules_raw '/api/v1/rules' ||
! recording_names="$(printf '%s' "$rules_raw" | extract_recording_rule_names -)" ||
! inventory_prometheus_json probe_raw '/api/v1/series' --get \
--data-urlencode 'match[]={__name__=~"probe_.+"}' ||
! printf '%s' "$probe_raw" | validate_probe_series_phase - "$phase"; then
inventory_reject
return
fi
if [[ "$phase" == post-substrate ]] && ! inventory_validate_post_substrate_metrics; then
inventory_reject
return
fi
dropped_raw=''
rules_raw=''
probe_raw=''
target_rows="$INVENTORY_WORK_DIR/target-rows.jsonl"
: >"$target_rows"
while IFS= read -r row; do
pool="$(jq -er '.scrapePool' <<<"$row" 2>/dev/null)" || { inventory_reject; return; }
scrape_url="$(jq -er '.scrapeUrl' <<<"$row" 2>/dev/null)" || { inventory_reject; return; }
if [[ "$phase" == post-substrate && "$pool" == probe/observability/* ]]; then
scrape_url="$(sanitize_scrape_url "$scrape_url" strip-query)" || return
else
scrape_url="$(sanitize_scrape_url "$scrape_url")" || return
fi
labels="$(jq -ceS '.labels' <<<"$row" 2>/dev/null)" || { inventory_reject; return; }
selector="$(prometheus_selector_for_labels "$labels")" || return
counter=$((counter + 1))
if ! inventory_prometheus_json metadata_raw '/api/v1/targets/metadata' --get \
--data-urlencode "match_target=$selector" --data 'limit=10000'; then
inventory_reject
return
fi
metric_names="$(printf '%s' "$metadata_raw" | extract_metric_names -)" || return
metadata_raw=''
reject_metric_name_collisions "$recording_names" "$metric_names" || return
validate_metric_names_for_pool "$metric_names" "$pool" "$phase" || return
metrics_path="$INVENTORY_WORK_DIR/metrics-$counter.jsonl"
: >"$metrics_path"
while IFS= read -r metric; do
series_selector="$(prometheus_selector_for_labels "$(
jq -ce --arg metric "$metric" '. + {"__name__":$metric}' <<<"$labels" 2>/dev/null
)")" || return
if ! inventory_prometheus_json series_raw '/api/v1/series' --get \
--data-urlencode "match[]=$series_selector"; then
inventory_reject
return
fi
series_count="$(printf '%s' "$series_raw" | jq -er '
if type == "object" and .status == "success" and
(.data | type == "array") and all(.data[]; type == "object")
then .data | length else error("invalid series") end
' 2>/dev/null)" || { inventory_reject; return; }
if (( series_count == 0 )); then
series_raw=''
continue
fi
metric_labels="$(printf '%s' "$series_raw" | extract_series_label_names -)" || return
series_raw=''
jq -cnS --arg name "$metric" --argjson label_names "$metric_labels" \
'{name:$name,label_names:$label_names}' >>"$metrics_path" || { inventory_reject; return; }
done <<<"$metric_names"
[[ -s "$metrics_path" ]] || { inventory_reject; return; }
metrics_json="$(jq -csS 'sort_by(.name)' "$metrics_path" 2>/dev/null)" || { inventory_reject; return; }
provenance="$(inventory_provenance "$pool")"
jq -cnS --arg scrape_pool "$pool" --arg scrape_url "$scrape_url" \
--arg provenance "$provenance" --argjson metrics "$metrics_json" \
'{scrape_pool:$scrape_pool,scrape_url:$scrape_url,health:"up",last_error:"",
provenance:$provenance,metrics:$metrics}' >>"$target_rows" || { inventory_reject; return; }
done < <(printf '%s' "$targets_raw" | jq -c '.data.activeTargets | sort_by(.scrapePool)[]')
targets_raw=''
timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" || { inventory_reject; return; }
inventory="$(build_inventory_json "$target_rows" "$phase" "$timestamp")" || return
validate_output_directory "$output_dir" || return
INVENTORY_STAGE_JSON="$(mktemp "$output_dir/.inventory.json.tmp.XXXXXX")" || { inventory_reject; return; }
chmod 0600 "$INVENTORY_STAGE_JSON" || { inventory_reject; return; }
printf '%s\n' "$inventory" >"$INVENTORY_STAGE_JSON" || { inventory_reject; return; }
sync -f "$INVENTORY_STAGE_JSON" >/dev/null 2>&1 || { inventory_reject; return; }
INVENTORY_JSON_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_JSON")" || { inventory_reject; return; }
hash="$(sha256sum -- "$INVENTORY_STAGE_JSON" | awk '{print $1}')" || { inventory_reject; return; }
INVENTORY_STAGE_SHA="$(mktemp "$output_dir/.inventory.sha256.tmp.XXXXXX")" || { inventory_reject; return; }
chmod 0600 "$INVENTORY_STAGE_SHA" || { inventory_reject; return; }
printf '%s inventory.json\n' "$hash" >"$INVENTORY_STAGE_SHA" || { inventory_reject; return; }
sync -f "$INVENTORY_STAGE_SHA" >/dev/null 2>&1 || { inventory_reject; return; }
INVENTORY_SHA_ID="$(stat -c %d:%i -- "$INVENTORY_STAGE_SHA")" || { inventory_reject; return; }
if ! ln -- "$INVENTORY_STAGE_JSON" "$output_dir/inventory.json" 2>/dev/null; then inventory_reject; return; fi
INVENTORY_CREATED_JSON=1
rm -- "$INVENTORY_STAGE_JSON" || { inventory_reject; return; }
INVENTORY_STAGE_JSON=''
if ! ln -- "$INVENTORY_STAGE_SHA" "$output_dir/inventory.sha256" 2>/dev/null; then inventory_reject; return; fi
INVENTORY_CREATED_SHA=1
rm -- "$INVENTORY_STAGE_SHA" || { inventory_reject; return; }
INVENTORY_STAGE_SHA=''
sync -f "$output_dir/inventory.json" >/dev/null 2>&1 || { inventory_reject; return; }
sync -f "$output_dir/inventory.sha256" >/dev/null 2>&1 || { inventory_reject; return; }
sync -f "$output_dir" >/dev/null 2>&1 || { inventory_reject; return; }
actual_files="$(find "$output_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)"
if [[ "$actual_files" != $'inventory.json\ninventory.sha256' ]] ||
[[ ! -f "$output_dir/inventory.json" || -L "$output_dir/inventory.json" ]] ||
[[ ! -f "$output_dir/inventory.sha256" || -L "$output_dir/inventory.sha256" ]] ||
[[ "$(stat -c %a -- "$output_dir/inventory.json")" != 600 ]] ||
[[ "$(stat -c %a -- "$output_dir/inventory.sha256")" != 600 ]]; then
inventory_reject
return
fi
INVENTORY_SUCCESS=1
printf 'OK: %s inventory captured\n' "$phase"
}
inventory_prometheus_json() {
local destination_name="$1" api_path="$2" response
local LC_ALL=C
local -n destination="$destination_name"
shift 2
if ! response="$(
set -o pipefail
curl --disable --silent --show-error --fail --connect-timeout 3 --max-time 10 \
--max-filesize "$INVENTORY_MAX_RESPONSE_BYTES" --output - \
"$@" "$INVENTORY_PROMETHEUS_BASE$api_path" 2>/dev/null |
LC_ALL=C head -c "$((INVENTORY_MAX_RESPONSE_BYTES + 1))"
pipeline_status=$?
printf '\034'
exit "$pipeline_status"
)"; then
response=''
inventory_reject
return
fi
[[ "$response" == *$'\034' ]] || { response=''; inventory_reject; return; }
response="${response%?}"
if (( ${#response} > INVENTORY_MAX_RESPONSE_BYTES )); then
response=''
inventory_reject
return
fi
destination="$response"
}
inventory_prometheus_ready() {
curl --disable --silent --show-error --fail --connect-timeout 1 --max-time 1 \
--max-filesize 1024 --output /dev/null "$INVENTORY_PROMETHEUS_BASE/-/ready" 2>/dev/null
}
inventory_capture_cleanup() {
local rc=$?
trap - EXIT HUP INT TERM
if [[ -n "${INVENTORY_PORT_FORWARD_PID:-}" ]]; then
inventory_stop_port_forward || true
fi
if [[ -n "${INVENTORY_STAGE_JSON:-}" && -f "$INVENTORY_STAGE_JSON" && ! -L "$INVENTORY_STAGE_JSON" ]]; then
rm -- "$INVENTORY_STAGE_JSON"
fi
if [[ -n "${INVENTORY_STAGE_SHA:-}" && -f "$INVENTORY_STAGE_SHA" && ! -L "$INVENTORY_STAGE_SHA" ]]; then
rm -- "$INVENTORY_STAGE_SHA"
fi
if [[ "${INVENTORY_SUCCESS:-0}" != 1 ]]; then
if [[ "${INVENTORY_CREATED_SHA:-0}" == 1 ]]; then
inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.sha256 "${INVENTORY_SHA_ID:-}"
fi
if [[ "${INVENTORY_CREATED_JSON:-0}" == 1 ]]; then
inventory_remove_created_file "${INVENTORY_OUTPUT_DIR:-}" inventory.json "${INVENTORY_JSON_ID:-}"
fi
if [[ -n "${INVENTORY_OUTPUT_DIR:-}" && -d "$INVENTORY_OUTPUT_DIR" && ! -L "$INVENTORY_OUTPUT_DIR" ]]; then
sync -f "$INVENTORY_OUTPUT_DIR" >/dev/null 2>&1 || true
fi
fi
if [[ -n "${INVENTORY_WORK_DIR:-}" && -d "$INVENTORY_WORK_DIR" && ! -L "$INVENTORY_WORK_DIR" ]]; then
case "$INVENTORY_WORK_DIR" in
/tmp/platform-observability-inventory-capture.*) rm -rf -- "$INVENTORY_WORK_DIR" ;;
esac
fi
exit "$rc"
}
inventory_remove_created_file() {
local directory="${1-}" basename="${2-}" expected_id="${3-}" path actual_id canonical
[[ -n "$directory" && -n "$basename" && -n "$expected_id" && -d "$directory" && ! -L "$directory" ]] || return 0
canonical="$(realpath -e -- "$directory" 2>/dev/null)" || return 0
[[ "$canonical" == "$directory" ]] || return 0
path="$directory/$basename"
[[ -f "$path" && ! -L "$path" ]] || return 0
actual_id="$(stat -c %d:%i -- "$path" 2>/dev/null)" || return 0
[[ "$actual_id" == "$expected_id" ]] || return 0
rm -- "$path"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
set -Eeuo pipefail
umask 077
inventory_capture_main "$@"
fi
@@ -0,0 +1,431 @@
#!/usr/bin/env bash
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
LC_ALL=C
export LC_ALL
umask 077
set -Eeuo pipefail
_k3slrf_initial_guard() {
local candidate_euid="${1-}" shell_flags="${2-}"
[[ "$candidate_euid" =~ ^[0-9]+$ ]] || return 1
(( candidate_euid != 0 )) || return 1
[[ "$shell_flags" != *x* ]]
}
if ! _k3slrf_initial_guard "$EUID" "$-"; then
printf 'Local recovery feasibility refused\n' >&2
if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then return 1; else exit 1; fi
fi
readonly K3SLRF_KEEPASSXC='/usr/bin/keepassxc-cli'
readonly K3SLRF_CRYPTSETUP='/usr/sbin/cryptsetup'
readonly K3SLRF_KEEPASS_PACKAGE='keepassxc'
readonly K3SLRF_KEEPASS_VERSION='2.7.6+dfsg.1-1build3'
readonly K3SLRF_CRYPTSETUP_PACKAGE='cryptsetup-bin'
readonly K3SLRF_CRYPTSETUP_VERSION='2:2.7.0-1ubuntu4.2'
readonly K3SLRF_SCRIPT_DIR="$(cd -P -- "${BASH_SOURCE[0]%/*}" && pwd -P)"
readonly K3SLRF_REPOSITORY_ROOT="$(cd -P -- "${K3SLRF_SCRIPT_DIR}/../.." && pwd -P)"
readonly K3SLRF_LIBRARY_PATH="${K3SLRF_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh"
# shellcheck source=../lib/k3s-local-recovery.sh
builtin source -- "$K3SLRF_LIBRARY_PATH" || {
printf 'Local recovery feasibility refused\n' >&2
if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then return 1; else exit 1; fi
}
_k3slrf_usage() {
printf 'Usage: bash scripts/validate/k3s-local-recovery-feasibility.sh [--execute]\n'
}
_k3slrf_fail() {
printf 'Local recovery feasibility failed\n' >&2
return 1
}
_k3slrf_require_interactive_stdin() {
[[ -t 0 ]]
}
_k3slrf_require_cached_sudo() {
_k3slr_cached_sudo_is_available
}
_k3slrf_package_version_is_exact() {
local package="$1" expected="$2" output
output="$(
_k3slr_command /usr/bin/dpkg-query --show --showformat='${db:Status-Abbrev}|${Version}\n' -- "$package" 2>/dev/null
printf '\034%s' "$?"
)" || return 1
[[ "$output" == "ii |${expected}"$'\n'$'\0340' ]]
}
_k3slrf_verify_prerequisites() {
_k3slr_trusted_root_executable /usr/bin/dpkg-query || return 1
_k3slr_trusted_root_executable /usr/bin/mktemp || return 1
_k3slr_trusted_root_executable /usr/bin/rm || return 1
_k3slr_trusted_root_executable /usr/bin/cmp || return 1
_k3slr_trusted_root_executable /usr/bin/mawk || return 1
_k3slr_trusted_root_executable /usr/bin/od || return 1
_k3slr_trusted_root_executable /usr/bin/ps || return 1
_k3slr_trusted_root_executable /usr/bin/sleep || return 1
_k3slr_trusted_root_executable /usr/bin/sha256sum || return 1
_k3slr_trusted_root_executable "$K3SLRF_KEEPASSXC" || return 1
_k3slr_trusted_root_executable "$K3SLRF_CRYPTSETUP" || return 1
_k3slrf_package_version_is_exact "$K3SLRF_KEEPASS_PACKAGE" "$K3SLRF_KEEPASS_VERSION" || return 1
_k3slrf_package_version_is_exact "$K3SLRF_CRYPTSETUP_PACKAGE" "$K3SLRF_CRYPTSETUP_VERSION"
}
_k3slrf_create_fixture_dir() {
local destination_name="$1" identity_name="$2" owned_name="$3" created physical metadata uid
created="$(_k3slr_command /usr/bin/mktemp -d /tmp/k3slr-feasibility.XXXXXX)" || return 1
printf -v "$destination_name" '%s' "$created"
printf -v "$owned_name" '%s' true
[[ "$created" =~ ^/tmp/k3slr-feasibility\.[A-Za-z0-9]+$ ]] || return 1
_k3slr_command /usr/bin/test -d "$created" || return 1
_k3slr_command /usr/bin/test ! -L "$created" || return 1
physical="$(cd -P -- "$created" && pwd -P)" || return 1
[[ "$physical" == "$created" ]] || return 1
_k3slr_capture_one_line uid /usr/bin/id -u || return 1
_k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$created" || return 1
[[ "$metadata" == "${uid}|700|directory|"* ]] || return 1
printf -v "$identity_name" '%s' "${metadata##*|}"
}
_k3slrf_cleanup_fixture_dir() {
local fixture_dir="$1" expected_identity="$2" metadata uid
[[ "$fixture_dir" =~ ^/tmp/k3slr-feasibility\.[A-Za-z0-9]+$ ]] || return 1
_k3slr_command /usr/bin/test -d "$fixture_dir" || return 1
_k3slr_command /usr/bin/test ! -L "$fixture_dir" || return 1
_k3slr_capture_one_line uid /usr/bin/id -u || return 1
_k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$fixture_dir" 2>/dev/null || return 1
[[ "$metadata" == "${uid}|700|directory|"* ]] || return 1
[[ -z "$expected_identity" || "${metadata##*|}" == "$expected_identity" ]] || return 1
_k3slr_command /usr/bin/rm --recursive --one-file-system -- "$fixture_dir"
}
_k3slrf_synthetic_master_once_stdout() {
printf '%s' 'k3slr-package-feasibility-master-v1' |
_k3slr_command /usr/bin/sha256sum |
_k3slr_command /usr/bin/mawk '{ print $1 }'
}
_k3slrf_synthetic_master_twice_stdout() {
_k3slrf_synthetic_master_once_stdout
_k3slrf_synthetic_master_once_stdout
}
_k3slrf_validate_generated_password_stdin() {
_k3slr_command /usr/bin/od -An -v -tx1 |
_k3slr_command /usr/bin/mawk '
{
for (i=1; i<=NF; i++) {
n++; byte=$i
if (n == 41) { if (byte != "0a") bad=1 }
else if (byte ~ /^3[0-9]$/) digit=1
else if (byte ~ /^(4[1-9a-f]|5[0-9a])$/) upper=1
else if (byte ~ /^(6[1-9a-f]|7[0-9a])$/) lower=1
else bad=1
}
}
END { exit !(n == 41 && !bad && lower && upper && digit) }
'
}
_k3slrf_file_identity() {
local destination_name="$1" path="$2" metadata uid
_k3slr_command /usr/bin/test -f "$path" || return 1
_k3slr_command /usr/bin/test ! -L "$path" || return 1
_k3slr_capture_one_line uid /usr/bin/id -u || return 1
_k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$path" || return 1
[[ "$metadata" == "${uid}|600|regular file|"* ||
"$metadata" == "${uid}|600|regular empty file|"* ]] || return 1
printf -v "$destination_name" '%s' "${metadata##*|}"
}
_k3slrf_file_identity_matches() {
local current=''
_k3slrf_file_identity current "$1" || return 1
[[ "$current" == "$2" ]]
}
_k3slrf_directory_identity() {
local destination_name="$1" path="$2" physical metadata uid
[[ "$path" =~ ^/tmp/k3slr-feasibility\.[A-Za-z0-9]+$ ]] || return 1
_k3slr_command /usr/bin/test -d "$path" || return 1
_k3slr_command /usr/bin/test ! -L "$path" || return 1
physical="$(cd -P -- "$path" && pwd -P)" || return 1
[[ "$physical" == "$path" ]] || return 1
_k3slr_capture_one_line uid /usr/bin/id -u || return 1
_k3slr_capture_one_line metadata /usr/bin/stat --format='%u|%a|%F|%d:%i' -- "$path" || return 1
[[ "$metadata" == "${uid}|700|directory|"* ]] || return 1
printf -v "$destination_name" '%s' "${metadata##*|}"
}
_k3slrf_directory_identity_matches() {
local current=''
_k3slrf_directory_identity current "$1" || return 1
[[ "$current" == "$2" ]]
}
_k3slrf_run_tracked() {
local rc caller_group='' current_identity='' stopped_identity=''
_k3slr_process_group_for_pid caller_group "$BASHPID" || return 1
K3SLRF_ACTIVE_CHILD_STATE='launching'
K3SLRF_LAST_CLEANUP_OK=false
_k3slr_lifecycle_boundary feasibility-launch-before
_k3slrf_pending_checkpoint || return $?
set -m
(
tracked_cancel=''
trap '[[ "$tracked_cancel" == TERM ]] || tracked_cancel=INT' INT
trap 'tracked_cancel=TERM' TERM
kill -STOP "$BASHPID"
tracked_worker_rc=143
if [[ -z "$tracked_cancel" ]]; then
if "$@"; then tracked_worker_rc=0; else tracked_worker_rc=$?; fi
fi
while [[ -n "$tracked_cancel" ]]; do
_k3slr_command /usr/bin/sleep 0.01 || :
done
exit "$tracked_worker_rc"
) &
K3SLRF_ACTIVE_CHILD_PID=$!
K3SLRF_ACTIVE_CHILD_STATE='published'
_k3slr_lifecycle_boundary feasibility-pid-published "$K3SLRF_ACTIVE_CHILD_PID"
if ! _k3slr_process_identity K3SLRF_ACTIVE_CHILD_IDENTITY "$K3SLRF_ACTIVE_CHILD_PID"; then
set +m
_k3slrf_cleanup_active_child || return 1
if [[ -n "$K3SLRF_PENDING_SIGNAL" ]]; then return "$K3SLRF_PENDING_STATUS"; fi
return 1
fi
if ! _k3slr_wait_for_stopped_group K3SLRF_ACTIVE_CHILD_PGID stopped_identity "$K3SLRF_ACTIVE_CHILD_PID" ||
[[ "$stopped_identity" != "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]]; then
set +m
_k3slrf_cleanup_active_child || return 1
if [[ -n "$K3SLRF_PENDING_SIGNAL" ]]; then return "$K3SLRF_PENDING_STATUS"; fi
return 1
fi
K3SLRF_ACTIVE_CHILD_STATE='stopped_pinned'
_k3slr_lifecycle_boundary feasibility-stop-query-complete
_k3slrf_pending_checkpoint || { rc=$?; set +m; return "$rc"; }
if [[ "$K3SLRF_ACTIVE_CHILD_PGID" == "$caller_group" ]]; then
set +m
_k3slrf_cleanup_active_child || return 1
return 1
fi
set +m
_k3slr_lifecycle_boundary feasibility-cont-before
_k3slrf_pending_checkpoint || return $?
_k3slr_continue_owned_child "$K3SLRF_ACTIVE_CHILD_PID" "$K3SLRF_ACTIVE_CHILD_PGID" \
"$K3SLRF_ACTIVE_CHILD_IDENTITY" || return 1
K3SLRF_ACTIVE_CHILD_STATE='continued'
_k3slr_lifecycle_boundary feasibility-cont-after
_k3slrf_pending_checkpoint || return $?
K3SLRF_ACTIVE_CHILD_STATE='waiting'
if _k3slr_wait_child "$K3SLRF_ACTIVE_CHILD_PID"; then rc=0; else rc=$?; fi
_k3slr_lifecycle_boundary feasibility-wait-reaped-before-clear
if _k3slr_process_identity current_identity "$K3SLRF_ACTIVE_CHILD_PID" 2>/dev/null &&
[[ "$current_identity" == "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]]; then
_k3slrf_cleanup_active_child || return 1
fi
K3SLRF_ACTIVE_CHILD_STATE='reaped' K3SLRF_ACTIVE_CHILD_PID='' \
K3SLRF_ACTIVE_CHILD_PGID='' K3SLRF_ACTIVE_CHILD_IDENTITY=''
K3SLRF_LAST_CLEANUP_OK=true
_k3slr_lifecycle_boundary feasibility-state-cleared
if [[ -n "$K3SLRF_PENDING_SIGNAL" ]]; then
rc="$K3SLRF_PENDING_STATUS"
fi
return "$rc"
}
_k3slrf_db_create() {
_k3slrf_synthetic_master_twice_stdout |
_k3slr_command "$K3SLRF_KEEPASSXC" db-create --quiet --set-password "$1" >/dev/null 2>&1
}
_k3slrf_entry_add() {
_k3slrf_synthetic_master_once_stdout |
_k3slr_command "$K3SLRF_KEEPASSXC" add --quiet --generate --length 40 \
--lower --upper --numeric --every-group "$1" "$K3SLR_KEEPASS_ENTRY" >/dev/null 2>&1
}
_k3slrf_password_check() {
_k3slrf_synthetic_master_once_stdout |
_k3slr_command "$K3SLRF_KEEPASSXC" show --show-protected \
--attributes Password "$1" "$K3SLR_KEEPASS_ENTRY" 2>/dev/null |
_k3slrf_validate_generated_password_stdin
}
_k3slrf_attachment_listing_stdout() {
(( $# == 1 )) || return 1
_k3slrf_synthetic_master_once_stdout |
_k3slr_command "$K3SLRF_KEEPASSXC" show --quiet --attributes Title \
--show-attachments "$1" "$K3SLR_KEEPASS_ENTRY"
}
_k3slrf_attachment_state_is() {
local database="${1-}" fixture_dir="${2-}" expected_state="${3-}"
local capture_base='' record='' state=''
(( $# == 3 )) || return 1
[[ "$expected_state" == absent || "$expected_state" == present ]] || return 1
capture_base="${fixture_dir}/keepass-listing.${BASHPID}"
_k3slr_capture_keepass_attachment_listing record \
"${capture_base}.stdout" "${capture_base}.stderr" \
_k3slrf_attachment_listing_stdout "$database" || return 1
_k3slr_parse_keepass_attachment_listing "$record" fixture.bin state || return 1
[[ "$state" == "$expected_state" ]]
}
_k3slrf_attachment_import() {
_k3slrf_synthetic_master_once_stdout |
_k3slr_command "$K3SLRF_KEEPASSXC" attachment-import --quiet "$1" "$K3SLR_KEEPASS_ENTRY" \
fixture.bin "$2" >/dev/null 2>&1
}
_k3slrf_attachment_export() {
_k3slrf_synthetic_master_once_stdout |
_k3slr_command "$K3SLRF_KEEPASSXC" attachment-export --quiet "$1" "$K3SLR_KEEPASS_ENTRY" \
fixture.bin "$2" >/dev/null 2>&1
}
_k3slrf_run_synthetic() {
local fixture_dir="$1" database input_file output_file database_identity input_identity output_identity fixture_identity export_fd
database="${fixture_dir}/synthetic.kdbx"
input_file="${fixture_dir}/input.bin"
output_file="${fixture_dir}/output.bin"
[[ ! -e "$database" && ! -L "$database" && ! -e "$input_file" && ! -L "$input_file" &&
! -e "$output_file" && ! -L "$output_file" ]] || return 1
_k3slrf_directory_identity fixture_identity "$fixture_dir" || return 1
( set -o noclobber; printf '%s\n' 'k3slr-attachment-roundtrip-v1' >"$input_file" ) || return 1
_k3slrf_file_identity input_identity "$input_file" || return 1
_k3slrf_run_tracked _k3slrf_db_create "$database" || return 1
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity database_identity "$database" || return 1
_k3slrf_run_tracked _k3slrf_entry_add "$database" || return 1
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity database_identity "$database" || return 1
_k3slrf_run_tracked _k3slrf_password_check "$database" || return 1
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity_matches "$database" "$database_identity" || return 1
_k3slrf_file_identity_matches "$input_file" "$input_identity" || return 1
_k3slrf_run_tracked _k3slrf_attachment_state_is "$database" "$fixture_dir" absent || return 1
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity_matches "$database" "$database_identity" || return 1
_k3slrf_file_identity_matches "$input_file" "$input_identity" || return 1
_k3slrf_run_tracked _k3slrf_attachment_import "$database" "$input_file" || return 1
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity database_identity "$database" || return 1
_k3slrf_run_tracked _k3slrf_attachment_state_is "$database" "$fixture_dir" present || return 1
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity_matches "$database" "$database_identity" || return 1
set -o noclobber
if ! exec {export_fd}>"$output_file"; then
set +o noclobber
return 1
fi
set +o noclobber
_k3slrf_file_identity output_identity "$output_file" || return 1
_k3slrf_run_tracked _k3slrf_attachment_export "$database" "/proc/${BASHPID}/fd/${export_fd}" || return 1
exec {export_fd}>&-
_k3slrf_directory_identity_matches "$fixture_dir" "$fixture_identity" || return 1
_k3slrf_file_identity_matches "$database" "$database_identity" || return 1
_k3slrf_file_identity_matches "$output_file" "$output_identity" || return 1
_k3slr_command /usr/bin/cmp --silent -- "$input_file" "$output_file"
}
k3slr_local_recovery_feasibility_main() (
local execute=false fixture_dir='' fixture_identity='' fixture_owned=false
local K3SLRF_ACTIVE_CHILD_PID='' K3SLRF_ACTIVE_CHILD_PGID='' K3SLRF_ACTIVE_CHILD_IDENTITY=''
local K3SLRF_ACTIVE_CHILD_STATE='empty' K3SLRF_PENDING_SIGNAL='' K3SLRF_PENDING_STATUS=''
local K3SLRF_LAST_CLEANUP_OK=false
case "$#" in
0) ;;
1) [[ "$1" == --execute ]] || { _k3slrf_usage >&2; return 2; }; execute=true ;;
*) _k3slrf_usage >&2; return 2 ;;
esac
if "$execute"; then
_k3slrf_require_interactive_stdin || { _k3slrf_fail; return 1; }
fi
_k3slrf_verify_prerequisites || { _k3slrf_fail; return 1; }
if ! "$execute"; then
printf 'Local recovery feasibility prerequisites: pass\n'
return 0
fi
_k3slrf_require_cached_sudo || { _k3slrf_fail; return 1; }
_k3slrf_cleanup_active_child() {
local cleanup_rc=0
if [[ -n "$K3SLRF_ACTIVE_CHILD_PID" ]]; then
if [[ "$K3SLRF_ACTIVE_CHILD_STATE" == published ]]; then
_k3slr_terminate_published_child "$K3SLRF_ACTIVE_CHILD_PID" || cleanup_rc=$?
elif [[ -n "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]]; then
_k3slr_terminate_and_reap "$K3SLRF_ACTIVE_CHILD_PID" "$K3SLRF_ACTIVE_CHILD_PGID" \
"$K3SLRF_ACTIVE_CHILD_IDENTITY" || cleanup_rc=$?
else
cleanup_rc=1
fi
if (( cleanup_rc == 0 )); then
K3SLRF_ACTIVE_CHILD_PID=''
K3SLRF_ACTIVE_CHILD_PGID=''
K3SLRF_ACTIVE_CHILD_IDENTITY=''
K3SLRF_ACTIVE_CHILD_STATE='reaped'
K3SLRF_LAST_CLEANUP_OK=true
fi
fi
return "$cleanup_rc"
}
_k3slrf_signal() {
if [[ "$1" == TERM || "$K3SLRF_PENDING_SIGNAL" != TERM ]]; then
K3SLRF_PENDING_SIGNAL="$1"
K3SLRF_PENDING_STATUS="$2"
fi
}
_k3slrf_pending_checkpoint() {
local signal_status cleanup_rc=0
[[ -n "$K3SLRF_PENDING_SIGNAL" ]] || return 0
signal_status="$K3SLRF_PENDING_STATUS"
if [[ -n "$K3SLRF_ACTIVE_CHILD_PID" ]]; then
[[ -n "$K3SLRF_ACTIVE_CHILD_IDENTITY" ]] || return 1
_k3slrf_cleanup_active_child || cleanup_rc=$?
else
K3SLRF_LAST_CLEANUP_OK=true
fi
(( cleanup_rc == 0 )) || return 1
return "$signal_status"
}
_k3slrf_exit_cleanup() {
local original_status="$1" cleanup_rc=0
trap - EXIT
_k3slrf_cleanup_active_child || cleanup_rc=$?
if "$fixture_owned"; then
_k3slrf_cleanup_fixture_dir "$fixture_dir" "$fixture_identity" >/dev/null 2>&1 || cleanup_rc=1
fi
(( cleanup_rc == 0 )) || exit 1
exit "$original_status"
}
trap '_k3slrf_exit_cleanup "$?"' EXIT
trap '_k3slrf_signal INT 130' INT
trap '_k3slrf_signal TERM 143' TERM
_k3slrf_create_fixture_dir fixture_dir fixture_identity fixture_owned || { _k3slrf_fail; return 1; }
if ! _k3slrf_run_synthetic "$fixture_dir"; then
if [[ -n "$K3SLRF_PENDING_STATUS" && -z "$K3SLRF_ACTIVE_CHILD_PID" &&
"$K3SLRF_LAST_CLEANUP_OK" == true ]]; then
_k3slrf_cleanup_fixture_dir "$fixture_dir" "$fixture_identity" || return 1
fixture_owned=false
trap - EXIT INT TERM
return "$K3SLRF_PENDING_STATUS"
fi
_k3slrf_fail
return 1
fi
_k3slrf_cleanup_fixture_dir "$fixture_dir" "$fixture_identity" || { _k3slrf_fail; return 1; }
fixture_owned=false
trap - EXIT INT TERM
printf 'Local recovery feasibility: pass\n'
)
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
k3slr_local_recovery_feasibility_main "$@"
fi
+718
View File
@@ -0,0 +1,718 @@
#!/usr/bin/env bash
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
LC_ALL=C
export LC_ALL
umask 077
set -Eeuo pipefail
shopt -s extglob
_k3slrv_initial_guard() {
local candidate_euid="${1-}" shell_flags="${2-}"
[[ "$candidate_euid" =~ ^[0-9]+$ ]] || return 1
(( candidate_euid != 0 )) || return 1
[[ "$shell_flags" != *x* ]]
}
if ! _k3slrv_initial_guard "$EUID" "$-"; then
printf 'Recovery validation refused\n' >&2
if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then return 1; else exit 1; fi
fi
readonly K3SLRV_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin'
readonly K3SLRV_TRUSTED_BINARIES=(
/usr/bin/findmnt /usr/bin/lsblk /usr/bin/stat /usr/bin/readlink
/usr/sbin/blkid /usr/bin/ntfs-3g.probe /usr/sbin/smartctl
/usr/sbin/losetup /usr/sbin/cryptsetup /usr/bin/du
/usr/bin/sha256sum /usr/bin/test /usr/bin/sudo /usr/bin/mawk
)
readonly K3SLRV_SCRIPT_DIR="$(cd -P -- "${BASH_SOURCE[0]%/*}" && pwd -P)"
readonly K3SLRV_REPOSITORY_ROOT="$(cd -P -- "${K3SLRV_SCRIPT_DIR}/../.." && pwd -P)"
readonly K3SLRV_LIBRARY_PATH="${K3SLRV_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh"
readonly K3SLRV_CONTRACT_PATH="${K3SLRV_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env"
_k3slrv_usage() {
printf 'Usage: bash scripts/validate/k3s-local-recovery.sh [--expect-device-ready|--expect-closed|--expect-open] [--check-latest-bundle]\n'
}
_k3slrv_fail() {
printf 'Recovery validation failed\n' >&2
return 1
}
_k3slrv_exec() {
local scope="$1"
shift
case "$scope" in
user) "$@" ;;
root) /usr/bin/sudo --non-interactive -- "$@" ;;
*) return 1 ;;
esac
}
# Bash command substitution normally discards every trailing newline. Append a
# non-newline sentinel inside the substitution, then remove only that sentinel
# so record parsers can distinguish one record from a record plus blank lines.
_k3slrv_capture() {
local destination="$1" captured command_rc sentinel=$'\036'
shift
if captured="$(
set +e
"$@"
command_rc=$?
printf '\036'
exit "$command_rc"
)"; then
command_rc=0
else
command_rc=$?
fi
[[ "$captured" == *"$sentinel" ]] || return 125
captured="${captured%"$sentinel"}"
[[ "$captured" != *"$sentinel"* ]] || return 125
printf -v "$destination" '%s' "$captured"
return "$command_rc"
}
_k3slrv_normalize_one_record() {
local destination="$1" value="$2"
[[ -n "$value" && "$value" != *$'\r'* ]] || return 1
value="${value%$'\n'}"
[[ -n "$value" && "$value" != *$'\n'* ]] || return 1
printf -v "$destination" '%s' "$value"
}
_k3slrv_verify_binary() {
local binary="$1" metadata uid mode kind group_digit other_digit
[[ "$binary" == /* ]] || return 1
_k3slrv_exec user /usr/bin/test -f "$binary" >/dev/null 2>&1 || return 1
_k3slrv_exec user /usr/bin/test ! -L "$binary" >/dev/null 2>&1 || return 1
_k3slrv_capture metadata _k3slrv_exec user /usr/bin/stat --format='%u|%a|%F' -- "$binary" 2>/dev/null || return 1
_k3slrv_normalize_one_record metadata "$metadata" || return 1
IFS='|' read -r uid mode kind <<<"$metadata"
[[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$kind" == 'regular file' ]] || return 1
group_digit="${mode: -2:1}"
other_digit="${mode: -1}"
(( (10#$group_digit & 2) == 0 && (10#$other_digit & 2) == 0 ))
}
_k3slrv_verify_trusted_binaries() {
local binary
[[ "$PATH" == "$K3SLRV_SECURE_PATH" ]] || return 1
for binary in "${K3SLRV_TRUSTED_BINARIES[@]}"; do
_k3slrv_verify_binary "$binary" || return 1
done
}
_k3slrv_require_cached_sudo() {
_k3slrv_exec user /usr/bin/sudo --non-interactive --validate >/dev/null 2>&1
}
_k3slrv_root() {
_k3slrv_exec root "$@"
}
_k3slrv_trim() {
local value="$1"
value="${value##+([[:space:]])}"
value="${value%%+([[:space:]])}"
printf '%s\n' "$value"
}
_k3slrv_one_line() {
local output
_k3slrv_capture output _k3slrv_exec user "$@" 2>/dev/null || return 1
_k3slrv_normalize_one_record output "$output" || return 1
_k3slrv_trim "$output"
}
_k3slrv_one_line_root() {
local output
_k3slrv_capture output _k3slrv_root "$@" 2>/dev/null || return 1
_k3slrv_normalize_one_record output "$output" || return 1
_k3slrv_trim "$output"
}
_k3slrv_blkid() {
_k3slrv_one_line_root /usr/sbin/blkid --output value --match-tag "$1" -- "$2"
}
_k3slrv_lsblk() {
_k3slrv_one_line /usr/bin/lsblk --noheadings --nodeps --paths --output "$1" -- "$2"
}
_k3slrv_smart_evidence() {
local disk="$1" output line id raw
local -a fields=() lines=()
local health='' reallocated='' pending='' uncorrectable=''
_k3slrv_capture output _k3slrv_root /usr/sbin/smartctl -H -A -- "$disk" 2>/dev/null || return 1
mapfile -t lines < <(printf '%s' "$output")
(( ${#lines[@]} > 0 )) || return 1
for line in "${lines[@]}"; do
[[ "$line" != *$'\r'* ]] || return 1
if [[ "$line" =~ ^SMART[[:space:]]overall-health[[:space:]]self-assessment[[:space:]]test[[:space:]]result:[[:space:]]*PASSED[[:space:]]*$ ]]; then
[[ -z "$health" ]] || return 1
health=PASSED
continue
fi
[[ "$line" =~ ^[[:space:]]*(5|197|198)[[:space:]] ]] || continue
id="${BASH_REMATCH[1]}"
read -r -a fields <<<"$line"
(( ${#fields[@]} >= 2 )) || return 1
raw="${fields[${#fields[@]} - 1]}"
[[ "$raw" =~ ^[0-9]+$ ]] || return 1
case "$id" in
5) [[ -z "$reallocated" ]] || return 1; reallocated="$raw" ;;
197) [[ -z "$pending" ]] || return 1; pending="$raw" ;;
198) [[ -z "$uncorrectable" ]] || return 1; uncorrectable="$raw" ;;
esac
done
[[ -n "$health" && -n "$reallocated" && -n "$pending" && -n "$uncorrectable" ]] || return 1
printf 'smart_health=%s\nsmart_reallocated=%s\nsmart_pending=%s\nsmart_uncorrectable=%s\n' \
"$health" "$reallocated" "$pending" "$uncorrectable"
}
_k3slrv_findmnt_line() {
local output
_k3slrv_capture output _k3slrv_root /usr/bin/findmnt --noheadings --raw --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint "$1" 2>/dev/null || return 1
_k3slrv_normalize_one_record output "$output" || return 1
output="$(_k3slrv_trim "$output")"
[[ -n "$output" ]] || return 1
printf '%s\n' "$output"
}
_k3slrv_findmnt_optional() {
local output rc=0
_k3slrv_capture output _k3slrv_root /usr/bin/findmnt --noheadings --raw --output SOURCE,FSTYPE,OPTIONS,ID,MAJ:MIN --mountpoint "$1" 2>/dev/null || rc=$?
case "$rc" in
0)
_k3slrv_normalize_one_record output "$output" || return 1
output="$(_k3slrv_trim "$output")"
[[ -n "$output" ]] || return 1
printf '%s\n' "$output"
;;
1) return 0 ;;
*) return 1 ;;
esac
}
_k3slrv_findmnt_source_optional() {
local output rc=0
_k3slrv_capture output _k3slrv_root /usr/bin/findmnt --noheadings --raw --output TARGET --source "$1" 2>/dev/null || rc=$?
case "$rc" in
0)
_k3slrv_normalize_one_record output "$output" || return 1
output="$(_k3slrv_trim "$output")"
[[ -n "$output" ]] || return 1
printf '%s\n' "$output"
;;
1) return 0 ;;
*) return 1 ;;
esac
}
_k3slrv_collect_device_evidence() {
local mode="$1" recovery_partition recovery_disk recovery_parent k3s_partition k3s_disk k3s_parent
local recovery_model recovery_serial recovery_wwn recovery_mm k3s_model k3s_serial k3s_wwn k3s_mm
local recovery_uuid recovery_partuuid recovery_type k3s_uuid k3s_partuuid k3s_type outer_mount source_mounts
recovery_partition="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID")" || return 1
recovery_disk="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_RECOVERY_DISK_BY_ID")" || return 1
recovery_parent="$(_k3slrv_lsblk PKNAME "$recovery_partition")" || return 1
k3s_partition="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_K3S_PARTITION_BY_ID")" || return 1
k3s_disk="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_K3S_DISK_BY_ID")" || return 1
k3s_parent="$(_k3slrv_lsblk PKNAME "$k3s_partition")" || return 1
recovery_model="$(_k3slrv_lsblk MODEL "$recovery_disk")" || return 1
recovery_serial="$(_k3slrv_lsblk SERIAL "$recovery_disk")" || return 1
recovery_wwn="$(_k3slrv_lsblk WWN "$recovery_disk")" || return 1
recovery_mm="$(_k3slrv_lsblk MAJ:MIN "$recovery_disk")" || return 1
k3s_model="$(_k3slrv_lsblk MODEL "$k3s_disk")" || return 1
k3s_serial="$(_k3slrv_lsblk SERIAL "$k3s_disk")" || return 1
k3s_wwn="$(_k3slrv_lsblk WWN "$k3s_disk")" || return 1
k3s_mm="$(_k3slrv_lsblk MAJ:MIN "$k3s_disk")" || return 1
recovery_uuid="$(_k3slrv_blkid UUID "$recovery_partition")" || return 1
recovery_partuuid="$(_k3slrv_blkid PARTUUID "$recovery_partition")" || return 1
recovery_type="$(_k3slrv_blkid TYPE "$recovery_partition")" || return 1
k3s_uuid="$(_k3slrv_blkid UUID "$k3s_partition")" || return 1
k3s_partuuid="$(_k3slrv_blkid PARTUUID "$k3s_partition")" || return 1
k3s_type="$(_k3slrv_blkid TYPE "$k3s_partition")" || return 1
printf 'recovery_partition=%s\nrecovery_disk=%s\nrecovery_partition_parent=%s\n' \
"$recovery_partition" "$recovery_disk" "$recovery_parent"
printf 'recovery_fs_uuid=%s\nrecovery_partuuid=%s\nrecovery_type=%s\n' \
"$recovery_uuid" "$recovery_partuuid" "$recovery_type"
printf 'recovery_model=%s\nrecovery_serial=%s\nrecovery_wwn=%s\nrecovery_major_minor=%s\n' \
"$recovery_model" "$recovery_serial" "$recovery_wwn" "$recovery_mm"
printf 'k3s_partition=%s\nk3s_disk=%s\nk3s_partition_parent=%s\n' \
"$k3s_partition" "$k3s_disk" "$k3s_parent"
printf 'k3s_fs_uuid=%s\nk3s_partuuid=%s\nk3s_type=%s\n' \
"$k3s_uuid" "$k3s_partuuid" "$k3s_type"
printf 'k3s_model=%s\nk3s_serial=%s\nk3s_wwn=%s\nk3s_major_minor=%s\n' \
"$k3s_model" "$k3s_serial" "$k3s_wwn" "$k3s_mm"
_k3slrv_smart_evidence "$recovery_disk" || return 1
if [[ "$mode" == open ]]; then
printf 'ntfs_probe=not_applicable\n'
else
outer_mount="$(_k3slrv_findmnt_optional "$K3SLR_OUTER_MOUNT")" || return 1
[[ -z "$outer_mount" ]] || return 1
source_mounts="$(_k3slrv_findmnt_source_optional "$recovery_partition")" || return 1
[[ -z "$source_mounts" ]] || return 1
if _k3slrv_root /usr/bin/ntfs-3g.probe --readwrite "$recovery_partition" >/dev/null 2>&1; then
printf 'ntfs_probe=pass\n'
else
printf 'ntfs_probe=fail\n'
fi
fi
}
_k3slrv_mapping_evidence() {
local name="$1" prefix="$2" output type='' device='' rc=0
local line
local -a lines=()
_k3slrv_capture output _k3slrv_root /usr/sbin/cryptsetup status -- "$name" 2>/dev/null || rc=$?
if (( rc == 4 )); then
printf '%s_present=0\n' "$prefix"
return 0
fi
(( rc == 0 )) || return 1
mapfile -t lines < <(printf '%s' "$output")
(( ${#lines[@]} > 0 )) || return 1
for line in "${lines[@]}"; do
[[ "$line" != *$'\r'* ]] || return 1
line="$(_k3slrv_trim "$line")"
case "$line" in
type:*) [[ -z "$type" ]] || return 1; type="$(_k3slrv_trim "${line#type:}")" ;;
device:*) [[ -z "$device" ]] || return 1; device="$(_k3slrv_trim "${line#device:}")" ;;
esac
done
[[ -n "$type" && -n "$device" ]] || return 1
printf '%s_present=1\n%s_type=%s\n%s_device=%s\n' "$prefix" "$prefix" "$type" "$prefix" "$device"
}
_k3slrv_pin_container_chain() {
local container="$1" current="$K3SLR_OUTER_MOUNT" canonical metadata component index=0 snapshot=''
local -a components=()
IFS='/' read -r -a components <<<"$K3SLR_CONTAINER_RELATIVE"
for component in '' "${components[@]}"; do
if (( index > 0 )); then current="${current}/${component}"; fi
_k3slrv_root /usr/bin/test ! -L "$current" >/dev/null 2>&1 || return 1
canonical="$(_k3slrv_one_line_root /usr/bin/readlink -e -- "$current")" || return 1
[[ "$canonical" == "$current" ]] || return 1
metadata="$(_k3slrv_one_line_root /usr/bin/stat --format='%d:%i|%F' -- "$current")" || return 1
if (( index < ${#components[@]} )); then
[[ "$metadata" == *'|directory' ]] || return 1
else
[[ "$current" == "$container" && "$metadata" == *'|regular file' ]] || return 1
fi
snapshot+="${metadata%%|*},"
index=$((index + 1))
done
printf '%s\n' "${snapshot%,}"
}
_k3slrv_mapping_value() {
local evidence="$1" key="$2" line found=''
while IFS= read -r line; do
[[ "$line" == "$key="* ]] || continue
[[ -z "$found" ]] || return 1
found="${line#*=}"
done <<<"$evidence"
[[ -n "$found" ]] || return 1
printf '%s\n' "$found"
}
_k3slrv_collect_state_evidence() {
local mode="$1" outer='' inner='' loops='' outer_source outer_fstype outer_options outer_mount_id outer_major_minor
local inner_source inner_fstype inner_options inner_mount_id inner_major_minor container metadata kind uid gid file_mode size blocks allocated inode container_device
local inner_type inner_label recovery_partition source_mounts chain_snapshot loop_device loop_back_inode loop_back_major_minor
local loop_major_minor loop_offset loop_sizelimit mapping_snapshot proof_snapshot mapping_device mapping_loop_major_minor mapping_major_minor
local outer_after inner_after source_after chain_after loops_after mapping_after proof_after mapping_mm_after
local container_metadata_after inner_type_after inner_label_after inner_root_metadata inner_root_after
local mapping_device_after mapping_loop_mm_after
local -a loop_records=()
outer="$(_k3slrv_findmnt_optional "$K3SLR_OUTER_MOUNT")" || return 1
inner="$(_k3slrv_findmnt_optional "$K3SLR_INNER_MOUNT")" || return 1
container="${K3SLR_OUTER_MOUNT}/${K3SLR_CONTAINER_RELATIVE}"
if [[ "$mode" != open ]]; then
_k3slrv_capture loops _k3slrv_root /usr/sbin/losetup --associated "$container" --noheadings --output NAME 2>/dev/null || return 1
printf 'outer_mounted=%s\ninner_mounted=%s\n' "$([[ -n "$outer" ]] && printf 1 || printf 0)" "$([[ -n "$inner" ]] && printf 1 || printf 0)"
if [[ -z "$loops" ]]; then
printf 'loop_count=0\n'
else
mapfile -t loop_records < <(printf '%s' "$loops")
if (( ${#loop_records[@]} == 1 )) && [[ -n "${loop_records[0]}" && "${loop_records[0]}" != *$'\r'* ]]; then
printf 'loop_count=1\n'
else
printf 'loop_count=2\n'
fi
fi
_k3slrv_mapping_evidence "$K3SLR_MAPPING_NAME" mapping || return 1
_k3slrv_mapping_evidence "$K3SLR_PROOF_MAPPING_NAME" proof_mapping || return 1
return 0
fi
_k3slrv_capture loops _k3slrv_root /usr/sbin/losetup --list --associated "$container" --noheadings --raw \
--output NAME,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT 2>/dev/null || return 1
_k3slrv_normalize_one_record loops "$loops" || return 1
[[ -n "$outer" && -n "$inner" ]] || return 1
read -r outer_source outer_fstype outer_options outer_mount_id outer_major_minor <<<"$outer"
read -r inner_source inner_fstype inner_options inner_mount_id inner_major_minor <<<"$inner"
read -r loop_device loop_back_inode loop_back_major_minor loop_major_minor loop_offset loop_sizelimit <<<"$loops"
[[ -n "$outer_mount_id" && -n "$outer_major_minor" && -n "$inner_mount_id" && -n "$inner_major_minor" &&
-n "$loop_device" && -n "$loop_back_inode" && -n "$loop_back_major_minor" && -n "$loop_major_minor" &&
-n "$loop_offset" && -n "$loop_sizelimit" ]] || return 1
outer_source="$(_k3slrv_one_line /usr/bin/readlink -f -- "$outer_source")" || return 1
recovery_partition="$(_k3slrv_one_line /usr/bin/readlink -f -- "$K3SLR_RECOVERY_PARTITION_BY_ID")" || return 1
source_mounts="$(_k3slrv_findmnt_source_optional "$recovery_partition")" || return 1
[[ "$source_mounts" == "$K3SLR_OUTER_MOUNT" ]] || return 1
chain_snapshot="$(_k3slrv_pin_container_chain "$container")" || return 1
metadata="$(_k3slrv_one_line_root /usr/bin/stat --format='%F|%u|%g|%a|%s|%b|%i|%Hd:%Ld' -- "$container")" || return 1
IFS='|' read -r kind uid gid file_mode size blocks inode container_device <<<"$metadata"
[[ "$container_device" =~ ^[0-9]+:[0-9]+$ ]] || return 1
[[ "$blocks" =~ ^[0-9]+$ && "$blocks" -le 18014398509481983 ]] || return 1
allocated=$((blocks * 512))
printf 'outer_mounted=1\nouter_source=%s\nouter_fstype=%s\nouter_options=%s\n' "$outer_source" "$outer_fstype" "$outer_options"
printf 'inner_mounted=1\ninner_source=%s\ninner_fstype=%s\ninner_options=%s\n' "$inner_source" "$inner_fstype" "$inner_options"
printf 'container_kind=%s\ncontainer_symlink=%s\ncontainer_uid=%s\ncontainer_gid=%s\ncontainer_mode=%s\n' \
"$([[ "$kind" == 'regular file' ]] && printf regular || printf other)" \
"$(_k3slrv_root /usr/bin/test -L "$container" >/dev/null 2>&1 && printf 1 || printf 0)" "$uid" "$gid" "$file_mode"
printf 'container_size=%s\ncontainer_allocated=%s\nloop_count=1\nloop_device=%s\n' "$size" "$allocated" "$loop_device"
_k3slrv_capture mapping_snapshot _k3slrv_mapping_evidence "$K3SLR_MAPPING_NAME" mapping || return 1
_k3slrv_capture proof_snapshot _k3slrv_mapping_evidence "$K3SLR_PROOF_MAPPING_NAME" proof_mapping || return 1
printf '%s%s' "$mapping_snapshot" "$proof_snapshot"
mapping_device="$(_k3slrv_mapping_value "$mapping_snapshot" mapping_device)" || return 1
mapping_loop_major_minor="$(_k3slrv_lsblk MAJ:MIN "$mapping_device")" || return 1
mapping_major_minor="$(_k3slrv_lsblk MAJ:MIN "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1
inner_type="$(_k3slrv_blkid TYPE "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1
inner_label="$(_k3slrv_blkid LABEL "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1
printf 'inner_type=%s\ninner_label=%s\n' "$inner_type" "$inner_label"
inner_root_metadata="$(_k3slrv_one_line_root /usr/bin/stat --format='%u|%g|%a|%F' -- "$K3SLR_INNER_MOUNT")" || return 1
IFS='|' read -r uid gid file_mode kind <<<"$inner_root_metadata"
printf 'inner_root_uid=%s\ninner_root_gid=%s\ninner_root_mode=%s\ninner_root_kind=%s\n' \
"$uid" "$gid" "$file_mode" "$kind"
outer_after="$(_k3slrv_findmnt_optional "$K3SLR_OUTER_MOUNT")" || return 1
inner_after="$(_k3slrv_findmnt_optional "$K3SLR_INNER_MOUNT")" || return 1
source_after="$(_k3slrv_findmnt_source_optional "$recovery_partition")" || return 1
chain_after="$(_k3slrv_pin_container_chain "$container")" || return 1
_k3slrv_capture loops_after _k3slrv_root /usr/sbin/losetup --list --associated "$container" --noheadings --raw \
--output NAME,BACK-INO,BACK-MAJ:MIN,MAJ:MIN,OFFSET,SIZELIMIT 2>/dev/null || return 1
_k3slrv_normalize_one_record loops_after "$loops_after" || return 1
_k3slrv_capture mapping_after _k3slrv_mapping_evidence "$K3SLR_MAPPING_NAME" mapping || return 1
_k3slrv_capture proof_after _k3slrv_mapping_evidence "$K3SLR_PROOF_MAPPING_NAME" proof_mapping || return 1
container_metadata_after="$(_k3slrv_one_line_root /usr/bin/stat --format='%F|%u|%g|%a|%s|%b|%i|%Hd:%Ld' -- "$container")" || return 1
inner_type_after="$(_k3slrv_blkid TYPE "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1
inner_label_after="$(_k3slrv_blkid LABEL "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1
inner_root_after="$(_k3slrv_one_line_root /usr/bin/stat --format='%u|%g|%a|%F' -- "$K3SLR_INNER_MOUNT")" || return 1
mapping_device_after="$(_k3slrv_mapping_value "$mapping_after" mapping_device)" || return 1
mapping_loop_mm_after="$(_k3slrv_lsblk MAJ:MIN "$mapping_device_after")" || return 1
mapping_mm_after="$(_k3slrv_lsblk MAJ:MIN "/dev/mapper/${K3SLR_MAPPING_NAME}")" || return 1
[[ "$outer_after" == "$outer" && "$inner_after" == "$inner" && "$source_after" == "$source_mounts" &&
"$chain_after" == "$chain_snapshot" && "$loops_after" == "$loops" && "$mapping_after" == "$mapping_snapshot" &&
"$proof_after" == "$proof_snapshot" && "$container_metadata_after" == "$metadata" &&
"$inner_type_after" == "$inner_type" && "$inner_label_after" == "$inner_label" &&
"$inner_root_after" == "$inner_root_metadata" && "$mapping_loop_mm_after" == "$mapping_loop_major_minor" &&
"$mapping_mm_after" == "$mapping_major_minor" ]] || return 1
printf '%s\n' \
'container_chain_pinned=1' \
"container_inode=${inode}" \
"container_device=${container_device}" \
"loop_back_inode=${loop_back_inode}" \
"loop_back_major_minor=${loop_back_major_minor}" \
"outer_major_minor=${outer_major_minor}" \
"loop_major_minor=${loop_major_minor}" \
"mapping_loop_major_minor=${mapping_loop_major_minor}" \
"mapping_major_minor=${mapping_major_minor}" \
"inner_major_minor=${inner_major_minor}" \
"loop_offset=${loop_offset}" \
"loop_sizelimit=${loop_sizelimit}" \
'snapshot_stable=1'
}
_k3slrv_parse_evidence() {
local input="$1" destination_name="$2"
shift 2
local -n destination="$destination_name"
local line key value expected
local -A allowed=() seen=()
local -a lines=()
destination=()
for expected in "$@"; do allowed["$expected"]=1; done
mapfile -t lines < <(printf '%s' "$input")
(( ${#lines[@]} > 0 )) || return 1
for line in "${lines[@]}"; do
[[ "$line" =~ ^([a-z][a-z0-9_]*)=([A-Za-z0-9._:/,+=-]*)$ ]] || return 1
key="${BASH_REMATCH[1]}"; value="${BASH_REMATCH[2]}"
[[ "${allowed[$key]+yes}" == yes && "${seen[$key]+yes}" != yes ]] || return 1
seen["$key"]=1; destination["$key"]="$value"
done
for expected in "${!allowed[@]}"; do [[ "${seen[$expected]+yes}" == yes ]] || return 1; done
}
_k3slrv_validate_device() {
local mode="$1" input="$2"
local -A e=()
local keys=(recovery_partition recovery_disk recovery_partition_parent recovery_fs_uuid recovery_partuuid recovery_type
recovery_model recovery_serial recovery_wwn recovery_major_minor k3s_partition k3s_disk k3s_partition_parent
k3s_fs_uuid k3s_partuuid k3s_type k3s_model k3s_serial k3s_wwn k3s_major_minor smart_health
smart_reallocated smart_pending smart_uncorrectable ntfs_probe)
_k3slrv_parse_evidence "$input" e "${keys[@]}" || return 1
[[ "${e[recovery_partition_parent]}" == "${e[recovery_disk]}" && "${e[k3s_partition_parent]}" == "${e[k3s_disk]}" ]] || return 1
[[ "${e[recovery_fs_uuid]}" == "$K3SLR_RECOVERY_FS_UUID" && "${e[recovery_partuuid]}" == "$K3SLR_RECOVERY_PARTUUID" && "${e[recovery_type]}" == ntfs ]] || return 1
[[ "${e[recovery_model]}" == "$K3SLR_RECOVERY_MODEL" && "${e[recovery_serial]}" == "$K3SLR_RECOVERY_SERIAL" && "${e[recovery_wwn]}" == "$K3SLR_RECOVERY_WWN" ]] || return 1
[[ "${e[k3s_fs_uuid]}" == "$K3SLR_K3S_FS_UUID" && "${e[k3s_partuuid]}" == "$K3SLR_K3S_PARTUUID" && "${e[k3s_type]}" == ext4 ]] || return 1
[[ "${e[k3s_model]}" == "$K3SLR_K3S_MODEL" && "${e[k3s_serial]}" == "$K3SLR_K3S_SERIAL" && "${e[k3s_wwn]}" == "$K3SLR_K3S_WWN" ]] || return 1
[[ "${e[recovery_major_minor]}" != "${e[k3s_major_minor]}" && "${e[smart_health]}" == PASSED ]] || return 1
[[ "${e[smart_reallocated]}" == 0 && "${e[smart_pending]}" == 0 && "${e[smart_uncorrectable]}" == 0 ]] || return 1
if [[ "$mode" == open ]]; then [[ "${e[ntfs_probe]}" == not_applicable || "${e[ntfs_probe]}" == pass ]];
else [[ "${e[ntfs_probe]}" == pass ]]; fi
}
_k3slrv_option_present() {
[[ ",${1}," == *",${2},"* ]]
}
_k3slrv_outer_masks_secure() {
local options="$1" option umask_count=0 dmask_count=0 fmask_count=0 invalid=false
local -a option_list=()
IFS=',' read -r -a option_list <<<"$options"
for option in "${option_list[@]}"; do
case "$option" in
umask=077|umask=0077) umask_count=$((umask_count + 1)) ;;
dmask=077|dmask=0077) dmask_count=$((dmask_count + 1)) ;;
fmask=077|fmask=0077|fmask=177|fmask=0177) fmask_count=$((fmask_count + 1)) ;;
umask=*|dmask=*|fmask=*) invalid=true ;;
esac
done
"$invalid" && return 1
if (( umask_count == 1 && dmask_count == 0 && fmask_count == 0 )); then return 0; fi
(( umask_count == 0 && dmask_count == 1 && fmask_count == 1 ))
}
_k3slrv_validate_open_snapshot() {
local input="$1"
local -A snapshot=()
_k3slrv_parse_evidence "$input" snapshot \
container_chain_pinned container_inode container_device loop_back_inode loop_back_major_minor outer_major_minor \
loop_major_minor mapping_loop_major_minor mapping_major_minor inner_major_minor \
loop_offset loop_sizelimit snapshot_stable || return 1
[[ "${snapshot[container_chain_pinned]}" == 1 && "${snapshot[snapshot_stable]}" == 1 ]] || return 1
[[ "${snapshot[container_inode]}" =~ ^[0-9]+$ &&
"${snapshot[loop_back_inode]}" == "${snapshot[container_inode]}" ]] || return 1
[[ "${snapshot[container_device]}" =~ ^[0-9]+:[0-9]+$ &&
"${snapshot[loop_back_major_minor]}" == "${snapshot[container_device]}" &&
"${snapshot[container_device]}" == "${snapshot[outer_major_minor]}" ]] || return 1
[[ "${snapshot[mapping_loop_major_minor]}" == "${snapshot[loop_major_minor]}" ]] || return 1
[[ "${snapshot[mapping_major_minor]}" == "${snapshot[inner_major_minor]}" ]] || return 1
[[ "${snapshot[loop_offset]}" == 0 && "${snapshot[loop_sizelimit]}" == 0 ]]
}
_k3slrv_validate_state() {
local mode="$1" input="$2" device_input="$3" approved_partition='' line snapshot_raw
local -A e=()
if [[ "$mode" != open ]]; then
_k3slrv_parse_evidence "$input" e outer_mounted inner_mounted loop_count mapping_present proof_mapping_present || return 1
[[ "${e[outer_mounted]}" == 0 && "${e[inner_mounted]}" == 0 && "${e[loop_count]}" == 0 &&
"${e[mapping_present]}" == 0 && "${e[proof_mapping_present]}" == 0 ]]
return
fi
local keys=(outer_mounted outer_source outer_fstype outer_options inner_mounted inner_source inner_fstype inner_options
container_kind container_symlink container_uid container_gid container_mode container_size container_allocated
loop_count loop_device mapping_present mapping_type mapping_device proof_mapping_present inner_type inner_label
inner_root_uid inner_root_gid inner_root_mode inner_root_kind container_chain_pinned container_inode
container_device loop_back_inode loop_back_major_minor outer_major_minor loop_major_minor mapping_loop_major_minor
mapping_major_minor inner_major_minor loop_offset loop_sizelimit snapshot_stable)
_k3slrv_parse_evidence "$input" e "${keys[@]}" || return 1
while IFS= read -r line; do
[[ "$line" == recovery_partition=* ]] || continue
[[ -z "$approved_partition" ]] || return 1
approved_partition="${line#recovery_partition=}"
done <<<"$device_input"
[[ -n "$approved_partition" && "${e[outer_mounted]}" == 1 &&
"${e[outer_source]}" == "$approved_partition" && "${e[outer_fstype]}" == ntfs3 ]] || return 1
for option in rw nodev nosuid noexec "uid=${K3SLR_OWNER_UID}" "gid=${K3SLR_OWNER_GID}"; do
_k3slrv_option_present "${e[outer_options]}" "$option" || return 1
done
_k3slrv_outer_masks_secure "${e[outer_options]}" || return 1
[[ "${e[inner_mounted]}" == 1 && "${e[inner_source]}" == "/dev/mapper/${K3SLR_MAPPING_NAME}" && "${e[inner_fstype]}" == ext4 ]] || return 1
for option in rw nodev nosuid noexec; do _k3slrv_option_present "${e[inner_options]}" "$option" || return 1; done
[[ "${e[container_kind]}" == regular && "${e[container_symlink]}" == 0 &&
"${e[container_uid]}" == "$K3SLR_OWNER_UID" && "${e[container_gid]}" == "$K3SLR_OWNER_GID" ]] || return 1
[[ "${e[container_mode]}" =~ ^[0-7]{3,4}$ ]] || return 1
(( (10#${e[container_mode]: -2:1} & 7) == 0 && (10#${e[container_mode]: -1:1} & 7) == 0 )) || return 1
[[ "${e[container_size]}" == "$K3SLR_CONTAINER_SIZE_BYTES" && "${e[container_allocated]}" =~ ^[0-9]+$ ]] || return 1
(( e[container_allocated] >= K3SLR_CONTAINER_SIZE_BYTES )) || return 1
[[ "${e[loop_count]}" == 1 && "${e[mapping_present]}" == 1 && "${e[mapping_type]}" == LUKS2 &&
"${e[mapping_device]}" == "${e[loop_device]}" && "${e[proof_mapping_present]}" == 0 ]] || return 1
[[ "${e[inner_type]}" == ext4 && "${e[inner_label]}" == "$K3SLR_INNER_LABEL" ]] || return 1
[[ "${e[inner_root_uid]}" == 0 && "${e[inner_root_gid]}" == 0 && "${e[inner_root_mode]}" == 700 && "${e[inner_root_kind]}" == directory ]] || return 1
printf -v snapshot_raw '%s\n' \
"container_chain_pinned=${e[container_chain_pinned]}" \
"container_inode=${e[container_inode]}" \
"container_device=${e[container_device]}" \
"loop_back_inode=${e[loop_back_inode]}" \
"loop_back_major_minor=${e[loop_back_major_minor]}" \
"outer_major_minor=${e[outer_major_minor]}" \
"loop_major_minor=${e[loop_major_minor]}" \
"mapping_loop_major_minor=${e[mapping_loop_major_minor]}" \
"mapping_major_minor=${e[mapping_major_minor]}" \
"inner_major_minor=${e[inner_major_minor]}" \
"loop_offset=${e[loop_offset]}" \
"loop_sizelimit=${e[loop_sizelimit]}" \
"snapshot_stable=${e[snapshot_stable]}"
_k3slrv_validate_open_snapshot "${snapshot_raw%$'\n'}"
}
_k3slrv_verify_manifest_targets() {
local bundle="$1" manifest="$2" records record expected_hash relative target component current canonical
local before after hash_line actual_hash count=0
local -a components=() manifest_records=()
_k3slrv_capture records _k3slrv_root /usr/bin/mawk '
NF!=2 || length($1)!=64 || $1 !~ /^[0-9a-f]+$/ || $2 !~ /^\.\/[A-Za-z0-9._\/-]+$/ ||
$2 ~ /\.\./ || $2 ~ /\/\// || $2 ~ /\/\.\// || $2 ~ /\/\.$/ || $2 ~ /\/$/ { bad=1; exit }
{ print $1 "|" $2 }
END { exit (bad || NR==0) }
' "$manifest" 2>/dev/null || return 1
mapfile -t manifest_records < <(printf '%s' "$records")
(( ${#manifest_records[@]} > 0 )) || return 1
for record in "${manifest_records[@]}"; do
[[ "$record" == *'|'* ]] || return 1
expected_hash="${record%%|*}"
relative="${record#*|}"
[[ "$expected_hash" =~ ^[0-9a-f]{64}$ && "$relative" =~ ^\./[A-Za-z0-9._/-]+$ ]] || return 1
target="${bundle}/${relative#./}"
current="$bundle"
IFS='/' read -r -a components <<<"${relative#./}"
for component in "${components[@]}"; do
[[ -n "$component" && "$component" != . && "$component" != .. ]] || return 1
current="${current}/${component}"
_k3slrv_root /usr/bin/test ! -L "$current" >/dev/null 2>&1 || return 1
canonical="$(_k3slrv_one_line_root /usr/bin/readlink -e -- "$current")" || return 1
[[ "$canonical" == "$current" ]] || return 1
done
_k3slrv_root /usr/bin/test -f "$target" >/dev/null 2>&1 || return 1
before="$(_k3slrv_one_line_root /usr/bin/stat --format='%d:%i|%F' -- "$target")" || return 1
[[ "$before" == *'|regular file' ]] || return 1
hash_line="$(_k3slrv_one_line_root /usr/bin/sha256sum -- "$target")" || return 1
actual_hash="${hash_line%% *}"
[[ "$actual_hash" == "$expected_hash" ]] || return 1
after="$(_k3slrv_one_line_root /usr/bin/stat --format='%d:%i|%F' -- "$target")" || return 1
[[ "$after" == "$before" ]] || return 1
count=$((count + 1))
done
(( count > 0 ))
}
_k3slrv_collect_latest_bundle_evidence() {
local metadata="${K3SLR_INNER_MOUNT}/.latest-post-bundle.env" parsed relative expected_identity bundle canonical_bundle actual_identity after_identity
local bundle_metadata manifest stat_value
_k3slrv_root /usr/bin/test -f "$metadata" >/dev/null 2>&1 || return 1
_k3slrv_root /usr/bin/test ! -L "$metadata" >/dev/null 2>&1 || return 1
stat_value="$(_k3slrv_one_line_root /usr/bin/stat --format='%u:%g:%a:%F' -- "$metadata")" || return 1
[[ "$stat_value" == '0:0:600:regular file' ]] || return 1
_k3slrv_capture parsed _k3slrv_root /usr/bin/mawk -F= '
BEGIN { ok=1 }
!/^[a-z_]+=[A-Za-z0-9._:\/-]+$/ { ok=0; next }
$1=="schema" && !s++ && $2=="k3slr-latest-post-bundle-v1" { schema=$2; next }
$1=="relative_path" && !p++ && $2 ~ /^k3s-secrets-encryption-[0-9]{8}T[0-9]{6}Z\/post$/ { path=$2; next }
$1=="directory_identity" && !i++ && $2 ~ /^[0-9]+:[0-9]+$/ { identity=$2; next }
{ ok=0 }
END { if (ok && NR==3 && s==1 && p==1 && i==1) print path "|" identity; else exit 1 }
' "$metadata" 2>/dev/null || return 1
_k3slrv_normalize_one_record parsed "$parsed" || return 1
[[ "$parsed" == *'|'* ]] || return 1
relative="${parsed%%|*}"; expected_identity="${parsed#*|}"
[[ "$relative" =~ ^k3s-secrets-encryption-[0-9]{8}T[0-9]{6}Z/post$ && "$relative" != *..* ]] || return 1
bundle="${K3SLR_INNER_MOUNT}/${relative}"
canonical_bundle="$(_k3slrv_one_line_root /usr/bin/readlink -f -- "$bundle")" || return 1
[[ "$canonical_bundle" == "$bundle" ]] || return 1
actual_identity="$(_k3slrv_one_line_root /usr/bin/stat --dereference --format='%d:%i' -- "$bundle")" || return 1
[[ "$actual_identity" == "$expected_identity" ]] || return 1
stat_value="$(_k3slrv_one_line_root /usr/bin/stat --dereference --format='%u:%g:%a:%F' -- "$bundle")" || return 1
[[ "$stat_value" == '0:0:700:directory' ]] || return 1
bundle_metadata="${bundle}/bundle.env"; manifest="${bundle}/verification.manifest"
for file in "$bundle_metadata" "$manifest"; do
_k3slrv_root /usr/bin/test -f "$file" >/dev/null 2>&1 || return 1
_k3slrv_root /usr/bin/test ! -L "$file" >/dev/null 2>&1 || return 1
stat_value="$(_k3slrv_one_line_root /usr/bin/stat --format='%u:%g:%a:%F' -- "$file")" || return 1
[[ "$stat_value" == '0:0:600:regular file' ]] || return 1
done
_k3slrv_root /usr/bin/mawk -F= '
BEGIN { ok=1 }
!/^[a-z][a-z0-9_]*=[A-Za-z0-9:+._-]+$/ { ok=0; next }
$1=="schema" && !a++ && $2=="platform-k3s-bundle-v1" { next }
$1=="bundle_id" && !b++ && $2 ~ /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ { next }
$1=="phase" && !c++ && $2=="post" { next }
$1=="k3s_version" && !d++ && $2=="v1.36.2+k3s1" { next }
$1=="datastore" && !e++ && ($2=="sqlite" || $2=="embedded-etcd") { next }
$1=="created_at_utc" && !f++ && $2 ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/ { next }
$1=="secret_count" && !g++ && $2 ~ /^[0-9]+$/ { next }
{ ok=0 }
END { exit !(ok && NR==7 && a==1 && b==1 && c==1 && d==1 && e==1 && f==1 && g==1) }
' "$bundle_metadata" >/dev/null 2>&1 || return 1
_k3slrv_verify_manifest_targets "$bundle" "$manifest" || return 1
after_identity="$(_k3slrv_one_line_root /usr/bin/stat --dereference --format='%d:%i' -- "$bundle")" || return 1
[[ "$after_identity" == "$expected_identity" ]] || return 1
printf '%s\n' \
'metadata_schema=k3slr-latest-post-bundle-v1' \
'relative_path_valid=1' \
'directory_identity_match=1' \
'directory_secure=1' \
'bundle_schema=platform-k3s-bundle-v1' \
'bundle_phase=post' \
'manifest_safe=1' \
'manifest_verified=1' \
'directory_identity_stable=1'
}
_k3slrv_verify_latest_bundle() {
local raw
local -A evidence=()
_k3slrv_capture raw _k3slrv_collect_latest_bundle_evidence || return 1
_k3slrv_parse_evidence "$raw" evidence \
metadata_schema relative_path_valid directory_identity_match directory_secure \
bundle_schema bundle_phase manifest_safe manifest_verified directory_identity_stable || return 1
[[ "${evidence[metadata_schema]}" == k3slr-latest-post-bundle-v1 &&
"${evidence[relative_path_valid]}" == 1 &&
"${evidence[directory_identity_match]}" == 1 &&
"${evidence[directory_secure]}" == 1 &&
"${evidence[bundle_schema]}" == platform-k3s-bundle-v1 &&
"${evidence[bundle_phase]}" == post &&
"${evidence[manifest_safe]}" == 1 &&
"${evidence[manifest_verified]}" == 1 &&
"${evidence[directory_identity_stable]}" == 1 ]]
}
k3slr_local_recovery_main() {
local mode='' check_latest=false argument device_raw state_raw latest=not_checked
for argument in "$@"; do
case "$argument" in
--expect-device-ready) [[ -z "$mode" ]] || { _k3slrv_usage >&2; return 2; }; mode=device_ready ;;
--expect-closed) [[ -z "$mode" ]] || { _k3slrv_usage >&2; return 2; }; mode=closed ;;
--expect-open) [[ -z "$mode" ]] || { _k3slrv_usage >&2; return 2; }; mode=open ;;
--check-latest-bundle) "$check_latest" && { _k3slrv_usage >&2; return 2; }; check_latest=true ;;
--help|-h) _k3slrv_usage; return 0 ;;
*) _k3slrv_usage >&2; return 2 ;;
esac
done
[[ -n "$mode" ]] || { _k3slrv_usage >&2; return 2; }
if "$check_latest" && [[ "$mode" != open ]]; then _k3slrv_usage >&2; return 2; fi
PATH="$K3SLRV_SECURE_PATH"; export PATH
_k3slrv_verify_trusted_binaries || { _k3slrv_fail; return 1; }
# Direct execution always replaces any inherited/predefined parser with the
# repository library in this unprivileged process.
# shellcheck source=/dev/null
builtin source -- "$K3SLRV_LIBRARY_PATH" || { _k3slrv_fail; return 1; }
_k3slr_load_contract "$K3SLRV_CONTRACT_PATH" || { _k3slrv_fail; return 1; }
_k3slrv_require_cached_sudo || { _k3slrv_fail; return 1; }
_k3slrv_capture device_raw _k3slrv_collect_device_evidence "$mode" || { _k3slrv_fail; return 1; }
_k3slrv_validate_device "$mode" "$device_raw" || { _k3slrv_fail; return 1; }
_k3slrv_capture state_raw _k3slrv_collect_state_evidence "$mode" || { _k3slrv_fail; return 1; }
_k3slrv_validate_state "$mode" "$state_raw" "$device_raw" || { _k3slrv_fail; return 1; }
if "$check_latest"; then
_k3slrv_verify_latest_bundle || { _k3slrv_fail; return 1; }
latest=verified
fi
printf 'Recovery device: match\nRecovery state: %s\nLineage: match\nLatest bundle: %s\n' "$mode" "$latest"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
k3slr_local_recovery_main "$@"
fi
+636
View File
@@ -0,0 +1,636 @@
#!/usr/bin/env bash
# 격리 복구 결과를 만들고 운영 host의 복구 증거를 등록·검사한다. source-safe
# 함수 경계는 fixture가 live/root side effect만 치환할 수 있게 유지한다.
set -Eeuo pipefail
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
readonly KSRE_SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
else
readonly KSRE_SCRIPT_DIR='.'
fi
readonly KSRE_ROOT="$(cd -- "${KSRE_SCRIPT_DIR}/../.." && pwd -P)"
readonly KSRE_STATUS_VALIDATOR="${KSRE_ROOT}/scripts/validate/k3s-secret-encryption.sh"
readonly KSRE_POST_BUNDLE='/etc/rancher/k3s/platform-post-bundle.env'
readonly KSRE_EVIDENCE='/etc/rancher/k3s/platform-restore-evidence.env'
readonly KSRE_VERSION='v1.36.2+k3s1'
readonly KSRE_KUBECTL='/usr/local/bin/kubectl'
readonly KSRE_LOCAL_API_SERVER='https://127.0.0.1:6443'
readonly KSRE_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin'
readonly KSRE_BROKER_RUNTIME_TIMEOUT='2s'
readonly KSRE_BROKER_KILL_AFTER='0.5s'
readonly KSRE_BROKER_READY_TIMEOUT='3'
readonly KSRE_BROKER_ACK_TIMEOUT='1.0'
readonly KSRE_BROKER_REAP_POLLS='300'
readonly KSRE_BROKER_SIGNAL_POLLS='50'
readonly KSRE_BROKER_POLL_INTERVAL='0.01'
_ksre_fail() {
printf 'ERROR: 복구 증거 검증에 실패했습니다. 수동 절차를 확인하세요.\n' >&2
return 1
}
_ksre_usage() {
cat <<'USAGE'
사용법:
bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \
--emit-result --bundle-metadata FILE --output FILE
bash scripts/validate/k3s-secret-encryption-restore-evidence.sh \
--record --bundle-metadata FILE --result-file FILE
bash scripts/validate/k3s-secret-encryption-restore-evidence.sh --check
--emit-result는 격리 복구 host의 live 상태를 검사한 뒤 결과 파일을 만듭니다.
--record는 복구 환경과 bundle 복사본 파기 확인 뒤 운영 host에 증거를 등록합니다.
--check는 권위 post bundle, live 암호화 상태와 등록된 증거를 읽기 전용 검사합니다.
USAGE
}
_ksre_allowed_key() {
local kind="$1" key="$2"
case "$kind:$key" in
bundle:schema|bundle:bundle_id|bundle:phase|bundle:k3s_version|bundle:datastore|bundle:created_at_utc|bundle:secret_count)
return 0
;;
result:schema|result:bundle_id|result:k3s_version|result:datastore|result:status_class|result:rotation_stage|result:hashes|result:local_integrity|result:api_ready|result:node_ready|result:isolation|result:secret_count_match|result:tested_at_utc)
return 0
;;
evidence:schema|evidence:bundle_id|evidence:k3s_version|evidence:datastore|evidence:rotation_stage|evidence:local_integrity|evidence:node_ready|evidence:isolation|evidence:tested_at_utc|evidence:recorded_at_utc|evidence:destroyed)
return 0
;;
*) return 1 ;;
esac
}
_ksre_expected_field_count() {
case "$1" in bundle) printf '7\n' ;; result) printf '13\n' ;; evidence) printf '11\n' ;; *) return 1 ;; esac
}
_ksre_values_valid() {
local kind="$1" destination="$2"
local -n values="$destination"
[[ "${values[bundle_id]:-}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]] || return 1
[[ "${values[k3s_version]:-}" == "$KSRE_VERSION" ]] || return 1
[[ "${values[datastore]:-}" == sqlite || "${values[datastore]:-}" == embedded-etcd ]] || return 1
case "$kind" in
bundle)
[[ "${values[schema]:-}" == platform-k3s-bundle-v1 &&
( "${values[phase]:-}" == pre || "${values[phase]:-}" == post ) &&
"${values[created_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ &&
"${values[secret_count]:-}" =~ ^[0-9]+$ ]]
;;
result)
[[ "${values[schema]:-}" == platform-k3s-restore-result-v1 &&
"${values[status_class]:-}" == enabled_stable &&
"${values[rotation_stage]:-}" == reencrypt_finished &&
"${values[hashes]:-}" == match && "${values[local_integrity]:-}" == match &&
"${values[api_ready]:-}" == pass && "${values[node_ready]:-}" == pass &&
"${values[isolation]:-}" == pass && "${values[secret_count_match]:-}" == pass &&
"${values[tested_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
;;
evidence)
[[ "${values[schema]:-}" == platform-k3s-restore-evidence-v1 &&
"${values[rotation_stage]:-}" == reencrypt_finished &&
"${values[local_integrity]:-}" == match && "${values[node_ready]:-}" == pass &&
"${values[isolation]:-}" == pass && "${values[destroyed]:-}" == confirmed &&
"${values[tested_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ &&
"${values[recorded_at_utc]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
;;
*) return 1 ;;
esac
}
_ksre_parse_stream() {
local kind="$1" destination="$2" line key value count=0 expected
local -n output_fields="$destination"
output_fields=()
expected="$(_ksre_expected_field_count "$kind")" || return 1
while IFS= read -r line || [[ -n "$line" ]]; do
[[ "$line" =~ ^([a-z0-9_]+)=([^=[:cntrl:]]+)$ ]] || return 1
key="${BASH_REMATCH[1]}"; value="${BASH_REMATCH[2]}"
_ksre_allowed_key "$kind" "$key" || return 1
[[ -z "${output_fields[$key]+present}" ]] || return 1
output_fields["$key"]="$value"
count=$((count + 1))
done
[[ "$count" == "$expected" ]] || return 1
_ksre_values_valid "$kind" "$destination"
}
_ksre_raw_stream_has_no_nul() {
/usr/bin/od -An -v -t u1 -- "$1" 2>/dev/null |
/usr/bin/awk '{ for (field = 1; field <= NF; field++) if ($field == 0) found = 1 }
END { exit(found ? 1 : 0) }'
}
_ksre_parse_external_fd() {
local kind="$1" input_fd="$2" destination="$3" handle metadata
[[ "$input_fd" =~ ^[0-9]+$ ]] || return 1
handle="/proc/self/fd/${input_fd}"
metadata="$(/usr/bin/stat --dereference --format='%u:%a:%F' -- "$handle" 2>/dev/null)" || return 1
[[ "$metadata" == "${EUID}:600:regular file" && -s "$handle" ]] || return 1
_ksre_raw_stream_has_no_nul "$handle" || return 1
_ksre_parse_stream "$kind" "$destination" <"$handle"
}
_ksre_broker_job_running() {
local expected_pid="$1" active_pid active_jobs
active_jobs="$(jobs -pr)"
while IFS= read -r active_pid; do
[[ "$active_pid" == "$expected_pid" ]] && return 0
done <<<"$active_jobs"
return 1
}
_ksre_reap_broker_bounded() {
local broker_pid="$1" attempt
for ((attempt = 0; attempt < KSRE_BROKER_REAP_POLLS; attempt++)); do
if ! _ksre_broker_job_running "$broker_pid"; then
if wait "$broker_pid"; then return 0; else return $?; fi
fi
/usr/bin/sleep "$KSRE_BROKER_POLL_INTERVAL"
done
/bin/kill -TERM "$broker_pid" 2>/dev/null || :
for ((attempt = 0; attempt < KSRE_BROKER_SIGNAL_POLLS; attempt++)); do
if ! _ksre_broker_job_running "$broker_pid"; then
wait "$broker_pid" 2>/dev/null || :
return 1
fi
/usr/bin/sleep "$KSRE_BROKER_POLL_INTERVAL"
done
/bin/kill -KILL "$broker_pid" 2>/dev/null || :
for ((attempt = 0; attempt < KSRE_BROKER_SIGNAL_POLLS; attempt++)); do
if ! _ksre_broker_job_running "$broker_pid"; then
wait "$broker_pid" 2>/dev/null || :
return 1
fi
/usr/bin/sleep "$KSRE_BROKER_POLL_INTERVAL"
done
return 1
}
_ksre_open_external_nofollow() {
local path="$1" destination="$2" broker_pid broker_read broker_write
local ready opener_pid opener_fd opened_descriptor='' rc=0
coproc KSRE_OPEN_BROKER {
exec /usr/bin/timeout --signal=TERM --kill-after="$KSRE_BROKER_KILL_AFTER" \
"$KSRE_BROKER_RUNTIME_TIMEOUT" /usr/bin/python3 -I -S -c '
import os
import select
import stat
import sys
required_flags = ("O_PATH", "O_NOFOLLOW", "O_CLOEXEC", "O_NONBLOCK")
if any(not hasattr(os, flag) for flag in required_flags):
sys.exit(1)
try:
path_descriptor = os.open(
sys.argv[1], os.O_PATH | os.O_NOFOLLOW | os.O_CLOEXEC
)
except OSError:
sys.exit(1)
try:
path_status = os.fstat(path_descriptor)
if not stat.S_ISREG(path_status.st_mode):
sys.exit(1)
try:
descriptor = os.open(
f"/proc/self/fd/{path_descriptor}",
os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK,
)
except OSError:
sys.exit(1)
try:
content_status = os.fstat(descriptor)
if (
not stat.S_ISREG(content_status.st_mode)
or content_status.st_dev != path_status.st_dev
or content_status.st_ino != path_status.st_ino
):
sys.exit(1)
print(f"{os.getpid()}:{descriptor}", flush=True)
readable, _, _ = select.select(
[sys.stdin.buffer], [], [], float(sys.argv[2])
)
if not readable or sys.stdin.buffer.read(1) != b"x":
sys.exit(1)
finally:
os.close(descriptor)
finally:
os.close(path_descriptor)
' "$path" "$KSRE_BROKER_ACK_TIMEOUT"
}
broker_pid="$KSRE_OPEN_BROKER_PID"
broker_read="${KSRE_OPEN_BROKER[0]}"
broker_write="${KSRE_OPEN_BROKER[1]}"
if IFS= read -r -t "$KSRE_BROKER_READY_TIMEOUT" ready <&"$broker_read"; then
if [[ "$ready" =~ ^([0-9]+):([0-9]+)$ ]]; then
opener_pid="${BASH_REMATCH[1]}"
opener_fd="${BASH_REMATCH[2]}"
exec {opened_descriptor}<"/proc/${opener_pid}/fd/${opener_fd}" || rc=1
else
rc=1
fi
else
rc=1
fi
if (( rc == 0 )); then
printf 'x' >&"$broker_write" || rc=1
fi
exec {broker_write}>&-
exec {broker_read}<&-
if (( rc != 0 )); then /bin/kill -TERM "$broker_pid" 2>/dev/null || :; fi
if ! _ksre_reap_broker_bounded "$broker_pid"; then rc=1; fi
if (( rc != 0 )); then
[[ -z "$opened_descriptor" ]] || exec {opened_descriptor}<&-
return 1
fi
printf -v "$destination" '%s' "$opened_descriptor"
}
_ksre_parse_external_file() {
local kind="$1" path="$2" destination="$3" input_fd rc=0
[[ -n "$path" ]] || return 1
_ksre_open_external_nofollow "$path" input_fd || return 1
if (( rc == 0 )); then _ksre_parse_external_fd "$kind" "$input_fd" "$destination" || rc=$?; fi
exec {input_fd}<&-
return "$rc"
}
_ksre_parse_trusted_content() {
local kind="$1" content="$2" destination="$3"
_ksre_parse_stream "$kind" "$destination" <<<"$content"
}
_ksre_current_context() {
/usr/bin/timeout --signal=TERM --kill-after=1s 9s \
"$KSRE_KUBECTL" config current-context 2>/dev/null
}
_ksre_require_same_context() {
local expected="$1" current
current="$(_ksre_current_context)" || return 1
[[ -n "$current" && "$current" == "$expected" && "$current" != *$'\n'* ]]
}
_ksre_validate_current_connection() {
[[ "$1" == "${KSRE_LOCAL_API_SERVER}|" ]]
}
_ksre_current_connection() {
local context="$1"
/usr/bin/timeout --signal=TERM --kill-after=1s 9s \
"$KSRE_KUBECTL" --context="$context" config view --minify \
-o 'jsonpath={.clusters[0].cluster.server}{"|"}{.clusters[0].cluster.proxy-url}' \
2>/dev/null
}
_ksre_require_same_connection() {
local context="$1" expected="$2" current
_ksre_validate_current_connection "$expected" || return 1
current="$(_ksre_current_connection "$context")" || return 1
[[ "$current" == "$expected" ]]
}
_ksre_now_rfc3339() { /usr/bin/date -u +%Y-%m-%dT%H:%M:%SZ; }
_ksre_now_epoch() { /usr/bin/date -u +%s; }
_ksre_timestamp_epoch() {
local timestamp="$1" normalized
normalized="$(/usr/bin/date -u -d "$timestamp" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || return 1
[[ "$normalized" == "$timestamp" ]] || return 1
/usr/bin/date -u -d "$timestamp" +%s
}
_ksre_age_within() {
local timestamp="$1" maximum="$2" now epoch age
now="$(_ksre_now_epoch)" || return 1
epoch="$(_ksre_timestamp_epoch "$timestamp")" || return 1
[[ "$now" =~ ^[0-9]+$ && "$epoch" =~ ^[0-9]+$ && "$maximum" =~ ^[0-9]+$ ]] || return 1
age=$((now - epoch))
(( age >= 0 && age <= maximum ))
}
_ksre_bundle_equal() {
local left_name="$1" right_name="$2" key
local -n left="$left_name" right="$right_name"
for key in schema bundle_id phase k3s_version datastore created_at_utc secret_count; do
[[ "${left[$key]}" == "${right[$key]}" ]] || return 1
done
}
_ksre_read_root_file() {
local path="$1" kind="$2" destination="$3" metadata temporary rc=0
/usr/bin/sudo --non-interactive -- /usr/bin/test ! -L "$path" || return 1
/usr/bin/sudo --non-interactive -- /usr/bin/test -f "$path" || return 1
/usr/bin/sudo --non-interactive -- /usr/bin/test -s "$path" || return 1
metadata="$(/usr/bin/sudo --non-interactive -- /usr/bin/stat \
--format='%u:%g:%a:%F' -- "$path" 2>/dev/null)" || return 1
[[ "$metadata" == '0:0:600:regular file' ]] || return 1
temporary="$(/usr/bin/mktemp /tmp/platform-k3s-restore-read.XXXXXX)" || return 1
/usr/bin/chmod 0600 "$temporary" || { /bin/rm -f -- "$temporary"; return 1; }
/usr/bin/sudo --non-interactive -- /bin/cat -- "$path" >"$temporary" || rc=$?
if (( rc == 0 )); then _ksre_parse_external_file "$kind" "$temporary" "$destination" || rc=$?; fi
/bin/rm -f -- "$temporary"
return "$rc"
}
_ksre_read_authoritative_bundle() { _ksre_read_root_file "$KSRE_POST_BUNDLE" bundle "$1"; }
_ksre_read_installed_evidence() { _ksre_read_root_file "$KSRE_EVIDENCE" evidence "$1"; }
_ksre_inventory_value() {
local file="$1" key="$2" line found=''
while IFS= read -r line; do
[[ "$line" =~ ^([a-z_]+)=([A-Za-z0-9_.+-]+)$ ]] || return 1
if [[ "${BASH_REMATCH[1]}" == "$key" ]]; then
[[ -z "$found" ]] || return 1
found="${BASH_REMATCH[2]}"
fi
done <"$file"
[[ -n "$found" ]] || return 1
printf '%s\n' "$found"
}
_ksre_secret_count_from_stream() {
/usr/bin/jq -er '.items | length' 2>/dev/null
}
_ksre_require_same_ready_node() {
local local_node="$1" local_node_uid="$2" nodes_json="$3"
[[ "$local_node" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || return 1
[[ "$local_node_uid" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]] || return 1
/usr/bin/jq -e --arg local_node "$local_node" --arg local_node_uid "$local_node_uid" '
[.items[] |
select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))] |
length == 1 and
.[0].metadata.name == $local_node and
.[0].metadata.uid == $local_node_uid
' >/dev/null 2>&1 <<<"$nodes_json"
}
_ksre_run_restore_status_validator() {
/usr/bin/bash "$KSRE_STATUS_VALIDATOR" --expect-reencrypted-restore \
--verified-output-dir "$1" >/dev/null
}
_ksre_current_user_api_ready() {
local context="$1" server="$2"
/usr/bin/timeout --signal=TERM --kill-after=1s 9s \
/usr/bin/env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u NO_PROXY \
-u http_proxy -u https_proxy -u all_proxy -u no_proxy \
"$KSRE_KUBECTL" --context="$context" --server="$server" \
get --raw=/readyz >/dev/null 2>&1
}
_ksre_current_user_nodes_json() {
local context="$1" server="$2"
/usr/bin/timeout --signal=TERM --kill-after=1s 15s \
/usr/bin/env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u NO_PROXY \
-u http_proxy -u https_proxy -u all_proxy -u no_proxy \
"$KSRE_KUBECTL" --context="$context" --server="$server" \
get nodes -o json 2>/dev/null
}
_ksre_current_user_secret_count() {
local context="$1" server="$2"
/usr/bin/timeout --signal=TERM --kill-after=1s 30s \
/usr/bin/env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u NO_PROXY \
-u http_proxy -u https_proxy -u all_proxy -u no_proxy \
"$KSRE_KUBECTL" --context="$context" --server="$server" \
get secrets --all-namespaces -o json 2>/dev/null |
_ksre_secret_count_from_stream
}
_ksre_live_restore_inventory() {
local context="$1" connection="$2" server
local handoff inventory nodes version datastore encryption rotation hashes integrity
local local_node local_node_uid local_node_ready
local api_ready node_ready secret_count rc=0
_ksre_validate_current_connection "$connection" || return 1
server="${connection%|}"
handoff="$(/usr/bin/mktemp -d /tmp/platform-k3s-encryption.XXXXXX)" || return 1
/usr/bin/chmod 0700 "$handoff" || { /bin/rm -rf -- "$handoff"; return 1; }
if ! _ksre_run_restore_status_validator "$handoff"; then
/bin/rm -rf -- "$handoff"; return 1
fi
inventory="$handoff/inventory.env"
version="$(_ksre_inventory_value "$inventory" version)" || rc=1
datastore="$(_ksre_inventory_value "$inventory" datastore)" || rc=1
encryption="$(_ksre_inventory_value "$inventory" encryption)" || rc=1
rotation="$(_ksre_inventory_value "$inventory" rotation)" || rc=1
hashes="$(_ksre_inventory_value "$inventory" server_hashes)" || rc=1
integrity="$(_ksre_inventory_value "$inventory" integrity)" || rc=1
local_node="$(_ksre_inventory_value "$inventory" node_name)" || rc=1
local_node_uid="$(_ksre_inventory_value "$inventory" node_uid)" || rc=1
local_node_ready="$(_ksre_inventory_value "$inventory" node_ready)" || rc=1
/bin/rm -rf -- "$handoff"
(( rc == 0 )) || return 1
[[ "$encryption" == Enabled && "$local_node_ready" == Ready ]] || return 1
if _ksre_current_user_api_ready "$context" "$server"; then api_ready=pass; else return 1; fi
nodes="$(_ksre_current_user_nodes_json "$context" "$server")" || return 1
_ksre_require_same_ready_node "$local_node" "$local_node_uid" "$nodes" || return 1
node_ready=pass
secret_count="$(_ksre_current_user_secret_count "$context" "$server")" || return 1
[[ "$secret_count" =~ ^[0-9]+$ ]] || return 1
_ksre_require_same_connection "$context" "$connection" || return 1
printf '%s|%s|enabled_stable|%s|%s|%s|%s|%s|%s\n' \
"$version" "$datastore" "$rotation" "$hashes" "$integrity" \
"$api_ready" "$node_ready" "$secret_count"
}
_ksre_isolation_attested() {
local context="$1" answer
[[ -t 0 ]] || return 1
printf '격리·egress 차단 시험 완료 후 Type ISOLATED %s: ' "$context" >&2
IFS= read -r answer
[[ "$answer" == "ISOLATED $context" ]]
}
_ksre_output_target_absent() {
local output="$1" parent base
[[ -n "$output" && ! -e "$output" && ! -L "$output" ]] || return 1
if [[ "$output" == */* ]]; then parent="${output%/*}"; base="${output##*/}"
else parent='.'; base="$output"; fi
[[ -n "$parent" && -n "$base" && "$base" != . && "$base" != .. ]] || return 1
[[ -d "$parent" && ! -L "$parent" ]] || return 1
}
_ksre_write_output() {
local output="$1" content="$2" kind="$3" parent base temporary
_ksre_output_target_absent "$output" || return 1
if [[ "$output" == */* ]]; then parent="${output%/*}"; base="${output##*/}"
else parent='.'; base="$output"; fi
temporary="$(/usr/bin/mktemp "${parent}/.${base}.tmp.XXXXXX")" || return 1
/usr/bin/chmod 0600 "$temporary" || { /bin/rm -f -- "$temporary"; return 1; }
printf '%s\n' "$content" >"$temporary" || { /bin/rm -f -- "$temporary"; return 1; }
declare -A verification=()
_ksre_parse_external_file "$kind" "$temporary" verification || { /bin/rm -f -- "$temporary"; return 1; }
/bin/ln -- "$temporary" "$output" 2>/dev/null || { /bin/rm -f -- "$temporary"; return 1; }
/bin/rm -f -- "$temporary"
}
_ksre_evidence_target_absent() {
/usr/bin/sudo --non-interactive -- /usr/bin/test ! -e "$KSRE_EVIDENCE" &&
/usr/bin/sudo --non-interactive -- /usr/bin/test ! -L "$KSRE_EVIDENCE"
}
_ksre_prompt_destroyed() {
local answer
[[ -t 0 ]] || return 1
printf '일회용 환경과 bundle 복사본 파기 후 Type DESTROYED default: ' >&2
IFS= read -r answer
[[ "$answer" == 'DESTROYED default' ]]
}
_ksre_install_evidence() {
local content="$1" local_file root_temporary
local_file="$(/usr/bin/mktemp /tmp/platform-k3s-restore-evidence.XXXXXX)" || return 1
/usr/bin/chmod 0600 "$local_file" || { /bin/rm -f -- "$local_file"; return 1; }
printf '%s\n' "$content" >"$local_file" || { /bin/rm -f -- "$local_file"; return 1; }
declare -A verification=()
_ksre_parse_external_file evidence "$local_file" verification || { /bin/rm -f -- "$local_file"; return 1; }
root_temporary="${KSRE_EVIDENCE}.new.$$"
/usr/bin/sudo --non-interactive -- /usr/bin/test ! -e "$KSRE_EVIDENCE" || { /bin/rm -f -- "$local_file"; return 1; }
/usr/bin/sudo --non-interactive -- /usr/bin/install -o root -g root -m 0600 -- \
"$local_file" "$root_temporary" || { /bin/rm -f -- "$local_file"; return 1; }
/usr/bin/sudo --non-interactive -- /usr/bin/cmp --silent -- "$local_file" "$root_temporary" || {
/usr/bin/sudo --non-interactive -- /bin/rm -f -- "$root_temporary" >/dev/null 2>&1 || :
/bin/rm -f -- "$local_file"; return 1
}
if ! /usr/bin/sudo --non-interactive -- /bin/ln -- "$root_temporary" "$KSRE_EVIDENCE" 2>/dev/null; then
/usr/bin/sudo --non-interactive -- /bin/rm -f -- "$root_temporary" >/dev/null 2>&1 || :
/bin/rm -f -- "$local_file"; return 1
fi
/usr/bin/sudo --non-interactive -- /bin/rm -f -- "$root_temporary" || return 1
/bin/rm -f -- "$local_file"
}
_ksre_render_result() {
local -n bundle_fields="$1"
local tested_at="$2"
printf 'schema=platform-k3s-restore-result-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nstatus_class=enabled_stable\nrotation_stage=reencrypt_finished\nhashes=match\nlocal_integrity=match\napi_ready=pass\nnode_ready=pass\nisolation=pass\nsecret_count_match=pass\ntested_at_utc=%s\n' \
"${bundle_fields[bundle_id]}" "${bundle_fields[k3s_version]}" "${bundle_fields[datastore]}" "$tested_at"
}
_ksre_render_evidence() {
local -n bundle_fields="$1" result_fields="$2"
local recorded_at="$3"
printf 'schema=platform-k3s-restore-evidence-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nrotation_stage=reencrypt_finished\nlocal_integrity=match\nnode_ready=pass\nisolation=pass\ntested_at_utc=%s\nrecorded_at_utc=%s\ndestroyed=confirmed\n' \
"${bundle_fields[bundle_id]}" "${bundle_fields[k3s_version]}" "${bundle_fields[datastore]}" \
"${result_fields[tested_at_utc]}" "$recorded_at"
}
_ksre_emit_result() {
local metadata="$1" output="$2" context connection live tested_at content
local version datastore status rotation hashes integrity api node count
declare -A bundle=()
context="$(_ksre_current_context)" || return 1
[[ -n "$context" && "$context" != *$'\n'* ]] || return 1
connection="$(_ksre_current_connection "$context")" || return 1
_ksre_validate_current_connection "$connection" || return 1
_ksre_parse_external_file bundle "$metadata" bundle || return 1
[[ "${bundle[phase]}" == post ]] || return 1
_ksre_output_target_absent "$output" || return 1
live="$(_ksre_live_restore_inventory "$context" "$connection")" || return 1
IFS='|' read -r version datastore status rotation hashes integrity api node count <<<"$live"
[[ "$version" == "${bundle[k3s_version]}" && "$datastore" == "${bundle[datastore]}" &&
"$status" == enabled_stable && "$rotation" == reencrypt_finished && "$hashes" == match &&
"$integrity" == match && "$api" == pass && "$node" == pass &&
"$count" == "${bundle[secret_count]}" ]] || return 1
_ksre_isolation_attested "$context" || return 1
_ksre_require_same_connection "$context" "$connection" || return 1
_ksre_require_same_context "$context" || return 1
tested_at="$(_ksre_now_rfc3339)" || return 1
content="$(_ksre_render_result bundle "$tested_at")" || return 1
_ksre_write_output "$output" "$content" result
}
_ksre_record() {
local metadata="$1" result_file="$2" context recorded_at content
declare -A bundle=() result=() authority=() verification=()
context="$(_ksre_current_context)" || return 1
[[ -n "$context" && "$context" != *$'\n'* ]] || return 1
_ksre_parse_external_file bundle "$metadata" bundle || return 1
_ksre_parse_external_file result "$result_file" result || return 1
[[ "${bundle[phase]}" == post && "${result[bundle_id]}" == "${bundle[bundle_id]}" &&
"${result[k3s_version]}" == "${bundle[k3s_version]}" &&
"${result[datastore]}" == "${bundle[datastore]}" ]] || return 1
_ksre_age_within "${result[tested_at_utc]}" 86400 || return 1
_ksre_read_authoritative_bundle authority || return 1
_ksre_bundle_equal bundle authority || return 1
_ksre_evidence_target_absent || return 1
_ksre_prompt_destroyed || return 1
_ksre_require_same_context "$context" || return 1
recorded_at="$(_ksre_now_rfc3339)" || return 1
content="$(_ksre_render_evidence bundle result "$recorded_at")" || return 1
_ksre_parse_trusted_content evidence "$content" verification || return 1
_ksre_install_evidence "$content"
}
_ksre_check() {
local context connection live version datastore status rotation hashes integrity api node count
declare -A authority=() evidence=()
context="$(_ksre_current_context)" || return 1
[[ -n "$context" && "$context" != *$'\n'* ]] || return 1
connection="$(_ksre_current_connection "$context")" || return 1
_ksre_validate_current_connection "$connection" || return 1
_ksre_read_authoritative_bundle authority || return 1
_ksre_read_installed_evidence evidence || return 1
[[ "${authority[phase]}" == post &&
"${evidence[bundle_id]}" == "${authority[bundle_id]}" &&
"${evidence[k3s_version]}" == "${authority[k3s_version]}" &&
"${evidence[datastore]}" == "${authority[datastore]}" ]] || return 1
_ksre_age_within "${evidence[tested_at_utc]}" 2592000 || return 1
live="$(_ksre_live_restore_inventory "$context" "$connection")" || return 1
IFS='|' read -r version datastore status rotation hashes integrity api node count <<<"$live"
[[ "$version" == "${evidence[k3s_version]}" && "$datastore" == "${evidence[datastore]}" &&
"$status" == enabled_stable && "$rotation" == reencrypt_finished &&
"$hashes" == match && "$integrity" == match && "$api" == pass && "$node" == pass ]] || return 1
_ksre_require_same_context "$context"
}
k3s_secret_encryption_restore_evidence_main() {
local mode='' metadata='' result_file='' output=''
while (( $# > 0 )); do
case "$1" in
--emit-result|--record|--check)
[[ -z "$mode" ]] || { _ksre_usage >&2; return 2; }
mode="$1"; shift
;;
--bundle-metadata)
[[ $# -ge 2 && -z "$metadata" ]] || { _ksre_usage >&2; return 2; }
metadata="$2"; shift 2
;;
--result-file)
[[ $# -ge 2 && -z "$result_file" ]] || { _ksre_usage >&2; return 2; }
result_file="$2"; shift 2
;;
--output)
[[ $# -ge 2 && -z "$output" ]] || { _ksre_usage >&2; return 2; }
output="$2"; shift 2
;;
-h|--help) _ksre_usage; return 0 ;;
*) _ksre_usage >&2; return 2 ;;
esac
done
case "$mode" in
--emit-result)
[[ -n "$metadata" && -n "$output" && -z "$result_file" ]] || { _ksre_usage >&2; return 2; }
_ksre_emit_result "$metadata" "$output" || _ksre_fail
;;
--record)
[[ -n "$metadata" && -n "$result_file" && -z "$output" ]] || { _ksre_usage >&2; return 2; }
_ksre_record "$metadata" "$result_file" || _ksre_fail
;;
--check)
[[ -z "$metadata" && -z "$result_file" && -z "$output" ]] || { _ksre_usage >&2; return 2; }
_ksre_check || _ksre_fail
;;
*) _ksre_usage >&2; return 2 ;;
esac
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
PATH="$KSRE_SECURE_PATH"
export PATH
k3s_secret_encryption_restore_evidence_main "$@"
fi
+344
View File
@@ -0,0 +1,344 @@
#!/usr/bin/env bash
# Read-only k3s Secret-encryption inventory. Privileged execution is limited to
# bounded invocations of validated, root-owned system binaries; workspace code
# is never executed or sourced by sudo.
set -Eeuo pipefail
readonly KSE_SECURE_PATH='/usr/sbin:/usr/bin:/sbin:/bin'
readonly KSE_EXPECTED_VERSION='v1.36.2+k3s1'
readonly KSE_EXPECTED_NODE='donghyeon-system-product-name'
readonly KSE_K3S='/usr/local/bin/k3s'
readonly KSE_TIMEOUT='/usr/bin/timeout'
readonly KSE_SUDO='/usr/bin/sudo'
readonly KSE_SYSTEMCTL='/usr/bin/systemctl'
readonly KSE_STAT='/usr/bin/stat'
readonly KSE_SHA256SUM='/usr/bin/sha256sum'
readonly KSE_CAT='/usr/bin/cat'
readonly KSE_JQ='/usr/bin/jq'
readonly KSE_FIND='/usr/bin/find'
readonly KSE_ID='/usr/bin/id'
readonly KSE_TEST='/usr/bin/test'
if [[ "${BASH_SOURCE[0]}" == */* ]]; then
readonly KSE_SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
else
readonly KSE_SCRIPT_DIR='.'
fi
readonly KSE_ROOT="$(cd -- "${KSE_SCRIPT_DIR}/../.." && pwd -P)"
readonly KSE_LIBRARY_PATH="${KSE_ROOT}/scripts/lib/k3s-secret-encryption.sh"
readonly KSE_CONFIG_PATH='/var/lib/rancher/k3s/server/cred/encryption-config.json'
readonly KSE_STATE_PATH='/var/lib/rancher/k3s/server/cred/encryption-state.json'
kse_fail() {
printf 'ERROR: %s\n' "$*" >&2
return 1
}
kse_usage() {
cat <<'USAGE'
사용법:
bash scripts/validate/k3s-secret-encryption.sh [기대 상태] [--verified-output-dir DIR]
기대 상태(하나만 지정):
--expect-disabled
--expect-transition-start
--expect-enabled
--expect-reencrypted
--expect-reencrypted-restore # 격리 복구 host의 다른 node 이름 허용
USAGE
}
_kse_trusted_executable() {
local path="$1" metadata uid mode file_type
[[ -f "$path" && ! -L "$path" && -x "$path" ]] || return 1
metadata="$($KSE_STAT --format='%u:%a:%F' -- "$path" 2>/dev/null)" || return 1
IFS=: read -r uid mode file_type <<<"$metadata"
[[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ && "$file_type" == 'regular file' ]] || return 1
(( (8#$mode & 8#022) == 0 ))
}
_kse_validate_runtime_dependencies() {
local binary
for binary in \
"$KSE_TIMEOUT" "$KSE_SUDO" "$KSE_K3S" "$KSE_SYSTEMCTL" \
"$KSE_STAT" "$KSE_SHA256SUM" "$KSE_CAT" "$KSE_JQ" \
"$KSE_FIND" "$KSE_ID" "$KSE_TEST" \
/usr/bin/sed /usr/bin/sort; do
_kse_trusted_executable "$binary" || return 1
done
}
_kse_privileged_capture() {
local binary="$1"
shift
"$KSE_TIMEOUT" --signal=TERM --kill-after=1s 9s \
"$KSE_SUDO" --non-interactive -- "$binary" "$@" 2>/dev/null
}
_kse_validate_sudo_credentials() {
"$KSE_TIMEOUT" --signal=TERM --kill-after=1s 9s \
"$KSE_SUDO" --non-interactive --validate >/dev/null 2>&1
}
_kse_privileged_quiet() {
_kse_privileged_capture "$@" >/dev/null 2>&1
}
_kse_read_version() {
local raw version
raw="$(_kse_privileged_capture "$KSE_K3S" --version)" || return 1
version="$(/usr/bin/sed -n -E 's/^k3s version (v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+)([[:space:]].*)?$/\1/p' <<<"$raw")"
[[ "$version" == "$KSE_EXPECTED_VERSION" ]] || return 1
printf '%s\n' "$version"
}
_kse_read_status() {
_kse_privileged_capture "$KSE_K3S" secrets-encrypt status --output json
}
_kse_read_nodes() {
_kse_privileged_capture "$KSE_K3S" kubectl get nodes -o json
}
_kse_read_systemctl_show() {
_kse_privileged_capture "$KSE_SYSTEMCTL" show k3s \
--property=ExecStart --property=Environment --property=EnvironmentFiles
}
_kse_api_ready() {
_kse_privileged_quiet "$KSE_K3S" kubectl get --raw=/readyz
}
_kse_allowed_root_text_path() {
local path="$1"
[[ "$path" == /etc/rancher/k3s/config.yaml ||
"$path" == /etc/systemd/system/k3s.service.env ||
"$path" =~ ^/etc/rancher/k3s/config\.yaml\.d/[A-Za-z0-9._-]+\.yaml$ ]]
}
_kse_read_root_text() {
local path="$1" metadata uid mode
_kse_allowed_root_text_path "$path" || return 1
_kse_privileged_quiet "$KSE_TEST" -f "$path" || return 1
_kse_privileged_quiet "$KSE_TEST" ! -L "$path" || return 1
metadata="$(_kse_privileged_capture "$KSE_STAT" --format='%u:%a' -- "$path")" || return 1
IFS=: read -r uid mode <<<"$metadata"
[[ "$uid" == 0 && "$mode" =~ ^[0-7]{3,4}$ ]] || return 1
(( (8#$mode & 8#022) == 0 )) || return 1
_kse_privileged_capture "$KSE_CAT" -- "$path"
}
_kse_root_path_kind() {
local path="$1"
if _kse_privileged_quiet "$KSE_TEST" -f "$path"; then
printf 'file\n'
elif _kse_privileged_quiet "$KSE_TEST" -d "$path"; then
printf 'directory\n'
else
printf 'absent\n'
fi
}
_kse_detect_datastore() {
local service_show="$1"
_k3s_systemctl_show() { printf '%s\n' "$service_show"; }
_k3s_systemd_dropin_lines() { return 0; }
_k3s_read_file() { _kse_read_root_text "$1"; }
_k3s_list_yaml_files() {
local config="$1" file listing=''
[[ "$config" == /etc/rancher/k3s/config.yaml ]] || return 2
if _kse_privileged_quiet "$KSE_TEST" -f "$config"; then printf '%s\n' "$config"; fi
if _kse_privileged_quiet "$KSE_TEST" -d "${config}.d"; then
listing="$(_kse_privileged_capture "$KSE_FIND" "${config}.d" -maxdepth 1 -type f -name '*.yaml' -print)" || return 2
while IFS= read -r file; do
[[ -z "$file" ]] || _kse_allowed_root_text_path "$file" || return 2
done <<<"$listing"
[[ -z "$listing" ]] || printf '%s\n' "$listing" | /usr/bin/sort
fi
}
_k3s_datastore_local_evidence() {
local data_dir="$1" state_kind etcd_kind
[[ "$data_dir" == /var/lib/rancher/k3s ]] || { printf 'ambiguous\n'; return 0; }
state_kind="$(_kse_root_path_kind "${data_dir}/server/db/state.db")" || return 1
etcd_kind="$(_kse_root_path_kind "${data_dir}/server/db/etcd")" || return 1
if [[ "$state_kind" == file && "$etcd_kind" == directory ]]; then printf 'ambiguous\n';
elif [[ "$state_kind" == file ]]; then printf 'sqlite\n';
elif [[ "$etcd_kind" == directory ]]; then printf 'embedded-etcd\n';
else printf 'none\n'; fi
}
detect_k3s_datastore
}
_kse_root_metadata() {
local path="$1" metadata
metadata="$(_kse_privileged_capture "$KSE_STAT" --format='%u:%a:%F' -- "$path")" || return 1
[[ "$metadata" == '0:600:regular file' ]] || return 1
printf '%s\n' "$metadata"
}
_kse_integrity_from_root_evidence() {
local annotation="$1" stage="$2" config_metadata state_metadata
local config_hash_line config_hash state_with_sentinel state_value result
config_metadata="$(_kse_root_metadata "$KSE_CONFIG_PATH")" || { printf 'mismatch\n'; return 1; }
state_metadata="$(_kse_root_metadata "$KSE_STATE_PATH")" || { printf 'mismatch\n'; return 1; }
config_hash_line="$(_kse_privileged_capture "$KSE_SHA256SUM" -- "$KSE_CONFIG_PATH")" || { printf 'mismatch\n'; return 1; }
config_hash="${config_hash_line%% *}"
[[ "$config_hash_line" == "${config_hash} ${KSE_CONFIG_PATH}" && "$config_hash" =~ ^[0-9a-f]{64}$ ]] || { printf 'mismatch\n'; return 1; }
state_with_sentinel="$(_kse_privileged_capture "$KSE_CAT" -- "$KSE_STATE_PATH"; printf .)" || { printf 'mismatch\n'; return 1; }
state_value="${state_with_sentinel%.}"
result="$(verify_local_encryption_config_integrity_evidence \
"$config_metadata" "$state_metadata" "$config_hash" "$state_value" \
"$annotation" "$stage" 2>/dev/null || true)"
[[ "$result" == match ]] || { printf 'mismatch\n'; return 1; }
printf 'match\n'
}
_kse_validate_output_dir() {
local output_dir="$1" physical metadata uid
[[ "$output_dir" =~ ^/tmp/platform-k3s-encryption\.[A-Za-z0-9]+$ ]] || return 1
[[ -d /tmp && ! -L /tmp && -d "$output_dir" && ! -L "$output_dir" ]] || return 1
[[ "$(cd -P -- /tmp && pwd -P)" == /tmp ]] || return 1
physical="$(cd -P -- "$output_dir" && pwd -P)" || return 1
[[ "$physical" == "$output_dir" ]] || return 1
uid="$($KSE_ID -u)" || return 1
metadata="$($KSE_STAT --format='%u:%a:%F' -- "$output_dir" 2>/dev/null)" || return 1
[[ "$metadata" == "${uid}:700:directory" ]] || return 1
[[ -z "$($KSE_FIND "$output_dir" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]
}
_kse_write_verified_output() {
local output_dir="$1" invoking_uid metadata physical entry
local -a entries=()
local directory_fd
_kse_validate_output_dir "$output_dir" || return 1
invoking_uid="$($KSE_ID -u)" || return 1
exec {directory_fd}<"$output_dir" || return 1
physical="$(cd -P -- "/proc/self/fd/${directory_fd}" && pwd -P)" || { exec {directory_fd}<&-; return 1; }
[[ "$physical" == "$output_dir" ]] || { exec {directory_fd}<&-; return 1; }
metadata="$($KSE_STAT -L --format='%u:%a:%F' -- "/proc/self/fd/${directory_fd}" 2>/dev/null)" || { exec {directory_fd}<&-; return 1; }
[[ "$metadata" == "${invoking_uid}:700:directory" ]] || { exec {directory_fd}<&-; return 1; }
(
umask 077
set -C
printf 'version=%s\nserver_count=%s\nnode_name=%s\nnode_uid=%s\nnode_ready=%s\ndatastore=%s\nencryption=%s\nrotation=%s\nserver_hashes=%s\nintegrity=%s\napi_readyz=%s\n' \
"$version" "$server_count" "$node_name" "$node_uid" "$node_ready" "$datastore" "$encryption" "$rotation" "$server_hashes" "$integrity" "$api_readyz" \
>"/proc/self/fd/${directory_fd}/inventory.env"
printf '%s\n' "$status_sha256" >"/proc/self/fd/${directory_fd}/status.sha256"
) || { exec {directory_fd}<&-; return 1; }
metadata="$($KSE_STAT --format='%u:%a:%F' -- "/proc/self/fd/${directory_fd}/inventory.env" "/proc/self/fd/${directory_fd}/status.sha256" 2>/dev/null)" || { exec {directory_fd}<&-; return 1; }
[[ "$metadata" == "${invoking_uid}:600:regular file"$'\n'"${invoking_uid}:600:regular file" ]] || { exec {directory_fd}<&-; return 1; }
while IFS= read -r -d '' entry; do entries+=("${entry##*/}"); done < <(
"$KSE_FIND" -H "/proc/self/fd/${directory_fd}" -mindepth 1 -maxdepth 1 -print0 2>/dev/null
)
exec {directory_fd}<&-
(( ${#entries[@]} == 2 )) || return 1
[[ " ${entries[*]} " == *' inventory.env '* && " ${entries[*]} " == *' status.sha256 '* ]]
}
k3s_secret_encryption_main() {
local expectation='' output_dir='' output_dir_seen=false
local raw_status canonical_status status_class stage status_hash_line
local nodes_json service_show annotations='' annotation='' first_annotation=''
local version server_count node_name node_uid node_ready datastore encryption rotation
local server_hashes=\mismatch integrity=mismatch api_readyz status_sha256
PATH="$KSE_SECURE_PATH"
export PATH
unset K3S_CONFIG_DIR K3S_SYSTEMD_DROPIN_DIR
while (( $# > 0 )); do
case "$1" in
--expect-disabled|--expect-transition-start|--expect-enabled|--expect-reencrypted|--expect-reencrypted-restore)
[[ -z "$expectation" ]] || { kse_usage >&2; return 2; }
expectation="$1"
;;
--verified-output-dir)
(( $# >= 2 )) || { kse_usage >&2; return 2; }
! "$output_dir_seen" && [[ -n "$2" ]] || { kse_usage >&2; return 2; }
output_dir="$2"; output_dir_seen=true; shift
;;
--help|-h) kse_usage; return 0 ;;
*) kse_usage >&2; return 2 ;;
esac
shift
done
_kse_validate_runtime_dependencies || { kse_fail 'trusted runtime dependency validation failed'; return 1; }
! "$output_dir_seen" || _kse_validate_output_dir "$output_dir" || { kse_fail 'verified output directory is unsafe'; return 1; }
_kse_validate_sudo_credentials || {
kse_fail 'noninteractive sudo authorization is unavailable; run sudo -v in the same terminal, then rerun the validator'
return 1
}
# shellcheck source=/dev/null
source "$KSE_LIBRARY_PATH"
version="$(_kse_read_version)" || { kse_fail 'k3s version is not the required production version'; return 1; }
raw_status="$(_kse_read_status)" || { kse_fail 'read-only encryption status failed'; return 1; }
canonical_status="$($KSE_JQ -ceS . <<<"$raw_status" 2>/dev/null)" || { kse_fail 'encryption status is invalid'; return 1; }
status_class="$(classify_encryption_status "$canonical_status")"
[[ "$status_class" != invalid ]] || { kse_fail 'encryption status is invalid'; return 1; }
status_hash_line="$(printf '%s' "$canonical_status" | "$KSE_SHA256SUM")" || return 1
status_sha256="${status_hash_line%% *}"
[[ "$status_sha256" =~ ^[0-9a-f]{64}$ ]] || return 1
encryption=Disabled; rotation=unsafe
case "$status_class" in
disabled_no_config) rotation=none ;;
transition_start) rotation=start ;;
enabled_stable)
encryption=Enabled
stage="$($KSE_JQ -er '.stage' <<<"$canonical_status")" || return 1
[[ "$stage" == start || "$stage" == reencrypt_finished ]] || return 1
rotation="$stage"
;;
unsafe_transition|hash_mismatch) encryption=Enabled ;;
*) return 1 ;;
esac
nodes_json="$(_kse_read_nodes)" || { kse_fail 'read-only node inventory failed'; return 1; }
server_count="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))] | length' <<<"$nodes_json" 2>/dev/null)" || return 1
[[ "$server_count" == 1 ]] || { kse_fail 'server count is not exactly one'; return 1; }
node_name="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))][0].metadata.name' <<<"$nodes_json" 2>/dev/null)" || return 1
node_uid="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null))][0].metadata.uid | if type == "string" and test("^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") then . else error("invalid node uid") end' <<<"$nodes_json" 2>/dev/null)" || { kse_fail 'server node UID is invalid'; return 1; }
if [[ "$expectation" != --expect-reencrypted-restore ]]; then
[[ "$node_name" == "$KSE_EXPECTED_NODE" ]] || { kse_fail 'server node identity mismatch'; return 1; }
fi
node_ready="$($KSE_JQ -er '[.items[] | select(.metadata.name == $name)][0].status.conditions | [ .[] | select(.type == "Ready") ] | if length == 1 and .[0].status == "True" then "Ready" else error("not ready") end' --arg name "$node_name" <<<"$nodes_json" 2>/dev/null)" || { kse_fail 'server node is not Ready'; return 1; }
annotations="$($KSE_JQ -er '[.items[] | select((.metadata.labels["node-role.kubernetes.io/control-plane"]? != null) or (.metadata.labels["node-role.kubernetes.io/master"]? != null)) | .metadata.annotations["k3s.io/encryption-config-hash"]?] | if length > 0 and all(.[]; type == "string" and test("^[a-z_]+-[0-9a-f]{64}$")) then .[] else empty end' <<<"$nodes_json" 2>/dev/null)" || annotations=''
if [[ -n "$annotations" ]]; then
first_annotation="${annotations%%$'\n'*}"; annotation="$first_annotation"
while IFS= read -r annotation; do [[ "$annotation" == "$first_annotation" ]] || first_annotation=''; done <<<"$annotations"
[[ -z "$first_annotation" ]] || server_hashes=match
fi
service_show="$(_kse_read_systemctl_show)" || { kse_fail 'read-only systemd inventory failed'; return 1; }
datastore="$(_kse_detect_datastore "$service_show")" || datastore=ambiguous
case "$datastore" in sqlite|embedded-etcd|external|ambiguous) ;; *) datastore=ambiguous ;; esac
if [[ "$server_hashes" == match && ( "$status_class" == transition_start || "$status_class" == enabled_stable ) ]]; then
integrity="$(_kse_integrity_from_root_evidence "$first_annotation" "$rotation" 2>/dev/null || true)"
[[ "$integrity" == match ]] || integrity=mismatch
fi
if _kse_api_ready; then api_readyz=pass; else api_readyz=fail; fi
printf 'K3s version: %s\nServer count: %s\nNode: %s %s\nDatastore: %s\nEncryption status: %s\nRotation stage: %s\nServer annotation hashes: %s\nLocal config/state/annotation integrity: %s\nAPI readyz: %s\n' \
"$version" "$server_count" "$node_name" "$node_ready" "$datastore" "$encryption" "$rotation" "$server_hashes" "$integrity" "$api_readyz"
[[ "$api_readyz" == pass ]] || return 1
case "$expectation" in
'') [[ "$status_class" != unsafe_transition && "$status_class" != hash_mismatch ]] || return 1 ;;
--expect-disabled) [[ "$status_class" == disabled_no_config ]] || return 1 ;;
--expect-transition-start) [[ "$status_class" == transition_start ]] || return 1 ;;
--expect-enabled) [[ "$status_class" == enabled_stable ]] || return 1 ;;
--expect-reencrypted|--expect-reencrypted-restore)
[[ "$status_class" == enabled_stable && "$rotation" == reencrypt_finished &&
"$server_hashes" == match && "$integrity" == match ]] || return 1
;;
esac
! "$output_dir_seen" || _kse_write_verified_output "$output_dir" || { kse_fail 'verified output write failed'; return 1; }
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
k3s_secret_encryption_main "$@"
fi
File diff suppressed because it is too large Load Diff
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
readonly KUBE_VERSION="1.36.2"
readonly PGADMIN_SHA="0f409becdc6a811e852f384cb53b3f11cfb652a19d77cb952a8a176071694f4c"
readonly AISTOR_SHA="50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e"
readonly IMAGE_SHA="sha256:66a300a7ecdcc1f325af0c430315329bca46cd4a7067227d6899802238167c6e"
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly PG_CACHE="$ROOT/services/pgadmin/.helm/charts/pgadmin4-helm-9.16.0"
readonly AI_CACHE="$ROOT/services/minio-aistor/base/deployment/.helm/charts/aistor-objectstore-1.0.16"
readonly NAMES="admin-namespace pgadmin-local-pv coredns-custom aistor-admin-oidc pgadmin"
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat <<'USAGE'
사용법:
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/validate/render-admin-services.sh
적용 스크립트 내부 handoff:
bash scripts/validate/render-admin-services.sh \
--verified-output-dir /tmp/platform-admin-apply.XXXXXX
고정한 공식 차트의 SHA-256과 렌더 결과를 검사합니다. 변경은 수행하지 않습니다.
USAGE
}
out=""
if (( $# == 0 )); then
:
elif (( $# == 2 )) && [[ "$1" == "--verified-output-dir" ]]; then
out="$2"
else
usage >&2
exit 2
fi
for cmd in awk find helm install kubectl rg sed sha256sum stat tar wc tr; do
if [[ "$cmd" == helm ]] && [[ -v PLATFORM_HELM_BIN ]]; then
continue
fi
command -v "$cmd" >/dev/null 2>&1 || fail "$cmd 명령이 필요합니다"
done
if [[ -v PLATFORM_HELM_BIN ]]; then
[[ "$PLATFORM_HELM_BIN" == /* && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN은 실행 가능한 절대 경로여야 합니다"
readonly HELM="$PLATFORM_HELM_BIN"
else
readonly HELM="$(command -v helm)"
fi
[[ "$("$HELM" version --template '{{.Version}}')" == "$EXPECTED_HELM_VERSION" ]] || \
fail "Helm은 정확히 $EXPECTED_HELM_VERSION이어야 합니다"
kv="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')"
[[ "$kv" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
fail "Kustomize는 정확히 $EXPECTED_KUSTOMIZE_VERSION이어야 합니다"
if [[ -n "$out" ]]; then
[[ "$out" == /tmp/platform-admin-apply.* ]] || fail "handoff 경로 형식이 잘못됐습니다"
[[ -d "$out" && ! -L "$out" && -O "$out" ]] || fail "handoff 경로가 안전하지 않습니다"
[[ "$(stat --format='%a' "$out")" == 700 ]] || fail "handoff 권한은 0700이어야 합니다"
[[ -z "$(find "$out" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail "handoff 경로가 비어 있지 않습니다"
fi
umask 077
work="$(mktemp -d /tmp/platform-admin-render.XXXXXX)"
pg_created=false
ai_created=false
cleanup() {
rc=$?
set +e
[[ "$pg_created" == true ]] && rm -rf -- "$PG_CACHE"
[[ "$ai_created" == true ]] && rm -rf -- "$AI_CACHE"
rmdir -- "$ROOT/services/pgadmin/.helm/charts" "$ROOT/services/pgadmin/.helm" 2>/dev/null || true
rmdir -- "$ROOT/services/minio-aistor/base/deployment/.helm/charts" \
"$ROOT/services/minio-aistor/base/deployment/.helm" 2>/dev/null || true
case "$work" in
/tmp/platform-admin-render.*) rm -rf -- "$work" ;;
esac
exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
prepare() {
label="$1"
ref="$2"
name="$3"
version="$4"
expected="$5"
cache="$6"
archive="$work/$name-$version.tgz"
[[ ! -e "$cache" && ! -L "$cache" ]] || fail "생성 캐시가 이미 있습니다: $cache"
if [[ "$ref" == oci://* ]]; then
"$HELM" pull "$ref" --version "$version" --destination "$work"
else
"$HELM" pull "$name" --repo "$ref" --version "$version" --destination "$work"
fi
[[ -f "$archive" ]] || fail "$label archive가 없습니다"
actual="$(sha256sum "$archive" | awk '{print $1}')"
[[ "$actual" == "$expected" ]] || fail "$label SHA-256 불일치: $actual"
mkdir -p -- "$cache"
[[ "$cache" == "$PG_CACHE" ]] && pg_created=true
[[ "$cache" == "$AI_CACHE" ]] && ai_created=true
tar -xzf "$archive" -C "$cache"
[[ -f "$cache/$name/Chart.yaml" ]] || fail "$label Chart.yaml이 없습니다"
printf '검증 %-18s SHA-256 %s\n' "$label" "$actual"
}
plain() {
kubectl kustomize "$ROOT/$2" >"$work/$1.yaml"
[[ -s "$work/$1.yaml" ]] || fail "$1 렌더 결과가 비었습니다"
}
with_helm() {
kubectl kustomize --enable-helm --helm-command "$HELM" \
--helm-kube-version "$KUBE_VERSION" "$ROOT/$2" >"$work/$1.yaml"
[[ -s "$work/$1.yaml" ]] || fail "$1 렌더 결과가 비었습니다"
}
cd -- "$ROOT"
if rg -n --glob '*.yaml' --glob '*.yml' --glob '!**/.helm/**' \
'^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' infrastructure services bootstrap; then
fail "Git 관리 대상에서 Secret manifest를 발견했습니다"
fi
if rg -n '^[[:space:]]*hostAliases:' services/gitea/profiles/oidc; then
fail "Gitea hostAliases가 남아 있습니다"
fi
prepare pgadmin oci://docker.io/dpage/pgadmin4-helm pgadmin4-helm 9.16.0 "$PGADMIN_SHA" "$PG_CACHE"
prepare aistor-objectstore https://helm.min.io/ aistor-objectstore 1.0.16 "$AISTOR_SHA" "$AI_CACHE"
plain admin-namespace infrastructure/namespaces/admin-tools
plain pgadmin-local-pv infrastructure/storage/pgadmin-local-pv
plain coredns-custom infrastructure/networking/private-dns/kubernetes
with_helm aistor-admin-oidc services/minio-aistor/profiles/admin-oidc
with_helm pgadmin services/pgadmin
if rg -n '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' "$work"/*.yaml; then
fail "렌더 결과에 Secret 리소스가 있습니다"
fi
[[ "$(rg -c -F "image: docker.io/dpage/pgadmin4@$IMAGE_SHA" "$work/pgadmin.yaml")" == 3 ]] || \
fail "pgAdmin main/init 이미지 세 개의 digest 고정이 깨졌습니다"
rg -q -F 'type: Recreate' "$work/pgadmin.yaml" || fail "pgAdmin Recreate 전략이 없습니다"
rg -q -F 'storageClassName: ssd-local-pgadmin-retain' "$work/pgadmin.yaml" || fail "pgAdmin StorageClass가 다릅니다"
[[ "$(rg -c -F 'value: db-admin.learn.hyeonworks.com' "$work/pgadmin.yaml")" == 3 ]] || \
fail "pgAdmin 세 probe의 Host header가 고정되지 않았습니다"
rg -q -F 'host: db-admin.learn.hyeonworks.com' "$work/pgadmin.yaml" || fail "pgAdmin Ingress가 없습니다"
[[ "$(rg -c -F 'ingressClassName: traefik' "$work/pgadmin.yaml")" == 1 ]] || \
fail "pgAdmin IngressClass가 traefik으로 고정되지 않았습니다"
if rg -q -F 'kubernetes.io/ingress.class' "$work/pgadmin.yaml"; then
fail "pgAdmin Ingress에 레거시 ingress.class annotation이 남아 있습니다"
fi
rg -q -F 'namespace: platform-data' "$work/pgadmin.yaml" || fail "PostgreSQL 정책 namespace가 다릅니다"
bad_pgadmin_namespaces="$(
awk '
function verify() {
if (pgadmin_instance && namespace != "platform-admin") {
printf "%s/%s=%s\n", kind, name, namespace == "" ? "missing" : namespace
}
}
function reset() {
kind=""
name=""
namespace=""
in_metadata=0
in_labels=0
pgadmin_instance=0
}
/^---$/ {
verify()
reset()
next
}
/^kind:/ {
kind=$2
next
}
/^metadata:$/ {
in_metadata=1
in_labels=0
next
}
in_metadata && /^ name:/ && name == "" {
name=$2
next
}
in_metadata && /^ namespace:/ && namespace == "" {
namespace=$2
next
}
in_metadata && /^ labels:$/ {
in_labels=1
next
}
in_labels && /^ app.kubernetes.io\/instance: pgadmin$/ {
pgadmin_instance=1
next
}
in_metadata && /^[^ ]/ {
in_metadata=0
in_labels=0
}
END {
verify()
}
' "$work/pgadmin.yaml"
)"
[[ -z "$bad_pgadmin_namespaces" ]] || \
fail "pgAdmin instance 리소스 namespace 오류: $bad_pgadmin_namespaces"
rg -q -F 'host: storage-admin.learn.hyeonworks.com' "$work/aistor-admin-oidc.yaml" || fail "AIStor Console Ingress가 없습니다"
rg -q -F 'name: MINIO_IDENTITY_OPENID_CLIENT_SECRET' "$work/aistor-admin-oidc.yaml" || fail "AIStor OIDC Secret 참조가 없습니다"
if rg -q -F 'name: MINIO_IDENTITY_OPENID_VENDOR' "$work/aistor-admin-oidc.yaml"; then
fail "claim-only OIDC profile에 Keycloak Admin API vendor 설정이 있습니다"
fi
if rg -q -F 'host: s3.' "$work/aistor-admin-oidc.yaml"; then
fail "S3 API Ingress가 생성됐습니다"
fi
for name in $NAMES; do
bytes="$(wc -c <"$work/$name.yaml" | tr -d '[:space:]')"
sha="$(sha256sum "$work/$name.yaml" | awk '{print $1}')"
printf '렌더 %-20s %8s bytes SHA-256 %s\n' "$name" "$bytes" "$sha"
if [[ -n "$out" ]]; then
install -m 0600 "$work/$name.yaml" "$out/$name.yaml"
fi
done
if [[ -n "$out" ]]; then
count="$(find "$out" -mindepth 1 -maxdepth 1 -type f | wc -l | tr -d '[:space:]')"
[[ "$count" == 5 ]] || fail "handoff 파일 수가 다릅니다"
fi
printf 'ADMIN SERVICES STATIC RENDER PASS\n'
@@ -0,0 +1,825 @@
#!/usr/bin/env bash
# Static renderer and sourceable assertion library for the observability access
# layer. It intentionally reuses the pinned core renderer for Helm/cache and
# image/credential boundaries instead of maintaining a second downloader.
set -o pipefail
readonly ACCESS_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
# shellcheck source=render-observability-core.sh
source "$ACCESS_ROOT/scripts/validate/render-observability-core.sh"
access_output_names() {
case "$1" in
grafana) printf '%s\n' grafana.yaml ;;
blackbox) printf '%s\n' blackbox.yaml ;;
targets) printf '%s\n' targets.yaml ;;
rules-alerts) printf '%s\n' dashboards.yaml rules.yaml alertmanager.yaml ;;
complete) printf '%s\n' grafana.yaml blackbox.yaml targets.yaml dashboards.yaml rules.yaml alertmanager.yaml private-dns.yaml ;;
*) return 1 ;;
esac
}
access_component_requires_inventory() {
[[ "$1" == rules-alerts || "$1" == complete ]]
}
access_secure_render_context() {
umask 077
}
_access_assert() {
local mode=$1
shift
python3 - "$mode" "$@" <<'PY'
import ipaddress
import json
import pathlib
import re
import sys
import yaml
class AccessSafeLoader(yaml.SafeLoader):
pass
AccessSafeLoader.add_constructor(
"tag:yaml.org,2002:value",
lambda loader, node: loader.construct_scalar(node),
)
mode = sys.argv[1]
paths = [pathlib.Path(value) for value in sys.argv[2:]]
def reject(message):
print(f"REJECT: {message}", file=sys.stderr)
raise SystemExit(1)
def load(path):
with path.open(encoding="utf-8") as stream:
return [item for item in yaml.load_all(stream, Loader=AccessSafeLoader) if item is not None]
def one(items, kind, name, namespace=None):
matches = [
item for item in items
if item.get("kind") == kind
and (item.get("metadata") or {}).get("name") == name
and (namespace is None or (item.get("metadata") or {}).get("namespace") == namespace)
]
if len(matches) != 1:
reject(f"expected one {kind}/{name}, found {len(matches)}")
return matches[0]
def labels(item):
return (item.get("metadata") or {}).get("labels") or {}
def exact_policy_specs(items, expected):
policies = {
(item.get("metadata") or {}).get("name"): item.get("spec") or {}
for item in items if item.get("kind") == "NetworkPolicy"
}
for name, spec in expected.items():
if policies.get(name) != spec:
reject(f"NetworkPolicy spec is not exact: {name}")
def selector_matches(selector, pod_labels):
selector = selector or {}
for key, value in (selector.get("matchLabels") or {}).items():
if pod_labels.get(key) != value:
return False
for expression in selector.get("matchExpressions") or []:
key = expression.get("key")
operator = expression.get("operator")
values = expression.get("values") or []
value = pod_labels.get(key)
if operator == "In" and value not in values:
return False
if operator == "NotIn" and value in values:
return False
if operator == "Exists" and value is None:
return False
if operator == "DoesNotExist" and value is not None:
return False
if operator not in {"In", "NotIn", "Exists", "DoesNotExist"}:
reject("NetworkPolicy selector operator is unsupported")
return True
def exact_selecting_policy_names(items, namespace, pod_labels, expected):
selected = {
(item.get("metadata") or {}).get("name")
for item in items
if item.get("kind") == "NetworkPolicy"
and (item.get("metadata") or {}).get("namespace") == namespace
and selector_matches((item.get("spec") or {}).get("podSelector") or {}, pod_labels)
}
if selected != set(expected):
reject("NetworkPolicy selecting set is not exact")
if mode == "grafana":
items, policies = load(paths[0]), load(paths[1])
service = one(items, "Service", "grafana", "observability")
if (service.get("spec") or {}).get("type") != "ClusterIP":
reject("Grafana Service is not ClusterIP")
ingress = one(items, "Ingress", "grafana", "observability")
spec = ingress.get("spec") or {}
rules = spec.get("rules") or []
if spec.get("ingressClassName") != "traefik" or spec.get("tls") not in (None, []):
reject("Grafana Ingress boundary is not exact")
if len(rules) != 1 or rules[0].get("host") != "grafana.learn.hyeonworks.com":
reject("Grafana host is not exact")
deployment = one(items, "Deployment", "grafana", "observability")
pod = (((deployment.get("spec") or {}).get("template") or {}).get("spec") or {})
if pod.get("automountServiceAccountToken") is not False:
reject("Grafana Pod token automount is not disabled")
containers = {container.get("name"): container for container in pod.get("containers") or []}
if set(containers) != {"grafana", "grafana-sc-dashboard"}:
reject("Grafana container set is not exact")
token_volume = "dashboard-sidecar-api-access"
main_mounts = {mount.get("name") for mount in containers["grafana"].get("volumeMounts") or []}
side_mounts = {mount.get("name") for mount in containers["grafana-sc-dashboard"].get("volumeMounts") or []}
if token_volume in main_mounts or token_volume not in side_mounts:
reject("dashboard API token mount is not sidecar-only")
volume = next((volume for volume in pod.get("volumes") or [] if volume.get("name") == token_volume), None)
sources = (((volume or {}).get("projected") or {}).get("sources") or [])
if len(sources) != 3 or not any("serviceAccountToken" in source for source in sources) \
or not any("configMap" in source for source in sources) \
or not any("downwardAPI" in source for source in sources):
reject("sidecar projected API volume is not exact")
one(items, "Role", "grafana-dashboard-sidecar", "observability")
if any(item.get("kind") == "ClusterRole" for item in items):
reject("cluster-wide Grafana RBAC is forbidden")
config = one(items, "ConfigMap", "grafana", "observability")
ini = ((config.get("data") or {}).get("grafana.ini") or "")
required = (
"allowed_groups = /platform-observability-admins /platform-observability-viewers",
"role_attribute_strict = true",
"allow_assign_grafana_admin = false",
"&& 'Admin' || contains(groups[*], '/platform-observability-viewers') && 'Viewer' || null",
)
if not all(token in ini for token in required) or "&& 'Editor'" in ini or "|| 'Viewer'" in ini:
reject("Grafana OIDC role mapping is not fail-closed")
exact_policy_specs(policies, {
"observability-default-deny": {
"podSelector": {}, "policyTypes": ["Ingress", "Egress"],
},
"observability-allow-grafana-ingress": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
}},
"policyTypes": ["Ingress"],
"ingress": [
{"from": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}},
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "traefik-kube-system",
"app.kubernetes.io/name": "traefik",
}},
}], "ports": [{"protocol": "TCP", "port": 3000}]},
{"from": [{"podSelector": {"matchLabels": {
"app.kubernetes.io/name": "prometheus",
}}}], "ports": [{"protocol": "TCP", "port": 3000}]},
{"from": [{"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "blackbox-exporter",
"app.kubernetes.io/name": "prometheus-blackbox-exporter",
}}}], "ports": [{"protocol": "TCP", "port": 3000}]},
],
},
"observability-allow-grafana-datasources": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
}},
"policyTypes": ["Egress"],
"egress": [
{"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "prometheus"}}}],
"ports": [{"protocol": "TCP", "port": 9090}]},
{"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "loki"}}}],
"ports": [{"protocol": "TCP", "port": 3100}]},
{"to": [{"podSelector": {"matchLabels": {"app.kubernetes.io/name": "tempo"}}}],
"ports": [{"protocol": "TCP", "port": 3200}]},
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}],
"ports": [{"protocol": "TCP", "port": 443}]},
],
},
"observability-allow-grafana-dashboard-api": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
}},
"policyTypes": ["Egress"],
"egress": [
{"to": [{"ipBlock": {"cidr": "10.43.0.1/32"}}],
"ports": [{"protocol": "TCP", "port": 443}]},
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}],
"ports": [{"protocol": "TCP", "port": 6443}]},
],
},
})
exact_selecting_policy_names(policies, "observability", {
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
}, {
"observability-default-deny", "observability-allow-dns",
"observability-allow-grafana-ingress", "observability-allow-grafana-datasources",
"observability-allow-grafana-dashboard-api",
})
elif mode == "blackbox":
items = load(paths[0])
service = one(items, "Service", "blackbox-exporter", "observability")
if (service.get("spec") or {}).get("type") != "ClusterIP":
reject("Blackbox Service is public")
deployment = one(items, "Deployment", "blackbox-exporter", "observability")
pod = (((deployment.get("spec") or {}).get("template") or {}).get("spec") or {})
if pod.get("automountServiceAccountToken") is not False or len(pod.get("containers") or []) != 1:
reject("Blackbox Pod attack surface is not exact")
expected = {
"platform-public-edge": {
"https://git.learn.hyeonworks.com/api/healthz",
"https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration",
},
"platform-private-edge": {
"https://grafana.learn.hyeonworks.com/", "https://storage-admin.learn.hyeonworks.com/",
"https://db-admin.learn.hyeonworks.com/",
},
"platform-private-internal": {
"http://grafana.observability.svc.cluster.local/api/health",
"http://pgadmin.platform-admin.svc.cluster.local/misc/ping",
"http://minio-aistor-console.object-storage.svc.cluster.local:9090/",
},
}
for name, targets in expected.items():
probe = one(items, "Probe", name, "observability")
actual = set((((probe.get("spec") or {}).get("targets") or {}).get("staticConfig") or {}).get("static") or [])
if actual != targets:
reject(f"Blackbox target set drifted: {name}")
exact_policy_specs(items, {
"observability-allow-prometheus-to-blackbox": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "blackbox-exporter",
"app.kubernetes.io/name": "prometheus-blackbox-exporter",
}},
"policyTypes": ["Ingress"],
"ingress": [{
"from": [{"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "observability-core-kube-pr-prometheus",
"app.kubernetes.io/name": "prometheus",
}}}],
"ports": [{"protocol": "TCP", "port": 9115}],
}],
},
"observability-allow-blackbox-egress": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "blackbox-exporter",
"app.kubernetes.io/name": "prometheus-blackbox-exporter",
}},
"policyTypes": ["Egress"],
"egress": [
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}},
"podSelector": {"matchLabels": {"k8s-app": "kube-dns"}},
}], "ports": [{"protocol": "UDP", "port": 53}, {"protocol": "TCP", "port": 53}]},
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}],
"ports": [{"protocol": "TCP", "port": 443}]},
{"to": [{"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "grafana", "app.kubernetes.io/name": "grafana",
}}}], "ports": [{"protocol": "TCP", "port": 3000}]},
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "platform-admin"}},
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "pgadmin", "app.kubernetes.io/name": "pgadmin4",
}},
}], "ports": [{"protocol": "TCP", "port": 5050}]},
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "object-storage"}},
"podSelector": {"matchLabels": {"aistor.min.io/objectStore": "minio-aistor"}},
}], "ports": [{"protocol": "TCP", "port": 9090}]},
],
},
})
if {
(item.get("metadata") or {}).get("name")
for item in items if item.get("kind") == "NetworkPolicy"
} != {"observability-allow-prometheus-to-blackbox", "observability-allow-blackbox-egress"}:
reject("Blackbox NetworkPolicy name set is not exact")
elif mode == "targets":
items = load(paths[0])
monitors = [(item.get("kind"), (item.get("metadata") or {}).get("namespace"), (item.get("metadata") or {}).get("name"))
for item in items if item.get("kind") in {"ServiceMonitor", "PodMonitor"}]
if sorted(monitors) != sorted([
("PodMonitor", "platform-data", "platform-postgres"),
("ServiceMonitor", "object-storage", "aistor-bucket-usage"),
]):
reject("manual target monitor set is not exact")
exact_policy_specs(items, {
"traefik-preserve-ingress-and-allow-prometheus-metrics": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik",
}},
"policyTypes": ["Ingress"],
"ingress": [
{"ports": [
{"protocol": "TCP", "port": 8000}, {"protocol": "TCP", "port": 8443},
{"protocol": "TCP", "port": 8080},
]},
{"from": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "observability"}},
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "observability-core-kube-pr-prometheus",
"app.kubernetes.io/name": "prometheus",
}},
}], "ports": [{"protocol": "TCP", "port": 9100}]},
],
},
"observability-allow-prometheus-platform-targets": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "observability-core-kube-pr-prometheus",
"app.kubernetes.io/name": "prometheus",
}},
"policyTypes": ["Egress"],
"egress": [
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "gitea"}},
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "gitea", "app.kubernetes.io/name": "gitea",
}},
}], "ports": [{"protocol": "TCP", "port": 3000}]},
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "keycloak"}},
"podSelector": {"matchLabels": {
"app": "keycloak", "app.kubernetes.io/instance": "keycloak",
"app.kubernetes.io/managed-by": "keycloak-operator",
}},
}], "ports": [{"protocol": "TCP", "port": 9000}]},
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "platform-data"}},
"podSelector": {"matchLabels": {"cnpg.io/cluster": "platform-postgres"}},
}], "ports": [{"protocol": "TCP", "port": 9187}]},
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "object-storage"}},
"podSelector": {"matchLabels": {"aistor.min.io/objectStore": "minio-aistor"}},
}], "ports": [{"protocol": "TCP", "port": 9000}]},
{"to": [{
"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}},
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik",
}},
}], "ports": [{"protocol": "TCP", "port": 9100}]},
],
},
})
if {
(item.get("metadata") or {}).get("name")
for item in items if item.get("kind") == "NetworkPolicy"
} != {
"traefik-preserve-ingress-and-allow-prometheus-metrics",
"observability-allow-prometheus-platform-targets",
}:
reject("target NetworkPolicy name set is not exact")
elif mode == "rules-alerts":
dashboards, rules, alertmanager = map(load, paths)
dashboard_names = sorted((item.get("metadata") or {}).get("name") for item in dashboards)
expected_dashboards = sorted([
"grafana-dashboard-kubernetes-node", "grafana-dashboard-workload-health",
"grafana-dashboard-platform-services", "grafana-dashboard-observability-backends",
"grafana-dashboard-https-endpoints",
])
if dashboard_names != expected_dashboards or any(labels(item).get("grafana_dashboard") != "1" for item in dashboards):
reject("dashboard ConfigMap set is not exact")
rule_names = sorted((item.get("metadata") or {}).get("name") for item in rules)
if rule_names != sorted(["platform-aistor-storage-quota", "platform-certificate-probes", "platform-observability-core", "platform-verified-services"]):
reject("PrometheusRule set is not exact")
am = one(alertmanager, "Alertmanager", "observability-core-kube-pr-alertmanager", "observability")
if (((am.get("spec") or {}).get("alertmanagerConfiguration") or {}).get("name")) != "platform-alertmanager":
reject("global AlertmanagerConfig reference is absent")
config = one(alertmanager, "AlertmanagerConfig", "platform-alertmanager", "observability")
receivers = (config.get("spec") or {}).get("receivers") or []
slack_receiver = next((receiver for receiver in receivers if receiver.get("name") == "platform-slack"), {})
slack = ((slack_receiver.get("slackConfigs") or [{}])[0])
api_url = slack.get("apiURL") or {}
if api_url != {"name": "alertmanager-slack-webhook", "key": "url"} or slack.get("sendResolved") is not True:
reject("Slack Secret selector or resolved route is not exact")
one(alertmanager, "NetworkPolicy", "observability-allow-alertmanager-public-https", "observability")
exact_policy_specs(alertmanager, {
"observability-allow-alertmanager-public-https": {
"podSelector": {"matchLabels": {
"app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager",
"app.kubernetes.io/name": "alertmanager",
}},
"policyTypes": ["Egress"],
"egress": [{
"to": [{"ipBlock": {
"cidr": "0.0.0.0/0",
"except": ["10.0.0.0/8", "100.64.0.0/10", "172.16.0.0/12", "192.168.0.0/16"],
}}],
"ports": [{"protocol": "TCP", "port": 443}],
}],
},
})
if {
(item.get("metadata") or {}).get("name")
for item in alertmanager if item.get("kind") == "NetworkPolicy"
} != {"observability-allow-alertmanager-public-https"}:
reject("Alertmanager NetworkPolicy name set is not exact")
elif mode == "scope":
items = []
for path in paths:
items.extend(load(path))
forbidden = re.compile(r"spring|jvm|kafka|batch|backup", re.I)
for item in items:
kind = item.get("kind")
name = (item.get("metadata") or {}).get("name") or ""
if kind in {"Probe", "ServiceMonitor", "PodMonitor", "PrometheusRule"}:
if labels(item).get("observability.hyeonworks.com/instance") != "home":
reject(f"missing monitor/rule instance label: {kind}/{name}")
semantic_names = [name]
if kind == "PrometheusRule":
for group in (item.get("spec") or {}).get("groups") or []:
semantic_names.append(group.get("name") or "")
for rule in group.get("rules") or []:
semantic_names.append(rule.get("alert") or rule.get("record") or "")
if any(forbidden.search(value) for value in semantic_names):
reject(f"forbidden product scope: {kind}/{name}")
if kind in {"ServiceMonitor", "PodMonitor"}:
namespace = (item.get("metadata") or {}).get("namespace")
allowed = {
("observability", "blackbox-exporter"),
("platform-data", "platform-postgres"),
("object-storage", "aistor-bucket-usage"),
}
if (namespace, name) not in allowed:
reject(f"out-of-scope access target: {namespace}/{name}")
elif mode == "private-dns":
items = load(paths[0])
config = one(items, "ConfigMap", "coredns-custom", "kube-system")
content = "\n".join((config.get("data") or {}).values())
lan = paths[1].read_text()
tail = paths[2].read_text()
hosts = ("git", "id", "storage-admin", "db-admin", "grafana")
for short in hosts:
host = f"{short}.learn.hyeonworks.com"
if f"192.168.0.107 {host}" not in content \
or f"address=/{host}/192.168.0.107" not in lan \
or f"address=/{host}/100.92.240.34" not in tail:
reject(f"private DNS mapping is not exact: {host}")
if re.search(r"(?:^|\s)(?!192\.168\.0\.107\b)\d+\.\d+\.\d+\.\d+\s+grafana\.learn", content):
reject("Grafana DNS assumes a non-private address")
elif mode == "traefik":
items = load(paths[0])
hcc = one(items, "HelmChartConfig", "traefik", "kube-system")
values = yaml.safe_load((hcc.get("spec") or {}).get("valuesContent") or "") or {}
web = ((((values.get("ports") or {}).get("web") or {}).get("forwardedHeaders") or {}))
if web.get("trustedIPs") != ["10.42.0.1/32"] or web.get("insecure") is True:
reject("Traefik trust overlay is not exact")
else:
reject(f"unknown assertion mode: {mode}")
PY
}
assert_access_grafana_contract() { _access_assert grafana "$@"; }
assert_access_blackbox_contract() { _access_assert blackbox "$@"; }
assert_access_targets_contract() { _access_assert targets "$@"; }
assert_access_rules_alerts_contract() {
(( $# == 3 )) || return 1
_access_assert rules-alerts "$@" || return 1
assert_alertmanager_routing_contract "$3"
}
assert_access_scope_contract() { _access_assert scope "$@"; }
assert_access_private_dns_contract() { _access_assert private-dns "$@"; }
assert_access_traefik_trust_contract() { _access_assert traefik "$@"; }
validate_access_inventory_root() {
local root=$1 phase directory expected actual filename extra entries current_uid
current_uid="$(id -u)" || return 1
[[ "$root" =~ ^/tmp/platform-observability-metrics\.[A-Za-z0-9]{6}$ ]] || return 1
[[ -d /tmp && ! -L /tmp && "$(readlink -f -- /tmp)" == /tmp &&
"$(stat -c %u:%a -- /tmp)" == 0:1777 ]] || return 1
[[ -d "$root" && ! -L "$root" && "$(readlink -f -- "$root")" == "$root" &&
"$(stat -c %u:%a -- "$root")" == "$current_uid:700" ]] || return 1
entries="$(find "$root" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1
[[ "$entries" == $'post-substrate\ntarget-initial' ]] || return 1
for phase in target-initial post-substrate; do
directory="$root/$phase"
[[ -d "$directory" && ! -L "$directory" && -f "$directory/inventory.json" &&
-f "$directory/inventory.sha256" && ! -L "$directory/inventory.json" &&
! -L "$directory/inventory.sha256" ]] || return 1
[[ "$(readlink -f -- "$directory")" == "$directory" &&
"$(stat -c %u:%a -- "$directory")" == "$current_uid:700" ]] || return 1
entries="$(find "$directory" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)" || return 1
[[ "$entries" == $'inventory.json\ninventory.sha256' ]] || return 1
for filename in inventory.json inventory.sha256; do
[[ -f "$directory/$filename" && ! -L "$directory/$filename" &&
"$(stat -c %u:%a:%h -- "$directory/$filename")" == "$current_uid:600:1" ]] || return 1
done
read -r expected filename extra <"$directory/inventory.sha256"
[[ "$expected" =~ ^[0-9a-f]{64}$ && "$filename" == inventory.json && -z "${extra:-}" ]] || return 1
actual="$(sha256sum "$directory/inventory.json" | awk '{print $1}')"
[[ "$actual" == "$expected" ]] || return 1
jq -e --arg phase "$phase" '
.schema == "platform-observability-metric-inventory/v1" and
.phase == $phase and (.targets | type == "array" and length > 0) and
all(.targets[]; .health == "up" and .last_error == "" and (.metrics | type == "array" and length > 0))
' "$directory/inventory.json" >/dev/null || return 1
done
}
publish_access_outputs() {
local root=$1 work=$2 component=$3
local -a names=()
mapfile -t names < <(access_output_names "$component") || return 2
(( ${#names[@]} > 0 )) || return 2
python3 - "$root" "$work" "${names[@]}" <<'PY'
import hashlib
import os
import re
import stat
import sys
class Rejected(Exception):
pass
def reject(reason):
raise Rejected(reason)
def same_identity(metadata, expected):
return (metadata.st_dev, metadata.st_ino) == expected
def sha256_descriptor(descriptor):
digest = hashlib.sha256()
os.lseek(descriptor, 0, os.SEEK_SET)
while True:
chunk = os.read(descriptor, 1024 * 1024)
if not chunk:
break
digest.update(chunk)
os.lseek(descriptor, 0, os.SEEK_SET)
return digest.hexdigest()
root, work, *names = sys.argv[1:]
root_descriptor = None
work_descriptor = None
source_descriptors = {}
temporary_names = []
created = {}
try:
if os.path.dirname(root) != "/tmp" or not re.fullmatch(
r"platform-observability-metrics\.[A-Za-z0-9]{6}", os.path.basename(root)
):
reject("output root is not the expected direct /tmp handoff")
tmp_metadata = os.lstat("/tmp")
root_metadata = os.lstat(root)
if not stat.S_ISDIR(tmp_metadata.st_mode) or stat.S_ISLNK(tmp_metadata.st_mode):
reject("/tmp is not a physical directory")
if tmp_metadata.st_uid != 0 or stat.S_IMODE(tmp_metadata.st_mode) != 0o1777:
reject("/tmp owner or mode differs")
if os.path.realpath("/tmp") != "/tmp" or os.path.realpath(root) != root:
reject("output root lineage is not physical")
if not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode):
reject("output root is not a physical directory")
if root_metadata.st_uid != os.getuid() or stat.S_IMODE(root_metadata.st_mode) != 0o700:
reject("output root owner or mode differs")
root_identity = (root_metadata.st_dev, root_metadata.st_ino)
root_descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
if not same_identity(os.fstat(root_descriptor), root_identity):
reject("opened output root identity differs")
base_entries = {"target-initial", "post-substrate"}
if set(os.listdir(root_descriptor)) != base_entries:
reject("output root is not pristine before publication")
for phase in base_entries:
phase_metadata = os.stat(phase, dir_fd=root_descriptor, follow_symlinks=False)
if not stat.S_ISDIR(phase_metadata.st_mode) or phase_metadata.st_uid != os.getuid():
reject("inventory phase type or owner differs")
if stat.S_IMODE(phase_metadata.st_mode) != 0o700:
reject("inventory phase mode differs")
phase_descriptor = os.open(
phase, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_descriptor
)
try:
if set(os.listdir(phase_descriptor)) != {"inventory.json", "inventory.sha256"}:
reject("inventory phase entry set differs")
for filename in ("inventory.json", "inventory.sha256"):
metadata = os.stat(filename, dir_fd=phase_descriptor, follow_symlinks=False)
if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid():
reject("inventory file type or owner differs")
if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1:
reject("inventory file mode or link count differs")
finally:
os.close(phase_descriptor)
if len(set(names)) != len(names) or any("/" in name or name in {".", ".."} for name in names):
reject("publication names are malformed")
if any(name in base_entries for name in names):
reject("publication name collides with inventory")
for name in names:
try:
os.stat(name, dir_fd=root_descriptor, follow_symlinks=False)
except FileNotFoundError:
continue
reject("output destination already exists")
work_metadata = os.lstat(work)
if not stat.S_ISDIR(work_metadata.st_mode) or stat.S_ISLNK(work_metadata.st_mode):
reject("verified render directory is not physical")
if work_metadata.st_uid != os.getuid() or stat.S_IMODE(work_metadata.st_mode) != 0o700:
reject("verified render directory owner or mode differs")
work_identity = (work_metadata.st_dev, work_metadata.st_ino)
work_descriptor = os.open(work, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
if not same_identity(os.fstat(work_descriptor), work_identity):
reject("opened render directory identity differs")
source_hashes = {}
for name in names:
descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=work_descriptor)
source_descriptors[name] = descriptor
metadata = os.fstat(descriptor)
if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid():
reject("verified output type or owner differs")
if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1:
reject("verified output mode or link count differs")
source_hashes[name] = sha256_descriptor(descriptor)
for position, name in enumerate(names):
temporary = f".access-handoff-{os.getpid()}-{position}"
target_descriptor = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
0o600,
dir_fd=root_descriptor,
)
temporary_names.append(temporary)
try:
source_descriptor = source_descriptors[name]
os.lseek(source_descriptor, 0, os.SEEK_SET)
while True:
chunk = os.read(source_descriptor, 1024 * 1024)
if not chunk:
break
view = memoryview(chunk)
while view:
view = view[os.write(target_descriptor, view):]
os.fchmod(target_descriptor, 0o600)
os.fsync(target_descriptor)
staged_identity = (os.fstat(target_descriptor).st_dev, os.fstat(target_descriptor).st_ino)
finally:
os.close(target_descriptor)
os.link(
temporary,
name,
src_dir_fd=root_descriptor,
dst_dir_fd=root_descriptor,
follow_symlinks=False,
)
created[name] = staged_identity
os.unlink(temporary, dir_fd=root_descriptor)
temporary_names.remove(temporary)
os.fsync(root_descriptor)
if set(os.listdir(root_descriptor)) != base_entries | set(names):
reject("published output entry set differs")
for name in names:
metadata = os.stat(name, dir_fd=root_descriptor, follow_symlinks=False)
if not same_identity(metadata, created[name]) or not stat.S_ISREG(metadata.st_mode):
reject("published output identity or type differs")
if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600:
reject("published output owner or mode differs")
descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root_descriptor)
try:
if sha256_descriptor(descriptor) != source_hashes[name]:
reject("published output content differs")
finally:
os.close(descriptor)
if not same_identity(os.lstat(root), root_identity):
reject("output root identity changed during publication")
except BaseException as error:
if root_descriptor is not None:
for name, identity in reversed(tuple(created.items())):
try:
metadata = os.stat(name, dir_fd=root_descriptor, follow_symlinks=False)
if same_identity(metadata, identity):
os.unlink(name, dir_fd=root_descriptor)
except OSError:
pass
for name in reversed(temporary_names):
try:
os.unlink(name, dir_fd=root_descriptor)
except OSError:
pass
try:
os.fsync(root_descriptor)
except OSError:
pass
if isinstance(error, Rejected):
print(f"REJECT: access output publication failed: {error}", file=sys.stderr)
else:
print("REJECT: access output publication failed: filesystem operation failed", file=sys.stderr)
raise SystemExit(23)
finally:
for descriptor in source_descriptors.values():
try:
os.close(descriptor)
except OSError:
pass
if work_descriptor is not None:
os.close(work_descriptor)
if root_descriptor is not None:
os.close(root_descriptor)
PY
}
_access_usage() {
cat <<'USAGE'
Usage:
PLATFORM_HELM_BIN=/home/donghyeon/.local/bin/helm \
bash scripts/validate/render-observability-access.sh \
--component grafana|blackbox|targets|rules-alerts|complete \
[--verified-output-dir /tmp/platform-observability-metrics.XXXXXX]
Static rendering only; no Kubernetes resource is applied.
USAGE
}
access_render_main() (
local component=complete output_root='' argument core_handoff work
access_secure_render_context
while (( $# > 0 )); do
argument=$1
case "$argument" in
--component) (( $# >= 2 )) || return 2; component=$2; shift 2 ;;
--verified-output-dir) (( $# >= 2 )) || return 2; output_root=$2; shift 2 ;;
-h|--help) _access_usage; return 0 ;;
*) _access_usage >&2; return 2 ;;
esac
done
access_output_names "$component" >/dev/null || return 2
if access_component_requires_inventory "$component"; then
[[ -n "$output_root" ]] || { printf 'REJECT: rules-alerts/complete requires --verified-output-dir\n' >&2; return 1; }
validate_access_inventory_root "$output_root" || { printf 'REJECT: verified inventory root is invalid\n' >&2; return 1; }
fi
work="$(mktemp -d /tmp/platform-observability-access-render.XXXXXX)"
chmod 0700 "$work"
core_handoff="$(mktemp -d /tmp/platform-observability-core-apply.XXXXXX)"
chmod 0700 "$core_handoff"
trap '[[ -n ${work:-} && $work == /tmp/platform-observability-access-render.* ]] && rm -rf -- "$work"; [[ -n ${core_handoff:-} && $core_handoff == /tmp/platform-observability-core-apply.* ]] && rm -rf -- "$core_handoff"' EXIT INT TERM
/usr/bin/bash "$ACCESS_ROOT/scripts/validate/render-observability-core.sh" \
--verified-output-dir "$core_handoff"
cp -- "$core_handoff/grafana.yaml" "$work/grafana.yaml"
cp -- "$core_handoff/blackbox.yaml" "$work/blackbox.yaml"
cp -- "$core_handoff/targets.yaml" "$work/targets.yaml"
cp -- "$core_handoff/dashboards.yaml" "$work/dashboards.yaml"
cp -- "$core_handoff/core-rules.yaml" "$work/rules.yaml"
kubectl kustomize "$ACCESS_ROOT/infrastructure/networking/private-dns/kubernetes" >"$work/private-dns.yaml"
python3 - "$core_handoff/kps.yaml" "$core_handoff/alerting.yaml" "$work/alertmanager.yaml" <<'PY'
import sys, yaml
class AccessExtractionSafeLoader(yaml.SafeLoader):
pass
AccessExtractionSafeLoader.add_constructor(
"tag:yaml.org,2002:value",
lambda loader, node: loader.construct_scalar(node),
)
items=[]
for path in sys.argv[1:3]:
with open(path, encoding="utf-8") as stream:
items.extend(
item
for item in yaml.load_all(stream, Loader=AccessExtractionSafeLoader)
if item is not None
)
selected=[item for item in items if item.get("kind") in {"Alertmanager", "AlertmanagerConfig", "NetworkPolicy"}]
with open(sys.argv[3], "w", encoding="utf-8") as stream:
yaml.safe_dump_all(selected, stream, explicit_start=True, sort_keys=False)
PY
assert_access_grafana_contract "$work/grafana.yaml" "$core_handoff/core-policies.yaml"
assert_pinned_images "$work/grafana.yaml"
assert_access_blackbox_contract "$work/blackbox.yaml"
assert_pinned_images "$work/blackbox.yaml"
assert_access_targets_contract "$work/targets.yaml"
assert_access_rules_alerts_contract "$work/dashboards.yaml" "$work/rules.yaml" "$work/alertmanager.yaml"
assert_access_scope_contract "$work/blackbox.yaml" "$work/targets.yaml" "$work/rules.yaml"
assert_access_private_dns_contract "$work/private-dns.yaml" \
"$ACCESS_ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf" \
"$ACCESS_ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf"
assert_no_credentials "$work"/*.yaml
if [[ -n "$output_root" ]]; then
publish_access_outputs "$output_root" "$work" "$component"
fi
printf 'OBSERVABILITY ACCESS STATIC RENDER PASS\n'
)
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
set -Eeuo pipefail
access_render_main "$@"
fi
File diff suppressed because it is too large Load Diff
+671
View File
@@ -0,0 +1,671 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly TARGET_KUBERNETES_VERSION="1.36.2"
readonly CNPG_CHART_NAME="cloudnative-pg"
readonly CNPG_CHART_VERSION="0.29.0"
readonly CNPG_CHART_REPOSITORY="https://cloudnative-pg.github.io/charts"
readonly EXPECTED_CNPG_CHART_SHA256="668e065ff53508d58238788fd35b355a925060843629a951df0e6a9362e6d32f"
readonly GITEA_CHART_NAME="gitea"
readonly GITEA_CHART_VERSION="12.7.0"
readonly GITEA_CHART_REPOSITORY="https://dl.gitea.com/charts/"
readonly EXPECTED_GITEA_CHART_SHA256="5881ef9c59400bee2d5547e77c4cd0efb925143c2f5d93fb4f38446db76b0167"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly -a RENDERED_MANIFEST_NAMES=(
namespaces
ssd-local-pv
cnpg-operator
platform-postgres
gitea
gitea-oidc
)
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
assert_exact_source_text() {
local file="$1"
local expected_text="$2"
local description="$3"
local count
count="$(rg --count-matches --fixed-strings -- "$expected_text" "$file" || true)"
[[ "$count" == "1" ]] || \
fail "${description} must appear exactly once in ${file#${REPOSITORY_ROOT}/}"
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/validate/render-phase1.sh
bash scripts/validate/render-phase1.sh \
--verified-output-dir /tmp/platform-phase1-apply.XXXXXX
The output option is an internal handoff used only by the Phase 1 apply
script. The destination must be an existing, empty, non-symlink directory
created directly below /tmp with the platform-phase1-apply.* prefix.
USAGE
}
verified_output_dir=""
case "$#" in
0)
;;
2)
[[ "$1" == "--verified-output-dir" ]] || {
usage >&2
exit 2
}
verified_output_dir="$2"
;;
*)
usage >&2
exit 2
;;
esac
if [[ -n "$verified_output_dir" ]]; then
[[ "$verified_output_dir" == /tmp/platform-phase1-apply.* ]] || \
fail "verified output directory must match /tmp/platform-phase1-apply.*"
[[ -d "$verified_output_dir" && ! -L "$verified_output_dir" ]] || \
fail "verified output directory must be an existing non-symlink directory"
[[ "$(cd -- "$verified_output_dir" && pwd -P)" == "$verified_output_dir" ]] || \
fail "verified output directory must be an absolute canonical path"
[[ -O "$verified_output_dir" ]] || \
fail "verified output directory must be owned by the current user"
[[ "$(stat --format='%a' -- "$verified_output_dir")" == "700" ]] || \
fail "verified output directory must have mode 0700"
[[ -z "$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \
fail "verified output directory must be empty"
fi
command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"
command -v jq >/dev/null 2>&1 || fail "jq is required"
command -v rg >/dev/null 2>&1 || fail "ripgrep (rg) is required"
command -v sha256sum >/dev/null 2>&1 || fail "sha256sum is required"
command -v tar >/dev/null 2>&1 || fail "tar is required"
command -v cmp >/dev/null 2>&1 || fail "cmp is required"
command -v find >/dev/null 2>&1 || fail "find is required"
command -v install >/dev/null 2>&1 || fail "install is required"
command -v stat >/dev/null 2>&1 || fail "stat is required"
if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then
[[ "$PLATFORM_HELM_BIN" == /* ]] || \
fail "PLATFORM_HELM_BIN must be an absolute path"
[[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}"
readonly HELM_BIN="$PLATFORM_HELM_BIN"
else
HELM_BIN="$(command -v helm 2>/dev/null)" || \
fail "Helm ${EXPECTED_HELM_VERSION} is required"
readonly HELM_BIN
fi
kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')"
helm_version="$("$HELM_BIN" version --template '{{.Version}}')"
[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}"
[[ "$helm_version" == "$EXPECTED_HELM_VERSION" ]] || \
fail "expected Helm ${EXPECTED_HELM_VERSION}, found ${helm_version:-unknown}"
render_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase1-render.XXXXXX")"
declare -a generated_chart_cache_dirs=()
declare -a generated_chart_cache_parent_dirs=()
cleanup() {
local cache_dir
local parent_dir
for cache_dir in "${generated_chart_cache_dirs[@]}"; do
case "$cache_dir" in
"${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm/charts/cloudnative-pg-0.29.0"|\
"${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm/charts/gitea-12.7.0")
rm -rf -- "$cache_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected chart cache: %s\n' \
"$cache_dir" >&2
;;
esac
done
for parent_dir in "${generated_chart_cache_parent_dirs[@]}"; do
case "$parent_dir" in
"${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm/charts"|\
"${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm"|\
"${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm/charts"|\
"${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm")
rmdir -- "$parent_dir" 2>/dev/null || true
;;
*)
printf 'WARNING: refusing to remove unexpected chart cache parent: %s\n' \
"$parent_dir" >&2
;;
esac
done
case "$render_dir" in
/tmp/platform-phase1-render.*|"${TMPDIR:-/tmp}"/platform-phase1-render.*)
rm -rf -- "$render_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render directory: %s\n' "$render_dir" >&2
;;
esac
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_plain() {
local label="$1"
local relative_path="$2"
local output="${render_dir}/${label}.yaml"
kubectl kustomize "${REPOSITORY_ROOT}/${relative_path}" >"$output"
[[ -s "$output" ]] || fail "${label} rendered an empty manifest"
printf 'Rendered %-20s %8s bytes\n' "$label" "$(wc -c <"$output" | tr -d '[:space:]')"
}
render_helm() {
local label="$1"
local relative_path="$2"
local output="${render_dir}/${label}.yaml"
kubectl kustomize \
--enable-helm \
--helm-command "$HELM_BIN" \
--helm-kube-version "$TARGET_KUBERNETES_VERSION" \
"${REPOSITORY_ROOT}/${relative_path}" >"$output"
[[ -s "$output" ]] || fail "${label} rendered an empty manifest"
printf 'Rendered %-20s %8s bytes\n' "$label" "$(wc -c <"$output" | tr -d '[:space:]')"
}
extract_rendered_document() {
local manifest="$1"
local wanted_kind="$2"
local wanted_name="$3"
local output="$4"
awk \
-v wanted_kind="$wanted_kind" \
-v wanted_name="$wanted_name" \
'
function reset_document() {
document = ""
document_kind = ""
document_name = ""
in_metadata = 0
}
function flush_document() {
if (document_kind == wanted_kind && document_name == wanted_name) {
matches++
printf "%s", document
}
}
BEGIN {
reset_document()
}
/^---[[:space:]]*$/ {
flush_document()
reset_document()
next
}
{
document = document $0 ORS
if ($0 ~ /^kind:[[:space:]]*/) {
document_kind = $0
sub(/^kind:[[:space:]]*/, "", document_kind)
sub(/[[:space:]]*$/, "", document_kind)
}
if ($0 == "metadata:") {
in_metadata = 1
next
}
if (in_metadata && $0 ~ /^ name:[[:space:]]*/) {
document_name = $0
sub(/^ name:[[:space:]]*/, "", document_name)
sub(/[[:space:]]*$/, "", document_name)
in_metadata = 0
} else if (in_metadata && $0 ~ /^[^[:space:]]/) {
in_metadata = 0
}
}
END {
flush_document()
if (matches != 1) {
exit 42
}
}
' \
"$manifest" >"$output"
}
prepare_verified_chart_cache() {
local label="$1"
local chart_name="$2"
local chart_repository="$3"
local chart_version="$4"
local build_root_relative_path="$5"
local expected_digest="$6"
local package_path="${render_dir}/${chart_name}-${chart_version}.tgz"
local cache_version_dir="${REPOSITORY_ROOT}/${build_root_relative_path}/.helm/charts/${chart_name}-${chart_version}"
local checksum_output
local actual_digest
local chart_cache_dir="${cache_version_dir%/*}"
local helm_cache_dir="${chart_cache_dir%/*}"
local cache_parent
if [[ ! -f "$package_path" ]]; then
"$HELM_BIN" pull "$chart_name" \
--repo "$chart_repository" \
--version "$chart_version" \
--destination "$render_dir"
fi
[[ -f "$package_path" ]] || fail "${label} chart package was not downloaded: ${package_path}"
checksum_output="$(sha256sum -- "$package_path")"
actual_digest="${checksum_output%% *}"
[[ "$actual_digest" == "$expected_digest" ]] || \
fail "${label} chart package SHA-256 mismatch: expected ${expected_digest}, found ${actual_digest}"
case "$cache_version_dir" in
"${REPOSITORY_ROOT}/infrastructure/controllers/cloudnative-pg/.helm/charts/cloudnative-pg-0.29.0"|\
"${REPOSITORY_ROOT}/services/gitea/profiles/oidc/.helm/charts/gitea-12.7.0")
;;
*)
fail "refusing to create unexpected chart cache directory: ${cache_version_dir}"
;;
esac
for cache_parent in "$helm_cache_dir" "$chart_cache_dir"; do
[[ ! -L "$cache_parent" ]] || \
fail "refusing symlinked chart cache parent: ${cache_parent}"
[[ ! -e "$cache_parent" || -d "$cache_parent" ]] || \
fail "chart cache parent is not a directory: ${cache_parent}"
done
[[ ! -e "$cache_version_dir" && ! -L "$cache_version_dir" ]] || \
fail "generated chart cache already exists; remove it only after confirming it is disposable: ${cache_version_dir}"
[[ -e "$chart_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$chart_cache_dir")
[[ -e "$helm_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$helm_cache_dir")
mkdir -p -- "$cache_version_dir"
generated_chart_cache_dirs+=("$cache_version_dir")
tar -xzf "$package_path" -C "$cache_version_dir"
[[ -f "${cache_version_dir}/${chart_name}/Chart.yaml" ]] || \
fail "${label} extracted chart is missing Chart.yaml"
printf 'Verified %-20s SHA-256 %s\n' "$label" "$actual_digest"
}
cd -- "$REPOSITORY_ROOT"
if rg --line-number --glob '*.yaml' --glob '*.yml' \
--glob '!**/.helm/**' --glob '!**/charts/**' \
'^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \
infrastructure services bootstrap clusters components; then
fail "a source-controlled Kubernetes Secret manifest was found"
fi
if rg --line-number --glob 'kustomization.yaml' 'LoadRestrictionsNone|load-restrictor' .; then
fail "the repository must keep Kustomize LoadRestrictionsRootOnly"
fi
readonly GITEA_BASELINE_KUSTOMIZATION="${REPOSITORY_ROOT}/services/gitea/kustomization.yaml"
readonly GITEA_OIDC_KUSTOMIZATION="${REPOSITORY_ROOT}/services/gitea/profiles/oidc/kustomization.yaml"
for gitea_kustomization in \
"$GITEA_BASELINE_KUSTOMIZATION" \
"$GITEA_OIDC_KUSTOMIZATION"; do
assert_exact_source_text "$gitea_kustomization" \
'repo: https://dl.gitea.com/charts/' \
"the pinned Gitea Chart repository"
assert_exact_source_text "$gitea_kustomization" \
'version: 12.7.0' \
"the pinned Gitea Chart version"
assert_exact_source_text "$gitea_kustomization" \
'includeCRDs: false' \
"the Gitea includeCRDs policy"
assert_exact_source_text "$gitea_kustomization" \
'skipTests: true' \
"the Gitea Helm test policy"
done
assert_exact_source_text "$GITEA_BASELINE_KUSTOMIZATION" \
'chartHome: profiles/oidc/.helm/charts' \
"the baseline Gitea verified Chart cache"
assert_exact_source_text "$GITEA_BASELINE_KUSTOMIZATION" \
'valuesFile: profiles/oidc/values/baseline.yaml' \
"the baseline Gitea values path"
assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \
'chartHome: .helm/charts' \
"the OIDC Gitea verified Chart cache"
assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \
'valuesFile: values/baseline.yaml' \
"the OIDC Gitea baseline values path"
assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \
'additionalValuesFiles:' \
"the OIDC Gitea values merge"
assert_exact_source_text "$GITEA_OIDC_KUSTOMIZATION" \
'values/oidc.yaml' \
"the OIDC Gitea override path"
render_plain namespaces infrastructure/namespaces/overlays/home
render_plain ssd-local-pv infrastructure/storage/ssd-local-pv
prepare_verified_chart_cache \
cloudnative-pg-chart \
"$CNPG_CHART_NAME" \
"$CNPG_CHART_REPOSITORY" \
"$CNPG_CHART_VERSION" \
infrastructure/controllers/cloudnative-pg \
"$EXPECTED_CNPG_CHART_SHA256"
render_helm cnpg-operator infrastructure/controllers/cloudnative-pg
render_plain platform-postgres services/platform-postgres
prepare_verified_chart_cache \
gitea-chart \
"$GITEA_CHART_NAME" \
"$GITEA_CHART_REPOSITORY" \
"$GITEA_CHART_VERSION" \
services/gitea/profiles/oidc \
"$EXPECTED_GITEA_CHART_SHA256"
render_helm gitea services/gitea
render_helm gitea-oidc services/gitea/profiles/oidc
[[ "$(awk '$0 == "kind: Cluster" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \
fail "the Phase 1 PostgreSQL root must contain exactly one Cluster"
[[ "$(awk '$0 == "kind: DatabaseRole" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \
fail "the Phase 1 PostgreSQL root must contain exactly one Gitea DatabaseRole"
[[ "$(awk '$0 == "kind: Database" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \
fail "the Phase 1 PostgreSQL root must contain exactly one Gitea Database"
[[ "$(awk '$0 == "kind: NetworkPolicy" { count++ } END { print count + 0 }' "${render_dir}/platform-postgres.yaml")" == "1" ]] || \
fail "the Phase 1 PostgreSQL root must contain exactly one base NetworkPolicy"
[[ "$(rg --count -- '^[[:space:]]*name:[[:space:]]platform-postgres-gitea[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "2" ]] || \
fail "the Phase 1 PostgreSQL root must contain the Gitea DatabaseRole and Database"
if rg --quiet '^[[:space:]]*name:[[:space:]]platform-postgres-keycloak[[:space:]]*$' "${render_dir}/platform-postgres.yaml"; then
fail "the Phase 1 PostgreSQL root must not contain a Keycloak DatabaseRole or Database"
fi
[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host gitea gitea all scram-sha-256[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \
fail "the Gitea role must authenticate only to the Gitea database"
[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host all gitea all reject[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \
fail "the Gitea role must be rejected from every other database"
[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host keycloak keycloak all scram-sha-256[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \
fail "the Keycloak role must authenticate only to the Keycloak database"
[[ "$(rg --count -- '^[[:space:]]*-[[:space:]]host all keycloak all reject[[:space:]]*$' "${render_dir}/platform-postgres.yaml" || true)" == "1" ]] || \
fail "the Keycloak role must be rejected from every other database"
rg --quiet '^kind: CustomResourceDefinition$' "${render_dir}/cnpg-operator.yaml" || \
fail "CloudNativePG CRDs are missing from the operator render"
rg --quiet '^kind: Cluster$' "${render_dir}/platform-postgres.yaml" || \
fail "the platform PostgreSQL Cluster is missing"
rg --quiet '^kind: DatabaseRole$' "${render_dir}/platform-postgres.yaml" || \
fail "the Gitea DatabaseRole is missing"
rg --quiet '^kind: Database$' "${render_dir}/platform-postgres.yaml" || \
fail "the Gitea Database is missing"
for gitea_profile in gitea gitea-oidc; do
gitea_profile_manifest="${render_dir}/${gitea_profile}.yaml"
gitea_profile_deployment_document="${render_dir}/${gitea_profile}-deployment.yaml"
gitea_profile_ingress_document="${render_dir}/${gitea_profile}-ingress.yaml"
gitea_profile_servicemonitor_document="${render_dir}/${gitea_profile}-servicemonitor.yaml"
rg --quiet '^kind: PersistentVolumeClaim$' "$gitea_profile_manifest" || \
fail "${gitea_profile} is missing the Gitea PVC"
rg --quiet '^kind: Ingress$' "$gitea_profile_manifest" || \
fail "${gitea_profile} is missing the Gitea Ingress"
rg --quiet \
'^[[:space:]]*-[[:space:]]*host:[[:space:]]*git\.learn\.hyeonworks\.com[[:space:]]*$' \
"$gitea_profile_manifest" || \
fail "${gitea_profile} Ingress host is not git.learn.hyeonworks.com"
[[ "$(rg --count-matches \
'^[[:space:]]*ROOT_URL=https://git\.learn\.hyeonworks\.com/[[:space:]]*$' \
"$gitea_profile_manifest" || true)" == "1" ]] || \
fail "${gitea_profile} must render the external HTTPS ROOT_URL exactly once"
extract_rendered_document \
"$gitea_profile_manifest" \
Deployment \
gitea \
"$gitea_profile_deployment_document" || \
fail "${gitea_profile} must contain exactly one gitea Deployment"
extract_rendered_document \
"$gitea_profile_manifest" \
Ingress \
gitea-http \
"$gitea_profile_ingress_document" || \
fail "${gitea_profile} must contain exactly one gitea-http Ingress"
extract_rendered_document \
"$gitea_profile_manifest" \
ServiceMonitor \
gitea \
"$gitea_profile_servicemonitor_document" || \
fail "${gitea_profile} must contain exactly one gitea ServiceMonitor"
if rg --quiet '^[[:space:]]{2}tls:[[:space:]]*' "$gitea_profile_ingress_document"; then
fail "${gitea_profile} must not render an in-cluster TLS section"
fi
if ! kubectl create --dry-run=client \
-f "$gitea_profile_servicemonitor_document" \
-o json | jq -e '
.apiVersion == "monitoring.coreos.com/v1" and
.kind == "ServiceMonitor" and
.metadata.name == "gitea" and
.metadata.namespace == "gitea" and
.metadata.labels["observability.hyeonworks.com/instance"] == "home" and
.spec.jobLabel == "app.kubernetes.io/name" and
.spec.selector.matchLabels == {
"app.kubernetes.io/instance": "gitea",
"app.kubernetes.io/name": "gitea"
} and
.spec.endpoints == [{
"interval": "30s",
"port": "http",
"scrapeTimeout": "10s"
}]
' >/dev/null; then
fail "${gitea_profile} ServiceMonitor contract is not exact"
fi
for restricted_setting in \
'allowPrivilegeEscalation: false' \
'runAsNonRoot: true' \
'type: RuntimeDefault' \
'- ALL'; do
[[ "$(rg --count-matches --fixed-strings -- "$restricted_setting" \
"$gitea_profile_deployment_document" || true)" == "4" ]] || \
fail "${gitea_profile} must apply ${restricted_setting} to all four containers"
done
done
for forbidden_baseline_marker in \
'gitea-keycloak-oidc' \
'id.learn.hyeonworks.com' \
'gitea-allow-host-nginx-keycloak' \
'gitea-branding-assets' \
'gitea-branding-templates' \
'ALLOW_ONLY_EXTERNAL_REGISTRATION=true'; do
if rg --quiet --fixed-strings -- "$forbidden_baseline_marker" "${render_dir}/gitea.yaml"; then
fail "the baseline Gitea render contains OIDC-only marker ${forbidden_baseline_marker}"
fi
done
for baseline_setting in \
'DISABLE_REGISTRATION=true' \
'ALLOW_ONLY_EXTERNAL_REGISTRATION=false' \
'SHOW_REGISTRATION_BUTTON=false' \
'ENABLE_PASSWORD_SIGNIN_FORM=true'; do
[[ "$(rg --count-matches --fixed-strings -- "$baseline_setting" \
"${render_dir}/gitea.yaml" || true)" == "1" ]] || \
fail "the baseline Gitea render must contain exactly one ${baseline_setting} setting"
done
gitea_deployment_document="${render_dir}/gitea-oidc-deployment.yaml"
gitea_keycloak_policy_document="${render_dir}/gitea-keycloak-egress-policy.yaml"
gitea_branding_assets_document="${render_dir}/gitea-branding-assets-configmap.yaml"
gitea_branding_templates_document="${render_dir}/gitea-branding-templates-configmap.yaml"
extract_rendered_document \
"${render_dir}/gitea-oidc.yaml" \
Deployment \
gitea \
"$gitea_deployment_document" || \
fail "the Gitea render must contain exactly one gitea Deployment"
extract_rendered_document \
"${render_dir}/gitea-oidc.yaml" \
NetworkPolicy \
gitea-allow-host-nginx-keycloak \
"$gitea_keycloak_policy_document" || \
fail "the Gitea render must contain exactly one dedicated Keycloak egress NetworkPolicy"
# Assert only references to the externally-created OIDC credential Secret.
# No credential payload is rendered, decoded, read, or printed by these checks.
rg --quiet --multiline \
'(?s)- name: GITEA_OAUTH_KEY_0\n[[:space:]]+valueFrom:\n[[:space:]]+secretKeyRef:\n[[:space:]]+key: key\n[[:space:]]+name: gitea-keycloak-oidc' \
"$gitea_deployment_document" || \
fail "the Gitea Deployment does not reference gitea-keycloak-oidc key=key"
rg --quiet --multiline \
'(?s)- name: GITEA_OAUTH_SECRET_0\n[[:space:]]+valueFrom:\n[[:space:]]+secretKeyRef:\n[[:space:]]+key: secret\n[[:space:]]+name: gitea-keycloak-oidc' \
"$gitea_deployment_document" || \
fail "the Gitea Deployment does not reference gitea-keycloak-oidc key=secret"
[[ "$(rg --count-matches --fixed-strings \
'https://id.learn.hyeonworks.com/realms/hyeonworks/.well-known/openid-configuration' \
"${render_dir}/gitea-oidc.yaml" || true)" == "2" ]] || \
fail "the Gitea OAuth add/update script must use the exact Keycloak discovery URL"
# These are non-sensitive app.ini policy values rendered by the pinned Chart.
for expected_setting in \
'ALLOW_ONLY_EXTERNAL_REGISTRATION=true' \
'DISABLE_REGISTRATION=false' \
'SHOW_REGISTRATION_BUTTON=false' \
'ENABLE_PASSWORD_SIGNIN_FORM=true' \
'ENABLE_AUTO_REGISTRATION=true' \
'USERNAME=preferred_username' \
'ACCOUNT_LINKING=login' \
'OPENID_CONNECT_SCOPES=profile email'; do
[[ "$(rg --count-matches --fixed-strings "$expected_setting" \
"${render_dir}/gitea-oidc.yaml" || true)" == "1" ]] || \
fail "the Gitea render must contain exactly one ${expected_setting} setting"
done
if rg --quiet '^[[:space:]]*hostAliases:' "$gitea_deployment_document"; then
fail "the Gitea Deployment must rely on CoreDNS and must not contain hostAliases"
fi
[[ "$(rg --count-matches \
'^[[:space:]]*cidr:[[:space:]]*192\.168\.0\.107/32[[:space:]]*$' \
"$gitea_keycloak_policy_document" || true)" == "1" ]] || \
fail "the dedicated Keycloak egress policy must allow exactly 192.168.0.107/32"
[[ "$(rg --count-matches \
'^[[:space:]]*-[[:space:]]*port:[[:space:]]*443[[:space:]]*$' \
"$gitea_keycloak_policy_document" || true)" == "1" ]] || \
fail "the dedicated Keycloak egress policy must allow exactly TCP port 443"
[[ "$(rg --count-matches \
'^[[:space:]]*protocol:[[:space:]]*TCP[[:space:]]*$' \
"$gitea_keycloak_policy_document" || true)" == "1" ]] || \
fail "the dedicated Keycloak egress policy must use TCP"
rg --quiet \
'^[[:space:]]*-[[:space:]]*Egress[[:space:]]*$' \
"$gitea_keycloak_policy_document" || \
fail "the dedicated Keycloak NetworkPolicy must select egress traffic"
gitea_branding_assets_configmap_name="$(
(rg --only-matching --no-filename \
'gitea-branding-assets-[a-z0-9]+' \
"${render_dir}/gitea-oidc.yaml" || true) |
LC_ALL=C sort --unique
)"
gitea_branding_templates_configmap_name="$(
(rg --only-matching --no-filename \
'gitea-branding-templates-[a-z0-9]+' \
"${render_dir}/gitea-oidc.yaml" || true) |
LC_ALL=C sort --unique
)"
[[ "$gitea_branding_assets_configmap_name" =~ ^gitea-branding-assets-[a-z0-9]{10}$ ]] || \
fail "the Gitea branding assets ConfigMap must have one Kustomize content hash"
[[ "$gitea_branding_templates_configmap_name" =~ ^gitea-branding-templates-[a-z0-9]{10}$ ]] || \
fail "the Gitea branding templates ConfigMap must have one Kustomize content hash"
extract_rendered_document \
"${render_dir}/gitea-oidc.yaml" \
ConfigMap \
"$gitea_branding_assets_configmap_name" \
"$gitea_branding_assets_document" || \
fail "the Gitea render must contain exactly one branding assets ConfigMap"
extract_rendered_document \
"${render_dir}/gitea-oidc.yaml" \
ConfigMap \
"$gitea_branding_templates_configmap_name" \
"$gitea_branding_templates_document" || \
fail "the Gitea render must contain exactly one branding templates ConfigMap"
for asset_key in hyeonworks.css logo.svg favicon.svg; do
[[ "$(rg --count-matches \
"^[[:space:]]{2}${asset_key//./\\.}:[[:space:]]*\\|[+-]?[[:space:]]*$" \
"$gitea_branding_assets_document" || true)" == "1" ]] || \
fail "the branding assets ConfigMap must contain exactly one ${asset_key}"
done
for template_key in header.tmpl extra_links.tmpl; do
[[ "$(rg --count-matches \
"^[[:space:]]{2}${template_key//./\\.}:[[:space:]]*\\|[+-]?[[:space:]]*$" \
"$gitea_branding_templates_document" || true)" == "1" ]] || \
fail "the branding templates ConfigMap must contain exactly one ${template_key}"
done
[[ "$(rg --count-matches --fixed-strings \
"$gitea_branding_assets_configmap_name" \
"$gitea_deployment_document" || true)" == "1" ]] || \
fail "the Gitea Deployment must reference the hashed branding assets ConfigMap once"
[[ "$(rg --count-matches --fixed-strings \
"$gitea_branding_templates_configmap_name" \
"$gitea_deployment_document" || true)" == "1" ]] || \
fail "the Gitea Deployment must reference the hashed branding templates ConfigMap once"
rg --quiet --multiline \
'(?s)- mountPath: /data/gitea/public/assets\n[[:space:]]+name: branding-assets\n[[:space:]]+readOnly: true' \
"$gitea_deployment_document" || \
fail "the Gitea branding assets must be mounted read-only at the official custom path"
rg --quiet --multiline \
'(?s)- mountPath: /data/gitea/templates/custom\n[[:space:]]+name: branding-templates\n[[:space:]]+readOnly: true' \
"$gitea_deployment_document" || \
fail "the Gitea branding templates must be mounted read-only at the official custom path"
if rg --quiet '^[[:space:]]*type:[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \
"${render_dir}/platform-postgres.yaml" \
"${render_dir}/gitea.yaml" \
"${render_dir}/gitea-oidc.yaml"; then
fail "an application service is exposed as NodePort or LoadBalancer"
fi
if rg --quiet '^[[:space:]]*name:[[:space:]]*gitea-ssh[[:space:]]*$' \
"${render_dir}/gitea.yaml" "${render_dir}/gitea-oidc.yaml"; then
fail "the disabled Gitea SSH Service is still rendered"
fi
if [[ -n "$verified_output_dir" ]]; then
for manifest_name in "${RENDERED_MANIFEST_NAMES[@]}"; do
source_manifest="${render_dir}/${manifest_name}.yaml"
output_manifest="${verified_output_dir}/${manifest_name}.yaml"
[[ -f "$source_manifest" && ! -L "$source_manifest" && -s "$source_manifest" ]] || \
fail "validated manifest is missing or unsafe: ${source_manifest}"
install -m 0600 -- "$source_manifest" "$output_manifest"
[[ -f "$output_manifest" && ! -L "$output_manifest" && -s "$output_manifest" ]] || \
fail "verified manifest handoff failed: ${output_manifest}"
cmp --silent -- "$source_manifest" "$output_manifest" || \
fail "verified manifest changed during handoff: ${manifest_name}.yaml"
done
verified_entry_count="$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]')"
[[ "$verified_entry_count" == "${#RENDERED_MANIFEST_NAMES[@]}" ]] || \
fail "verified output directory does not contain exactly six manifest files"
printf 'Preserved six verified manifests for the apply handoff.\n'
fi
printf 'Phase 1 baseline and Gitea OIDC desired rendering invariants passed.\n'
printf 'Temporary rendered manifests and generated chart caches will be removed on exit.\n'
+439
View File
@@ -0,0 +1,439 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly EXPECTED_KUSTOMIZE_VERSION="v5.8.1"
readonly EXPECTED_HELM_VERSION="v3.19.4"
readonly TARGET_KUBERNETES_VERSION="1.36.2"
readonly AISTOR_OPERATOR_CHART_NAME="aistor-operator"
readonly AISTOR_OPERATOR_CHART_VERSION="5.10.0"
readonly EXPECTED_AISTOR_OPERATOR_SHA256="e5534f5ae4f6f12a3528a8cda5954d73280cba1a8e979e1628fbf3aac76babd1"
readonly AISTOR_OBJECTSTORE_CHART_NAME="aistor-objectstore"
readonly AISTOR_OBJECTSTORE_CHART_VERSION="1.0.16"
readonly EXPECTED_AISTOR_OBJECTSTORE_SHA256="50ffa6a4e014cdc48566b237593baab41d039f103cdc50c1b1ac173b8d8bf71e"
readonly AISTOR_CHART_REPOSITORY="https://helm.min.io/"
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly -a RENDERED_MANIFEST_NAMES=(
phase2-namespaces
aistor-local-pv
keycloak-operator
platform-postgres-keycloak
keycloak
aistor-operator
minio-aistor
aistor-network-policies
)
fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
assert_regex_count() {
local file="$1"
local pattern="$2"
local expected="$3"
local description="$4"
local actual
[[ -f "$file" ]] || fail "missing validation input: ${file}"
actual="$(rg --count --no-filename -- "$pattern" "$file" || true)"
actual="${actual:-0}"
[[ "$actual" == "$expected" ]] || \
fail "${description}: expected ${expected}, found ${actual}"
}
usage() {
cat <<'USAGE'
Usage:
bash scripts/validate/render-phase2.sh
bash scripts/validate/render-phase2.sh \
--verified-output-dir /tmp/platform-phase2-apply.XXXXXX
The output option is an internal handoff used only by the AIStor apply script.
The destination must be an existing, empty, non-symlink directory created
below /tmp with the platform-phase2-apply.* prefix.
USAGE
}
verified_output_dir=""
case "$#" in
0)
;;
2)
[[ "$1" == "--verified-output-dir" ]] || {
usage >&2
exit 2
}
verified_output_dir="$2"
;;
*)
usage >&2
exit 2
;;
esac
if [[ -n "$verified_output_dir" ]]; then
[[ "$verified_output_dir" == /tmp/platform-phase2-apply.* ]] || \
fail "verified output directory must match /tmp/platform-phase2-apply.*"
[[ -d "$verified_output_dir" && ! -L "$verified_output_dir" ]] || \
fail "verified output directory must be an existing non-symlink directory"
[[ "$(cd -- "$verified_output_dir" && pwd -P)" == "$verified_output_dir" ]] || \
fail "verified output directory must be an absolute canonical path"
[[ -O "$verified_output_dir" ]] || \
fail "verified output directory must be owned by the current user"
[[ "$(stat --format='%a' -- "$verified_output_dir")" == "700" ]] || \
fail "verified output directory must have mode 0700"
[[ -z "$(find "$verified_output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || \
fail "verified output directory must be empty"
fi
for command_name in cmp find install kubectl rg sha256sum stat tar; do
command -v "$command_name" >/dev/null 2>&1 || \
fail "${command_name} is required"
done
if [[ -n "${PLATFORM_HELM_BIN:-}" ]]; then
[[ "$PLATFORM_HELM_BIN" == /* ]] || \
fail "PLATFORM_HELM_BIN must be an absolute path"
[[ -f "$PLATFORM_HELM_BIN" && -x "$PLATFORM_HELM_BIN" ]] || \
fail "PLATFORM_HELM_BIN is not an executable file: ${PLATFORM_HELM_BIN}"
readonly HELM_BIN="$PLATFORM_HELM_BIN"
else
HELM_BIN="$(command -v helm 2>/dev/null)" || \
fail "Helm ${EXPECTED_HELM_VERSION} is required"
readonly HELM_BIN
fi
kustomize_version="$(kubectl version --client --output=yaml | sed -n 's/^kustomizeVersion: //p')"
helm_version="$("$HELM_BIN" version --template '{{.Version}}')"
[[ "$kustomize_version" == "$EXPECTED_KUSTOMIZE_VERSION" ]] || \
fail "expected Kustomize ${EXPECTED_KUSTOMIZE_VERSION}, found ${kustomize_version:-unknown}"
[[ "$helm_version" == "$EXPECTED_HELM_VERSION" ]] || \
fail "expected Helm ${EXPECTED_HELM_VERSION}, found ${helm_version:-unknown}"
printf 'Validating the Phase 1 baseline and Gitea OIDC desired profile first.\n'
PLATFORM_HELM_BIN="$HELM_BIN" \
bash "${REPOSITORY_ROOT}/scripts/validate/render-phase1.sh"
render_dir="$(mktemp -d "${TMPDIR:-/tmp}/platform-phase2-render.XXXXXX")"
declare -a generated_chart_cache_dirs=()
declare -a generated_chart_cache_parent_dirs=()
cleanup() {
local cache_dir
local parent_dir
for cache_dir in "${generated_chart_cache_dirs[@]}"; do
case "$cache_dir" in
"${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm/charts/aistor-operator-5.10.0"|\
"${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm/charts/aistor-objectstore-1.0.16")
rm -rf -- "$cache_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected chart cache: %s\n' \
"$cache_dir" >&2
;;
esac
done
for parent_dir in "${generated_chart_cache_parent_dirs[@]}"; do
case "$parent_dir" in
"${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm/charts"|\
"${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm"|\
"${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm/charts"|\
"${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm")
rmdir -- "$parent_dir" 2>/dev/null || true
;;
*)
printf 'WARNING: refusing to remove unexpected chart cache parent: %s\n' \
"$parent_dir" >&2
;;
esac
done
case "$render_dir" in
/tmp/platform-phase2-render.*|"${TMPDIR:-/tmp}"/platform-phase2-render.*)
rm -rf -- "$render_dir"
;;
*)
printf 'WARNING: refusing to remove unexpected render directory: %s\n' \
"$render_dir" >&2
;;
esac
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_plain() {
local label="$1"
local relative_path="$2"
local output="${render_dir}/${label}.yaml"
kubectl kustomize "${REPOSITORY_ROOT}/${relative_path}" >"$output"
[[ -s "$output" ]] || fail "${label} rendered an empty manifest"
printf 'Rendered %-24s %8s bytes\n' \
"$label" "$(wc -c <"$output" | tr -d '[:space:]')"
}
render_helm() {
local label="$1"
local relative_path="$2"
local output="${render_dir}/${label}.yaml"
kubectl kustomize \
--enable-helm \
--helm-command "$HELM_BIN" \
--helm-kube-version "$TARGET_KUBERNETES_VERSION" \
"${REPOSITORY_ROOT}/${relative_path}" >"$output"
[[ -s "$output" ]] || fail "${label} rendered an empty manifest"
printf 'Rendered %-24s %8s bytes\n' \
"$label" "$(wc -c <"$output" | tr -d '[:space:]')"
}
prepare_verified_chart_cache() {
local label="$1"
local chart_name="$2"
local chart_version="$3"
local build_root_relative_path="$4"
local expected_digest="$5"
local package_path="${render_dir}/${chart_name}-${chart_version}.tgz"
local cache_version_dir="${REPOSITORY_ROOT}/${build_root_relative_path}/.helm/charts/${chart_name}-${chart_version}"
local checksum_output
local actual_digest
local chart_cache_dir="${cache_version_dir%/*}"
local helm_cache_dir="${chart_cache_dir%/*}"
local cache_parent
case "$cache_version_dir" in
"${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/.helm/charts/aistor-operator-5.10.0"|\
"${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/.helm/charts/aistor-objectstore-1.0.16")
;;
*)
fail "refusing to create unexpected chart cache: ${cache_version_dir}"
;;
esac
for cache_parent in "$helm_cache_dir" "$chart_cache_dir"; do
[[ ! -L "$cache_parent" ]] || \
fail "refusing symlinked chart cache parent: ${cache_parent}"
[[ ! -e "$cache_parent" || -d "$cache_parent" ]] || \
fail "chart cache parent is not a directory: ${cache_parent}"
done
[[ ! -e "$cache_version_dir" && ! -L "$cache_version_dir" ]] || \
fail "generated chart cache already exists; remove it after confirming it is disposable: ${cache_version_dir}"
"$HELM_BIN" pull "$chart_name" \
--repo "$AISTOR_CHART_REPOSITORY" \
--version "$chart_version" \
--destination "$render_dir"
[[ -f "$package_path" ]] || \
fail "${label} chart package was not downloaded: ${package_path}"
checksum_output="$(sha256sum -- "$package_path")"
actual_digest="${checksum_output%% *}"
[[ "$actual_digest" == "$expected_digest" ]] || \
fail "${label} chart SHA-256 mismatch: expected ${expected_digest}, found ${actual_digest}"
[[ -e "$chart_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$chart_cache_dir")
[[ -e "$helm_cache_dir" ]] || generated_chart_cache_parent_dirs+=("$helm_cache_dir")
mkdir -p -- "$cache_version_dir"
generated_chart_cache_dirs+=("$cache_version_dir")
tar -xzf "$package_path" -C "$cache_version_dir"
[[ -f "${cache_version_dir}/${chart_name}/Chart.yaml" ]] || \
fail "${label} extracted chart is missing Chart.yaml"
printf 'Verified %-24s SHA-256 %s\n' "$label" "$actual_digest"
}
cd -- "$REPOSITORY_ROOT"
if rg --line-number --glob '*.yaml' --glob '*.yml' \
--glob '!**/.helm/**' --glob '!**/charts/**' \
'^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \
infrastructure services bootstrap clusters components; then
fail "a source-controlled Kubernetes Secret manifest was found"
fi
if rg --line-number --glob 'kustomization.yaml' \
'LoadRestrictionsNone|load-restrictor' .; then
fail "the repository must keep Kustomize LoadRestrictionsRootOnly"
fi
keycloak_operator_kustomization="${REPOSITORY_ROOT}/infrastructure/controllers/keycloak-operator/kustomization.yaml"
aistor_operator_kustomization="${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/kustomization.yaml"
aistor_objectstore_kustomization="${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/kustomization.yaml"
aistor_operator_values="${REPOSITORY_ROOT}/infrastructure/controllers/aistor-operator/values/home.yaml"
aistor_objectstore_values="${REPOSITORY_ROOT}/services/minio-aistor/base/deployment/values/home.yaml"
assert_regex_count "$keycloak_operator_kustomization" '^[[:space:]]*-[[:space:]]github\.com/keycloak/keycloak-k8s-resources/kubernetes\?ref=26\.7\.0[[:space:]]*$' 1 "Keycloak Operator remote ref must be 26.7.0"
assert_regex_count "$keycloak_operator_kustomization" 'github\.com/keycloak/keycloak-k8s-resources/kubernetes' 1 "exactly one Keycloak Operator remote resource is allowed"
assert_regex_count "$keycloak_operator_kustomization" '\?ref=' 1 "exactly one pinned remote ref is allowed in the Keycloak Operator root"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*-[[:space:]]name:' 1 "AIStor Operator root must contain one Helm chart"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*-[[:space:]]name:[[:space:]]aistor-operator[[:space:]]*$' 1 "AIStor Operator chart name is pinned"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*repo:[[:space:]]https://helm\.min\.io/[[:space:]]*$' 1 "AIStor Operator chart repository is pinned"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*version:[[:space:]]5\.10\.0[[:space:]]*$' 1 "AIStor Operator chart version is pinned"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*valuesFile:[[:space:]]values/home\.yaml[[:space:]]*$' 1 "AIStor Operator values file is pinned"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*includeCRDs:[[:space:]]true[[:space:]]*$' 1 "AIStor Operator CRDs must be included"
assert_regex_count "$aistor_operator_kustomization" '^[[:space:]]*chartHome:[[:space:]]\.helm/charts[[:space:]]*$' 1 "AIStor Operator chartHome is pinned"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*-[[:space:]]name:' 1 "AIStor ObjectStore root must contain one Helm chart"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*-[[:space:]]name:[[:space:]]aistor-objectstore[[:space:]]*$' 1 "AIStor ObjectStore chart name is pinned"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*repo:[[:space:]]https://helm\.min\.io/[[:space:]]*$' 1 "AIStor ObjectStore chart repository is pinned"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*version:[[:space:]]1\.0\.16[[:space:]]*$' 1 "AIStor ObjectStore chart version is pinned"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*valuesFile:[[:space:]]values/home\.yaml[[:space:]]*$' 1 "AIStor ObjectStore values file is pinned"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*includeCRDs:[[:space:]]false[[:space:]]*$' 1 "AIStor ObjectStore root must not duplicate CRDs"
assert_regex_count "$aistor_objectstore_kustomization" '^[[:space:]]*chartHome:[[:space:]]\.helm/charts[[:space:]]*$' 1 "AIStor ObjectStore chartHome is pinned"
assert_regex_count "$aistor_operator_values" '^license:[[:space:]]*""[[:space:]]*$' 1 "AIStor license payload must not be rendered from values"
assert_regex_count "$aistor_objectstore_values" '^[[:space:]]*existingSecret:[[:space:]]true[[:space:]]*$' 1 "AIStor ObjectStore must use an existing Secret"
assert_regex_count "$aistor_objectstore_values" '^[[:space:]]*name:[[:space:]]aistor-root-configuration[[:space:]]*$' 2 "AIStor configuration Secret references are pinned"
assert_regex_count "$aistor_objectstore_values" '^[[:space:]]*serviceType:[[:space:]]ClusterIP[[:space:]]*$' 2 "AIStor S3 and Console services must be ClusterIP"
render_plain phase2-namespaces infrastructure/namespaces/phase2
render_plain aistor-local-pv infrastructure/storage/aistor-local-pv
render_plain aistor-network-policies infrastructure/networking/aistor
render_plain keycloak-operator infrastructure/controllers/keycloak-operator
render_plain platform-postgres-keycloak services/platform-postgres-keycloak
render_plain keycloak services/keycloak
prepare_verified_chart_cache \
aistor-operator-chart \
"$AISTOR_OPERATOR_CHART_NAME" \
"$AISTOR_OPERATOR_CHART_VERSION" \
infrastructure/controllers/aistor-operator \
"$EXPECTED_AISTOR_OPERATOR_SHA256"
render_helm aistor-operator infrastructure/controllers/aistor-operator
prepare_verified_chart_cache \
aistor-objectstore-chart \
"$AISTOR_OBJECTSTORE_CHART_NAME" \
"$AISTOR_OBJECTSTORE_CHART_VERSION" \
services/minio-aistor/base/deployment \
"$EXPECTED_AISTOR_OBJECTSTORE_SHA256"
render_helm minio-aistor services/minio-aistor
if rg --line-number '^[[:space:]]*kind:[[:space:]]*Secret[[:space:]]*$' \
"${render_dir}/phase2-namespaces.yaml" \
"${render_dir}/aistor-local-pv.yaml" \
"${render_dir}/keycloak-operator.yaml" \
"${render_dir}/platform-postgres-keycloak.yaml" \
"${render_dir}/keycloak.yaml" \
"${render_dir}/aistor-operator.yaml" \
"${render_dir}/minio-aistor.yaml" \
"${render_dir}/aistor-network-policies.yaml"; then
fail "a Phase 2 render unexpectedly contains a Secret"
fi
assert_regex_count "${render_dir}/phase2-namespaces.yaml" '^kind: Namespace$' 3 "Phase 2 namespace count"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^kind: StorageClass$' 1 "AIStor StorageClass count"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^kind: PersistentVolume$' 1 "AIStor PersistentVolume count"
assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^kind: NetworkPolicy$' 11 "AIStor NetworkPolicy count"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: DatabaseRole$' 1 "Keycloak DatabaseRole count"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: Database$' 1 "Keycloak Database count"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: NetworkPolicy$' 1 "Keycloak PostgreSQL NetworkPolicy count"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^kind: Cluster$' 0 "the Phase 2 PostgreSQL extension must not contain a Cluster"
assert_regex_count "${render_dir}/keycloak.yaml" '^kind: Keycloak$' 1 "Keycloak custom resource count"
assert_regex_count "${render_dir}/keycloak.yaml" '^kind: Ingress$' 1 "Keycloak Ingress count"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^kind: ObjectStore$' 1 "AIStor ObjectStore count"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*instances:[[:space:]]1[[:space:]]*$' 1 "Keycloak instance count field"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*hostname:[[:space:]]https://id\.learn\.hyeonworks\.com[[:space:]]*$' 1 "Keycloak external hostname"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*strict:[[:space:]]true[[:space:]]*$' 1 "Keycloak strict hostname mode"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*httpEnabled:[[:space:]]true[[:space:]]*$' 1 "Keycloak internal HTTP mode"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*httpPort:[[:space:]]8080[[:space:]]*$' 1 "Keycloak HTTP port"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*serviceHttpPort:[[:space:]]8080[[:space:]]*$' 1 "Keycloak Service HTTP port"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*headers:[[:space:]]xforwarded[[:space:]]*$' 1 "Keycloak forwarded-header mode"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*enabled:[[:space:]]false[[:space:]]*$' 1 "Keycloak Operator-managed Ingress must be disabled"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*ingressClassName:[[:space:]]traefik[[:space:]]*$' 1 "Keycloak Ingress class"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*-[[:space:]]host:[[:space:]]id\.learn\.hyeonworks\.com[[:space:]]*$' 1 "Keycloak Ingress host"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*name:[[:space:]]keycloak-service[[:space:]]*$' 1 "Keycloak Ingress backend Service"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*number:[[:space:]]8080[[:space:]]*$' 1 "Keycloak Ingress backend port"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*kubernetes\.io/metadata\.name:[[:space:]]kube-system[[:space:]]*$' 2 "Keycloak Traefik namespace selectors"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*app\.kubernetes\.io/name:[[:space:]]traefik[[:space:]]*$' 2 "Keycloak Traefik pod selectors"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*kubernetes\.io/metadata\.name:[[:space:]]keycloak[[:space:]]*$' 1 "Keycloak management namespace selector"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*app\.kubernetes\.io/name:[[:space:]]keycloak-operator[[:space:]]*$' 1 "Keycloak management Operator selector"
assert_regex_count "${render_dir}/keycloak.yaml" '^[[:space:]]*xaEnabled:[[:space:]]false[[:space:]]*$' 1 "Keycloak XA transaction mode"
assert_regex_count "${render_dir}/keycloak.yaml" ':[[:space:]]*9000[[:space:]]*$' 0 "Keycloak management port must not be exposed"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*name:[[:space:]]platform-postgres-keycloak[[:space:]]*$' 2 "Keycloak database and role names"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*name:[[:space:]]keycloak-db-credentials[[:space:]]*$' 1 "Keycloak DatabaseRole Secret reference"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*kubernetes\.io/metadata\.name:[[:space:]]keycloak[[:space:]]*$' 1 "Keycloak PostgreSQL namespace selector"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*-[[:space:]]port:[[:space:]]5432[[:space:]]*$' 1 "Keycloak PostgreSQL ingress port"
assert_regex_count "${render_dir}/platform-postgres-keycloak.yaml" '^[[:space:]]*cnpg\.io/cluster:[[:space:]]platform-postgres[[:space:]]*$' 1 "Keycloak PostgreSQL pod selector"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^provisioner:[[:space:]]kubernetes\.io/no-provisioner[[:space:]]*$' 1 "AIStor static StorageClass provisioner"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^reclaimPolicy:[[:space:]]Retain[[:space:]]*$' 1 "AIStor StorageClass reclaim policy"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^volumeBindingMode:[[:space:]]WaitForFirstConsumer[[:space:]]*$' 1 "AIStor volume binding mode"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*persistentVolumeReclaimPolicy:[[:space:]]Retain[[:space:]]*$' 1 "AIStor PersistentVolume reclaim policy"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*storage:[[:space:]]900Gi[[:space:]]*$' 1 "AIStor Local PV capacity"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*path:[[:space:]]/srv/k3s/aistor[[:space:]]*$' 1 "AIStor Local PV host path"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*-[[:space:]]donghyeon-system-product-name[[:space:]]*$' 1 "AIStor Local PV node affinity"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*platform\.hyeonworks\.com/filesystem:[[:space:]]xfs[[:space:]]*$' 1 "AIStor Local PV filesystem label"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*storageClassName:[[:space:]]aistor-local-xfs-retain[[:space:]]*$' 1 "AIStor Local PV StorageClass binding"
assert_regex_count "${render_dir}/aistor-local-pv.yaml" '^[[:space:]]*-[[:space:]]ReadWriteOnce[[:space:]]*$' 1 "AIStor Local PV access mode"
assert_regex_count "${render_dir}/aistor-operator.yaml" '^kind: CustomResourceDefinition$' 3 "AIStor Operator CRD count"
assert_regex_count "${render_dir}/aistor-operator.yaml" '^kind: Deployment$' 3 "AIStor Operator Deployment count"
assert_regex_count "${render_dir}/aistor-operator.yaml" '^[[:space:]]*caBundle:' 0 "AIStor webhook runtime CA must not be declaratively owned"
assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*namespace:[[:space:]]aistor$' 5 "AIStor control-plane NetworkPolicy namespace count"
assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*namespace:[[:space:]]object-storage$' 6 "AIStor data-plane NetworkPolicy namespace count"
assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*-[[:space:]]port:[[:space:]]8443$' 1 "AIStor admission webhook policy port"
assert_regex_count "${render_dir}/aistor-network-policies.yaml" '^[[:space:]]*-[[:space:]]port:[[:space:]]4221$' 2 "AIStor upgrade service policy port"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*servers:[[:space:]]1[[:space:]]*$' 1 "AIStor server count"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*volumesPerServer:[[:space:]]1[[:space:]]*$' 1 "AIStor volume count"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*storage:[[:space:]]900Gi[[:space:]]*$' 1 "AIStor claim size"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*storageClassName:[[:space:]]aistor-local-xfs-retain[[:space:]]*$' 1 "AIStor claim StorageClass"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*name:[[:space:]]aistor-root-configuration[[:space:]]*$' 1 "AIStor configuration Secret reference"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*pvcProtection:[[:space:]]true[[:space:]]*$' 1 "AIStor PVC protection"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*serviceType:[[:space:]]ClusterIP[[:space:]]*$' 2 "AIStor S3 and Console Service types"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*disableAutoCert:[[:space:]]true[[:space:]]*$' 1 "AIStor in-cluster TLS mode"
assert_regex_count "${render_dir}/minio-aistor.yaml" '^[[:space:]]*kubernetes\.io/hostname:[[:space:]]donghyeon-system-product-name[[:space:]]*$' 1 "AIStor ObjectStore node selector"
if rg --quiet '^[[:space:]]*kind:[[:space:]]*Ingress[[:space:]]*$|^[[:space:]]*(serviceType|type):[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \
"${render_dir}/minio-aistor.yaml"; then
fail "AIStor must not render Ingress, NodePort, or LoadBalancer exposure"
fi
if rg --quiet '^[[:space:]]*tls:[[:space:]]*(\[\])?[[:space:]]*$' \
"${render_dir}/keycloak.yaml"; then
fail "Keycloak must not render in-cluster TLS"
fi
if rg --quiet '^[[:space:]]*(serviceType|type):[[:space:]]*(NodePort|LoadBalancer)[[:space:]]*$' \
"${render_dir}/keycloak.yaml"; then
fail "Keycloak must not render NodePort or LoadBalancer exposure"
fi
if rg --quiet 'cidr:[[:space:]]*0\.0\.0\.0/0' \
"${render_dir}/aistor-network-policies.yaml"; then
fail "AIStor NetworkPolicies must not allow unrestricted Internet egress"
fi
if [[ -n "$verified_output_dir" ]]; then
for manifest_name in "${RENDERED_MANIFEST_NAMES[@]}"; do
source_manifest="${render_dir}/${manifest_name}.yaml"
output_manifest="${verified_output_dir}/${manifest_name}.yaml"
[[ -f "$source_manifest" && ! -L "$source_manifest" && -s "$source_manifest" ]] || \
fail "validated manifest is missing or unsafe: ${source_manifest}"
install -m 0600 -- "$source_manifest" "$output_manifest"
[[ -f "$output_manifest" && ! -L "$output_manifest" && -s "$output_manifest" ]] || \
fail "verified manifest handoff failed: ${output_manifest}"
cmp --silent -- "$source_manifest" "$output_manifest" || \
fail "verified manifest changed during handoff: ${manifest_name}.yaml"
done
verified_entry_count="$(
find "$verified_output_dir" -mindepth 1 -maxdepth 1 |
wc -l | tr -d '[:space:]'
)"
[[ "$verified_entry_count" == "${#RENDERED_MANIFEST_NAMES[@]}" ]] || \
fail "verified output directory does not contain exactly eight manifest files"
printf 'Preserved eight verified manifests for the AIStor apply handoff.\n'
fi
printf 'Phase 2 rendering and source invariants passed.\n'
printf 'No live ObjectStore CRD or Kubernetes cluster access was required.\n'
printf 'Temporary Phase 2 rendered manifests and generated chart caches will be removed on exit.\n'
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,751 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly CAPTURE="$ROOT/scripts/validate/capture-observability-metric-inventory.sh"
readonly REJECT_STATUS=23
readonly ORIGINAL_PATH="$PATH"
fail() {
printf 'TEST FAILURE: %s\n' "$*" >&2
exit 1
}
expect_pass() {
local name="$1" output
shift
if ! output="$({ "$@"; } 2>&1)"; then
fail "$name (expected success): $output"
fi
pass_count=$((pass_count + 1))
}
expect_reject() {
local name="$1" output rc=0
shift
output="$({ "$@"; } 2>&1)" || rc=$?
[[ "$rc" == "$REJECT_STATUS" ]] || fail "$name (expected $REJECT_STATUS, got $rc): $output"
[[ "$output" == REJECT:* ]] || fail "$name (unsanitized rejection): $output"
[[ "$output" != *DO-NOT-LEAK-FIXTURE* ]] || fail "$name leaked a fixture value"
pass_count=$((pass_count + 1))
}
[[ -f "$CAPTURE" ]] || fail "EXPECTED RED: capture command is missing: $CAPTURE"
# shellcheck source=/dev/null
source "$CAPTURE"
fixture_root="$(mktemp -d /tmp/platform-observability-inventory-test.XXXXXX)"
test_helper_pids=()
cleanup() {
local rc=$?
local helper_pid
for helper_pid in "${test_helper_pids[@]}"; do
if [[ "$helper_pid" =~ ^[0-9]+$ ]]; then kill -KILL "$helper_pid" 2>/dev/null || true; fi
done
if [[ "${KEEP_INVENTORY_TEST_FIXTURE:-0}" == 1 ]]; then
printf 'KEPT FIXTURE: %s\n' "$fixture_root" >&2
exit "$rc"
fi
case "$fixture_root" in
/tmp/platform-observability-inventory-test.*) rm -rf -- "$fixture_root" ;;
*) printf 'TEST FAILURE: refusing unsafe fixture cleanup\n' >&2; rc=1 ;;
esac
exit "$rc"
}
trap cleanup EXIT
pass_count=0
mapfile -t exact_pools <<'POOLS'
podMonitor/platform-data/platform-postgres/0
serviceMonitor/gitea/gitea/0
serviceMonitor/keycloak/keycloak/0
serviceMonitor/kube-system/traefik/0
serviceMonitor/object-storage/aistor-bucket-usage/0
serviceMonitor/observability-agent/alloy/0
serviceMonitor/observability-agent/node-exporter/0
serviceMonitor/observability/grafana/0
serviceMonitor/observability/loki/0
serviceMonitor/observability/observability-core-kube-pr-alertmanager/0
serviceMonitor/observability/observability-core-kube-pr-alertmanager/1
serviceMonitor/observability/observability-core-kube-pr-apiserver/0
serviceMonitor/observability/observability-core-kube-pr-coredns/0
serviceMonitor/observability/observability-core-kube-pr-kubelet/0
serviceMonitor/observability/observability-core-kube-pr-kubelet/1
serviceMonitor/observability/observability-core-kube-pr-kubelet/2
serviceMonitor/observability/observability-core-kube-pr-operator/0
serviceMonitor/observability/observability-core-kube-pr-prometheus/0
serviceMonitor/observability/observability-core-kube-pr-prometheus/1
serviceMonitor/observability/observability-core-kube-state-metrics/0
serviceMonitor/observability/tempo/0
POOLS
expected_target_selectors="$fixture_root/expected-target-selectors"
expected_series_selectors="$fixture_root/expected-series-selectors"
: >"$expected_target_selectors"
: >"$expected_series_selectors"
make_targets() {
local destination="$1" pool labels selector series_selector index=0
local rows="$fixture_root/target-rows.jsonl"
: >"$rows"
for pool in "${exact_pools[@]}"; do
labels="$(jq -cn --arg instance "10.0.0.$((index + 1)):9090" --arg target "target-$index" \
'{job:"fixture-job",instance:$instance,sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')"
selector="{instance=\"10.0.0.$((index + 1)):9090\",job=\"fixture-job\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",target_fixture=\"target-$index\"}"
if [[ "$pool" == serviceMonitor/object-storage/aistor-bucket-usage/0 ]]; then
labels="$(jq -cn --arg target "target-$index" \
'{job:"minio-aistor",namespace:"object-storage",service:"minio",endpoint:"http-minio",
container:"minio",instance:"10.42.7.19:9000",pod:"minio-aistor-pool-0-0",
sensitive_fixture:"DO-NOT-LEAK-FIXTURE",target_fixture:$target}')"
selector="{container=\"minio\",endpoint=\"http-minio\",instance=\"10.42.7.19:9000\",job=\"minio-aistor\",namespace=\"object-storage\",pod=\"minio-aistor-pool-0-0\",sensitive_fixture=\"DO-NOT-LEAK-FIXTURE\",service=\"minio\",target_fixture=\"target-$index\"}"
fi
printf '%s\n' "$selector" >>"$expected_target_selectors"
series_selector="{__name__=\"fixture_metric\",${selector#\{}"
printf '%s\n' "$series_selector" >>"$expected_series_selectors"
jq -cn \
--arg pool "$pool" \
--arg url "http://10.42.0.$((index + 1)):9090/metrics" \
--argjson labels "$labels" \
'{discoveredLabels:$labels,labels:$labels,scrapePool:$pool,scrapeUrl:$url,
globalUrl:$url,lastError:"",lastScrape:"2026-08-12T00:00:00Z",
lastScrapeDuration:0.01,health:"up",scrapeInterval:"30s",scrapeTimeout:"10s"}' \
>>"$rows"
index=$((index + 1))
done
jq -cs '{status:"success",data:{activeTargets:.,droppedTargets:[]}}' "$rows" >"$destination"
}
valid_targets="$fixture_root/targets.json"
make_targets "$valid_targets"
oversized_padding="$fixture_root/oversized-padding"
head -c 8388609 /dev/zero | tr '\000' x >"$oversized_padding"
oversized_targets="$fixture_root/oversized-targets.json"
jq --rawfile padding "$oversized_padding" '. + {padding:$padding}' "$valid_targets" >"$oversized_targets"
expect_pass 'exact 21 active pools accepted' validate_targets_json "$valid_targets"
post_targets="$fixture_root/post-targets.json"
post_probe_rows="$fixture_root/post-probe-rows.json"
post_substrate_expected_probe_targets | jq '[.[] as $target | $target + {
discoveredLabels:$target.labels,globalUrl:$target.scrapeUrl,lastError:"",
lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01,
health:"up",scrapeInterval:"30s",scrapeTimeout:"15s"}]' >"$post_probe_rows"
jq --slurpfile probes "$post_probe_rows" '
.data.activeTargets += ($probes[0] + [{
discoveredLabels:{},
labels:{job:"blackbox-exporter",namespace:"observability",service:"blackbox-exporter",
endpoint:"http",container:"blackbox-exporter",pod:"blackbox-exporter-6d8444ffdb-svsck",
instance:"10.42.0.123:9115"},
scrapePool:"serviceMonitor/observability/blackbox-exporter/0",
scrapeUrl:"http://10.42.0.123:9115/metrics",globalUrl:"http://10.42.0.123:9115/metrics",
lastError:"",lastScrape:"2026-08-12T00:00:00Z",lastScrapeDuration:0.01,
health:"up",scrapeInterval:"30s",scrapeTimeout:"10s"
}])
' "$valid_targets" >"$post_targets"
expect_pass 'exact post-substrate 30 targets and multiplicities accepted' \
validate_targets_json "$post_targets" post-substrate
expect_reject 'post-substrate targets rejected by target-initial phase' \
validate_targets_json "$post_targets" target-initial
post_missing_probe="$fixture_root/post-missing-probe.json"
jq 'del(.data.activeTargets[] | select(.scrapePool=="probe/observability/platform-public-edge"))' \
"$post_targets" >"$post_missing_probe"
expect_reject 'post-substrate missing Probe target rejected' \
validate_targets_json "$post_missing_probe" post-substrate
expect_pass 'Probe query is removed from persisted scrape URL' \
test "$(sanitize_scrape_url 'http://blackbox-exporter.observability.svc.cluster.local:9115/probe?module=http_2xx&target=https%3A%2F%2Fexample.test' strip-query)" = \
'http://blackbox-exporter.observability.svc.cluster.local:9115/probe'
expect_pass 'UTF-8 Prometheus label key is quoted in selector' \
test "$(prometheus_selector_for_labels '{"job":"blackbox","observability.hyeonworks.com/probe-group":"public-edge"}')" = \
'{job="blackbox","observability.hyeonworks.com/probe-group"="public-edge"}'
mutate_and_reject() {
local name="$1" filter="$2" destination
destination="$fixture_root/${name// /-}.json"
jq "$filter" "$valid_targets" >"$destination"
expect_reject "$name" validate_targets_json "$destination"
}
mutate_and_reject 'missing pool rejected' 'del(.data.activeTargets[0])'
mutate_and_reject 'extra pool rejected' '.data.activeTargets += [.data.activeTargets[0] | .scrapePool="serviceMonitor/extra/extra/0"]'
mutate_and_reject 'duplicate pool rejected' '.data.activeTargets += [.data.activeTargets[0]]'
mutate_and_reject 'down target rejected' '.data.activeTargets[0].health="down"'
mutate_and_reject 'last error rejected' '.data.activeTargets[0].lastError="DO-NOT-LEAK-FIXTURE"'
mutate_and_reject 'blackbox pool rejected' '.data.activeTargets[0].scrapePool="serviceMonitor/observability/blackbox-exporter/0"'
mutate_and_reject 'Probe pool rejected' '.data.activeTargets[0].scrapePool="probe/observability/public-edge/0"'
dropped_blackbox="$fixture_root/dropped-blackbox.json"
jq '.data.droppedTargets = [{
"discoveredLabels": {
"__meta_kubernetes_namespace": "observability",
"__meta_kubernetes_service_name": "blackbox-exporter",
"job": "blackbox-exporter"
},
"labels": {"job": "blackbox-exporter"},
"scrapePool": "serviceMonitor/observability/blackbox-exporter/0",
"scrapeUrl": "http://blackbox-exporter.observability.svc:9115/metrics",
"globalUrl": "http://blackbox-exporter.observability.svc:9115/metrics",
"lastError": "",
"health": "unknown"
}]' "$valid_targets" >"$dropped_blackbox"
expect_reject 'dropped blackbox identity rejected' validate_targets_json "$dropped_blackbox"
expect_pass 'post-substrate allows blackbox self discovery drops' \
validate_dropped_targets_json "$dropped_blackbox" post-substrate
dropped_probe="$fixture_root/dropped-probe.json"
jq '.data.droppedTargets = [{
"discoveredLabels": {
"__meta_kubernetes_namespace": "observability",
"__meta_kubernetes_service_name": "synthetic-exporter",
"monitor_kind": "Probe"
},
"labels": {"job": "synthetic-checks"},
"scrapePool": "probe/observability/public-edge/0",
"scrapeUrl": "http://synthetic-exporter.observability.svc:9115/metrics",
"globalUrl": "http://synthetic-exporter.observability.svc:9115/metrics",
"lastError": "",
"health": "unknown"
}]' "$valid_targets" >"$dropped_probe"
expect_reject 'dropped Probe identity rejected' validate_targets_json "$dropped_probe"
legitimate_probe_word="$fixture_root/legitimate-probe-word.json"
jq '.data.droppedTargets = [{
"discoveredLabels": {
"__meta_kubernetes_pod_annotation_fixture": "{\"livenessProbe\":true}",
"__metrics_path__": "/metrics/probes"
},
"labels": {"job": "platform-postgres"},
"scrapePool": "podMonitor/platform-data/platform-postgres/0",
"scrapeUrl": "http://10.42.0.1:9187/metrics/probes",
"globalUrl": "http://10.42.0.1:9187/metrics/probes",
"lastError": "",
"health": "unknown"
}]' "$valid_targets" >"$legitimate_probe_word"
expect_pass 'legitimate dropped metadata containing probe words accepted in active response' \
validate_targets_json "$legitimate_probe_word"
expect_pass 'legitimate dropped metadata containing probe words accepted in dropped response' \
validate_dropped_targets_json "$legitimate_probe_word"
empty_dropped_targets="$fixture_root/empty-dropped-targets.json"
printf '%s\n' '{"status":"success","data":{"activeTargets":[],"droppedTargets":[]}}' >"$empty_dropped_targets"
mutate_and_reject 'AIStor wrong job rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.job)="wrong"'
mutate_and_reject 'AIStor wrong namespace rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.namespace)="wrong"'
mutate_and_reject 'AIStor wrong service rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.service)="wrong"'
mutate_and_reject 'AIStor wrong endpoint rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.endpoint)="wrong"'
mutate_and_reject 'AIStor wrong container rejected' '(.data.activeTargets[] | select(.scrapePool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .labels.container)="wrong"'
expect_pass 'safe HTTP URL accepted' test "$(sanitize_scrape_url 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets')" = 'http://10.42.7.19:9000/minio/metrics/v3/cluster/usage/buckets'
expect_pass 'safe HTTPS URL accepted' test "$(sanitize_scrape_url 'https://metrics.example.test/metrics')" = 'https://metrics.example.test/metrics'
for unsafe_url in \
'http://user:DO-NOT-LEAK-FIXTURE@host/metrics' \
'http://host/metrics?token=DO-NOT-LEAK-FIXTURE' \
'http://host/metrics#DO-NOT-LEAK-FIXTURE' \
'ftp://host/metrics' \
'http://host/space here'; do
expect_reject 'unsafe scrape URL rejected' sanitize_scrape_url "$unsafe_url"
done
metadata="$fixture_root/metadata.json"
printf '%s\n' '{"status":"success","data":{"z_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}],"a_metric":[{"type":"counter","help":"ignored","unit":""}]}}' >"$metadata"
expect_pass 'metadata metric names sorted' test "$(extract_metric_names "$metadata")" = $'a_metric\nz_metric'
metadata_array="$fixture_root/metadata-array.json"
printf '%s\n' '{"status":"success","data":[{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""},{"target":{"job":"fixture"},"metric":"a_metric","type":"counter","help":"ignored","unit":""},{"target":{"job":"fixture"},"metric":"z_metric","type":"gauge","help":"duplicate","unit":""}]}' >"$metadata_array"
expect_pass 'current Prometheus metadata array yields unique sorted metric names' \
test "$(extract_metric_names "$metadata_array")" = $'a_metric\nz_metric'
bad_metadata_array="$fixture_root/bad-metadata-array.json"
printf '%s\n' '{"status":"success","data":[{"target":{},"metric":"bad metric","type":"gauge","help":"ignored","unit":""}]}' >"$bad_metadata_array"
expect_reject 'invalid current metadata array metric name rejected' extract_metric_names "$bad_metadata_array"
expect_pass 'post-substrate Probe pool accepts probe metrics' \
validate_metric_names_for_pool $'probe_duration_seconds\nprobe_success' \
probe/observability/platform-public-edge post-substrate
expect_reject 'target-initial rejects probe metrics' \
validate_metric_names_for_pool probe_success serviceMonitor/observability/grafana/0 target-initial
expect_reject 'post-substrate non-Probe pool rejects probe metrics' \
validate_metric_names_for_pool probe_success serviceMonitor/observability/blackbox-exporter/0 post-substrate
expect_reject 'post-substrate Probe pool requires probe_success descriptor' \
validate_metric_names_for_pool probe_duration_seconds probe/observability/platform-public-edge post-substrate
bad_metadata="$fixture_root/bad-metadata.json"
printf '%s\n' '{"status":"success","data":{"bad metric":[]}}' >"$bad_metadata"
expect_reject 'invalid metadata metric name rejected' extract_metric_names "$bad_metadata"
rules="$fixture_root/rules.json"
printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"z_metric","query":"DO-NOT-LEAK-FIXTURE"},{"type":"alerting","name":"a_metric"}]}]}}' >"$rules"
expect_reject 'recording-rule collision rejected' reject_recording_rule_collisions "$rules" "$metadata"
printf '%s\n' '{"status":"success","data":{"groups":[{"name":"fixture","rules":[{"type":"recording","name":"recorded_metric"}]}]}}' >"$rules"
expect_pass 'non-colliding recording rule accepted' reject_recording_rule_collisions "$rules" "$metadata"
series="$fixture_root/series.json"
printf '%s\n' '{"status":"success","data":[{"z_label":"DO-NOT-LEAK-FIXTURE","__name__":"a_metric","a_label":"private-value"},{"__name__":"a_metric","middle":"another-private-value"}]}' >"$series"
expect_pass 'series parser emits only sorted label keys' test "$(extract_series_label_names "$series")" = '["__name__","a_label","middle","z_label"]'
[[ "$(extract_series_label_names "$series")" != *private-value* ]] || fail 'series parser persisted label values'
dotted_series="$fixture_root/dotted-series.json"
printf '%s\n' '{"status":"success","data":[{"__name__":"probe_success","instance":"https://example.test/","job":"blackbox-public-edge","namespace":"observability","observability.hyeonworks.com/probe-group":"public-edge"}]}' >"$dotted_series"
expect_pass 'series parser preserves safe quoted Prometheus label names' \
test "$(extract_series_label_names "$dotted_series")" = \
'["__name__","instance","job","namespace","observability.hyeonworks.com/probe-group"]'
empty_probe_series="$fixture_root/empty-probe-series.json"
printf '%s\n' '{"status":"success","data":[]}' >"$empty_probe_series"
expect_pass 'target-initial requires probe series absence' \
validate_probe_series_phase "$empty_probe_series" target-initial
expect_reject 'post-substrate rejects probe series absence' \
validate_probe_series_phase "$empty_probe_series" post-substrate
expect_pass 'post-substrate accepts observability probe series' \
validate_probe_series_phase "$dotted_series" post-substrate
expect_reject 'target-initial rejects existing probe series' \
validate_probe_series_phase "$dotted_series" target-initial
post_labels="$fixture_root/post-labels.json"
post_substrate_expected_probe_targets | jq '[.[].labels]' >"$post_labels"
for query_kind in success status duration tls; do
query_fixture="$fixture_root/post-query-$query_kind.json"
jq -cn --slurpfile labels "$post_labels" --arg kind "$query_kind" '
($labels[0] | if $kind=="tls" then map(select(.instance|startswith("https://"))) else . end) as $selected |
{status:"success",data:{resultType:"vector",result:[$selected[] |
{metric:(. + {__name__:(if $kind=="success" then "probe_success"
elif $kind=="status" then "probe_http_status_code"
elif $kind=="duration" then "probe_duration_seconds"
else "probe_ssl_earliest_cert_expiry" end)}),
value:[1770000000,(if $kind=="success" then "1"
elif $kind=="status" then (if .job=="blackbox-private-edge" then "403" else "200" end)
elif $kind=="duration" then "0.1" else "4102444800" end)]}]}}
' >"$query_fixture"
expect_pass "post-substrate $query_kind query contract accepted" \
validate_post_substrate_query_body "$query_fixture" "$query_kind"
done
large_target_rows="$fixture_root/large-target-rows.jsonl"
awk 'BEGIN {
for (i = 0; i < 20000; i++)
printf "{\"scrape_pool\":\"pool/%d\",\"scrape_url\":\"http://10.42.0.1:9090/metrics\",\"health\":\"up\",\"last_error\":\"\",\"provenance\":\"core\",\"metrics\":[{\"name\":\"fixture_metric\",\"label_names\":[\"__name__\",\"instance\"]}]}\n", i
}' >"$large_target_rows"
large_inventory_contract() {
[[ "$(stat -c %s "$large_target_rows")" -gt 2097152 ]] || return 1
build_inventory_json "$large_target_rows" target-initial 2026-08-12T00:00:00Z |
jq -e '.phase=="target-initial" and (.targets|length)==20000' >/dev/null
}
expect_pass 'inventory builder reads target rows from file beyond argv size limit' large_inventory_contract
fake_bin="$fixture_root/bin"
mkdir -p "$fake_bin"
fake_log="$fixture_root/fake-commands.log"
pf_log="$fixture_root/port-forward-pids.log"
: >"$fake_log"
: >"$pf_log"
cat >"$fake_bin/kubectl" <<'FAKE_KUBECTL'
#!/usr/bin/env bash
set -Eeuo pipefail
printf 'kubectl' >>"$FAKE_LOG"
printf ' <%s>' "$@" >>"$FAKE_LOG"
printf '\n' >>"$FAKE_LOG"
args=" $* "
if [[ "$args" =~ [[:space:]](apply|delete|patch|replace|exec|rollout|scale)[[:space:]] ]]; then
exit 91
fi
if [[ "$args" == *' config current-context '* ]]; then printf 'default\n'; exit 0; fi
if [[ "$args" == *' config view '* ]]; then printf '%s\n' '{"clusters":[{"cluster":{"server":"https://127.0.0.1:6443"}}]}'; exit 0; fi
if [[ "$args" == *' get --raw=/readyz '* ]]; then printf 'ok\n'; exit 0; fi
if [[ "$args" == *' auth can-i '* ]]; then printf 'yes\n'; exit 0; fi
if [[ "$args" == *' get prometheus '* ]]; then printf '%s\n' '{"apiVersion":"monitoring.coreos.com/v1","kind":"Prometheus","metadata":{"name":"observability-core-kube-pr-prometheus","namespace":"observability"}}'; exit 0; fi
if [[ "$args" == *' get pods '* ]]; then printf '%s\n' '{"items":[{"metadata":{"name":"prometheus-observability-core-kube-pr-prometheus-0","namespace":"observability"},"status":{"phase":"Running","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"prometheus","ready":true}]}}]}'; exit 0; fi
if [[ "$args" == *' port-forward '* ]]; then
printf '%s\n' "$$" >>"$PF_LOG"
pf_stderr_path="$(readlink -- "/proc/$$/fd/2")"
printf '%s\n' "${pf_stderr_path%/*}" >>"$PF_WORKDIR_LOG"
if [[ "${PF_IGNORE_TERM:-0}" == 1 ]]; then
trap 'printf "ignored %s TERM\n" "$$" >>"$PF_LOG"' TERM
else
trap 'printf "stopped %s TERM\n" "$$" >>"$PF_LOG"; exit 0' TERM
fi
trap 'printf "stopped %s INT\n" "$$" >>"$PF_LOG"; exit 0' INT
trap 'printf "stopped %s HUP\n" "$$" >>"$PF_LOG"; exit 0' HUP
if [[ "${PF_NO_ANNOUNCEMENT:-0}" != 1 ]]; then
printf 'Forwarding from 127.0.0.1:43123 -> 9090\n' >&2
fi
while :; do /usr/bin/sleep 0.05; done
fi
exit 1
FAKE_KUBECTL
cat >"$fake_bin/curl" <<'FAKE_CURL'
#!/usr/bin/env bash
set -Eeuo pipefail
[[ "${1-}" == --disable ]] || exit 92
printf 'curl' >>"$FAKE_LOG"
printf ' <%s>' "$@" >>"$FAKE_LOG"
printf '\n' >>"$FAKE_LOG"
destination=''
url=''
max_filesize=''
all_arguments=" $* "
declare -a data_values=()
declare -a urlencode_values=()
while (($#)); do
case "$1" in
--output) destination="$2"; shift 2 ;;
--max-filesize) max_filesize="$2"; shift 2 ;;
--data) data_values+=("$2"); shift 2 ;;
--data-urlencode) urlencode_values+=("$2"); shift 2 ;;
http://*) url="$1"; shift ;;
*) shift ;;
esac
done
[[ -n "$destination" && -n "$url" ]] || exit 2
emit_text() {
if [[ "$destination" == - ]]; then printf '%s\n' "$1"; else printf '%s\n' "$1" >"$destination"; fi
}
emit_file() {
if [[ "$destination" == - ]]; then cat -- "$1"; else cp -- "$1" "$destination"; fi
}
case "$url" in
*/-/ready) emit_text 'ready' ;;
*/api/v1/targets)
[[ "${#data_values[@]}" == 1 ]] || exit 80
case "${data_values[0]}" in
state=dropped)
if [[ "${FAKE_CURL_MODE:-success}" == dropped_blackbox ]]; then
emit_file "$DROPPED_BLACKBOX_FIXTURE"
else
emit_file "$DROPPED_TARGETS_FIXTURE"
fi
exit 0
;;
state=active) ;;
*) exit 81 ;;
esac
if [[ "${FAKE_CURL_MODE:-success}" == hang ]]; then
trap 'exit 128' INT TERM HUP
while :; do /usr/bin/sleep 0.05; done
fi
if [[ "${FAKE_CURL_MODE:-success}" == error ]]; then exit 22; fi
if [[ "${FAKE_CURL_MODE:-success}" == oversized_targets ]]; then
[[ "$max_filesize" =~ ^[0-9]+$ ]] || exit 88
if (( $(stat -c %s "$OVERSIZED_TARGETS_FIXTURE") > max_filesize )); then exit 63; fi
emit_file "$OVERSIZED_TARGETS_FIXTURE"
else
emit_file "$TARGETS_FIXTURE"
fi
;;
*/api/v1/rules) emit_text '{"status":"success","data":{"groups":[]}}' ;;
*/api/v1/targets/metadata)
[[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match_target=* ]] || exit 84
[[ "${#data_values[@]}" == 1 && "${data_values[0]}" == 'limit=10000' ]] || exit 85
target_selector="${urlencode_values[0]#match_target=}"
grep -Fqx -- "$target_selector" "$EXPECTED_TARGET_SELECTORS" || exit 86
printf '%s\n' "$target_selector" >>"$METADATA_SELECTOR_LOG"
emit_text '{"status":"success","data":{"descriptor_only_metric":[{"type":"gauge","help":"registered without a current series","unit":""}],"fixture_metric":[{"type":"gauge","help":"DO-NOT-LEAK-FIXTURE","unit":""}]}}'
;;
*/api/v1/series)
if [[ "$all_arguments" == *'probe_.+'* ]]; then
[[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == 'match[]={__name__=~"probe_.+"}' ]] || exit 87
emit_text '{"status":"success","data":[]}'
else
[[ "${#urlencode_values[@]}" == 1 && "${urlencode_values[0]}" == match\[\]=* ]] || exit 89
series_selector="${urlencode_values[0]#match[]=}"
if [[ "$series_selector" == *'__name__="descriptor_only_metric"'* ]]; then
fixture_selector="${series_selector/descriptor_only_metric/fixture_metric}"
grep -Fqx -- "$fixture_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90
emit_text '{"status":"success","data":[]}'
exit 0
fi
grep -Fqx -- "$series_selector" "$EXPECTED_SERIES_SELECTORS" || exit 90
if [[ "${FAKE_CURL_MODE:-success}" == all_series_empty ]]; then
emit_text '{"status":"success","data":[]}'
exit 0
fi
printf '%s\n' "$series_selector" >>"$SERIES_SELECTOR_LOG"
if [[ "${FAKE_CURL_MODE:-success}" == hang_series ]]; then
printf '%s\n' "$$" >>"$CURL_PID_LOG"
: >"$CURL_HANG_MARKER"
trap 'exit 128' INT TERM HUP
while :; do /usr/bin/sleep 0.05; done
fi
emit_text '{"status":"success","data":[{"__name__":"fixture_metric","instance":"10.42.0.1:9090","secret_label":"DO-NOT-LEAK-FIXTURE"}]}'
fi
;;
*) exit 3 ;;
esac
FAKE_CURL
cat >"$fake_bin/ln" <<'FAKE_LN'
#!/usr/bin/env bash
set -Eeuo pipefail
if [[ "${FAIL_SHA_LINK:-0}" == 1 && "${*: -1}" == */inventory.sha256 ]]; then
exit 1
fi
exec /usr/bin/ln "$@"
FAKE_LN
chmod 0700 "$fake_bin/kubectl" "$fake_bin/curl" "$fake_bin/ln"
export PATH="$fake_bin:$ORIGINAL_PATH"
pf_workdir_log="$fixture_root/port-forward-workdirs.log"
curl_pid_log="$fixture_root/curl-pids.log"
metadata_selector_log="$fixture_root/metadata-selectors.log"
series_selector_log="$fixture_root/series-selectors.log"
: >"$pf_workdir_log"
: >"$curl_pid_log"
: >"$metadata_selector_log"
: >"$series_selector_log"
export FAKE_LOG="$fake_log" PF_LOG="$pf_log" PF_WORKDIR_LOG="$pf_workdir_log"
export CURL_PID_LOG="$curl_pid_log" TARGETS_FIXTURE="$valid_targets" OVERSIZED_TARGETS_FIXTURE="$oversized_targets"
export DROPPED_TARGETS_FIXTURE="$empty_dropped_targets" DROPPED_BLACKBOX_FIXTURE="$dropped_blackbox"
export EXPECTED_TARGET_SELECTORS="$expected_target_selectors" EXPECTED_SERIES_SELECTORS="$expected_series_selectors"
export METADATA_SELECTOR_LOG="$metadata_selector_log" SERIES_SELECTOR_LOG="$series_selector_log"
run_capture() {
local output_dir="$1"
shift
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" \
PF_WORKDIR_LOG="$PF_WORKDIR_LOG" CURL_PID_LOG="$CURL_PID_LOG" \
EXPECTED_TARGET_SELECTORS="$EXPECTED_TARGET_SELECTORS" EXPECTED_SERIES_SELECTORS="$EXPECTED_SERIES_SELECTORS" \
METADATA_SELECTOR_LOG="$METADATA_SELECTOR_LOG" SERIES_SELECTOR_LOG="$SERIES_SELECTOR_LOG" \
DROPPED_TARGETS_FIXTURE="$DROPPED_TARGETS_FIXTURE" DROPPED_BLACKBOX_FIXTURE="$DROPPED_BLACKBOX_FIXTURE" \
OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \
FAKE_CURL_MODE="${FAKE_CURL_MODE:-success}" \
bash "$CAPTURE" --execute --phase target-initial --output-dir "$output_dir" "$@"
}
inventory_kubectl_alias() { inventory_kubectl "$@"; }
mutation_log_lines="$(wc -l <"$fake_log")"
for wrapper_name in inventory_kubectl inventory_kubectl_alias; do
for mutation_verb in apply create delete patch replace exec rollout scale; do
expect_reject "$wrapper_name default-denies $mutation_verb" \
"$wrapper_name" "$mutation_verb" fixture
done
done
[[ "$(wc -l <"$fake_log")" == "$mutation_log_lines" ]] || fail 'denied wrapper mutation reached kubectl'
ownership_signal_log="$fixture_root/ownership-signals.log"
ownership_ready="$fixture_root/ownership-ready"
bash -c '
trap '\''printf "TERM\n" >>"$1"; exit 0'\'' TERM
: >"$2"
while :; do /usr/bin/sleep 0.05; done
' bash "$ownership_signal_log" "$ownership_ready" &
ownership_pid=$!
test_helper_pids+=("$ownership_pid")
for ((attempt=0; attempt<40; attempt++)); do [[ -e "$ownership_ready" ]] && break; sleep 0.025; done
[[ -e "$ownership_ready" ]] || fail 'ownership fixture did not start'
ownership_snapshot="$(inventory_process_snapshot "$ownership_pid")" || fail 'ownership fixture snapshot failed'
INVENTORY_PORT_FORWARD_PID="$ownership_pid"
INVENTORY_PORT_FORWARD_START="$((10#${ownership_snapshot#*:} + 1))"
expect_pass 'wrong start-time PID is not signaled' inventory_stop_port_forward
[[ ! -s "$ownership_signal_log" ]] || fail 'wrong start-time process received TERM'
kill -0 "$ownership_pid" 2>/dev/null || fail 'wrong start-time process was killed'
kill -TERM "$ownership_pid" 2>/dev/null || true
for ((attempt=0; attempt<40; attempt++)); do kill -0 "$ownership_pid" 2>/dev/null || break; sleep 0.025; done
cli_dir="$fixture_root/cli"
mkdir -m 0700 "$cli_dir"
expect_reject 'missing CLI arguments rejected' bash "$CAPTURE"
expect_reject 'missing execute rejected' bash "$CAPTURE" --phase target-initial --output-dir "$cli_dir"
expect_reject 'duplicate argument rejected' bash "$CAPTURE" --execute --phase target-initial --phase target-initial --output-dir "$cli_dir"
expect_reject 'unknown phase rejected' bash "$CAPTURE" --execute --phase future-phase --output-dir "$cli_dir"
expect_reject 'unknown argument rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$cli_dir" --unknown
relative_dir="${cli_dir#/}"
expect_reject 'relative output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$relative_dir"
noncanonical="$fixture_root/../$(basename "$fixture_root")/cli"
expect_reject 'noncanonical output path rejected' bash "$CAPTURE" --execute --phase target-initial --output-dir "$noncanonical"
chmod 0755 "$cli_dir"
expect_reject 'non-0700 output directory rejected' run_capture "$cli_dir"
chmod 0700 "$cli_dir"
ln -s "$cli_dir" "$fixture_root/cli-link"
expect_reject 'symlink output directory rejected' run_capture "$fixture_root/cli-link"
mkdir -m 0700 "$fixture_root/unsafe-parent"
mkdir -m 0700 "$fixture_root/real-child"
ln -s "$fixture_root/unsafe-parent" "$fixture_root/parent-link"
mkdir -m 0700 "$fixture_root/unsafe-parent/child"
expect_reject 'symlink path component rejected' run_capture "$fixture_root/parent-link/child"
printf 'preserve-me\n' >"$cli_dir/preexisting"
expect_reject 'nonempty output directory rejected without clobber' run_capture "$cli_dir"
[[ "$(cat "$cli_dir/preexisting")" == preserve-me ]] || fail 'preexisting file was modified'
rm "$cli_dir/preexisting"
empty_series_dir="$fixture_root/empty-series"
mkdir -m 0700 "$empty_series_dir"
FAKE_CURL_MODE=all_series_empty
export FAKE_CURL_MODE
expect_reject 'target with metadata descriptors but no actual series rejected' run_capture "$empty_series_dir"
unset FAKE_CURL_MODE
[[ -z "$(find "$empty_series_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'empty-series response produced accepted output'
: >"$metadata_selector_log"
: >"$series_selector_log"
success_dir="$fixture_root/success"
mkdir -m 0700 "$success_dir"
expect_pass 'full capture succeeds with fixture APIs' run_capture "$success_dir"
cmp -s <(sort "$expected_target_selectors") <(sort "$metadata_selector_log") || fail 'metadata calls did not cover every exact full-label selector once'
cmp -s <(sort "$expected_series_selectors") <(sort "$series_selector_log") || fail 'series calls did not cover every exact metric selector once'
mapfile -t output_names < <(find "$success_dir" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)
[[ "${output_names[*]}" == 'inventory.json inventory.sha256' ]] || fail 'capture did not create exactly two files'
for output_name in "${output_names[@]}"; do
[[ -f "$success_dir/$output_name" && ! -L "$success_dir/$output_name" ]] || fail "$output_name is not a regular non-symlink"
[[ "$(stat -c %a "$success_dir/$output_name")" == 600 ]] || fail "$output_name mode is not 0600"
done
[[ "$(jq -cS . "$success_dir/inventory.json")"$'\n' == "$(cat "$success_dir/inventory.json")"$'\n' ]] || fail 'inventory is not canonical jq -cS JSON'
(cd "$success_dir" && sha256sum --check inventory.sha256 >/dev/null) || fail 'inventory hash does not bind JSON bytes'
jq -e '
.schema == "platform-observability-metric-inventory/v1" and
.phase == "target-initial" and .context == "default" and
.api_server == "https://127.0.0.1:6443" and
.prometheus == "observability/observability-core-kube-pr-prometheus" and
.preexisting_verify_only == ["grafana"] and
(.targets | length == 21) and
([.targets[].scrape_pool] == ([.targets[].scrape_pool] | sort)) and
(all(.targets[]; .health == "up" and .last_error == "" and
.metrics == [{"label_names":["__name__","instance","secret_label"],"name":"fixture_metric"}])) and
([.targets[] | select(.scrape_pool=="serviceMonitor/observability/grafana/0") | .provenance] == ["preexisting-verify-only"]) and
([.targets[] | select(.scrape_pool=="serviceMonitor/object-storage/aistor-bucket-usage/0") | .provenance] == ["task6-owned"])
' "$success_dir/inventory.json" >/dev/null || fail 'inventory schema/provenance/content mismatch'
if rg -q 'DO-NOT-LEAK-FIXTURE|private-value|help|query|fragment|exemplar' "$success_dir/inventory.json"; then
fail 'inventory persisted a prohibited value or field'
fi
success_pf_pid="$(head -n 1 "$pf_log")"
if kill -0 "$success_pf_pid" 2>/dev/null; then fail 'port-forward survived success'; fi
oversized_dir="$fixture_root/oversized-response"
mkdir -m 0700 "$oversized_dir"
FAKE_CURL_MODE=oversized_targets
export FAKE_CURL_MODE
expect_reject 'Prometheus response above the byte ceiling rejected' run_capture "$oversized_dir"
unset FAKE_CURL_MODE
[[ -z "$(find "$oversized_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'oversized response produced accepted output'
dropped_live_dir="$fixture_root/dropped-live-response"
mkdir -m 0700 "$dropped_live_dir"
FAKE_CURL_MODE=dropped_blackbox
export FAKE_CURL_MODE
expect_reject 'live dropped blackbox response rejected with exact active set' run_capture "$dropped_live_dir"
unset FAKE_CURL_MODE
[[ -z "$(find "$dropped_live_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'dropped blackbox response produced accepted output'
sigkill_dir="$fixture_root/sigkill"
mkdir -m 0700 "$sigkill_dir"
sigkill_marker="$fixture_root/sigkill-series-started"
sigkill_pf_lines="$(wc -l <"$pf_log")"
sigkill_workdir_lines="$(wc -l <"$pf_workdir_log")"
sigkill_curl_lines="$(wc -l <"$curl_pid_log")"
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" PF_WORKDIR_LOG="$PF_WORKDIR_LOG" \
CURL_PID_LOG="$CURL_PID_LOG" CURL_HANG_MARKER="$sigkill_marker" \
TARGETS_FIXTURE="$TARGETS_FIXTURE" OVERSIZED_TARGETS_FIXTURE="$OVERSIZED_TARGETS_FIXTURE" \
FAKE_CURL_MODE=hang_series bash "$CAPTURE" --execute --phase target-initial --output-dir "$sigkill_dir" \
>/dev/null 2>&1 &
sigkill_capture_pid=$!
for ((attempt=0; attempt<200; attempt++)); do
[[ -e "$sigkill_marker" ]] && break
kill -0 "$sigkill_capture_pid" 2>/dev/null || break
sleep 0.025
done
[[ -e "$sigkill_marker" ]] || fail 'SIGKILL residue fixture did not reach the series request'
sigkill_workdir="$(sed -n "$((sigkill_workdir_lines + 1))p" "$pf_workdir_log")"
sigkill_pf_pid="$(sed -n "$((sigkill_pf_lines + 1))p" "$pf_log")"
sigkill_curl_pid="$(sed -n "$((sigkill_curl_lines + 1))p" "$curl_pid_log")"
[[ "$sigkill_workdir" == /tmp/platform-observability-inventory-capture.* && -d "$sigkill_workdir" && ! -L "$sigkill_workdir" ]] || fail 'SIGKILL fixture work directory was not identified safely'
kill -KILL "$sigkill_capture_pid" 2>/dev/null || true
wait "$sigkill_capture_pid" 2>/dev/null || true
for owned_pid in "$sigkill_curl_pid" "$sigkill_pf_pid"; do
[[ "$owned_pid" =~ ^[0-9]+$ ]] || fail 'SIGKILL fixture lost an owned child PID'
kill -KILL "$owned_pid" 2>/dev/null || true
for ((attempt=0; attempt<40; attempt++)); do
kill -0 "$owned_pid" 2>/dev/null || break
sleep 0.025
done
done
raw_residue="$(find "$sigkill_workdir" -type f \( -name 'targets*.json' -o -name 'rules*.json' -o -name 'metadata*.json' -o -name 'series*.json' \) -print)"
secret_residue=0
if rg -q 'DO-NOT-LEAK-FIXTURE' "$sigkill_workdir"; then secret_residue=1; fi
find "$sigkill_workdir" -depth -delete
[[ -z "$raw_residue" ]] || fail "SIGKILL left raw Prometheus response files: $raw_residue"
[[ "$secret_residue" == 0 ]] || fail 'SIGKILL residue persisted a raw label/help value'
pass_count=$((pass_count + 1))
stubborn_pf_dir="$fixture_root/stubborn-port-forward"
mkdir -m 0700 "$stubborn_pf_dir"
stubborn_lines="$(wc -l <"$pf_log")"
set +e
timeout --preserve-status --signal=TERM --kill-after=0.5s 5s \
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" PF_IGNORE_TERM=1 \
bash "$CAPTURE" --execute --phase target-initial --output-dir "$stubborn_pf_dir" >/dev/null 2>&1
stubborn_rc=$?
set -e
stubborn_pf_pid="$(sed -n "$((stubborn_lines + 1))p" "$pf_log")"
[[ "$stubborn_pf_pid" =~ ^[0-9]+$ ]] || fail 'TERM-ignoring fixture did not start a port-forward'
stubborn_survived_production_cleanup=0
if kill -0 "$stubborn_pf_pid" 2>/dev/null; then
stubborn_survived_production_cleanup=1
kill -KILL "$stubborn_pf_pid" 2>/dev/null || true
for ((attempt=0; attempt<40; attempt++)); do
kill -0 "$stubborn_pf_pid" 2>/dev/null || break
sleep 0.025
done
fi
[[ "$stubborn_rc" == 0 ]] || fail "TERM-ignoring port-forward cleanup exceeded its bound (status $stubborn_rc)"
grep -Fqx "ignored $stubborn_pf_pid TERM" "$pf_log" || fail 'TERM-ignoring fixture did not receive TERM before escalation'
[[ "$stubborn_survived_production_cleanup" == 0 ]] || fail 'TERM-ignoring port-forward survived production cleanup'
pass_count=$((pass_count + 1))
silent_pf_dir="$fixture_root/silent-port-forward"
mkdir -m 0700 "$silent_pf_dir"
PF_NO_ANNOUNCEMENT=1
export PF_NO_ANNOUNCEMENT
expect_pass 'readiness does not depend on port-forward status text' run_capture "$silent_pf_dir"
unset PF_NO_ANNOUNCEMENT
publish_failure_dir="$fixture_root/publish-failure"
mkdir -m 0700 "$publish_failure_dir"
FAIL_SHA_LINK=1
export FAIL_SHA_LINK
expect_reject 'second-file publication failure removes only run-owned output' run_capture "$publish_failure_dir"
unset FAIL_SHA_LINK
[[ -z "$(find "$publish_failure_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'publication failure left a partial inventory'
error_dir="$fixture_root/error"
mkdir -m 0700 "$error_dir"
error_pf_lines="$(wc -l <"$pf_log")"
FAKE_CURL_MODE=error
export FAKE_CURL_MODE
expect_reject 'ordinary API error is sanitized' run_capture "$error_dir"
[[ -z "$(find "$error_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail 'ordinary error left partial output'
error_pf_pid="$(sed -n "$((error_pf_lines + 1))p" "$pf_log")"
[[ "$error_pf_pid" =~ ^[0-9]+$ ]] || fail 'ordinary error did not start a port-forward'
if kill -0 "$error_pf_pid" 2>/dev/null; then fail 'port-forward survived ordinary error'; fi
unset FAKE_CURL_MODE
for signal in INT TERM HUP; do
signal_dir="$fixture_root/signal-${signal,,}"
mkdir -m 0700 "$signal_dir"
before_lines="$(wc -l <"$pf_log")"
set +e
timeout --preserve-status --signal="$signal" --kill-after=2s 0.3s \
env PATH="$PATH" FAKE_LOG="$FAKE_LOG" PF_LOG="$PF_LOG" TARGETS_FIXTURE="$TARGETS_FIXTURE" FAKE_CURL_MODE=hang \
bash "$CAPTURE" --execute --phase target-initial --output-dir "$signal_dir" >/dev/null 2>&1
signal_rc=$?
set -e
[[ "$signal_rc" != 0 ]] || fail "$signal unexpectedly succeeded"
[[ -z "$(find "$signal_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || fail "$signal left partial output"
signal_pf_pid="$(sed -n "$((before_lines + 1))p" "$pf_log")"
[[ "$signal_pf_pid" =~ ^[0-9]+$ ]] || fail "$signal did not start a port-forward"
if kill -0 "$signal_pf_pid" 2>/dev/null; then fail "port-forward survived $signal"; fi
pass_count=$((pass_count + 1))
done
if awk '
/kubectl/ {
line=$0
if (line ~ / <(apply|delete|patch|replace|exec|rollout|scale)>/) exit 1
if (line ~ / <create>/ && line !~ / <auth> <can-i> <create> <pods\/portforward>/) exit 1
}
' "$fake_log"; then :; else fail 'dynamic command trace contains a mutation verb'; fi
# Whole-source proof complements the runtime trace. Removing line boundaries
# prevents a backslash-newline, command prefix, or wrapper alias from hiding a
# mutation verb. The exact RBAC question is the sole create exception.
[[ "$(rg -Fxc " 'create pods/portforward'; do" "$CAPTURE")" == 1 ]] || fail 'port-forward RBAC exception is not pinned exactly once'
mutation_free_source="$(sed "s/'create pods\\/portforward'/'RBAC_PORTFORWARD'/" "$CAPTURE")"
if rg -q '(^|[^[:alnum:]_])(apply|create|delete|patch|replace|exec|rollout|scale)($|[^[:alnum:]_])' <<<"$mutation_free_source"; then
fail 'whole-source proof found a Kubernetes mutation verb'
fi
[[ "$(rg -c '^[[:space:]]*kubectl --request-timeout=' "$CAPTURE")" == 2 ]] || fail 'kubectl invocation count escaped the wrapper/port-forward allowlist'
if rg -q '^[[:space:]]*(alias[[:space:]]+[^=]+=.*kubectl|[A-Za-z_][A-Za-z0-9_]*=.*kubectl)' "$CAPTURE"; then
fail 'kubectl executable alias bypasses the default-deny wrapper'
fi
pass_count=$((pass_count + 1))
printf 'PASS: %d target-initial inventory assertions\n' "$pass_count"
+651
View File
@@ -0,0 +1,651 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly SCRIPT="$ROOT/scripts/bootstrap/configure-keycloak-grafana-oidc.sh"
readonly NOW_UTC='2026-08-12T07:00:00Z'
readonly CLIENT_SECRET='GrafanaFixtureClientSecret-123456789'
WORK=''
ASSERTIONS=0
fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
pass() { ASSERTIONS=$((ASSERTIONS + 1)); printf 'PASS: %s\n' "$1"; }
cleanup() {
trap - EXIT HUP INT TERM
case "$WORK" in /tmp/platform-grafana-oidc-test.??????) rm -rf -- "$WORK" ;; esac
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
write_executable() {
local path=$1
shift
printf '%s\n' "$@" >"$path"
chmod 0755 "$path"
}
make_fakes() {
local fixture=$1
mkdir -p "$fixture/bin" "$fixture/state" "$fixture/evidence-parent"
chmod 0700 "$fixture" "$fixture/state" "$fixture/evidence-parent"
chmod 0755 "$fixture/bin"
: >"$fixture/commands.log"
write_executable "$fixture/bin/encryption" \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'printf '\''encryption'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''--\n'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \
'n=0; [[ ! -f "$PLATFORM_TEST_ENCRYPTION_COUNT" ]] || read -r n <"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
'n=$((n + 1)); printf '\''%s\n'\'' "$n" >"$PLATFORM_TEST_ENCRYPTION_COUNT"' \
'[[ "$n" != "${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" ]]'
write_executable "$fixture/bin/restore" \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'printf '\''restore'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
'if [[ -n "${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" ]]; then /usr/bin/env | LC_ALL=C /usr/bin/sort >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; printf '\''--\n'\'' >>"$PLATFORM_TEST_VALIDATOR_ENV_LOG"; fi' \
'n=0; [[ ! -f "$PLATFORM_TEST_RESTORE_COUNT" ]] || read -r n <"$PLATFORM_TEST_RESTORE_COUNT"' \
'n=$((n + 1)); printf '\''%s\n'\'' "$n" >"$PLATFORM_TEST_RESTORE_COUNT"' \
'if [[ "$n" == 1 && -f "$PLATFORM_TEST_STATE/remove-evidence-parent" ]]; then /usr/bin/mv -- "${PLATFORM_TEST_STATE%/state}/evidence-parent" "${PLATFORM_TEST_STATE%/state}/evidence-parent-missing"; fi' \
'[[ "$n" != "${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" ]]'
write_executable "$fixture/bin/sudo" \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'printf '\''sudo'\'' >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\'' %q'\'' "$@" >>"$PLATFORM_TEST_COMMAND_LOG"; printf '\''\n'\'' >>"$PLATFORM_TEST_COMMAND_LOG"' \
'if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED"; exit 0; fi' \
'[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92' \
'[[ "${1:-}" != --non-interactive && "${1:-}" != -n ]] || shift' \
'exec "$@"'
cat >"$fixture/bin/keycloak-api" <<'PY'
#!/usr/bin/env python3
import http.server, json, os, pathlib, signal, sys, time, urllib.parse
port = int(sys.argv[1]); root = pathlib.Path(os.environ["PLATFORM_TEST_STATE"])
state_path = root / "keycloak.json"; log = pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"])
def load(): return json.loads(state_path.read_text())
def save(value): state_path.write_text(json.dumps(value, sort_keys=True))
def public_client(item): return {k:v for k,v in item.items() if k != "_secret"}
def maybe_fault(label, handler):
fault = os.environ.get("PLATFORM_TEST_FAULT", "")
marker = root / ("fault-fired-" + label)
if marker.exists():
return False
if fault == "timeout-" + label:
marker.write_text("1")
time.sleep(5)
return True
if fault == "loss-" + label:
marker.write_text("1")
handler.close_connection = True
return True
if fault == "http500-" + label:
marker.write_text("1")
handler.send_response(500); handler.end_headers(); return True
if fault == "drift-secret-" + label:
marker.write_text("1")
secret_path=root/"secret.json"
item=json.loads(secret_path.read_text())
item["metadata"]["uid"]="secret-concurrent-replacement"
item["metadata"]["resourceVersion"]="99"
secret_path.write_text(json.dumps(item,sort_keys=True))
handler.send_response(500); handler.end_headers(); return True
if fault.startswith("signal-") and fault.endswith("-" + label):
marker.write_text("1")
sig = fault.split("-", 2)[1]
os.kill(int(os.environ["PLATFORM_TEST_TARGET_PID"]), getattr(signal, "SIG" + sig))
handler.close_connection = True
return True
return False
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, *_): return
def body(self):
length = int(self.headers.get("Content-Length", "0")); data = self.rfile.read(length)
return json.loads(data) if data else None
def send_json(self, code, value):
payload = json.dumps(value, separators=(",", ":")).encode()
self.send_response(code); self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload))); self.end_headers(); self.wfile.write(payload)
def empty(self, code): self.send_response(code); self.end_headers()
def parts(self): return urllib.parse.urlparse(self.path), urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query)
def do_GET(self):
parsed, query = self.parts(); path = parsed.path; state = load()
if path.endswith("/.well-known/openid-configuration"):
return self.send_json(200, {"issuer":"https://id.learn.hyeonworks.com/realms/hyeonworks"})
if path.endswith("/clients"):
matches=[public_client(x) for x in state["clients"] if x.get("clientId") == query.get("clientId",[""])[0]]
return self.send_json(200, matches[:int(query.get("max",["2"])[0])])
if "/clients/" in path and path.endswith("/client-secret"):
cid=path.split("/clients/",1)[1].split("/",1)[0]
matches=[x for x in state["clients"] if x["id"] == cid]
return self.send_json(200,{"type":"secret","value":matches[0]["_secret"]}) if len(matches)==1 else self.empty(404)
if "/clients/" in path and path.endswith("/protocol-mappers/models"):
cid=path.split("/clients/",1)[1].split("/",1)[0]
return self.send_json(200,state["mappers"].get(cid,[]))
if "/clients/" in path:
cid=path.split("/clients/",1)[1].split("/",1)[0]; matches=[public_client(x) for x in state["clients"] if x["id"]==cid]
return self.send_json(200,matches[0]) if len(matches)==1 else self.empty(404)
if path.endswith("/groups") and "/users/" not in path:
search=query.get("search",[""])[0]
matches=[x for x in state["groups"] if x.get("name")==search or x.get("path")=="/"+search]
return self.send_json(200,matches[:int(query.get("max",["2"])[0])])
if "/groups/" in path:
gid=path.split("/groups/",1)[1].split("/",1)[0]; matches=[x for x in state["groups"] if x["id"]==gid]
return self.send_json(200,matches[0]) if len(matches)==1 else self.empty(404)
if path.endswith("/users"):
username=query.get("username",[""])[0]; matches=[{"id":x["id"],"username":x["username"]} for x in state["users"] if x["username"]==username]
return self.send_json(200,matches[:int(query.get("max",["2"])[0])])
if "/users/" in path and path.endswith("/groups"):
uid=path.split("/users/",1)[1].split("/",1)[0]; users=[x for x in state["users"] if x["id"]==uid]
groups=[x for x in state["groups"] if x["id"] in users[0]["groups"]] if len(users)==1 else []
return self.send_json(200,groups[:int(query.get("max",["101"])[0])])
self.empty(404)
def do_POST(self):
with log.open("a",encoding="utf-8") as stream: stream.write("keycloak POST "+self.path+"\n")
parsed,_=self.parts(); path=parsed.path
if path.endswith("/protocol/openid-connect/token"):
self.rfile.read(int(self.headers.get("Content-Length", "0")))
return self.send_json(200,{"access_token":"fixture-admin-token","token_type":"Bearer"})
body=self.body(); state=load()
if path.endswith("/clients"):
item=body; item["id"]="client-created"; item["_secret"]=os.environ["PLATFORM_TEST_CLIENT_SECRET"]
state["clients"].append(item); state["mappers"][item["id"]]=[]; save(state)
if maybe_fault("client-create",self): return
self.send_response(201); self.send_header("Location",self.path+"/"+item["id"]); self.end_headers(); return
if path.endswith("/groups"):
item=body; item["id"]="group-created-"+str(len(state["groups"])+1); item["path"]="/"+item["name"]
state["groups"].append(item); save(state)
if maybe_fault("group-create",self): return
self.send_response(201); self.send_header("Location",self.path+"/"+item["id"]); self.end_headers(); return
if path.endswith("/protocol-mappers/models"):
cid=path.split("/clients/",1)[1].split("/",1)[0]; item=body; item["id"]="mapper-created"
state["mappers"].setdefault(cid,[]).append(item); save(state)
if maybe_fault("mapper-create",self): return
self.send_response(201); self.send_header("Location",self.path+"/"+item["id"]); self.end_headers(); return
self.empty(404)
def do_PUT(self):
with log.open("a",encoding="utf-8") as stream: stream.write("keycloak PUT "+self.path+"\n")
parsed,_=self.parts(); path=parsed.path; body=self.body(); state=load()
if "/protocol-mappers/models/" in path:
cid=path.split("/clients/",1)[1].split("/",1)[0]; mid=path.rsplit("/",1)[1]
for i,item in enumerate(state["mappers"].get(cid,[])):
if item["id"]==mid: body["id"]=mid; state["mappers"][cid][i]=body; save(state); break
if maybe_fault("mapper-put",self): return
return self.empty(204)
if "/clients/" in path:
cid=path.split("/clients/",1)[1].split("/",1)[0]
for i,item in enumerate(state["clients"]):
if item["id"]==cid:
secret=body.pop("secret",item["_secret"]); body["id"]=cid; body["_secret"]=secret; state["clients"][i]=body; save(state); break
if maybe_fault("client-put",self): return
return self.empty(204)
if "/groups/" in path and "/users/" not in path:
gid=path.split("/groups/",1)[1].split("/",1)[0]
for i,item in enumerate(state["groups"]):
if item["id"]==gid:
body["id"]=gid; body["path"]="/"+body["name"]; state["groups"][i]=body; save(state); break
if maybe_fault("group-put",self): return
return self.empty(204)
if "/users/" in path and "/groups/" in path:
uid=path.split("/users/",1)[1].split("/",1)[0]; gid=path.rsplit("/",1)[1]
for user in state["users"]:
if user["id"]==uid and gid not in user["groups"]: user["groups"].append(gid)
save(state)
label="admin-member-put" if gid.startswith("group-admin") else "viewer-member-put"
if maybe_fault(label,self): return
return self.empty(204)
self.empty(404)
def do_DELETE(self):
with log.open("a",encoding="utf-8") as stream: stream.write("keycloak DELETE "+self.path+"\n")
parsed,_=self.parts(); path=parsed.path; state=load()
if "/users/" in path and "/groups/" in path:
uid=path.split("/users/",1)[1].split("/",1)[0]; gid=path.rsplit("/",1)[1]
for user in state["users"]:
if user["id"]==uid and gid in user["groups"]: user["groups"].remove(gid)
save(state); return self.empty(204)
if "/protocol-mappers/models/" in path:
cid=path.split("/clients/",1)[1].split("/",1)[0]; mid=path.rsplit("/",1)[1]
if os.environ.get("PLATFORM_TEST_ROLLBACK_FAULT","")=="noop-mapper-delete":
(root/"rollback-fault-fired-noop-mapper-delete").write_text("1")
return self.empty(204)
state["mappers"][cid]=[x for x in state["mappers"].get(cid,[]) if x["id"]!=mid]; save(state); return self.empty(204)
if "/clients/" in path:
cid=path.split("/clients/",1)[1].split("/",1)[0]; state["clients"]=[x for x in state["clients"] if x["id"]!=cid]; state["mappers"].pop(cid,None); save(state); return self.empty(204)
if "/groups/" in path:
gid=path.rsplit("/",1)[1]
if os.environ.get("PLATFORM_TEST_ROLLBACK_FAULT","")=="noop-group-delete":
(root/"rollback-fault-fired-noop-group-delete").write_text("1")
return self.empty(204)
state["groups"]=[x for x in state["groups"] if x["id"]!=gid]
for user in state["users"]: user["groups"]=[x for x in user["groups"] if x!=gid]
save(state); return self.empty(204)
self.empty(404)
try:
server=http.server.ThreadingHTTPServer(("127.0.0.1",port),Handler)
server.serve_forever()
except BaseException as error:
(root/"keycloak-api-error").write_text(repr(error))
raise
PY
chmod 0755 "$fixture/bin/keycloak-api"
/usr/bin/python3 -m py_compile "$fixture/bin/keycloak-api"
cat >"$fixture/bin/kube-api" <<'PY'
#!/usr/bin/env python3
import base64, http.server, json, os, pathlib, socketserver, sys
sock=sys.argv[1]; root=pathlib.Path(os.environ["PLATFORM_TEST_STATE"]); path=root/"secret.json"
def load(): return json.loads(path.read_text()) if path.exists() else None
class Server(socketserver.UnixStreamServer): pass
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self,*_): return
def payload(self):
n=int(self.headers.get("Content-Length","0")); raw=self.rfile.read(n); return json.loads(raw) if raw else None
def send_json(self,code,item):
raw=json.dumps(item,separators=(",",":")).encode(); self.send_response(code); self.send_header("Content-Type","application/json"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
def do_GET(self):
item=load()
if self.path.endswith("/grafana-keycloak-oidc"):
return self.send_json(200,item) if item else self.send_json(404,{"kind":"Status","reason":"NotFound"})
self.send_json(405,{"kind":"Status"})
def do_POST(self):
if path.exists(): return self.send_json(409,{"kind":"Status","reason":"AlreadyExists"})
item=self.payload(); item["metadata"]["uid"]="secret-created-uid"; item["metadata"]["resourceVersion"]="1"; path.write_text(json.dumps(item,sort_keys=True))
fault=os.environ.get("PLATFORM_TEST_FAULT","")
if fault=="loss-secret-create": self.close_connection=True; return
if fault=="conflict-secret-create": path.unlink(); return self.send_json(409,{"kind":"Status","reason":"Conflict"})
self.send_json(201,item)
def do_DELETE(self):
item=load(); options=self.payload() or {}; pre=(options.get("preconditions") or {})
if not item: return self.send_json(404,{"kind":"Status","reason":"NotFound"})
if pre.get("uid")!=item["metadata"]["uid"] or pre.get("resourceVersion")!=item["metadata"]["resourceVersion"]:
return self.send_json(409,{"kind":"Status","reason":"Conflict"})
path.unlink(); self.send_json(200,{"kind":"Status","status":"Success"})
with Server(sock,Handler) as server: server.serve_forever()
PY
chmod 0755 "$fixture/bin/kube-api"
/usr/bin/python3 -m py_compile "$fixture/bin/kube-api"
cat >"$fixture/bin/kubectl" <<'PY'
#!/usr/bin/env python3
import base64,json,os,pathlib,sys
args=sys.argv[1:]; log=pathlib.Path(os.environ["PLATFORM_TEST_COMMAND_LOG"])
with log.open("a") as f: f.write("kubectl "+" ".join(args)+"\n")
if "--request-timeout=5s" not in args: raise SystemExit(84)
args=[x for x in args if x!="--request-timeout=5s"]
def opt(name):
for i,x in enumerate(args):
if x==name and i+1<len(args): return args[i+1]
if x.startswith(name+"="): return x.split("=",1)[1]
return None
if args==["config","current-context"]: print(os.environ.get("PLATFORM_TEST_CONTEXT","default")); raise SystemExit()
if args[:3]==["config","view","--minify"]: print("https://127.0.0.1:6443",end=""); raise SystemExit()
if args[:3]==["get","node","donghyeon-system-product-name"]:
print(json.dumps({"apiVersion":"v1","kind":"Node","metadata":{"name":"donghyeon-system-product-name","uid":"node-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]}})); raise SystemExit()
if args[:2]==["auth","can-i"]: print("yes"); raise SystemExit()
if args[:3]==["get","namespace","keycloak"] or args[:3]==["get","namespace","observability"]: print("namespace/"+args[2]); raise SystemExit()
if "get" in args and "keycloak.k8s.keycloak.org/keycloak" in args:
print(json.dumps({"apiVersion":"k8s.keycloak.org/v2beta1","kind":"Keycloak","metadata":{"name":"keycloak","namespace":"keycloak","uid":"kc-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]}})); raise SystemExit()
if "get" in args and "service/keycloak-service" in args:
print(json.dumps({"apiVersion":"v1","kind":"Service","metadata":{"name":"keycloak-service","namespace":"keycloak","uid":"svc-uid"},"spec":{"ports":[{"port":8080,"targetPort":8080}]}})); raise SystemExit()
if "get" in args and "secret/keycloak-initial-admin" in args:
item={"apiVersion":"v1","kind":"Secret","type":"kubernetes.io/basic-auth","metadata":{"name":"keycloak-initial-admin","namespace":"keycloak","uid":"admin-uid","resourceVersion":"1"},"data":{"username":base64.b64encode(b"temp-admin").decode(),"password":base64.b64encode(b"TempAdminPassword-123").decode()}}
print(json.dumps(item)); raise SystemExit()
if any(x.startswith("--accept-methods") for x in args): raise SystemExit(86)
if args and args[0]=="port-forward":
mapping=next(x for x in args if x.endswith(":8080")); port=mapping.split(":",1)[0]
print("Forwarding from 127.0.0.1:"+port+" -> 8080",flush=True)
os.execv(os.environ["PLATFORM_TEST_KEYCLOAK_API"],[os.environ["PLATFORM_TEST_KEYCLOAK_API"],port])
if args and args[0]=="proxy":
sock=opt("--unix-socket"); os.execv(os.environ["PLATFORM_TEST_KUBE_API"],[os.environ["PLATFORM_TEST_KUBE_API"],sock])
raise SystemExit(83)
PY
chmod 0755 "$fixture/bin/kubectl"
}
seed_state() {
local fixture=$1 profile=${2:-absent}
python3 -I -S - "$fixture/state/keycloak.json" "$profile" "$CLIENT_SECRET" <<'PY'
import json,pathlib,sys
path,profile,secret=sys.argv[1:]
client={"id":"client-existing","clientId":"grafana","name":"Drifted Grafana","enabled":False,"protocol":"openid-connect","publicClient":True,"standardFlowEnabled":False,"implicitFlowEnabled":True,"directAccessGrantsEnabled":True,"serviceAccountsEnabled":True,"authorizationServicesEnabled":True,"fullScopeAllowed":True,"rootUrl":"https://wrong.invalid","baseUrl":"https://wrong.invalid","redirectUris":["https://wrong.invalid/cb"],"webOrigins":["+"],"attributes":{},"_secret":secret}
groups=[{"id":"group-admin-existing","name":"platform-observability-admins","path":"/platform-observability-admins"},{"id":"group-viewer-existing","name":"platform-observability-viewers","path":"/platform-observability-viewers"}]
mapper={"id":"mapper-existing","name":"grafana-groups","protocol":"openid-connect","protocolMapper":"oidc-hardcoded-claim-mapper","consentRequired":True,"config":{"claim.name":"wrong"}}
state={"clients":[],"groups":[],"mappers":{},"users":[{"id":"user-admin","username":"admin-user","groups":[]},{"id":"user-viewer","username":"viewer-user","groups":[]}]}
if profile in {"existing","exact-secret","preexisting-admin","duplicate-client","duplicate-user","duplicate-group","mapper-duplicate"}:
state["clients"]=[client]; state["groups"]=groups; state["mappers"]={"client-existing":[mapper]}
if profile=="exact-secret":
state["clients"][0].update({"name":"Grafana","description":"Grafana confidential OIDC client managed by the platform bootstrap","enabled":True,"protocol":"openid-connect","clientAuthenticatorType":"client-secret","publicClient":False,"standardFlowEnabled":True,"implicitFlowEnabled":False,"directAccessGrantsEnabled":False,"serviceAccountsEnabled":False,"authorizationServicesEnabled":False,"consentRequired":False,"fullScopeAllowed":False,"rootUrl":"https://grafana.learn.hyeonworks.com","baseUrl":"https://grafana.learn.hyeonworks.com","redirectUris":["https://grafana.learn.hyeonworks.com/login/generic_oauth"],"webOrigins":["https://grafana.learn.hyeonworks.com"],"attributes":{"post.logout.redirect.uris":"https://grafana.learn.hyeonworks.com/*","oauth2.device.authorization.grant.enabled":"false","oidc.ciba.grant.enabled":"false"}})
state["mappers"]["client-existing"]=[{"id":"mapper-existing","name":"grafana-groups","protocol":"openid-connect","protocolMapper":"oidc-group-membership-mapper","consentRequired":False,"config":{"claim.name":"groups","full.path":"true","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true"}}]
if profile=="preexisting-admin": state["users"][0]["groups"]=["group-admin-existing"]
if profile=="duplicate-client": state["clients"].append(dict(client,id="client-second"))
if profile=="duplicate-user": state["users"].append({"id":"user-admin-second","username":"admin-user","groups":[]})
if profile=="duplicate-group": state["groups"].append({"id":"group-admin-second","name":"platform-observability-admins","path":"/platform-observability-admins"})
if profile=="mapper-duplicate": state["mappers"]["client-existing"].append(dict(mapper,id="mapper-second",name="other",config={"claim.name":"groups"}))
pathlib.Path(path).write_text(json.dumps(state,sort_keys=True))
PY
}
seed_secret() {
local fixture=$1 secret=${2:-$CLIENT_SECRET}
python3 -I -S - "$fixture/state/secret.json" "$secret" <<'PY'
import base64,json,pathlib,sys
path,secret=sys.argv[1:]
item={"apiVersion":"v1","kind":"Secret","type":"Opaque","metadata":{"namespace":"observability","name":"grafana-keycloak-oidc","uid":"secret-existing-uid","resourceVersion":"7"},"data":{"client-id":base64.b64encode(b"grafana").decode(),"client-secret":base64.b64encode(secret.encode()).decode()}}
pathlib.Path(path).write_text(json.dumps(item,sort_keys=True))
PY
}
new_fixture() {
local name=$1 profile=${2:-absent} fixture
fixture="$WORK/$name"
mkdir -p "$fixture"; make_fakes "$fixture"; seed_state "$fixture" "$profile"; printf '%s\n' "$fixture"
}
run_script() {
local fixture=$1; shift
env \
PATH="$fixture/bin:$PATH" \
PLATFORM_GRAFANA_OIDC_CONFIRMATIONS="${PLATFORM_TEST_CONFIRMATIONS:-}" \
PLATFORM_GRAFANA_OIDC_NOW_UTC="$NOW_UTC" \
PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
PLATFORM_TEST_STATE="$fixture/state" \
PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \
PLATFORM_TEST_ENCRYPTION_COUNT="$fixture/encryption-count" \
PLATFORM_TEST_RESTORE_COUNT="$fixture/restore-count" \
PLATFORM_TEST_ENCRYPTION_FAIL_AT="${PLATFORM_TEST_ENCRYPTION_FAIL_AT:-0}" \
PLATFORM_TEST_RESTORE_FAIL_AT="${PLATFORM_TEST_RESTORE_FAIL_AT:-0}" \
PLATFORM_TEST_VALIDATOR_ENV_LOG="${PLATFORM_TEST_VALIDATOR_ENV_LOG:-}" \
PLATFORM_TEST_CONTEXT="${PLATFORM_TEST_CONTEXT:-default}" \
PLATFORM_TEST_FAULT="${PLATFORM_TEST_FAULT:-}" \
PLATFORM_TEST_ROLLBACK_FAULT="${PLATFORM_TEST_ROLLBACK_FAULT:-}" \
PLATFORM_TEST_CLIENT_SECRET="$CLIENT_SECRET" \
PLATFORM_TEST_KEYCLOAK_API="$fixture/bin/keycloak-api" \
PLATFORM_TEST_KUBE_API="$fixture/bin/kube-api" \
bash -c 'source "$1"; shift; platform_grafana_oidc_fixture_main "$@"' \
platform-grafana-oidc-test "$SCRIPT" "$fixture" "$@"
}
assert_exact_contract() {
local fixture=$1
python3 -I -S - "$fixture/state/keycloak.json" "$fixture/state/secret.json" "$CLIENT_SECRET" <<'PY' || return 1
import base64,json,pathlib,sys
kc=json.loads(pathlib.Path(sys.argv[1]).read_text()); secret=json.loads(pathlib.Path(sys.argv[2]).read_text()); expected=sys.argv[3]
assert len(kc["clients"])==1
c=kc["clients"][0]
assert c["clientId"]=="grafana" and c["publicClient"] is False and c["standardFlowEnabled"] is True
assert all(c[x] is False for x in ("implicitFlowEnabled","directAccessGrantsEnabled","serviceAccountsEnabled","authorizationServicesEnabled"))
assert c["rootUrl"]==c["baseUrl"]=="https://grafana.learn.hyeonworks.com"
assert c["redirectUris"]==["https://grafana.learn.hyeonworks.com/login/generic_oauth"]
assert c["webOrigins"]==["https://grafana.learn.hyeonworks.com"] and c["fullScopeAllowed"] is False
assert c["attributes"]["post.logout.redirect.uris"]=="https://grafana.learn.hyeonworks.com/*"
assert c["attributes"]["oauth2.device.authorization.grant.enabled"]=="false" and c["attributes"]["oidc.ciba.grant.enabled"]=="false"
assert c["_secret"]==expected
assert sorted(x["path"] for x in kc["groups"])==["/platform-observability-admins","/platform-observability-viewers"]
m=kc["mappers"][c["id"]]; assert len(m)==1
assert m[0]["name"]=="grafana-groups" and m[0]["protocolMapper"]=="oidc-group-membership-mapper"
assert m[0]["config"]=={"access.token.claim":"true","claim.name":"groups","full.path":"true","id.token.claim":"true","userinfo.token.claim":"true"}
assert secret["type"]=="Opaque" and sorted(secret["data"])==["client-id","client-secret"]
assert base64.b64decode(secret["data"]["client-id"])==b"grafana"
assert base64.b64decode(secret["data"]["client-secret"]).decode()==expected
PY
}
WORK="$(mktemp -d /tmp/platform-grafana-oidc-test.XXXXXX)"; chmod 0700 "$WORK"
[[ -f "$SCRIPT" ]] || fail 'production Grafana OIDC bootstrap is absent'
fixture="$(new_fixture dry-run)"
output="$(run_script "$fixture")" || fail 'no-argument dry-run failed'
grep -Fq 'GRAFANA_OIDC_DRY_RUN=PASS' <<<"$output" || fail 'dry-run marker absent'
[[ ! -s "$fixture/commands.log" ]] || fail 'dry-run crossed a system boundary'
[[ "$output" != *"$CLIENT_SECRET"* ]] || fail 'dry-run leaked a payload'
pass 'no-argument mode is payload-free and read-only'
fixture="$(new_fixture boundary)"
for argv in '--admin admin-user' '--execute --check-recovery-evidence' '--check-recovery-evidence --viewer viewer-user' '--execute --admin' '--execute --admin admin-user --admin admin-user'; do
read -r -a args <<<"$argv"
if run_script "$fixture" "${args[@]}" >"$fixture/out" 2>&1; then fail "unsafe CLI accepted: $argv"; fi
[[ ! -s "$fixture/commands.log" ]] || fail "invalid CLI crossed a boundary: $argv"
done
if PLATFORM_GRAFANA_OIDC_TEST_MODE=1 bash "$SCRIPT" >"$fixture/direct" 2>&1; then fail 'production entrypoint accepted an environment test bypass'; fi
pass 'CLI modes are closed and production rejects test overrides'
fixture="$(new_fixture last-gate)"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_RESTORE_FAIL_AT=2 \
run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail 'last restore gate failure was accepted'; fi
[[ "$(grep -c '^encryption --expect-reencrypted$' "$fixture/commands.log")" == 2 ]] || {
sed -n '1,200p' "$fixture/out" >&2; sed -n '1,240p' "$fixture/commands.log" >&2
[[ ! -f "$fixture/state/keycloak-api-error" ]] || cat "$fixture/state/keycloak-api-error" >&2
fail 'encryption gate was not fresh twice'
}
[[ "$(grep -c '^restore --check$' "$fixture/commands.log")" == 2 ]] || {
sed -n '1,200p' "$fixture/out" >&2; sed -n '1,240p' "$fixture/commands.log" >&2
fail 'restore gate was not fresh twice'
}
python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail 'last gate mutated Keycloak'
import json,pathlib,sys
x=json.loads(pathlib.Path(sys.argv[1]).read_text()); assert x["clients"]==[] and x["groups"]==[]
PY
[[ ! -e "$fixture/state/secret.json" ]] || fail 'last gate mutated Kubernetes'
pass 'both validator pairs run in fresh processes and last-gate failure is zero-mutation'
fixture="$(new_fixture missing-evidence-parent)"
: >"$fixture/state/remove-evidence-parent"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \
run_script "$fixture" --execute >"$fixture/out" 2>&1; then
fail 'missing recovery-evidence parent was accepted'
fi
! grep -Fq 'Type APPLY default:' "$fixture/out" ||
fail 'missing recovery-evidence parent reached confirmation'
! grep -Eq '^keycloak (POST|PUT|DELETE) /admin/realms/hyeonworks/' "$fixture/commands.log" ||
fail 'missing recovery-evidence parent reached Keycloak mutation'
[[ ! -e "$fixture/state/secret.json" ]] ||
fail 'missing recovery-evidence parent reached Kubernetes Secret mutation'
pass 'missing recovery-evidence parent fails closed before object mutation'
fixture="$(new_fixture create)"
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute >"$fixture/out" 2>&1 || { sed -n '1,220p' "$fixture/out" >&2; fail 'create path failed'; }
assert_exact_contract "$fixture" || fail 'created contract differs'
[[ "$(<"$fixture/out")" != *"$CLIENT_SECRET"* ]] || fail 'success output leaked client secret'
! grep -Fq "$CLIENT_SECRET" "$fixture/commands.log" || fail 'client secret reached argv log'
grep -Fq 'GRAFANA_OIDC_TRANSACTION=PASS' "$fixture/out" || fail 'success marker absent'
marker="$fixture/evidence-parent/recovery-evidence/keycloak.env"
[[ -f "$marker" && ! -L "$marker" && "$(stat -c '%u:%a:%h' "$marker")" == "$(id -u):600:1" ]] || fail 'marker metadata differs'
[[ "$(wc -l <"$marker" | tr -d '[:space:]')" == 4 ]] || fail 'marker key set differs'
grep -Fqx 'schema=platform-observability-recovery-evidence-v1' "$marker" || fail 'marker schema differs'
grep -Fqx 'context=default' "$marker" || fail 'marker context differs'
grep -Fqx 'resource=keycloak/hyeonworks/client/grafana' "$marker" || fail 'marker resource differs'
grep -Fqx "checked_at_utc=$NOW_UTC" "$marker" || fail 'marker timestamp differs'
pass 'create path establishes the exact client, groups, mapper, Secret, and evidence marker'
fixture="$(new_fixture hostile-curlrc)"
mkdir -m 0700 -- "$fixture/curl-home"
printf 'trace-ascii = "%s"\n' "$fixture/curl-trace" >"$fixture/curl-home/.curlrc"
chmod 0600 -- "$fixture/curl-home/.curlrc"
CURL_HOME="$fixture/curl-home" \
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \
run_script "$fixture" --execute >"$fixture/out" 2>&1 || {
sed -n '1,220p' "$fixture/out" >&2
fail 'hostile curl default-config fixture did not complete'
}
[[ ! -e "$fixture/curl-trace" ]] || fail 'curl loaded a user default config and wrote a credential trace'
pass 'curl ignores user default configuration across credential-bearing requests'
marker="$fixture/evidence-parent/recovery-evidence/keycloak.env"
: >"$fixture/commands.log"
run_script "$fixture" --check-recovery-evidence >"$fixture/check" 2>&1 || fail 'fresh evidence rejected'
grep -Fq 'KEYCLOAK_RECOVERY_EVIDENCE=PASS' "$fixture/check" || fail 'check marker absent'
! grep -q '^encryption\|^restore' "$fixture/commands.log" || fail 'check mode ran mutation gates'
grep -q '^sudo ' "$fixture/commands.log" || fail 'check did not use narrow sudo'
if PLATFORM_TEST_CONTEXT=other run_script "$fixture" --check-recovery-evidence >"$fixture/wrong" 2>&1; then fail 'wrong context accepted evidence'; fi
ln -- "$marker" "$marker.second"
if run_script "$fixture" --check-recovery-evidence >"$fixture/link" 2>&1; then fail 'multiply linked marker accepted'; fi
unlink -- "$marker.second"
cp -- "$marker" "$fixture/marker-valid"
sed -i 's/checked_at_utc=.*/checked_at_utc=2026-06-01T00:00:00Z/' "$marker"
if run_script "$fixture" --check-recovery-evidence >"$fixture/stale" 2>&1; then fail 'stale recovery evidence accepted'; fi
cp -- "$fixture/marker-valid" "$marker"
printf 'extra=forbidden\n' >>"$marker"
if run_script "$fixture" --check-recovery-evidence >"$fixture/extra" 2>&1; then fail 'extra recovery evidence field accepted'; fi
cp -- "$fixture/marker-valid" "$marker"
mv -- "$fixture/evidence-parent/recovery-evidence" "$fixture/evidence-parent/recovery-evidence-real"
ln -s -- recovery-evidence-real "$fixture/evidence-parent/recovery-evidence"
if run_script "$fixture" --check-recovery-evidence >"$fixture/symlink" 2>&1; then fail 'symlink evidence directory accepted'; fi
unlink -- "$fixture/evidence-parent/recovery-evidence"
mv -- "$fixture/evidence-parent/recovery-evidence-real" "$fixture/evidence-parent/recovery-evidence"
pass 'evidence check binds context, age, schema, and no-follow metadata'
fixture="$(new_fixture existing existing)"; seed_secret "$fixture"
before_secret="$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')"
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1 || {
sed -n '1,240p' "$fixture/out" >&2; sed -n '1,300p' "$fixture/commands.log" >&2
fail 'existing update failed'
}
assert_exact_contract "$fixture" || fail 'existing update contract differs'
[[ "$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" == "$before_secret" ]] || fail 'exact existing Secret was rotated or rewritten'
python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail 'requested memberships absent'
import json,pathlib,sys
x=json.loads(pathlib.Path(sys.argv[1]).read_text()); ids={g["path"]:g["id"] for g in x["groups"]}; users={u["username"]:u for u in x["users"]}
assert ids["/platform-observability-admins"] in users["admin-user"]["groups"]
assert ids["/platform-observability-viewers"] in users["viewer-user"]["groups"]
PY
pass 'existing client is declaratively updated without credential rotation and memberships are optional'
fixture="$(new_fixture exact-noop exact-secret)"; seed_secret "$fixture"
before_keycloak="$(sha256sum "$fixture/state/keycloak.json" | awk '{print $1}')"
before_exact_secret="$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')"
PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute >"$fixture/out" 2>&1 || {
sed -n '1,240p' "$fixture/out" >&2; fail 'exact-existing no-op failed'
}
[[ "$(sha256sum "$fixture/state/keycloak.json" | awk '{print $1}')" == "$before_keycloak" ]] || fail 'exact-existing Keycloak state was rewritten'
[[ "$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" == "$before_exact_secret" ]] || fail 'exact-existing Secret was rewritten'
! grep -q '^keycloak PUT ' "$fixture/commands.log" || fail 'exact-existing Keycloak state was PUT'
[[ -f "$fixture/evidence-parent/recovery-evidence/keycloak.env" ]] || fail 'exact-existing no-op omitted recovery evidence'
pass 'exact existing live-style state is a no-op except recovery evidence validation'
for profile in duplicate-client duplicate-group duplicate-user mapper-duplicate; do
fixture="$(new_fixture "$profile" "$profile")"; seed_secret "$fixture"
args=(--execute); [[ "$profile" != duplicate-user ]] || args+=(--admin admin-user)
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" "${args[@]}" >"$fixture/out" 2>&1; then fail "$profile ambiguity accepted"; fi
! grep -q 'request=\(POST\|PUT\|DELETE\)' "$fixture/commands.log" || fail "$profile reached mutation"
done
pass 'duplicate client, group, mapper, and user states fail before mutation'
fixture="$(new_fixture mismatch existing)"; seed_secret "$fixture" 'DifferentClientSecret-123456789'
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail 'mismatched existing Secret accepted'; fi
python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail 'mismatch mutated client'
import json,pathlib,sys
x=json.loads(pathlib.Path(sys.argv[1]).read_text()); assert x["clients"][0]["name"]=="Drifted Grafana"
PY
pass 'existing Secret/client mismatch refuses rotation with zero mutation'
for malformed_client_id in trailing-newline trailing-nul; do
fixture="$(new_fixture "client-id-$malformed_client_id" existing)"; seed_secret "$fixture"
python3 -I -S - "$fixture/state/secret.json" "$malformed_client_id" <<'PY'
import base64,json,pathlib,sys
path=pathlib.Path(sys.argv[1]); variant=sys.argv[2]
item=json.loads(path.read_text())
payload={"trailing-newline":b"grafana\n","trailing-nul":b"grafana\0"}[variant]
item["data"]["client-id"]=base64.b64encode(payload).decode()
path.write_text(json.dumps(item,sort_keys=True))
PY
cp -- "$fixture/state/keycloak.json" "$fixture/before.json"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \
run_script "$fixture" --execute >"$fixture/out" 2>&1; then
fail "$malformed_client_id OIDC Secret client-id was accepted"
fi
cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" ||
fail "$malformed_client_id OIDC Secret reached Keycloak mutation"
done
pass 'OIDC Secret client-id must decode to the exact grafana byte sequence'
fixture="$(new_fixture rollback preexisting-admin)"; seed_secret "$fixture"
cp -- "$fixture/state/keycloak.json" "$fixture/before.json"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=http500-viewer-member-put \
run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1; then fail 'membership failure accepted'; fi
cmp --silent "$fixture/before.json" "$fixture/state/keycloak.json" || fail 'rollback did not restore exact Keycloak prestate'
[[ "$(sha256sum "$fixture/state/secret.json" | awk '{print $1}')" == "$before_secret" ]] || fail 'rollback rewrote prior Secret'
grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || fail 'rollback marker absent'
pass 'rollback preserves preexisting membership and removes only transaction-added membership'
for rollback_fault in noop-group-delete noop-mapper-delete; do
fixture="$(new_fixture "$rollback_fault")"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' \
PLATFORM_TEST_FAULT=http500-viewer-member-put \
PLATFORM_TEST_ROLLBACK_FAULT="$rollback_fault" \
run_script "$fixture" --execute --viewer viewer-user >"$fixture/out" 2>&1; then
fail "$rollback_fault trigger was accepted"
fi
[[ -f "$fixture/state/rollback-fault-fired-$rollback_fault" ]] ||
fail "$rollback_fault fixture did not intercept the rollback DELETE"
! grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" ||
fail "$rollback_fault produced a false rollback PASS"
grep -Fq 'GRAFANA_OIDC_ROLLBACK=FAIL' "$fixture/out" ||
fail "$rollback_fault omitted rollback failure evidence"
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/out" ||
fail "$rollback_fault omitted manual recovery evidence"
done
pass 'created Keycloak object rollback requires exact post-delete absence'
for fault in loss-group-put loss-client-create loss-client-put loss-mapper-put loss-secret-create conflict-secret-create timeout-client-create; do
fixture="$(new_fixture "fault-$fault")"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT="$fault" run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail "$fault accepted"; fi
python3 -I -S - "$fixture/state/keycloak.json" <<'PY' || fail "$fault left Keycloak objects"
import json,pathlib,sys
x=json.loads(pathlib.Path(sys.argv[1]).read_text()); assert x["clients"]==[] and x["groups"]==[]
PY
[[ ! -e "$fixture/state/secret.json" ]] || fail "$fault left transaction Secret"
done
pass 'response loss and Kubernetes conflict classify ownership and reverse exact transaction state'
fixture="$(new_fixture update-response-loss existing)"; seed_secret "$fixture"; cp -- "$fixture/state/keycloak.json" "$fixture/before.json"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=loss-client-put \
run_script "$fixture" --execute >"$fixture/out" 2>&1; then fail 'lost client update response was accepted'; fi
cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || fail 'lost update response did not restore exact client prestate'
grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || {
sed -n '1,240p' "$fixture/out" >&2; fail 'lost update response omitted rollback proof'
}
pass 'lost Keycloak update response is classified and exact prior client state is restored'
for sig in HUP INT TERM; do
fixture="$(new_fixture "signal-${sig,,}" existing)"; seed_secret "$fixture"; cp -- "$fixture/state/keycloak.json" "$fixture/before.json"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT="signal-$sig-viewer-member-put" \
run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1; then fail "$sig was accepted"; fi
cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || fail "$sig did not restore exact Keycloak prestate"
grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || fail "$sig did not complete the rollback path"
! grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/out" || fail "$sig exact ownership was misclassified as ambiguous"
done
pass 'HUP, INT, and TERM classify exact ownership and restore the transaction prestate'
fixture="$(new_fixture exit-fallback existing)"; seed_secret "$fixture"; cp -- "$fixture/state/keycloak.json" "$fixture/before.json"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=signal-USR1-viewer-member-put \
run_script "$fixture" --execute --admin admin-user --viewer viewer-user >"$fixture/out" 2>&1; then fail 'unhandled fatal signal was accepted'; fi
cmp --silent -- "$fixture/before.json" "$fixture/state/keycloak.json" || fail 'EXIT fallback did not restore exact Keycloak prestate'
grep -Fq 'GRAFANA_OIDC_ROLLBACK=PASS' "$fixture/out" || fail 'EXIT fallback omitted rollback proof'
pass 'EXIT fallback classifies pending ownership and restores exact prestate'
fixture="$(new_fixture secret-uid-conflict)"
if PLATFORM_TEST_CONFIRMATIONS=$'APPLY default\nRECOVERY KEYCLOAK default' PLATFORM_TEST_FAULT=drift-secret-viewer-member-put \
run_script "$fixture" --execute --viewer viewer-user >"$fixture/out" 2>&1; then fail 'concurrent Secret replacement was accepted'; fi
[[ -f "$fixture/state/secret.json" ]] || fail 'concurrent Secret replacement was deleted'
python3 -I -S - "$fixture/state/secret.json" <<'PY' || fail 'concurrent Secret replacement identity was not preserved'
import json,pathlib,sys
item=json.loads(pathlib.Path(sys.argv[1]).read_text())
assert item["metadata"]["uid"]=="secret-concurrent-replacement" and item["metadata"]["resourceVersion"]=="99"
PY
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/out" || fail 'Secret UID/RV conflict omitted manual recovery marker'
pass 'Secret UID and resourceVersion preconditions preserve a concurrent replacement'
for line in $(grep '^kubectl ' "$fixture/commands.log"); do :; done
grep '^kubectl ' "$fixture/commands.log" | grep -vq -- '--request-timeout=5s' && fail 'an unbounded kubectl invocation was observed'
[[ "$(<"$fixture/out")" != *"$CLIENT_SECRET"* ]] || fail 'fault output leaked payload'
! grep -Fq "$CLIENT_SECRET" "$fixture/commands.log" || fail 'fault argv leaked payload'
pass 'kubectl and curl boundaries are bounded and payloads stay out of output and argv'
printf 'KEYCLOAK GRAFANA OIDC TEST PASS (%s assertions)\n' "$ASSERTIONS"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,56 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly DENY="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf"
readonly FULL="$ROOT/infrastructure/networking/host-nginx/learn-services-observability.conf"
fail() {
printf 'FAIL: %s\n' "$*" >&2
exit 1
}
[[ -f "$DENY" && ! -L "$DENY" ]] || fail 'deny-guard source is missing'
[[ -f "$FULL" && ! -L "$FULL" ]] || fail 'full observability source is missing'
python3 - "$DENY" "$FULL" <<'PY'
import pathlib
import sys
deny_path, full_path = map(pathlib.Path, sys.argv[1:])
deny = deny_path.read_text()
full = full_path.read_text()
old = " proxy_pass http://127.0.0.1:1;"
new = " proxy_pass http://127.0.0.1:30080;"
if deny.count(old) != 1:
raise SystemExit("FAIL: deny guard does not have one closed Grafana upstream")
if deny.count(new) != 4:
raise SystemExit("FAIL: deny guard existing service upstream set changed")
expected = deny.replace(old, new, 1)
if full != expected:
raise SystemExit("FAIL: full candidate differs from deny guard outside the Grafana upstream")
if full.count(old) != 0 or full.count(new) != 5:
raise SystemExit("FAIL: full candidate Grafana upstream is not exact")
required = (
"ssl_reject_handshake on;",
"server_name git.learn.hyeonworks.com;",
"server_name id.learn.hyeonworks.com;",
"server_name storage-admin.learn.hyeonworks.com;",
"server_name db-admin.learn.hyeonworks.com;",
"server_name grafana.learn.hyeonworks.com;",
"location = /metrics {",
"allow 192.168.0.0/24;",
"allow 100.64.0.0/10;",
"deny all;",
)
for token in required:
if token not in full:
raise SystemExit(f"FAIL: full candidate lacks {token}")
if full.count("location = /metrics {") != 2:
raise SystemExit("FAIL: Gitea and Grafana metrics guards are not exact")
PY
printf 'HOST NGINX OBSERVABILITY SOURCE CONTRACT PASS\n'
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
# Regression test for the NTFS mask used by the recovery volume. The 0177
# file mask removes owner execute plus every group/other permission, producing
# mode 0600 for the KeePass database and LUKS container.
set -Eeuo pipefail
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh"
# shellcheck source=/dev/null
source "$VALIDATOR_PATH"
readonly LIVE_SAFE_OPTIONS='rw,nosuid,nodev,noexec,relatime,uid=1000,gid=1000,dmask=0077,fmask=0177,iocharset=utf8'
if ! _k3slrv_outer_masks_secure "$LIVE_SAFE_OPTIONS"; then
printf '%s\n' \
'TEST FAILURE: recovery validator rejected secure fmask=0177' >&2
exit 1
fi
printf '%s\n' 'PASS: recovery validator accepts secure fmask=0177'
@@ -0,0 +1,797 @@
#!/usr/bin/env bash
# Source-safe TDD coverage for the standalone K3s recovery LUKS header proof.
set -Eeuo pipefail
readonly K3SLRH_TEST_DIRECTORY="${BASH_SOURCE[0]%/*}"
readonly K3SLRH_REPOSITORY_ROOT="$(cd -- "${K3SLRH_TEST_DIRECTORY}/../.." && pwd -P)"
readonly K3SLRH_SHARED_LIBRARY="${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery.sh"
readonly K3SLRH_A1_LIBRARY="${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-a1.sh"
readonly K3SLRH_A1_HELPER="${K3SLRH_REPOSITORY_ROOT}/scripts/libexec/k3s-local-recovery-a1-check.sh"
readonly K3SLRH_PROOF_LIBRARY="${K3SLRH_REPOSITORY_ROOT}/scripts/lib/k3s-local-recovery-header-proof.sh"
readonly K3SLRH_WRAPPER="${K3SLRH_REPOSITORY_ROOT}/scripts/bootstrap/prove-k3s-local-recovery-header.sh"
readonly K3SLRH_CONTRACT="${K3SLRH_REPOSITORY_ROOT}/infrastructure/security/k3s/local-recovery.env"
readonly K3SLRH_VALIDATOR="${K3SLRH_REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh"
readonly K3SLRH_AUTHORITY_DESIGN='/home/donghyeon/workspace/docs/platform/plans/2026-08-09-k3s-luks-header-proof-design.md'
readonly K3SLRH_AUTHORITY_PLAN='/home/donghyeon/workspace/docs/platform/plans/2026-08-09-k3s-luks-header-proof-implementation.md'
readonly K3SLRH_AUTHORITY_DESIGN_SHA256='0d1546795da59490956ba2e260bf12df210083514ae55c0551de632d54e278ec'
readonly K3SLRH_AUTHORITY_PLAN_SHA256='126f7fa43fb857dd9ff753aac4ce6528f7a227d7151d688f8cce2b6e0ff8fbc9'
readonly K3SLRH_EXPECTED_SHARED_SHA256='a206d81539e6c37fa085457ba01cff9de57de50f7235e3a4085989780e99585d'
readonly K3SLRH_EXPECTED_A1_SHA256='8b39fc46229ebd4c77ff20040f942c4ef7b756d564d344a8ecdd11a7428e2968'
readonly K3SLRH_EXPECTED_A1_HELPER_SHA256='06831fa35943a328ecd183d3c6f721b24c4607b2f349abf6201a12e690d08ebe'
readonly K3SLRH_EXPECTED_CONTRACT_SHA256='c045f42c2dc3eda09c37a123bcc05d88d94f275e61504a2d5816948237e516df'
readonly K3SLRH_EXPECTED_VALIDATOR_SHA256='6b869032fc325759ee25b219fad4ea6190a3389a561c496f0664a51b802f73e6'
k3slrh_test_fail() {
printf 'TEST FAILURE: %s\n' "$*" >&2
exit 1
}
# This RED gate intentionally precedes authority hashing, source scanning,
# sourcing, mktemp, command doubles, and every capability or live path.
[[ -f "$K3SLRH_PROOF_LIBRARY" && ! -L "$K3SLRH_PROOF_LIBRARY" &&
-r "$K3SLRH_PROOF_LIBRARY" ]] || k3slrh_test_fail 'header-proof library missing'
[[ -f "$K3SLRH_WRAPPER" && ! -L "$K3SLRH_WRAPPER" &&
-r "$K3SLRH_WRAPPER" ]] || k3slrh_test_fail 'header-proof wrapper missing'
[[ "${K3SLRH_FOCUS:-entrypoint}" == entrypoint ]] ||
k3slrh_test_fail 'unsupported header-proof focus'
k3slrh_sha_is_exact() {
local path="${1-}" expected="${2-}" record
(( $# == 2 )) || return 97
record="$(/usr/bin/sha256sum -- "$path")" || return 1
[[ "$record" == "${expected} ${path}" ]]
}
for k3slrh_checkpoint in \
"$K3SLRH_AUTHORITY_DESIGN:$K3SLRH_AUTHORITY_DESIGN_SHA256" \
"$K3SLRH_AUTHORITY_PLAN:$K3SLRH_AUTHORITY_PLAN_SHA256" \
"$K3SLRH_SHARED_LIBRARY:$K3SLRH_EXPECTED_SHARED_SHA256" \
"$K3SLRH_A1_LIBRARY:$K3SLRH_EXPECTED_A1_SHA256" \
"$K3SLRH_A1_HELPER:$K3SLRH_EXPECTED_A1_HELPER_SHA256" \
"$K3SLRH_CONTRACT:$K3SLRH_EXPECTED_CONTRACT_SHA256" \
"$K3SLRH_VALIDATOR:$K3SLRH_EXPECTED_VALIDATOR_SHA256"; do
k3slrh_checkpoint_path="${k3slrh_checkpoint%:*}"
k3slrh_checkpoint_hash="${k3slrh_checkpoint##*:}"
k3slrh_sha_is_exact "$k3slrh_checkpoint_path" "$k3slrh_checkpoint_hash" ||
k3slrh_test_fail 'authority or frozen dependency drift'
done
unset k3slrh_checkpoint k3slrh_checkpoint_path k3slrh_checkpoint_hash
k3slrh_library_static_is_safe() {
local source_path="${1-}"
(( $# == 1 )) || return 97
[[ -f "$source_path" && ! -L "$source_path" ]] || return 1
/usr/bin/awk '
function trim(value) {
sub(/^[[:space:]]+/, "", value)
sub(/[[:space:]]+$/, "", value)
return value
}
function external_path(value) {
return value ~ /\/(usr\/)?(s?bin)\/[A-Za-z0-9._+:-]+/
}
function sensitive_bare(value) {
return value ~ /(^|(&&|\|\||;)[[:space:]]*)(sudo|mount|umount|losetup|cryptsetup|keepassxc-cli|sqlite3|systemctl|timeout|k3s|findmnt|lsblk|blockdev|dmsetup)([[:space:]]|$)/
}
{
raw = $0
value = trim(raw)
if (value == "" || value ~ /^#/) next
if (value ~ /<Task-[0-9]+-[^>]+>/) bad = 1
if (value ~ /_k3slr_collect_prepackage_snapshot|_k3slr_lifecycle_dispatch|_k3slr_prove_header_restore/) bad = 1
if (in_function) {
if (raw == "}") { in_function = 0; function_name = ""; next }
if (value ~ /`/ || value ~ /(^|[;[:space:]])(eval|command|exec)[[:space:]]/) bad = 1
if (sensitive_bare(value) && value !~ /_k3slrh_command[[:space:]]/) bad = 1
if (external_path(value) && value !~ /_k3slrh_command[[:space:]]/ &&
value !~ /_k3slr[a-z0-9_]+[[:space:]]/) bad = 1
next
}
if (value ~ /^[A-Za-z_][A-Za-z0-9_]*\(\)[[:space:]]*\{$/) {
in_function = 1
function_name = value
sub(/\(\)[[:space:]]*\{$/, "", function_name)
next
}
if (value ~ /^readonly[[:space:]]+[A-Za-z_][A-Za-z0-9_]*=(\047[^\047]*\047|[A-Za-z0-9_.:+\/-]+)$/) next
bad = 1
}
END { if (in_function || bad) exit 1 }
' "$source_path"
}
k3slrh_wrapper_static_is_safe() {
local source_path="${1-}"
(( $# == 1 )) || return 97
[[ -f "$source_path" && ! -L "$source_path" ]] || return 1
/usr/bin/awk '
{
if ($0 ~ /_k3slr_collect_prepackage_snapshot|_k3slr_lifecycle_dispatch|_k3slr_prove_header_restore/) bad = 1
if ($0 ~ /(^|[[:space:]])(sudo|mount|umount|losetup|cryptsetup|keepassxc-cli|sqlite3)([[:space:]]|$)/) bad = 1
if ($0 ~ /scripts\/lib\/k3s-local-recovery\.sh/) shared = NR
if ($0 ~ /scripts\/lib\/k3s-local-recovery-a1\.sh/) a1 = NR
if ($0 ~ /scripts\/lib\/k3s-local-recovery-header-proof\.sh/) proof = NR
if ($0 ~ /^if \[\[ "\$\{BASH_SOURCE\[0\]\}" == "\$0" \]\]; then$/) direct_guard++
if ($0 ~ /_k3slrh_header_proof_main "\$@"/) direct_main++
if ($0 ~ /_k3slrh_wrapper_initial_guard/) initial_guard++
if ($0 ~ /unset -f[[:space:]]+_k3slrh_/) bad = 1
if ($0 ~ /_k3slrh_wrapper_environment_is_clean/) inherited_guard = NR
}
END {
if (bad || shared == 0 || a1 == 0 || proof == 0 ||
!(shared < a1 && a1 < proof) || direct_guard != 1 ||
direct_main != 1 || initial_guard < 2 || inherited_guard == 0 ||
inherited_guard >= shared) exit 1
}
' "$source_path"
}
# No production source is evaluated until both conservative scans pass.
k3slrh_library_static_is_safe "$K3SLRH_PROOF_LIBRARY" ||
k3slrh_test_fail 'header-proof library static preflight failed'
k3slrh_wrapper_static_is_safe "$K3SLRH_WRAPPER" ||
k3slrh_test_fail 'header-proof wrapper static preflight failed'
# shellcheck source=/dev/null
source "$K3SLRH_SHARED_LIBRARY" || k3slrh_test_fail 'shared library source failed'
# shellcheck source=/dev/null
source "$K3SLRH_A1_LIBRARY" || k3slrh_test_fail 'A1 library source failed'
# shellcheck source=/dev/null
source "$K3SLRH_PROOF_LIBRARY" || k3slrh_test_fail 'header-proof library source failed'
_k3slr_load_contract "$K3SLRH_CONTRACT" || k3slrh_test_fail 'strict contract load failed'
k3slrh_require_function() {
local name="${1-}"
(( $# == 1 )) || return 97
declare -F "$name" >/dev/null || k3slrh_test_fail "header-proof API missing: ${name}"
}
for k3slrh_api in \
_k3slrh_parse_cli \
_k3slrh_derive_targets \
_k3slrh_run_closed_validator \
_k3slrh_k3s_health \
_k3slrh_pre_mount_capability \
_k3slrh_header_proof_main \
_k3slrh_target_state_is_consistent; do
k3slrh_require_function "$k3slrh_api"
done
unset k3slrh_api
k3slrh_assert_readonly_literal() {
local name="${1-}" expected="${2-}" declaration
(( $# == 2 )) || return 97
declaration="$(declare -p "$name" 2>/dev/null)" || return 1
[[ "$declaration" == 'declare -r '* || "$declaration" == 'declare -rx '* ]] || return 1
[[ "${!name}" == "$expected" ]]
}
k3slrh_assert_readonly_literal K3SLRH_PROOF_RELATIVE \
'HyeonworksRecovery/containers/k3s-recovery.header-proof.luks' ||
k3slrh_test_fail 'proof relative name is not immutable'
k3slrh_assert_readonly_literal K3SLRH_PRE_KDBX_PREFIX \
'hyeonworks-recovery.header-proof-pre.' ||
k3slrh_test_fail 'pre KDBX prefix is not immutable'
k3slrh_assert_readonly_literal K3SLRH_POST_KDBX_PREFIX \
'hyeonworks-recovery.header-proof-post.' ||
k3slrh_test_fail 'post KDBX prefix is not immutable'
k3slrh_assert_readonly_literal K3SLRH_HEADER_BACKUP_NAME 'header-original.bin' ||
k3slrh_test_fail 'header backup name is not immutable'
k3slrh_assert_readonly_literal K3SLRH_HEADER_EXPORT_NAME 'header-exported.bin' ||
k3slrh_test_fail 'header export name is not immutable'
k3slrh_assert_readonly_literal K3SLRH_SQLITE_PACKAGE_VERSION '3.45.1-1ubuntu2.7' ||
k3slrh_test_fail 'sqlite package version is not immutable'
k3slrh_assert_readonly_literal K3SLRH_SHARED_LIBRARY_SHA256 "$K3SLRH_EXPECTED_SHARED_SHA256" ||
k3slrh_test_fail 'shared source checkpoint placeholder or drift'
k3slrh_assert_readonly_literal K3SLRH_A1_LIBRARY_SHA256 "$K3SLRH_EXPECTED_A1_SHA256" ||
k3slrh_test_fail 'A1 source checkpoint placeholder or drift'
k3slrh_assert_readonly_literal K3SLRH_A1_HELPER_SHA256 "$K3SLRH_EXPECTED_A1_HELPER_SHA256" ||
k3slrh_test_fail 'A1 helper checkpoint placeholder or drift'
k3slrh_assert_readonly_literal K3SLRH_CONTRACT_SHA256 "$K3SLRH_EXPECTED_CONTRACT_SHA256" ||
k3slrh_test_fail 'contract checkpoint placeholder or drift'
k3slrh_assert_readonly_literal K3SLRH_VALIDATOR_SHA256 "$K3SLRH_EXPECTED_VALIDATOR_SHA256" ||
k3slrh_test_fail 'validator checkpoint placeholder or drift'
readonly K3SLRH_EXPECTED_UTC='20260809T120304Z'
readonly K3SLRH_RAW_IDENTIFIER_SENTINEL='RAW_IDENTIFIER_SENTINEL_DO_NOT_PRINT'
readonly K3SLRH_EXPECTED_CLOSED_RECORD=$'Recovery device: match\nRecovery state: closed\nLineage: match\nLatest bundle: not_checked\n'
readonly K3SLRH_EXPECTED_DRY_RUN=$'Header proof state: closed\nHeader proof capability: pass\nExecute-time checks: pending\nDRY RUN PASS: no changes\n'
k3slrh_fixture_root="$(/usr/bin/mktemp -d /tmp/k3slrh-entrypoint-test.XXXXXXXX)" ||
k3slrh_test_fail 'cannot allocate fixture root'
readonly k3slrh_fixture_root
readonly k3slrh_stdout="${k3slrh_fixture_root}/stdout"
readonly k3slrh_stderr="${k3slrh_fixture_root}/stderr"
readonly k3slrh_command_log="${k3slrh_fixture_root}/commands"
readonly k3slrh_mutation_log="${k3slrh_fixture_root}/mutations"
readonly k3slrh_validator_call_log="${k3slrh_fixture_root}/validator-calls"
readonly k3slrh_validator_hash_log="${k3slrh_fixture_root}/validator-hashes"
readonly k3slrh_sudo_validate_log="${k3slrh_fixture_root}/sudo-validates"
readonly k3slrh_readyz_log="${k3slrh_fixture_root}/readyz-calls"
readonly k3slrh_validator_file_log="${k3slrh_fixture_root}/validator-file-checks"
readonly k3slrh_validator_symlink_log="${k3slrh_fixture_root}/validator-symlink-checks"
readonly k3slrh_validator_stat_log="${k3slrh_fixture_root}/validator-stat-checks"
readonly k3slrh_binary_stat_log="${k3slrh_fixture_root}/binary-stat-checks"
k3slrh_cleanup() {
case "$k3slrh_fixture_root" in
/tmp/k3slrh-entrypoint-test.[A-Za-z0-9]*) /usr/bin/rm -rf -- "$k3slrh_fixture_root" ;;
*) return 1 ;;
esac
}
trap k3slrh_cleanup EXIT
k3slrh_assert_eq() {
local expected="${1-}" actual="${2-}" label="${3-}"
(( $# == 3 )) || k3slrh_test_fail 'invalid assert_eq invocation'
[[ "$actual" == "$expected" ]] || k3slrh_test_fail "$label"
}
k3slrh_assert_rc() {
local expected="${1-}" label="${2-}" rc=0
shift 2 || k3slrh_test_fail 'invalid assert_rc invocation'
if "$@"; then rc=0; else rc=$?; fi
[[ "$rc" == "$expected" ]] || k3slrh_test_fail "${label}: rc=${rc}, expected=${expected}"
}
k3slrh_assert_succeeds() {
local label="${1-}"
shift || k3slrh_test_fail 'invalid assert_succeeds invocation'
"$@" || k3slrh_test_fail "$label"
}
k3slrh_wrapper_clean_composition() {
local rc=0
: >"$k3slrh_stdout"
: >"$k3slrh_stderr"
if /usr/bin/env -i \
PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C \
/usr/bin/bash --noprofile --norc -c '
set -Eeuo pipefail
source "$1"
declare -F _k3slrh_header_proof_main >/dev/null
declare -F _k3slrh_wrapper_initial_guard >/dev/null
if _k3slrh_wrapper_initial_guard 0 hB; then exit 41; fi
if _k3slrh_wrapper_initial_guard 1000 hxB; then exit 42; fi
' k3slrh-wrapper-probe "$K3SLRH_WRAPPER" \
>"$k3slrh_stdout" 2>"$k3slrh_stderr"; then
rc=0
else
rc=$?
fi
[[ "$rc" == 0 && ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]]
}
k3slrh_wrapper_rejects_inherited_route() {
local rc=0 marker="${k3slrh_fixture_root}/inherited-route-marker"
: >"$k3slrh_stdout"
: >"$k3slrh_stderr"
if /usr/bin/env -i \
PATH=/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C \
/usr/bin/bash --noprofile --norc -c '
set -Eeuo pipefail
K3SLRH_PROOF_RELATIVE=attacker-controlled
test_marker=$2
_k3slrh_header_proof_main() { : >"$test_marker"; }
set +e
source "$1" >/dev/null 2>&1
wrapper_rc=$?
set -e
[[ "$wrapper_rc" == 1 && ! -e "$test_marker" ]]
' k3slrh-wrapper-probe "$K3SLRH_WRAPPER" "$marker" \
>"$k3slrh_stdout" 2>"$k3slrh_stderr"; then
rc=0
else
rc=$?
fi
[[ "$rc" == 0 && ! -e "$marker" && ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]]
}
k3slrh_run_parse() {
local parsed='' rc=0
if _k3slrh_parse_cli parsed "$@"; then rc=0; else rc=$?; fi
(( rc == 0 )) || return "$rc"
printf '%s' "$parsed"
}
k3slrh_run_timestamp() (
_k3slrh_derive_targets "$@"
)
k3slrh_targets_are_exact() (
_k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" || return 1
[[ "${K3SLRH_RUN_UTC:-}" == "$K3SLRH_EXPECTED_UTC" &&
"${K3SLRH_PRE_KDBX_RELATIVE:-}" == \
"HyeonworksRecovery/vault/backups/${K3SLRH_PRE_KDBX_PREFIX}${K3SLRH_EXPECTED_UTC}.kdbx" &&
"${K3SLRH_POST_KDBX_RELATIVE:-}" == \
"HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}${K3SLRH_EXPECTED_UTC}.kdbx" ]] || return 1
_k3slrh_target_state_is_consistent \
"$K3SLRH_RUN_UTC" "$K3SLRH_PRE_KDBX_RELATIVE" "$K3SLRH_POST_KDBX_RELATIVE"
)
k3slrh_run_targets_same_second_collision() (
_k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" || return 1
_k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC"
)
k3slrh_run_targets_pre_post_timestamp_drift() (
local drifted_post
_k3slrh_derive_targets "$K3SLRH_EXPECTED_UTC" || return 1
drifted_post="HyeonworksRecovery/vault/backups/${K3SLRH_POST_KDBX_PREFIX}20260809T120305Z.kdbx"
_k3slrh_target_state_is_consistent \
"$K3SLRH_RUN_UTC" "$K3SLRH_PRE_KDBX_RELATIVE" "$drifted_post"
)
k3slrh_model_case=valid
k3slrh_validator_count=0
k3slrh_mutation_count=0
k3slrh_log_command() {
local argument
for argument in "$@"; do printf '%s\0' "$argument" >>"$k3slrh_command_log"; done
printf '\0' >>"$k3slrh_command_log"
}
k3slrh_path_is_known_binary() {
local candidate
for candidate in \
/usr/bin/awk /usr/bin/bash /usr/bin/chmod /usr/bin/chown /usr/bin/cmp \
/usr/bin/date /usr/bin/dd /usr/bin/dpkg-query /usr/bin/env /usr/bin/find \
/usr/bin/findmnt /usr/bin/grep /usr/bin/keepassxc-cli /usr/bin/lsblk \
/usr/bin/mkdir /usr/bin/mktemp /usr/bin/mount /usr/bin/ntfsinfo \
/usr/bin/od /usr/bin/readlink /usr/bin/rm /usr/bin/rmdir /usr/bin/sha256sum \
/usr/bin/sqlite3 /usr/bin/stat /usr/bin/sudo /usr/bin/sync /usr/bin/systemctl \
/usr/bin/test /usr/bin/timeout /usr/bin/umount /usr/bin/unlink /usr/bin/wc \
/usr/local/bin/k3s /usr/sbin/blockdev /usr/sbin/cryptsetup /usr/sbin/dmsetup \
/usr/sbin/losetup; do
[[ "${1-}" != "$candidate" ]] || return 0
done
return 1
}
_k3slrh_command() {
local package='' path='' expected_backing='' hash_count=0
k3slrh_log_command "$@"
case "${1-}" in
/usr/bin/sudo)
if (( $# == 3 )) && [[ "$2" == --non-interactive && "$3" == --validate ]]; then
printf x >>"$k3slrh_sudo_validate_log"
if [[ "$k3slrh_model_case" == uncached_sudo ]]; then
printf 'SUDO_PROMPT_SENTINEL\n' >&2
return 1
fi
return 0
fi
if (( $# == 15 )) && [[ "$2" == --non-interactive && "$3" == -- &&
"$4" == /usr/bin/env && "$5" == -i &&
"$6" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' && "$7" == LC_ALL=C &&
"$8" == /usr/bin/timeout && "$9" == --signal=TERM &&
"${10}" == --kill-after=1s && "${11}" == 30s &&
"${12}" == /usr/local/bin/k3s && "${13}" == kubectl &&
"${14}" == get && "${15}" == --raw=/readyz ]]; then
printf x >>"$k3slrh_readyz_log"
case "$k3slrh_model_case" in
readyz_extra) printf 'ok\nextra\n' ;;
readyz_missing_lf) printf 'ok' ;;
readyz_cr) printf 'ok\r\n' ;;
readyz_nul) printf 'ok\0\n' ;;
readyz_empty) : ;;
readyz_rc1) return 1 ;;
readyz_rc124) return 124 ;;
readyz_rc137) return 137 ;;
readyz_stderr) printf 'READYZ_STDERR_SENTINEL\n' >&2; printf 'ok\n' ;;
*) printf 'ok\n' ;;
esac
return 0
fi
printf 'mutation:sudo\n' >>"$k3slrh_mutation_log"
return 97
;;
/usr/bin/systemctl)
[[ "$#" == 4 && "$2" == is-active && "$3" == --quiet && "$4" == k3s ]] || return 97
[[ "$k3slrh_model_case" != k3s_inactive ]] || return 3
;;
/usr/bin/timeout)
# Direct non-root readyz is forbidden by the authority contract.
return 97
;;
/usr/bin/env)
[[ "$#" == 10 && "$2" == -i && "$3" == 'PATH=/usr/sbin:/usr/bin:/sbin:/bin' &&
"$4" == LC_ALL=C && "$5" == /usr/bin/bash && "$6" == --noprofile &&
"$7" == --norc && "$8" == -- && "$9" == "$K3SLRH_VALIDATOR" &&
"${10}" == --expect-closed ]] || return 97
k3slrh_validator_count=$((k3slrh_validator_count + 1))
printf x >>"$k3slrh_validator_call_log"
case "$k3slrh_model_case" in
validator_cr) printf '%s\r\n' 'Recovery device: match' 'Recovery state: closed' 'Lineage: match' 'Latest bundle: not_checked' ;;
validator_nul) printf 'Recovery device: match\0Recovery state: closed\nLineage: match\nLatest bundle: not_checked\n' ;;
validator_missing_lf) printf '%s' "${K3SLRH_EXPECTED_CLOSED_RECORD%$'\n'}" ;;
validator_extra) printf '%sextra\n' "$K3SLRH_EXPECTED_CLOSED_RECORD" ;;
validator_raw_id) printf '%s%s\n' "$K3SLRH_EXPECTED_CLOSED_RECORD" "$K3SLRH_RAW_IDENTIFIER_SENTINEL" ;;
validator_rc1) return 1 ;;
validator_rc124) return 124 ;;
validator_rc137) return 137 ;;
validator_stderr) printf 'x' >&2; printf '%s' "$K3SLRH_EXPECTED_CLOSED_RECORD" ;;
*) printf '%s' "$K3SLRH_EXPECTED_CLOSED_RECORD" ;;
esac
;;
/usr/bin/dpkg-query)
[[ "$#" == 4 && "$2" == --show &&
"$3" == '--showformat=${Status}|${Version}\n' ]] || return 97
package="${!#}"
case "$package" in
keepassxc) path='2.7.6+dfsg.1-1build3'; [[ "$k3slrh_model_case" != keepass_package_drift ]] || path='0.invalid' ;;
cryptsetup-bin) path='2:2.7.0-1ubuntu4.2'; [[ "$k3slrh_model_case" != cryptsetup_package_drift ]] || path='0.invalid' ;;
sqlite3) path='3.45.1-1ubuntu2.7'; [[ "$k3slrh_model_case" != sqlite_package_drift ]] || path='0.invalid' ;;
*) return 97 ;;
esac
printf 'install ok installed|%s\n' "$path"
;;
/usr/bin/date)
[[ "$#" == 3 && "$2" == --utc && "$3" == +%Y%m%dT%H%M%SZ ]] || return 97
printf '%s\n' "$K3SLRH_EXPECTED_UTC"
;;
/usr/bin/findmnt)
[[ "$#" == 6 && "$2" == -rn && "$3" == -M && "$5" == -o && "$6" == TARGET ]] || return 97
case "$4|$k3slrh_model_case" in
"$K3SLR_OUTER_MOUNT|outer_mount_collision"|"$K3SLR_INNER_MOUNT|inner_mount_collision")
printf '%s\n' "$4"
return 0
;;
"$K3SLR_OUTER_MOUNT|"*|"$K3SLR_INNER_MOUNT|"*) return 1 ;;
*) return 97 ;;
esac
;;
/usr/sbin/losetup)
[[ "$#" == 5 && "$2" == --list && "$3" == --noheadings &&
"$4" == --output && "$5" == NAME,BACK-FILE ]] || {
printf 'mutation:losetup\n' >>"$k3slrh_mutation_log"
return 97
}
case "$k3slrh_model_case" in
main_loop_residue)
expected_backing="${K3SLR_OUTER_MOUNT}/${K3SLR_CONTAINER_RELATIVE}"
printf '/dev/loop91 %s\n' "$expected_backing"
;;
proof_loop_residue)
expected_backing="${K3SLR_OUTER_MOUNT}/${K3SLRH_PROOF_RELATIVE}"
printf '/dev/loop92 %s\n' "$expected_backing"
;;
esac
;;
/usr/sbin/dmsetup)
return 97
;;
/usr/bin/find)
if (( $# == 8 )) && [[ "$2" == "$K3SLR_OUTER_MOUNT" &&
"$3" == -mindepth && "$4" == 1 && "$5" == -maxdepth &&
"$6" == 1 && "$7" == -print && "$8" == -quit ]]; then
[[ "$k3slrh_model_case" != outer_mountpoint_not_empty ]] ||
printf '%s/unexpected\n' "$K3SLR_OUTER_MOUNT"
return 0
fi
if (( $# == 10 )) && [[ "$2" == /run && "$3" == -mindepth &&
"$4" == 1 && "$5" == -maxdepth && "$6" == 1 && "$7" == -name &&
"$8" == 'hyeonworks-k3s-recovery.*' && "$9" == -print &&
"${10}" == -quit ]]; then
[[ "$k3slrh_model_case" != tmpfs_residue ]] ||
printf '/run/hyeonworks-k3s-recovery.ABCDEFGH\n'
return 0
fi
return 97
;;
/usr/bin/test)
if (( $# == 3 )) && [[ "$2" == ! && "$3" == -L ]]; then return 97; fi
if (( $# == 4 )) && [[ "$2" == ! && "$3" == -e ]]; then
case "$4" in
"/dev/mapper/${K3SLR_MAPPING_NAME}")
[[ "$k3slrh_model_case" != main_mapping_residue ]]
return
;;
"/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}")
[[ "$k3slrh_model_case" != proof_mapping_residue ]]
return
;;
*) return 97 ;;
esac
fi
if (( $# == 3 )); then
case "$2" in
-d) [[ "$3" == "$K3SLR_OUTER_MOUNT" ]] || return 97 ;;
-f|-x|-r)
case "$3" in
"$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT"|"$K3SLRH_VALIDATOR") ;;
*) k3slrh_path_is_known_binary "$3" || return 97 ;;
esac
if [[ "$2" == -f && "$3" == "$K3SLRH_VALIDATOR" ]]; then
printf x >>"$k3slrh_validator_file_log"
fi
;;
*) return 97 ;;
esac
return 0
fi
if (( $# == 4 )) && [[ "$2" == ! && "$3" == -L ]]; then
case "$4" in
"/dev/mapper/${K3SLR_MAPPING_NAME}")
[[ "$k3slrh_model_case" != main_mapping_dangling ]]
return
;;
"/dev/mapper/${K3SLR_PROOF_MAPPING_NAME}")
[[ "$k3slrh_model_case" != proof_mapping_dangling ]]
return
;;
"$K3SLRH_VALIDATOR") printf x >>"$k3slrh_validator_symlink_log"; return 0 ;;
"$K3SLR_OUTER_MOUNT"|"$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT") return 0 ;;
*) k3slrh_path_is_known_binary "$4" || return 97; return 0 ;;
esac
fi
return 97
;;
/usr/bin/stat)
path="${!#}"
if (( $# == 4 )) && [[ "$2" == "--format=%u|%a|%F" && "$3" == -- ]]; then
k3slrh_path_is_known_binary "$path" || return 97
printf '%s\n' "$path" >>"$k3slrh_binary_stat_log"
if [[ "$k3slrh_model_case" == binary_untrusted && "$path" == /usr/bin/keepassxc-cli ]]; then
printf '0|775|regular file\n'
else
printf '0|755|regular file\n'
fi
return 0
fi
if (( $# == 4 )) && [[ "$2" == "--format=%F|%u|%g|%a" && "$3" == -- ]]; then
case "$path" in
"$K3SLR_OUTER_MOUNT") printf 'directory|0|0|755\n' ;;
"$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT"|"$K3SLRH_VALIDATOR")
[[ "$path" != "$K3SLRH_VALIDATOR" ]] || printf x >>"$k3slrh_validator_stat_log"
printf 'regular file|1000|1000|664\n'
;;
*) return 97 ;;
esac
return 0
fi
return 97
;;
/usr/bin/readlink)
[[ "$#" == 4 && "$2" == -e && "$3" == -- ]] || return 97
path="$4"
case "$path" in
"$K3SLR_OUTER_MOUNT"|"$K3SLRH_SHARED_LIBRARY"|"$K3SLRH_A1_LIBRARY"|"$K3SLRH_A1_HELPER"|"$K3SLRH_CONTRACT"|"$K3SLRH_VALIDATOR") printf '%s\n' "$path" ;;
*) return 97 ;;
esac
;;
/usr/bin/sha256sum)
[[ "$#" == 3 && "$2" == -- ]] || return 97
path="${!#}"
case "$path" in
"$K3SLRH_SHARED_LIBRARY") printf '%s %s\n' "$K3SLRH_EXPECTED_SHARED_SHA256" "$path" ;;
"$K3SLRH_A1_LIBRARY") printf '%s %s\n' "$K3SLRH_EXPECTED_A1_SHA256" "$path" ;;
"$K3SLRH_A1_HELPER") printf '%s %s\n' "$K3SLRH_EXPECTED_A1_HELPER_SHA256" "$path" ;;
"$K3SLRH_CONTRACT") printf '%s %s\n' "$K3SLRH_EXPECTED_CONTRACT_SHA256" "$path" ;;
"$K3SLRH_VALIDATOR")
printf x >>"$k3slrh_validator_hash_log"
hash_count="$(/usr/bin/wc -c <"$k3slrh_validator_hash_log")" || return 97
if [[ "$k3slrh_model_case" == validator_hash_pre_drift ||
( "$k3slrh_model_case" == validator_hash_post_drift && "$hash_count" -ge 2 ) ]]; then
printf '%064d %s\n' 0 "$path"
else
printf '%s %s\n' "$K3SLRH_EXPECTED_VALIDATOR_SHA256" "$path"
fi
;;
*) return 97 ;;
esac
;;
/usr/bin/mount|/usr/bin/umount|/usr/sbin/cryptsetup)
k3slrh_mutation_count=$((k3slrh_mutation_count + 1))
printf 'mutation\n' >>"$k3slrh_mutation_log"
return 97
;;
*) return 97 ;;
esac
}
_k3slr_command() { _k3slrh_command "$@"; }
_k3slr_collect_prepackage_snapshot() { printf 'legacy-collector\n' >>"$k3slrh_mutation_log"; return 97; }
_k3slr_prove_header_restore() { printf 'legacy-marker\n' >>"$k3slrh_mutation_log"; return 97; }
_k3slr_lifecycle_dispatch() { printf 'prepare-dispatch\n' >>"$k3slrh_mutation_log"; return 97; }
k3slrh_reset_model_logs() {
: >"$k3slrh_command_log"
: >"$k3slrh_mutation_log"
: >"$k3slrh_validator_call_log"
: >"$k3slrh_validator_hash_log"
: >"$k3slrh_sudo_validate_log"
: >"$k3slrh_readyz_log"
: >"$k3slrh_validator_file_log"
: >"$k3slrh_validator_symlink_log"
: >"$k3slrh_validator_stat_log"
: >"$k3slrh_binary_stat_log"
}
k3slrh_capture_call() {
: >"$k3slrh_stdout"
: >"$k3slrh_stderr"
"$@" >"$k3slrh_stdout" 2>"$k3slrh_stderr"
}
k3slrh_validator_case() {
local case_name="${1-}" expected_rc="${2-}" expected_calls="${3-}"
local expected_hashes="${4-}" rc=0 expected_physical=2
(( $# == 4 )) || return 97
k3slrh_reset_model_logs
k3slrh_model_case="$case_name"
[[ "$case_name" != validator_hash_pre_drift ]] || expected_physical=1
if k3slrh_capture_call _k3slrh_run_closed_validator; then rc=0; else rc=$?; fi
[[ "$rc" == "$expected_rc" &&
"$(/usr/bin/wc -c <"$k3slrh_validator_call_log")" == "$expected_calls" &&
"$(/usr/bin/wc -c <"$k3slrh_validator_hash_log")" == "$expected_hashes" &&
"$(/usr/bin/wc -c <"$k3slrh_validator_file_log")" == "$expected_physical" &&
"$(/usr/bin/wc -c <"$k3slrh_validator_symlink_log")" == "$expected_physical" &&
"$(/usr/bin/wc -c <"$k3slrh_validator_stat_log")" == "$expected_physical" ]] || return 1
[[ ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] || return 1
! /usr/bin/grep -aFq -- "$K3SLRH_RAW_IDENTIFIER_SENTINEL" "$k3slrh_stdout" "$k3slrh_stderr"
}
k3slrh_health_case() {
local case_name="${1-}" expected_rc="${2-}" expected_readyz="${3-}" rc=0
(( $# == 3 )) || return 97
k3slrh_reset_model_logs
k3slrh_model_case="$case_name"
if k3slrh_capture_call _k3slrh_k3s_health; then rc=0; else rc=$?; fi
[[ "$rc" == "$expected_rc" && ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" &&
"$(/usr/bin/wc -c <"$k3slrh_readyz_log")" == "$expected_readyz" ]]
}
k3slrh_capability_case() {
local case_name="${1-}" expected_rc="${2-}" rc=0
k3slrh_reset_model_logs
k3slrh_model_case="$case_name"
if k3slrh_capture_call _k3slrh_pre_mount_capability; then rc=0; else rc=$?; fi
[[ "$rc" == "$expected_rc" ]] || return 1
if [[ "$expected_rc" == 0 ]]; then
[[ "$(<"$k3slrh_stdout")" == 'Header proof capability: pass' && ! -s "$k3slrh_stderr" ]] || return 1
else
[[ ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] || return 1
fi
[[ ! -s "$k3slrh_mutation_log" ]] || return 1
if [[ "$case_name" == uncached_sudo ]]; then
[[ "$(/usr/bin/wc -c <"$k3slrh_sudo_validate_log")" == 1 &&
! -s "$k3slrh_readyz_log" && ! -s "$k3slrh_validator_call_log" &&
! -s "$k3slrh_validator_hash_log" ]]
fi
}
k3slrh_command_model_is_closed() {
local rc=0 k3slrh_bad_command
local -a k3slrh_bad_argv=()
k3slrh_reset_model_logs
for k3slrh_bad_command in \
'/usr/bin/timeout --signal=TERM 30s /usr/local/bin/k3s kubectl get --raw=/readyz' \
'/usr/bin/findmnt --target /fixture' \
'/usr/sbin/losetup --find --show /fixture' \
'/usr/bin/stat --format=%F -- /dev/loop0' \
'/usr/bin/test -e /dev/mapper/k3s-recovery'; do
read -r -a k3slrh_bad_argv <<<"$k3slrh_bad_command"
if _k3slrh_command "${k3slrh_bad_argv[@]}" >/dev/null 2>&1; then rc=0; else rc=$?; fi
[[ "$rc" == 97 ]] || return 1
done
[[ -s "$k3slrh_mutation_log" ]]
}
k3slrh_invalid_main() {
local rc=0
: >"$k3slrh_stdout"
: >"$k3slrh_stderr"
if _k3slrh_header_proof_main "$@" >"$k3slrh_stdout" 2>"$k3slrh_stderr"; then
rc=0
else
rc=$?
fi
! /usr/bin/grep -aFq -- "$K3SLRH_RAW_IDENTIFIER_SENTINEL" \
"$k3slrh_stdout" "$k3slrh_stderr" || return 97
return "$rc"
}
k3slrh_execute_without_tty() (
local rc=0
_k3slrh_tty_capability() { return 1; }
_k3slr_require_execute_tty() { return 1; }
: >"$k3slrh_stdout"
: >"$k3slrh_stderr"
if _k3slrh_header_proof_main --execute >"$k3slrh_stdout" 2>"$k3slrh_stderr"; then
rc=0
else
rc=$?
fi
[[ ! -s "$k3slrh_stdout" && ! -s "$k3slrh_stderr" ]] || return 97
return "$rc"
)
k3slrh_dry_run_closed() {
local rc=0 validator_calls validator_hashes sudo_validates readyz_calls
local validator_files validator_symlinks validator_stats binary_count k3slrh_required_binary
k3slrh_model_case=valid
k3slrh_validator_count=0
k3slrh_mutation_count=0
k3slrh_reset_model_logs
if k3slrh_capture_call _k3slrh_header_proof_main; then rc=0; else rc=$?; fi
validator_calls="$(/usr/bin/wc -c <"$k3slrh_validator_call_log")" || return 1
validator_hashes="$(/usr/bin/wc -c <"$k3slrh_validator_hash_log")" || return 1
sudo_validates="$(/usr/bin/wc -c <"$k3slrh_sudo_validate_log")" || return 1
readyz_calls="$(/usr/bin/wc -c <"$k3slrh_readyz_log")" || return 1
validator_files="$(/usr/bin/wc -c <"$k3slrh_validator_file_log")" || return 1
validator_symlinks="$(/usr/bin/wc -c <"$k3slrh_validator_symlink_log")" || return 1
validator_stats="$(/usr/bin/wc -c <"$k3slrh_validator_stat_log")" || return 1
for k3slrh_required_binary in /usr/bin/keepassxc-cli /usr/sbin/cryptsetup /usr/bin/sqlite3; do
binary_count="$(/usr/bin/grep -Fxc -- "$k3slrh_required_binary" "$k3slrh_binary_stat_log")" || return 1
[[ "$binary_count" == 1 ]] || return 1
done
[[ "$rc" == 0 && "$(<"$k3slrh_stdout")"$'\n' == "$K3SLRH_EXPECTED_DRY_RUN" &&
! -s "$k3slrh_stderr" && ! -s "$k3slrh_mutation_log" &&
"$validator_calls" == 1 && "$validator_hashes" == 2 &&
"$validator_files" == 2 && "$validator_symlinks" == 2 && "$validator_stats" == 2 &&
"$sudo_validates" == 1 && "$readyz_calls" == 1 &&
"$k3slrh_mutation_count" == 0 ]]
}
k3slrh_assert_eq dry-run "$(k3slrh_run_parse)" 'no argument must select dry-run'
k3slrh_assert_eq execute "$(k3slrh_run_parse --execute)" 'one execute must be accepted'
k3slrh_assert_rc 1 'duplicate parser execute must fail safely' k3slrh_run_parse --execute --execute
k3slrh_assert_rc 1 'caller parser path must fail safely' k3slrh_run_parse --container /fixture
k3slrh_assert_rc 2 'duplicate main execute must be usage rc2' k3slrh_invalid_main --execute --execute
k3slrh_assert_rc 2 'caller main path must be usage rc2' k3slrh_invalid_main --container /fixture
k3slrh_assert_succeeds 'actual clean wrapper composition and root/xtrace guards must pass' \
k3slrh_wrapper_clean_composition
k3slrh_assert_succeeds 'actual wrapper must reject inherited route and constants' \
k3slrh_wrapper_rejects_inherited_route
k3slrh_assert_rc 1 'execute without TTY must be safety rc1' k3slrh_execute_without_tty
k3slrh_assert_rc 1 'malformed timestamp must be safety rc1' k3slrh_run_timestamp malformed
k3slrh_assert_succeeds 'valid target names must derive exactly once' k3slrh_targets_are_exact
k3slrh_assert_rc 1 'same-second target reservation must be safety rc1' \
k3slrh_run_targets_same_second_collision
k3slrh_assert_rc 1 'pre/post timestamp drift must be safety rc1' \
k3slrh_run_targets_pre_post_timestamp_drift
k3slrh_assert_succeeds 'closed external command model must reject unknown argv' \
k3slrh_command_model_is_closed
for k3slrh_bad_health in k3s_inactive readyz_extra readyz_missing_lf readyz_cr \
readyz_nul readyz_empty readyz_rc1 readyz_rc124 readyz_rc137 readyz_stderr; do
if [[ "$k3slrh_bad_health" == k3s_inactive ]]; then k3slrh_expected_readyz=0; else k3slrh_expected_readyz=1; fi
k3slrh_assert_succeeds "K3s health must reject ${k3slrh_bad_health}" \
k3slrh_health_case "$k3slrh_bad_health" 1 "$k3slrh_expected_readyz"
done
unset k3slrh_bad_health k3slrh_expected_readyz
k3slrh_assert_succeeds 'exact cached-sudo readyz must pass silently' \
k3slrh_health_case valid 0 1
for k3slrh_bad_validator in validator_cr validator_nul validator_missing_lf \
validator_extra validator_raw_id validator_rc1 validator_rc124 validator_rc137 \
validator_stderr; do
k3slrh_assert_succeeds "closed validator must reject ${k3slrh_bad_validator}" \
k3slrh_validator_case "$k3slrh_bad_validator" 1 1 2
done
unset k3slrh_bad_validator
k3slrh_assert_succeeds 'validator pre-call hash drift must stop before dispatch' \
k3slrh_validator_case validator_hash_pre_drift 1 0 1
k3slrh_assert_succeeds 'validator post-call hash drift must fail closed' \
k3slrh_validator_case validator_hash_post_drift 1 1 2
k3slrh_assert_succeeds 'exact pinned closed validator record must pass' \
k3slrh_validator_case valid 0 1 2
for k3slrh_bad_capability in uncached_sudo k3s_inactive readyz_extra \
keepass_package_drift cryptsetup_package_drift sqlite_package_drift \
binary_untrusted \
outer_mount_collision inner_mount_collision outer_mountpoint_not_empty tmpfs_residue \
main_mapping_residue proof_mapping_residue main_mapping_dangling proof_mapping_dangling \
main_loop_residue proof_loop_residue; do
k3slrh_assert_succeeds "capability must reject ${k3slrh_bad_capability}" \
k3slrh_capability_case "$k3slrh_bad_capability" 1
done
unset k3slrh_bad_capability
k3slrh_assert_succeeds 'closed dry-run must pass without mutation' k3slrh_dry_run_closed
printf 'PASS: header-proof entrypoint and closed capability fixtures\n'
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Regression test for whole-disk lsblk queries. Without --nodeps, lsblk emits
# the disk and all child partitions, while the validator requires one record.
set -Eeuo pipefail
readonly REPOSITORY_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly VALIDATOR_PATH="${REPOSITORY_ROOT}/scripts/validate/k3s-local-recovery.sh"
# shellcheck source=/dev/null
source "$VALIDATOR_PATH"
_k3slrv_exec() {
local scope="${1-}"
shift || return 1
[[ "$scope" == user ]] || return 1
if (( $# == 8 )) &&
[[ "$1" == /usr/bin/lsblk && "$2" == --noheadings &&
"$3" == --nodeps && "$4" == --paths && "$5" == --output &&
"$6" == MODEL && "$7" == -- && "$8" == /dev/fixture-disk ]]; then
printf '%s\n' FIXTURE_DISK_MODEL
else
# This mirrors lsblk against a whole disk when dependency traversal is
# not disabled: the disk row is followed by a child-partition row.
printf '%s\n%s\n' FIXTURE_DISK_MODEL FIXTURE_CHILD_ROW
fi
}
actual=''
if ! actual="$(_k3slrv_lsblk MODEL /dev/fixture-disk)"; then
printf '%s\n' \
'TEST FAILURE: whole-disk lsblk query did not produce one record' >&2
exit 1
fi
if [[ "$actual" != FIXTURE_DISK_MODEL ]]; then
printf '%s\n' \
'TEST FAILURE: whole-disk lsblk query returned an unexpected record' >&2
exit 1
fi
printf '%s\n' 'PASS: recovery validator excludes lsblk child rows'
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,705 @@
#!/usr/bin/env bash
# k3s Secret 복구 증거의 parser, 세 mode와 권한 경계를 검증한다.
# 이 파일은 source-safe 함수만 호출하며 live host나 Kubernetes API를 건드리지 않는다.
set -Eeuo pipefail
readonly TEST_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly BOOTSTRAP_PATH="${TEST_ROOT}/scripts/bootstrap/apply-k3s-secret-encryption.sh"
readonly RESTORE_VALIDATOR_PATH="${TEST_ROOT}/scripts/validate/k3s-secret-encryption-restore-evidence.sh"
fail() {
printf 'RESTORE EVIDENCE TEST FAILURE: %s\n' "$*" >&2
exit 1
}
assert_eq() {
local expected="$1" actual="$2" description="$3"
[[ "$actual" == "$expected" ]] || fail "$description"
}
# 잘못된 legacy field를 쓰는 producer 변경을 잡는다. 기대값은 Task 5 계약에서
# 손으로 적었으며 production parser나 formatter로 만들지 않는다.
probe_canonical_bundle_metadata_producer() (
# shellcheck source=/dev/null
source "$BOOTSTRAP_PATH"
declare -F _kseb_render_bundle_metadata >/dev/null || return 1
local actual
actual="$(_kseb_render_bundle_metadata \
'01234567-89ab-4cde-8fab-0123456789ab' post embedded-etcd \
'2026-08-01T00:00:00Z' 7 'v1.36.2+k3s1')" || return 1
assert_eq $'schema=platform-k3s-bundle-v1\nbundle_id=01234567-89ab-4cde-8fab-0123456789ab\nphase=post\nk3s_version=v1.36.2+k3s1\ndatastore=embedded-etcd\ncreated_at_utc=2026-08-01T00:00:00Z\nsecret_count=7' \
"$actual" 'canonical bundle metadata fields/order'
)
probe_canonical_bundle_metadata_producer || fail 'canonical bundle metadata producer'
# 아래 production 변경이 깨질 때 잡는 결함은 allowlist parser 우회, mode 간
# 검증 누락, sudo 전 입력 검증 누락과 기존 evidence overwrite다.
[[ -f "$RESTORE_VALIDATOR_PATH" ]] || fail 'restore evidence validator is absent'
# shellcheck source=/dev/null
source "$RESTORE_VALIDATOR_PATH"
fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-restore-evidence-test.XXXXXX")"
cleanup() {
case "$fixture_root" in
/tmp/k3s-restore-evidence-test.*|"${TMPDIR:-/tmp}"/k3s-restore-evidence-test.*)
rm -rf -- "$fixture_root"
;;
*) fail 'unexpected fixture root' ;;
esac
}
trap cleanup EXIT
readonly FIXTURE_BUNDLE_ID='01234567-89ab-4cde-8fab-0123456789ab'
readonly FIXTURE_OTHER_ID='11234567-89ab-4cde-8fab-0123456789ab'
readonly FIXTURE_LOCAL_NODE_UID='aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'
readonly FIXTURE_OTHER_NODE_UID='11111111-2222-4333-8444-555555555555'
readonly FIXTURE_TESTED_AT='2026-08-01T00:00:00Z'
readonly FIXTURE_RECORDED_AT='2026-08-01T01:00:00Z'
# Fix round 1 / finding 4: invalid compact timestamp는 production transaction의
# directory/backup/marker mutation 전에 거부해야 한다.
probe_invalid_timestamp_precedes_transaction_mutation() (
local parent="${fixture_root}/invalid-stamp" root data fake_k3s command_log rc=0
parent="$(mktemp -d "${fixture_root}/invalid-stamp.XXXXXX")"
root="$parent/recovery"; data="$parent/data"; fake_k3s="$parent/k3s"
command_log="$parent/commands.log"
mkdir -p "$root" "$data/server/cred"
chmod 0700 "$root"
printf 'fixture\n' >"$data/server/token"
printf '{}\n' >"$data/server/cred/encryption-config.json"
: >"$command_log"
printf '%s\n' '#!/usr/bin/env bash' \
'printf "called\n" >>"${KSEB_INVALID_STAMP_LOG:?}"' \
'exit 1' >"$fake_k3s"
chmod 0700 "$fake_k3s"
# shellcheck source=/dev/null
source "$BOOTSTRAP_PATH"
KSEB_BACKUP_ROOT_IDENTITY="$(stat --format='%d:%i' -- "$root")"
_kseb_run_privileged_shell() { /usr/bin/bash "$@"; }
_kseb_transaction_data_dir() { printf '%s\n' "$data"; }
_kseb_transaction_config_dir() { printf '%s\n' "$parent/no-config"; }
_kseb_transaction_systemd_paths() { :; }
_kseb_transaction_k3s_binary() { printf '%s\n' "$fake_k3s"; }
_kseb_transaction_post_marker() { printf '%s\n' "$parent/post-marker"; }
_kseb_transaction_restore_evidence() { printf '%s\n' "$parent/restore-evidence"; }
export KSEB_INVALID_STAMP_LOG="$command_log"
_kseb_pinned_phase_transaction "$root" bundle/pre embedded-etcd \
"$FIXTURE_BUNDLE_ID" pre 20261301T000000Z 7 v1.36.2+k3s1 \
>/dev/null 2>&1 || rc=$?
[[ "$rc" -ne 0 && ! -e "$root/bundle" && ! -s "$command_log" ]]
)
write_metadata() {
local path="$1" phase="${2:-post}" bundle_id="${3:-$FIXTURE_BUNDLE_ID}"
local version="${4:-v1.36.2+k3s1}" datastore="${5:-embedded-etcd}"
local created="${6:-2026-07-31T23:00:00Z}" count="${7:-7}"
printf 'schema=platform-k3s-bundle-v1\nbundle_id=%s\nphase=%s\nk3s_version=%s\ndatastore=%s\ncreated_at_utc=%s\nsecret_count=%s\n' \
"$bundle_id" "$phase" "$version" "$datastore" "$created" "$count" >"$path"
chmod 0600 "$path"
}
write_result() {
local path="$1" bundle_id="${2:-$FIXTURE_BUNDLE_ID}" version="${3:-v1.36.2+k3s1}"
local datastore="${4:-embedded-etcd}" node="${5:-pass}" isolation="${6:-pass}"
local integrity="${7:-match}" count="${8:-pass}" tested="${9:-$FIXTURE_TESTED_AT}"
printf 'schema=platform-k3s-restore-result-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nstatus_class=enabled_stable\nrotation_stage=reencrypt_finished\nhashes=match\nlocal_integrity=%s\napi_ready=pass\nnode_ready=%s\nisolation=%s\nsecret_count_match=%s\ntested_at_utc=%s\n' \
"$bundle_id" "$version" "$datastore" "$integrity" "$node" "$isolation" "$count" "$tested" >"$path"
chmod 0600 "$path"
}
write_evidence() {
local path="$1" bundle_id="${2:-$FIXTURE_BUNDLE_ID}" version="${3:-v1.36.2+k3s1}"
local datastore="${4:-embedded-etcd}" integrity="${5:-match}" destroyed="${6:-confirmed}"
local tested="${7:-$FIXTURE_TESTED_AT}" recorded="${8:-$FIXTURE_RECORDED_AT}"
printf 'schema=platform-k3s-restore-evidence-v1\nbundle_id=%s\nk3s_version=%s\ndatastore=%s\nrotation_stage=reencrypt_finished\nlocal_integrity=%s\nnode_ready=pass\nisolation=pass\ntested_at_utc=%s\nrecorded_at_utc=%s\ndestroyed=%s\n' \
"$bundle_id" "$version" "$datastore" "$integrity" "$tested" "$recorded" "$destroyed" >"$path"
chmod 0600 "$path"
}
assert_fails() {
if "$@" >/dev/null 2>&1; then fail "expected failure: $*"; fi
}
assert_succeeds() {
"$@" || fail "expected success: $*"
}
assert_succeeds probe_invalid_timestamp_precedes_transaction_mutation
metadata="${fixture_root}/bundle.env"
result="${fixture_root}/result.env"
evidence_fixture="${fixture_root}/evidence.env"
write_metadata "$metadata"
write_result "$result"
write_evidence "$evidence_fixture"
probe_parse() (
local kind="$1" path="$2"
declare -A parsed=()
_ksre_parse_external_file "$kind" "$path" parsed
)
assert_succeeds probe_parse bundle "$metadata"
assert_succeeds probe_parse result "$result"
assert_succeeds probe_parse evidence "$evidence_fixture"
# Fix round 1 / finding 1: local restore validator와 current-user API가
# 서로 다른 cluster를 보면 production identity gate가 실패해야 한다.
matching_nodes='{"items":[{"metadata":{"name":"isolated-ready","uid":"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
mismatched_nodes='{"items":[{"metadata":{"name":"other-ready","uid":"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
multiple_ready_nodes='{"items":[{"metadata":{"name":"isolated-ready","uid":"aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"other-ready","uid":"11111111-2222-4333-8444-555555555555"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
assert_succeeds _ksre_require_same_ready_node isolated-ready "$FIXTURE_LOCAL_NODE_UID" "$matching_nodes"
assert_fails _ksre_require_same_ready_node isolated-ready "$FIXTURE_LOCAL_NODE_UID" "$mismatched_nodes"
assert_fails _ksre_require_same_ready_node isolated-ready "$FIXTURE_LOCAL_NODE_UID" "$multiple_ready_nodes"
# Fix round 2 / finding 1: 같은 UID가 복제된 다른 API도 허용하지 않도록
# current-user context의 effective endpoint는 exact loopback이며 proxy가 없어야 한다.
assert_succeeds _ksre_validate_current_connection 'https://127.0.0.1:6443|'
assert_fails _ksre_validate_current_connection 'https://192.0.2.10:6443|'
assert_fails _ksre_validate_current_connection 'https://localhost:6443|'
assert_fails _ksre_validate_current_connection 'https://127.0.0.1:6443/|'
assert_fails _ksre_validate_current_connection 'https://127.0.0.1:6443|http://127.0.0.1:8080'
probe_live_inventory_cluster_binding() (
local api_node="$1" api_node_uid="$2" output
declare -F _ksre_current_connection >/dev/null || return 1
declare -F _ksre_require_same_connection >/dev/null || return 1
declare -F _ksre_run_restore_status_validator >/dev/null || return 1
declare -F _ksre_current_user_api_ready >/dev/null || return 1
declare -F _ksre_current_user_nodes_json >/dev/null || return 1
declare -F _ksre_current_user_secret_count >/dev/null || return 1
_ksre_run_restore_status_validator() {
local handoff="$1"
printf 'version=v1.36.2+k3s1\nserver_count=1\nnode_name=isolated-ready\nnode_uid=%s\nnode_ready=Ready\ndatastore=embedded-etcd\nencryption=Enabled\nrotation=reencrypt_finished\nserver_hashes=match\nintegrity=match\napi_readyz=pass\n' \
"$FIXTURE_LOCAL_NODE_UID" \
>"$handoff/inventory.env"
chmod 0600 "$handoff/inventory.env"
}
_ksre_current_connection() {
[[ "${1:-}" == fixture-context ]] || return 1
printf 'https://127.0.0.1:6443|\n'
}
_ksre_current_user_api_ready() {
[[ "${1:-}" == fixture-context && "${2:-}" == https://127.0.0.1:6443 ]]
}
_ksre_current_user_nodes_json() {
[[ "${1:-}" == fixture-context && "${2:-}" == https://127.0.0.1:6443 ]] || return 1
printf '{"items":[{"metadata":{"name":"%s","uid":"%s"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}\n' \
"$api_node" "$api_node_uid"
}
_ksre_current_user_secret_count() {
[[ "${1:-}" == fixture-context && "${2:-}" == https://127.0.0.1:6443 ]] || return 1
printf '7\n'
}
output="$(_ksre_live_restore_inventory fixture-context 'https://127.0.0.1:6443|')" || return 1
[[ "$output" == 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|pass|7' ]]
)
assert_succeeds probe_live_inventory_cluster_binding isolated-ready "$FIXTURE_LOCAL_NODE_UID"
assert_fails probe_live_inventory_cluster_binding isolated-ready "$FIXTURE_OTHER_NODE_UID"
assert_fails probe_live_inventory_cluster_binding other-ready "$FIXTURE_LOCAL_NODE_UID"
# Fix round 1 / finding 2: Bash read가 버리는 NUL도 raw byte 단계에서 거부한다.
nul_metadata="${fixture_root}/metadata-nul.env"
{
printf 'schema=platform-k3s-bundle-v1\n'
printf 'bundle_id=%s\n' "$FIXTURE_BUNDLE_ID"
printf 'phase=po\0st\n'
printf 'k3s_version=v1.36.2+k3s1\n'
printf 'datastore=embedded-etcd\n'
printf 'created_at_utc=2026-07-31T23:00:00Z\n'
printf 'secret_count=7\n'
} >"$nul_metadata"
chmod 0600 "$nul_metadata"
assert_fails probe_parse bundle "$nul_metadata"
# Fix round 1 / finding 3: pathname이 아니라 실제로 열린 FD identity를 검사한다.
probe_pinned_external_fd_survives_path_swap() (
local input="${fixture_root}/toctou-valid.env" moved="${fixture_root}/toctou-valid-opened.env"
local input_fd
write_metadata "$input"
exec {input_fd}<"$input"
/bin/mv -- "$input" "$moved"
printf 'unsafe replacement\n' >"$input"
chmod 0644 "$input"
declare -A parsed=()
_ksre_parse_external_fd bundle "$input_fd" parsed || return 1
exec {input_fd}<&-
[[ "${parsed[bundle_id]}" == "$FIXTURE_BUNDLE_ID" ]]
)
probe_unsafe_opened_fd_cannot_borrow_safe_path() (
local input="${fixture_root}/toctou-unsafe.env" moved="${fixture_root}/toctou-unsafe-opened.env"
local input_fd
write_metadata "$input"
chmod 0644 "$input"
exec {input_fd}<"$input"
/bin/mv -- "$input" "$moved"
write_metadata "$input"
declare -A parsed=()
if _ksre_parse_external_fd bundle "$input_fd" parsed; then return 1; fi
exec {input_fd}<&-
)
assert_succeeds probe_pinned_external_fd_survives_path_swap
assert_succeeds probe_unsafe_opened_fd_cannot_borrow_safe_path
# Fix round 2 / finding 2: production final-component open의 정확한 순간을
# test-only interposer로 멈춘다. precheck 뒤 symlink로 바꾸고 open 뒤 같은
# target inode의 hard link로 되돌려도 parser가 반드시 실패해야 한다.
race_interposer_source="${fixture_root}/nofollow-race.c"
race_interposer="${fixture_root}/nofollow-race.so"
cat >"$race_interposer_source" <<'RACE_INTERPOSER'
#define _GNU_SOURCE
#include <dlfcn.h>
#include <fcntl.h>
#include <limits.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
static int triggered;
static int (*real_open_fn)(const char *, int, ...);
static int (*real_open64_fn)(const char *, int, ...);
static int (*real_openat_fn)(int, const char *, int, ...);
static int (*real_openat64_fn)(int, const char *, int, ...);
static void resolve_symbols(void) {
if (!real_open_fn) real_open_fn = dlsym(RTLD_NEXT, "open");
if (!real_open64_fn) real_open64_fn = dlsym(RTLD_NEXT, "open64");
if (!real_openat_fn) real_openat_fn = dlsym(RTLD_NEXT, "openat");
if (!real_openat64_fn) real_openat64_fn = dlsym(RTLD_NEXT, "openat64");
}
static void marker_path(char *buffer, size_t size, const char *name) {
const char *directory = getenv("KSRE_RACE_SYNC");
if (!directory || snprintf(buffer, size, "%s/%s", directory, name) >= (int)size) _exit(125);
}
static void touch_marker(const char *name) {
char path[PATH_MAX];
int fd;
marker_path(path, sizeof(path), name);
resolve_symbols();
fd = real_open_fn(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0) _exit(125);
dprintf(fd, "%ld\n", (long)getpid());
close(fd);
}
static void wait_for_marker(const char *name) {
char path[PATH_MAX];
marker_path(path, sizeof(path), name);
while (access(path, F_OK) != 0) usleep(1000);
}
static int should_race(const char *path) {
const char *target = getenv("KSRE_RACE_PATH");
if (!triggered && target && path && strcmp(path, target) == 0) {
triggered = 1;
touch_marker("entered-open");
wait_for_marker("continue-open");
return 1;
}
return 0;
}
static void after_open(void) {
touch_marker("opened");
wait_for_marker("continue-postcheck");
}
int open(const char *path, int flags, ...) {
mode_t mode = 0;
int raced, result;
va_list arguments;
resolve_symbols();
if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) {
va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments);
}
raced = should_race(path);
result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE))
? real_open_fn(path, flags, mode) : real_open_fn(path, flags);
if (raced) after_open();
return result;
}
int open64(const char *path, int flags, ...) {
mode_t mode = 0;
int raced, result;
va_list arguments;
resolve_symbols();
if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) {
va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments);
}
raced = should_race(path);
if (!real_open64_fn) real_open64_fn = real_open_fn;
result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE))
? real_open64_fn(path, flags, mode) : real_open64_fn(path, flags);
if (raced) after_open();
return result;
}
int openat(int directory_fd, const char *path, int flags, ...) {
mode_t mode = 0;
int raced, result;
va_list arguments;
resolve_symbols();
if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) {
va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments);
}
raced = should_race(path);
result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE))
? real_openat_fn(directory_fd, path, flags, mode) : real_openat_fn(directory_fd, path, flags);
if (raced) after_open();
return result;
}
int openat64(int directory_fd, const char *path, int flags, ...) {
mode_t mode = 0;
int raced, result;
va_list arguments;
resolve_symbols();
if ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) {
va_start(arguments, flags); mode = va_arg(arguments, mode_t); va_end(arguments);
}
raced = should_race(path);
if (!real_openat64_fn) real_openat64_fn = real_openat_fn;
result = ((flags & O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE))
? real_openat64_fn(directory_fd, path, flags, mode) : real_openat64_fn(directory_fd, path, flags);
if (raced) after_open();
return result;
}
RACE_INTERPOSER
/usr/bin/cc -shared -fPIC -O2 -o "$race_interposer" "$race_interposer_source" -ldl ||
fail 'compile no-follow race interposer'
wait_for_race_marker() {
local marker="$1" process_id="$2" attempt
for ((attempt = 0; attempt < 5000; attempt++)); do
[[ -e "$marker" ]] && return 0
/bin/kill -0 "$process_id" 2>/dev/null || return 1
/usr/bin/sleep 0.001
done
return 1
}
probe_open_instant_symlink_replacement_rejected() (
local target="${fixture_root}/race-target.env" input="${fixture_root}/race-input.env"
local original="${fixture_root}/race-input-original.env" sync="${fixture_root}/race-sync"
local process_id parse_rc=0
write_metadata "$target"
write_metadata "$input"
mkdir "$sync"
/usr/bin/env KSRE_RACE_PATH="$input" KSRE_RACE_SYNC="$sync" LD_PRELOAD="$race_interposer" \
/usr/bin/timeout --signal=TERM --kill-after=1s 10s \
/usr/bin/bash -c '
set -Eeuo pipefail
source "$1"
declare -A parsed=()
_ksre_parse_external_file bundle "$2" parsed
' bash "$RESTORE_VALIDATOR_PATH" "$input" &
process_id=$!
if ! wait_for_race_marker "$sync/entered-open" "$process_id"; then
: >"$sync/continue-open"; : >"$sync/continue-postcheck"
wait "$process_id" 2>/dev/null || :
return 1
fi
/bin/mv -- "$input" "$original"
/bin/ln -s -- "$target" "$input"
: >"$sync/continue-open"
if ! wait_for_race_marker "$sync/opened" "$process_id"; then
: >"$sync/continue-postcheck"
wait "$process_id" 2>/dev/null || :
return 1
fi
/bin/rm -- "$input"
/bin/ln -- "$target" "$input"
: >"$sync/continue-postcheck"
if wait "$process_id"; then parse_rc=0; else parse_rc=$?; fi
[[ "$parse_rc" -ne 0 ]]
)
assert_succeeds probe_open_instant_symlink_replacement_rejected
# Fix round 3: production broker가 readiness 전에 멈춰도 parent 자체 상한 안에
# nonzero로 끝나고 partial success output과 살아 있는 broker를 남기면 안 된다.
probe_broker_readiness_delay_is_bounded() (
local input="${fixture_root}/broker-delay.env" sync="${fixture_root}/broker-delay-sync"
local partial_output="${fixture_root}/broker-delay.out" broker_process='' parser_rc=0 attempt
write_metadata "$input"
mkdir "$sync"
/usr/bin/env KSRE_RACE_PATH="$input" KSRE_RACE_SYNC="$sync" LD_PRELOAD="$race_interposer" \
/usr/bin/timeout --signal=TERM --kill-after=0.5s 4s \
/usr/bin/bash -c '
set -Eeuo pipefail
source "$1"
declare -A parsed=()
if _ksre_parse_external_file bundle "$2" parsed; then
printf "unexpected-success\n"
exit 0
fi
exit 23
' bash "$RESTORE_VALIDATOR_PATH" "$input" >"$partial_output" 2>/dev/null || parser_rc=$?
[[ -s "$sync/entered-open" ]] || return 1
broker_process="$(<"$sync/entered-open")"
[[ "$broker_process" =~ ^[0-9]+$ ]] || return 1
: >"$sync/continue-open"
: >"$sync/continue-postcheck"
for ((attempt = 0; attempt < 1000; attempt++)); do
/bin/kill -0 "$broker_process" 2>/dev/null || break
/usr/bin/sleep 0.001
done
if [[ "$parser_rc" == 124 || "$parser_rc" == 137 ]]; then
printf 'broker readiness required the outer watchdog\n' >&2
return 1
fi
[[ "$parser_rc" -ne 0 && ! -s "$partial_output" ]]
! /bin/kill -0 "$broker_process" 2>/dev/null
)
assert_succeeds probe_broker_readiness_delay_is_bounded
# Fix round 2 pre-review concern: writer가 없는 FIFO는 regular-file 검사 전에
# block하지 않고 즉시 거부되어야 한다. timeout은 실패가 아니라 RED 증거다.
probe_fifo_rejected_without_blocking() (
local fifo="${fixture_root}/external-input.fifo" parser_rc=0
/usr/bin/mkfifo -- "$fifo"
/usr/bin/chmod 0600 "$fifo"
/usr/bin/timeout --signal=TERM --kill-after=0.25s 0.25s \
/usr/bin/bash -c '
set -Eeuo pipefail
source "$1"
declare -A parsed=()
_ksre_parse_external_file bundle "$2" parsed
' bash "$RESTORE_VALIDATOR_PATH" "$fifo" >/dev/null 2>&1 || parser_rc=$?
if [[ "$parser_rc" == 124 || "$parser_rc" == 137 ]]; then
/usr/bin/timeout --signal=TERM --kill-after=0.25s 1s \
/usr/bin/bash -c 'printf x >"$1"' bash "$fifo" >/dev/null 2>&1 || :
printf 'FIFO parser blocked before regular-file rejection\n' >&2
return 1
fi
[[ "$parser_rc" -ne 0 ]]
)
assert_succeeds probe_fifo_rejected_without_blocking
secret_stream="${fixture_root}/secret-stream.json"
secret_trace="${fixture_root}/secret-stream.trace"
printf '{"items":[{"data":{"value":"fixture-sensitive-payload"}}]}\n' >"$secret_stream"
probe_stream_secret_count() (
exec 9>"$secret_trace"
BASH_XTRACEFD=9
set -x
local count
count="$(_ksre_secret_count_from_stream <"$secret_stream")"
set +x
[[ "$count" == 1 ]]
)
assert_succeeds probe_stream_secret_count
[[ "$(<"$secret_trace")" != *fixture-sensitive-payload* ]] || fail 'Secret payload reached xtrace'
# Parser/security matrix. 각 fixture는 parser의 실제 branch를 지나며 privileged
# helper가 한 번이라도 호출되면 별도 marker를 남기도록 mode fixture에서 확인한다.
for attack in command_substitution backtick duplicate unknown empty control malformed trailing; do
candidate="${fixture_root}/metadata-${attack}.env"
write_metadata "$candidate"
case "$attack" in
command_substitution) /usr/bin/sed -i '$c\secret_count=$(id)' "$candidate" ;;
backtick) /usr/bin/sed -i '$c\secret_count=`id`' "$candidate" ;;
duplicate) printf 'phase=post\n' >>"$candidate" ;;
unknown) printf 'credential=forbidden\n' >>"$candidate" ;;
empty) /usr/bin/sed -i 's/^phase=.*/phase=/' "$candidate" ;;
control) /usr/bin/sed -i 's/^phase=.*/phase=post\r/' "$candidate" ;;
malformed) printf 'not-an-assignment\n' >>"$candidate" ;;
trailing) printf 'trailing-data\n' >>"$candidate" ;;
esac
assert_fails probe_parse bundle "$candidate"
done
empty_file="${fixture_root}/empty.env"; : >"$empty_file"; chmod 0600 "$empty_file"
assert_fails probe_parse bundle "$empty_file"
mode_file="${fixture_root}/mode.env"; write_metadata "$mode_file"; chmod 0644 "$mode_file"
assert_fails probe_parse bundle "$mode_file"
symlink_file="${fixture_root}/link.env"; ln -s -- "$metadata" "$symlink_file"
assert_fails probe_parse bundle "$symlink_file"
probe_mode_rejects_before_privilege() (
local path="$1" marker="${fixture_root}/unexpected-privilege"
_ksre_current_context() { printf 'default\n'; }
_ksre_read_authoritative_bundle() { : >"$marker"; return 1; }
_ksre_install_evidence() { : >"$marker"; return 1; }
_ksre_record "$path" "$result" <<<"DESTROYED default" >/dev/null 2>&1 && return 1
[[ ! -e "$marker" ]]
)
for attack in command_substitution backtick duplicate unknown control malformed trailing; do
assert_succeeds probe_mode_rejects_before_privilege "${fixture_root}/metadata-${attack}.env"
done
assert_succeeds probe_mode_rejects_before_privilege "$mode_file"
assert_succeeds probe_mode_rejects_before_privilege "$symlink_file"
mode_events="${fixture_root}/mode-events.log"
installed_evidence="${fixture_root}/installed-evidence.env"
install_mode_fakes() {
: >"$mode_events"
rm -f -- "$installed_evidence"
active_evidence_file="$evidence_fixture"
_ksre_current_context() { printf 'context\n' >>"$mode_events"; printf 'default\n'; }
_ksre_current_connection() { printf 'connection\n' >>"$mode_events"; printf 'https://127.0.0.1:6443|\n'; }
_ksre_require_same_context() { printf 'context-recheck\n' >>"$mode_events"; }
_ksre_now_rfc3339() { printf '%s\n' "$FIXTURE_RECORDED_AT"; }
_ksre_now_epoch() { printf '1785546000\n'; }
_ksre_live_restore_inventory() {
printf 'live\n' >>"$mode_events"
printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|pass|7\n'
}
_ksre_isolation_attested() { printf 'isolation\n' >>"$mode_events"; }
_ksre_read_authoritative_bundle() {
local destination="$1"
printf 'authority\n' >>"$mode_events"
_ksre_parse_trusted_content bundle "$(<"$metadata")" "$destination"
}
_ksre_read_installed_evidence() {
local destination="$1"
printf 'evidence\n' >>"$mode_events"
_ksre_parse_trusted_content evidence "$(<"$active_evidence_file")" "$destination"
}
_ksre_evidence_target_absent() { printf 'target-check\n' >>"$mode_events"; }
_ksre_prompt_destroyed() { printf 'destroyed\n' >>"$mode_events"; }
_ksre_install_evidence() {
local content="$1"
printf 'install\n' >>"$mode_events"
printf '%s\n' "$content" >"$installed_evidence"
chmod 0600 "$installed_evidence"
}
}
probe_emit_success() (
install_mode_fakes
local output="${fixture_root}/emitted.env"
rm -f -- "$output"
_ksre_emit_result "$metadata" "$output" >/dev/null
declare -A parsed=()
_ksre_parse_external_file result "$output" parsed
[[ "${parsed[bundle_id]}" == "$FIXTURE_BUNDLE_ID" &&
"${parsed[secret_count_match]}" == pass && -z "${parsed[destroyed]+present}" &&
"$(<"$mode_events")" == $'context\nconnection\nlive\nisolation\nconnection\ncontext-recheck' ]]
)
assert_succeeds probe_emit_success
probe_emit_relative_output() (
install_mode_fakes
cd -- "$fixture_root"
rm -f -- relative-result.env
_ksre_emit_result "$metadata" relative-result.env >/dev/null
[[ -f relative-result.env ]]
)
assert_succeeds probe_emit_relative_output
probe_emit_existing_output() (
install_mode_fakes
local output="${fixture_root}/existing-output.env"
printf 'keep\n' >"$output"; chmod 0600 "$output"
_ksre_emit_result "$metadata" "$output" >/dev/null 2>&1 && return 1
[[ "$(<"$output")" == keep ]]
)
assert_succeeds probe_emit_existing_output
probe_emit_live_failure() (
local field="$1"
install_mode_fakes
local output="${fixture_root}/emit-failure-${field}.env"
_ksre_live_restore_inventory() {
case "$field" in
count) printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|pass|6\n' ;;
node) printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|match|pass|fail|7\n' ;;
integrity) printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|mismatch|pass|pass|7\n' ;;
esac
}
if [[ "$field" == isolation ]]; then _ksre_isolation_attested() { return 1; }; fi
_ksre_emit_result "$metadata" "$output" >/dev/null 2>&1 && return 1
[[ ! -e "$output" ]]
)
for field in count node isolation integrity; do assert_succeeds probe_emit_live_failure "$field"; done
probe_record_success() (
install_mode_fakes
_ksre_record "$metadata" "$result" <<<"DESTROYED default" >/dev/null
declare -A parsed=()
_ksre_parse_external_file evidence "$installed_evidence" parsed
[[ "${parsed[destroyed]}" == confirmed && "$(head -n 1 "$mode_events")" == context ]]
)
assert_succeeds probe_record_success
probe_record_failure() (
local scenario="$1" local_metadata="${fixture_root}/record-${scenario}-metadata.env"
local local_result="${fixture_root}/record-${scenario}-result.env"
install_mode_fakes
write_metadata "$local_metadata"; write_result "$local_result"
case "$scenario" in
metadata_id) write_metadata "$local_metadata" post "$FIXTURE_OTHER_ID" ;;
result_id) write_result "$local_result" "$FIXTURE_OTHER_ID" ;;
result_version) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.1+k3s1 ;;
result_backend) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 sqlite ;;
pre) write_metadata "$local_metadata" pre ;;
stale) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass pass match pass 2026-07-30T00:00:00Z ;;
secret_count) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass pass match fail ;;
node) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd fail ;;
isolation) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass fail ;;
integrity) write_result "$local_result" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd pass pass mismatch ;;
authority_id)
_ksre_read_authoritative_bundle() {
local destination="$1" alternate="${fixture_root}/alternate-authority.env"
write_metadata "$alternate" post "$FIXTURE_OTHER_ID"
_ksre_parse_trusted_content bundle "$(<"$alternate")" "$destination"
}
;;
destroyed) _ksre_prompt_destroyed() { return 1; } ;;
target_exists) _ksre_evidence_target_absent() { return 1; } ;;
esac
_ksre_record "$local_metadata" "$local_result" <<<"DESTROYED default" >/dev/null 2>&1 && return 1
[[ ! -e "$installed_evidence" ]]
)
for scenario in metadata_id result_id result_version result_backend pre stale secret_count node isolation integrity authority_id destroyed target_exists; do
assert_succeeds probe_record_failure "$scenario"
done
probe_check_success() (
install_mode_fakes
_ksre_check >/dev/null
[[ "$(head -n 1 "$mode_events")" == context ]]
)
assert_succeeds probe_check_success
probe_check_failure() (
local scenario="$1" altered="${fixture_root}/check-${scenario}.env"
install_mode_fakes
case "$scenario" in
authority_id)
_ksre_read_authoritative_bundle() {
local destination="$1" alternate="${fixture_root}/check-authority.env"
write_metadata "$alternate" post "$FIXTURE_OTHER_ID"
_ksre_parse_trusted_content bundle "$(<"$alternate")" "$destination"
}
;;
version) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.1+k3s1; active_evidence_file="$altered" ;;
backend) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 sqlite; active_evidence_file="$altered" ;;
integrity) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd mismatch; active_evidence_file="$altered" ;;
destroyed) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd match pending; active_evidence_file="$altered" ;;
stale) write_evidence "$altered" "$FIXTURE_BUNDLE_ID" v1.36.2+k3s1 embedded-etcd match confirmed 2026-06-01T00:00:00Z 2026-06-01T01:00:00Z; active_evidence_file="$altered" ;;
live_stage) _ksre_live_restore_inventory() { printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|start|match|match|pass|pass|7\n'; } ;;
live_integrity) _ksre_live_restore_inventory() { printf 'v1.36.2+k3s1|embedded-etcd|enabled_stable|reencrypt_finished|match|mismatch|pass|pass|7\n'; } ;;
esac
_ksre_check >/dev/null 2>&1 && return 1
return 0
)
for scenario in authority_id version backend integrity destroyed stale live_stage live_integrity; do
assert_succeeds probe_check_failure "$scenario"
done
printf 'K3S SECRET ENCRYPTION RESTORE EVIDENCE TEST PASS\n'
File diff suppressed because it is too large Load Diff
+273
View File
@@ -0,0 +1,273 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly ALERTING_ROOT="$ROOT/services/observability/alerting"
readonly ALERTMANAGER_CONFIG="$ALERTING_ROOT/platform-alertmanager.yaml"
readonly PUBLIC_HTTPS_POLICY="$ALERTING_ROOT/alertmanager-public-https-egress.yaml"
readonly ALERTING_KUSTOMIZATION="$ALERTING_ROOT/kustomization.yaml"
readonly OBSERVABILITY_KUSTOMIZATION="$ROOT/services/observability/kustomization.yaml"
readonly KPS_ROOT="$ROOT/services/observability/kube-prometheus-stack"
readonly KPS_VALUES="$ROOT/services/observability/kube-prometheus-stack/values/home.yaml"
usage() {
printf 'usage: %s [--server-dry-run]\n' "${0##*/}" >&2
exit 2
}
server_dry_run=false
case $# in
0) ;;
1)
[[ "$1" == --server-dry-run ]] || usage
server_dry_run=true
;;
*) usage ;;
esac
python3 - \
"$ALERTMANAGER_CONFIG" \
"$PUBLIC_HTTPS_POLICY" \
"$ALERTING_KUSTOMIZATION" \
"$OBSERVABILITY_KUSTOMIZATION" \
"$KPS_VALUES" <<'PY'
import pathlib
import sys
import yaml
def fail(reason):
print(f"FAIL: observability alerting contract: {reason}", file=sys.stderr)
raise SystemExit(1)
def load_one(path):
source = pathlib.Path(path)
try:
with source.open(encoding="utf-8") as stream:
documents = [item for item in yaml.safe_load_all(stream) if item is not None]
except (OSError, yaml.YAMLError):
fail(f"{source.name} is missing or invalid YAML")
if len(documents) != 1 or not isinstance(documents[0], dict):
fail(f"{source.name} must contain exactly one object")
return documents[0]
alert_path, policy_path, child_kustomization_path, root_kustomization_path, values_path = sys.argv[1:]
alert = load_one(alert_path)
policy = load_one(policy_path)
child = load_one(child_kustomization_path)
root = load_one(root_kustomization_path)
values = load_one(values_path)
if alert != {
"apiVersion": "monitoring.coreos.com/v1alpha1",
"kind": "AlertmanagerConfig",
"metadata": {
"name": "platform-alertmanager",
"namespace": "observability",
"labels": {"observability.hyeonworks.com/instance": "home"},
},
"spec": {
"route": {
"receiver": "platform-slack",
"groupBy": ["cluster", "namespace", "alertname", "severity"],
"groupWait": "30s",
"groupInterval": "5m",
"repeatInterval": "4h",
"routes": [
{
"receiver": "platform-null",
"matchers": [
{
"name": "alertname",
"matchType": "=",
"value": "InfoInhibitor",
}
],
}
],
},
"inhibitRules": [
{
"sourceMatch": [
{
"name": "alertname",
"matchType": "=",
"value": "InfoInhibitor",
}
],
"targetMatch": [
{
"name": "severity",
"matchType": "=",
"value": "info",
}
],
"equal": ["namespace"],
}
],
"receivers": [
{"name": "platform-null"},
{
"name": "platform-slack",
"slackConfigs": [
{
"apiURL": {
"name": "alertmanager-slack-webhook",
"key": "url",
},
"sendResolved": True,
"linkNames": False,
"mrkdwnIn": ["text", "fields"],
"fallback": '{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }}: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}',
"title": '{{ if eq .Status "firing" }}[FIRING:{{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }} · {{ .CommonLabels.alertname }}',
"titleLink": "https://grafana.learn.hyeonworks.com/",
"color": '{{ if eq .Status "resolved" }}good{{ else if or (eq .CommonLabels.severity "emergency") (eq .CommonLabels.severity "critical") }}danger{{ else if eq .CommonLabels.severity "warning" }}warning{{ else }}#439FE0{{ end }}',
"fields": [
{
"title": "Status",
"value": "{{ .Status | toUpper }}",
"short": True,
},
{
"title": "Severity",
"value": "{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}UNKNOWN{{ end }}",
"short": True,
},
{
"title": "Location",
"value": "{{ if .CommonLabels.cluster }}{{ .CommonLabels.cluster }}{{ else }}unknown-cluster{{ end }} / {{ if .CommonLabels.namespace }}{{ .CommonLabels.namespace }}{{ else }}cluster-scoped{{ end }}",
"short": True,
},
{
"title": "Alert count",
"value": "{{ len .Alerts }}",
"short": True,
},
],
"text": "{{ range .Alerts }}\n*Alert status:* {{ .Status | toUpper }}\n*Target:* {{ $target := .Labels.Remove $.GroupLabels.Names }}{{ if $target }}{{ range $target.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }}{{ else }}unknown{{ end }}\n*Summary:* {{ with .Annotations.summary }}{{ . }}{{ else }}No summary provided{{ end }}\n*Details:* {{ with .Annotations.description }}{{ . }}{{ else }}No description provided{{ end }}\n*Started:* {{ .StartsAt.Format \"2006-01-02T15:04:05Z07:00\" }}\n{{ if eq .Status \"resolved\" }}*Ended:* {{ .EndsAt.Format \"2006-01-02T15:04:05Z07:00\" }}{{ end }}\n{{ with .Annotations.runbook_url }}*Runbook:* <{{ . }}|대응 절차 열기>{{ end }}\n\n{{ end }}\n<https://grafana.learn.hyeonworks.com/|Grafana> · <https://grafana.learn.hyeonworks.com/explore|Explore>",
"footer": "hyeonworks observability · Alertmanager",
}
],
}
],
},
}:
fail("platform-alertmanager must match the exact global Slack contract")
if policy != {
"apiVersion": "networking.k8s.io/v1",
"kind": "NetworkPolicy",
"metadata": {
"name": "observability-allow-alertmanager-public-https",
"namespace": "observability",
},
"spec": {
"podSelector": {
"matchLabels": {
"app.kubernetes.io/name": "alertmanager",
"app.kubernetes.io/instance": "observability-core-kube-pr-alertmanager",
}
},
"policyTypes": ["Egress"],
"egress": [
{
"to": [
{
"ipBlock": {
"cidr": "0.0.0.0/0",
"except": [
"10.0.0.0/8",
"100.64.0.0/10",
"172.16.0.0/12",
"192.168.0.0/16",
],
}
}
],
"ports": [{"protocol": "TCP", "port": 443}],
}
],
},
}:
fail("Alertmanager egress must allow only public IPv4 TCP/443")
if child.get("apiVersion") != "kustomize.config.k8s.io/v1beta1" or child.get("kind") != "Kustomization":
fail("alerting kustomization identity is invalid")
if child.get("namespace") != "observability":
fail("alerting kustomization namespace must be observability")
if child.get("resources") != [
"platform-alertmanager.yaml",
"alertmanager-public-https-egress.yaml",
]:
fail("alerting kustomization resources are not exact")
if set(child) != {"apiVersion", "kind", "namespace", "resources"}:
fail("alerting kustomization contains unapproved behavior")
root_resources = root.get("resources")
if not isinstance(root_resources, list) or len(root_resources) != len(set(root_resources)):
fail("observability root resources must be a unique list")
for required in ("targets", "rules/platform", "alerting", "dashboards/platform"):
if root_resources.count(required) != 1:
fail(f"observability root must include {required} exactly once")
try:
alertmanager_spec = values["alertmanager"]["alertmanagerSpec"]
except (KeyError, TypeError):
fail("kube-prometheus-stack alertmanagerSpec is missing")
if alertmanager_spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}:
fail("global alertmanagerConfiguration reference is not exact")
for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"):
if forbidden in alertmanager_spec:
fail(f"{forbidden} must remain absent when using the global reference")
for document, label in ((alert, "AlertmanagerConfig"), (policy, "NetworkPolicy")):
if document.get("kind") == "Secret":
fail(f"{label} source must not contain a Secret")
print("OBSERVABILITY ALERTING STATIC CONTRACT PASS")
PY
render_dir="$(mktemp -d /tmp/platform-observability-alerting-test.XXXXXX)"
cleanup() {
case "$render_dir" in
/tmp/platform-observability-alerting-test.*) rm -rf -- "$render_dir" ;;
esac
}
trap cleanup EXIT
chmod 0700 "$render_dir"
kubectl kustomize --enable-helm "$KPS_ROOT" >"$render_dir/kps.yaml"
chmod 0600 "$render_dir/kps.yaml"
python3 - "$render_dir/kps.yaml" <<'PY'
import sys
import yaml
with open(sys.argv[1], encoding="utf-8") as stream:
items = [item for item in yaml.safe_load_all(stream) if item is not None]
alertmanagers = [item for item in items if item.get("kind") == "Alertmanager"]
if len(alertmanagers) != 1:
raise SystemExit("FAIL: rendered KPS must contain exactly one Alertmanager")
spec = alertmanagers[0].get("spec") or {}
if spec.get("alertmanagerConfiguration") != {"name": "platform-alertmanager"}:
raise SystemExit("FAIL: rendered Alertmanager global reference is not exact")
for forbidden in ("alertmanagerConfigSelector", "alertmanagerConfigNamespaceSelector"):
if forbidden in spec:
raise SystemExit(f"FAIL: rendered Alertmanager retains forbidden {forbidden}")
print("OBSERVABILITY ALERTMANAGER GLOBAL RENDER CONTRACT PASS")
PY
if [[ "$server_dry_run" == true ]]; then
readonly KUBECTL_BIN="${PLATFORM_KUBECTL_BIN:-/usr/local/bin/kubectl}"
[[ "$KUBECTL_BIN" == /* && -f "$KUBECTL_BIN" && -x "$KUBECTL_BIN" && ! -L "$KUBECTL_BIN" ]] || {
printf 'FAIL: PLATFORM_KUBECTL_BIN must be an executable absolute regular file\n' >&2
exit 1
}
"$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$ALERTMANAGER_CONFIG" >/dev/null
"$KUBECTL_BIN" --request-timeout=10s apply --dry-run=server -f "$PUBLIC_HTTPS_POLICY" >/dev/null
printf 'OBSERVABILITY ALERTING SERVER DRY RUN PASS\n'
fi
+612
View File
@@ -0,0 +1,612 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
DASHBOARD_ROOT="$ROOT/services/observability/dashboards/platform"
METRIC_ROOT=${PLATFORM_OBSERVABILITY_METRIC_ROOT:-/tmp/platform-observability-metrics.VUpsZn}
TARGET_INVENTORY="$METRIC_ROOT/target-initial/inventory.json"
POST_INVENTORY="$METRIC_ROOT/post-substrate/inventory.json"
PROMETHEUS_IMAGE='quay.io/prometheus/prometheus@sha256:ce95cfa77eff5aad28bd7a65aff19868cf78d9e17e4c254da7dfe22ade78318b'
WORK_DIR=''
PROMETHEUS_BASE_URL=''
PORT_FORWARD_PID=''
fail() {
printf 'FAIL: %s\n' "$*" >&2
exit 1
}
pass() {
printf 'PASS: %s\n' "$*"
}
require_command() {
command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
}
usage() {
printf 'Usage: %s [--prometheus-base-url http://127.0.0.1:PORT]\n' "${0##*/}"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--prometheus-base-url)
[[ $# -ge 2 ]] || fail '--prometheus-base-url requires a value'
[[ -z "$PROMETHEUS_BASE_URL" ]] || fail '--prometheus-base-url may be specified only once'
PROMETHEUS_BASE_URL=$2
shift 2
;;
--help)
usage
exit 0
;;
*)
fail "unknown argument: $1"
;;
esac
done
if [[ -n "$PROMETHEUS_BASE_URL" ]]; then
[[ "$PROMETHEUS_BASE_URL" =~ ^http://(127\.0\.0\.1|localhost):([1-9][0-9]{0,4})$ ]] \
|| fail '--prometheus-base-url must be an exact loopback HTTP origin without a path'
(( BASH_REMATCH[2] <= 65535 )) \
|| fail '--prometheus-base-url port exceeds 65535'
fi
}
cleanup() {
local rc=$?
trap - EXIT
if [[ -n "$PORT_FORWARD_PID" ]] && kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
kill "$PORT_FORWARD_PID" 2>/dev/null || true
wait "$PORT_FORWARD_PID" 2>/dev/null || true
fi
if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then
rm -rf -- "$WORK_DIR"
fi
exit "$rc"
}
verify_inventory() {
local phase=$1
local inventory="$METRIC_ROOT/$phase/inventory.json"
local checksum="$METRIC_ROOT/$phase/inventory.sha256"
local expected actual
[[ -f "$inventory" ]] || fail "$phase inventory is absent"
[[ -f "$checksum" ]] || fail "$phase inventory checksum is absent"
jq -e --arg phase "$phase" '
.schema == "platform-observability-metric-inventory/v1"
and .phase == $phase
and (.targets | type == "array" and length > 0)
and all(.targets[]; .health == "up" and (.metrics | type == "array" and length > 0))
' "$inventory" >/dev/null || fail "$phase inventory contract is invalid"
read -r expected checksum_name < "$checksum"
[[ "$checksum_name" == 'inventory.json' ]] || fail "$phase checksum names an unexpected file"
actual=$(sha256sum "$inventory" | awk '{print $1}')
[[ "$actual" == "$expected" ]] || fail "$phase inventory checksum mismatch"
pass "$phase inventory contract and checksum"
}
inventory_has_metric() {
local inventory=$1
local metric=$2
jq -e --arg metric "$metric" 'any(.targets[].metrics[]; .name == $metric)' "$inventory" >/dev/null
}
inventory_metric_has_label() {
local inventory=$1
local metric=$2
local label=$3
jq -e --arg metric "$metric" --arg label "$label" '
any(.targets[].metrics[]; .name == $metric and (.label_names | index($label) != null))
' "$inventory" >/dev/null
}
validate_inventory_references() {
local dashboard=$1
local inventory=$2
local expression selector metric matchers matcher label token
local inventory_metrics referenced_metrics found
inventory_metrics=$(jq -r '[.targets[].metrics[].name] | unique[]' "$inventory")
while IFS= read -r expression; do
[[ -n "$expression" ]] || fail "empty PromQL expression in $dashboard"
referenced_metrics=''
while IFS= read -r selector; do
[[ -n "$selector" ]] || continue
metric=${selector%%\{*}
grep -Fxq "$metric" <<<"$inventory_metrics" \
|| fail "$dashboard references absent metric $metric"
if ! grep -Fxq "$metric" <<<"$referenced_metrics"; then
referenced_metrics+="${metric}"$'\n'
fi
matchers=${selector#*\{}
matchers=${matchers%\}}
while IFS= read -r matcher; do
matcher=${matcher#"${matcher%%[![:space:]]*}"}
[[ -n "$matcher" ]] || continue
label=$(sed -E 's/^([A-Za-z_][A-Za-z0-9_]*).*/\1/' <<<"$matcher")
[[ "$label" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \
|| fail "$dashboard has an unauditable label matcher in $expression"
inventory_metric_has_label "$inventory" "$metric" "$label" \
|| fail "$dashboard references absent label $metric.$label"
done < <(tr ',' '\n' <<<"$matchers")
done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]*\}' <<<"$expression" || true)
[[ -n "$referenced_metrics" ]] || fail "$dashboard expression has no inventory metric: $expression"
while IFS= read -r token; do
[[ -n "$token" ]] || continue
if grep -Fxq "$token" <<<"$inventory_metrics"; then
grep -Fxq "$token" <<<"$referenced_metrics" \
|| fail "$dashboard uses inventory metric $token without an auditable selector"
fi
done < <(grep -oE '[A-Za-z_:][A-Za-z0-9_:]*' <<<"$expression" | sort -u)
while IFS= read -r label; do
[[ -n "$label" ]] || continue
found=0
while IFS= read -r metric; do
[[ -n "$metric" ]] || continue
if inventory_metric_has_label "$inventory" "$metric" "$label"; then
found=1
break
fi
done <<<"$referenced_metrics"
[[ "$found" -eq 1 ]] || fail "$dashboard groups by absent inventory label $label"
done < <(
grep -oE '(by|without)[[:space:]]*\([^)]*\)' <<<"$expression" \
| sed -E 's/^[^(]*\(([^)]*)\)$/\1/' \
| tr ',' '\n' \
| sed -E 's/^[[:space:]]+|[[:space:]]+$//g' \
|| true
)
done < <(jq -r '.panels[].targets[].expr' "$dashboard")
}
validate_dashboard() {
local file=$1
local expected_uid=$2
local inventory=$3
local path="$DASHBOARD_ROOT/$file"
[[ -f "$path" ]] || fail "dashboard source is absent: $file"
jq -e --arg uid "$expected_uid" '
type == "object"
and .uid == $uid
and (.title | type == "string" and length > 0)
and .schemaVersion == 42
and .editable == false
and .refresh == "30s"
and (.templating.list == [])
and (.panels | type == "array" and length > 0)
and ([.panels[].id] | length == (unique | length))
and all(.panels[];
.type != "row"
and (.title | type == "string" and length > 0)
and (.gridPos.w > 0 and .gridPos.h > 0)
and .datasource.type == "prometheus"
and .datasource.uid == "prometheus"
and (.targets | type == "array" and length > 0)
and all(.targets[];
.datasource.type == "prometheus"
and .datasource.uid == "prometheus"
and (.expr | type == "string" and length > 0)
and (.refId | type == "string" and length > 0)
)
)
' "$path" >/dev/null || fail "dashboard structure is invalid: $file"
if jq -r '.title, .tags[], .panels[].title' "$path" \
| grep -Eiq '(^|[^[:alnum:]_])(spring|jvm|kafka|batch|backup)([^[:alnum:]_]|$)'; then
fail "dashboard contains a forbidden product area: $file"
fi
if jq -r '.panels[].targets[].expr' "$path" \
| grep -Eiq '(^|[^[:alnum:]_])(spring_|jvm_|kafka_|batch_|backup_)'; then
fail "dashboard contains a forbidden product query: $file"
fi
if jq -r '.. | strings' "$path" \
| grep -Eiq '(request[_ -]?id|trace[_ -]?id|username|raw[_ -]?url)'; then
fail "dashboard contains forbidden or high-cardinality content: $file"
fi
validate_inventory_references "$path" "$inventory"
pass "$file structure and inventory references"
}
validate_rendered_configmaps() {
local rendered=$1
local file name block
local -a files=(
kubernetes-node
workload-health
platform-services
observability-backends
https-endpoints
)
kubectl kustomize "$DASHBOARD_ROOT" > "$rendered"
[[ $(grep -c '^kind: ConfigMap$' "$rendered") -eq 5 ]] \
|| fail 'kustomization must render exactly five ConfigMaps'
for file in "${files[@]}"; do
name="grafana-dashboard-$file"
block=$(awk -v name="$name" '
BEGIN { RS="---" }
$0 ~ "name: " name "([[:space:]]|$)" { print }
' "$rendered")
[[ -n "$block" ]] || fail "rendered ConfigMap is absent: $name"
grep -Eq 'grafana_dashboard: ("1"|1)$' <<<"$block" \
|| fail "$name lacks grafana_dashboard=1"
grep -q 'observability.hyeonworks.com/instance: home' <<<"$block" \
|| fail "$name lacks the observability instance label"
grep -q 'observability.hyeonworks.com/owner: platform-observability' <<<"$block" \
|| fail "$name lacks the observability owner label"
grep -q " $file.json:" <<<"$block" \
|| fail "$name does not embed $file.json"
done
pass 'kustomization renders five stable labeled dashboard ConfigMaps'
}
validate_inventory_driven_omissions() {
local workload="$DASHBOARD_ROOT/workload-health.json"
local services="$DASHBOARD_ROOT/platform-services.json"
local endpoints="$DASHBOARD_ROOT/https-endpoints.json"
if inventory_has_metric "$TARGET_INVENTORY" kube_pod_container_status_last_terminated_reason; then
fail 'the OOM omission contract no longer matches target-initial inventory'
fi
if jq -r '.panels[].targets[].expr' "$workload" \
| grep -Fq 'kube_pod_container_status_last_terminated_reason'; then
fail 'workload dashboard guessed an OOM metric absent from target-initial inventory'
fi
pass 'OOM panel omitted: kube_pod_container_status_last_terminated_reason is absent from target-initial inventory'
jq -e '
([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})") != null)
and
([.panels[].targets[].expr] | index("sum(kube_persistentvolumeclaim_status_phase{phase=\"Lost\"})") != null)
' "$workload" >/dev/null \
|| fail 'PVC health panel must retain zero-valued Pending and Lost series instead of filtering healthy state away'
pass 'PVC health panel keeps visible zero-valued Pending and Lost aggregates'
if jq -e '
any(.targets[].metrics[];
(.name | startswith("traefik_"))
and (.name | endswith("_bucket"))
and (.name | test("duration|request"))
)
' "$TARGET_INVENTORY" >/dev/null; then
fail 'the Traefik latency omission contract no longer matches target-initial inventory'
fi
if jq -r '.panels[].targets[].expr' "$services" \
| grep -Eq 'histogram_quantile|traefik_.*_bucket'; then
fail 'service dashboard guessed a Traefik latency histogram absent from target-initial inventory'
fi
pass 'Traefik p95/p99 omitted: no request duration histogram bucket exists in target-initial inventory'
inventory_has_metric "$POST_INVENTORY" probe_success \
|| fail 'post-substrate inventory lacks probe_success for the platform boundary summary'
jq -e '
any(.panels[].targets[];
.expr == "min by (job) (probe_success{})"
)
' "$services" >/dev/null \
|| fail 'platform service dashboard lacks the inventory-backed boundary health summary'
pass 'platform service boundary summary uses post-substrate probe_success'
jq -e '
any(.panels[].targets[];
.expr == "max by (instance) (probe_http_status_code{job=\"blackbox-private-edge\"})"
)
' "$endpoints" >/dev/null \
|| fail 'HTTPS dashboard does not isolate the observed private-edge 403 boundary'
pass 'HTTPS dashboard isolates the observed blackbox-private-edge status boundary'
}
validate_promql_syntax() {
local rules=$1
local file expression index=0
local -a files=(
kubernetes-node.json
workload-health.json
platform-services.json
observability-backends.json
https-endpoints.json
)
{
printf 'groups:\n'
printf ' - name: dashboard.promql.syntax\n'
printf ' rules:\n'
for file in "${files[@]}"; do
while IFS= read -r expression; do
index=$((index + 1))
printf ' - record: dashboard_syntax_%d\n' "$index"
printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
done < <(jq -r '.panels[].targets[].expr' "$DASHBOARD_ROOT/$file")
done
} > "$rules"
if [[ -n ${PROMTOOL_BIN:-} ]]; then
"$PROMTOOL_BIN" check rules "$rules" >/dev/null
else
require_command docker
docker run --rm \
--entrypoint=/bin/promtool \
-v "$rules:/tmp/dashboard-rules.yaml:ro" \
"$PROMETHEUS_IMAGE" check rules /tmp/dashboard-rules.yaml >/dev/null
fi
pass "promtool parsed $index dashboard expressions"
}
validate_traefik_low_traffic_ratio() {
local rules="$WORK_DIR/traefik-ratio-rules.yaml"
local tests="$WORK_DIR/traefik-ratio-tests.yaml"
local expression
expression=$(jq -er '
.panels[]
| select(.title == "Traefik 5xx Ratio")
| .targets[]
| select(.refId == "A")
| .expr
' "$DASHBOARD_ROOT/platform-services.json") \
|| fail 'Traefik 5xx ratio expression is absent'
{
printf 'groups:\n'
printf ' - name: dashboard.traefik.ratio\n'
printf ' interval: 1m\n'
printf ' rules:\n'
printf ' - record: dashboard_traefik_5xx_ratio_percent\n'
printf ' expr: %s\n' "$(jq -Rn --arg expr "$expression" '$expr')"
} > "$rules"
{
printf 'rule_files:\n'
printf ' - traefik-ratio-rules.yaml\n'
printf 'evaluation_interval: 1m\n'
printf 'tests:\n'
printf ' - interval: 1m\n'
printf ' input_series:\n'
printf ' - series: '\''traefik_entrypoint_requests_total{code="200",entrypoint="websecure"}'\''\n'
printf ' values: '\''0+1x10'\''\n'
printf ' - series: '\''traefik_entrypoint_requests_total{code="500",entrypoint="websecure"}'\''\n'
printf ' values: '\''0+1x10'\''\n'
printf ' promql_expr_test:\n'
printf ' - expr: dashboard_traefik_5xx_ratio_percent\n'
printf ' eval_time: 10m\n'
printf ' exp_samples:\n'
printf ' - labels: '\''dashboard_traefik_5xx_ratio_percent{}'\''\n'
printf ' value: 50\n'
} > "$tests"
if [[ -n ${PROMTOOL_BIN:-} ]]; then
(cd "$WORK_DIR" && "$PROMTOOL_BIN" test rules traefik-ratio-tests.yaml) >/dev/null
else
require_command docker
docker run --rm \
--entrypoint=/bin/promtool \
-v "$WORK_DIR:/tmp/dashboard-validation:ro" \
-w /tmp/dashboard-validation \
"$PROMETHEUS_IMAGE" test rules traefik-ratio-tests.yaml >/dev/null
fi
pass 'Traefik 5xx ratio preserves 50 percent at low request rates'
}
start_prometheus_port_forward() {
local log="$WORK_DIR/prometheus-port-forward.log"
local port=''
local attempt
kubectl -n observability port-forward \
--address=127.0.0.1 \
service/observability-core-kube-pr-prometheus \
:9090 > "$log" 2>&1 &
PORT_FORWARD_PID=$!
for ((attempt = 1; attempt <= 100; attempt++)); do
if grep -Eq '^Forwarding from 127\.0\.0\.1:[0-9]+ -> 9090$' "$log"; then
port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> 9090$/\1/p' "$log" | head -n 1)
break
fi
if ! kill -0 "$PORT_FORWARD_PID" 2>/dev/null; then
fail "Prometheus port-forward exited before readiness: $(tr '\n' ' ' < "$log")"
fi
sleep 0.1
done
[[ "$port" =~ ^[1-9][0-9]*$ ]] \
|| fail 'Prometheus port-forward did not bind a loopback port within 10 seconds'
PROMETHEUS_BASE_URL="http://127.0.0.1:$port"
}
prometheus_get() {
local endpoint=$1
shift
curl \
--fail \
--silent \
--show-error \
--connect-timeout 2 \
--max-time 10 \
--get \
"$PROMETHEUS_BASE_URL$endpoint" \
"$@"
}
wait_for_prometheus() {
local attempt
for ((attempt = 1; attempt <= 50; attempt++)); do
if prometheus_get '/-/ready' >/dev/null 2>&1; then
return 0
fi
sleep 0.1
done
fail 'Prometheus did not become ready within 5 seconds'
}
live_query_nonempty() {
local expression=$1
local context=$2
local expected_scalar=${3:-}
local quiet=${4:-0}
local response="$WORK_DIR/live-query-response.json"
if ! prometheus_get '/api/v1/query' \
--data-urlencode "query=$expression" > "$response"; then
fail "live Prometheus query request failed: $context"
fi
jq -e '
.status == "success"
and (.data.resultType == "vector" or .data.resultType == "scalar")
and (.data.result | type == "array" and length > 0)
' "$response" >/dev/null \
|| fail "live Prometheus query returned no vector/scalar result: $context"
if [[ -n "$expected_scalar" ]] && ! jq -e --arg expected "$expected_scalar" '
.data.resultType == "scalar"
and .data.result[1] == $expected
' "$response" >/dev/null; then
if [[ "$quiet" -eq 1 ]]; then
return 1
fi
fail "live Prometheus scalar result was not exact $expected_scalar: $context"
fi
}
validate_scalar_one_gate_regression() {
if (live_query_nonempty 'scalar(vector(0))' 'scalar-zero regression fixture' 1 1); then
fail 'scalar-zero comparison fixture passed the success gate'
fi
live_query_nonempty 'scalar(vector(1))' 'scalar-one regression fixture' 1
pass 'success comparison gate rejects scalar 0 and accepts exact scalar 1'
}
validate_live_dashboard_queries() {
local dashboard uid panel ref expression
local count=0
for dashboard in "$DASHBOARD_ROOT"/*.json; do
while IFS=$'\t' read -r uid panel ref expression; do
live_query_nonempty "$expression" "$uid / $panel / $ref"
count=$((count + 1))
done < <(
jq -r '
.uid as $uid
| .panels[]
| .title as $panel
| .targets[]
| [$uid, $panel, .refId, .expr]
| @tsv
' "$dashboard"
)
done
[[ "$count" -eq 37 ]] || fail "expected 37 live dashboard queries, got $count"
pass 'live Prometheus accepted 37 dashboard queries with nonempty vector/scalar results'
}
validate_live_matcher_selectors() {
local dashboard expression selector
local selectors="$WORK_DIR/dashboard-matchers.txt"
local count=0
: > "$selectors"
for dashboard in "$DASHBOARD_ROOT"/*.json; do
while IFS= read -r expression; do
grep -oE '[A-Za-z_:][A-Za-z0-9_:]*\{[^}]+\}' <<<"$expression" >> "$selectors" || true
done < <(jq -r '.panels[].targets[].expr' "$dashboard")
done
sort -u -o "$selectors" "$selectors"
while IFS= read -r selector; do
[[ -n "$selector" ]] || continue
live_query_nonempty "$selector" "live matcher selector $selector"
count=$((count + 1))
done < "$selectors"
[[ "$count" -gt 0 ]] || fail 'no dashboard matcher selectors were discovered'
pass "live Prometheus found series for $count dashboard matcher selectors"
}
validate_live_matcher_values() {
local -a presence_contracts=(
'node_cpu_seconds_total{mode="idle"}'
'node_filesystem_avail_bytes{mountpoint="/"}'
'node_filesystem_size_bytes{mountpoint="/"}'
'node_filesystem_files_free{mountpoint="/"}'
'node_filesystem_files{mountpoint="/"}'
'kube_persistentvolumeclaim_status_phase{phase="Pending"}'
'kube_persistentvolumeclaim_status_phase{phase="Lost"}'
)
local -a success_contracts=(
'scalar(count(count by (bucket) (minio_cluster_usage_buckets_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
'scalar(count(count by (bucket) (minio_cluster_usage_buckets_quota_total_bytes{bucket=~"loki|tempo"}))) == bool 2'
'scalar(min(probe_success{job="blackbox-private-edge"})) == bool 1'
'scalar(count(probe_http_status_code{job="blackbox-private-edge"})) == bool scalar(count(probe_http_status_code{job="blackbox-private-edge"} == 403))'
)
local contract
for contract in "${presence_contracts[@]}"; do
live_query_nonempty "$contract" "live matcher-value contract $contract"
done
for contract in "${success_contracts[@]}"; do
live_query_nonempty "$contract" "live matcher-value contract $contract" 1
done
pass 'live matcher values cover idle CPU, root filesystems, PVC phases, both buckets, and private-edge 403'
}
validate_live_prometheus() {
require_command curl
if [[ -z "$PROMETHEUS_BASE_URL" ]]; then
start_prometheus_port_forward
fi
wait_for_prometheus
validate_scalar_one_gate_regression
validate_live_dashboard_queries
validate_live_matcher_selectors
validate_live_matcher_values
}
main() {
local work rendered rules
parse_args "$@"
require_command jq
require_command sha256sum
require_command kubectl
verify_inventory target-initial
verify_inventory post-substrate
[[ -d "$DASHBOARD_ROOT" ]] || fail 'dashboard source directory is absent'
validate_dashboard kubernetes-node.json platform-kubernetes-node "$TARGET_INVENTORY"
validate_dashboard workload-health.json platform-workload-health "$TARGET_INVENTORY"
validate_dashboard platform-services.json platform-services "$POST_INVENTORY"
validate_dashboard observability-backends.json platform-observability-backends "$TARGET_INVENTORY"
validate_dashboard https-endpoints.json platform-https-endpoints "$POST_INVENTORY"
validate_inventory_driven_omissions
work=$(mktemp -d)
WORK_DIR=$work
chmod 0755 "$WORK_DIR"
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
rendered="$work/rendered.yaml"
rules="$work/dashboard-rules.yaml"
validate_rendered_configmaps "$rendered"
validate_promql_syntax "$rules"
validate_traefik_low_traffic_ratio
validate_live_prometheus
pass 'observability dashboard contract'
}
main "$@"
@@ -0,0 +1,163 @@
#!/usr/bin/env bash
set -euo pipefail
# This closed policy check intentionally reports only labels and token counts.
readonly ROOT='/home/donghyeon/workspace/platform'
readonly CENTRAL_DOCS='/home/donghyeon/workspace/docs/platform'
readonly MASTER="$CENTRAL_DOCS/plans/2026-07-31-observability-phase4.md"
readonly ACCESS="$CENTRAL_DOCS/plans/2026-07-31-observability-access-alerting.md"
readonly MANUAL="$ROOT/bootstrap/manual/phase4-observability-access.md"
readonly WORKLOG="$CENTRAL_DOCS/runbooks/2026-08-13-observability-phase4-resume-worklog.md"
readonly HISTORICAL_PLAN="$CENTRAL_DOCS/plans/2026-08-11-gitea-metrics-checkpoint.md"
readonly HISTORICAL_DESIGN="$CENTRAL_DOCS/plans/2026-08-11-gitea-metrics-checkpoint-design.md"
failures=0
fail() {
printf 'FAIL: %s\n' "$1" >&2
failures=$((failures + 1))
}
require_file() {
[[ -f "$1" && ! -L "$1" ]] || fail "$2 is not a regular file"
}
count_token() {
local file=$1 token=$2
grep -Foc -- "$token" "$file" || true
}
require_count() {
local file=$1 label=$2 token=$3 expected=$4 actual
actual="$(count_token "$file" "$token")"
[[ "$actual" == "$expected" ]] || fail "$label token_count=$actual expected=$expected"
}
require_hash() {
local file=$1 label=$2 expected=$3 actual
actual="$(sha256sum -- "$file" | awk '{print $1}')"
[[ "$actual" == "$expected" ]] || fail "$label sha256=$actual expected=$expected"
}
require_execution_relationships() {
python3 - "$ACCESS" "$MANUAL" <<'PY'
import pathlib
import sys
access = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')
manual = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')
def commands(text):
result = []
lines = text.splitlines()
index = 0
while index < len(lines):
line = lines[index]
if line.lstrip().startswith('bash scripts/bootstrap/create-observability-secrets.sh'):
start = index
parts = [line.strip()]
while parts[-1].rstrip().endswith('\\') and index + 1 < len(lines):
index += 1
parts.append(lines[index].strip())
result.append((start, ' '.join(parts)))
index += 1
return result
def slack_execute(items):
return [item for item in items if '--execute' in item[1] and '--slack-webhook' in item[1]]
def checker_after_count(text, command_index):
return sum(
index > command_index and '--check-slack-deployment-evidence' in line
for index, line in enumerate(text.splitlines())
)
failures = 0
def require(label, actual, expected):
global failures
if actual != expected:
print(f'FAIL: {label} count={actual} expected={expected}')
failures += 1
hostile = '''padding padding padding padding padding padding padding padding padding padding padding padding
--check-slack-deployment-evidence
bash scripts/bootstrap/create-observability-secrets.sh --execute --slack-webhook --accept-no-off-host-slack-recovery
'''
hostile_slack = slack_execute(commands(hostile))
require('hostile checker-before-Slack ordering', checker_after_count(hostile, hostile_slack[0][0]), 0)
step = '- [ ] **Step 3: 실행 순서를 manual 문서에 고정한다**'
start = access.find(step)
if start < 0:
require('authoritative access cutover block', 0, 1)
block = ''
else:
fence_start = access.find(' ```bash', start)
fence_end = access.find(' ```', fence_start + len(' ```bash'))
block = access[fence_start:fence_end] if fence_start >= 0 and fence_end >= 0 else ''
require('authoritative access cutover block', int(bool(block)), 1)
access_slack = slack_execute(commands(block))
require('authoritative access Slack execute command', len(access_slack), 1)
if len(access_slack) == 1:
require('authoritative access Slack risk flag', int('--accept-no-off-host-slack-recovery' in access_slack[0][1]), 1)
manual_slack = slack_execute(commands(manual))
require('manual Slack execute command', len(manual_slack), 1)
if len(manual_slack) == 1:
command_index, command = manual_slack[0]
require('manual Slack risk flag', int('--accept-no-off-host-slack-recovery' in command), 1)
require('manual deployment checker after Slack bootstrap', checker_after_count(manual, command_index), 1)
sys.exit(1 if failures else 0)
PY
}
require_file "$MASTER" master
require_file "$ACCESS" access
require_file "$MANUAL" manual
require_file "$WORKLOG" worklog
require_file "$HISTORICAL_PLAN" historical_plan
require_file "$HISTORICAL_DESIGN" historical_design
if (( failures == 0 )); then
require_count "$MASTER" 'master deployment gate' '--check-slack-deployment-evidence' 1
require_count "$ACCESS" 'access deployment gate' '--check-slack-deployment-evidence' 2
require_count "$ACCESS" 'access strict recovery gate' '--check-slack-recovery-evidence' 1
require_count "$MANUAL" 'manual deployment gate' '--check-slack-deployment-evidence' 1
require_count "$MANUAL" 'manual strict recovery gate' '--check-slack-recovery-evidence' 1
require_count "$MASTER" 'master strict DR-only wording' 'off-host Slack DR-complete 판정에만 사용' 1
require_count "$ACCESS" 'access strict DR-only wording' 'operational apply를 허용하지 않는다' 1
require_count "$MANUAL" 'manual risk-path DR-deferred wording' '이 경로는 DR-complete을 주장하지 않습니다' 1
require_count "$MANUAL" 'manual risk flag' '--accept-no-off-host-slack-recovery' 1
require_count "$MANUAL" 'manual exact risk confirmation' 'ACCEPT NO OFF-HOST SLACK RECOVERY default' 1
require_count "$MANUAL" 'manual kubectl prohibition' 'kubectl로 Secret을 수동 생성하지 않습니다' 1
require_count "$MANUAL" 'manual false recovery prohibition' '거짓 `RECOVERY SLACK default` 확인을 입력하지 않습니다' 1
require_count "$MANUAL" 'manual two deployment-gate calls' '두 번 호출' 1
require_count "$MANUAL" 'manual v2 acceptance schema' 'platform-observability-rules-alerts-v2' 1
require_count "$MANUAL" 'manual bare risk ledger value' '`RISK_ACCEPTED`' 1
require_count "$MANUAL" 'manual bare recovery ledger value' '`RECOVERY`' 1
require_count "$ACCESS" 'access risk flag order' '--accept-no-off-host-slack-recovery' 3
require_count "$MASTER" 'master operational acceptance status' 'Phase 4 operational acceptance: complete with active Slack DR exception' 1
require_count "$MASTER" 'master deferred DR status' 'Slack off-host disaster recovery: deferred / not complete' 1
require_count "$ACCESS" 'access operational acceptance status' 'Phase 4 operational acceptance: complete with active Slack DR exception' 1
require_count "$ACCESS" 'access deferred DR status' 'Slack off-host disaster recovery: deferred / not complete' 1
require_count "$WORKLOG" 'worklog user decision' 'Slack risk acceptance decision' 1
require_count "$WORKLOG" 'worklog KeePass deferred' 'KeePass work remains deferred' 1
require_count "$WORKLOG" 'worklog absent Slack Secret' 'alertmanager-slack-webhook Secret remains absent' 1
require_hash "$HISTORICAL_PLAN" 'historical plan byte preservation' '10ccc38835928304bd631a60680ecc8e6c1d565d51cbe7047169eb609aa38ce5'
require_hash "$HISTORICAL_DESIGN" 'historical design byte preservation' 'a9fd69e0241e5b56c5328076648a8ecf4a6f71d416aae7d53453ce7ff969249d'
if ! require_execution_relationships; then
failures=$((failures + 1))
fi
fi
if [[ -d "$ROOT/docs" ]]; then
fail 'platform docs boundary token_count=1 expected=0'
fi
if (( failures != 0 )); then
exit 1
fi
printf 'PASS: observability Slack risk policy token_counts=exact\n'
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -Eeuo pipefail
readonly ROOT="$(cd -- "$(dirname -- "$BASH_SOURCE")/../.." && pwd -P)"
readonly APPLY="$ROOT/scripts/bootstrap/apply-private-dns.sh"
readonly LAN="$ROOT/infrastructure/networking/private-dns/host/dnsmasq-lan.conf"
readonly TAIL="$ROOT/infrastructure/networking/private-dns/host/dnsmasq-tailscale.conf"
readonly CORE="$ROOT/infrastructure/networking/private-dns/kubernetes/coredns-custom.yaml"
readonly BUSYBOX='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0'
readonly -a HOSTS=(
git.learn.hyeonworks.com
id.learn.hyeonworks.com
storage-admin.learn.hyeonworks.com
db-admin.learn.hyeonworks.com
grafana.learn.hyeonworks.com
)
fail() {
printf 'FAIL: %s\n' "$*" >&2
exit 1
}
pass() {
printf 'PASS: %s\n' "$*"
}
for file in "$APPLY" "$LAN" "$TAIL" "$CORE"; do
[[ -f "$file" && ! -L "$file" ]] || fail "missing regular source: $file"
done
bash -n "$APPLY"
for host in "${HOSTS[@]}"; do
grep -Fqx "address=/$host/192.168.0.107" "$LAN" || fail "LAN source lacks $host"
grep -Fqx "address=/$host/100.92.240.34" "$TAIL" || fail "Tail source lacks $host"
grep -Fq "192.168.0.107 $host" "$CORE" || fail "CoreDNS source lacks $host"
done
pass 'all five private names are source-bound for LAN, Tail, and Pods'
grep -Fq 'PUBLIC_PRIVATE_HOSTS=(storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com)' "$APPLY" ||
fail 'public A/AAAA absence set is not exact'
grep -Fq 'PRIVATE_HOSTS=(git.learn.hyeonworks.com id.learn.hyeonworks.com storage-admin.learn.hyeonworks.com db-admin.learn.hyeonworks.com grafana.learn.hyeonworks.com)' "$APPLY" ||
fail 'private resolver verification set is not exact'
grep -Fq "readonly BUSYBOX_IMAGE=\"$BUSYBOX\"" "$APPLY" || fail 'BusyBox pin constant is not exact'
grep -Fq -- '--image="$BUSYBOX_IMAGE"' "$APPLY" || fail 'Pod smoke does not use the pinned BusyBox constant'
for host in "${HOSTS[@]}"; do
grep -Fq "nslookup $host" "$APPLY" || fail "Pod smoke omits $host"
done
pass 'apply source checks public absence and all five resolver paths with pinned BusyBox'
grep -Fq 'managed_listeners_are_exact' "$APPLY" || fail 'repeat execution has no exact managed-listener classifier'
grep -Fq 'lan_was_active=' "$APPLY" || fail 'rollback does not snapshot LAN active state'
grep -Fq 'tail_was_active=' "$APPLY" || fail 'rollback does not snapshot Tail active state'
grep -Fq 'lan_was_enabled=' "$APPLY" || fail 'rollback does not snapshot LAN enabled state'
grep -Fq 'tail_was_enabled=' "$APPLY" || fail 'rollback does not snapshot Tail enabled state'
grep -Fq 'restore_service_state' "$APPLY" || fail 'rollback does not restore managed service state'
pass 'repeat execution and rollback service-state contracts are present'
printf 'PRIVATE DNS OBSERVABILITY CONTRACT PASS\n'
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,562 @@
#!/usr/bin/env bash
# Production-boundary test for the temporary blackbox source proof. Every
# external side effect is replaced only below the Kubernetes/root boundaries;
# the validator CLI and its production control flow remain real.
set -Eeuo pipefail
set +x
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly VALIDATOR="$ROOT/scripts/validate/validate-blackbox-edge-source.sh"
fail() { printf 'BLACKBOX EDGE SOURCE TEST FAILURE: %s\n' "$*" >&2; exit 1; }
assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected=$1 actual=$2"; }
assert_contains() { [[ "$1" == *"$2"* ]] || fail "$3"; }
assert_not_contains() { [[ "$1" != *"$2"* ]] || fail "$3"; }
assert_no_object_create() {
[[ "$1" != *' create --dry-run=server '* && "$1" != *' create -f '* ]] || fail "$2"
}
assert_no_name_delete() {
[[ "$1" != *' delete pod '* && "$1" != *' delete networkpolicy '* ]] || fail "$2"
}
[[ -f "$VALIDATOR" && ! -L "$VALIDATOR" ]] || fail 'production validator is missing (RED: create validate-blackbox-edge-source.sh)'
# RED 8: production uses / as its rollback anchor. A canonical absolute
# descendant of that anchor must remain valid without invoking sudo or any
# Kubernetes boundary; non-root fixture-anchor checks are exercised below.
production_root_anchor_rc=0
PLATFORM_BLACKBOX_EDGE_TEST_MODE=0 bash -c '
source "$1"
root_run() { "$@"; }
validate_root_chain /var/lib /
' bash "$VALIDATOR" || production_root_anchor_rc=$?
assert_eq 0 "$production_root_anchor_rc" 'production root anchor must accept a canonical descendant'
fixture_root="$(mktemp -d /tmp/platform-blackbox-edge-test.XXXXXX)"
chmod 0700 "$fixture_root"
trap 'rm -rf -- "$fixture_root"' EXIT
mkdir -p "$fixture_root/bin" "$fixture_root/state" "$fixture_root/active" \
"$fixture_root/rollbacks/observability-20260812T000000Z"
chmod 0700 "$fixture_root/bin" "$fixture_root/state" "$fixture_root/active" \
"$fixture_root/rollbacks" "$fixture_root/rollbacks/observability-20260812T000000Z"
cp -- "$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" \
"$fixture_root/active/learn-services"
chmod 0600 "$fixture_root/active/learn-services"
cat >"$fixture_root/bin/kubectl-proxy.py" <<'PY'
import json
import os
import pathlib
import socket
import sys
sock_path = pathlib.Path(sys.argv[1])
state = pathlib.Path(sys.argv[2])
if sock_path.exists() or sock_path.is_symlink():
raise SystemExit(91)
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(str(sock_path))
os.chmod(sock_path, 0o600)
server.listen(1)
connection, _ = server.accept()
request = b""
while b"\r\n\r\n" not in request:
piece = connection.recv(4096)
if not piece:
break
request += piece
head, _, body = request.partition(b"\r\n\r\n")
lines = head.decode("ascii", "replace").split("\r\n")
method, path, _ = lines[0].split(" ", 2)
length = 0
for line in lines[1:]:
if line.lower().startswith("content-length:"):
length = int(line.split(":", 1)[1].strip())
while len(body) < length:
piece = connection.recv(4096)
if not piece:
break
body += piece
try:
expected = json.loads(body.decode("utf-8"))["preconditions"]["uid"]
except Exception:
connection.sendall(b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n")
connection.close(); server.close(); raise SystemExit(0)
kind = "pod" if "/pods/" in path else "networkpolicy" if "/networkpolicies/" in path else ""
item_path = state / f"{kind}.json"
log = pathlib.Path(os.environ["BB_TEST_PROXY_LOG"])
with log.open("a", encoding="utf-8") as stream:
stream.write(f"{method} {path} uid={expected}\n")
if not kind or not item_path.exists():
connection.sendall(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")
elif os.environ.get("BB_TEST_DELETE_REPLACE_KIND") == kind or os.environ.get("BB_TEST_FOREIGN_UID") == "1":
item = json.loads(item_path.read_text(encoding="utf-8"))
item["metadata"]["uid"] = "uid-external-replacement"
item["metadata"]["labels"]["platform.hyeonworks.com/source-proof-run"] = "foreign-run"
item_path.write_text(json.dumps(item), encoding="utf-8")
connection.sendall(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\n\r\n")
elif json.loads(item_path.read_text(encoding="utf-8"))["metadata"]["uid"] != expected:
connection.sendall(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\n\r\n")
elif os.environ.get("BB_TEST_DELETE_AMBIGUOUS_KIND") == kind:
item_path.unlink()
connection.close(); server.close(); raise SystemExit(0)
elif os.environ.get("BB_TEST_DELETE_READ_AMBIGUOUS_KIND") == kind:
(state / f"{kind}.delete-read-ambiguous").touch()
connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}")
elif os.environ.get("BB_TEST_DELETE_NEVER_DISAPPEARS_KIND") == kind:
connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}")
elif os.environ.get("BB_TEST_DELETE_TERMINATING_KIND") == kind:
pending_polls = int(os.environ.get("BB_TEST_DELETE_PENDING_POLLS", "1"))
if pending_polls < 1:
raise SystemExit(92)
(state / f"{kind}.delete-pending").write_text(str(pending_polls), encoding="utf-8")
connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}")
else:
item_path.unlink()
connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nContent-Type: application/json\r\n\r\n{\"status\":\"Success\"}")
connection.close()
server.close()
PY
cat >"$fixture_root/bin/kubectl" <<'KUBECTL'
#!/usr/bin/env bash
set -Eeuo pipefail
log_command() { printf '%q ' "$@" >>"${BB_TEST_LOG:?}"; printf '\n' >>"${BB_TEST_LOG:?}"; }
state_file() { printf '%s/%s.json\n' "${BB_TEST_STATE:?}" "$1"; }
render_object() {
local kind=$1 name=$2 run=$3 uid=$4 bad=${5:-} live_defaults=${6:-0} mutate=${7:-} policy_mutate=${8:-}
/usr/bin/python3 - "$kind" "$name" "$run" "$uid" "$bad" "$live_defaults" "$mutate" "$policy_mutate" <<'PY'
import json
import sys
kind, name, run, uid, bad, live_defaults, mutate, policy_mutate = sys.argv[1:]
labels = {
"app.kubernetes.io/managed-by": "platform-blackbox-edge-source",
"platform.hyeonworks.com/source-proof-run": run,
}
metadata = {"namespace": "observability", "name": name, "uid": uid, "labels": labels}
if kind == "networkpolicy":
spec = {
"podSelector": {"matchLabels": {"platform.hyeonworks.com/source-proof-run": run}},
"policyTypes": ["Egress"],
"egress": [
{"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, "podSelector": {"matchLabels": {"k8s-app": "kube-dns"}}}], "ports": [{"protocol": "UDP", "port": 53}, {"protocol": "TCP", "port": 53}]},
{"to": [{"ipBlock": {"cidr": "192.168.0.107/32"}}], "ports": [{"protocol": "TCP", "port": 443}]},
],
}
item = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": metadata, "spec": spec}
else:
spec = {
"automountServiceAccountToken": False,
"restartPolicy": "Never",
"securityContext": {"runAsNonRoot": True, "runAsUser": 65534, "runAsGroup": 65534, "seccompProfile": {"type": "RuntimeDefault"}},
"containers": [{"name": "probe", "image": "docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0", "imagePullPolicy": "IfNotPresent", "command": ["sh", "-c", "sleep 120"], "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}}],
}
item = {"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}
if live_defaults == "1":
spec.update({
"dnsPolicy": "ClusterFirst",
"enableServiceLinks": True,
"preemptionPolicy": "PreemptLowerPriority",
"priority": 0,
"schedulerName": "default-scheduler",
"serviceAccount": "default",
"serviceAccountName": "default",
"terminationGracePeriodSeconds": 30,
"tolerations": [
{"effect": "NoExecute", "key": "node.kubernetes.io/not-ready", "operator": "Exists", "tolerationSeconds": 300},
{"effect": "NoExecute", "key": "node.kubernetes.io/unreachable", "operator": "Exists", "tolerationSeconds": 300},
],
})
spec["containers"][0].update({
"resources": {},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
})
metadata.update({"creationTimestamp": "2026-08-12T17:20:51Z", "generation": 1})
item["status"] = {"phase": "Pending", "qosClass": "BestEffort"}
if mutate == "security":
spec["containers"][0]["securityContext"]["privileged"] = True
elif mutate == "command":
spec["containers"][0]["command"] = ["sh", "-c", "sleep 120; id"]
elif mutate == "image":
spec["containers"][0]["image"] = "docker.io/library/busybox:latest"
if (kind == "pod" and mutate == "label") or (kind == "networkpolicy" and policy_mutate == "label"):
labels["platform.hyeonworks.com/unowned"] = "unexpected"
if bad == kind:
item["metadata"]["labels"]["platform.hyeonworks.com/source-proof-run"] = "foreign-run"
print(json.dumps(item))
PY
}
read_manifest() {
local body kind name run
body="$(/bin/cat)"
kind="$(printf '%s\n' "$body" | /usr/bin/awk '/^kind: / { print $2; exit }')"
name="$(printf '%s\n' "$body" | /usr/bin/awk '/^ name: / { print $2; exit }')"
run="$(printf '%s\n' "$body" | /usr/bin/awk '/source-proof-run:/ { gsub(/"/, "", $2); print $2; exit }')"
case "$kind" in NetworkPolicy) printf 'networkpolicy|%s|%s\n' "$name" "$run" ;; Pod) printf 'pod|%s|%s\n' "$name" "$run" ;; *) exit 95 ;; esac
}
log_command "$@"
args=" $* "
if [[ "$args" == *' config current-context '* ]]; then printf 'fixture-context\n'; exit 0; fi
if [[ "$args" == *' get --raw=/readyz '* ]]; then printf 'ok\n'; exit 0; fi
if [[ "$args" == *' auth can-i '* ]]; then printf '%s\n' "${BB_TEST_AUTH:-yes}"; exit 0; fi
if [[ "$args" == *' create '* ]]; then
IFS='|' read -r kind name run <<<"$(read_manifest)"
mode=create
[[ "$args" == *' --dry-run=server '* ]] && mode=dry-run
printf '%s %s\n' "$kind" "$mode" >>"${BB_TEST_CREATE_LOG:?}"
uid="uid-${kind}-${run}"
bad="${BB_TEST_BAD_SPEC_KIND:-}"
if [[ "$mode" == dry-run ]]; then render_object "$kind" "$name" "$run" dry-run "$bad" "${BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN:-0}" "${BB_TEST_MUTATE_POD_FIELD:-}" "${BB_TEST_MUTATE_NETWORKPOLICY_FIELD:-}"; exit 0; fi
path="$(state_file "$kind")"
[[ ! -e "$path" && ! -L "$path" ]] || exit 1
render_object "$kind" "$name" "$run" "$uid" "$bad" >"$path"
if [[ "${BB_TEST_CREATE_AMBIGUOUS_KIND:-}" == "$kind" || "${BB_TEST_CREATE_FAILURE_KIND:-}" == "$kind" ]]; then exit 1; fi
if [[ "${BB_TEST_BAD_RESPONSE_KIND:-}" == "$kind" ]]; then
render_object "$kind" "$name" "$run" "$uid" "$kind"
exit 0
fi
/bin/cat -- "$path"
exit 0
fi
if [[ "$args" == *' get pod '* || "$args" == *' get networkpolicy '* ]]; then
if [[ "$args" =~ get\ (pod|networkpolicy)\ ([^[:space:]]+) ]]; then kind=${BASH_REMATCH[1]}; name=${BASH_REMATCH[2]}; else exit 96; fi
path="$(state_file "$kind")"
[[ ! -e "${BB_TEST_STATE:?}/${kind}.delete-read-ambiguous" ]] || exit 1
if [[ -e "$path" && ! -L "$path" ]]; then
if [[ "${BB_TEST_FOREIGN_UID:-0}" == 1 ]]; then
/usr/bin/python3 - "$path" <<'PY'
import json, pathlib, sys
p = pathlib.Path(sys.argv[1]); item = json.loads(p.read_text()); item["metadata"]["uid"] = "foreign-uid"; item["metadata"]["labels"]["platform.hyeonworks.com/source-proof-run"] = "foreign-run"; p.write_text(json.dumps(item))
PY
fi
if [[ -e "${BB_TEST_STATE:?}/${kind}.delete-pending" ]]; then
remaining="$(<"${BB_TEST_STATE:?}/${kind}.delete-pending")"
[[ "$remaining" =~ ^[1-9][0-9]*$ ]] || exit 99
remaining=$((remaining - 1))
/bin/cat -- "$path"
if (( remaining == 0 )); then
/bin/rm -f -- "${BB_TEST_STATE:?}/${kind}.delete-pending" "$path"
else
printf '%s\n' "$remaining" >"${BB_TEST_STATE:?}/${kind}.delete-pending"
fi
exit 0
fi
/bin/cat -- "$path"; exit 0
fi
[[ "$args" == *'--ignore-not-found'* ]] && exit 0
exit 1
fi
if [[ "$args" == *' wait --for=condition=Ready '* && "$args" == *' pod/'* ]]; then
if [[ -n "${BB_TEST_SCHEDULED_POD_NODE:-}" && -e "$(state_file pod)" ]]; then
/usr/bin/python3 - "$(state_file pod)" "${BB_TEST_SCHEDULED_POD_NODE}" <<'PY'
import json, pathlib, sys
p = pathlib.Path(sys.argv[1]); item = json.loads(p.read_text()); item["spec"]["nodeName"] = sys.argv[2]; p.write_text(json.dumps(item))
PY
fi
[[ -e "$(state_file pod)" ]]; exit $?
fi
if [[ "$args" == *' exec '* ]]; then
[[ "${BB_TEST_EXEC_SLEEP:-0}" == 1 ]] && /usr/bin/sleep 20
case "${BB_TEST_CLIENT_STATUS:-403}" in
403) printf 'HTTP/1.1 403 Forbidden\n' >&2 ;;
200) printf 'HTTP/1.1 200 OK\n' >&2 ;;
302) printf 'HTTP/1.1 302 Found\n' >&2 ;;
*) printf 'HTTP/1.1 500 Error\n' >&2 ;;
esac
exit "${BB_TEST_CLIENT_RC:-1}"
fi
if [[ "$args" == *' proxy '* ]]; then
socket=''
for argument in "$@"; do [[ "$argument" == --unix-socket=* ]] && socket=${argument#--unix-socket=}; done
[[ "$socket" == /tmp/platform-blackbox-edge-source.*/*.sock ]] || exit 97
exec /usr/bin/python3 "${BB_TEST_PROXY_HELPER:?}" "$socket" "${BB_TEST_STATE:?}"
fi
if [[ "$args" == *' delete '* || "$args" == *' apply '* || "$args" == *' replace '* || "$args" == *' patch '* ]]; then exit 98; fi
exit 0
KUBECTL
cat >"$fixture_root/bin/sudo" <<'SUDO'
#!/usr/bin/env bash
set -Eeuo pipefail
while [[ "${1:-}" == -n || "${1:-}" == -- ]]; do shift; done
command_path=${1:-}
[[ "$command_path" == /* ]] || { printf 'non-absolute sudo child: %s\n' "$command_path" >&2; exit 97; }
printf '%q ' "$@" >>"${BB_TEST_ROOT_LOG:?}"; printf '\n' >>"${BB_TEST_ROOT_LOG:?}"
if [[ "$command_path" == /usr/bin/awk ]]; then
printf '%q ' "$@" >>"${BB_TEST_SUDO_LOG:?}"; printf '\n' >>"${BB_TEST_SUDO_LOG:?}"
case "${BB_TEST_LOG_MATCHES:-1}" in
0) exit 0 ;;
1) printf '10.42.0.55 403\n10.42.0.55 403\n10.42.0.55 403\n' ;;
2) printf '10.42.0.55 403\n10.42.0.55 403\n10.42.0.55 403\n10.42.0.55 403\n' ;;
*) exit 98 ;;
esac
exit 0
fi
if [[ "$command_path" == /usr/bin/install ]]; then
shift
filtered=()
while (( $# )); do case "$1" in -o|-g) shift 2 ;; *) filtered+=("$1"); shift ;; esac; done
exec /usr/bin/install "${filtered[@]}"
fi
if [[ "$command_path" == /bin/ln && "${BB_TEST_PUBLISH_RACE:-0}" == 1 ]]; then
/bin/mkdir -p -- "${BB_TEST_PROOF:?}.race-dir"
/bin/ln -s -- "${BB_TEST_PROOF}.race-dir" "${BB_TEST_PROOF:?}"
fi
exec "$@"
SUDO
cat >"$fixture_root/bin/date" <<'DATE'
#!/usr/bin/env bash
set -Eeuo pipefail
if [[ ( "${BB_TEST_POST_PROMPT_ACTIVE_DRIFT:-0}" == 1 || "${BB_TEST_POST_PROMPT_PROOF_APPEARS:-0}" == 1 ) && ! -e "${BB_TEST_POST_PROMPT_MARKER:?}" ]]; then
: >"${BB_TEST_POST_PROMPT_MARKER:?}"
[[ "${BB_TEST_POST_PROMPT_ACTIVE_DRIFT:-0}" == 1 ]] && printf 'active drift\n' >"${BB_TEST_ACTIVE:?}"
[[ "${BB_TEST_POST_PROMPT_PROOF_APPEARS:-0}" == 1 ]] && printf 'unsafe existing proof\n' >"${BB_TEST_PROOF:?}"
fi
if [[ "$*" == *'+%Y-%m-%dT%H:%M:%SZ'* ]]; then printf '2026-08-12T00:00:00Z\n'; else /usr/bin/date "$@"; fi
DATE
cat >"$fixture_root/bin/sleep" <<'SLEEP'
#!/usr/bin/env bash
set -Eeuo pipefail
[[ "$#" == 1 && "$1" == 0.05 ]] || exit 97
exec /usr/bin/sleep 0.001
SLEEP
chmod 0700 "$fixture_root/bin"/*
proof="$fixture_root/rollbacks/observability-20260812T000000Z/blackbox-source-proof.env"
run_validator() {
local output_file="$fixture_root/out" rc=0
/bin/rm -f -- "$fixture_root/state"/* "$fixture_root/commands.log" "$fixture_root/creates.log" \
"$fixture_root/deletes.log" "$fixture_root/sudo.log" "$fixture_root/root.log" \
"$fixture_root/proxy.log" "$fixture_root/post-prompt"
: >"$fixture_root/commands.log"; : >"$fixture_root/creates.log"; : >"$fixture_root/deletes.log"
: >"$fixture_root/sudo.log"; : >"$fixture_root/root.log"; : >"$fixture_root/proxy.log"
PLATFORM_BLACKBOX_EDGE_TEST_MODE=1 \
PLATFORM_BLACKBOX_EDGE_KUBECTL="$fixture_root/bin/kubectl" \
PLATFORM_BLACKBOX_EDGE_SUDO="$fixture_root/bin/sudo" \
PLATFORM_BLACKBOX_EDGE_DATE="$fixture_root/bin/date" \
PLATFORM_BLACKBOX_EDGE_SLEEP="$fixture_root/bin/sleep" \
PLATFORM_BLACKBOX_EDGE_ACTIVE="$fixture_root/active/learn-services" \
PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE="$fixture_root/rollbacks" \
PLATFORM_BLACKBOX_EDGE_CONFIRMATION='PROVE BLACKBOX PRIVATE EDGE fixture-context' \
PLATFORM_OBSERVABILITY_ROLLBACK_ID=20260812T000000Z \
BB_TEST_LOG="$fixture_root/commands.log" BB_TEST_CREATE_LOG="$fixture_root/creates.log" \
BB_TEST_DELETE_LOG="$fixture_root/deletes.log" BB_TEST_SUDO_LOG="$fixture_root/sudo.log" \
BB_TEST_ROOT_LOG="$fixture_root/root.log" BB_TEST_PROXY_LOG="$fixture_root/proxy.log" \
BB_TEST_PROXY_HELPER="$fixture_root/bin/kubectl-proxy.py" BB_TEST_STATE="$fixture_root/state" \
BB_TEST_TMP="$fixture_root/state" BB_TEST_ACTIVE="$fixture_root/active/learn-services" \
BB_TEST_PROOF="$proof" BB_TEST_POST_PROMPT_MARKER="$fixture_root/post-prompt" \
BB_TEST_SCHEDULED_POD_NODE="${BB_TEST_SCHEDULED_POD_NODE:-}" \
BB_TEST_DELETE_TERMINATING_KIND="${BB_TEST_DELETE_TERMINATING_KIND:-}" \
BB_TEST_DELETE_PENDING_POLLS="${BB_TEST_DELETE_PENDING_POLLS:-1}" \
BB_TEST_DELETE_READ_AMBIGUOUS_KIND="${BB_TEST_DELETE_READ_AMBIGUOUS_KIND:-}" \
BB_TEST_DELETE_NEVER_DISAPPEARS_KIND="${BB_TEST_DELETE_NEVER_DISAPPEARS_KIND:-}" \
bash "$VALIDATOR" --execute --context fixture-context >"$output_file" 2>&1 || rc=$?
RUN_OUTPUT="$(<"$output_file")"; RUN_RC=$rc
}
# Production must reject escape hatches before a dry-run can contact a fake.
override_rc=0
PLATFORM_BLACKBOX_EDGE_ACTIVE=/tmp/override bash "$VALIDATOR" >"$fixture_root/override.out" 2>&1 || override_rc=$?
assert_eq 1 "$override_rc" 'production override boundary'
# Dry-run is read-only and contains neither token nor log payload.
: >"$fixture_root/commands.log"
dry_output="$(PLATFORM_BLACKBOX_EDGE_TEST_MODE=1 PLATFORM_BLACKBOX_EDGE_KUBECTL="$fixture_root/bin/kubectl" PLATFORM_BLACKBOX_EDGE_SUDO="$fixture_root/bin/sudo" PLATFORM_BLACKBOX_EDGE_DATE="$fixture_root/bin/date" PLATFORM_BLACKBOX_EDGE_SLEEP="$fixture_root/bin/sleep" PLATFORM_BLACKBOX_EDGE_ACTIVE="$fixture_root/active/learn-services" PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE="$fixture_root/rollbacks" BB_TEST_LOG="$fixture_root/commands.log" bash "$VALIDATOR")"
assert_contains "$dry_output" 'BLACKBOX_PRIVATE_EDGE_SOURCE_DRY_RUN=PASS' 'dry-run result label'
[[ ! -s "$fixture_root/commands.log" ]] || fail 'dry-run contacted Kubernetes'
assert_not_contains "$dry_output" 'hyeonworks_probe=' 'dry-run disclosed a token'
# Bad active SHA and authorization fail before any create.
printf 'wrong active source\n' >"$fixture_root/active/learn-services"
run_validator
assert_eq 1 "$RUN_RC" 'wrong active guard must use fixed failure status'
assert_no_object_create "$(<"$fixture_root/commands.log")" 'wrong active guard created an object'
cp -- "$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" "$fixture_root/active/learn-services"
BB_TEST_AUTH=no run_validator
assert_eq 1 "$RUN_RC" 'authorization denial must use fixed failure status'
assert_no_object_create "$(<"$fixture_root/commands.log")" 'authorization denial created an object'
# RED 1: BusyBox wget returns 1 for HTTP 403. The pinned client must pass only
# for exactly that result and an exact final parsed 403—not GNU wget's rc 8.
# RED 1a: Kubernetes server-side Pod dry-run adds its default fields. They must
# not make the validator reject an otherwise exact restricted probe Pod.
BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN=1 BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=1 run_validator
assert_eq 0 "$RUN_RC" 'live-defaulted Pod server dry-run must pass'
/bin/rm -f -- "$proof"
BB_TEST_MUTATE_NETWORKPOLICY_FIELD=label run_validator
assert_eq 1 "$RUN_RC" 'NetworkPolicy extra metadata label must fail closed'
unset BB_TEST_MUTATE_NETWORKPOLICY_FIELD
for mutated_field in label security command image; do
BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN=1 BB_TEST_MUTATE_POD_FIELD="$mutated_field" run_validator
assert_eq 1 "$RUN_RC" "live-defaulted Pod $mutated_field mutation must fail closed"
done
unset BB_TEST_LIVE_DEFAULTED_POD_DRY_RUN BB_TEST_MUTATE_POD_FIELD
BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=1 run_validator
assert_eq 0 "$RUN_RC" 'BusyBox 403 exit 1 with final parsed 403 must pass'
assert_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'BusyBox success label'
assert_not_contains "$RUN_OUTPUT" 'hyeonworks_probe=' 'BusyBox success disclosed token'
assert_eq 1 "$(wc -l <"$fixture_root/sudo.log")" 'exactly one sudo awk invocation'
assert_eq $'networkpolicy dry-run\nnetworkpolicy create\npod dry-run\npod create' "$(<"$fixture_root/creates.log")" 'create-only safe order'
assert_contains "$(<"$fixture_root/commands.log")" ' proxy ' 'cleanup must use bounded raw-delete proxy'
assert_no_name_delete "$(<"$fixture_root/commands.log")" 'cleanup must never name-delete'
assert_contains "$(<"$fixture_root/root.log")" '/usr/bin/mktemp' 'proof must stage under root mktemp'
assert_contains "$(<"$fixture_root/root.log")" "--tmpdir=$fixture_root/rollbacks/observability-20260812T000000Z" 'proof staging directory'
assert_contains "$(<"$fixture_root/root.log")" '/bin/ln -nT --' 'proof publication must not dereference a raced destination'
assert_eq $'schema=platform-blackbox-source-v1\nrollback_id=20260812T000000Z\nnginx_sha256='"$(sha256sum "$fixture_root/active/learn-services" | awk '{print $1}')"$'\ntested_at_utc=2026-08-12T00:00:00Z\ngrafana_remote_addr=10.42.0.55\ngrafana_status=403\nstorage_admin_remote_addr=10.42.0.55\nstorage_admin_status=403\ndb_admin_remote_addr=10.42.0.55\ndb_admin_status=403' "$(<"$proof")" 'exact proof schema and order'
[[ "$(stat -c %a "$proof")" == 600 && "$(stat -c %h "$proof")" == 1 ]] || fail 'proof must be unlinked 0600 evidence'
/bin/rm -f -- "$proof"
BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=0 run_validator
assert_eq 1 "$RUN_RC" '403 with non-BusyBox success rc must fail'
BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=8 run_validator
assert_eq 1 "$RUN_RC" '403 with GNU-wget rc must fail'
BB_TEST_CLIENT_STATUS=200 BB_TEST_CLIENT_RC=1 run_validator
assert_eq 1 "$RUN_RC" '200 source masquerade must fail'
assert_not_contains "$RUN_OUTPUT" 'hyeonworks_probe=' 'masquerade failure disclosed token'
# RED 2: values that drift after confirmation but before the first create are
# rejected by a repeated commit gate.
BB_TEST_POST_PROMPT_ACTIVE_DRIFT=1 run_validator
assert_eq 1 "$RUN_RC" 'post-prompt active SHA drift must fail'
assert_no_object_create "$(<"$fixture_root/commands.log")" 'active drift reached create'
cp -- "$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" "$fixture_root/active/learn-services"
/bin/rm -f -- "$proof"
BB_TEST_POST_PROMPT_PROOF_APPEARS=1 run_validator
assert_eq 1 "$RUN_RC" 'post-prompt proof appearance must fail'
assert_no_object_create "$(<"$fixture_root/commands.log")" 'proof appearance reached create'
/bin/rm -f -- "$proof"
# RED 3/4: a nonzero create may have committed. Exact owned state is cleaned
# through UID-preconditioned raw DELETE; never apply/update or name-delete.
BB_TEST_CREATE_AMBIGUOUS_KIND=networkpolicy run_validator
assert_eq 1 "$RUN_RC" 'ambiguous NetworkPolicy create must fail closed'
[[ ! -e "$fixture_root/state/networkpolicy.json" && ! -e "$fixture_root/state/pod.json" ]] || fail 'owned ambiguous NetworkPolicy was not cleaned'
assert_contains "$(<"$fixture_root/proxy.log")" '/networkpolicies/' 'NetworkPolicy raw UID delete'
BB_TEST_CREATE_AMBIGUOUS_KIND=pod run_validator
assert_eq 1 "$RUN_RC" 'ambiguous Pod create must fail closed'
[[ ! -e "$fixture_root/state/networkpolicy.json" && ! -e "$fixture_root/state/pod.json" ]] || fail 'owned partial create was not cleaned'
assert_contains "$(<"$fixture_root/proxy.log")" '/networkpolicies/' 'partial cleanup NetworkPolicy raw UID delete'
assert_contains "$(<"$fixture_root/proxy.log")" '/pods/' 'partial cleanup Pod raw UID delete'
BB_TEST_BAD_RESPONSE_KIND=pod run_validator
assert_eq 1 "$RUN_RC" 'malformed successful Pod create response must reclassify and fail closed'
[[ ! -e "$fixture_root/state/networkpolicy.json" && ! -e "$fixture_root/state/pod.json" ]] || fail 'owned malformed-response objects were not cleaned'
# RED 4a: catch a cleanup wait that gives up while a normally terminating,
# exact-owned scheduled Pod remains present beyond 640 cleanup observations,
# then disappears while still requiring two consecutive absence reads.
export BB_TEST_SCHEDULED_POD_NODE=donghyeon-system-product-name \
BB_TEST_DELETE_TERMINATING_KIND=pod BB_TEST_DELETE_PENDING_POLLS=700
run_validator
assert_eq 0 "$RUN_RC" 'expected scheduled Pod node during terminating cleanup must pass'
/bin/rm -f -- "$proof"
export BB_TEST_SCHEDULED_POD_NODE=foreign-node BB_TEST_DELETE_TERMINATING_KIND=pod \
BB_TEST_DELETE_PENDING_POLLS=2
run_validator
assert_eq 1 "$RUN_RC" 'foreign scheduled Pod node must fail closed'
assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'foreign scheduled Pod node became proof-command PASS'
[[ -e "$fixture_root/state/pod.json" ]] || fail 'foreign scheduled Pod disappeared'
unset BB_TEST_SCHEDULED_POD_NODE BB_TEST_DELETE_TERMINATING_KIND BB_TEST_DELETE_PENDING_POLLS
/bin/rm -f -- "$proof"
# Cleanup GET ambiguity must fail closed with the exact-owned object retained.
BB_TEST_DELETE_READ_AMBIGUOUS_KIND=pod run_validator
assert_eq 1 "$RUN_RC" 'cleanup API/read ambiguity must fail closed'
assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'cleanup API/read ambiguity became proof-command PASS'
[[ -e "$fixture_root/state/pod.json" ]] || fail 'read-ambiguous exact-owned Pod disappeared'
/bin/rm -f -- "$proof"
# An exact-owned object that never disappears must exhaust the bounded wait,
# fail cleanup, and never become a proof-command PASS.
BB_TEST_DELETE_NEVER_DISAPPEARS_KIND=pod run_validator
assert_eq 1 "$RUN_RC" 'never-disappearing exact-owned Pod must fail bounded cleanup'
assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'never-disappearing exact-owned Pod became proof-command PASS'
[[ -e "$fixture_root/state/pod.json" ]] || fail 'never-disappearing exact-owned Pod disappeared'
/bin/rm -f -- "$proof"
# A transport-ambiguous raw delete is accepted only after bounded stable
# absence; a UID/label replacement remains untouched and fails safely.
BB_TEST_DELETE_AMBIGUOUS_KIND=pod run_validator
assert_eq 0 "$RUN_RC" 'ambiguous raw delete with stable absence must pass'
/bin/rm -f -- "$proof"
BB_TEST_DELETE_REPLACE_KIND=pod run_validator
assert_eq 1 "$RUN_RC" 'replacement during raw delete must fail safely'
[[ -e "$fixture_root/state/pod.json" ]] || fail 'replacement was deleted'
/bin/rm -f -- "$proof" "$fixture_root/state/pod.json" "$fixture_root/state/networkpolicy.json"
# RED 5: a proof target that becomes a directory symlink only at publication
# must fail without allowing ln to create a hard link below that foreign path.
BB_TEST_PUBLISH_RACE=1 run_validator
assert_eq 1 "$RUN_RC" 'publication symlink race must fail safely'
[[ -L "$proof" ]] || fail 'publication race did not preserve the unsafe target'
[[ ! -e "$proof.race-dir"/* && ! -L "$proof.race-dir"/* ]] || fail 'publication followed a raced destination symlink'
/bin/rm -f -- "$proof"; /bin/rm -rf -- "$proof.race-dir"
# Existing proof symlink/hardlink and any rollback-root symlink ancestor are
# unsafe. The validator must leave all temporary objects uncreated.
ln -s /dev/null "$proof"
run_validator
assert_eq 1 "$RUN_RC" 'symlink proof target must fail safely'
/bin/rm -f -- "$proof"; printf 'existing\n' >"$proof"; ln "$proof" "$proof.link"
run_validator
assert_eq 1 "$RUN_RC" 'hard-linked proof target must fail safely'
/bin/rm -f -- "$proof" "$proof.link"
/bin/mv -- "$fixture_root/rollbacks" "$fixture_root/rollbacks-real"
/bin/ln -s -- "$fixture_root/rollbacks-real" "$fixture_root/rollbacks"
run_validator
assert_eq 1 "$RUN_RC" 'rollback root symlink ancestor must fail safely'
assert_no_object_create "$(<"$fixture_root/commands.log")" 'symlink ancestor reached create'
/bin/rm -f -- "$fixture_root/rollbacks"; /bin/mv -- "$fixture_root/rollbacks-real" "$fixture_root/rollbacks"
# A foreign object is never deleted, including on a signal path.
BB_TEST_FOREIGN_UID=1 run_validator
assert_eq 1 "$RUN_RC" 'foreign UID must fail safely'
assert_not_contains "$RUN_OUTPUT" 'BLACKBOX PRIVATE EDGE SOURCE PASS' 'foreign UID became proof-command PASS'
[[ -e "$fixture_root/state/networkpolicy.json" || -e "$fixture_root/state/pod.json" ]] || fail 'foreign object disappeared'
/bin/rm -f -- "$fixture_root/state"/*.json "$proof"
: >"$fixture_root/commands.log"; : >"$fixture_root/creates.log"; : >"$fixture_root/root.log"; : >"$fixture_root/proxy.log"
setsid env \
PLATFORM_BLACKBOX_EDGE_TEST_MODE=1 \
PLATFORM_BLACKBOX_EDGE_KUBECTL="$fixture_root/bin/kubectl" \
PLATFORM_BLACKBOX_EDGE_SUDO="$fixture_root/bin/sudo" \
PLATFORM_BLACKBOX_EDGE_DATE="$fixture_root/bin/date" \
PLATFORM_BLACKBOX_EDGE_SLEEP="$fixture_root/bin/sleep" \
PLATFORM_BLACKBOX_EDGE_ACTIVE="$fixture_root/active/learn-services" \
PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE="$fixture_root/rollbacks" \
PLATFORM_BLACKBOX_EDGE_CONFIRMATION='PROVE BLACKBOX PRIVATE EDGE fixture-context' \
PLATFORM_OBSERVABILITY_ROLLBACK_ID=20260812T000000Z \
BB_TEST_LOG="$fixture_root/commands.log" BB_TEST_CREATE_LOG="$fixture_root/creates.log" \
BB_TEST_DELETE_LOG="$fixture_root/deletes.log" BB_TEST_SUDO_LOG="$fixture_root/sudo.log" \
BB_TEST_ROOT_LOG="$fixture_root/root.log" BB_TEST_PROXY_LOG="$fixture_root/proxy.log" \
BB_TEST_PROXY_HELPER="$fixture_root/bin/kubectl-proxy.py" BB_TEST_STATE="$fixture_root/state" \
BB_TEST_TMP="$fixture_root/state" BB_TEST_ACTIVE="$fixture_root/active/learn-services" \
BB_TEST_PROOF="$proof" BB_TEST_POST_PROMPT_MARKER="$fixture_root/post-prompt" \
BB_TEST_CLIENT_STATUS=403 BB_TEST_CLIENT_RC=1 BB_TEST_EXEC_SLEEP=1 \
bash "$VALIDATOR" --execute --context fixture-context >"$fixture_root/signal.out" 2>&1 &
signal_pid=$!
for ((attempt = 0; attempt < 100; attempt++)); do
[[ "$(<"$fixture_root/commands.log")" == *' exec '* ]] && break
/usr/bin/sleep 0.02
done
[[ "$(<"$fixture_root/commands.log")" == *' exec '* ]] || fail 'signal fixture did not reach client request'
/bin/kill -TERM -- "-$signal_pid"
signal_rc=0; wait "$signal_pid" || signal_rc=$?
assert_eq 143 "$signal_rc" 'SIGTERM must retain fixed status'
assert_contains "$(<"$fixture_root/proxy.log")" '/pods/' 'signal raw Pod cleanup'
assert_contains "$(<"$fixture_root/proxy.log")" '/networkpolicies/' 'signal raw NetworkPolicy cleanup'
printf 'BLACKBOX EDGE SOURCE PRODUCTION-BOUNDARY TEST PASS\n'
@@ -0,0 +1,728 @@
#!/usr/bin/env bash
# Prove that a restricted in-cluster source is denied by the active Grafana
# Nginx guard. This is deliberately a short-lived prerequisite transaction,
# not a reusable HTTP probing tool.
set -Eeuo pipefail
set +x
umask 077
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
readonly DENY_GUARD="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf"
readonly NAMESPACE=observability
readonly ROLLBACK_BASE_DEFAULT=/var/lib/hyeonworks/platform-rollbacks
readonly ACTIVE_DEFAULT=/etc/nginx/sites-available/learn-services
readonly KUBECTL_DEFAULT=/usr/local/bin/kubectl
readonly SUDO_DEFAULT=/usr/bin/sudo
readonly DATE_DEFAULT=/usr/bin/date
readonly SLEEP_DEFAULT=/usr/bin/sleep
readonly AWK_BIN=/usr/bin/awk
readonly CHMOD_BIN=/usr/bin/chmod
readonly CMP_BIN=/usr/bin/cmp
readonly CURL_BIN=/usr/bin/curl
readonly ID_BIN=/usr/bin/id
readonly INSTALL_BIN=/usr/bin/install
readonly JQ_BIN=/usr/bin/jq
readonly KILL_BIN=/bin/kill
readonly LN_BIN=/bin/ln
readonly MKTEMP_BIN=/usr/bin/mktemp
readonly REALPATH_BIN=/usr/bin/realpath
readonly RM_BIN=/bin/rm
readonly SHA256SUM_BIN=/usr/bin/sha256sum
readonly STAT_BIN=/usr/bin/stat
readonly TEST_BIN=/usr/bin/test
readonly TRUE_BIN=/usr/bin/true
readonly UNLINK_BIN=/usr/bin/unlink
readonly PROOF_SCHEMA=platform-blackbox-source-v1
readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0'
readonly EXPECTED_NODE='donghyeon-system-product-name'
readonly REQUEST_TIMEOUT=10s
readonly READY_TIMEOUT=30s
readonly DELETE_WAIT_POLLS=1200
readonly PROXY_WAIT_POLLS=50
readonly GRAFANA_HOST=grafana.learn.hyeonworks.com
readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com
readonly DB_HOST=db-admin.learn.hyeonworks.com
readonly TEST_MODE="${PLATFORM_BLACKBOX_EDGE_TEST_MODE:-0}"
readonly KUBECTL="${PLATFORM_BLACKBOX_EDGE_KUBECTL:-$KUBECTL_DEFAULT}"
readonly SUDO="${PLATFORM_BLACKBOX_EDGE_SUDO:-$SUDO_DEFAULT}"
readonly DATE_BIN="${PLATFORM_BLACKBOX_EDGE_DATE:-$DATE_DEFAULT}"
readonly SLEEP_BIN="${PLATFORM_BLACKBOX_EDGE_SLEEP:-$SLEEP_DEFAULT}"
readonly ACTIVE="${PLATFORM_BLACKBOX_EDGE_ACTIVE:-$ACTIVE_DEFAULT}"
readonly ROLLBACK_BASE="${PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE:-$ROLLBACK_BASE_DEFAULT}"
execute=false
context=''
fixture_root=''
rollback_anchor=''
rollback_root=''
proof_path=''
run_temp=''
run_label=''
pod_name=''
policy_name=''
pod_uid=''
policy_uid=''
cleanup_started=false
declare -a LOG_RESULTS=()
fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
usage() {
cat <<'USAGE'
Usage:
bash scripts/validate/validate-blackbox-edge-source.sh
bash scripts/validate/validate-blackbox-edge-source.sh --execute --context CONTEXT
The default is read-only. Execute requires an exact context-qualified
confirmation, creates only a temporary restricted NetworkPolicy and Pod, and
never prints request tokens or Nginx log lines.
USAGE
}
sha_of() { "$SHA256SUM_BIN" -- "$1" | "$AWK_BIN" '{print $1}'; }
is_sha256() { [[ "$1" =~ ^[0-9a-f]{64}$ ]]; }
require_regular_source() { [[ -f "$1" && ! -L "$1" ]] || fail "unsafe or missing source: $1"; }
safe_fixture_command() {
local path=$1 fixture_bin=$2 resolved metadata type owner mode mode_value
[[ "$path" != /usr/bin/sudo ]] || return 1
resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$path" 2>/dev/null)" || return 1
[[ "$resolved" == "$path" && "${path%/*}" == "$fixture_bin" && -f "$path" && ! -L "$path" && -x "$path" ]] || return 1
metadata="$("$STAT_BIN" -c '%F|%u|%a' -- "$path")" || return 1
IFS='|' read -r type owner mode <<<"$metadata"
mode_value=$((8#$mode))
[[ "$type" == 'regular file' && "$owner" == "$EUID" && $((mode_value & 0022)) == 0 && $((mode_value & 07000)) == 0 ]]
}
validate_test_mode() {
local active_parent resolved key
[[ "$TEST_MODE" == 0 || "$TEST_MODE" == 1 ]] || { fail 'invalid test-mode flag'; return 1; }
if [[ "$TEST_MODE" == 0 ]]; then
for key in ${!PLATFORM_BLACKBOX_EDGE_@}; do
fail "production rejects Blackbox edge override: $key"
return 1
done
[[ "$KUBECTL" == "$KUBECTL_DEFAULT" && "$SUDO" == "$SUDO_DEFAULT" && "$DATE_BIN" == "$DATE_DEFAULT" && "$SLEEP_BIN" == "$SLEEP_DEFAULT" && "$ACTIVE" == "$ACTIVE_DEFAULT" && "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || {
fail 'production Blackbox edge constants are not exact'; return 1;
}
rollback_anchor=/
return 0
fi
active_parent=${ACTIVE%/*}
resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$active_parent/.." 2>/dev/null)" || {
fail 'test fixture root does not resolve'; return 1;
}
fixture_root=$resolved
[[ "$fixture_root" == /tmp/platform-blackbox-edge-test.* && "$ACTIVE" == "$fixture_root/active/learn-services" && "$ROLLBACK_BASE" == "$fixture_root/rollbacks" && -d "$fixture_root/bin" && ! -L "$fixture_root/bin" ]] || {
fail 'test mode is restricted to one isolated fixture tree'; return 1;
}
safe_fixture_command "$KUBECTL" "$fixture_root/bin" &&
safe_fixture_command "$SUDO" "$fixture_root/bin" &&
safe_fixture_command "$DATE_BIN" "$fixture_root/bin" &&
safe_fixture_command "$SLEEP_BIN" "$fixture_root/bin" || {
fail 'test command boundary is unsafe'; return 1;
}
rollback_anchor=$fixture_root
}
parse_args() {
local execute_count=0 context_count=0
while (( $# > 0 )); do
case "$1" in
--execute) execute=true; execute_count=$((execute_count + 1)); shift ;;
--context)
(( $# >= 2 )) || { fail '--context requires a value'; return 1; }
context=$2; context_count=$((context_count + 1)); shift 2
;;
-h|--help) usage; exit 0 ;;
*) usage >&2; fail "unsupported argument: $1"; return 1 ;;
esac
done
(( execute_count <= 1 && context_count <= 1 )) || { fail 'duplicate execute argument'; return 1; }
if [[ "$execute" == true ]]; then
[[ "$context" =~ ^[A-Za-z0-9._:-]+$ ]] || { fail '--execute requires an exact --context'; return 1; }
else
[[ -z "$context" ]] || { fail '--context is execute-only'; return 1; }
fi
}
root_run() { "$SUDO" -n "$@"; }
root_metadata() { root_run "$STAT_BIN" -c '%F|%u:%g|%a|%h' -- "$1"; }
root_owner_expected() {
if [[ "$TEST_MODE" == 0 ]]; then [[ "$1" == 0:0 ]]; else [[ "$1" == "$EUID:$($ID_BIN -g)" ]]; fi
}
root_safe_ancestor_dir() {
local path=$1 metadata type owner mode links mode_value
root_run "$TEST_BIN" ! -L "$path" || return 1
root_run "$TEST_BIN" -d "$path" || return 1
metadata="$(root_metadata "$path")" || return 1
IFS='|' read -r type owner mode links <<<"$metadata"
mode_value=$((8#$mode))
[[ "$type" == directory && "$links" =~ ^[1-9][0-9]*$ ]] || return 1
root_owner_expected "$owner" || return 1
(( (mode_value & 0022) == 0 ))
}
root_safe_dir() {
local path=$1 expected_mode=$2 metadata type owner mode links
root_safe_ancestor_dir "$path" || return 1
metadata="$(root_metadata "$path")" || return 1
IFS='|' read -r type owner mode links <<<"$metadata"
[[ "$mode" == "$expected_mode" && "$links" =~ ^[1-9][0-9]*$ ]]
}
root_safe_file() {
local path=$1 expected_mode=$2 metadata type owner mode links
root_run "$TEST_BIN" ! -L "$path" || return 1
root_run "$TEST_BIN" -f "$path" || return 1
metadata="$(root_metadata "$path")" || return 1
IFS='|' read -r type owner mode links <<<"$metadata"
[[ ( "$type" == 'regular file' || "$type" == 'regular empty file' ) && "$mode" == "$expected_mode" && "$links" == 1 ]] || return 1
root_owner_expected "$owner"
}
validate_root_chain() {
local target=$1 anchor=$2 canonical relative current component
[[ "$target" == /* && "$anchor" == /* && "$target" != *'//' && "$target" != */./* && "$target" != */../* ]] || return 1
canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$target")" || return 1
[[ "$canonical" == "$target" && "$canonical" != *$'\n'* ]] || return 1
canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$anchor")" || return 1
[[ "$canonical" == "$anchor" && "$canonical" != *$'\n'* ]] || return 1
if [[ "$anchor" == / ]]; then
# Root has no separator to append: "$anchor/" would become "//" and
# reject every normal absolute descendant after canonicalization.
[[ "$target" == /* ]] || return 1
else
[[ "$target" == "$anchor" || "$target" == "$anchor/"* ]] || return 1
fi
current=$anchor
root_safe_ancestor_dir "$current" || return 1
relative=${target#"$anchor"}
relative=${relative#/}
[[ -z "$relative" ]] && return 0
IFS='/' read -r -a _blackbox_path_parts <<<"$relative"
for component in "${_blackbox_path_parts[@]}"; do
[[ -n "$component" && "$component" != . && "$component" != .. ]] || return 1
if [[ "$current" == / ]]; then current="/$component"; else current="$current/$component"; fi
root_safe_ancestor_dir "$current" || return 1
done
}
validate_rollback_root() {
local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-}
[[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || { fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid'; return 1; }
if [[ "$TEST_MODE" == 0 ]]; then [[ "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || { fail 'production rollback base is not exact'; return 1; }; fi
rollback_root="$ROLLBACK_BASE/observability-$id"
proof_path="$rollback_root/blackbox-source-proof.env"
validate_root_chain "$ROLLBACK_BASE" "$rollback_anchor" || { fail 'rollback base lineage is unsafe'; return 1; }
validate_root_chain "$rollback_root" "$rollback_anchor" || { fail 'rollback root lineage is unsafe'; return 1; }
root_safe_dir "$rollback_root" 700 || { fail 'rollback root metadata is unsafe'; return 1; }
}
validate_proof_target_absent() {
validate_rollback_root || return 1
[[ "$proof_path" == "$rollback_root/blackbox-source-proof.env" ]] || { fail 'source-proof path is not exact'; return 1; }
root_run "$TEST_BIN" ! -e "$proof_path" && root_run "$TEST_BIN" ! -L "$proof_path" || {
fail 'source-proof evidence already exists or is unsafe'; return 1;
}
}
validate_active_guard_expected() {
local expected=$1 deny_sha metadata type owner mode links active_sha
require_regular_source "$DENY_GUARD" || return 1
is_sha256 "$expected" || { fail 'expected deny guard hash is invalid'; return 1; }
deny_sha="$(sha_of "$DENY_GUARD")" || { fail 'cannot hash deny guard source'; return 1; }
[[ "$deny_sha" == "$expected" ]] || { fail 'deny guard source changed during proof'; return 1; }
root_run "$TEST_BIN" ! -L "$ACTIVE" && root_run "$TEST_BIN" -f "$ACTIVE" || {
fail 'active Nginx config is missing or symlinked'; return 1;
}
metadata="$(root_metadata "$ACTIVE")" || { fail 'cannot stat active Nginx config'; return 1; }
IFS='|' read -r type owner mode links <<<"$metadata"
[[ "$type" == 'regular file' && "$links" == 1 ]] || { fail 'active Nginx config metadata is unsafe'; return 1; }
root_owner_expected "$owner" || { fail 'active Nginx config owner is unsafe'; return 1; }
if [[ "$TEST_MODE" == 0 ]]; then [[ "$mode" == 644 ]] || { fail 'active Nginx config mode is unsafe'; return 1; }; fi
active_sha="$(root_run "$SHA256SUM_BIN" -- "$ACTIVE" | "$AWK_BIN" '{print $1}')" || {
fail 'cannot hash active Nginx config'; return 1;
}
[[ "$active_sha" == "$expected" ]] || { fail 'active Nginx config is not the exact Grafana deny guard'; return 1; }
}
validate_active_guard() {
local deny_sha
require_regular_source "$DENY_GUARD" || return 1
deny_sha="$(sha_of "$DENY_GUARD")" || return 1
is_sha256 "$deny_sha" || { fail 'deny guard hash is invalid'; return 1; }
validate_active_guard_expected "$deny_sha" || return 1
printf '%s\n' "$deny_sha"
}
confirm() {
local expected="PROVE BLACKBOX PRIVATE EDGE $context" answer
printf 'Type %s: ' "$expected" >&2
if [[ "$TEST_MODE" == 1 ]]; then
answer=${PLATFORM_BLACKBOX_EDGE_CONFIRMATION:-}
printf '%s\n' "$answer" >&2
else
[[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; }
IFS= read -r answer
fi
[[ "$answer" == "$expected" ]] || { fail 'cancelled'; return 1; }
}
require_context_api_auth() {
local current answer
current="$("$KUBECTL" --request-timeout="$REQUEST_TIMEOUT" config current-context)" || { fail 'cannot read Kubernetes context'; return 1; }
[[ "$current" == "$context" && "$current" != *$'\n'* ]] || { fail 'current Kubernetes context does not match --context'; return 1; }
"$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; }
for answer in \
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create pods -n "$NAMESPACE")" \
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create networkpolicies.networking.k8s.io -n "$NAMESPACE")" \
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete pods -n "$NAMESPACE")" \
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete networkpolicies.networking.k8s.io -n "$NAMESPACE")"; do
[[ "$answer" == yes ]] || { fail 'Kubernetes authorization is insufficient'; return 1; }
done
root_run "$TRUE_BIN" || { fail 'sudo non-interactive authentication failed'; return 1; }
}
prepare_local_temp() {
run_temp="$("$MKTEMP_BIN" -d /tmp/platform-blackbox-edge-source.XXXXXXXX)" || return 1
[[ "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 1
"$CHMOD_BIN" 0700 -- "$run_temp"
}
cleanup_local_temp() {
[[ -n "$run_temp" && "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 0
"$RM_BIN" -rf -- "$run_temp"
run_temp=''
}
render_network_policy() {
/bin/cat <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: $policy_name
namespace: $NAMESPACE
labels:
app.kubernetes.io/managed-by: platform-blackbox-edge-source
platform.hyeonworks.com/source-proof-run: "$run_label"
spec:
podSelector:
matchLabels:
platform.hyeonworks.com/source-proof-run: "$run_label"
policyTypes: ["Egress"]
egress:
- to:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: kube-system}
podSelector:
matchLabels: {k8s-app: kube-dns}
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
- to:
- ipBlock: {cidr: 192.168.0.107/32}
ports: [{protocol: TCP, port: 443}]
EOF
}
render_pod() {
/bin/cat <<EOF
apiVersion: v1
kind: Pod
metadata:
name: $pod_name
namespace: $NAMESPACE
labels:
app.kubernetes.io/managed-by: platform-blackbox-edge-source
platform.hyeonworks.com/source-proof-run: "$run_label"
spec:
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
seccompProfile: {type: RuntimeDefault}
containers:
- name: probe
image: $BUSYBOX_IMAGE
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "sleep 120"]
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: ["ALL"]}
EOF
}
validate_object_json() {
local kind=$1 name=$2 run=$3 source=$4 require_uid=$5 allow_scheduled_node=${6:-false} uid_test
uid_test='true'
[[ "$require_uid" == true ]] && uid_test='(.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))'
case "$kind" in
networkpolicy)
"$JQ_BIN" -e --arg namespace "$NAMESPACE" --arg name "$name" --arg run "$run" --argjson uid_required "$([[ "$require_uid" == true ]] && printf true || printf false)" '
.apiVersion == "networking.k8s.io/v1" and .kind == "NetworkPolicy" and
.metadata.namespace == $namespace and .metadata.name == $name and
.metadata.labels == {
"app.kubernetes.io/managed-by":"platform-blackbox-edge-source",
"platform.hyeonworks.com/source-proof-run":$run
} and
(($uid_required | not) or (.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))) and
.spec.podSelector == {"matchLabels":{"platform.hyeonworks.com/source-proof-run":$run}} and
.spec.policyTypes == ["Egress"] and
.spec.egress == [
{"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"kube-system"}},"podSelector":{"matchLabels":{"k8s-app":"kube-dns"}}}],"ports":[{"protocol":"UDP","port":53},{"protocol":"TCP","port":53}]},
{"to":[{"ipBlock":{"cidr":"192.168.0.107/32"}}],"ports":[{"protocol":"TCP","port":443}]}
]
' "$source" >/dev/null
;;
pod)
"$JQ_BIN" -e --arg namespace "$NAMESPACE" --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg node "$EXPECTED_NODE" --argjson uid_required "$([[ "$require_uid" == true ]] && printf true || printf false)" --argjson scheduled_node_allowed "$([[ "$allow_scheduled_node" == true ]] && printf true || printf false)" '
def optional_default($key; $value):
(has($key) | not) or .[$key] == $value;
.apiVersion == "v1" and .kind == "Pod" and
.metadata.namespace == $namespace and .metadata.name == $name and
.metadata.labels == {
"app.kubernetes.io/managed-by":"platform-blackbox-edge-source",
"platform.hyeonworks.com/source-proof-run":$run
} and
(($uid_required | not) or (.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))) and
(.spec |
type == "object" and
optional_default("dnsPolicy"; "ClusterFirst") and
optional_default("enableServiceLinks"; true) and
optional_default("preemptionPolicy"; "PreemptLowerPriority") and
optional_default("priority"; 0) and
optional_default("schedulerName"; "default-scheduler") and
optional_default("serviceAccount"; "default") and
optional_default("serviceAccountName"; "default") and
optional_default("terminationGracePeriodSeconds"; 30) and
((has("nodeName") | not) or ($scheduled_node_allowed and .nodeName == $node)) and
optional_default("tolerations"; [
{"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300},
{"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300}
]) and
(.containers | type == "array" and length == 1) and
(.containers[0] |
optional_default("resources"; {}) and
optional_default("terminationMessagePath"; "/dev/termination-log") and
optional_default("terminationMessagePolicy"; "File")) and
(del(
.dnsPolicy, .enableServiceLinks, .preemptionPolicy, .priority,
.schedulerName, .serviceAccount, .serviceAccountName,
.terminationGracePeriodSeconds, .tolerations, .nodeName,
.containers[0].resources, .containers[0].terminationMessagePath,
.containers[0].terminationMessagePolicy
) == {
"automountServiceAccountToken":false,
"restartPolicy":"Never",
"securityContext":{"runAsNonRoot":true,"runAsUser":65534,"runAsGroup":65534,"seccompProfile":{"type":"RuntimeDefault"}},
"containers":[{"name":"probe","image":$image,"imagePullPolicy":"IfNotPresent","command":["sh","-c","sleep 120"],"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}}]
})
)
' "$source" >/dev/null
;;
*) return 1 ;;
esac
}
response_uid() {
local source=$1 uid
uid="$("$JQ_BIN" -er '.metadata.uid' "$source")" || return 1
[[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
printf '%s\n' "$uid"
}
local_response_file() {
"$MKTEMP_BIN" "$run_temp/${1}.response.XXXXXXXX"
}
render_resource() {
case "$1" in networkpolicy) render_network_policy ;; pod) render_pod ;; *) return 1 ;; esac
}
resource_name() { case "$1" in networkpolicy) printf '%s\n' "$policy_name" ;; pod) printf '%s\n' "$pod_name" ;; *) return 1 ;; esac; }
server_dry_run_resource() {
local kind=$1 name response
name="$(resource_name "$kind")" || return 1
response="$(local_response_file "${kind}.dry")" || return 1
"$CHMOD_BIN" 0600 -- "$response"
if ! render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create --dry-run=server -f - -o json >"$response"; then
"$RM_BIN" -f -- "$response"; return 1
fi
validate_object_json "$kind" "$name" "$run_label" "$response" false || { "$RM_BIN" -f -- "$response"; return 1; }
"$RM_BIN" -f -- "$response"
}
inspect_owned_object() {
local kind=$1 name=$2 destination=$3 response uid rc=0
response="$(local_response_file "${kind}.inspect")" || return 3
"$CHMOD_BIN" 0600 -- "$response"
"$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$?
if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 3; fi
if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi
validate_object_json "$kind" "$name" "$run_label" "$response" true || { "$RM_BIN" -f -- "$response"; return 2; }
uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 2; }
"$RM_BIN" -f -- "$response"
printf -v "$destination" '%s' "$uid"
}
create_resource() {
local kind=$1 destination=$2 name response uid rc=0 classification
name="$(resource_name "$kind")" || return 1
response="$(local_response_file "${kind}.create")" || return 1
"$CHMOD_BIN" 0600 -- "$response"
if render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create -f - -o json >"$response"; then
if validate_object_json "$kind" "$name" "$run_label" "$response" true && uid="$(response_uid "$response")"; then
"$RM_BIN" -f -- "$response"
printf -v "$destination" '%s' "$uid"
return 0
fi
fi
"$RM_BIN" -f -- "$response"
# A syntactically bad success response is still ambiguous: the API may have
# committed the exact object, so classify the live state before cleanup.
if inspect_owned_object "$kind" "$name" "$destination"; then
return 2
else
classification=$?
[[ "$classification" == 1 ]] && return 1
return 1
fi
}
create_temporary_objects() {
local result
server_dry_run_resource networkpolicy || { fail 'temporary NetworkPolicy server dry-run failed'; return 1; }
if create_resource networkpolicy policy_uid; then :; else
result=$?
[[ "$result" == 2 ]] && fail 'temporary NetworkPolicy create outcome was ambiguous but owned'
[[ "$result" != 2 ]] && fail 'temporary NetworkPolicy create failed'
return 1
fi
server_dry_run_resource pod || { fail 'temporary Pod server dry-run failed'; return 1; }
if create_resource pod pod_uid; then :; else
result=$?
[[ "$result" == 2 ]] && fail 'temporary Pod create outcome was ambiguous but owned'
[[ "$result" != 2 ]] && fail 'temporary Pod create failed'
return 1
fi
}
stop_proxy() {
local pid=$1 attempt
"$KILL_BIN" -TERM "$pid" 2>/dev/null || true
for ((attempt=0; attempt<20; attempt++)); do
if ! "$KILL_BIN" -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi
"$SLEEP_BIN" 0.05
done
"$KILL_BIN" -KILL "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
}
api_path_for() {
case "$1" in
pod) printf '/api/v1/namespaces/%s/pods/%s\n' "$NAMESPACE" "$2" ;;
networkpolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s\n' "$NAMESPACE" "$2" ;;
*) return 1 ;;
esac
}
object_state() {
local kind=$1 name=$2 expected_uid=$3 response uid rc=0
response="$(local_response_file "${kind}.state")" || return 4
"$CHMOD_BIN" 0600 -- "$response"
"$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$?
if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 4; fi
if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi
uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 4; }
[[ "$uid" == "$expected_uid" ]] || { "$RM_BIN" -f -- "$response"; return 3; }
validate_object_json "$kind" "$name" "$run_label" "$response" true true || { "$RM_BIN" -f -- "$response"; return 4; }
"$RM_BIN" -f -- "$response"
return 0
}
wait_for_stable_absence() {
local kind=$1 name=$2 uid=$3 attempt absent=0 state
for ((attempt=0; attempt<DELETE_WAIT_POLLS; attempt++)); do
if object_state "$kind" "$name" "$uid"; then
absent=0
else
state=$?
if [[ "$state" == 1 ]]; then
absent=$((absent + 1))
(( absent >= 2 )) && return 0
else
return 1
fi
fi
"$SLEEP_BIN" 0.05
done
return 1
}
delete_with_uid_precondition() {
local kind=$1 name=$2 uid=$3 api_path socket options response proxy_log proxy_pid='' attempt http_code='' curl_rc=0
[[ "$name" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ && "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
api_path="$(api_path_for "$kind" "$name")" || return 1
socket="$run_temp/proxy-${kind}.sock"
options="$run_temp/delete-${kind}.json"
response="$run_temp/delete-${kind}.response"
proxy_log="$run_temp/proxy-${kind}.log"
[[ ! -e "$socket" && ! -L "$socket" ]] || return 1
printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' "$uid" >"$options"
"$CHMOD_BIN" 0600 -- "$options"
: >"$response"; "$CHMOD_BIN" 0600 -- "$response"
"$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" proxy \
--unix-socket="$socket" --api-prefix=/ --accept-paths="^${api_path}$" \
--reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 &
proxy_pid=$!
for ((attempt=0; attempt<PROXY_WAIT_POLLS; attempt++)); do
[[ -S "$socket" ]] && break
"$KILL_BIN" -0 "$proxy_pid" 2>/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; }
"$SLEEP_BIN" 0.05
done
[[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; }
http_code="$("$CURL_BIN" --silent --show-error --max-time 5 --unix-socket "$socket" --output "$response" --write-out '%{http_code}' \
--request DELETE --header 'Content-Type: application/json' --data-binary "@$options" "http://localhost${api_path}")" || curl_rc=$?
stop_proxy "$proxy_pid"
# A transport/status ambiguity is not treated as success on its own. Only
# two consecutive exact absence reads reclassify it as a completed delete.
wait_for_stable_absence "$kind" "$name" "$uid"
}
cleanup() {
local rc=0
[[ "$cleanup_started" == false ]] || return 0
cleanup_started=true
if [[ -n "$pod_uid" ]]; then delete_with_uid_precondition pod "$pod_name" "$pod_uid" || rc=1; fi
if [[ -n "$policy_uid" ]]; then delete_with_uid_precondition networkpolicy "$policy_name" "$policy_uid" || rc=1; fi
pod_uid=''; policy_uid=''
(( rc == 0 )) || printf 'ERROR: temporary Blackbox proof object identity is unknown; manual review required\n' >&2
return "$rc"
}
on_exit() {
local rc=$?
trap - EXIT HUP INT TERM
if [[ -n "$pod_uid" || -n "$policy_uid" ]]; then
cleanup || { [[ "$rc" == 0 ]] && rc=1; }
fi
cleanup_local_temp || { [[ "$rc" == 0 ]] && rc=1; }
exit "$rc"
}
run_client_request() {
local host=$1 token=$2 output status rc=0
output="$("$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" exec "$pod_name" -- sh -ec \
"wget --no-check-certificate --server-response --spider 'https://$host/?hyeonworks_probe=$token'" 2>&1)" || rc=$?
status="$("$AWK_BIN" '/^ HTTP\// { code=$2 } /^HTTP\// { code=$2 } END { print code }' <<<"$output")"
[[ "$rc" == 1 && "$status" == 403 ]]
}
collect_log_results() {
local one=$1 two=$2 three=$3 output pair addr status
output="$(root_run "$AWK_BIN" -v one="$one" -v two="$two" -v three="$three" '
$0 ~ ("hyeonworks_probe=" one "([& ]| HTTP)") { if (++count[1] == 1) value[1]=$1 " " $9 }
$0 ~ ("hyeonworks_probe=" two "([& ]| HTTP)") { if (++count[2] == 1) value[2]=$1 " " $9 }
$0 ~ ("hyeonworks_probe=" three "([& ]| HTTP)") { if (++count[3] == 1) value[3]=$1 " " $9 }
END { if (count[1] != 1 || count[2] != 1 || count[3] != 1) exit 42; print value[1]; print value[2]; print value[3] }
' /var/log/nginx/access.log)" || return 1
mapfile -t LOG_RESULTS <<<"$output"
(( ${#LOG_RESULTS[@]} == 3 )) || return 1
for pair in "${LOG_RESULTS[@]}"; do
read -r addr status <<<"$pair"
[[ "$addr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$status" == 403 ]] || return 1
done
}
write_proof() {
local nginx_sha=$1 tested=$2 local_file root_temporary canonical suffix
local_file="$("$MKTEMP_BIN" "$run_temp/proof.XXXXXXXX")" || return 1
"$CHMOD_BIN" 0600 -- "$local_file"
printf 'schema=%s\nrollback_id=%s\nnginx_sha256=%s\ntested_at_utc=%s\ngrafana_remote_addr=%s\ngrafana_status=403\nstorage_admin_remote_addr=%s\nstorage_admin_status=403\ndb_admin_remote_addr=%s\ndb_admin_status=403\n' \
"$PROOF_SCHEMA" "${PLATFORM_OBSERVABILITY_ROLLBACK_ID:?}" "$nginx_sha" "$tested" \
"${LOG_RESULTS[0]%% *}" "${LOG_RESULTS[1]%% *}" "${LOG_RESULTS[2]%% *}" >"$local_file"
root_temporary="$(root_run "$MKTEMP_BIN" --tmpdir="$rollback_root" '.blackbox-source-proof.env.new.XXXXXXXX')" || { "$RM_BIN" -f -- "$local_file"; return 1; }
suffix=${root_temporary#"$rollback_root/.blackbox-source-proof.env.new."}
[[ "$root_temporary" == "$rollback_root/.blackbox-source-proof.env.new."* && "$suffix" =~ ^[A-Za-z0-9]{8}$ && "$root_temporary" != *$'\n'* ]] || {
"$RM_BIN" -f -- "$local_file"; return 1;
}
canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$root_temporary")" || { "$RM_BIN" -f -- "$local_file"; return 1; }
[[ "$canonical" == "$root_temporary" ]] && root_safe_file "$root_temporary" 600 || {
"$RM_BIN" -f -- "$local_file"; return 1;
}
if ! root_run "$INSTALL_BIN" -o root -g root -m 0600 -- "$local_file" "$root_temporary" ||
! root_safe_file "$root_temporary" 600 ||
! root_run "$CMP_BIN" --silent -- "$local_file" "$root_temporary" ||
! root_run "$LN_BIN" -nT -- "$root_temporary" "$proof_path"; then
root_safe_file "$root_temporary" 600 && root_run "$UNLINK_BIN" -- "$root_temporary" >/dev/null 2>&1 || true
"$RM_BIN" -f -- "$local_file"
return 1
fi
root_run "$UNLINK_BIN" -- "$root_temporary" || { "$RM_BIN" -f -- "$local_file"; return 1; }
root_safe_file "$proof_path" 600 && root_run "$CMP_BIN" --silent -- "$local_file" "$proof_path" || {
"$RM_BIN" -f -- "$local_file"; return 1;
}
"$RM_BIN" -f -- "$local_file"
}
execute_proof() {
local nginx_sha stamp random token_one token_two token_three tested
nginx_sha="$(validate_active_guard)" || return 1
validate_rollback_root || return 1
require_context_api_auth || return 1
confirm || return 1
prepare_local_temp || { fail 'cannot create private validator temporary directory'; return 1; }
stamp="$("$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1
random="${RANDOM}${RANDOM}"
run_label="${stamp,,}-${random}"
policy_name="blackbox-edge-source-egress-${run_label}"
pod_name="blackbox-edge-source-${run_label}"
token_one="${stamp}-${RANDOM}${RANDOM}"
token_two="${stamp}-${RANDOM}${RANDOM}"
token_three="${stamp}-${RANDOM}${RANDOM}"
[[ "$token_one" != "$token_two" && "$token_one" != "$token_three" && "$token_two" != "$token_three" ]] || { fail 'could not create unique probe tokens'; return 1; }
# This is the commit gate: it is immediately after confirmation and all
# local preparation, and immediately before the first Kubernetes mutation.
validate_active_guard_expected "$nginx_sha" || return 1
validate_proof_target_absent || return 1
create_temporary_objects || return 1
"$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" wait --for=condition=Ready --timeout="$READY_TIMEOUT" "pod/$pod_name" >/dev/null || {
fail 'temporary Pod did not become Ready'; return 1;
}
run_client_request "$GRAFANA_HOST" "$token_one" || { fail 'Grafana client status is not exact BusyBox 403'; return 1; }
run_client_request "$STORAGE_HOST" "$token_two" || { fail 'AIStor client status is not exact BusyBox 403'; return 1; }
run_client_request "$DB_HOST" "$token_three" || { fail 'pgAdmin client status is not exact BusyBox 403'; return 1; }
collect_log_results "$token_one" "$token_two" "$token_three" || { fail 'Nginx log did not contain exactly one 403 result per probe'; return 1; }
tested="$("$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1
write_proof "$nginx_sha" "$tested" || { fail 'cannot atomically write source-proof evidence'; return 1; }
cleanup || { fail 'temporary object cleanup failed'; return 1; }
printf 'BLACKBOX PRIVATE EDGE SOURCE PASS\n'
}
main() {
parse_args "$@" || return 1
validate_test_mode || return 1
require_regular_source "$DENY_GUARD" || return 1
if [[ "$execute" == false ]]; then
printf 'BLACKBOX_EDGE_SOURCE_DENY_GUARD_SHA256=%s\nBLACKBOX_EDGE_SOURCE_BUSYBOX_IMAGE=%s\nBLACKBOX_PRIVATE_EDGE_SOURCE_DRY_RUN=PASS\n' "$(sha_of "$DENY_GUARD")" "$BUSYBOX_IMAGE"
return 0
fi
for command_path in "$AWK_BIN" "$CHMOD_BIN" "$CMP_BIN" "$CURL_BIN" "$ID_BIN" "$INSTALL_BIN" "$JQ_BIN" "$KILL_BIN" "$LN_BIN" "$MKTEMP_BIN" "$REALPATH_BIN" "$RM_BIN" "$SHA256SUM_BIN" "$SLEEP_BIN" "$STAT_BIN" "$TEST_BIN" "$TRUE_BIN" "$UNLINK_BIN"; do
[[ -x "$command_path" ]] || { fail "required command is missing: $command_path"; return 1; }
done
[[ -x "$KUBECTL" && -x "$SUDO" && -x "$DATE_BIN" ]] || { fail 'required command boundary is missing'; return 1; }
trap on_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
execute_proof
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi