#!/usr/bin/env bash # Prove that a restricted in-cluster source is denied by the active Grafana # Nginx guard. This is deliberately a short-lived prerequisite transaction, # not a reusable HTTP probing tool. set -Eeuo pipefail set +x umask 077 readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly DENY_GUARD="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" readonly NAMESPACE=observability readonly ROLLBACK_BASE_DEFAULT=/var/lib/hyeonworks/platform-rollbacks readonly ACTIVE_DEFAULT=/etc/nginx/sites-available/learn-services readonly KUBECTL_DEFAULT=/usr/local/bin/kubectl readonly SUDO_DEFAULT=/usr/bin/sudo readonly DATE_DEFAULT=/usr/bin/date readonly SLEEP_DEFAULT=/usr/bin/sleep readonly AWK_BIN=/usr/bin/awk readonly CHMOD_BIN=/usr/bin/chmod readonly CMP_BIN=/usr/bin/cmp readonly CURL_BIN=/usr/bin/curl readonly ID_BIN=/usr/bin/id readonly INSTALL_BIN=/usr/bin/install readonly JQ_BIN=/usr/bin/jq readonly KILL_BIN=/bin/kill readonly LN_BIN=/bin/ln readonly MKTEMP_BIN=/usr/bin/mktemp readonly REALPATH_BIN=/usr/bin/realpath readonly RM_BIN=/bin/rm readonly SHA256SUM_BIN=/usr/bin/sha256sum readonly STAT_BIN=/usr/bin/stat readonly TEST_BIN=/usr/bin/test readonly TRUE_BIN=/usr/bin/true readonly UNLINK_BIN=/usr/bin/unlink readonly PROOF_SCHEMA=platform-blackbox-source-v1 readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0' readonly EXPECTED_NODE='donghyeon-system-product-name' readonly REQUEST_TIMEOUT=10s readonly READY_TIMEOUT=30s readonly DELETE_WAIT_POLLS=1200 readonly PROXY_WAIT_POLLS=50 readonly GRAFANA_HOST=grafana.learn.hyeonworks.com readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com readonly DB_HOST=db-admin.learn.hyeonworks.com readonly TEST_MODE="${PLATFORM_BLACKBOX_EDGE_TEST_MODE:-0}" readonly KUBECTL="${PLATFORM_BLACKBOX_EDGE_KUBECTL:-$KUBECTL_DEFAULT}" readonly SUDO="${PLATFORM_BLACKBOX_EDGE_SUDO:-$SUDO_DEFAULT}" readonly DATE_BIN="${PLATFORM_BLACKBOX_EDGE_DATE:-$DATE_DEFAULT}" readonly SLEEP_BIN="${PLATFORM_BLACKBOX_EDGE_SLEEP:-$SLEEP_DEFAULT}" readonly ACTIVE="${PLATFORM_BLACKBOX_EDGE_ACTIVE:-$ACTIVE_DEFAULT}" readonly ROLLBACK_BASE="${PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE:-$ROLLBACK_BASE_DEFAULT}" execute=false context='' fixture_root='' rollback_anchor='' rollback_root='' proof_path='' run_temp='' run_label='' pod_name='' policy_name='' pod_uid='' policy_uid='' cleanup_started=false declare -a LOG_RESULTS=() fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; } usage() { cat <<'USAGE' Usage: bash scripts/validate/validate-blackbox-edge-source.sh bash scripts/validate/validate-blackbox-edge-source.sh --execute --context CONTEXT The default is read-only. Execute requires an exact context-qualified confirmation, creates only a temporary restricted NetworkPolicy and Pod, and never prints request tokens or Nginx log lines. USAGE } sha_of() { "$SHA256SUM_BIN" -- "$1" | "$AWK_BIN" '{print $1}'; } is_sha256() { [[ "$1" =~ ^[0-9a-f]{64}$ ]]; } require_regular_source() { [[ -f "$1" && ! -L "$1" ]] || fail "unsafe or missing source: $1"; } safe_fixture_command() { local path=$1 fixture_bin=$2 resolved metadata type owner mode mode_value [[ "$path" != /usr/bin/sudo ]] || return 1 resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$path" 2>/dev/null)" || return 1 [[ "$resolved" == "$path" && "${path%/*}" == "$fixture_bin" && -f "$path" && ! -L "$path" && -x "$path" ]] || return 1 metadata="$("$STAT_BIN" -c '%F|%u|%a' -- "$path")" || return 1 IFS='|' read -r type owner mode <<<"$metadata" mode_value=$((8#$mode)) [[ "$type" == 'regular file' && "$owner" == "$EUID" && $((mode_value & 0022)) == 0 && $((mode_value & 07000)) == 0 ]] } validate_test_mode() { local active_parent resolved key [[ "$TEST_MODE" == 0 || "$TEST_MODE" == 1 ]] || { fail 'invalid test-mode flag'; return 1; } if [[ "$TEST_MODE" == 0 ]]; then for key in ${!PLATFORM_BLACKBOX_EDGE_@}; do fail "production rejects Blackbox edge override: $key" return 1 done [[ "$KUBECTL" == "$KUBECTL_DEFAULT" && "$SUDO" == "$SUDO_DEFAULT" && "$DATE_BIN" == "$DATE_DEFAULT" && "$SLEEP_BIN" == "$SLEEP_DEFAULT" && "$ACTIVE" == "$ACTIVE_DEFAULT" && "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || { fail 'production Blackbox edge constants are not exact'; return 1; } rollback_anchor=/ return 0 fi active_parent=${ACTIVE%/*} resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$active_parent/.." 2>/dev/null)" || { fail 'test fixture root does not resolve'; return 1; } fixture_root=$resolved [[ "$fixture_root" == /tmp/platform-blackbox-edge-test.* && "$ACTIVE" == "$fixture_root/active/learn-services" && "$ROLLBACK_BASE" == "$fixture_root/rollbacks" && -d "$fixture_root/bin" && ! -L "$fixture_root/bin" ]] || { fail 'test mode is restricted to one isolated fixture tree'; return 1; } safe_fixture_command "$KUBECTL" "$fixture_root/bin" && safe_fixture_command "$SUDO" "$fixture_root/bin" && safe_fixture_command "$DATE_BIN" "$fixture_root/bin" && safe_fixture_command "$SLEEP_BIN" "$fixture_root/bin" || { fail 'test command boundary is unsafe'; return 1; } rollback_anchor=$fixture_root } parse_args() { local execute_count=0 context_count=0 while (( $# > 0 )); do case "$1" in --execute) execute=true; execute_count=$((execute_count + 1)); shift ;; --context) (( $# >= 2 )) || { fail '--context requires a value'; return 1; } context=$2; context_count=$((context_count + 1)); shift 2 ;; -h|--help) usage; exit 0 ;; *) usage >&2; fail "unsupported argument: $1"; return 1 ;; esac done (( execute_count <= 1 && context_count <= 1 )) || { fail 'duplicate execute argument'; return 1; } if [[ "$execute" == true ]]; then [[ "$context" =~ ^[A-Za-z0-9._:-]+$ ]] || { fail '--execute requires an exact --context'; return 1; } else [[ -z "$context" ]] || { fail '--context is execute-only'; return 1; } fi } root_run() { "$SUDO" -n "$@"; } root_metadata() { root_run "$STAT_BIN" -c '%F|%u:%g|%a|%h' -- "$1"; } root_owner_expected() { if [[ "$TEST_MODE" == 0 ]]; then [[ "$1" == 0:0 ]]; else [[ "$1" == "$EUID:$($ID_BIN -g)" ]]; fi } root_safe_ancestor_dir() { local path=$1 metadata type owner mode links mode_value root_run "$TEST_BIN" ! -L "$path" || return 1 root_run "$TEST_BIN" -d "$path" || return 1 metadata="$(root_metadata "$path")" || return 1 IFS='|' read -r type owner mode links <<<"$metadata" mode_value=$((8#$mode)) [[ "$type" == directory && "$links" =~ ^[1-9][0-9]*$ ]] || return 1 root_owner_expected "$owner" || return 1 (( (mode_value & 0022) == 0 )) } root_safe_dir() { local path=$1 expected_mode=$2 metadata type owner mode links root_safe_ancestor_dir "$path" || return 1 metadata="$(root_metadata "$path")" || return 1 IFS='|' read -r type owner mode links <<<"$metadata" [[ "$mode" == "$expected_mode" && "$links" =~ ^[1-9][0-9]*$ ]] } root_safe_file() { local path=$1 expected_mode=$2 metadata type owner mode links root_run "$TEST_BIN" ! -L "$path" || return 1 root_run "$TEST_BIN" -f "$path" || return 1 metadata="$(root_metadata "$path")" || return 1 IFS='|' read -r type owner mode links <<<"$metadata" [[ ( "$type" == 'regular file' || "$type" == 'regular empty file' ) && "$mode" == "$expected_mode" && "$links" == 1 ]] || return 1 root_owner_expected "$owner" } validate_root_chain() { local target=$1 anchor=$2 canonical relative current component [[ "$target" == /* && "$anchor" == /* && "$target" != *'//' && "$target" != */./* && "$target" != */../* ]] || return 1 canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$target")" || return 1 [[ "$canonical" == "$target" && "$canonical" != *$'\n'* ]] || return 1 canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$anchor")" || return 1 [[ "$canonical" == "$anchor" && "$canonical" != *$'\n'* ]] || return 1 if [[ "$anchor" == / ]]; then # Root has no separator to append: "$anchor/" would become "//" and # reject every normal absolute descendant after canonicalization. [[ "$target" == /* ]] || return 1 else [[ "$target" == "$anchor" || "$target" == "$anchor/"* ]] || return 1 fi current=$anchor root_safe_ancestor_dir "$current" || return 1 relative=${target#"$anchor"} relative=${relative#/} [[ -z "$relative" ]] && return 0 IFS='/' read -r -a _blackbox_path_parts <<<"$relative" for component in "${_blackbox_path_parts[@]}"; do [[ -n "$component" && "$component" != . && "$component" != .. ]] || return 1 if [[ "$current" == / ]]; then current="/$component"; else current="$current/$component"; fi root_safe_ancestor_dir "$current" || return 1 done } validate_rollback_root() { local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-} [[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || { fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid'; return 1; } if [[ "$TEST_MODE" == 0 ]]; then [[ "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || { fail 'production rollback base is not exact'; return 1; }; fi rollback_root="$ROLLBACK_BASE/observability-$id" proof_path="$rollback_root/blackbox-source-proof.env" validate_root_chain "$ROLLBACK_BASE" "$rollback_anchor" || { fail 'rollback base lineage is unsafe'; return 1; } validate_root_chain "$rollback_root" "$rollback_anchor" || { fail 'rollback root lineage is unsafe'; return 1; } root_safe_dir "$rollback_root" 700 || { fail 'rollback root metadata is unsafe'; return 1; } } validate_proof_target_absent() { validate_rollback_root || return 1 [[ "$proof_path" == "$rollback_root/blackbox-source-proof.env" ]] || { fail 'source-proof path is not exact'; return 1; } root_run "$TEST_BIN" ! -e "$proof_path" && root_run "$TEST_BIN" ! -L "$proof_path" || { fail 'source-proof evidence already exists or is unsafe'; return 1; } } validate_active_guard_expected() { local expected=$1 deny_sha metadata type owner mode links active_sha require_regular_source "$DENY_GUARD" || return 1 is_sha256 "$expected" || { fail 'expected deny guard hash is invalid'; return 1; } deny_sha="$(sha_of "$DENY_GUARD")" || { fail 'cannot hash deny guard source'; return 1; } [[ "$deny_sha" == "$expected" ]] || { fail 'deny guard source changed during proof'; return 1; } root_run "$TEST_BIN" ! -L "$ACTIVE" && root_run "$TEST_BIN" -f "$ACTIVE" || { fail 'active Nginx config is missing or symlinked'; return 1; } metadata="$(root_metadata "$ACTIVE")" || { fail 'cannot stat active Nginx config'; return 1; } IFS='|' read -r type owner mode links <<<"$metadata" [[ "$type" == 'regular file' && "$links" == 1 ]] || { fail 'active Nginx config metadata is unsafe'; return 1; } root_owner_expected "$owner" || { fail 'active Nginx config owner is unsafe'; return 1; } if [[ "$TEST_MODE" == 0 ]]; then [[ "$mode" == 644 ]] || { fail 'active Nginx config mode is unsafe'; return 1; }; fi active_sha="$(root_run "$SHA256SUM_BIN" -- "$ACTIVE" | "$AWK_BIN" '{print $1}')" || { fail 'cannot hash active Nginx config'; return 1; } [[ "$active_sha" == "$expected" ]] || { fail 'active Nginx config is not the exact Grafana deny guard'; return 1; } } validate_active_guard() { local deny_sha require_regular_source "$DENY_GUARD" || return 1 deny_sha="$(sha_of "$DENY_GUARD")" || return 1 is_sha256 "$deny_sha" || { fail 'deny guard hash is invalid'; return 1; } validate_active_guard_expected "$deny_sha" || return 1 printf '%s\n' "$deny_sha" } confirm() { local expected="PROVE BLACKBOX PRIVATE EDGE $context" answer printf 'Type %s: ' "$expected" >&2 if [[ "$TEST_MODE" == 1 ]]; then answer=${PLATFORM_BLACKBOX_EDGE_CONFIRMATION:-} printf '%s\n' "$answer" >&2 else [[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; } IFS= read -r answer fi [[ "$answer" == "$expected" ]] || { fail 'cancelled'; return 1; } } require_context_api_auth() { local current answer current="$("$KUBECTL" --request-timeout="$REQUEST_TIMEOUT" config current-context)" || { fail 'cannot read Kubernetes context'; return 1; } [[ "$current" == "$context" && "$current" != *$'\n'* ]] || { fail 'current Kubernetes context does not match --context'; return 1; } "$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; } for answer in \ "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create pods -n "$NAMESPACE")" \ "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create networkpolicies.networking.k8s.io -n "$NAMESPACE")" \ "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete pods -n "$NAMESPACE")" \ "$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete networkpolicies.networking.k8s.io -n "$NAMESPACE")"; do [[ "$answer" == yes ]] || { fail 'Kubernetes authorization is insufficient'; return 1; } done root_run "$TRUE_BIN" || { fail 'sudo non-interactive authentication failed'; return 1; } } prepare_local_temp() { run_temp="$("$MKTEMP_BIN" -d /tmp/platform-blackbox-edge-source.XXXXXXXX)" || return 1 [[ "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 1 "$CHMOD_BIN" 0700 -- "$run_temp" } cleanup_local_temp() { [[ -n "$run_temp" && "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 0 "$RM_BIN" -rf -- "$run_temp" run_temp='' } render_network_policy() { /bin/cat </dev/null ;; pod) "$JQ_BIN" -e --arg namespace "$NAMESPACE" --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg node "$EXPECTED_NODE" --argjson uid_required "$([[ "$require_uid" == true ]] && printf true || printf false)" --argjson scheduled_node_allowed "$([[ "$allow_scheduled_node" == true ]] && printf true || printf false)" ' def optional_default($key; $value): (has($key) | not) or .[$key] == $value; .apiVersion == "v1" and .kind == "Pod" and .metadata.namespace == $namespace and .metadata.name == $name and .metadata.labels == { "app.kubernetes.io/managed-by":"platform-blackbox-edge-source", "platform.hyeonworks.com/source-proof-run":$run } and (($uid_required | not) or (.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))) and (.spec | type == "object" and optional_default("dnsPolicy"; "ClusterFirst") and optional_default("enableServiceLinks"; true) and optional_default("preemptionPolicy"; "PreemptLowerPriority") and optional_default("priority"; 0) and optional_default("schedulerName"; "default-scheduler") and optional_default("serviceAccount"; "default") and optional_default("serviceAccountName"; "default") and optional_default("terminationGracePeriodSeconds"; 30) and ((has("nodeName") | not) or ($scheduled_node_allowed and .nodeName == $node)) and optional_default("tolerations"; [ {"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300}, {"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300} ]) and (.containers | type == "array" and length == 1) and (.containers[0] | optional_default("resources"; {}) and optional_default("terminationMessagePath"; "/dev/termination-log") and optional_default("terminationMessagePolicy"; "File")) and (del( .dnsPolicy, .enableServiceLinks, .preemptionPolicy, .priority, .schedulerName, .serviceAccount, .serviceAccountName, .terminationGracePeriodSeconds, .tolerations, .nodeName, .containers[0].resources, .containers[0].terminationMessagePath, .containers[0].terminationMessagePolicy ) == { "automountServiceAccountToken":false, "restartPolicy":"Never", "securityContext":{"runAsNonRoot":true,"runAsUser":65534,"runAsGroup":65534,"seccompProfile":{"type":"RuntimeDefault"}}, "containers":[{"name":"probe","image":$image,"imagePullPolicy":"IfNotPresent","command":["sh","-c","sleep 120"],"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}}] }) ) ' "$source" >/dev/null ;; *) return 1 ;; esac } response_uid() { local source=$1 uid uid="$("$JQ_BIN" -er '.metadata.uid' "$source")" || return 1 [[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 printf '%s\n' "$uid" } local_response_file() { "$MKTEMP_BIN" "$run_temp/${1}.response.XXXXXXXX" } render_resource() { case "$1" in networkpolicy) render_network_policy ;; pod) render_pod ;; *) return 1 ;; esac } resource_name() { case "$1" in networkpolicy) printf '%s\n' "$policy_name" ;; pod) printf '%s\n' "$pod_name" ;; *) return 1 ;; esac; } server_dry_run_resource() { local kind=$1 name response name="$(resource_name "$kind")" || return 1 response="$(local_response_file "${kind}.dry")" || return 1 "$CHMOD_BIN" 0600 -- "$response" if ! render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create --dry-run=server -f - -o json >"$response"; then "$RM_BIN" -f -- "$response"; return 1 fi validate_object_json "$kind" "$name" "$run_label" "$response" false || { "$RM_BIN" -f -- "$response"; return 1; } "$RM_BIN" -f -- "$response" } inspect_owned_object() { local kind=$1 name=$2 destination=$3 response uid rc=0 response="$(local_response_file "${kind}.inspect")" || return 3 "$CHMOD_BIN" 0600 -- "$response" "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$? if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 3; fi if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi validate_object_json "$kind" "$name" "$run_label" "$response" true || { "$RM_BIN" -f -- "$response"; return 2; } uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 2; } "$RM_BIN" -f -- "$response" printf -v "$destination" '%s' "$uid" } create_resource() { local kind=$1 destination=$2 name response uid rc=0 classification name="$(resource_name "$kind")" || return 1 response="$(local_response_file "${kind}.create")" || return 1 "$CHMOD_BIN" 0600 -- "$response" if render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create -f - -o json >"$response"; then if validate_object_json "$kind" "$name" "$run_label" "$response" true && uid="$(response_uid "$response")"; then "$RM_BIN" -f -- "$response" printf -v "$destination" '%s' "$uid" return 0 fi fi "$RM_BIN" -f -- "$response" # A syntactically bad success response is still ambiguous: the API may have # committed the exact object, so classify the live state before cleanup. if inspect_owned_object "$kind" "$name" "$destination"; then return 2 else classification=$? [[ "$classification" == 1 ]] && return 1 return 1 fi } create_temporary_objects() { local result server_dry_run_resource networkpolicy || { fail 'temporary NetworkPolicy server dry-run failed'; return 1; } if create_resource networkpolicy policy_uid; then :; else result=$? [[ "$result" == 2 ]] && fail 'temporary NetworkPolicy create outcome was ambiguous but owned' [[ "$result" != 2 ]] && fail 'temporary NetworkPolicy create failed' return 1 fi server_dry_run_resource pod || { fail 'temporary Pod server dry-run failed'; return 1; } if create_resource pod pod_uid; then :; else result=$? [[ "$result" == 2 ]] && fail 'temporary Pod create outcome was ambiguous but owned' [[ "$result" != 2 ]] && fail 'temporary Pod create failed' return 1 fi } stop_proxy() { local pid=$1 attempt "$KILL_BIN" -TERM "$pid" 2>/dev/null || true for ((attempt=0; attempt<20; attempt++)); do if ! "$KILL_BIN" -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi "$SLEEP_BIN" 0.05 done "$KILL_BIN" -KILL "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true } api_path_for() { case "$1" in pod) printf '/api/v1/namespaces/%s/pods/%s\n' "$NAMESPACE" "$2" ;; networkpolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s\n' "$NAMESPACE" "$2" ;; *) return 1 ;; esac } object_state() { local kind=$1 name=$2 expected_uid=$3 response uid rc=0 response="$(local_response_file "${kind}.state")" || return 4 "$CHMOD_BIN" 0600 -- "$response" "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$? if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 4; fi if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 4; } [[ "$uid" == "$expected_uid" ]] || { "$RM_BIN" -f -- "$response"; return 3; } validate_object_json "$kind" "$name" "$run_label" "$response" true true || { "$RM_BIN" -f -- "$response"; return 4; } "$RM_BIN" -f -- "$response" return 0 } wait_for_stable_absence() { local kind=$1 name=$2 uid=$3 attempt absent=0 state for ((attempt=0; attempt= 2 )) && return 0 else return 1 fi fi "$SLEEP_BIN" 0.05 done return 1 } delete_with_uid_precondition() { local kind=$1 name=$2 uid=$3 api_path socket options response proxy_log proxy_pid='' attempt http_code='' curl_rc=0 [[ "$name" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ && "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1 api_path="$(api_path_for "$kind" "$name")" || return 1 socket="$run_temp/proxy-${kind}.sock" options="$run_temp/delete-${kind}.json" response="$run_temp/delete-${kind}.response" proxy_log="$run_temp/proxy-${kind}.log" [[ ! -e "$socket" && ! -L "$socket" ]] || return 1 printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' "$uid" >"$options" "$CHMOD_BIN" 0600 -- "$options" : >"$response"; "$CHMOD_BIN" 0600 -- "$response" "$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" proxy \ --unix-socket="$socket" --api-prefix=/ --accept-paths="^${api_path}$" \ --reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 & proxy_pid=$! for ((attempt=0; attempt/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; } "$SLEEP_BIN" 0.05 done [[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; } http_code="$("$CURL_BIN" --silent --show-error --max-time 5 --unix-socket "$socket" --output "$response" --write-out '%{http_code}' \ --request DELETE --header 'Content-Type: application/json' --data-binary "@$options" "http://localhost${api_path}")" || curl_rc=$? stop_proxy "$proxy_pid" # A transport/status ambiguity is not treated as success on its own. Only # two consecutive exact absence reads reclassify it as a completed delete. wait_for_stable_absence "$kind" "$name" "$uid" } cleanup() { local rc=0 [[ "$cleanup_started" == false ]] || return 0 cleanup_started=true if [[ -n "$pod_uid" ]]; then delete_with_uid_precondition pod "$pod_name" "$pod_uid" || rc=1; fi if [[ -n "$policy_uid" ]]; then delete_with_uid_precondition networkpolicy "$policy_name" "$policy_uid" || rc=1; fi pod_uid=''; policy_uid='' (( rc == 0 )) || printf 'ERROR: temporary Blackbox proof object identity is unknown; manual review required\n' >&2 return "$rc" } on_exit() { local rc=$? trap - EXIT HUP INT TERM if [[ -n "$pod_uid" || -n "$policy_uid" ]]; then cleanup || { [[ "$rc" == 0 ]] && rc=1; } fi cleanup_local_temp || { [[ "$rc" == 0 ]] && rc=1; } exit "$rc" } run_client_request() { local host=$1 token=$2 output status rc=0 output="$("$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" exec "$pod_name" -- sh -ec \ "wget --no-check-certificate --server-response --spider 'https://$host/?hyeonworks_probe=$token'" 2>&1)" || rc=$? status="$("$AWK_BIN" '/^ HTTP\// { code=$2 } /^HTTP\// { code=$2 } END { print code }' <<<"$output")" [[ "$rc" == 1 && "$status" == 403 ]] } collect_log_results() { local one=$1 two=$2 three=$3 output pair addr status output="$(root_run "$AWK_BIN" -v one="$one" -v two="$two" -v three="$three" ' $0 ~ ("hyeonworks_probe=" one "([& ]| HTTP)") { if (++count[1] == 1) value[1]=$1 " " $9 } $0 ~ ("hyeonworks_probe=" two "([& ]| HTTP)") { if (++count[2] == 1) value[2]=$1 " " $9 } $0 ~ ("hyeonworks_probe=" three "([& ]| HTTP)") { if (++count[3] == 1) value[3]=$1 " " $9 } END { if (count[1] != 1 || count[2] != 1 || count[3] != 1) exit 42; print value[1]; print value[2]; print value[3] } ' /var/log/nginx/access.log)" || return 1 mapfile -t LOG_RESULTS <<<"$output" (( ${#LOG_RESULTS[@]} == 3 )) || return 1 for pair in "${LOG_RESULTS[@]}"; do read -r addr status <<<"$pair" [[ "$addr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$status" == 403 ]] || return 1 done } write_proof() { local nginx_sha=$1 tested=$2 local_file root_temporary canonical suffix local_file="$("$MKTEMP_BIN" "$run_temp/proof.XXXXXXXX")" || return 1 "$CHMOD_BIN" 0600 -- "$local_file" printf 'schema=%s\nrollback_id=%s\nnginx_sha256=%s\ntested_at_utc=%s\ngrafana_remote_addr=%s\ngrafana_status=403\nstorage_admin_remote_addr=%s\nstorage_admin_status=403\ndb_admin_remote_addr=%s\ndb_admin_status=403\n' \ "$PROOF_SCHEMA" "${PLATFORM_OBSERVABILITY_ROLLBACK_ID:?}" "$nginx_sha" "$tested" \ "${LOG_RESULTS[0]%% *}" "${LOG_RESULTS[1]%% *}" "${LOG_RESULTS[2]%% *}" >"$local_file" root_temporary="$(root_run "$MKTEMP_BIN" --tmpdir="$rollback_root" '.blackbox-source-proof.env.new.XXXXXXXX')" || { "$RM_BIN" -f -- "$local_file"; return 1; } suffix=${root_temporary#"$rollback_root/.blackbox-source-proof.env.new."} [[ "$root_temporary" == "$rollback_root/.blackbox-source-proof.env.new."* && "$suffix" =~ ^[A-Za-z0-9]{8}$ && "$root_temporary" != *$'\n'* ]] || { "$RM_BIN" -f -- "$local_file"; return 1; } canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$root_temporary")" || { "$RM_BIN" -f -- "$local_file"; return 1; } [[ "$canonical" == "$root_temporary" ]] && root_safe_file "$root_temporary" 600 || { "$RM_BIN" -f -- "$local_file"; return 1; } if ! root_run "$INSTALL_BIN" -o root -g root -m 0600 -- "$local_file" "$root_temporary" || ! root_safe_file "$root_temporary" 600 || ! root_run "$CMP_BIN" --silent -- "$local_file" "$root_temporary" || ! root_run "$LN_BIN" -nT -- "$root_temporary" "$proof_path"; then root_safe_file "$root_temporary" 600 && root_run "$UNLINK_BIN" -- "$root_temporary" >/dev/null 2>&1 || true "$RM_BIN" -f -- "$local_file" return 1 fi root_run "$UNLINK_BIN" -- "$root_temporary" || { "$RM_BIN" -f -- "$local_file"; return 1; } root_safe_file "$proof_path" 600 && root_run "$CMP_BIN" --silent -- "$local_file" "$proof_path" || { "$RM_BIN" -f -- "$local_file"; return 1; } "$RM_BIN" -f -- "$local_file" } execute_proof() { local nginx_sha stamp random token_one token_two token_three tested nginx_sha="$(validate_active_guard)" || return 1 validate_rollback_root || return 1 require_context_api_auth || return 1 confirm || return 1 prepare_local_temp || { fail 'cannot create private validator temporary directory'; return 1; } stamp="$("$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1 random="${RANDOM}${RANDOM}" run_label="${stamp,,}-${random}" policy_name="blackbox-edge-source-egress-${run_label}" pod_name="blackbox-edge-source-${run_label}" token_one="${stamp}-${RANDOM}${RANDOM}" token_two="${stamp}-${RANDOM}${RANDOM}" token_three="${stamp}-${RANDOM}${RANDOM}" [[ "$token_one" != "$token_two" && "$token_one" != "$token_three" && "$token_two" != "$token_three" ]] || { fail 'could not create unique probe tokens'; return 1; } # This is the commit gate: it is immediately after confirmation and all # local preparation, and immediately before the first Kubernetes mutation. validate_active_guard_expected "$nginx_sha" || return 1 validate_proof_target_absent || return 1 create_temporary_objects || return 1 "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" wait --for=condition=Ready --timeout="$READY_TIMEOUT" "pod/$pod_name" >/dev/null || { fail 'temporary Pod did not become Ready'; return 1; } run_client_request "$GRAFANA_HOST" "$token_one" || { fail 'Grafana client status is not exact BusyBox 403'; return 1; } run_client_request "$STORAGE_HOST" "$token_two" || { fail 'AIStor client status is not exact BusyBox 403'; return 1; } run_client_request "$DB_HOST" "$token_three" || { fail 'pgAdmin client status is not exact BusyBox 403'; return 1; } collect_log_results "$token_one" "$token_two" "$token_three" || { fail 'Nginx log did not contain exactly one 403 result per probe'; return 1; } tested="$("$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1 write_proof "$nginx_sha" "$tested" || { fail 'cannot atomically write source-proof evidence'; return 1; } cleanup || { fail 'temporary object cleanup failed'; return 1; } printf 'BLACKBOX PRIVATE EDGE SOURCE PASS\n' } main() { parse_args "$@" || return 1 validate_test_mode || return 1 require_regular_source "$DENY_GUARD" || return 1 if [[ "$execute" == false ]]; then printf 'BLACKBOX_EDGE_SOURCE_DENY_GUARD_SHA256=%s\nBLACKBOX_EDGE_SOURCE_BUSYBOX_IMAGE=%s\nBLACKBOX_PRIVATE_EDGE_SOURCE_DRY_RUN=PASS\n' "$(sha_of "$DENY_GUARD")" "$BUSYBOX_IMAGE" return 0 fi for command_path in "$AWK_BIN" "$CHMOD_BIN" "$CMP_BIN" "$CURL_BIN" "$ID_BIN" "$INSTALL_BIN" "$JQ_BIN" "$KILL_BIN" "$LN_BIN" "$MKTEMP_BIN" "$REALPATH_BIN" "$RM_BIN" "$SHA256SUM_BIN" "$SLEEP_BIN" "$STAT_BIN" "$TEST_BIN" "$TRUE_BIN" "$UNLINK_BIN"; do [[ -x "$command_path" ]] || { fail "required command is missing: $command_path"; return 1; } done [[ -x "$KUBECTL" && -x "$SUDO" && -x "$DATE_BIN" ]] || { fail 'required command boundary is missing'; return 1; } trap on_exit EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM execute_proof } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi