729 lines
33 KiB
Bash
729 lines
33 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Prove that a restricted in-cluster source is denied by the active Grafana
|
|
# Nginx guard. This is deliberately a short-lived prerequisite transaction,
|
|
# not a reusable HTTP probing tool.
|
|
set -Eeuo pipefail
|
|
set +x
|
|
umask 077
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly DENY_GUARD="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf"
|
|
readonly NAMESPACE=observability
|
|
readonly ROLLBACK_BASE_DEFAULT=/var/lib/hyeonworks/platform-rollbacks
|
|
readonly ACTIVE_DEFAULT=/etc/nginx/sites-available/learn-services
|
|
readonly KUBECTL_DEFAULT=/usr/local/bin/kubectl
|
|
readonly SUDO_DEFAULT=/usr/bin/sudo
|
|
readonly DATE_DEFAULT=/usr/bin/date
|
|
readonly SLEEP_DEFAULT=/usr/bin/sleep
|
|
readonly AWK_BIN=/usr/bin/awk
|
|
readonly CHMOD_BIN=/usr/bin/chmod
|
|
readonly CMP_BIN=/usr/bin/cmp
|
|
readonly CURL_BIN=/usr/bin/curl
|
|
readonly ID_BIN=/usr/bin/id
|
|
readonly INSTALL_BIN=/usr/bin/install
|
|
readonly JQ_BIN=/usr/bin/jq
|
|
readonly KILL_BIN=/bin/kill
|
|
readonly LN_BIN=/bin/ln
|
|
readonly MKTEMP_BIN=/usr/bin/mktemp
|
|
readonly REALPATH_BIN=/usr/bin/realpath
|
|
readonly RM_BIN=/bin/rm
|
|
readonly SHA256SUM_BIN=/usr/bin/sha256sum
|
|
readonly STAT_BIN=/usr/bin/stat
|
|
readonly TEST_BIN=/usr/bin/test
|
|
readonly TRUE_BIN=/usr/bin/true
|
|
readonly UNLINK_BIN=/usr/bin/unlink
|
|
readonly PROOF_SCHEMA=platform-blackbox-source-v1
|
|
readonly BUSYBOX_IMAGE='docker.io/library/busybox:1.37.0@sha256:7a3ebe5bfd1a4a19797d20b0c0bb39d44393e9a03fd852c0865b0f540d868df0'
|
|
readonly EXPECTED_NODE='donghyeon-system-product-name'
|
|
readonly REQUEST_TIMEOUT=10s
|
|
readonly READY_TIMEOUT=30s
|
|
readonly DELETE_WAIT_POLLS=1200
|
|
readonly PROXY_WAIT_POLLS=50
|
|
readonly GRAFANA_HOST=grafana.learn.hyeonworks.com
|
|
readonly STORAGE_HOST=storage-admin.learn.hyeonworks.com
|
|
readonly DB_HOST=db-admin.learn.hyeonworks.com
|
|
|
|
readonly TEST_MODE="${PLATFORM_BLACKBOX_EDGE_TEST_MODE:-0}"
|
|
readonly KUBECTL="${PLATFORM_BLACKBOX_EDGE_KUBECTL:-$KUBECTL_DEFAULT}"
|
|
readonly SUDO="${PLATFORM_BLACKBOX_EDGE_SUDO:-$SUDO_DEFAULT}"
|
|
readonly DATE_BIN="${PLATFORM_BLACKBOX_EDGE_DATE:-$DATE_DEFAULT}"
|
|
readonly SLEEP_BIN="${PLATFORM_BLACKBOX_EDGE_SLEEP:-$SLEEP_DEFAULT}"
|
|
readonly ACTIVE="${PLATFORM_BLACKBOX_EDGE_ACTIVE:-$ACTIVE_DEFAULT}"
|
|
readonly ROLLBACK_BASE="${PLATFORM_BLACKBOX_EDGE_ROLLBACK_BASE:-$ROLLBACK_BASE_DEFAULT}"
|
|
|
|
execute=false
|
|
context=''
|
|
fixture_root=''
|
|
rollback_anchor=''
|
|
rollback_root=''
|
|
proof_path=''
|
|
run_temp=''
|
|
run_label=''
|
|
pod_name=''
|
|
policy_name=''
|
|
pod_uid=''
|
|
policy_uid=''
|
|
cleanup_started=false
|
|
declare -a LOG_RESULTS=()
|
|
|
|
fail() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
bash scripts/validate/validate-blackbox-edge-source.sh
|
|
bash scripts/validate/validate-blackbox-edge-source.sh --execute --context CONTEXT
|
|
|
|
The default is read-only. Execute requires an exact context-qualified
|
|
confirmation, creates only a temporary restricted NetworkPolicy and Pod, and
|
|
never prints request tokens or Nginx log lines.
|
|
USAGE
|
|
}
|
|
|
|
sha_of() { "$SHA256SUM_BIN" -- "$1" | "$AWK_BIN" '{print $1}'; }
|
|
is_sha256() { [[ "$1" =~ ^[0-9a-f]{64}$ ]]; }
|
|
require_regular_source() { [[ -f "$1" && ! -L "$1" ]] || fail "unsafe or missing source: $1"; }
|
|
|
|
safe_fixture_command() {
|
|
local path=$1 fixture_bin=$2 resolved metadata type owner mode mode_value
|
|
[[ "$path" != /usr/bin/sudo ]] || return 1
|
|
resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$path" 2>/dev/null)" || return 1
|
|
[[ "$resolved" == "$path" && "${path%/*}" == "$fixture_bin" && -f "$path" && ! -L "$path" && -x "$path" ]] || return 1
|
|
metadata="$("$STAT_BIN" -c '%F|%u|%a' -- "$path")" || return 1
|
|
IFS='|' read -r type owner mode <<<"$metadata"
|
|
mode_value=$((8#$mode))
|
|
[[ "$type" == 'regular file' && "$owner" == "$EUID" && $((mode_value & 0022)) == 0 && $((mode_value & 07000)) == 0 ]]
|
|
}
|
|
|
|
validate_test_mode() {
|
|
local active_parent resolved key
|
|
[[ "$TEST_MODE" == 0 || "$TEST_MODE" == 1 ]] || { fail 'invalid test-mode flag'; return 1; }
|
|
if [[ "$TEST_MODE" == 0 ]]; then
|
|
for key in ${!PLATFORM_BLACKBOX_EDGE_@}; do
|
|
fail "production rejects Blackbox edge override: $key"
|
|
return 1
|
|
done
|
|
[[ "$KUBECTL" == "$KUBECTL_DEFAULT" && "$SUDO" == "$SUDO_DEFAULT" && "$DATE_BIN" == "$DATE_DEFAULT" && "$SLEEP_BIN" == "$SLEEP_DEFAULT" && "$ACTIVE" == "$ACTIVE_DEFAULT" && "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || {
|
|
fail 'production Blackbox edge constants are not exact'; return 1;
|
|
}
|
|
rollback_anchor=/
|
|
return 0
|
|
fi
|
|
active_parent=${ACTIVE%/*}
|
|
resolved="$("$REALPATH_BIN" --canonicalize-existing -- "$active_parent/.." 2>/dev/null)" || {
|
|
fail 'test fixture root does not resolve'; return 1;
|
|
}
|
|
fixture_root=$resolved
|
|
[[ "$fixture_root" == /tmp/platform-blackbox-edge-test.* && "$ACTIVE" == "$fixture_root/active/learn-services" && "$ROLLBACK_BASE" == "$fixture_root/rollbacks" && -d "$fixture_root/bin" && ! -L "$fixture_root/bin" ]] || {
|
|
fail 'test mode is restricted to one isolated fixture tree'; return 1;
|
|
}
|
|
safe_fixture_command "$KUBECTL" "$fixture_root/bin" &&
|
|
safe_fixture_command "$SUDO" "$fixture_root/bin" &&
|
|
safe_fixture_command "$DATE_BIN" "$fixture_root/bin" &&
|
|
safe_fixture_command "$SLEEP_BIN" "$fixture_root/bin" || {
|
|
fail 'test command boundary is unsafe'; return 1;
|
|
}
|
|
rollback_anchor=$fixture_root
|
|
}
|
|
|
|
parse_args() {
|
|
local execute_count=0 context_count=0
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--execute) execute=true; execute_count=$((execute_count + 1)); shift ;;
|
|
--context)
|
|
(( $# >= 2 )) || { fail '--context requires a value'; return 1; }
|
|
context=$2; context_count=$((context_count + 1)); shift 2
|
|
;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; fail "unsupported argument: $1"; return 1 ;;
|
|
esac
|
|
done
|
|
(( execute_count <= 1 && context_count <= 1 )) || { fail 'duplicate execute argument'; return 1; }
|
|
if [[ "$execute" == true ]]; then
|
|
[[ "$context" =~ ^[A-Za-z0-9._:-]+$ ]] || { fail '--execute requires an exact --context'; return 1; }
|
|
else
|
|
[[ -z "$context" ]] || { fail '--context is execute-only'; return 1; }
|
|
fi
|
|
}
|
|
|
|
root_run() { "$SUDO" -n "$@"; }
|
|
root_metadata() { root_run "$STAT_BIN" -c '%F|%u:%g|%a|%h' -- "$1"; }
|
|
root_owner_expected() {
|
|
if [[ "$TEST_MODE" == 0 ]]; then [[ "$1" == 0:0 ]]; else [[ "$1" == "$EUID:$($ID_BIN -g)" ]]; fi
|
|
}
|
|
|
|
root_safe_ancestor_dir() {
|
|
local path=$1 metadata type owner mode links mode_value
|
|
root_run "$TEST_BIN" ! -L "$path" || return 1
|
|
root_run "$TEST_BIN" -d "$path" || return 1
|
|
metadata="$(root_metadata "$path")" || return 1
|
|
IFS='|' read -r type owner mode links <<<"$metadata"
|
|
mode_value=$((8#$mode))
|
|
[[ "$type" == directory && "$links" =~ ^[1-9][0-9]*$ ]] || return 1
|
|
root_owner_expected "$owner" || return 1
|
|
(( (mode_value & 0022) == 0 ))
|
|
}
|
|
|
|
root_safe_dir() {
|
|
local path=$1 expected_mode=$2 metadata type owner mode links
|
|
root_safe_ancestor_dir "$path" || return 1
|
|
metadata="$(root_metadata "$path")" || return 1
|
|
IFS='|' read -r type owner mode links <<<"$metadata"
|
|
[[ "$mode" == "$expected_mode" && "$links" =~ ^[1-9][0-9]*$ ]]
|
|
}
|
|
|
|
root_safe_file() {
|
|
local path=$1 expected_mode=$2 metadata type owner mode links
|
|
root_run "$TEST_BIN" ! -L "$path" || return 1
|
|
root_run "$TEST_BIN" -f "$path" || return 1
|
|
metadata="$(root_metadata "$path")" || return 1
|
|
IFS='|' read -r type owner mode links <<<"$metadata"
|
|
[[ ( "$type" == 'regular file' || "$type" == 'regular empty file' ) && "$mode" == "$expected_mode" && "$links" == 1 ]] || return 1
|
|
root_owner_expected "$owner"
|
|
}
|
|
|
|
validate_root_chain() {
|
|
local target=$1 anchor=$2 canonical relative current component
|
|
[[ "$target" == /* && "$anchor" == /* && "$target" != *'//' && "$target" != */./* && "$target" != */../* ]] || return 1
|
|
canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$target")" || return 1
|
|
[[ "$canonical" == "$target" && "$canonical" != *$'\n'* ]] || return 1
|
|
canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$anchor")" || return 1
|
|
[[ "$canonical" == "$anchor" && "$canonical" != *$'\n'* ]] || return 1
|
|
if [[ "$anchor" == / ]]; then
|
|
# Root has no separator to append: "$anchor/" would become "//" and
|
|
# reject every normal absolute descendant after canonicalization.
|
|
[[ "$target" == /* ]] || return 1
|
|
else
|
|
[[ "$target" == "$anchor" || "$target" == "$anchor/"* ]] || return 1
|
|
fi
|
|
current=$anchor
|
|
root_safe_ancestor_dir "$current" || return 1
|
|
relative=${target#"$anchor"}
|
|
relative=${relative#/}
|
|
[[ -z "$relative" ]] && return 0
|
|
IFS='/' read -r -a _blackbox_path_parts <<<"$relative"
|
|
for component in "${_blackbox_path_parts[@]}"; do
|
|
[[ -n "$component" && "$component" != . && "$component" != .. ]] || return 1
|
|
if [[ "$current" == / ]]; then current="/$component"; else current="$current/$component"; fi
|
|
root_safe_ancestor_dir "$current" || return 1
|
|
done
|
|
}
|
|
|
|
validate_rollback_root() {
|
|
local id=${PLATFORM_OBSERVABILITY_ROLLBACK_ID:-}
|
|
[[ "$id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || { fail 'PLATFORM_OBSERVABILITY_ROLLBACK_ID format is invalid'; return 1; }
|
|
if [[ "$TEST_MODE" == 0 ]]; then [[ "$ROLLBACK_BASE" == "$ROLLBACK_BASE_DEFAULT" ]] || { fail 'production rollback base is not exact'; return 1; }; fi
|
|
rollback_root="$ROLLBACK_BASE/observability-$id"
|
|
proof_path="$rollback_root/blackbox-source-proof.env"
|
|
validate_root_chain "$ROLLBACK_BASE" "$rollback_anchor" || { fail 'rollback base lineage is unsafe'; return 1; }
|
|
validate_root_chain "$rollback_root" "$rollback_anchor" || { fail 'rollback root lineage is unsafe'; return 1; }
|
|
root_safe_dir "$rollback_root" 700 || { fail 'rollback root metadata is unsafe'; return 1; }
|
|
}
|
|
|
|
validate_proof_target_absent() {
|
|
validate_rollback_root || return 1
|
|
[[ "$proof_path" == "$rollback_root/blackbox-source-proof.env" ]] || { fail 'source-proof path is not exact'; return 1; }
|
|
root_run "$TEST_BIN" ! -e "$proof_path" && root_run "$TEST_BIN" ! -L "$proof_path" || {
|
|
fail 'source-proof evidence already exists or is unsafe'; return 1;
|
|
}
|
|
}
|
|
|
|
validate_active_guard_expected() {
|
|
local expected=$1 deny_sha metadata type owner mode links active_sha
|
|
require_regular_source "$DENY_GUARD" || return 1
|
|
is_sha256 "$expected" || { fail 'expected deny guard hash is invalid'; return 1; }
|
|
deny_sha="$(sha_of "$DENY_GUARD")" || { fail 'cannot hash deny guard source'; return 1; }
|
|
[[ "$deny_sha" == "$expected" ]] || { fail 'deny guard source changed during proof'; return 1; }
|
|
root_run "$TEST_BIN" ! -L "$ACTIVE" && root_run "$TEST_BIN" -f "$ACTIVE" || {
|
|
fail 'active Nginx config is missing or symlinked'; return 1;
|
|
}
|
|
metadata="$(root_metadata "$ACTIVE")" || { fail 'cannot stat active Nginx config'; return 1; }
|
|
IFS='|' read -r type owner mode links <<<"$metadata"
|
|
[[ "$type" == 'regular file' && "$links" == 1 ]] || { fail 'active Nginx config metadata is unsafe'; return 1; }
|
|
root_owner_expected "$owner" || { fail 'active Nginx config owner is unsafe'; return 1; }
|
|
if [[ "$TEST_MODE" == 0 ]]; then [[ "$mode" == 644 ]] || { fail 'active Nginx config mode is unsafe'; return 1; }; fi
|
|
active_sha="$(root_run "$SHA256SUM_BIN" -- "$ACTIVE" | "$AWK_BIN" '{print $1}')" || {
|
|
fail 'cannot hash active Nginx config'; return 1;
|
|
}
|
|
[[ "$active_sha" == "$expected" ]] || { fail 'active Nginx config is not the exact Grafana deny guard'; return 1; }
|
|
}
|
|
|
|
validate_active_guard() {
|
|
local deny_sha
|
|
require_regular_source "$DENY_GUARD" || return 1
|
|
deny_sha="$(sha_of "$DENY_GUARD")" || return 1
|
|
is_sha256 "$deny_sha" || { fail 'deny guard hash is invalid'; return 1; }
|
|
validate_active_guard_expected "$deny_sha" || return 1
|
|
printf '%s\n' "$deny_sha"
|
|
}
|
|
|
|
confirm() {
|
|
local expected="PROVE BLACKBOX PRIVATE EDGE $context" answer
|
|
printf 'Type %s: ' "$expected" >&2
|
|
if [[ "$TEST_MODE" == 1 ]]; then
|
|
answer=${PLATFORM_BLACKBOX_EDGE_CONFIRMATION:-}
|
|
printf '%s\n' "$answer" >&2
|
|
else
|
|
[[ -t 0 ]] || { fail '--execute requires an interactive terminal'; return 1; }
|
|
IFS= read -r answer
|
|
fi
|
|
[[ "$answer" == "$expected" ]] || { fail 'cancelled'; return 1; }
|
|
}
|
|
|
|
require_context_api_auth() {
|
|
local current answer
|
|
current="$("$KUBECTL" --request-timeout="$REQUEST_TIMEOUT" config current-context)" || { fail 'cannot read Kubernetes context'; return 1; }
|
|
[[ "$current" == "$context" && "$current" != *$'\n'* ]] || { fail 'current Kubernetes context does not match --context'; return 1; }
|
|
"$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" get --raw=/readyz >/dev/null || { fail 'Kubernetes API is not ready'; return 1; }
|
|
for answer in \
|
|
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create pods -n "$NAMESPACE")" \
|
|
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i create networkpolicies.networking.k8s.io -n "$NAMESPACE")" \
|
|
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete pods -n "$NAMESPACE")" \
|
|
"$("$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" auth can-i delete networkpolicies.networking.k8s.io -n "$NAMESPACE")"; do
|
|
[[ "$answer" == yes ]] || { fail 'Kubernetes authorization is insufficient'; return 1; }
|
|
done
|
|
root_run "$TRUE_BIN" || { fail 'sudo non-interactive authentication failed'; return 1; }
|
|
}
|
|
|
|
prepare_local_temp() {
|
|
run_temp="$("$MKTEMP_BIN" -d /tmp/platform-blackbox-edge-source.XXXXXXXX)" || return 1
|
|
[[ "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 1
|
|
"$CHMOD_BIN" 0700 -- "$run_temp"
|
|
}
|
|
|
|
cleanup_local_temp() {
|
|
[[ -n "$run_temp" && "$run_temp" == /tmp/platform-blackbox-edge-source.???????? && -d "$run_temp" && ! -L "$run_temp" ]] || return 0
|
|
"$RM_BIN" -rf -- "$run_temp"
|
|
run_temp=''
|
|
}
|
|
|
|
render_network_policy() {
|
|
/bin/cat <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: $policy_name
|
|
namespace: $NAMESPACE
|
|
labels:
|
|
app.kubernetes.io/managed-by: platform-blackbox-edge-source
|
|
platform.hyeonworks.com/source-proof-run: "$run_label"
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
platform.hyeonworks.com/source-proof-run: "$run_label"
|
|
policyTypes: ["Egress"]
|
|
egress:
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels: {kubernetes.io/metadata.name: kube-system}
|
|
podSelector:
|
|
matchLabels: {k8s-app: kube-dns}
|
|
ports:
|
|
- {protocol: UDP, port: 53}
|
|
- {protocol: TCP, port: 53}
|
|
- to:
|
|
- ipBlock: {cidr: 192.168.0.107/32}
|
|
ports: [{protocol: TCP, port: 443}]
|
|
EOF
|
|
}
|
|
|
|
render_pod() {
|
|
/bin/cat <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $pod_name
|
|
namespace: $NAMESPACE
|
|
labels:
|
|
app.kubernetes.io/managed-by: platform-blackbox-edge-source
|
|
platform.hyeonworks.com/source-proof-run: "$run_label"
|
|
spec:
|
|
automountServiceAccountToken: false
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
seccompProfile: {type: RuntimeDefault}
|
|
containers:
|
|
- name: probe
|
|
image: $BUSYBOX_IMAGE
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["sh", "-c", "sleep 120"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: {drop: ["ALL"]}
|
|
EOF
|
|
}
|
|
|
|
validate_object_json() {
|
|
local kind=$1 name=$2 run=$3 source=$4 require_uid=$5 allow_scheduled_node=${6:-false} uid_test
|
|
uid_test='true'
|
|
[[ "$require_uid" == true ]] && uid_test='(.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))'
|
|
case "$kind" in
|
|
networkpolicy)
|
|
"$JQ_BIN" -e --arg namespace "$NAMESPACE" --arg name "$name" --arg run "$run" --argjson uid_required "$([[ "$require_uid" == true ]] && printf true || printf false)" '
|
|
.apiVersion == "networking.k8s.io/v1" and .kind == "NetworkPolicy" and
|
|
.metadata.namespace == $namespace and .metadata.name == $name and
|
|
.metadata.labels == {
|
|
"app.kubernetes.io/managed-by":"platform-blackbox-edge-source",
|
|
"platform.hyeonworks.com/source-proof-run":$run
|
|
} and
|
|
(($uid_required | not) or (.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))) and
|
|
.spec.podSelector == {"matchLabels":{"platform.hyeonworks.com/source-proof-run":$run}} and
|
|
.spec.policyTypes == ["Egress"] and
|
|
.spec.egress == [
|
|
{"to":[{"namespaceSelector":{"matchLabels":{"kubernetes.io/metadata.name":"kube-system"}},"podSelector":{"matchLabels":{"k8s-app":"kube-dns"}}}],"ports":[{"protocol":"UDP","port":53},{"protocol":"TCP","port":53}]},
|
|
{"to":[{"ipBlock":{"cidr":"192.168.0.107/32"}}],"ports":[{"protocol":"TCP","port":443}]}
|
|
]
|
|
' "$source" >/dev/null
|
|
;;
|
|
pod)
|
|
"$JQ_BIN" -e --arg namespace "$NAMESPACE" --arg name "$name" --arg run "$run" --arg image "$BUSYBOX_IMAGE" --arg node "$EXPECTED_NODE" --argjson uid_required "$([[ "$require_uid" == true ]] && printf true || printf false)" --argjson scheduled_node_allowed "$([[ "$allow_scheduled_node" == true ]] && printf true || printf false)" '
|
|
def optional_default($key; $value):
|
|
(has($key) | not) or .[$key] == $value;
|
|
.apiVersion == "v1" and .kind == "Pod" and
|
|
.metadata.namespace == $namespace and .metadata.name == $name and
|
|
.metadata.labels == {
|
|
"app.kubernetes.io/managed-by":"platform-blackbox-edge-source",
|
|
"platform.hyeonworks.com/source-proof-run":$run
|
|
} and
|
|
(($uid_required | not) or (.metadata.uid | type == "string" and test("^[A-Za-z0-9][A-Za-z0-9._-]*$"))) and
|
|
(.spec |
|
|
type == "object" and
|
|
optional_default("dnsPolicy"; "ClusterFirst") and
|
|
optional_default("enableServiceLinks"; true) and
|
|
optional_default("preemptionPolicy"; "PreemptLowerPriority") and
|
|
optional_default("priority"; 0) and
|
|
optional_default("schedulerName"; "default-scheduler") and
|
|
optional_default("serviceAccount"; "default") and
|
|
optional_default("serviceAccountName"; "default") and
|
|
optional_default("terminationGracePeriodSeconds"; 30) and
|
|
((has("nodeName") | not) or ($scheduled_node_allowed and .nodeName == $node)) and
|
|
optional_default("tolerations"; [
|
|
{"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300},
|
|
{"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300}
|
|
]) and
|
|
(.containers | type == "array" and length == 1) and
|
|
(.containers[0] |
|
|
optional_default("resources"; {}) and
|
|
optional_default("terminationMessagePath"; "/dev/termination-log") and
|
|
optional_default("terminationMessagePolicy"; "File")) and
|
|
(del(
|
|
.dnsPolicy, .enableServiceLinks, .preemptionPolicy, .priority,
|
|
.schedulerName, .serviceAccount, .serviceAccountName,
|
|
.terminationGracePeriodSeconds, .tolerations, .nodeName,
|
|
.containers[0].resources, .containers[0].terminationMessagePath,
|
|
.containers[0].terminationMessagePolicy
|
|
) == {
|
|
"automountServiceAccountToken":false,
|
|
"restartPolicy":"Never",
|
|
"securityContext":{"runAsNonRoot":true,"runAsUser":65534,"runAsGroup":65534,"seccompProfile":{"type":"RuntimeDefault"}},
|
|
"containers":[{"name":"probe","image":$image,"imagePullPolicy":"IfNotPresent","command":["sh","-c","sleep 120"],"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}}]
|
|
})
|
|
)
|
|
' "$source" >/dev/null
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
response_uid() {
|
|
local source=$1 uid
|
|
uid="$("$JQ_BIN" -er '.metadata.uid' "$source")" || return 1
|
|
[[ "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
|
|
printf '%s\n' "$uid"
|
|
}
|
|
|
|
local_response_file() {
|
|
"$MKTEMP_BIN" "$run_temp/${1}.response.XXXXXXXX"
|
|
}
|
|
|
|
render_resource() {
|
|
case "$1" in networkpolicy) render_network_policy ;; pod) render_pod ;; *) return 1 ;; esac
|
|
}
|
|
resource_name() { case "$1" in networkpolicy) printf '%s\n' "$policy_name" ;; pod) printf '%s\n' "$pod_name" ;; *) return 1 ;; esac; }
|
|
|
|
server_dry_run_resource() {
|
|
local kind=$1 name response
|
|
name="$(resource_name "$kind")" || return 1
|
|
response="$(local_response_file "${kind}.dry")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$response"
|
|
if ! render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create --dry-run=server -f - -o json >"$response"; then
|
|
"$RM_BIN" -f -- "$response"; return 1
|
|
fi
|
|
validate_object_json "$kind" "$name" "$run_label" "$response" false || { "$RM_BIN" -f -- "$response"; return 1; }
|
|
"$RM_BIN" -f -- "$response"
|
|
}
|
|
|
|
inspect_owned_object() {
|
|
local kind=$1 name=$2 destination=$3 response uid rc=0
|
|
response="$(local_response_file "${kind}.inspect")" || return 3
|
|
"$CHMOD_BIN" 0600 -- "$response"
|
|
"$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$?
|
|
if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 3; fi
|
|
if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi
|
|
validate_object_json "$kind" "$name" "$run_label" "$response" true || { "$RM_BIN" -f -- "$response"; return 2; }
|
|
uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 2; }
|
|
"$RM_BIN" -f -- "$response"
|
|
printf -v "$destination" '%s' "$uid"
|
|
}
|
|
|
|
create_resource() {
|
|
local kind=$1 destination=$2 name response uid rc=0 classification
|
|
name="$(resource_name "$kind")" || return 1
|
|
response="$(local_response_file "${kind}.create")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$response"
|
|
if render_resource "$kind" | "$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" create -f - -o json >"$response"; then
|
|
if validate_object_json "$kind" "$name" "$run_label" "$response" true && uid="$(response_uid "$response")"; then
|
|
"$RM_BIN" -f -- "$response"
|
|
printf -v "$destination" '%s' "$uid"
|
|
return 0
|
|
fi
|
|
fi
|
|
"$RM_BIN" -f -- "$response"
|
|
# A syntactically bad success response is still ambiguous: the API may have
|
|
# committed the exact object, so classify the live state before cleanup.
|
|
if inspect_owned_object "$kind" "$name" "$destination"; then
|
|
return 2
|
|
else
|
|
classification=$?
|
|
[[ "$classification" == 1 ]] && return 1
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
create_temporary_objects() {
|
|
local result
|
|
server_dry_run_resource networkpolicy || { fail 'temporary NetworkPolicy server dry-run failed'; return 1; }
|
|
if create_resource networkpolicy policy_uid; then :; else
|
|
result=$?
|
|
[[ "$result" == 2 ]] && fail 'temporary NetworkPolicy create outcome was ambiguous but owned'
|
|
[[ "$result" != 2 ]] && fail 'temporary NetworkPolicy create failed'
|
|
return 1
|
|
fi
|
|
server_dry_run_resource pod || { fail 'temporary Pod server dry-run failed'; return 1; }
|
|
if create_resource pod pod_uid; then :; else
|
|
result=$?
|
|
[[ "$result" == 2 ]] && fail 'temporary Pod create outcome was ambiguous but owned'
|
|
[[ "$result" != 2 ]] && fail 'temporary Pod create failed'
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
stop_proxy() {
|
|
local pid=$1 attempt
|
|
"$KILL_BIN" -TERM "$pid" 2>/dev/null || true
|
|
for ((attempt=0; attempt<20; attempt++)); do
|
|
if ! "$KILL_BIN" -0 "$pid" 2>/dev/null; then wait "$pid" 2>/dev/null || true; return 0; fi
|
|
"$SLEEP_BIN" 0.05
|
|
done
|
|
"$KILL_BIN" -KILL "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
}
|
|
|
|
api_path_for() {
|
|
case "$1" in
|
|
pod) printf '/api/v1/namespaces/%s/pods/%s\n' "$NAMESPACE" "$2" ;;
|
|
networkpolicy) printf '/apis/networking.k8s.io/v1/namespaces/%s/networkpolicies/%s\n' "$NAMESPACE" "$2" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
object_state() {
|
|
local kind=$1 name=$2 expected_uid=$3 response uid rc=0
|
|
response="$(local_response_file "${kind}.state")" || return 4
|
|
"$CHMOD_BIN" 0600 -- "$response"
|
|
"$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" get "$kind" "$name" --ignore-not-found -o json >"$response" || rc=$?
|
|
if (( rc != 0 )); then "$RM_BIN" -f -- "$response"; return 4; fi
|
|
if [[ ! -s "$response" ]]; then "$RM_BIN" -f -- "$response"; return 1; fi
|
|
uid="$(response_uid "$response")" || { "$RM_BIN" -f -- "$response"; return 4; }
|
|
[[ "$uid" == "$expected_uid" ]] || { "$RM_BIN" -f -- "$response"; return 3; }
|
|
validate_object_json "$kind" "$name" "$run_label" "$response" true true || { "$RM_BIN" -f -- "$response"; return 4; }
|
|
"$RM_BIN" -f -- "$response"
|
|
return 0
|
|
}
|
|
|
|
wait_for_stable_absence() {
|
|
local kind=$1 name=$2 uid=$3 attempt absent=0 state
|
|
for ((attempt=0; attempt<DELETE_WAIT_POLLS; attempt++)); do
|
|
if object_state "$kind" "$name" "$uid"; then
|
|
absent=0
|
|
else
|
|
state=$?
|
|
if [[ "$state" == 1 ]]; then
|
|
absent=$((absent + 1))
|
|
(( absent >= 2 )) && return 0
|
|
else
|
|
return 1
|
|
fi
|
|
fi
|
|
"$SLEEP_BIN" 0.05
|
|
done
|
|
return 1
|
|
}
|
|
|
|
delete_with_uid_precondition() {
|
|
local kind=$1 name=$2 uid=$3 api_path socket options response proxy_log proxy_pid='' attempt http_code='' curl_rc=0
|
|
[[ "$name" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ && "$uid" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
|
|
api_path="$(api_path_for "$kind" "$name")" || return 1
|
|
socket="$run_temp/proxy-${kind}.sock"
|
|
options="$run_temp/delete-${kind}.json"
|
|
response="$run_temp/delete-${kind}.response"
|
|
proxy_log="$run_temp/proxy-${kind}.log"
|
|
[[ ! -e "$socket" && ! -L "$socket" ]] || return 1
|
|
printf '{"apiVersion":"meta.k8s.io/v1","kind":"DeleteOptions","propagationPolicy":"Background","preconditions":{"uid":"%s"}}\n' "$uid" >"$options"
|
|
"$CHMOD_BIN" 0600 -- "$options"
|
|
: >"$response"; "$CHMOD_BIN" 0600 -- "$response"
|
|
"$KUBECTL" --context "$context" --request-timeout="$REQUEST_TIMEOUT" proxy \
|
|
--unix-socket="$socket" --api-prefix=/ --accept-paths="^${api_path}$" \
|
|
--reject-methods='^(GET|POST|PUT|PATCH)$' >"$proxy_log" 2>&1 &
|
|
proxy_pid=$!
|
|
for ((attempt=0; attempt<PROXY_WAIT_POLLS; attempt++)); do
|
|
[[ -S "$socket" ]] && break
|
|
"$KILL_BIN" -0 "$proxy_pid" 2>/dev/null || { wait "$proxy_pid" 2>/dev/null || true; return 1; }
|
|
"$SLEEP_BIN" 0.05
|
|
done
|
|
[[ -S "$socket" ]] || { stop_proxy "$proxy_pid"; return 1; }
|
|
http_code="$("$CURL_BIN" --silent --show-error --max-time 5 --unix-socket "$socket" --output "$response" --write-out '%{http_code}' \
|
|
--request DELETE --header 'Content-Type: application/json' --data-binary "@$options" "http://localhost${api_path}")" || curl_rc=$?
|
|
stop_proxy "$proxy_pid"
|
|
# A transport/status ambiguity is not treated as success on its own. Only
|
|
# two consecutive exact absence reads reclassify it as a completed delete.
|
|
wait_for_stable_absence "$kind" "$name" "$uid"
|
|
}
|
|
|
|
cleanup() {
|
|
local rc=0
|
|
[[ "$cleanup_started" == false ]] || return 0
|
|
cleanup_started=true
|
|
if [[ -n "$pod_uid" ]]; then delete_with_uid_precondition pod "$pod_name" "$pod_uid" || rc=1; fi
|
|
if [[ -n "$policy_uid" ]]; then delete_with_uid_precondition networkpolicy "$policy_name" "$policy_uid" || rc=1; fi
|
|
pod_uid=''; policy_uid=''
|
|
(( rc == 0 )) || printf 'ERROR: temporary Blackbox proof object identity is unknown; manual review required\n' >&2
|
|
return "$rc"
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$?
|
|
trap - EXIT HUP INT TERM
|
|
if [[ -n "$pod_uid" || -n "$policy_uid" ]]; then
|
|
cleanup || { [[ "$rc" == 0 ]] && rc=1; }
|
|
fi
|
|
cleanup_local_temp || { [[ "$rc" == 0 ]] && rc=1; }
|
|
exit "$rc"
|
|
}
|
|
|
|
run_client_request() {
|
|
local host=$1 token=$2 output status rc=0
|
|
output="$("$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" exec "$pod_name" -- sh -ec \
|
|
"wget --no-check-certificate --server-response --spider 'https://$host/?hyeonworks_probe=$token'" 2>&1)" || rc=$?
|
|
status="$("$AWK_BIN" '/^ HTTP\// { code=$2 } /^HTTP\// { code=$2 } END { print code }' <<<"$output")"
|
|
[[ "$rc" == 1 && "$status" == 403 ]]
|
|
}
|
|
|
|
collect_log_results() {
|
|
local one=$1 two=$2 three=$3 output pair addr status
|
|
output="$(root_run "$AWK_BIN" -v one="$one" -v two="$two" -v three="$three" '
|
|
$0 ~ ("hyeonworks_probe=" one "([& ]| HTTP)") { if (++count[1] == 1) value[1]=$1 " " $9 }
|
|
$0 ~ ("hyeonworks_probe=" two "([& ]| HTTP)") { if (++count[2] == 1) value[2]=$1 " " $9 }
|
|
$0 ~ ("hyeonworks_probe=" three "([& ]| HTTP)") { if (++count[3] == 1) value[3]=$1 " " $9 }
|
|
END { if (count[1] != 1 || count[2] != 1 || count[3] != 1) exit 42; print value[1]; print value[2]; print value[3] }
|
|
' /var/log/nginx/access.log)" || return 1
|
|
mapfile -t LOG_RESULTS <<<"$output"
|
|
(( ${#LOG_RESULTS[@]} == 3 )) || return 1
|
|
for pair in "${LOG_RESULTS[@]}"; do
|
|
read -r addr status <<<"$pair"
|
|
[[ "$addr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ && "$status" == 403 ]] || return 1
|
|
done
|
|
}
|
|
|
|
write_proof() {
|
|
local nginx_sha=$1 tested=$2 local_file root_temporary canonical suffix
|
|
local_file="$("$MKTEMP_BIN" "$run_temp/proof.XXXXXXXX")" || return 1
|
|
"$CHMOD_BIN" 0600 -- "$local_file"
|
|
printf 'schema=%s\nrollback_id=%s\nnginx_sha256=%s\ntested_at_utc=%s\ngrafana_remote_addr=%s\ngrafana_status=403\nstorage_admin_remote_addr=%s\nstorage_admin_status=403\ndb_admin_remote_addr=%s\ndb_admin_status=403\n' \
|
|
"$PROOF_SCHEMA" "${PLATFORM_OBSERVABILITY_ROLLBACK_ID:?}" "$nginx_sha" "$tested" \
|
|
"${LOG_RESULTS[0]%% *}" "${LOG_RESULTS[1]%% *}" "${LOG_RESULTS[2]%% *}" >"$local_file"
|
|
root_temporary="$(root_run "$MKTEMP_BIN" --tmpdir="$rollback_root" '.blackbox-source-proof.env.new.XXXXXXXX')" || { "$RM_BIN" -f -- "$local_file"; return 1; }
|
|
suffix=${root_temporary#"$rollback_root/.blackbox-source-proof.env.new."}
|
|
[[ "$root_temporary" == "$rollback_root/.blackbox-source-proof.env.new."* && "$suffix" =~ ^[A-Za-z0-9]{8}$ && "$root_temporary" != *$'\n'* ]] || {
|
|
"$RM_BIN" -f -- "$local_file"; return 1;
|
|
}
|
|
canonical="$(root_run "$REALPATH_BIN" --canonicalize-existing -- "$root_temporary")" || { "$RM_BIN" -f -- "$local_file"; return 1; }
|
|
[[ "$canonical" == "$root_temporary" ]] && root_safe_file "$root_temporary" 600 || {
|
|
"$RM_BIN" -f -- "$local_file"; return 1;
|
|
}
|
|
if ! root_run "$INSTALL_BIN" -o root -g root -m 0600 -- "$local_file" "$root_temporary" ||
|
|
! root_safe_file "$root_temporary" 600 ||
|
|
! root_run "$CMP_BIN" --silent -- "$local_file" "$root_temporary" ||
|
|
! root_run "$LN_BIN" -nT -- "$root_temporary" "$proof_path"; then
|
|
root_safe_file "$root_temporary" 600 && root_run "$UNLINK_BIN" -- "$root_temporary" >/dev/null 2>&1 || true
|
|
"$RM_BIN" -f -- "$local_file"
|
|
return 1
|
|
fi
|
|
root_run "$UNLINK_BIN" -- "$root_temporary" || { "$RM_BIN" -f -- "$local_file"; return 1; }
|
|
root_safe_file "$proof_path" 600 && root_run "$CMP_BIN" --silent -- "$local_file" "$proof_path" || {
|
|
"$RM_BIN" -f -- "$local_file"; return 1;
|
|
}
|
|
"$RM_BIN" -f -- "$local_file"
|
|
}
|
|
|
|
execute_proof() {
|
|
local nginx_sha stamp random token_one token_two token_three tested
|
|
nginx_sha="$(validate_active_guard)" || return 1
|
|
validate_rollback_root || return 1
|
|
require_context_api_auth || return 1
|
|
confirm || return 1
|
|
prepare_local_temp || { fail 'cannot create private validator temporary directory'; return 1; }
|
|
stamp="$("$DATE_BIN" -u +%Y%m%dT%H%M%SZ)" || return 1
|
|
random="${RANDOM}${RANDOM}"
|
|
run_label="${stamp,,}-${random}"
|
|
policy_name="blackbox-edge-source-egress-${run_label}"
|
|
pod_name="blackbox-edge-source-${run_label}"
|
|
token_one="${stamp}-${RANDOM}${RANDOM}"
|
|
token_two="${stamp}-${RANDOM}${RANDOM}"
|
|
token_three="${stamp}-${RANDOM}${RANDOM}"
|
|
[[ "$token_one" != "$token_two" && "$token_one" != "$token_three" && "$token_two" != "$token_three" ]] || { fail 'could not create unique probe tokens'; return 1; }
|
|
# This is the commit gate: it is immediately after confirmation and all
|
|
# local preparation, and immediately before the first Kubernetes mutation.
|
|
validate_active_guard_expected "$nginx_sha" || return 1
|
|
validate_proof_target_absent || return 1
|
|
create_temporary_objects || return 1
|
|
"$KUBECTL" --context "$context" -n "$NAMESPACE" --request-timeout="$REQUEST_TIMEOUT" wait --for=condition=Ready --timeout="$READY_TIMEOUT" "pod/$pod_name" >/dev/null || {
|
|
fail 'temporary Pod did not become Ready'; return 1;
|
|
}
|
|
run_client_request "$GRAFANA_HOST" "$token_one" || { fail 'Grafana client status is not exact BusyBox 403'; return 1; }
|
|
run_client_request "$STORAGE_HOST" "$token_two" || { fail 'AIStor client status is not exact BusyBox 403'; return 1; }
|
|
run_client_request "$DB_HOST" "$token_three" || { fail 'pgAdmin client status is not exact BusyBox 403'; return 1; }
|
|
collect_log_results "$token_one" "$token_two" "$token_three" || { fail 'Nginx log did not contain exactly one 403 result per probe'; return 1; }
|
|
tested="$("$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" || return 1
|
|
write_proof "$nginx_sha" "$tested" || { fail 'cannot atomically write source-proof evidence'; return 1; }
|
|
cleanup || { fail 'temporary object cleanup failed'; return 1; }
|
|
printf 'BLACKBOX PRIVATE EDGE SOURCE PASS\n'
|
|
}
|
|
|
|
main() {
|
|
parse_args "$@" || return 1
|
|
validate_test_mode || return 1
|
|
require_regular_source "$DENY_GUARD" || return 1
|
|
if [[ "$execute" == false ]]; then
|
|
printf 'BLACKBOX_EDGE_SOURCE_DENY_GUARD_SHA256=%s\nBLACKBOX_EDGE_SOURCE_BUSYBOX_IMAGE=%s\nBLACKBOX_PRIVATE_EDGE_SOURCE_DRY_RUN=PASS\n' "$(sha_of "$DENY_GUARD")" "$BUSYBOX_IMAGE"
|
|
return 0
|
|
fi
|
|
for command_path in "$AWK_BIN" "$CHMOD_BIN" "$CMP_BIN" "$CURL_BIN" "$ID_BIN" "$INSTALL_BIN" "$JQ_BIN" "$KILL_BIN" "$LN_BIN" "$MKTEMP_BIN" "$REALPATH_BIN" "$RM_BIN" "$SHA256SUM_BIN" "$SLEEP_BIN" "$STAT_BIN" "$TEST_BIN" "$TRUE_BIN" "$UNLINK_BIN"; do
|
|
[[ -x "$command_path" ]] || { fail "required command is missing: $command_path"; return 1; }
|
|
done
|
|
[[ -x "$KUBECTL" && -x "$SUDO" && -x "$DATE_BIN" ]] || { fail 'required command boundary is missing'; return 1; }
|
|
trap on_exit EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
execute_proof
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi
|