1182 lines
48 KiB
Bash
Executable File
1182 lines
48 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
|
|
readonly APPLY="$ROOT/scripts/bootstrap/apply-host-nginx-observability.sh"
|
|
readonly DENY="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf"
|
|
readonly FULL="$ROOT/infrastructure/networking/host-nginx/learn-services-observability.conf"
|
|
readonly ROLLBACK_ID=20260812T120000Z
|
|
WORK=''
|
|
ASSERTIONS=0
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
pass() {
|
|
ASSERTIONS=$((ASSERTIONS + 1))
|
|
printf 'PASS: %s\n' "$1"
|
|
}
|
|
|
|
cleanup() {
|
|
local rc=$?
|
|
trap - EXIT HUP INT TERM
|
|
case "$WORK" in
|
|
/tmp/platform-host-nginx-test.*) rm -rf -- "$WORK" ;;
|
|
esac
|
|
exit "$rc"
|
|
}
|
|
trap cleanup EXIT HUP INT TERM
|
|
|
|
make_fakes() {
|
|
local root=$1
|
|
mkdir -m 0700 -p "$root/bin"
|
|
cat >"$root/bin/nginx" <<'SH'
|
|
#!/usr/bin/env bash
|
|
[[ "${PLATFORM_TEST_VIA_SUDO:-0}" == 1 ]] || {
|
|
printf 'nginx-without-sudo %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
exit 91
|
|
}
|
|
printf 'nginx %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
exit 0
|
|
SH
|
|
cat >"$root/bin/systemctl" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'systemctl %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
if [[ "${1:-} ${2:-}" == 'reload nginx' ]]; then
|
|
count=0
|
|
[[ ! -f "$PLATFORM_TEST_RELOAD_COUNTER" ]] || read -r count <"$PLATFORM_TEST_RELOAD_COUNTER"
|
|
count=$((count + 1))
|
|
printf '%d\n' "$count" >"$PLATFORM_TEST_RELOAD_COUNTER"
|
|
if [[ "${PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT:-0}" == "$count" ]]; then
|
|
exit 94
|
|
fi
|
|
fi
|
|
exit 0
|
|
SH
|
|
cat >"$root/bin/sudo" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'sudo %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
if [[ "${1:-}" == -v ]]; then
|
|
: >"$PLATFORM_TEST_SUDO_REFRESHED"
|
|
exit 0
|
|
fi
|
|
if [[ "${1:-}" == -n ]]; then shift; fi
|
|
(( $# == 0 )) && exit 0
|
|
[[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92
|
|
export PLATFORM_TEST_VIA_SUDO=1
|
|
exec "$@"
|
|
SH
|
|
cat >"$root/bin/test" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'test %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
exec /usr/bin/test "$@"
|
|
SH
|
|
cat >"$root/bin/install" <<'SH'
|
|
#!/usr/bin/env bash
|
|
args=()
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
-o|-g) shift 2 ;;
|
|
*) args+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
if (( ${#args[@]} > 0 )); then
|
|
destination=${args[$((${#args[@]} - 1))]}
|
|
if [[ -n "${PLATFORM_TEST_INSTALL_FAIL_MATCH:-}" &&
|
|
"${args[*]}" == *"$PLATFORM_TEST_INSTALL_FAIL_MATCH"* ]]; then
|
|
exit 93
|
|
fi
|
|
if [[ -n "${PLATFORM_HOST_NGINX_ACTIVE:-}" &&
|
|
"$destination" == "$PLATFORM_HOST_NGINX_ACTIVE" ]]; then
|
|
count=0
|
|
[[ ! -f "$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" ]] ||
|
|
read -r count <"$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER"
|
|
count=$((count + 1))
|
|
printf '%d\n' "$count" >"$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER"
|
|
if [[ "${PLATFORM_TEST_FAIL_ACTIVE_INSTALL_AT:-0}" == "$count" ]]; then
|
|
exit 95
|
|
fi
|
|
fi
|
|
fi
|
|
/usr/bin/install "${args[@]}"
|
|
rc=$?
|
|
if (( rc == 0 )) && [[ -n "${PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH:-}" &&
|
|
"${args[*]}" == *"$PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH"* ]]; then
|
|
chmod 0644 "$destination"
|
|
fi
|
|
exit "$rc"
|
|
SH
|
|
cat >"$root/bin/openssl" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'openssl %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
# This boundary deliberately reports a valid exact SAN and successful checkend.
|
|
# A missing privkey must therefore be rejected by our validator, not by OpenSSL.
|
|
if [[ " $* " == *' -ext subjectAltName '* ]]; then
|
|
printf 'X509v3 Subject Alternative Name:\n DNS:grafana.learn.hyeonworks.com\n'
|
|
fi
|
|
exit 0
|
|
SH
|
|
for command in dig curl kubectl; do
|
|
cat >"$root/bin/$command" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf '%s %s\n' "${0##*/}" "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
exit 0
|
|
SH
|
|
done
|
|
cat >"$root/bin/preflight" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'preflight\n' >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
exit "${PLATFORM_TEST_PREFLIGHT_RC:-0}"
|
|
SH
|
|
cat >"$root/bin/postcheck" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'postcheck\n' >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
exit "${PLATFORM_TEST_POSTCHECK_RC:-0}"
|
|
SH
|
|
cat >"$root/bin/certbot" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'certbot %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG"
|
|
if [[ "${1:-}" == plugins ]]; then
|
|
printf 'dns-cloudflare\n'
|
|
fi
|
|
exit 0
|
|
SH
|
|
cat >"$root/bin/date" <<'SH'
|
|
#!/usr/bin/env bash
|
|
if [[ -n "${PLATFORM_TEST_DATE_NOW_EPOCH:-}" && "$*" == '-u +%s' ]]; then
|
|
printf '%s\n' "$PLATFORM_TEST_DATE_NOW_EPOCH"
|
|
exit 0
|
|
fi
|
|
exec /usr/bin/date "$@"
|
|
SH
|
|
chmod 0755 "$root/bin/"*
|
|
}
|
|
|
|
write_inventory() {
|
|
local root=$1 phase=$2 sha
|
|
mkdir -m 0700 -p -- "$root/$phase"
|
|
printf '{"phase":"%s"}\n' "$phase" >"$root/$phase/inventory.json"
|
|
chmod 0600 "$root/$phase/inventory.json"
|
|
sha="$(sha256sum "$root/$phase/inventory.json" | awk '{print $1}')"
|
|
printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256"
|
|
chmod 0600 "$root/$phase/inventory.sha256"
|
|
printf '%s' "$sha"
|
|
}
|
|
|
|
write_acceptance_v2() {
|
|
local path=$1 initial_sha=$2 post_sha=$3
|
|
local gate=${4:-RISK_ACCEPTED} ref uid
|
|
case "$gate" in
|
|
RECOVERY)
|
|
ref=strict-recovery-evidence-v1
|
|
uid=not-applicable
|
|
;;
|
|
RISK_ACCEPTED)
|
|
ref=2026-08-14-observability-slack-recovery-risk-acceptance-design
|
|
uid=1000
|
|
;;
|
|
*) return 2 ;;
|
|
esac
|
|
cat >"$path" <<EOF
|
|
schema=platform-observability-rules-alerts-v2
|
|
rollback_id=$ROLLBACK_ID
|
|
target_initial_sha256=$initial_sha
|
|
post_substrate_sha256=$post_sha
|
|
slack_deployment_gate=$gate
|
|
slack_gate_approval_ref=$ref
|
|
slack_gate_accepted_by_uid=$uid
|
|
accepted_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
state=accepted
|
|
EOF
|
|
chmod 0600 "$path"
|
|
}
|
|
|
|
write_acceptance_v1() {
|
|
local path=$1 initial_sha=$2 post_sha=$3
|
|
cat >"$path" <<EOF
|
|
schema=platform-observability-rules-alerts-v1
|
|
rollback_id=$ROLLBACK_ID
|
|
target_initial_sha256=$initial_sha
|
|
post_substrate_sha256=$post_sha
|
|
accepted_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
state=accepted
|
|
EOF
|
|
chmod 0600 "$path"
|
|
}
|
|
|
|
write_evidence() {
|
|
local rollback_root=$1 metric_root=$2 deny_sha=$3 initial_sha post_sha now
|
|
mkdir -p -- "$metric_root"
|
|
chmod 0700 "$metric_root"
|
|
initial_sha="$(write_inventory "$metric_root" target-initial)"
|
|
post_sha="$(write_inventory "$metric_root" post-substrate)"
|
|
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
mkdir -m 0700 -p -- "$rollback_root/access-rules-alerts"
|
|
write_acceptance_v2 "$rollback_root/access-rules-alerts/acceptance.env" \
|
|
"$initial_sha" "$post_sha"
|
|
cat >"$rollback_root/blackbox-source-proof.env" <<EOF
|
|
schema=platform-blackbox-source-v1
|
|
rollback_id=$ROLLBACK_ID
|
|
nginx_sha256=$deny_sha
|
|
tested_at_utc=$now
|
|
grafana_remote_addr=10.42.0.42
|
|
grafana_status=403
|
|
storage_admin_remote_addr=10.42.0.42
|
|
storage_admin_status=403
|
|
db_admin_remote_addr=10.42.0.42
|
|
db_admin_status=403
|
|
EOF
|
|
chmod 0600 "$rollback_root/access-rules-alerts/acceptance.env" \
|
|
"$rollback_root/blackbox-source-proof.env"
|
|
}
|
|
|
|
run_apply() {
|
|
local fixture=$1
|
|
shift
|
|
env \
|
|
PATH="$fixture/bin:$PATH" \
|
|
PLATFORM_HOST_NGINX_TEST_MODE=1 \
|
|
PLATFORM_HOST_NGINX_ACTIVE="$fixture/active/learn-services" \
|
|
PLATFORM_HOST_NGINX_ENABLED="$fixture/enabled/learn-services" \
|
|
PLATFORM_HOST_NGINX_ROLLBACK_BASE="$fixture/rollbacks" \
|
|
PLATFORM_HOST_NGINX_CERT_DIR="$fixture/cert" \
|
|
PLATFORM_HOST_NGINX_NGINX_BIN="${PLATFORM_TEST_NGINX_BIN:-$fixture/bin/nginx}" \
|
|
PLATFORM_HOST_NGINX_SYSTEMCTL_BIN="${PLATFORM_TEST_SYSTEMCTL_BIN:-$fixture/bin/systemctl}" \
|
|
PLATFORM_HOST_NGINX_SUDO_BIN="${PLATFORM_TEST_SUDO_BIN:-$fixture/bin/sudo}" \
|
|
PLATFORM_HOST_NGINX_TEST_BIN="${PLATFORM_TEST_TEST_BIN:-$fixture/bin/test}" \
|
|
PLATFORM_HOST_NGINX_CERTBOT_BIN="${PLATFORM_TEST_CERTBOT_BIN:-$fixture/bin/certbot}" \
|
|
PLATFORM_HOST_NGINX_PREFLIGHT_COMMAND="${PLATFORM_TEST_PREFLIGHT_COMMAND:-$fixture/bin/preflight}" \
|
|
PLATFORM_HOST_NGINX_POSTCHECK_COMMAND="${PLATFORM_TEST_POSTCHECK_COMMAND:-$fixture/bin/postcheck}" \
|
|
PLATFORM_HOST_NGINX_CONFIRMATION=APPLY \
|
|
PLATFORM_TEST_COMMAND_LOG="$fixture/commands.log" \
|
|
PLATFORM_TEST_SUDO_REFRESHED="$fixture/sudo-refreshed" \
|
|
PLATFORM_TEST_RELOAD_COUNTER="$fixture/reload-counter" \
|
|
PLATFORM_TEST_ACTIVE_INSTALL_COUNTER="$fixture/active-install-counter" \
|
|
PLATFORM_TEST_PREFLIGHT_RC="${PLATFORM_TEST_PREFLIGHT_RC:-0}" \
|
|
PLATFORM_TEST_POSTCHECK_RC="${PLATFORM_TEST_POSTCHECK_RC:-0}" \
|
|
PLATFORM_TEST_INSTALL_FAIL_MATCH="${PLATFORM_TEST_INSTALL_FAIL_MATCH:-}" \
|
|
PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH="${PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH:-}" \
|
|
PLATFORM_TEST_FAIL_ACTIVE_INSTALL_AT="${PLATFORM_TEST_FAIL_ACTIVE_INSTALL_AT:-0}" \
|
|
PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT="${PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT:-0}" \
|
|
PLATFORM_TEST_DATE_NOW_EPOCH="${PLATFORM_TEST_DATE_NOW_EPOCH:-}" \
|
|
PLATFORM_OBSERVABILITY_ROLLBACK_ID="$ROLLBACK_ID" \
|
|
bash "$APPLY" "$@"
|
|
}
|
|
|
|
assert_test_command_override_rejected() {
|
|
local label=$1 override=$2 fixture rc
|
|
fixture="$(new_fixture "rejects-$label-override")"
|
|
case "$label" in
|
|
nginx) PLATFORM_TEST_NGINX_BIN=$override ;;
|
|
systemctl) PLATFORM_TEST_SYSTEMCTL_BIN=$override ;;
|
|
test) PLATFORM_TEST_TEST_BIN=$override ;;
|
|
certbot) PLATFORM_TEST_CERTBOT_BIN=$override ;;
|
|
preflight) PLATFORM_TEST_PREFLIGHT_COMMAND=$override ;;
|
|
postcheck) PLATFORM_TEST_POSTCHECK_COMMAND=$override ;;
|
|
*) fail "unknown test command override: $label" ;;
|
|
esac
|
|
if run_apply "$fixture" >"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
unset PLATFORM_TEST_NGINX_BIN PLATFORM_TEST_SYSTEMCTL_BIN PLATFORM_TEST_TEST_BIN PLATFORM_TEST_CERTBOT_BIN \
|
|
PLATFORM_TEST_PREFLIGHT_COMMAND PLATFORM_TEST_POSTCHECK_COMMAND
|
|
(( rc != 0 )) || fail "test mode accepted an escaped $label command override"
|
|
}
|
|
|
|
assert_test_command_boundary_mutation_rejected() {
|
|
local mutation=$1 fixture rc
|
|
fixture="$(new_fixture "test-boundary-$mutation")"
|
|
case "$mutation" in
|
|
missing) rm -f -- "$fixture/bin/test" ;;
|
|
unsafe-mode) chmod 0775 "$fixture/bin/test" ;;
|
|
*) fail "unknown test command boundary mutation: $mutation" ;;
|
|
esac
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
(( rc != 0 )) || fail "test mode accepted $mutation test command boundary"
|
|
grep -Fq 'test command boundary is unsafe: test' "$fixture/output" ||
|
|
fail "$mutation test command boundary did not identify the test boundary"
|
|
cmp -s "$fixture/active/learn-services" "$DENY" ||
|
|
fail "$mutation test command boundary changed active bytes"
|
|
[[ ! -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" &&
|
|
! -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]] ||
|
|
fail "$mutation test command boundary created a Host ledger"
|
|
[[ ! -s "$fixture/commands.log" ]] ||
|
|
fail "$mutation test command boundary reached a command boundary"
|
|
! grep -Fq 'Type APPLY' "$fixture/output" ||
|
|
fail "$mutation test command boundary reached the APPLY prompt"
|
|
pass "test mode rejects $mutation test command boundary before contact, ledger, or prompt"
|
|
}
|
|
|
|
new_fixture() {
|
|
local name=$1 fixture deny_sha
|
|
fixture="$WORK/$name"
|
|
mkdir -m 0700 -p "$fixture"
|
|
mkdir -p "$fixture/active" "$fixture/enabled" "$fixture/rollbacks/observability-$ROLLBACK_ID"
|
|
cp "$DENY" "$fixture/active/learn-services"
|
|
ln -s "$fixture/active/learn-services" "$fixture/enabled/learn-services"
|
|
chmod 0644 "$fixture/active/learn-services"
|
|
chmod 0700 "$fixture/rollbacks/observability-$ROLLBACK_ID"
|
|
: >"$fixture/commands.log"
|
|
make_fakes "$fixture"
|
|
deny_sha="$(sha256sum "$DENY" | awk '{print $1}')"
|
|
write_evidence "$fixture/rollbacks/observability-$ROLLBACK_ID" "$fixture/metrics" "$deny_sha"
|
|
printf '%s' "$fixture"
|
|
}
|
|
|
|
acceptance_inventory_shas() {
|
|
local fixture=$1 initial_sha post_sha
|
|
initial_sha="$(sha256sum "$fixture/metrics/target-initial/inventory.json" | awk '{print $1}')"
|
|
post_sha="$(sha256sum "$fixture/metrics/post-substrate/inventory.json" | awk '{print $1}')"
|
|
printf '%s|%s' "$initial_sha" "$post_sha"
|
|
}
|
|
|
|
evidence_contract_failures=()
|
|
task2_contract_failures=()
|
|
|
|
record_evidence_contract_failure() {
|
|
evidence_contract_failures+=("$1")
|
|
printf 'RED: %s\n' "$1" >&2
|
|
}
|
|
|
|
record_task2_contract_failure() {
|
|
task2_contract_failures+=("$1")
|
|
printf 'RED: %s\n' "$1" >&2
|
|
}
|
|
|
|
host_tree_fingerprint() {
|
|
local root=$1 entry
|
|
if [[ ! -e "$root" && ! -L "$root" ]]; then
|
|
printf 'ABSENT'
|
|
return 0
|
|
fi
|
|
{
|
|
stat -c '.|%F|%u:%g|%a|%h|%N' -- "$root"
|
|
if [[ -d "$root" && ! -L "$root" ]]; then
|
|
find -P "$root" -mindepth 1 -printf '%P|%y|%U:%G|%m|%n|%l\n' | sort
|
|
while IFS= read -r entry; do
|
|
printf '%s|' "${entry#"$root"/}"
|
|
sha256sum -- "$entry" | awk '{print $1}'
|
|
done < <(find -P "$root" -type f -links 1 -print | sort)
|
|
fi
|
|
} | sha256sum | awk '{print $1}'
|
|
}
|
|
|
|
exercise_first_cutover_residue() {
|
|
local mutation=$1 fixture host_root payload_root ledger before after rc failed=false
|
|
fixture="$(new_fixture "first-cutover-residue-$mutation")"
|
|
host_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx"
|
|
payload_root="$host_root/payloads"
|
|
ledger="$host_root/stages.tsv"
|
|
case "$mutation" in
|
|
empty-root)
|
|
mkdir -m 0700 "$host_root"
|
|
;;
|
|
normal-ledger)
|
|
mkdir -m 0700 "$host_root" "$payload_root"
|
|
printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' >"$ledger"
|
|
chmod 0600 "$ledger"
|
|
;;
|
|
normal-payload)
|
|
mkdir -m 0700 "$host_root" "$payload_root"
|
|
cp "$DENY" "$payload_root/full-prior-0001.conf"
|
|
chmod 0600 "$payload_root/full-prior-0001.conf"
|
|
;;
|
|
*) fail "unknown first-cutover residue mutation: $mutation" ;;
|
|
esac
|
|
before="$(host_tree_fingerprint "$host_root")"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
after="$(host_tree_fingerprint "$host_root")"
|
|
if (( rc == 0 )); then
|
|
record_task2_contract_failure "$mutation residue returned success"
|
|
failed=true
|
|
fi
|
|
if ! cmp -s "$fixture/active/learn-services" "$DENY"; then
|
|
record_task2_contract_failure "$mutation residue changed active deny bytes"
|
|
failed=true
|
|
fi
|
|
if [[ "$before" != "$after" ]]; then
|
|
record_task2_contract_failure "$mutation residue changed the pre-existing Host evidence tree"
|
|
failed=true
|
|
fi
|
|
if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
record_task2_contract_failure "$mutation residue reached preflight or mutation"
|
|
failed=true
|
|
fi
|
|
if grep -Fq 'Type APPLY' "$fixture/output"; then
|
|
record_task2_contract_failure "$mutation residue reached the APPLY prompt"
|
|
failed=true
|
|
fi
|
|
if [[ "$failed" == false ]]; then
|
|
pass "full first cutover rejects $mutation residue before preflight, mutation, and prompt"
|
|
fi
|
|
}
|
|
|
|
exercise_active_state_gate() {
|
|
local state=$1 fixture before rc failed=false expected_prefix attempt
|
|
fixture="$(new_fixture "active-state-$state")"
|
|
case "$state" in
|
|
full) cp "$FULL" "$fixture/active/learn-services" ;;
|
|
third)
|
|
expected_prefix="$(sha256sum "$DENY" | cut -c1)"
|
|
for ((attempt=0; attempt<256; attempt++)); do
|
|
printf 'server { listen 127.0.0.1:65534; } # %d\n' "$attempt" \
|
|
>"$fixture/active/learn-services"
|
|
[[ "$(sha256sum "$fixture/active/learn-services" | cut -c1)" == "$expected_prefix" ]] && break
|
|
done
|
|
[[ "$attempt" -lt 256 ]] || fail 'cannot construct deterministic same-prefix third state'
|
|
;;
|
|
*) fail "unknown active state: $state" ;;
|
|
esac
|
|
chmod 0644 "$fixture/active/learn-services"
|
|
before="$(sha256sum "$fixture/active/learn-services")"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
if (( rc == 0 )); then
|
|
record_task2_contract_failure "$state active state returned success"
|
|
failed=true
|
|
fi
|
|
if [[ "$(sha256sum "$fixture/active/learn-services")" != "$before" ]]; then
|
|
record_task2_contract_failure "$state active state changed active bytes"
|
|
failed=true
|
|
fi
|
|
if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
record_task2_contract_failure "$state active state reached preflight or mutation"
|
|
failed=true
|
|
fi
|
|
if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ||
|
|
-L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then
|
|
record_task2_contract_failure "$state active state created a Host ledger"
|
|
failed=true
|
|
fi
|
|
if grep -Fq 'Type APPLY' "$fixture/output"; then
|
|
record_task2_contract_failure "$state active state reached the APPLY prompt"
|
|
failed=true
|
|
fi
|
|
if [[ "$state" == full ]] && grep -Fq 'ALREADY_ACTIVE' "$fixture/output"; then
|
|
record_task2_contract_failure 'full active state emitted ALREADY_ACTIVE'
|
|
failed=true
|
|
fi
|
|
if [[ "$failed" == false ]]; then
|
|
pass "full cutover rejects $state active state before preflight, ledger, and prompt"
|
|
fi
|
|
}
|
|
|
|
exercise_source_boundary_contract() {
|
|
local output rc=0
|
|
output="$(env APPLY="$APPLY" /usr/bin/bash <<'SH' 2>&1
|
|
set -Eeuo pipefail
|
|
source "$APPLY"
|
|
[[ "$PATH" == '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' ]]
|
|
[[ "$TEST_BIN" == /usr/bin/test ]]
|
|
print_plan_body="$(declare -f print_plan)"
|
|
if grep -Eq '(^|[[:space:];(])(dig|curl|kubectl|sudo|nginx|systemctl)([[:space:];|)])' <<<"$print_plan_body"; then
|
|
exit 1
|
|
fi
|
|
expected=$'awk=/usr/bin/awk\nbash=/usr/bin/bash\ncat=/usr/bin/cat\ncurl=/usr/bin/curl\ndate=/usr/bin/date\ndig=/usr/bin/dig\ndirname=/usr/bin/dirname\ngrep=/usr/bin/grep\nhead=/usr/bin/head\nid=/usr/bin/id\ninstall=/usr/bin/install\njq=/usr/bin/jq\nkubectl=/usr/local/bin/kubectl\nmktemp=/usr/bin/mktemp\nopenssl=/usr/bin/openssl\npython3=/usr/bin/python3\nreadlink=/usr/bin/readlink\nrm=/usr/bin/rm\nsed=/usr/bin/sed\nsha256sum=/usr/bin/sha256sum\nsleep=/usr/bin/sleep\nsort=/usr/bin/sort\nss=/usr/bin/ss\nstat=/usr/bin/stat\ntail=/usr/bin/tail\ntest=/usr/bin/test\ntr=/usr/bin/tr'
|
|
actual="$(for name in "${!HOST_NGINX_PRODUCTION_COMMANDS[@]}"; do printf '%s=%s\n' "$name" "${HOST_NGINX_PRODUCTION_COMMANDS[$name]}"; done | sort)"
|
|
if (( ${#HOST_NGINX_PRODUCTION_COMMANDS[@]} != 27 )); then
|
|
printf 'production command allowlist cardinality is not 27\n' >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
printf 'production command allowlist lacks exact test=/usr/bin/test boundary\n' >&2
|
|
exit 1
|
|
fi
|
|
declare -F production_executable_metadata_is_safe >/dev/null
|
|
reject_production_metadata() {
|
|
if production_executable_metadata_is_safe "$@"; then
|
|
return 1
|
|
fi
|
|
}
|
|
production_executable_metadata_is_safe sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4755 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 0755 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 1 0 4755 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 1 4755 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4755 2 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/local/bin/sudo 'regular file' 0 0 4755 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 6755 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4775 1 sudo
|
|
reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4757 1 sudo
|
|
production_executable_metadata_is_safe awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 0755 1 ordinary
|
|
reject_production_metadata awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 4755 1 ordinary
|
|
reject_production_metadata awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 2755 1 ordinary
|
|
validate_production_command_boundaries
|
|
SH
|
|
)" || rc=$?
|
|
if (( rc != 0 )); then
|
|
record_task2_contract_failure "fixed production command boundary contract failed: ${output//$'\n'/; }"
|
|
else
|
|
pass 'fixed production PATH, exact inventory, sudo exception, and ordinary metadata policy are enforced'
|
|
fi
|
|
}
|
|
|
|
assert_privileged_test_source_boundary() {
|
|
local source=$1
|
|
/usr/bin/python3 - "$source" <<'PY'
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
text = pathlib.Path(sys.argv[1]).read_text()
|
|
exact = '"$SUDO_BIN" -n "$TEST_BIN"'
|
|
expected_count = 27
|
|
commands = re.findall(r'"\$SUDO_BIN"\s+-n\s+([^\s;&|()]+)', text)
|
|
test_commands = [
|
|
command for command in commands
|
|
if command == '"$TEST_BIN"'
|
|
or command.strip('"\'') == 'test'
|
|
or command.strip('"\'').endswith('/test')
|
|
]
|
|
if text.count(exact) != expected_count:
|
|
raise SystemExit(
|
|
f'privileged test boundary exact-call cardinality is {text.count(exact)}, expected {expected_count}'
|
|
)
|
|
if len(test_commands) != expected_count:
|
|
raise SystemExit(
|
|
f'privileged test boundary candidate cardinality is {len(test_commands)}, expected {expected_count}'
|
|
)
|
|
if any(command != '"$TEST_BIN"' for command in test_commands):
|
|
raise SystemExit('privileged test boundary includes a non-fixed test command')
|
|
for operand in (
|
|
'-s "$CERT_DIR/fullchain.pem"',
|
|
'-s "$CERT_DIR/privkey.pem"',
|
|
'-f "$CREDENTIALS"',
|
|
):
|
|
if text.count(f'{exact} {operand}') != 1:
|
|
raise SystemExit(f'privileged test boundary misses exact certificate/credential probe: {operand}')
|
|
PY
|
|
}
|
|
|
|
exercise_privileged_test_source_mutation_matrix() {
|
|
local mutation ordinal rc
|
|
assert_privileged_test_source_boundary "$APPLY" ||
|
|
fail 'current Host source violates the closed privileged test boundary'
|
|
for (( ordinal=0; ordinal<27; ordinal++ )); do
|
|
mutation="$WORK/privileged-test-bare-$ordinal.sh"
|
|
/usr/bin/python3 - "$APPLY" "$mutation" "$ordinal" <<'PY'
|
|
import pathlib
|
|
import sys
|
|
|
|
source = pathlib.Path(sys.argv[1])
|
|
destination = pathlib.Path(sys.argv[2])
|
|
ordinal = int(sys.argv[3])
|
|
text = source.read_text()
|
|
exact = '"$SUDO_BIN" -n "$TEST_BIN"'
|
|
bare = '"$SUDO_BIN" -n test'
|
|
parts = text.split(exact)
|
|
if len(parts) - 1 != 27:
|
|
raise SystemExit('unexpected privileged test boundary cardinality while mutating')
|
|
if ordinal < 0 or ordinal >= len(parts) - 1:
|
|
raise SystemExit('privileged test mutation ordinal is out of range')
|
|
mutated = exact.join(parts[:ordinal + 1]) + bare + exact.join(parts[ordinal + 1:])
|
|
destination.write_text(mutated)
|
|
destination.chmod(0o600)
|
|
PY
|
|
if assert_privileged_test_source_boundary "$mutation" >"$mutation.output" 2>&1; then
|
|
fail "privileged test boundary accepted bare-test mutation $ordinal"
|
|
fi
|
|
grep -Eq 'privileged test boundary (exact-call|candidate) cardinality is|privileged test boundary includes a non-fixed test command' \
|
|
"$mutation.output" ||
|
|
fail "privileged test boundary mutation $ordinal failed outside the boundary assertion"
|
|
done
|
|
pass 'closed source matrix rejects every privileged bare-test mutation including certificate and credentials probes'
|
|
}
|
|
|
|
exercise_active_site_symlink_diagnostic() {
|
|
local fixture rc expected
|
|
fixture="$(new_fixture active-site-symlink-diagnostic)"
|
|
rm -f -- "$fixture/active/learn-services"
|
|
ln -s "$DENY" "$fixture/active/learn-services"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
(( rc != 0 )) || fail 'active-site symlink was accepted'
|
|
expected="ERROR: active site is a symlink: $fixture/active/learn-services"
|
|
grep -Fqx "$expected" "$fixture/output" ||
|
|
fail 'active-site symlink diagnostic drifted from its exact contract'
|
|
[[ ! -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" &&
|
|
! -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]] ||
|
|
fail 'active-site symlink diagnostic created a Host ledger'
|
|
! grep -Fq 'Type APPLY' "$fixture/output" ||
|
|
fail 'active-site symlink diagnostic reached the APPLY prompt'
|
|
pass 'active-site symlink retains its exact diagnostic before ledger or prompt'
|
|
}
|
|
|
|
exercise_precommit_staging_failure() {
|
|
local stage=$1 fixture host_root rc failed=false reloads=0
|
|
fixture="$(new_fixture "staging-failure-$stage")"
|
|
host_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx"
|
|
unset PLATFORM_TEST_INSTALL_FAIL_MATCH PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH
|
|
case "$stage" in
|
|
directory) PLATFORM_TEST_INSTALL_FAIL_MATCH="$host_root" ;;
|
|
ledger) PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH="$host_root/stages.tsv" ;;
|
|
payload) PLATFORM_TEST_INSTALL_FAIL_MATCH="$host_root/payloads/full-prior-0001.conf" ;;
|
|
timestamp) PLATFORM_TEST_INSTALL_FAIL_MATCH='.before-observability-full-' ;;
|
|
*) fail "unknown precommit staging failure: $stage" ;;
|
|
esac
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
unset PLATFORM_TEST_INSTALL_FAIL_MATCH PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH
|
|
[[ ! -f "$fixture/reload-counter" ]] || read -r reloads <"$fixture/reload-counter"
|
|
if (( rc == 0 )); then
|
|
record_task2_contract_failure "$stage staging failure returned success"
|
|
failed=true
|
|
fi
|
|
if ! cmp -s "$fixture/active/learn-services" "$DENY"; then
|
|
record_task2_contract_failure "$stage staging failure changed active deny bytes"
|
|
failed=true
|
|
fi
|
|
if [[ "$reloads" != 0 ]]; then
|
|
record_task2_contract_failure "$stage staging failure reloaded Nginx"
|
|
failed=true
|
|
fi
|
|
if grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=' "$fixture/output"; then
|
|
record_task2_contract_failure "$stage pre-transaction staging failure emitted a rollback marker"
|
|
failed=true
|
|
fi
|
|
if ! grep -Fq 'fresh rollback ID + source proof + complete Task 6 required' "$fixture/output"; then
|
|
record_task2_contract_failure "$stage staging failure omitted the frozen-ID terminal classification"
|
|
failed=true
|
|
fi
|
|
case "$stage" in
|
|
directory)
|
|
if [[ -e "$host_root" || -L "$host_root" ]]; then
|
|
record_task2_contract_failure 'directory staging failure unexpectedly created the Host root'
|
|
failed=true
|
|
fi
|
|
;;
|
|
ledger)
|
|
if [[ ! -d "$host_root/payloads" || ! -f "$host_root/stages.tsv" ||
|
|
"$(stat -c %a "$host_root/stages.tsv")" != 644 ]]; then
|
|
record_task2_contract_failure 'ledger staging failure did not preserve its unsafe staged evidence for review'
|
|
failed=true
|
|
fi
|
|
;;
|
|
payload)
|
|
if [[ ! -f "$host_root/stages.tsv" || -e "$host_root/payloads/full-prior-0001.conf" ]]; then
|
|
record_task2_contract_failure 'payload staging failure evidence topology is not exact'
|
|
failed=true
|
|
fi
|
|
;;
|
|
timestamp)
|
|
if [[ ! -f "$host_root/stages.tsv" || ! -f "$host_root/payloads/full-prior-0001.conf" ]] ||
|
|
compgen -G "$fixture/active/learn-services.before-observability-full-*" >/dev/null; then
|
|
record_task2_contract_failure 'timestamp staging failure evidence topology is not exact'
|
|
failed=true
|
|
fi
|
|
;;
|
|
esac
|
|
if [[ "$failed" == false ]]; then
|
|
pass "$stage pre-transaction staging failure freezes the ID without a rollback claim"
|
|
fi
|
|
}
|
|
|
|
exercise_valid_acceptance_tuple() {
|
|
local gate=$1 fixture acceptance inventory_shas initial_sha post_sha rc failed=false
|
|
fixture="$(new_fixture "valid-${gate,,}-acceptance")"
|
|
acceptance="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
|
|
inventory_shas="$(acceptance_inventory_shas "$fixture")"
|
|
initial_sha=${inventory_shas%%|*}
|
|
post_sha=${inventory_shas#*|}
|
|
write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" "$gate"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
if (( rc != 0 )); then
|
|
record_evidence_contract_failure "valid $gate v2 acceptance was rejected with RC $rc"
|
|
failed=true
|
|
if cmp -s "$fixture/active/learn-services" "$DENY"; then
|
|
printf 'RED-PROOF: valid %s v2 rejection preserved exact deny bytes\n' "$gate" >&2
|
|
else
|
|
record_evidence_contract_failure "valid $gate v2 rejection changed the active deny bytes"
|
|
fi
|
|
fi
|
|
if ! cmp -s "$fixture/active/learn-services" "$FULL"; then
|
|
record_evidence_contract_failure "valid $gate v2 acceptance did not install the full candidate"
|
|
failed=true
|
|
fi
|
|
if [[ ! -f "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/stages.tsv" ]]; then
|
|
record_evidence_contract_failure "valid $gate v2 acceptance did not create the Host ledger"
|
|
failed=true
|
|
fi
|
|
if ! grep -Fq 'Type APPLY' "$fixture/output"; then
|
|
record_evidence_contract_failure "valid $gate v2 acceptance did not reach the APPLY prompt"
|
|
failed=true
|
|
fi
|
|
if [[ "$failed" == false ]]; then
|
|
pass "full evidence accepts the exact $gate v2 tuple"
|
|
fi
|
|
}
|
|
|
|
exercise_invalid_acceptance() {
|
|
local mutation=$1 fixture acceptance inventory_shas initial_sha post_sha rc failed=false
|
|
fixture="$(new_fixture "invalid-${mutation}")"
|
|
acceptance="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
|
|
inventory_shas="$(acceptance_inventory_shas "$fixture")"
|
|
initial_sha=${inventory_shas%%|*}
|
|
post_sha=${inventory_shas#*|}
|
|
case "$mutation" in
|
|
v1)
|
|
write_acceptance_v1 "$acceptance" "$initial_sha" "$post_sha"
|
|
;;
|
|
wrong-schema)
|
|
sed -i 's/^schema=.*/schema=wrong-schema/' "$acceptance"
|
|
;;
|
|
missing-slack-gate)
|
|
sed -i '/^slack_deployment_gate=/d' "$acceptance"
|
|
;;
|
|
extra-field)
|
|
printf 'unexpected=value\n' >>"$acceptance"
|
|
;;
|
|
duplicate-field)
|
|
printf 'state=accepted\n' >>"$acceptance"
|
|
;;
|
|
recovery-with-risk-ref)
|
|
write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" RECOVERY
|
|
sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design/' "$acceptance"
|
|
;;
|
|
recovery-with-uid-1000)
|
|
write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" RECOVERY
|
|
sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=1000/' "$acceptance"
|
|
;;
|
|
risk-with-strict-ref)
|
|
sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=strict-recovery-evidence-v1/' "$acceptance"
|
|
;;
|
|
risk-with-not-applicable)
|
|
sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=not-applicable/' "$acceptance"
|
|
;;
|
|
arbitrary-gate)
|
|
sed -i 's/^slack_deployment_gate=.*/slack_deployment_gate=ARBITRARY/' "$acceptance"
|
|
;;
|
|
arbitrary-ref)
|
|
sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=arbitrary-ref/' "$acceptance"
|
|
;;
|
|
arbitrary-uid)
|
|
sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=2000/' "$acceptance"
|
|
;;
|
|
rollback-id)
|
|
sed -i 's/^rollback_id=.*/rollback_id=20260812T120001Z/' "$acceptance"
|
|
;;
|
|
target-initial-hash)
|
|
sed -i 's/^target_initial_sha256=.*/target_initial_sha256=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/' "$acceptance"
|
|
;;
|
|
post-substrate-hash)
|
|
sed -i 's/^post_substrate_sha256=.*/post_substrate_sha256=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc/' "$acceptance"
|
|
;;
|
|
state)
|
|
sed -i 's/^state=.*/state=pending/' "$acceptance"
|
|
;;
|
|
malformed-accepted-at)
|
|
sed -i 's/^accepted_at_utc=.*/accepted_at_utc=2026-02-30T00:00:00Z/' "$acceptance"
|
|
;;
|
|
*) fail "unknown acceptance mutation: $mutation" ;;
|
|
esac
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
if (( rc == 0 )); then
|
|
record_evidence_contract_failure "$mutation acceptance returned success"
|
|
failed=true
|
|
fi
|
|
if ! cmp -s "$fixture/active/learn-services" "$DENY"; then
|
|
record_evidence_contract_failure "$mutation acceptance changed the active deny bytes"
|
|
failed=true
|
|
fi
|
|
if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
record_evidence_contract_failure "$mutation acceptance reached preflight or mutation"
|
|
failed=true
|
|
fi
|
|
if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ||
|
|
-L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then
|
|
record_evidence_contract_failure "$mutation acceptance created a Host ledger"
|
|
failed=true
|
|
fi
|
|
if grep -Fq 'Type APPLY' "$fixture/output"; then
|
|
record_evidence_contract_failure "$mutation acceptance reached the APPLY prompt"
|
|
failed=true
|
|
fi
|
|
if [[ "$failed" == false ]]; then
|
|
pass "full evidence rejects $mutation before preflight, mutation, ledger, and prompt"
|
|
fi
|
|
}
|
|
|
|
exercise_invalid_source_proof() {
|
|
local mutation=$1 fixture proof rc failed=false mutated_time fixed_now=''
|
|
fixture="$(new_fixture "invalid-source-${mutation}")"
|
|
proof="$fixture/rollbacks/observability-$ROLLBACK_ID/blackbox-source-proof.env"
|
|
case "$mutation" in
|
|
deny-sha)
|
|
sed -i 's/^nginx_sha256=.*/nginx_sha256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' "$proof"
|
|
;;
|
|
status)
|
|
sed -i 's/^grafana_status=403$/grafana_status=200/' "$proof"
|
|
;;
|
|
stale)
|
|
mutated_time="$(date -u -d 'now - 86401 seconds' +%Y-%m-%dT%H:%M:%SZ)"
|
|
sed -i "s/^tested_at_utc=.*/tested_at_utc=$mutated_time/" "$proof"
|
|
;;
|
|
future)
|
|
fixed_now="$(date -u +%s)"
|
|
mutated_time="$(date -u -d "@$((fixed_now + 301))" +%Y-%m-%dT%H:%M:%SZ)"
|
|
sed -i "s/^tested_at_utc=.*/tested_at_utc=$mutated_time/" "$proof"
|
|
;;
|
|
invalid-calendar)
|
|
sed -i 's/^tested_at_utc=.*/tested_at_utc=2026-02-30T00:00:00Z/' "$proof"
|
|
;;
|
|
*) fail "unknown source-proof mutation: $mutation" ;;
|
|
esac
|
|
if PLATFORM_TEST_DATE_NOW_EPOCH="$fixed_now" run_apply "$fixture" \
|
|
--execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
if (( rc == 0 )); then
|
|
record_evidence_contract_failure "source-proof $mutation returned success"
|
|
failed=true
|
|
fi
|
|
if ! cmp -s "$fixture/active/learn-services" "$DENY"; then
|
|
record_evidence_contract_failure "source-proof $mutation changed the active deny bytes"
|
|
failed=true
|
|
fi
|
|
if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
record_evidence_contract_failure "source-proof $mutation reached preflight or mutation"
|
|
failed=true
|
|
fi
|
|
if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ||
|
|
-L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then
|
|
record_evidence_contract_failure "source-proof $mutation created a Host ledger"
|
|
failed=true
|
|
fi
|
|
if grep -Fq 'Type APPLY' "$fixture/output"; then
|
|
record_evidence_contract_failure "source-proof $mutation reached the APPLY prompt"
|
|
failed=true
|
|
fi
|
|
if [[ "$failed" == false ]]; then
|
|
pass "full evidence rejects source-proof $mutation before preflight, mutation, ledger, and prompt"
|
|
fi
|
|
}
|
|
|
|
WORK="$(mktemp -d /tmp/platform-host-nginx-test.XXXXXX)"
|
|
chmod 0700 "$WORK"
|
|
|
|
if env PLATFORM_HOST_NGINX_ACTIVE="$DENY" bash "$APPLY" >"$WORK/production-path-override.out" 2>&1; then
|
|
fail 'production dry-run accepted a Host Nginx path override'
|
|
fi
|
|
pass 'production rejects Host Nginx path overrides'
|
|
|
|
if env PLATFORM_HOST_NGINX_NGINX_BIN=/bin/true bash "$APPLY" >"$WORK/production-binary-override.out" 2>&1; then
|
|
fail 'production dry-run accepted a Host Nginx binary override'
|
|
fi
|
|
pass 'production rejects Host Nginx binary overrides'
|
|
|
|
fixture="$(new_fixture rejects-real-sudo)"
|
|
if PLATFORM_TEST_SUDO_BIN=/usr/bin/sudo run_apply "$fixture" >"$fixture/output" 2>&1; then
|
|
fail 'test-mode dry-run accepted the real sudo binary'
|
|
fi
|
|
grep -Fq 'test command boundary is unsafe' "$fixture/output" ||
|
|
fail 'test-mode real sudo rejection did not identify the command boundary'
|
|
pass 'test mode rejects the real sudo binary before it can execute'
|
|
|
|
for command in nginx systemctl test certbot preflight postcheck; do
|
|
assert_test_command_override_rejected "$command" /usr/bin/true
|
|
fixture="$WORK/rejects-$command-override"
|
|
grep -Fq 'test command boundary is unsafe' "$fixture/output" ||
|
|
fail "test-mode escaped $command rejection did not identify the command boundary"
|
|
pass "test mode rejects escaped $command command overrides"
|
|
done
|
|
|
|
fixture="$(new_fixture rejects-foreign-fixture-bin)"
|
|
foreign_fixture="$(new_fixture foreign-fixture-bin)"
|
|
if PLATFORM_TEST_SUDO_BIN="$foreign_fixture/bin/sudo" \
|
|
PLATFORM_TEST_NGINX_BIN="$foreign_fixture/bin/nginx" \
|
|
PLATFORM_TEST_SYSTEMCTL_BIN="$foreign_fixture/bin/systemctl" \
|
|
PLATFORM_TEST_TEST_BIN="$foreign_fixture/bin/test" \
|
|
PLATFORM_TEST_CERTBOT_BIN="$foreign_fixture/bin/certbot" \
|
|
PLATFORM_TEST_PREFLIGHT_COMMAND="$foreign_fixture/bin/preflight" \
|
|
PLATFORM_TEST_POSTCHECK_COMMAND="$foreign_fixture/bin/postcheck" \
|
|
run_apply "$fixture" >"$fixture/output" 2>&1; then
|
|
fail 'test mode accepted command boundaries from a different fixture tree'
|
|
fi
|
|
grep -Fq 'test command boundary is unsafe: sudo escapes the fixture bin' "$fixture/output" ||
|
|
fail 'foreign fixture command boundary rejection did not identify the fixture mismatch'
|
|
pass 'test mode requires command boundaries from the active and rollback fixture tree'
|
|
|
|
for mutation in missing unsafe-mode; do
|
|
assert_test_command_boundary_mutation_rejected "$mutation"
|
|
done
|
|
|
|
fixture="$(new_fixture dry-run)"
|
|
before="$(sha256sum "$fixture/active/learn-services")"
|
|
output="$(run_apply "$fixture")" || fail 'dry-run failed'
|
|
[[ "$output" == *'HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS'* ]] || fail 'dry-run marker missing'
|
|
[[ "$(sha256sum "$fixture/active/learn-services")" == "$before" ]] || fail 'dry-run mutated active config'
|
|
[[ ! -s "$fixture/commands.log" ]] || fail 'dry-run invoked a mutating boundary'
|
|
dry_run_contract_ok=true
|
|
for marker in \
|
|
'HOST_NGINX_CERTIFICATE_EXPECTED_SAN=grafana.learn.hyeonworks.com' \
|
|
'HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN' \
|
|
'HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN'; do
|
|
if [[ "$(grep -Fxc "$marker" <<<"$output")" != 1 ]]; then
|
|
record_task2_contract_failure "dry-run marker is not exact-once: $marker"
|
|
dry_run_contract_ok=false
|
|
fi
|
|
done
|
|
for forbidden in HOST_NGINX_CERTIFICATE_EXACT_SAN= HOST_NGINX_GRAFANA_PUBLIC_DNS=ABSENT; do
|
|
if grep -Fq "$forbidden" <<<"$output"; then
|
|
record_task2_contract_failure "dry-run retained false live claim: $forbidden"
|
|
dry_run_contract_ok=false
|
|
fi
|
|
done
|
|
if [[ "$dry_run_contract_ok" == true ]]; then
|
|
pass 'dry-run is no-contact and emits truthful expected/not-checked markers'
|
|
fi
|
|
|
|
hostile="$WORK/hostile-path"
|
|
mkdir -m 0700 "$hostile"
|
|
cat >"$hostile/dirname" <<'SH'
|
|
#!/usr/bin/env bash
|
|
printf 'dirname %s\n' "$*" >>"$PLATFORM_TEST_HOSTILE_LOG"
|
|
exec /usr/bin/dirname "$@"
|
|
SH
|
|
chmod 0700 "$hostile/dirname"
|
|
: >"$WORK/hostile-path.log"
|
|
if env PATH="$hostile:$PATH" PLATFORM_TEST_HOSTILE_LOG="$WORK/hostile-path.log" \
|
|
PLATFORM_HOST_NGINX_TEST_MODE=invalid /usr/bin/bash "$APPLY" >"$WORK/hostile-path.out" 2>&1; then
|
|
record_task2_contract_failure 'invalid test mode unexpectedly returned success under hostile PATH'
|
|
elif [[ -s "$WORK/hostile-path.log" ]]; then
|
|
record_task2_contract_failure 'ambient dirname executed before hostile test mode rejection'
|
|
else
|
|
pass 'fixed PATH is installed before ROOT resolution and invalid-mode rejection'
|
|
fi
|
|
|
|
exercise_source_boundary_contract
|
|
exercise_privileged_test_source_mutation_matrix
|
|
exercise_active_site_symlink_diagnostic
|
|
|
|
fixture="$(new_fixture missing-acceptance)"
|
|
rm -f "$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'missing acceptance was accepted'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'failed preflight changed active config'
|
|
if grep -Eq '^(nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
fail 'missing acceptance reached a mutation boundary'
|
|
fi
|
|
pass 'full mode refuses missing acceptance with zero mutation'
|
|
|
|
fixture="$(new_fixture unsafe-inventory-mode)"
|
|
chmod 0755 "$fixture/metrics/target-initial"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'unsafe inventory mode was hidden by the later inventory hash'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'unsafe inventory mode changed active config'
|
|
if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
fail 'unsafe inventory mode reached a mutation or preflight boundary'
|
|
fi
|
|
pass 'unsafe inventory metadata cannot be hidden by a valid hash'
|
|
|
|
fixture="$(new_fixture symlinked-inventory-json)"
|
|
mv "$fixture/metrics/target-initial/inventory.json" \
|
|
"$fixture/metrics/target-initial/inventory.real.json"
|
|
ln -s inventory.real.json "$fixture/metrics/target-initial/inventory.json"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'symlinked inventory JSON was hidden by its valid target hash'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked inventory JSON changed active config'
|
|
if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then
|
|
fail 'symlinked inventory JSON reached a mutation or preflight boundary'
|
|
fi
|
|
pass 'symlinked inventory JSON cannot be hidden by a valid target hash'
|
|
|
|
fixture="$(new_fixture missing-certificate-key)"
|
|
mkdir -p "$fixture/cert"
|
|
printf 'test certificate\n' >"$fixture/cert/fullchain.pem"
|
|
if run_apply "$fixture" --execute --certificate-only >"$fixture/output" 2>&1; then
|
|
fail 'missing certificate private key was hidden by later SAN validation'
|
|
fi
|
|
if grep -Fq 'HOST_NGINX_GRAFANA_CERTIFICATE=READY' "$fixture/output"; then
|
|
fail 'missing certificate private key emitted READY'
|
|
fi
|
|
pass 'exact certificate validation cannot hide a missing private key'
|
|
|
|
for gate in RECOVERY RISK_ACCEPTED; do
|
|
exercise_valid_acceptance_tuple "$gate"
|
|
done
|
|
|
|
for mutation in \
|
|
v1 wrong-schema missing-slack-gate extra-field duplicate-field \
|
|
recovery-with-risk-ref recovery-with-uid-1000 \
|
|
risk-with-strict-ref risk-with-not-applicable \
|
|
arbitrary-gate arbitrary-ref arbitrary-uid \
|
|
rollback-id target-initial-hash post-substrate-hash state malformed-accepted-at; do
|
|
exercise_invalid_acceptance "$mutation"
|
|
done
|
|
|
|
for mutation in deny-sha status stale future invalid-calendar; do
|
|
exercise_invalid_source_proof "$mutation"
|
|
done
|
|
|
|
exercise_active_state_gate full
|
|
exercise_active_state_gate third
|
|
for mutation in empty-root normal-ledger normal-payload; do
|
|
exercise_first_cutover_residue "$mutation"
|
|
done
|
|
|
|
if (( ${#evidence_contract_failures[@]} > 0 )); then
|
|
fail "full evidence contract mismatches: ${evidence_contract_failures[*]}"
|
|
fi
|
|
|
|
fixture="$(new_fixture symlinked-payload-leaf)"
|
|
mkdir -m 0700 "$fixture/escaped-payloads" \
|
|
"$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx"
|
|
ln -s "$fixture/escaped-payloads" \
|
|
"$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/payloads"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'symlinked payloads ledger leaf was followed'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked payload leaf changed active config'
|
|
[[ ! -e "$fixture/escaped-payloads/full-prior-0001.conf" ]] ||
|
|
fail 'prior payload escaped through a symlinked leaf'
|
|
pass 'ledger payload leaf symlink is rejected before any write'
|
|
|
|
fixture="$(new_fixture symlinked-ledger-parent)"
|
|
mkdir -m 0700 "$fixture/escaped-host-nginx"
|
|
ln -s "$fixture/escaped-host-nginx" \
|
|
"$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'symlinked host-nginx ledger parent was followed'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked ledger parent changed active config'
|
|
[[ ! -e "$fixture/escaped-host-nginx/stages.tsv" ]] || fail 'ledger escaped through a symlinked parent'
|
|
pass 'ledger parent symlink is rejected before any write'
|
|
|
|
fixture="$(new_fixture symlinked-ledger-leaf)"
|
|
ledger_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx"
|
|
mkdir -m 0700 "$ledger_root" "$ledger_root/payloads"
|
|
printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' \
|
|
>"$fixture/escaped-stages.tsv"
|
|
chmod 0600 "$fixture/escaped-stages.tsv"
|
|
ln -s "$fixture/escaped-stages.tsv" "$ledger_root/stages.tsv"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'symlinked stages.tsv ledger leaf was accepted'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked ledger leaf changed active config'
|
|
if grep -Fq "sudo -n cat -- $ledger_root/stages.tsv" "$fixture/commands.log"; then
|
|
fail 'symlinked stages.tsv was read before no-follow rejection'
|
|
fi
|
|
pass 'ledger file symlink is rejected before read or write'
|
|
|
|
fixture="$(new_fixture hardlinked-ledger-leaf)"
|
|
ledger_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx"
|
|
mkdir -m 0700 "$ledger_root" "$ledger_root/payloads"
|
|
printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' \
|
|
>"$ledger_root/stages.tsv"
|
|
chmod 0600 "$ledger_root/stages.tsv"
|
|
ln "$ledger_root/stages.tsv" "$fixture/linked-stages.tsv"
|
|
if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'multiply-linked stages.tsv ledger leaf was accepted'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'multiply-linked ledger leaf changed active config'
|
|
pass 'ledger file requires exactly one link before read or write'
|
|
|
|
fixture="$(new_fixture success)"
|
|
run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1 || {
|
|
sed -n '1,260p' "$fixture/output" >&2
|
|
fail 'full transition failed'
|
|
}
|
|
cmp -s "$fixture/active/learn-services" "$FULL" || fail 'full candidate was not installed'
|
|
ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/stages.tsv"
|
|
[[ -f "$ledger" && ! -L "$ledger" ]] || fail 'authoritative ledger missing'
|
|
[[ "$(wc -l <"$ledger" | tr -d ' ')" == 2 ]] || fail 'ledger does not have header plus exact stage'
|
|
awk -F '\t' 'NR == 1 { exit($0 != "stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc") }
|
|
NR == 2 { exit(NF != 6 || $1 != "full" || $4 != "payloads/full-prior-0001.conf") }' "$ledger" ||
|
|
fail 'ledger schema or full row is not exact'
|
|
payload="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/payloads/full-prior-0001.conf"
|
|
cmp -s "$payload" "$DENY" || fail 'ledger payload is not exact prior active config'
|
|
[[ "$(stat -c %a "$ledger")" == 600 && "$(stat -c %a "$payload")" == 600 ]] ||
|
|
fail 'ledger or payload is not mode 0600'
|
|
grep -Fqx 'systemctl reload nginx' "$fixture/commands.log" || fail 'successful transition did not reload nginx'
|
|
[[ "$(grep -Fxc 'postcheck' "$fixture/commands.log")" == 3 ]] ||
|
|
fail 'postcheck did not require three consecutive stable results'
|
|
first_sudo="$(grep '^sudo ' "$fixture/commands.log" | head -n1)"
|
|
[[ "$first_sudo" == 'sudo -v' ]] || fail 'sudo refresh did not precede root metadata reads'
|
|
awk -v nginx="$fixture/bin/nginx" '
|
|
$0 == "sudo -n " nginx " -t" { sudo_test=NR }
|
|
$0 == "preflight" { exit(!(sudo_test > 0 && sudo_test < NR)) }
|
|
END { if (!sudo_test) exit 1 }
|
|
' "$fixture/commands.log" || fail 'preflight nginx -t did not route through sudo before the probe hook'
|
|
! grep -q '^nginx-without-sudo ' "$fixture/commands.log" || fail 'nginx was invoked outside sudo'
|
|
grep -Eq "^sudo -n $fixture/bin/test " "$fixture/commands.log" ||
|
|
fail 'full transition did not route test probes through the fixed test executable'
|
|
! grep -Eq '^sudo -n test( |$)' "$fixture/commands.log" ||
|
|
fail 'full transition invoked a bare sudo test subcommand'
|
|
pass 'full transition routes all sudo test probes through the fixed test executable'
|
|
pass 'full transition records exact prior payload and installs candidate'
|
|
|
|
fixture="$(new_fixture rollback)"
|
|
if PLATFORM_TEST_POSTCHECK_RC=1 run_apply "$fixture" --execute \
|
|
--verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then
|
|
fail 'failed postcheck returned success'
|
|
fi
|
|
cmp -s "$fixture/active/learn-services" "$DENY" || fail 'rollback did not restore exact ledger payload'
|
|
grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS' "$fixture/output" || fail 'rollback PASS marker missing'
|
|
[[ "$(grep -Fxc 'systemctl reload nginx' "$fixture/commands.log")" == 2 ]] ||
|
|
fail 'apply and rollback did not each reload nginx'
|
|
pass 'failed postcheck restores the exact prior payload'
|
|
|
|
fixture="$(new_fixture rollback-failure)"
|
|
if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT=2 \
|
|
run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \
|
|
>"$fixture/output" 2>&1; then
|
|
fail 'failed rollback returned success'
|
|
fi
|
|
grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=FAIL' "$fixture/output" ||
|
|
fail 'rollback failure marker missing'
|
|
grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" ||
|
|
fail 'manual recovery marker missing after rollback failure'
|
|
pass 'rollback failure is truthfully classified for manual recovery'
|
|
|
|
for stage in directory ledger payload timestamp; do
|
|
exercise_precommit_staging_failure "$stage"
|
|
done
|
|
|
|
if (( ${#task2_contract_failures[@]} > 0 )); then
|
|
fail "Task 2 contract mismatches: ${task2_contract_failures[*]}"
|
|
fi
|
|
|
|
printf 'Assertions: %d\n' "$ASSERTIONS"
|
|
printf 'APPLY HOST NGINX OBSERVABILITY TEST PASS\n'
|