#!/usr/bin/env bash set -Eeuo pipefail readonly ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" readonly APPLY="$ROOT/scripts/bootstrap/apply-host-nginx-observability.sh" readonly DENY="$ROOT/infrastructure/networking/host-nginx/learn-services-grafana-deny-guard.conf" readonly FULL="$ROOT/infrastructure/networking/host-nginx/learn-services-observability.conf" readonly ROLLBACK_ID=20260812T120000Z WORK='' ASSERTIONS=0 fail() { printf 'FAIL: %s\n' "$*" >&2 exit 1 } pass() { ASSERTIONS=$((ASSERTIONS + 1)) printf 'PASS: %s\n' "$1" } cleanup() { local rc=$? trap - EXIT HUP INT TERM case "$WORK" in /tmp/platform-host-nginx-test.*) rm -rf -- "$WORK" ;; esac exit "$rc" } trap cleanup EXIT HUP INT TERM make_fakes() { local root=$1 mkdir -m 0700 -p "$root/bin" cat >"$root/bin/nginx" <<'SH' #!/usr/bin/env bash [[ "${PLATFORM_TEST_VIA_SUDO:-0}" == 1 ]] || { printf 'nginx-without-sudo %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" exit 91 } printf 'nginx %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" exit 0 SH cat >"$root/bin/systemctl" <<'SH' #!/usr/bin/env bash printf 'systemctl %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" if [[ "${1:-} ${2:-}" == 'reload nginx' ]]; then count=0 [[ ! -f "$PLATFORM_TEST_RELOAD_COUNTER" ]] || read -r count <"$PLATFORM_TEST_RELOAD_COUNTER" count=$((count + 1)) printf '%d\n' "$count" >"$PLATFORM_TEST_RELOAD_COUNTER" if [[ "${PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT:-0}" == "$count" ]]; then exit 94 fi fi exit 0 SH cat >"$root/bin/sudo" <<'SH' #!/usr/bin/env bash printf 'sudo %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" if [[ "${1:-}" == -v ]]; then : >"$PLATFORM_TEST_SUDO_REFRESHED" exit 0 fi if [[ "${1:-}" == -n ]]; then shift; fi (( $# == 0 )) && exit 0 [[ -f "$PLATFORM_TEST_SUDO_REFRESHED" ]] || exit 92 export PLATFORM_TEST_VIA_SUDO=1 exec "$@" SH cat >"$root/bin/test" <<'SH' #!/usr/bin/env bash printf 'test %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" exec /usr/bin/test "$@" SH cat >"$root/bin/install" <<'SH' #!/usr/bin/env bash args=() while (( $# > 0 )); do case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac done if (( ${#args[@]} > 0 )); then destination=${args[$((${#args[@]} - 1))]} if [[ -n "${PLATFORM_TEST_INSTALL_FAIL_MATCH:-}" && "${args[*]}" == *"$PLATFORM_TEST_INSTALL_FAIL_MATCH"* ]]; then exit 93 fi if [[ -n "${PLATFORM_HOST_NGINX_ACTIVE:-}" && "$destination" == "$PLATFORM_HOST_NGINX_ACTIVE" ]]; then count=0 [[ ! -f "$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" ]] || read -r count <"$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" count=$((count + 1)) printf '%d\n' "$count" >"$PLATFORM_TEST_ACTIVE_INSTALL_COUNTER" if [[ "${PLATFORM_TEST_FAIL_ACTIVE_INSTALL_AT:-0}" == "$count" ]]; then exit 95 fi fi fi /usr/bin/install "${args[@]}" rc=$? if (( rc == 0 )) && [[ -n "${PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH:-}" && "${args[*]}" == *"$PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH"* ]]; then chmod 0644 "$destination" fi exit "$rc" SH cat >"$root/bin/openssl" <<'SH' #!/usr/bin/env bash printf 'openssl %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" # This boundary deliberately reports a valid exact SAN and successful checkend. # A missing privkey must therefore be rejected by our validator, not by OpenSSL. if [[ " $* " == *' -ext subjectAltName '* ]]; then printf 'X509v3 Subject Alternative Name:\n DNS:grafana.learn.hyeonworks.com\n' fi exit 0 SH for command in dig curl kubectl; do cat >"$root/bin/$command" <<'SH' #!/usr/bin/env bash printf '%s %s\n' "${0##*/}" "$*" >>"$PLATFORM_TEST_COMMAND_LOG" exit 0 SH done cat >"$root/bin/preflight" <<'SH' #!/usr/bin/env bash printf 'preflight\n' >>"$PLATFORM_TEST_COMMAND_LOG" exit "${PLATFORM_TEST_PREFLIGHT_RC:-0}" SH cat >"$root/bin/postcheck" <<'SH' #!/usr/bin/env bash printf 'postcheck\n' >>"$PLATFORM_TEST_COMMAND_LOG" exit "${PLATFORM_TEST_POSTCHECK_RC:-0}" SH cat >"$root/bin/certbot" <<'SH' #!/usr/bin/env bash printf 'certbot %s\n' "$*" >>"$PLATFORM_TEST_COMMAND_LOG" if [[ "${1:-}" == plugins ]]; then printf 'dns-cloudflare\n' fi exit 0 SH cat >"$root/bin/date" <<'SH' #!/usr/bin/env bash if [[ -n "${PLATFORM_TEST_DATE_NOW_EPOCH:-}" && "$*" == '-u +%s' ]]; then printf '%s\n' "$PLATFORM_TEST_DATE_NOW_EPOCH" exit 0 fi exec /usr/bin/date "$@" SH chmod 0755 "$root/bin/"* } write_inventory() { local root=$1 phase=$2 sha mkdir -m 0700 -p -- "$root/$phase" printf '{"phase":"%s"}\n' "$phase" >"$root/$phase/inventory.json" chmod 0600 "$root/$phase/inventory.json" sha="$(sha256sum "$root/$phase/inventory.json" | awk '{print $1}')" printf '%s inventory.json\n' "$sha" >"$root/$phase/inventory.sha256" chmod 0600 "$root/$phase/inventory.sha256" printf '%s' "$sha" } write_acceptance_v2() { local path=$1 initial_sha=$2 post_sha=$3 local gate=${4:-RISK_ACCEPTED} ref uid case "$gate" in RECOVERY) ref=strict-recovery-evidence-v1 uid=not-applicable ;; RISK_ACCEPTED) ref=2026-08-14-observability-slack-recovery-risk-acceptance-design uid=1000 ;; *) return 2 ;; esac cat >"$path" <"$path" <"$rollback_root/blackbox-source-proof.env" <"$fixture/output" 2>&1; then rc=0 else rc=$? fi unset PLATFORM_TEST_NGINX_BIN PLATFORM_TEST_SYSTEMCTL_BIN PLATFORM_TEST_TEST_BIN PLATFORM_TEST_CERTBOT_BIN \ PLATFORM_TEST_PREFLIGHT_COMMAND PLATFORM_TEST_POSTCHECK_COMMAND (( rc != 0 )) || fail "test mode accepted an escaped $label command override" } assert_test_command_boundary_mutation_rejected() { local mutation=$1 fixture rc fixture="$(new_fixture "test-boundary-$mutation")" case "$mutation" in missing) rm -f -- "$fixture/bin/test" ;; unsafe-mode) chmod 0775 "$fixture/bin/test" ;; *) fail "unknown test command boundary mutation: $mutation" ;; esac if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi (( rc != 0 )) || fail "test mode accepted $mutation test command boundary" grep -Fq 'test command boundary is unsafe: test' "$fixture/output" || fail "$mutation test command boundary did not identify the test boundary" cmp -s "$fixture/active/learn-services" "$DENY" || fail "$mutation test command boundary changed active bytes" [[ ! -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" && ! -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]] || fail "$mutation test command boundary created a Host ledger" [[ ! -s "$fixture/commands.log" ]] || fail "$mutation test command boundary reached a command boundary" ! grep -Fq 'Type APPLY' "$fixture/output" || fail "$mutation test command boundary reached the APPLY prompt" pass "test mode rejects $mutation test command boundary before contact, ledger, or prompt" } new_fixture() { local name=$1 fixture deny_sha fixture="$WORK/$name" mkdir -m 0700 -p "$fixture" mkdir -p "$fixture/active" "$fixture/enabled" "$fixture/rollbacks/observability-$ROLLBACK_ID" cp "$DENY" "$fixture/active/learn-services" ln -s "$fixture/active/learn-services" "$fixture/enabled/learn-services" chmod 0644 "$fixture/active/learn-services" chmod 0700 "$fixture/rollbacks/observability-$ROLLBACK_ID" : >"$fixture/commands.log" make_fakes "$fixture" deny_sha="$(sha256sum "$DENY" | awk '{print $1}')" write_evidence "$fixture/rollbacks/observability-$ROLLBACK_ID" "$fixture/metrics" "$deny_sha" printf '%s' "$fixture" } acceptance_inventory_shas() { local fixture=$1 initial_sha post_sha initial_sha="$(sha256sum "$fixture/metrics/target-initial/inventory.json" | awk '{print $1}')" post_sha="$(sha256sum "$fixture/metrics/post-substrate/inventory.json" | awk '{print $1}')" printf '%s|%s' "$initial_sha" "$post_sha" } evidence_contract_failures=() task2_contract_failures=() record_evidence_contract_failure() { evidence_contract_failures+=("$1") printf 'RED: %s\n' "$1" >&2 } record_task2_contract_failure() { task2_contract_failures+=("$1") printf 'RED: %s\n' "$1" >&2 } host_tree_fingerprint() { local root=$1 entry if [[ ! -e "$root" && ! -L "$root" ]]; then printf 'ABSENT' return 0 fi { stat -c '.|%F|%u:%g|%a|%h|%N' -- "$root" if [[ -d "$root" && ! -L "$root" ]]; then find -P "$root" -mindepth 1 -printf '%P|%y|%U:%G|%m|%n|%l\n' | sort while IFS= read -r entry; do printf '%s|' "${entry#"$root"/}" sha256sum -- "$entry" | awk '{print $1}' done < <(find -P "$root" -type f -links 1 -print | sort) fi } | sha256sum | awk '{print $1}' } exercise_first_cutover_residue() { local mutation=$1 fixture host_root payload_root ledger before after rc failed=false fixture="$(new_fixture "first-cutover-residue-$mutation")" host_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" payload_root="$host_root/payloads" ledger="$host_root/stages.tsv" case "$mutation" in empty-root) mkdir -m 0700 "$host_root" ;; normal-ledger) mkdir -m 0700 "$host_root" "$payload_root" printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' >"$ledger" chmod 0600 "$ledger" ;; normal-payload) mkdir -m 0700 "$host_root" "$payload_root" cp "$DENY" "$payload_root/full-prior-0001.conf" chmod 0600 "$payload_root/full-prior-0001.conf" ;; *) fail "unknown first-cutover residue mutation: $mutation" ;; esac before="$(host_tree_fingerprint "$host_root")" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi after="$(host_tree_fingerprint "$host_root")" if (( rc == 0 )); then record_task2_contract_failure "$mutation residue returned success" failed=true fi if ! cmp -s "$fixture/active/learn-services" "$DENY"; then record_task2_contract_failure "$mutation residue changed active deny bytes" failed=true fi if [[ "$before" != "$after" ]]; then record_task2_contract_failure "$mutation residue changed the pre-existing Host evidence tree" failed=true fi if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then record_task2_contract_failure "$mutation residue reached preflight or mutation" failed=true fi if grep -Fq 'Type APPLY' "$fixture/output"; then record_task2_contract_failure "$mutation residue reached the APPLY prompt" failed=true fi if [[ "$failed" == false ]]; then pass "full first cutover rejects $mutation residue before preflight, mutation, and prompt" fi } exercise_active_state_gate() { local state=$1 fixture before rc failed=false expected_prefix attempt fixture="$(new_fixture "active-state-$state")" case "$state" in full) cp "$FULL" "$fixture/active/learn-services" ;; third) expected_prefix="$(sha256sum "$DENY" | cut -c1)" for ((attempt=0; attempt<256; attempt++)); do printf 'server { listen 127.0.0.1:65534; } # %d\n' "$attempt" \ >"$fixture/active/learn-services" [[ "$(sha256sum "$fixture/active/learn-services" | cut -c1)" == "$expected_prefix" ]] && break done [[ "$attempt" -lt 256 ]] || fail 'cannot construct deterministic same-prefix third state' ;; *) fail "unknown active state: $state" ;; esac chmod 0644 "$fixture/active/learn-services" before="$(sha256sum "$fixture/active/learn-services")" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi if (( rc == 0 )); then record_task2_contract_failure "$state active state returned success" failed=true fi if [[ "$(sha256sum "$fixture/active/learn-services")" != "$before" ]]; then record_task2_contract_failure "$state active state changed active bytes" failed=true fi if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then record_task2_contract_failure "$state active state reached preflight or mutation" failed=true fi if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" || -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then record_task2_contract_failure "$state active state created a Host ledger" failed=true fi if grep -Fq 'Type APPLY' "$fixture/output"; then record_task2_contract_failure "$state active state reached the APPLY prompt" failed=true fi if [[ "$state" == full ]] && grep -Fq 'ALREADY_ACTIVE' "$fixture/output"; then record_task2_contract_failure 'full active state emitted ALREADY_ACTIVE' failed=true fi if [[ "$failed" == false ]]; then pass "full cutover rejects $state active state before preflight, ledger, and prompt" fi } exercise_source_boundary_contract() { local output rc=0 output="$(env APPLY="$APPLY" /usr/bin/bash <<'SH' 2>&1 set -Eeuo pipefail source "$APPLY" [[ "$PATH" == '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' ]] [[ "$TEST_BIN" == /usr/bin/test ]] print_plan_body="$(declare -f print_plan)" if grep -Eq '(^|[[:space:];(])(dig|curl|kubectl|sudo|nginx|systemctl)([[:space:];|)])' <<<"$print_plan_body"; then exit 1 fi expected=$'awk=/usr/bin/awk\nbash=/usr/bin/bash\ncat=/usr/bin/cat\ncurl=/usr/bin/curl\ndate=/usr/bin/date\ndig=/usr/bin/dig\ndirname=/usr/bin/dirname\ngrep=/usr/bin/grep\nhead=/usr/bin/head\nid=/usr/bin/id\ninstall=/usr/bin/install\njq=/usr/bin/jq\nkubectl=/usr/local/bin/kubectl\nmktemp=/usr/bin/mktemp\nopenssl=/usr/bin/openssl\npython3=/usr/bin/python3\nreadlink=/usr/bin/readlink\nrm=/usr/bin/rm\nsed=/usr/bin/sed\nsha256sum=/usr/bin/sha256sum\nsleep=/usr/bin/sleep\nsort=/usr/bin/sort\nss=/usr/bin/ss\nstat=/usr/bin/stat\ntail=/usr/bin/tail\ntest=/usr/bin/test\ntr=/usr/bin/tr' actual="$(for name in "${!HOST_NGINX_PRODUCTION_COMMANDS[@]}"; do printf '%s=%s\n' "$name" "${HOST_NGINX_PRODUCTION_COMMANDS[$name]}"; done | sort)" if (( ${#HOST_NGINX_PRODUCTION_COMMANDS[@]} != 27 )); then printf 'production command allowlist cardinality is not 27\n' >&2 exit 1 fi if [[ "$actual" != "$expected" ]]; then printf 'production command allowlist lacks exact test=/usr/bin/test boundary\n' >&2 exit 1 fi declare -F production_executable_metadata_is_safe >/dev/null reject_production_metadata() { if production_executable_metadata_is_safe "$@"; then return 1 fi } production_executable_metadata_is_safe sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4755 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 0755 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 1 0 4755 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 1 4755 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4755 2 sudo reject_production_metadata sudo /usr/bin/sudo /usr/local/bin/sudo 'regular file' 0 0 4755 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 6755 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4775 1 sudo reject_production_metadata sudo /usr/bin/sudo /usr/bin/sudo 'regular file' 0 0 4757 1 sudo production_executable_metadata_is_safe awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 0755 1 ordinary reject_production_metadata awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 4755 1 ordinary reject_production_metadata awk /usr/bin/awk /usr/bin/awk 'regular file' 0 0 2755 1 ordinary validate_production_command_boundaries SH )" || rc=$? if (( rc != 0 )); then record_task2_contract_failure "fixed production command boundary contract failed: ${output//$'\n'/; }" else pass 'fixed production PATH, exact inventory, sudo exception, and ordinary metadata policy are enforced' fi } assert_privileged_test_source_boundary() { local source=$1 /usr/bin/python3 - "$source" <<'PY' import pathlib import re import sys text = pathlib.Path(sys.argv[1]).read_text() exact = '"$SUDO_BIN" -n "$TEST_BIN"' expected_count = 27 commands = re.findall(r'"\$SUDO_BIN"\s+-n\s+([^\s;&|()]+)', text) test_commands = [ command for command in commands if command == '"$TEST_BIN"' or command.strip('"\'') == 'test' or command.strip('"\'').endswith('/test') ] if text.count(exact) != expected_count: raise SystemExit( f'privileged test boundary exact-call cardinality is {text.count(exact)}, expected {expected_count}' ) if len(test_commands) != expected_count: raise SystemExit( f'privileged test boundary candidate cardinality is {len(test_commands)}, expected {expected_count}' ) if any(command != '"$TEST_BIN"' for command in test_commands): raise SystemExit('privileged test boundary includes a non-fixed test command') for operand in ( '-s "$CERT_DIR/fullchain.pem"', '-s "$CERT_DIR/privkey.pem"', '-f "$CREDENTIALS"', ): if text.count(f'{exact} {operand}') != 1: raise SystemExit(f'privileged test boundary misses exact certificate/credential probe: {operand}') PY } exercise_privileged_test_source_mutation_matrix() { local mutation ordinal rc assert_privileged_test_source_boundary "$APPLY" || fail 'current Host source violates the closed privileged test boundary' for (( ordinal=0; ordinal<27; ordinal++ )); do mutation="$WORK/privileged-test-bare-$ordinal.sh" /usr/bin/python3 - "$APPLY" "$mutation" "$ordinal" <<'PY' import pathlib import sys source = pathlib.Path(sys.argv[1]) destination = pathlib.Path(sys.argv[2]) ordinal = int(sys.argv[3]) text = source.read_text() exact = '"$SUDO_BIN" -n "$TEST_BIN"' bare = '"$SUDO_BIN" -n test' parts = text.split(exact) if len(parts) - 1 != 27: raise SystemExit('unexpected privileged test boundary cardinality while mutating') if ordinal < 0 or ordinal >= len(parts) - 1: raise SystemExit('privileged test mutation ordinal is out of range') mutated = exact.join(parts[:ordinal + 1]) + bare + exact.join(parts[ordinal + 1:]) destination.write_text(mutated) destination.chmod(0o600) PY if assert_privileged_test_source_boundary "$mutation" >"$mutation.output" 2>&1; then fail "privileged test boundary accepted bare-test mutation $ordinal" fi grep -Eq 'privileged test boundary (exact-call|candidate) cardinality is|privileged test boundary includes a non-fixed test command' \ "$mutation.output" || fail "privileged test boundary mutation $ordinal failed outside the boundary assertion" done pass 'closed source matrix rejects every privileged bare-test mutation including certificate and credentials probes' } exercise_active_site_symlink_diagnostic() { local fixture rc expected fixture="$(new_fixture active-site-symlink-diagnostic)" rm -f -- "$fixture/active/learn-services" ln -s "$DENY" "$fixture/active/learn-services" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi (( rc != 0 )) || fail 'active-site symlink was accepted' expected="ERROR: active site is a symlink: $fixture/active/learn-services" grep -Fqx "$expected" "$fixture/output" || fail 'active-site symlink diagnostic drifted from its exact contract' [[ ! -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" && ! -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]] || fail 'active-site symlink diagnostic created a Host ledger' ! grep -Fq 'Type APPLY' "$fixture/output" || fail 'active-site symlink diagnostic reached the APPLY prompt' pass 'active-site symlink retains its exact diagnostic before ledger or prompt' } exercise_precommit_staging_failure() { local stage=$1 fixture host_root rc failed=false reloads=0 fixture="$(new_fixture "staging-failure-$stage")" host_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" unset PLATFORM_TEST_INSTALL_FAIL_MATCH PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH case "$stage" in directory) PLATFORM_TEST_INSTALL_FAIL_MATCH="$host_root" ;; ledger) PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH="$host_root/stages.tsv" ;; payload) PLATFORM_TEST_INSTALL_FAIL_MATCH="$host_root/payloads/full-prior-0001.conf" ;; timestamp) PLATFORM_TEST_INSTALL_FAIL_MATCH='.before-observability-full-' ;; *) fail "unknown precommit staging failure: $stage" ;; esac if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi unset PLATFORM_TEST_INSTALL_FAIL_MATCH PLATFORM_TEST_INSTALL_UNSAFE_MODE_MATCH [[ ! -f "$fixture/reload-counter" ]] || read -r reloads <"$fixture/reload-counter" if (( rc == 0 )); then record_task2_contract_failure "$stage staging failure returned success" failed=true fi if ! cmp -s "$fixture/active/learn-services" "$DENY"; then record_task2_contract_failure "$stage staging failure changed active deny bytes" failed=true fi if [[ "$reloads" != 0 ]]; then record_task2_contract_failure "$stage staging failure reloaded Nginx" failed=true fi if grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=' "$fixture/output"; then record_task2_contract_failure "$stage pre-transaction staging failure emitted a rollback marker" failed=true fi if ! grep -Fq 'fresh rollback ID + source proof + complete Task 6 required' "$fixture/output"; then record_task2_contract_failure "$stage staging failure omitted the frozen-ID terminal classification" failed=true fi case "$stage" in directory) if [[ -e "$host_root" || -L "$host_root" ]]; then record_task2_contract_failure 'directory staging failure unexpectedly created the Host root' failed=true fi ;; ledger) if [[ ! -d "$host_root/payloads" || ! -f "$host_root/stages.tsv" || "$(stat -c %a "$host_root/stages.tsv")" != 644 ]]; then record_task2_contract_failure 'ledger staging failure did not preserve its unsafe staged evidence for review' failed=true fi ;; payload) if [[ ! -f "$host_root/stages.tsv" || -e "$host_root/payloads/full-prior-0001.conf" ]]; then record_task2_contract_failure 'payload staging failure evidence topology is not exact' failed=true fi ;; timestamp) if [[ ! -f "$host_root/stages.tsv" || ! -f "$host_root/payloads/full-prior-0001.conf" ]] || compgen -G "$fixture/active/learn-services.before-observability-full-*" >/dev/null; then record_task2_contract_failure 'timestamp staging failure evidence topology is not exact' failed=true fi ;; esac if [[ "$failed" == false ]]; then pass "$stage pre-transaction staging failure freezes the ID without a rollback claim" fi } exercise_valid_acceptance_tuple() { local gate=$1 fixture acceptance inventory_shas initial_sha post_sha rc failed=false fixture="$(new_fixture "valid-${gate,,}-acceptance")" acceptance="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" inventory_shas="$(acceptance_inventory_shas "$fixture")" initial_sha=${inventory_shas%%|*} post_sha=${inventory_shas#*|} write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" "$gate" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi if (( rc != 0 )); then record_evidence_contract_failure "valid $gate v2 acceptance was rejected with RC $rc" failed=true if cmp -s "$fixture/active/learn-services" "$DENY"; then printf 'RED-PROOF: valid %s v2 rejection preserved exact deny bytes\n' "$gate" >&2 else record_evidence_contract_failure "valid $gate v2 rejection changed the active deny bytes" fi fi if ! cmp -s "$fixture/active/learn-services" "$FULL"; then record_evidence_contract_failure "valid $gate v2 acceptance did not install the full candidate" failed=true fi if [[ ! -f "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/stages.tsv" ]]; then record_evidence_contract_failure "valid $gate v2 acceptance did not create the Host ledger" failed=true fi if ! grep -Fq 'Type APPLY' "$fixture/output"; then record_evidence_contract_failure "valid $gate v2 acceptance did not reach the APPLY prompt" failed=true fi if [[ "$failed" == false ]]; then pass "full evidence accepts the exact $gate v2 tuple" fi } exercise_invalid_acceptance() { local mutation=$1 fixture acceptance inventory_shas initial_sha post_sha rc failed=false fixture="$(new_fixture "invalid-${mutation}")" acceptance="$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" inventory_shas="$(acceptance_inventory_shas "$fixture")" initial_sha=${inventory_shas%%|*} post_sha=${inventory_shas#*|} case "$mutation" in v1) write_acceptance_v1 "$acceptance" "$initial_sha" "$post_sha" ;; wrong-schema) sed -i 's/^schema=.*/schema=wrong-schema/' "$acceptance" ;; missing-slack-gate) sed -i '/^slack_deployment_gate=/d' "$acceptance" ;; extra-field) printf 'unexpected=value\n' >>"$acceptance" ;; duplicate-field) printf 'state=accepted\n' >>"$acceptance" ;; recovery-with-risk-ref) write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" RECOVERY sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=2026-08-14-observability-slack-recovery-risk-acceptance-design/' "$acceptance" ;; recovery-with-uid-1000) write_acceptance_v2 "$acceptance" "$initial_sha" "$post_sha" RECOVERY sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=1000/' "$acceptance" ;; risk-with-strict-ref) sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=strict-recovery-evidence-v1/' "$acceptance" ;; risk-with-not-applicable) sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=not-applicable/' "$acceptance" ;; arbitrary-gate) sed -i 's/^slack_deployment_gate=.*/slack_deployment_gate=ARBITRARY/' "$acceptance" ;; arbitrary-ref) sed -i 's/^slack_gate_approval_ref=.*/slack_gate_approval_ref=arbitrary-ref/' "$acceptance" ;; arbitrary-uid) sed -i 's/^slack_gate_accepted_by_uid=.*/slack_gate_accepted_by_uid=2000/' "$acceptance" ;; rollback-id) sed -i 's/^rollback_id=.*/rollback_id=20260812T120001Z/' "$acceptance" ;; target-initial-hash) sed -i 's/^target_initial_sha256=.*/target_initial_sha256=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/' "$acceptance" ;; post-substrate-hash) sed -i 's/^post_substrate_sha256=.*/post_substrate_sha256=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc/' "$acceptance" ;; state) sed -i 's/^state=.*/state=pending/' "$acceptance" ;; malformed-accepted-at) sed -i 's/^accepted_at_utc=.*/accepted_at_utc=2026-02-30T00:00:00Z/' "$acceptance" ;; *) fail "unknown acceptance mutation: $mutation" ;; esac if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi if (( rc == 0 )); then record_evidence_contract_failure "$mutation acceptance returned success" failed=true fi if ! cmp -s "$fixture/active/learn-services" "$DENY"; then record_evidence_contract_failure "$mutation acceptance changed the active deny bytes" failed=true fi if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then record_evidence_contract_failure "$mutation acceptance reached preflight or mutation" failed=true fi if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" || -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then record_evidence_contract_failure "$mutation acceptance created a Host ledger" failed=true fi if grep -Fq 'Type APPLY' "$fixture/output"; then record_evidence_contract_failure "$mutation acceptance reached the APPLY prompt" failed=true fi if [[ "$failed" == false ]]; then pass "full evidence rejects $mutation before preflight, mutation, ledger, and prompt" fi } exercise_invalid_source_proof() { local mutation=$1 fixture proof rc failed=false mutated_time fixed_now='' fixture="$(new_fixture "invalid-source-${mutation}")" proof="$fixture/rollbacks/observability-$ROLLBACK_ID/blackbox-source-proof.env" case "$mutation" in deny-sha) sed -i 's/^nginx_sha256=.*/nginx_sha256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' "$proof" ;; status) sed -i 's/^grafana_status=403$/grafana_status=200/' "$proof" ;; stale) mutated_time="$(date -u -d 'now - 86401 seconds' +%Y-%m-%dT%H:%M:%SZ)" sed -i "s/^tested_at_utc=.*/tested_at_utc=$mutated_time/" "$proof" ;; future) fixed_now="$(date -u +%s)" mutated_time="$(date -u -d "@$((fixed_now + 301))" +%Y-%m-%dT%H:%M:%SZ)" sed -i "s/^tested_at_utc=.*/tested_at_utc=$mutated_time/" "$proof" ;; invalid-calendar) sed -i 's/^tested_at_utc=.*/tested_at_utc=2026-02-30T00:00:00Z/' "$proof" ;; *) fail "unknown source-proof mutation: $mutation" ;; esac if PLATFORM_TEST_DATE_NOW_EPOCH="$fixed_now" run_apply "$fixture" \ --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then rc=0 else rc=$? fi if (( rc == 0 )); then record_evidence_contract_failure "source-proof $mutation returned success" failed=true fi if ! cmp -s "$fixture/active/learn-services" "$DENY"; then record_evidence_contract_failure "source-proof $mutation changed the active deny bytes" failed=true fi if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then record_evidence_contract_failure "source-proof $mutation reached preflight or mutation" failed=true fi if [[ -e "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" || -L "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ]]; then record_evidence_contract_failure "source-proof $mutation created a Host ledger" failed=true fi if grep -Fq 'Type APPLY' "$fixture/output"; then record_evidence_contract_failure "source-proof $mutation reached the APPLY prompt" failed=true fi if [[ "$failed" == false ]]; then pass "full evidence rejects source-proof $mutation before preflight, mutation, ledger, and prompt" fi } WORK="$(mktemp -d /tmp/platform-host-nginx-test.XXXXXX)" chmod 0700 "$WORK" if env PLATFORM_HOST_NGINX_ACTIVE="$DENY" bash "$APPLY" >"$WORK/production-path-override.out" 2>&1; then fail 'production dry-run accepted a Host Nginx path override' fi pass 'production rejects Host Nginx path overrides' if env PLATFORM_HOST_NGINX_NGINX_BIN=/bin/true bash "$APPLY" >"$WORK/production-binary-override.out" 2>&1; then fail 'production dry-run accepted a Host Nginx binary override' fi pass 'production rejects Host Nginx binary overrides' fixture="$(new_fixture rejects-real-sudo)" if PLATFORM_TEST_SUDO_BIN=/usr/bin/sudo run_apply "$fixture" >"$fixture/output" 2>&1; then fail 'test-mode dry-run accepted the real sudo binary' fi grep -Fq 'test command boundary is unsafe' "$fixture/output" || fail 'test-mode real sudo rejection did not identify the command boundary' pass 'test mode rejects the real sudo binary before it can execute' for command in nginx systemctl test certbot preflight postcheck; do assert_test_command_override_rejected "$command" /usr/bin/true fixture="$WORK/rejects-$command-override" grep -Fq 'test command boundary is unsafe' "$fixture/output" || fail "test-mode escaped $command rejection did not identify the command boundary" pass "test mode rejects escaped $command command overrides" done fixture="$(new_fixture rejects-foreign-fixture-bin)" foreign_fixture="$(new_fixture foreign-fixture-bin)" if PLATFORM_TEST_SUDO_BIN="$foreign_fixture/bin/sudo" \ PLATFORM_TEST_NGINX_BIN="$foreign_fixture/bin/nginx" \ PLATFORM_TEST_SYSTEMCTL_BIN="$foreign_fixture/bin/systemctl" \ PLATFORM_TEST_TEST_BIN="$foreign_fixture/bin/test" \ PLATFORM_TEST_CERTBOT_BIN="$foreign_fixture/bin/certbot" \ PLATFORM_TEST_PREFLIGHT_COMMAND="$foreign_fixture/bin/preflight" \ PLATFORM_TEST_POSTCHECK_COMMAND="$foreign_fixture/bin/postcheck" \ run_apply "$fixture" >"$fixture/output" 2>&1; then fail 'test mode accepted command boundaries from a different fixture tree' fi grep -Fq 'test command boundary is unsafe: sudo escapes the fixture bin' "$fixture/output" || fail 'foreign fixture command boundary rejection did not identify the fixture mismatch' pass 'test mode requires command boundaries from the active and rollback fixture tree' for mutation in missing unsafe-mode; do assert_test_command_boundary_mutation_rejected "$mutation" done fixture="$(new_fixture dry-run)" before="$(sha256sum "$fixture/active/learn-services")" output="$(run_apply "$fixture")" || fail 'dry-run failed' [[ "$output" == *'HOST_NGINX_OBSERVABILITY_DRY_RUN=PASS'* ]] || fail 'dry-run marker missing' [[ "$(sha256sum "$fixture/active/learn-services")" == "$before" ]] || fail 'dry-run mutated active config' [[ ! -s "$fixture/commands.log" ]] || fail 'dry-run invoked a mutating boundary' dry_run_contract_ok=true for marker in \ 'HOST_NGINX_CERTIFICATE_EXPECTED_SAN=grafana.learn.hyeonworks.com' \ 'HOST_NGINX_CERTIFICATE_SAN=NOT_CHECKED_DRY_RUN' \ 'HOST_NGINX_GRAFANA_PUBLIC_DNS=NOT_CHECKED_DRY_RUN'; do if [[ "$(grep -Fxc "$marker" <<<"$output")" != 1 ]]; then record_task2_contract_failure "dry-run marker is not exact-once: $marker" dry_run_contract_ok=false fi done for forbidden in HOST_NGINX_CERTIFICATE_EXACT_SAN= HOST_NGINX_GRAFANA_PUBLIC_DNS=ABSENT; do if grep -Fq "$forbidden" <<<"$output"; then record_task2_contract_failure "dry-run retained false live claim: $forbidden" dry_run_contract_ok=false fi done if [[ "$dry_run_contract_ok" == true ]]; then pass 'dry-run is no-contact and emits truthful expected/not-checked markers' fi hostile="$WORK/hostile-path" mkdir -m 0700 "$hostile" cat >"$hostile/dirname" <<'SH' #!/usr/bin/env bash printf 'dirname %s\n' "$*" >>"$PLATFORM_TEST_HOSTILE_LOG" exec /usr/bin/dirname "$@" SH chmod 0700 "$hostile/dirname" : >"$WORK/hostile-path.log" if env PATH="$hostile:$PATH" PLATFORM_TEST_HOSTILE_LOG="$WORK/hostile-path.log" \ PLATFORM_HOST_NGINX_TEST_MODE=invalid /usr/bin/bash "$APPLY" >"$WORK/hostile-path.out" 2>&1; then record_task2_contract_failure 'invalid test mode unexpectedly returned success under hostile PATH' elif [[ -s "$WORK/hostile-path.log" ]]; then record_task2_contract_failure 'ambient dirname executed before hostile test mode rejection' else pass 'fixed PATH is installed before ROOT resolution and invalid-mode rejection' fi exercise_source_boundary_contract exercise_privileged_test_source_mutation_matrix exercise_active_site_symlink_diagnostic fixture="$(new_fixture missing-acceptance)" rm -f "$fixture/rollbacks/observability-$ROLLBACK_ID/access-rules-alerts/acceptance.env" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'missing acceptance was accepted' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'failed preflight changed active config' if grep -Eq '^(nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then fail 'missing acceptance reached a mutation boundary' fi pass 'full mode refuses missing acceptance with zero mutation' fixture="$(new_fixture unsafe-inventory-mode)" chmod 0755 "$fixture/metrics/target-initial" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'unsafe inventory mode was hidden by the later inventory hash' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'unsafe inventory mode changed active config' if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then fail 'unsafe inventory mode reached a mutation or preflight boundary' fi pass 'unsafe inventory metadata cannot be hidden by a valid hash' fixture="$(new_fixture symlinked-inventory-json)" mv "$fixture/metrics/target-initial/inventory.json" \ "$fixture/metrics/target-initial/inventory.real.json" ln -s inventory.real.json "$fixture/metrics/target-initial/inventory.json" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'symlinked inventory JSON was hidden by its valid target hash' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked inventory JSON changed active config' if grep -Eq '^(preflight|nginx|systemctl|sudo .*install)' "$fixture/commands.log"; then fail 'symlinked inventory JSON reached a mutation or preflight boundary' fi pass 'symlinked inventory JSON cannot be hidden by a valid target hash' fixture="$(new_fixture missing-certificate-key)" mkdir -p "$fixture/cert" printf 'test certificate\n' >"$fixture/cert/fullchain.pem" if run_apply "$fixture" --execute --certificate-only >"$fixture/output" 2>&1; then fail 'missing certificate private key was hidden by later SAN validation' fi if grep -Fq 'HOST_NGINX_GRAFANA_CERTIFICATE=READY' "$fixture/output"; then fail 'missing certificate private key emitted READY' fi pass 'exact certificate validation cannot hide a missing private key' for gate in RECOVERY RISK_ACCEPTED; do exercise_valid_acceptance_tuple "$gate" done for mutation in \ v1 wrong-schema missing-slack-gate extra-field duplicate-field \ recovery-with-risk-ref recovery-with-uid-1000 \ risk-with-strict-ref risk-with-not-applicable \ arbitrary-gate arbitrary-ref arbitrary-uid \ rollback-id target-initial-hash post-substrate-hash state malformed-accepted-at; do exercise_invalid_acceptance "$mutation" done for mutation in deny-sha status stale future invalid-calendar; do exercise_invalid_source_proof "$mutation" done exercise_active_state_gate full exercise_active_state_gate third for mutation in empty-root normal-ledger normal-payload; do exercise_first_cutover_residue "$mutation" done if (( ${#evidence_contract_failures[@]} > 0 )); then fail "full evidence contract mismatches: ${evidence_contract_failures[*]}" fi fixture="$(new_fixture symlinked-payload-leaf)" mkdir -m 0700 "$fixture/escaped-payloads" \ "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" ln -s "$fixture/escaped-payloads" \ "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/payloads" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'symlinked payloads ledger leaf was followed' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked payload leaf changed active config' [[ ! -e "$fixture/escaped-payloads/full-prior-0001.conf" ]] || fail 'prior payload escaped through a symlinked leaf' pass 'ledger payload leaf symlink is rejected before any write' fixture="$(new_fixture symlinked-ledger-parent)" mkdir -m 0700 "$fixture/escaped-host-nginx" ln -s "$fixture/escaped-host-nginx" \ "$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'symlinked host-nginx ledger parent was followed' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked ledger parent changed active config' [[ ! -e "$fixture/escaped-host-nginx/stages.tsv" ]] || fail 'ledger escaped through a symlinked parent' pass 'ledger parent symlink is rejected before any write' fixture="$(new_fixture symlinked-ledger-leaf)" ledger_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" mkdir -m 0700 "$ledger_root" "$ledger_root/payloads" printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' \ >"$fixture/escaped-stages.tsv" chmod 0600 "$fixture/escaped-stages.tsv" ln -s "$fixture/escaped-stages.tsv" "$ledger_root/stages.tsv" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'symlinked stages.tsv ledger leaf was accepted' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'symlinked ledger leaf changed active config' if grep -Fq "sudo -n cat -- $ledger_root/stages.tsv" "$fixture/commands.log"; then fail 'symlinked stages.tsv was read before no-follow rejection' fi pass 'ledger file symlink is rejected before read or write' fixture="$(new_fixture hardlinked-ledger-leaf)" ledger_root="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx" mkdir -m 0700 "$ledger_root" "$ledger_root/payloads" printf '%s\n' $'stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc' \ >"$ledger_root/stages.tsv" chmod 0600 "$ledger_root/stages.tsv" ln "$ledger_root/stages.tsv" "$fixture/linked-stages.tsv" if run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'multiply-linked stages.tsv ledger leaf was accepted' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'multiply-linked ledger leaf changed active config' pass 'ledger file requires exactly one link before read or write' fixture="$(new_fixture success)" run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1 || { sed -n '1,260p' "$fixture/output" >&2 fail 'full transition failed' } cmp -s "$fixture/active/learn-services" "$FULL" || fail 'full candidate was not installed' ledger="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/stages.tsv" [[ -f "$ledger" && ! -L "$ledger" ]] || fail 'authoritative ledger missing' [[ "$(wc -l <"$ledger" | tr -d ' ')" == 2 ]] || fail 'ledger does not have header plus exact stage' awk -F '\t' 'NR == 1 { exit($0 != "stage\tprevious_sha256\tcandidate_sha256\tpayload_file\tpayload_sha256\tapplied_at_utc") } NR == 2 { exit(NF != 6 || $1 != "full" || $4 != "payloads/full-prior-0001.conf") }' "$ledger" || fail 'ledger schema or full row is not exact' payload="$fixture/rollbacks/observability-$ROLLBACK_ID/host-nginx/payloads/full-prior-0001.conf" cmp -s "$payload" "$DENY" || fail 'ledger payload is not exact prior active config' [[ "$(stat -c %a "$ledger")" == 600 && "$(stat -c %a "$payload")" == 600 ]] || fail 'ledger or payload is not mode 0600' grep -Fqx 'systemctl reload nginx' "$fixture/commands.log" || fail 'successful transition did not reload nginx' [[ "$(grep -Fxc 'postcheck' "$fixture/commands.log")" == 3 ]] || fail 'postcheck did not require three consecutive stable results' first_sudo="$(grep '^sudo ' "$fixture/commands.log" | head -n1)" [[ "$first_sudo" == 'sudo -v' ]] || fail 'sudo refresh did not precede root metadata reads' awk -v nginx="$fixture/bin/nginx" ' $0 == "sudo -n " nginx " -t" { sudo_test=NR } $0 == "preflight" { exit(!(sudo_test > 0 && sudo_test < NR)) } END { if (!sudo_test) exit 1 } ' "$fixture/commands.log" || fail 'preflight nginx -t did not route through sudo before the probe hook' ! grep -q '^nginx-without-sudo ' "$fixture/commands.log" || fail 'nginx was invoked outside sudo' grep -Eq "^sudo -n $fixture/bin/test " "$fixture/commands.log" || fail 'full transition did not route test probes through the fixed test executable' ! grep -Eq '^sudo -n test( |$)' "$fixture/commands.log" || fail 'full transition invoked a bare sudo test subcommand' pass 'full transition routes all sudo test probes through the fixed test executable' pass 'full transition records exact prior payload and installs candidate' fixture="$(new_fixture rollback)" if PLATFORM_TEST_POSTCHECK_RC=1 run_apply "$fixture" --execute \ --verified-output-dir "$fixture/metrics" >"$fixture/output" 2>&1; then fail 'failed postcheck returned success' fi cmp -s "$fixture/active/learn-services" "$DENY" || fail 'rollback did not restore exact ledger payload' grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=PASS' "$fixture/output" || fail 'rollback PASS marker missing' [[ "$(grep -Fxc 'systemctl reload nginx' "$fixture/commands.log")" == 2 ]] || fail 'apply and rollback did not each reload nginx' pass 'failed postcheck restores the exact prior payload' fixture="$(new_fixture rollback-failure)" if PLATFORM_TEST_POSTCHECK_RC=1 PLATFORM_TEST_SYSTEMCTL_FAIL_RELOAD_AT=2 \ run_apply "$fixture" --execute --verified-output-dir "$fixture/metrics" \ >"$fixture/output" 2>&1; then fail 'failed rollback returned success' fi grep -Fq 'HOST_NGINX_OBSERVABILITY_ROLLBACK=FAIL' "$fixture/output" || fail 'rollback failure marker missing' grep -Fq 'MANUAL_RECOVERY_REQUIRED=YES' "$fixture/output" || fail 'manual recovery marker missing after rollback failure' pass 'rollback failure is truthfully classified for manual recovery' for stage in directory ledger payload timestamp; do exercise_precommit_staging_failure "$stage" done if (( ${#task2_contract_failures[@]} > 0 )); then fail "Task 2 contract mismatches: ${task2_contract_failures[*]}" fi printf 'Assertions: %d\n' "$ASSERTIONS" printf 'APPLY HOST NGINX OBSERVABILITY TEST PASS\n'