#!/usr/bin/env bash set -Eeuo pipefail umask 077 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" SOURCE_REPOSITORY_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd -P)" ACTUAL_RUNTIME_ROOT="${SOURCE_REPOSITORY_ROOT}/src/build/redis-lab" FIXTURE_ROOT="${SOURCE_REPOSITORY_ROOT}/src/build/redis-lab-contract" REPOSITORY_ROOT="${FIXTURE_ROOT}/repository" LAB_SCRIPT="${REPOSITORY_ROOT}/infra/redis-lab/bin/redis-lab" RUNTIME_ROOT="${REPOSITORY_ROOT}/src/build/redis-lab" FAKE_BIN="${FIXTURE_ROOT}/bin" FAKE_LOG="${FIXTURE_ROOT}/commands.log" FAKE_INVENTORY="${FIXTURE_ROOT}/inventory" FAKE_OWNERSHIP="${FIXTURE_ROOT}/ownership" FAKE_LATE_CREATE="${FIXTURE_ROOT}/late-create" FAKE_INFO_COUNT="${FIXTURE_ROOT}/info-count" FAKE_VIOLATIONS="${FIXTURE_ROOT}/violations" EXPECTED_STATE="${RUNTIME_ROOT}/run.state" EXPECTED_KUBECONFIG="${RUNTIME_ROOT}/kubeconfig" EXPECTED_HOST_KUBECONFIG="${RUNTIME_ROOT}/observations/host-kubeconfig" EXPECTED_CLOUD_INIT_ROOT="${RUNTIME_ROOT}/cloud-init" EXPECTED_RUN_HANDOFF_MARKER="${RUNTIME_ROOT}/run-handoff.started" TEST_HOME_ROOT="${FIXTURE_ROOT}/home" DEFAULT_KUBECONFIG_SNAPSHOT="${FIXTURE_ROOT}/default-kubeconfig.snapshot" ACTUAL_RUNTIME_BEFORE="${FIXTURE_ROOT}/actual-runtime.before" ACTUAL_RUNTIME_AFTER="${FIXTURE_ROOT}/actual-runtime.after" KUBECONFIG_WITH_NAMESPACE_GOLDEN="${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/test/fixtures/kubeconfig-with-namespace.expected.yaml" KUBECONFIG_WITHOUT_NAMESPACE_SOURCE="${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/test/fixtures/kubeconfig-without-namespace.source.yaml" KUBECONFIG_WITHOUT_NAMESPACE_GOLDEN="${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/test/fixtures/kubeconfig-without-namespace.expected.yaml" KUBECONFIG_WITHOUT_NAMESPACE_ACTUAL="${FIXTURE_ROOT}/kubeconfig-without-namespace.actual" export REDIS_LAB_FAKE_OWNERSHIP="${FAKE_OWNERSHIP}" export REDIS_LAB_FAKE_LATE_CREATE="${FAKE_LATE_CREATE}" export REDIS_LAB_FAKE_INFO_COUNT="${FAKE_INFO_COUNT}" export REDIS_LAB_EXPECTED_CLOUD_INIT_ROOT="${EXPECTED_CLOUD_INIT_ROOT}" fail() { printf 'redis-lab-contract: %s\n' "$1" >&2 exit 1 } assert_file_contains() { local file="$1" local literal="$2" grep -Fqx -- "${literal}" "${file}" || fail "missing expected line: ${literal}" } assert_file_not_contains_pattern() { local file="$1" local pattern="$2" if grep -Eq -- "${pattern}" "${file}"; then fail "forbidden command matched: ${pattern}" fi } assert_count() { local expected="$1" local pattern="$2" local file="$3" local actual actual="$(grep -Ec -- "${pattern}" "${file}" || true)" [[ "${actual}" == "${expected}" ]] || fail "expected ${expected} matches for ${pattern}, found ${actual}" } assert_fails() { if "$@"; then fail "command unexpectedly succeeded: $*" fi } assert_fd9_static_contract() { local lifecycle_script="$1" local violations="${FIXTURE_ROOT}/fd9-static.violations" local closed_timeout_count local keep_timeout_count local acquisition_count local timeout_count /usr/bin/awk ' function inspect() { candidate = logical sub(/^[[:space:]]+/, "", candidate) if (candidate == "" || candidate ~ /^#/ || candidate ~ /^\047/) { logical = "" return } if (candidate ~ /(^|[[:space:];|!&(])(mkdir|chmod|rm|cp|mv|awk|sort|grep|tr|cat|cmp|sleep|multipass|kubectl|ip|curl|sha256sum|openssl|uname)([[:space:]]|$)/ && candidate !~ /(^|[[:space:];|!&(])(run_child|bounded)([[:space:]]|$)/) { print start_line ":" candidate } logical = "" } { if (logical == "") { start_line = NR } logical = logical " " $0 if ($0 ~ /\\[[:space:]]*$/ || $0 ~ /\|[[:space:]]*$/ || $0 ~ /\|\|[[:space:]]*$/) { next } inspect() } END { if (logical != "") { inspect() } } ' "${lifecycle_script}" >"${violations}" if [[ -s "${violations}" ]]; then IFS= read -r first_violation <"${violations}" fail "raw post-lock child path is not FD9-closed: ${first_violation}" fi closed_timeout_count="$( grep -Fxc -- \ ' timeout --signal=TERM --kill-after=5s "${seconds}s" "$@" 9>&-' \ "${lifecycle_script}" || true )" keep_timeout_count="$( grep -Fxc -- \ ' timeout --signal=TERM --kill-after=5s "${seconds}s" "$@"' \ "${lifecycle_script}" || true )" acquisition_count="$( grep -Fc -- 'bounded_keep_lock 5 flock -n 9' "${lifecycle_script}" || true )" timeout_count="$( grep -Fc -- 'timeout --signal=TERM --kill-after=5s' "${lifecycle_script}" || true )" if [[ "${closed_timeout_count}" != 1 || "${keep_timeout_count}" != 1 || "${acquisition_count}" != 1 || "${timeout_count}" != 2 ]]; then fail 'FD9 timeout/lock acquisition inventory changed' fi } reset_case() { : >"${FAKE_LOG}" : >"${FAKE_INVENTORY}" : >"${FAKE_OWNERSHIP}" : >"${FAKE_LATE_CREATE}" : >"${FAKE_INFO_COUNT}" rm -rf -- \ "${RUNTIME_ROOT}/cloud-init" \ "${RUNTIME_ROOT}/observations" \ "${RUNTIME_ROOT}/fingerprint.before" \ "${RUNTIME_ROOT}/fingerprint.after" \ "${RUNTIME_ROOT}/k3s-token" \ "${RUNTIME_ROOT}/kubeconfig" \ "${RUNTIME_ROOT}/kubeconfig.next" \ "${EXPECTED_RUN_HANDOFF_MARKER}" \ "${RUNTIME_ROOT}/run.state" mkdir -p -- "${TEST_HOME_ROOT}/.kube" printf '%s\n' \ 'apiVersion: v1' \ 'clusters:' \ '- cluster:' \ ' server: https://host.invalid:6443' \ ' name: host-cluster' \ 'contexts:' \ '- context:' \ ' cluster: host-cluster' \ ' user: host-user' \ ' name: host-context' \ 'current-context: host-context' \ 'kind: Config' \ 'users:' \ '- name: host-user' \ ' user: {}' >"${TEST_HOME_ROOT}/.kube/config" cp -- "${TEST_HOME_ROOT}/.kube/config" "${DEFAULT_KUBECONFIG_SNAPSHOT}" } prepare_tracked_input_repository() { local repository_root="$1" rm -rf -- "${repository_root}" mkdir -p -- \ "${repository_root}/infra/redis-lab/bin" \ "${repository_root}/infra/redis-lab/cloud-init" \ "${repository_root}/infra/redis-lab/lib" \ "${repository_root}/src" cp -- "${LAB_SCRIPT}" "${repository_root}/infra/redis-lab/bin/redis-lab" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" \ "${repository_root}/infra/redis-lab/cloud-init/node.yaml" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk" \ "${repository_root}/infra/redis-lab/lib/render-kubeconfig.awk" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/versions.env" \ "${repository_root}/infra/redis-lab/versions.env" chmod 0700 -- "${repository_root}/infra/redis-lab/bin/redis-lab" } snapshot_actual_runtime() { local destination="$1" local path if [[ ! -e "${ACTUAL_RUNTIME_ROOT}" && ! -L "${ACTUAL_RUNTIME_ROOT}" ]]; then printf '%s\n' 'ABSENT' >"${destination}" return fi { find "${ACTUAL_RUNTIME_ROOT}" -printf '%y|%m|%p|%l\n' | LC_ALL=C sort while IFS= read -r path; do /usr/bin/sha256sum "${path}" done < <(find "${ACTUAL_RUNTIME_ROOT}" -type f -print | LC_ALL=C sort) } >"${destination}" } run_lab() { local expected_state="${RUN_LAB_EXPECTED_STATE:-${EXPECTED_STATE}}" local expected_kubeconfig="${RUN_LAB_EXPECTED_KUBECONFIG:-${EXPECTED_KUBECONFIG}}" local expected_host_kubeconfig="${RUN_LAB_EXPECTED_HOST_KUBECONFIG:-${EXPECTED_HOST_KUBECONFIG}}" local expected_cloud_init_root="${RUN_LAB_EXPECTED_CLOUD_INIT_ROOT:-${EXPECTED_CLOUD_INIT_ROOT}}" env \ PATH="${FAKE_BIN}" \ REDIS_LAB_CONTRACT_TEST=1 \ REDIS_LAB_HOST_SERVICE_CIDRS='10.81.0.0/16' \ REDIS_LAB_HOME_DIR="${TEST_HOME_ROOT}" \ REDIS_LAB_FAKE_LOG="${FAKE_LOG}" \ REDIS_LAB_FAKE_INVENTORY="${FAKE_INVENTORY}" \ REDIS_LAB_FAKE_OWNERSHIP="${FAKE_OWNERSHIP}" \ REDIS_LAB_FAKE_LATE_CREATE="${FAKE_LATE_CREATE}" \ REDIS_LAB_FAKE_INFO_COUNT="${FAKE_INFO_COUNT}" \ REDIS_LAB_FAKE_VIOLATIONS="${FAKE_VIOLATIONS}" \ REDIS_LAB_EXPECTED_STATE="${expected_state}" \ REDIS_LAB_EXPECTED_KUBECONFIG="${expected_kubeconfig}" \ REDIS_LAB_EXPECTED_HOST_KUBECONFIG="${expected_host_kubeconfig}" \ REDIS_LAB_EXPECTED_CLOUD_INIT_ROOT="${expected_cloud_init_root}" \ "$@" } rm -rf -- "${FIXTURE_ROOT}" mkdir -p -- \ "${REPOSITORY_ROOT}/infra/redis-lab/bin" \ "${REPOSITORY_ROOT}/infra/redis-lab/cloud-init" \ "${REPOSITORY_ROOT}/infra/redis-lab/lib" \ "${REPOSITORY_ROOT}/src" \ "${FAKE_BIN}" \ "${TEST_HOME_ROOT}/.kube" snapshot_actual_runtime "${ACTUAL_RUNTIME_BEFORE}" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/bin/redis-lab" \ "${REPOSITORY_ROOT}/infra/redis-lab/bin/redis-lab" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" \ "${REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk" \ "${REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/versions.env" \ "${REPOSITORY_ROOT}/infra/redis-lab/versions.env" : >"${FAKE_LOG}" : >"${FAKE_INVENTORY}" : >"${FAKE_OWNERSHIP}" : >"${FAKE_LATE_CREATE}" : >"${FAKE_INFO_COUNT}" : >"${FAKE_VIOLATIONS}" if ! /usr/bin/awk -v address=192.0.2.10 -v target=ca-redis-lab \ -f "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk" \ "${KUBECONFIG_WITHOUT_NAMESPACE_SOURCE}" \ >"${KUBECONFIG_WITHOUT_NAMESPACE_ACTUAL}"; then fail 'namespace-absent kubeconfig was rejected' fi cmp -s -- \ "${KUBECONFIG_WITHOUT_NAMESPACE_ACTUAL}" \ "${KUBECONFIG_WITHOUT_NAMESPACE_GOLDEN}" || fail 'namespace-absent kubeconfig output changed' cat >"${FAKE_BIN}/safe-command" <<'FAKE_SAFE_COMMAND' #!/bin/bash set -Eeuo pipefail command_name="${0##*/}" if [[ "${command_name}" != bash ]]; then printf 'child <%s>\n' "${command_name}" >>"${REDIS_LAB_FAKE_LOG}" fi if [[ -e "/proc/${BASHPID}/fd/9" || -e /dev/fd/9 ]]; then printf 'fd9-leak <%s>\n' "${command_name}" >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 116 fi exec "/usr/bin/${command_name}" "$@" FAKE_SAFE_COMMAND for safe_command in bash dirname awk sort grep rm mkdir cp tr cat cmp env; do ln -s -- "${FAKE_BIN}/safe-command" "${FAKE_BIN}/${safe_command}" done cat >"${FAKE_BIN}/flock" <<'FAKE_FLOCK' #!/bin/bash set -Eeuo pipefail if [[ ! -e "/proc/${BASHPID}/fd/9" && ! -e /dev/fd/9 ]] || [[ "$#" != 2 || "$1" != -n || "$2" != 9 ]]; then printf '%s\n' 'fd9-lock-acquisition-contract' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 117 fi exec /usr/bin/flock "$@" FAKE_FLOCK for denied_command in docker sudo systemctl k3s wget; do cat >"${FAKE_BIN}/${denied_command}" <<'FAKE_DENIED' #!/usr/bin/env bash printf '%s\n' 'sealed-path-denied-command' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 127 FAKE_DENIED done cat >"${FAKE_BIN}/multipass" <<'FAKE_MULTIPASS' #!/usr/bin/env bash set -Eeuo pipefail printf 'multipass' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" command_name="${1:-}" ownership_file="${REDIS_LAB_FAKE_OWNERSHIP:-${REDIS_LAB_FAKE_INVENTORY}.ownership}" late_create_file="${REDIS_LAB_FAKE_LATE_CREATE:-${REDIS_LAB_FAKE_INVENTORY}.late-create}" info_count_file="${REDIS_LAB_FAKE_INFO_COUNT:-${REDIS_LAB_FAKE_INVENTORY}.info-count}" expected_cloud_init_root="${REDIS_LAB_EXPECTED_CLOUD_INIT_ROOT:-$(dirname -- "${REDIS_LAB_EXPECTED_STATE}")/cloud-init}" record_ownership() { local target_name="$1" local marker="$2" local ownership_next="${ownership_file}.next" grep -Ev "^${target_name}\\|" "${ownership_file}" >"${ownership_next}" || true printf '%s|%s\n' "${target_name}" "${marker}" >>"${ownership_next}" mv -- "${ownership_next}" "${ownership_file}" } emit_kubeconfig() { local variant="$1" local swap local -a lines=( 'apiVersion: v1' 'clusters:' '- cluster:' ' certificate-authority-data: preserve-default-ca-canary' ' server: https://127.0.0.1:6443' ' name: default' 'contexts:' '- context:' ' cluster: default' ' namespace: team-default' ' user: default' ' name: default' 'current-context: default' 'kind: Config' 'preferences: {}' 'users:' '- name: default' ' user:' ' client-certificate-data: preserve-default-client-cert-canary' ' client-key-data: preserve-default-client-key-canary' ) case "${variant}" in valid) ;; missing-server) unset 'lines[4]' ;; duplicate-server) lines[4]+=$'\n server: https://127.0.0.1:6443' ;; reordered-kind-preferences) swap="${lines[13]}" lines[13]="${lines[14]}" lines[14]="${swap}" ;; unknown-cluster-key) lines[4]+=$'\n proxy-url: https://proxy.invalid' ;; whitespace-before-colon) lines[4]=' server : https://127.0.0.1:6443' ;; quoted-key) lines[4]=' "server": https://127.0.0.1:6443' ;; tagged-key) lines[4]=' !!str server: https://127.0.0.1:6443' ;; explicit-key) lines[4]=$' ? server\n : https://127.0.0.1:6443' ;; anchor) lines[2]='- cluster: &default-cluster' ;; alias) lines[4]=' server: *loopback-server' ;; merge) lines[4]+=$'\n <<: *default-cluster' ;; flow-map) lines[4]=' server: {value: https://127.0.0.1:6443}' ;; flow-sequence) lines[4]=' server: [https://127.0.0.1:6443]' ;; tab-indentation) lines[4]=$'\tserver: https://127.0.0.1:6443' ;; crlf) printf '%s\r\n' "${lines[@]}" return ;; document-start) lines[0]=$'---\napiVersion: v1' ;; document-end) lines[19]+=$'\n...' ;; trailing-content) lines[19]+=$'\nmetadata: forbidden' ;; sibling-flow-cluster) lines[5]+=$'\n cluster : {server: https://foreign.invalid:6443}' ;; sibling-flow-context) lines[11]+=$'\n context : {cluster: foreign, user: foreign}' ;; *) printf '%s\n' 'multipass-kubeconfig-variant' \ >>"${REDIS_LAB_FAKE_VIOLATIONS}" return 115 ;; esac printf '%s\n' "${lines[@]}" } case "${command_name}" in list) if [[ -n "${REDIS_LAB_FAKE_INFRA_BACKGROUND_PID_FILE:-}" && ! -e "${REDIS_LAB_FAKE_INFRA_BACKGROUND_PID_FILE}" ]]; then /bin/sleep 3 >/dev/null 2>&1 & printf '%s\n' "$!" >"${REDIS_LAB_FAKE_INFRA_BACKGROUND_PID_FILE}" fi [[ "$#" == 3 && "$2" == '--format' && "$3" == 'csv' ]] || { printf '%s\n' 'multipass-list-arguments' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 90 } printf '%s\n' 'Name,State,IPv4,Image' while IFS= read -r name; do [[ -n "${name}" ]] || continue printf '%s\n' "${name},Running,192.0.2.10,Ubuntu 24.04 LTS" done <"${REDIS_LAB_FAKE_INVENTORY}" ;; launch) name='' cpus='' memory='' disk='' cloud_init='' image='' shift while (($#)); do case "$1" in --name) name="${2:-}" shift 2 ;; --cpus) cpus="${2:-}" shift 2 ;; --memory) memory="${2:-}" shift 2 ;; --disk) disk="${2:-}" shift 2 ;; --cloud-init) cloud_init="${2:-}" shift 2 ;; *) image="$1" shift ;; esac done case "${name}:${cpus}:${memory}:${disk}" in ca-redis-lab-server:2:3G:12G | \ ca-redis-lab-agent-1:2:2560M:12G | \ ca-redis-lab-agent-2:2:2560M:12G) ;; *) printf '%s\n' 'multipass-launch-contract' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 91 ;; esac [[ "${image}" == '24.04' ]] || { printf '%s\n' 'multipass-launch-inputs' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 92 } [[ "${cloud_init}" == "${expected_cloud_init_root}/${name}.yaml" && "$(/usr/bin/stat -c '%a' "${cloud_init}")" == 600 ]] || { printf '%s\n' 'multipass-rendered-cloud-init' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 111 } ownership_marker="$(awk -v target="${name}" ' $0 ~ "^ run-[0-9]+-[0-9]+-[0-9]+\\|" target "$" { sub(/^ /, "") print found = 1 exit } END { if (!found) { exit 1 } } ' "${cloud_init}")" || { printf '%s\n' 'multipass-cloud-init-ownership' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 112 } grep -Eq "^PENDING\\|run-[0-9]+-[0-9]+-[0-9]+\\|${name}$" \ "${REDIS_LAB_EXPECTED_STATE}" || { printf '%s\n' 'launch-attempt-not-reserved' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 99 } if [[ "${name}" == 'ca-redis-lab-agent-1' ]]; then grep -Eq '^CREATED\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-server$' \ "${REDIS_LAB_EXPECTED_STATE}" || { printf '%s\n' 'server-state-not-recorded-before-next-launch' \ >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 93 } fi if [[ "${name}" == 'ca-redis-lab-agent-2' ]]; then grep -Eq '^CREATED\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-agent-1$' \ "${REDIS_LAB_EXPECTED_STATE}" || { printf '%s\n' 'agent-state-not-recorded-before-next-launch' \ >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 94 } fi if [[ "${REDIS_LAB_FAIL_LAUNCH_NAME:-}" == "${name}" ]]; then exit 42 fi printf '%s\n' "${name}" >>"${REDIS_LAB_FAKE_INVENTORY}" if [[ "${name}" == 'ca-redis-lab-server' && "${REDIS_LAB_FAKE_LAUNCH_MARKER_MODE:-}" == missing ]]; then : elif [[ "${name}" == 'ca-redis-lab-server' && "${REDIS_LAB_FAKE_LAUNCH_MARKER_MODE:-}" == foreign ]]; then record_ownership "${name}" "run-999-999-999|${name}" else record_ownership "${name}" "${ownership_marker}" fi if [[ "${REDIS_LAB_FAKE_LAUNCH_SIGNAL_PARENT:-}" == 1 && "${name}" == 'ca-redis-lab-server' ]]; then : >"${REDIS_LAB_FAKE_LAUNCH_BARRIER}.started" kill -TERM "${PPID}" exit 143 fi if [[ -n "${REDIS_LAB_FAKE_LAUNCH_BARRIER:-}" && "${name}" == 'ca-redis-lab-server' ]]; then : >"${REDIS_LAB_FAKE_LAUNCH_BARRIER}.started" barrier_attempt=0 while [[ ! -f "${REDIS_LAB_FAKE_LAUNCH_BARRIER}.release" ]]; do /bin/sleep 0.02 ((barrier_attempt += 1)) ((barrier_attempt < 250)) || exit 43 done fi ;; info) [[ "$#" == 4 && "$2" == '--format' && "$3" == 'csv' ]] || { printf '%s\n' 'multipass-info-arguments' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 95 } name="$4" if ! grep -Fqx -- "${name}" "${REDIS_LAB_FAKE_INVENTORY}" && [[ -n "${REDIS_LAB_FAKE_LATE_CREATE_MODE:-}" ]]; then info_count="$(cat "${info_count_file}" 2>/dev/null || true)" info_count="${info_count:-0}" ((info_count += 1)) printf '%s\n' "${info_count}" >"${info_count_file}" if ((info_count >= 2)) && [[ "${REDIS_LAB_FAKE_LATE_CREATE_MODE}" != absent ]]; then IFS='|' read -r late_name late_run_id late_vm_name <"${late_create_file}" [[ "${late_name}" == "${name}" && "${late_vm_name}" == "${name}" ]] || { printf '%s\n' 'multipass-late-create-contract' \ >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 113 } if [[ "${REDIS_LAB_FAKE_LATE_CREATE_MODE}" == foreign ]]; then late_run_id='run-999-999-999' fi printf '%s\n' "${name}" >>"${REDIS_LAB_FAKE_INVENTORY}" record_ownership "${name}" "${late_run_id}|${late_vm_name}" fi fi grep -Fqx -- "${name}" "${REDIS_LAB_FAKE_INVENTORY}" || exit 2 printf '%s\n' 'Name,State,IPv4,Release,Image hash,Load,Disk usage,Memory usage,Mounts' if [[ "${REDIS_LAB_FAKE_BAD_SERVER_IP:-}" == 1 && "${name}" == 'ca-redis-lab-server' ]]; then printf '%s\n' "${name},Running,999.0.2.10,Ubuntu 24.04 LTS,fake,0.00,1G,128M,--" else printf '%s\n' "${name},Running,192.0.2.10,Ubuntu 24.04 LTS,fake,0.00,1G,128M,--" fi ;; transfer) [[ "$#" == 3 && -f "$2" ]] || { printf '%s\n' 'multipass-transfer-arguments' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 100 } case "$3" in ca-redis-lab-server:/home/ubuntu/ca-redis-lab-k3s | \ ca-redis-lab-agent-1:/home/ubuntu/ca-redis-lab-k3s | \ ca-redis-lab-agent-2:/home/ubuntu/ca-redis-lab-k3s) ;; *) printf '%s\n' 'multipass-transfer-target' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 101 ;; esac ;; exec) if [[ "$#" == 6 && "$3" == '--' && "$4" == sudo && "$5" == cat && "$6" == '/var/lib/ca-redis-lab/ownership' ]]; then name="$2" ownership_marker="$(awk -F'|' -v target="${name}" ' $1 == target { print $2 "|" $3 found = 1 exit } END { if (!found) { exit 1 } } ' "${ownership_file}")" || exit 2 printf '%s\n' "${ownership_marker}" exit 0 fi if [[ "$*" == *'install-k3s-server'* || "$*" == *'install-k3s-agent'* ]]; then IFS= read -r received_token || { printf '%s\n' 'multipass-exec-token-missing' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 104 } [[ "${received_token}" == '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' ]] || { printf '%s\n' 'multipass-exec-token-invalid' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 105 } if [[ "$*" == *'curl '* || "$*" != *'/usr/local/bin/k3s'* || "$*" == *"${received_token}"* ]]; then printf '%s\n' 'multipass-exec-install-contract' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 106 fi if [[ "$*" != *'sha256sum /home/ubuntu/ca-redis-lab-k3s'* || "$*" != *'f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc'* ]]; then printf '%s\n' 'multipass-exec-transferred-sha-contract' \ >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 109 fi if [[ "${REDIS_LAB_FAKE_TRANSFER_SHA_MISMATCH:-}" == 1 ]]; then exit 110 fi if [[ "$*" == *'install-k3s-server'* ]]; then [[ "$*" == *'install-k3s-server f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc v1.33.3+k3s1 server --cluster-cidr=10.52.0.0/16 --service-cidr=10.53.0.0/16 --disable=traefik --disable=servicelb --write-kubeconfig-mode=0600'* ]] || { printf '%s\n' 'multipass-exec-server-contract' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 107 } else [[ "$*" == *'install-k3s-agent https://192.0.2.10:6443 f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc v1.33.3+k3s1 agent'* ]] || { printf '%s\n' 'multipass-exec-agent-contract' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 108 } fi printf 'vm-install <%s>\n' "${2:-}" >>"${REDIS_LAB_FAKE_LOG}" fi if [[ "$*" == *'/etc/rancher/k3s/k3s.yaml'* ]]; then emit_kubeconfig "${REDIS_LAB_FAKE_KUBECONFIG_SCHEMA_VARIANT:-valid}" fi exit 0 ;; delete) [[ "$#" == 3 && "$2" == '--purge' ]] || { printf '%s\n' 'multipass-delete-arguments' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 96 } name="$3" case "${name}" in ca-redis-lab-server | ca-redis-lab-agent-1 | ca-redis-lab-agent-2) ;; *) printf '%s\n' 'multipass-delete-name' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 97 ;; esac if [[ "${REDIS_LAB_FAKE_DELETE_FAILURE_NAME:-}" == "${name}" ]]; then exit 114 fi inventory_next="${REDIS_LAB_FAKE_INVENTORY}.next" grep -Fvx -- "${name}" "${REDIS_LAB_FAKE_INVENTORY}" >"${inventory_next}" || true mv -- "${inventory_next}" "${REDIS_LAB_FAKE_INVENTORY}" ownership_next="${ownership_file}.next" grep -Ev "^${name}\\|" "${ownership_file}" >"${ownership_next}" || true mv -- "${ownership_next}" "${ownership_file}" ;; *) printf '%s\n' 'multipass-subcommand' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 98 ;; esac FAKE_MULTIPASS cat >"${FAKE_BIN}/kubectl" <<'FAKE_KUBECTL' #!/usr/bin/env bash set -Eeuo pipefail printf 'kubectl' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" [[ "${KUBECONFIG+x}" != x ]] || { printf '%s\n' 'kubectl-env-kubeconfig' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 80 } [[ "$#" -ge 4 && "$1" == '--kubeconfig' && "$3" == '--context' ]] || { printf '%s\n' 'kubectl-explicit-target' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 81 } target='unknown' if [[ "$2" == "${REDIS_LAB_EXPECTED_HOST_KUBECONFIG}" && "$4" == 'host-context' ]]; then target='host' elif [[ "$2" == "${REDIS_LAB_EXPECTED_KUBECONFIG}" && "$4" == 'ca-redis-lab' ]]; then target='lab' else printf '%s\n' 'kubectl-explicit-target' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 81 fi shift 4 case "${1:-}" in apply | create | delete | edit | patch | replace | rollout | scale | taint) printf '%s\n' 'host-mutating-kubectl' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 82 ;; config) case "${2:-}" in view) if [[ "${target}" == host ]]; then printf '%s\n' 'https://host.invalid:6443' else printf '%s\n' 'https://192.0.2.10:6443' fi ;; *) printf '%s\n' 'kubectl-config-command' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 83 ;; esac ;; get) case "${2:-}" in nodes) if [[ "${target}" == lab ]]; then if [[ "${REDIS_LAB_FAKE_NOT_READY:-}" == 1 ]]; then printf '%s\n' \ 'ca-redis-lab-server|True' \ 'ca-redis-lab-agent-1|False' else printf '%s\n' \ 'ca-redis-lab-server|True' \ 'ca-redis-lab-agent-1|True' \ 'ca-redis-lab-agent-2|True' fi elif [[ "${REDIS_LAB_FAKE_CIDR_OVERLAP:-}" == 1 ]]; then printf '%s\n' 'host-node|k3s://host|10.52.8.0/24' else printf '%s\n' 'host-node|k3s://host|10.80.0.0/24' fi ;; deployments,statefulsets,daemonsets) printf '%s\n' 'kube-system|deployment|coredns|1' ;; services) printf '%s\n' 'default|kubernetes|443' ;; *) printf '%s\n' 'kubectl-get-resource' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 84 ;; esac ;; *) printf '%s\n' 'kubectl-command' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 85 ;; esac FAKE_KUBECTL cat >"${FAKE_BIN}/ip" <<'FAKE_IP' #!/usr/bin/env bash set -Eeuo pipefail printf 'ip' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" if [[ "$*" == '-o -4 addr show' ]]; then printf '%s\n' '1: lo inet 127.0.0.1/8 scope host lo' printf '%s\n' '2: eth0 inet 192.0.2.20/24 scope global eth0' elif [[ "$*" == '-4 route show table all' ]]; then printf '%s\n' 'default via 192.0.2.1 dev eth0' printf '%s\n' '192.0.2.0/24 dev eth0 scope link' else printf '%s\n' 'ip-arguments' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 70 fi FAKE_IP cat >"${FAKE_BIN}/sha256sum" <<'FAKE_SHA256SUM' #!/usr/bin/env bash set -Eeuo pipefail printf 'sha256sum' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" digest='1111111111111111111111111111111111111111111111111111111111111111' if [[ "${1:-}" == */k3s-amd64 ]]; then digest='f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' if [[ "${REDIS_LAB_FAKE_K3S_SHA_MISMATCH:-}" == 1 ]]; then digest='0000000000000000000000000000000000000000000000000000000000000000' fi fi if [[ "${REDIS_LAB_FAKE_FINGERPRINT_MISMATCH:-}" == 1 && "${REDIS_LAB_CAPTURE_PHASE:-}" == after ]]; then digest='2222222222222222222222222222222222222222222222222222222222222222' fi printf '%s %s\n' "${digest}" "${1:-}" FAKE_SHA256SUM cat >"${FAKE_BIN}/curl" <<'FAKE_CURL' #!/usr/bin/env bash set -Eeuo pipefail printf 'curl' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" destination='' url='' while (($#)); do case "$1" in --fail | --location | --silent | --show-error) shift ;; --output) destination="${2:-}" shift 2 ;; *) url="$1" shift ;; esac done [[ -n "${destination}" && "${url}" == 'https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' ]] || { printf '%s\n' 'curl-pinned-artifact-contract' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 102 } printf '%s\n' 'fake pinned k3s amd64 binary' >"${destination}" FAKE_CURL cat >"${FAKE_BIN}/uname" <<'FAKE_UNAME' #!/usr/bin/env bash set -Eeuo pipefail printf 'uname' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" [[ "$#" == 1 && "$1" == '-m' ]] || exit 103 printf '%s\n' 'x86_64' FAKE_UNAME cat >"${FAKE_BIN}/timeout" <<'FAKE_TIMEOUT' #!/bin/bash set -Eeuo pipefail printf 'timeout' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" while (($#)); do case "$1" in --signal=* | --kill-after=*) shift ;; *) shift break ;; esac done (("$#" > 0)) || exit 64 if [[ -e "/proc/${BASHPID}/fd/9" || -e /dev/fd/9 ]]; then if [[ "$#" != 3 || "$1" != flock || "$2" != -n || "$3" != 9 ]]; then printf '%s\n' 'fd9-timeout-child-leak' >>"${REDIS_LAB_FAKE_VIOLATIONS}" exit 118 fi fi if [[ "${REDIS_LAB_FAKE_TIMEOUT_COMMAND:-}" == 'multipass-launch' && "${1:-}" == multipass && "${2:-}" == launch ]]; then if [[ -n "${REDIS_LAB_FAKE_LATE_CREATE_MODE:-}" ]]; then late_name='' late_cloud_init='' shift 2 while (($#)); do case "$1" in --name) late_name="${2:-}" shift 2 ;; --cloud-init) late_cloud_init="${2:-}" shift 2 ;; --cpus | --memory | --disk) shift 2 ;; *) shift ;; esac done late_marker="$(awk -v target="${late_name}" ' $0 ~ "^ run-[0-9]+-[0-9]+-[0-9]+\\|" target "$" { sub(/^ /, "") print found = 1 exit } END { if (!found) { exit 1 } } ' "${late_cloud_init}")" printf '%s|%s\n' "${late_name}" "${late_marker}" \ >"${REDIS_LAB_FAKE_LATE_CREATE}" fi exit 124 fi exec "$@" FAKE_TIMEOUT cat >"${FAKE_BIN}/sleep" <<'FAKE_SLEEP' #!/usr/bin/env bash set -Eeuo pipefail printf 'sleep' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" exit 0 FAKE_SLEEP cat >"${FAKE_BIN}/mv" <<'FAKE_MV' #!/usr/bin/env bash set -Eeuo pipefail source_path="${@: -2:1}" destination="${@: -1}" if [[ "${destination}" == "${REDIS_LAB_EXPECTED_STATE}" && -f "${source_path}" ]]; then if [[ "${REDIS_LAB_FAIL_STATE_RESERVATION:-}" == 1 ]] && grep -Fq 'PENDING|' "${source_path}"; then exit 51 fi if [[ "${REDIS_LAB_FAIL_STATE_PROMOTION:-}" == 1 ]] && grep -Fq 'CREATED|' "${source_path}"; then exit 52 fi fi if [[ "${REDIS_LAB_FAIL_KUBECONFIG_MV:-}" == 1 && "${source_path}" == "${REDIS_LAB_EXPECTED_KUBECONFIG}.next" && "${destination}" == "${REDIS_LAB_EXPECTED_KUBECONFIG}" ]]; then exit 54 fi exec /usr/bin/mv "$@" FAKE_MV cat >"${FAKE_BIN}/chmod" <<'FAKE_CHMOD' #!/usr/bin/env bash set -Eeuo pipefail target="${@: -1}" if [[ -n "${REDIS_LAB_FAKE_DIRECT_TOOL_BACKGROUND_PID_FILE:-}" && "${target}" == */redis-lab/observations && ! -e "${REDIS_LAB_FAKE_DIRECT_TOOL_BACKGROUND_PID_FILE}" ]]; then /bin/sleep 3 >/dev/null 2>&1 & printf '%s\n' "$!" >"${REDIS_LAB_FAKE_DIRECT_TOOL_BACKGROUND_PID_FILE}" fi if [[ "${REDIS_LAB_FAIL_STATE_CHMOD:-}" == 1 && "${target}" == "${REDIS_LAB_EXPECTED_STATE}.next" && -f "${target}" ]] && grep -Fq 'CREATED|' "${target}"; then exit 53 fi if [[ "${REDIS_LAB_FAIL_KUBECONFIG_CHMOD:-}" == 1 && "${target}" == "${REDIS_LAB_EXPECTED_KUBECONFIG}.next" ]]; then exit 55 fi exec /usr/bin/chmod "$@" FAKE_CHMOD cat >"${FAKE_BIN}/openssl" <<'FAKE_OPENSSL' #!/usr/bin/env bash set -Eeuo pipefail printf 'openssl' >>"${REDIS_LAB_FAKE_LOG}" printf ' <%s>' "$@" >>"${REDIS_LAB_FAKE_LOG}" printf '\n' >>"${REDIS_LAB_FAKE_LOG}" [[ "$*" == 'rand -hex 32' ]] || exit 60 printf '%s\n' '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' FAKE_OPENSSL cat >"${FAKE_BIN}/scenario-success" <<'FAKE_SUCCESS' #!/usr/bin/env bash if [[ -n "${REDIS_LAB_FAKE_SCENARIO_MARKER:-}" ]]; then : >"${REDIS_LAB_FAKE_SCENARIO_MARKER}" fi exit 0 FAKE_SUCCESS cat >"${FAKE_BIN}/scenario-failure" <<'FAKE_FAILURE' #!/usr/bin/env bash exit 23 FAKE_FAILURE cat >"${FAKE_BIN}/scenario-background-child" <<'FAKE_BACKGROUND' #!/usr/bin/env bash set -Eeuo pipefail /bin/sleep 3 & printf '%s\n' "$!" >"${REDIS_LAB_FAKE_BACKGROUND_PID_FILE}" exit 0 FAKE_BACKGROUND find "${FAKE_BIN}" -maxdepth 1 -type f -exec chmod 0700 -- {} + [[ -x "${LAB_SCRIPT}" ]] || fail "lifecycle script is missing or not executable" assert_fd9_static_contract "${LAB_SCRIPT}" SEALED_STRENGTH_VIOLATIONS="${FIXTURE_ROOT}/sealed-strength.violations" SEALED_STRENGTH_SCRIPT="${FIXTURE_ROOT}/unsafe-fixture-lifecycle" : >"${SEALED_STRENGTH_VIOLATIONS}" cat >"${SEALED_STRENGTH_SCRIPT}" <<'SEALED_STRENGTH' #!/usr/bin/env bash set -Eeuo pipefail docker version SEALED_STRENGTH chmod 0700 -- "${SEALED_STRENGTH_SCRIPT}" assert_fails env \ PATH="${FAKE_BIN}" \ REDIS_LAB_FAKE_VIOLATIONS="${SEALED_STRENGTH_VIOLATIONS}" \ "${SEALED_STRENGTH_SCRIPT}" assert_file_contains "${SEALED_STRENGTH_VIOLATIONS}" 'sealed-path-denied-command' BAD_VERSION_REPOSITORY="${FIXTURE_ROOT}/bad-version-repository" prepare_tracked_input_repository "${BAD_VERSION_REPOSITORY}" printf '%s\n' 'K3S_VERSION=not-pinned' 'MULTIPASS_IMAGE=24.04' \ >"${BAD_VERSION_REPOSITORY}/infra/redis-lab/versions.env" reset_case assert_fails run_lab "${BAD_VERSION_REPOSITORY}/infra/redis-lab/bin/redis-lab" preflight assert_count 0 '^(multipass|kubectl|ip|sha256sum|openssl|timeout) ' "${FAKE_LOG}" SYMLINK_CLOUD_INIT_REPOSITORY="${FIXTURE_ROOT}/symlink-cloud-init-repository" SYMLINK_CLOUD_INIT_ESCAPE="${FIXTURE_ROOT}/symlink-cloud-init-escape" prepare_tracked_input_repository "${SYMLINK_CLOUD_INIT_REPOSITORY}" rm -f -- "${SYMLINK_CLOUD_INIT_REPOSITORY}/infra/redis-lab/cloud-init/node.yaml" printf '%s\n' \ '#cloud-config' \ 'write_files:' \ 'runcmd:' \ ' - [ sh, -c, "printf injected-bootstrap > /var/lib/redis-lab-canary" ]' \ >"${SYMLINK_CLOUD_INIT_ESCAPE}" ln -s -- "${SYMLINK_CLOUD_INIT_ESCAPE}" \ "${SYMLINK_CLOUD_INIT_REPOSITORY}/infra/redis-lab/cloud-init/node.yaml" reset_case CLOUD_INIT_SYMLINK_ACCEPTED=0 SYMLINK_CLOUD_INIT_RUNTIME_ROOT="${SYMLINK_CLOUD_INIT_REPOSITORY}/src/build/redis-lab" if RUN_LAB_EXPECTED_STATE="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/run.state" \ RUN_LAB_EXPECTED_KUBECONFIG="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/kubeconfig" \ RUN_LAB_EXPECTED_HOST_KUBECONFIG="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/observations/host-kubeconfig" \ RUN_LAB_EXPECTED_CLOUD_INIT_ROOT="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/cloud-init" \ run_lab \ "${SYMLINK_CLOUD_INIT_REPOSITORY}/infra/redis-lab/bin/redis-lab" up; then CLOUD_INIT_SYMLINK_ACCEPTED=1 RUN_LAB_EXPECTED_STATE="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/run.state" \ RUN_LAB_EXPECTED_KUBECONFIG="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/kubeconfig" \ RUN_LAB_EXPECTED_HOST_KUBECONFIG="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/observations/host-kubeconfig" \ RUN_LAB_EXPECTED_CLOUD_INIT_ROOT="${SYMLINK_CLOUD_INIT_RUNTIME_ROOT}/cloud-init" \ run_lab \ "${SYMLINK_CLOUD_INIT_REPOSITORY}/infra/redis-lab/bin/redis-lab" down fi CLOUD_INIT_PREVALIDATION_CHILD_COUNT="$( grep -Ec \ '^(child|multipass|kubectl|ip|sha256sum|openssl|timeout|curl|uname) ' \ "${FAKE_LOG}" || true )" SYMLINK_VERSION_REPOSITORY="${FIXTURE_ROOT}/symlink-version-repository" SYMLINK_VERSION_ESCAPE="${FIXTURE_ROOT}/symlink-version-escape" SYMLINK_VERSION_CANARY="${FIXTURE_ROOT}/symlink-version-source.executed" prepare_tracked_input_repository "${SYMLINK_VERSION_REPOSITORY}" rm -f -- \ "${SYMLINK_VERSION_REPOSITORY}/infra/redis-lab/versions.env" \ "${SYMLINK_VERSION_CANARY}" printf '%s\n' \ 'K3S_VERSION=v1.33.3+k3s1' \ 'MULTIPASS_IMAGE=24.04' \ 'K3S_AMD64_URL=https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' \ 'K3S_AMD64_SHA256=f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' \ "printf '%s\\n' 'versions-source-executed' >'${SYMLINK_VERSION_CANARY}'" \ 'return 97' \ >"${SYMLINK_VERSION_ESCAPE}" ln -s -- "${SYMLINK_VERSION_ESCAPE}" \ "${SYMLINK_VERSION_REPOSITORY}/infra/redis-lab/versions.env" reset_case assert_fails run_lab \ "${SYMLINK_VERSION_REPOSITORY}/infra/redis-lab/bin/redis-lab" preflight VERSION_SYMLINK_EXECUTED=0 [[ ! -e "${SYMLINK_VERSION_CANARY}" ]] || VERSION_SYMLINK_EXECUTED=1 VERSION_PREVALIDATION_CHILD_COUNT="$( grep -Ec \ '^(child|multipass|kubectl|ip|sha256sum|openssl|timeout|curl|uname) ' \ "${FAKE_LOG}" || true )" if ((CLOUD_INIT_SYMLINK_ACCEPTED == 1)); then printf '%s\n' \ 'redis-lab-contract: symlinked cloud-init was accepted for VM bootstrap' >&2 fi if ((VERSION_SYMLINK_EXECUTED == 1)); then printf '%s\n' \ 'redis-lab-contract: symlinked versions file executed host shell' >&2 fi if ((CLOUD_INIT_SYMLINK_ACCEPTED == 1 || VERSION_SYMLINK_EXECUTED == 1)) || [[ "${CLOUD_INIT_PREVALIDATION_CHILD_COUNT}" != 0 || "${VERSION_PREVALIDATION_CHILD_COUNT}" != 0 ]]; then fail 'tracked input symlink validation was not fail-closed' fi INVALID_VERSIONS_REPOSITORY="${FIXTURE_ROOT}/invalid-versions-repository" for versions_variant in unknown duplicate missing malformed; do prepare_tracked_input_repository "${INVALID_VERSIONS_REPOSITORY}" case "${versions_variant}" in unknown) printf '%s\n' \ 'K3S_VERSION=v1.33.3+k3s1' \ 'MULTIPASS_IMAGE=24.04' \ 'K3S_AMD64_URL=https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' \ 'K3S_AMD64_SHA256=f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' \ 'REDIS_LAB_UNKNOWN=value' \ >"${INVALID_VERSIONS_REPOSITORY}/infra/redis-lab/versions.env" ;; duplicate) printf '%s\n' \ 'K3S_VERSION=v1.33.3+k3s1' \ 'K3S_VERSION=v1.33.3+k3s1' \ 'MULTIPASS_IMAGE=24.04' \ 'K3S_AMD64_URL=https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' \ 'K3S_AMD64_SHA256=f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' \ >"${INVALID_VERSIONS_REPOSITORY}/infra/redis-lab/versions.env" ;; missing) printf '%s\n' \ 'K3S_VERSION=v1.33.3+k3s1' \ 'MULTIPASS_IMAGE=24.04' \ 'K3S_AMD64_URL=https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' \ >"${INVALID_VERSIONS_REPOSITORY}/infra/redis-lab/versions.env" ;; malformed) printf '%s\n' \ 'K3S_VERSION =v1.33.3+k3s1' \ 'MULTIPASS_IMAGE=24.04' \ 'K3S_AMD64_URL=https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' \ 'K3S_AMD64_SHA256=f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' \ >"${INVALID_VERSIONS_REPOSITORY}/infra/redis-lab/versions.env" ;; esac reset_case assert_fails run_lab \ "${INVALID_VERSIONS_REPOSITORY}/infra/redis-lab/bin/redis-lab" preflight assert_count 0 \ '^(multipass|kubectl|ip|sha256sum|openssl|timeout|curl|uname) ' \ "${FAKE_LOG}" [[ ! -e "${INVALID_VERSIONS_REPOSITORY}/src/build/redis-lab" ]] || fail "${versions_variant} versions input published runtime state" done MISSING_RENDERER_REPOSITORY="${FIXTURE_ROOT}/missing-renderer-repository" rm -rf -- "${MISSING_RENDERER_REPOSITORY}" mkdir -p -- \ "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/bin" \ "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/cloud-init" \ "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/lib" \ "${MISSING_RENDERER_REPOSITORY}/src" cp -- "${LAB_SCRIPT}" "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/bin/redis-lab" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" \ "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/cloud-init/node.yaml" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/versions.env" \ "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/versions.env" chmod 0700 -- "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/bin/redis-lab" reset_case assert_fails run_lab \ "${MISSING_RENDERER_REPOSITORY}/infra/redis-lab/bin/redis-lab" preflight assert_count 0 \ '^(multipass|kubectl|ip|sha256sum|openssl|timeout|curl|uname) ' \ "${FAKE_LOG}" [[ ! -e "${MISSING_RENDERER_REPOSITORY}/src/build/redis-lab" ]] || fail 'missing renderer published runtime state' SYMLINK_RENDERER_REPOSITORY="${FIXTURE_ROOT}/symlink-renderer-repository" SYMLINK_RENDERER_ESCAPE="${FIXTURE_ROOT}/symlink-renderer-escape" rm -rf -- "${SYMLINK_RENDERER_REPOSITORY}" mkdir -p -- \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/bin" \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/cloud-init" \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/lib" \ "${SYMLINK_RENDERER_REPOSITORY}/src" cp -- "${LAB_SCRIPT}" "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/bin/redis-lab" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/cloud-init/node.yaml" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/versions.env" \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/versions.env" printf '%s\n' 'renderer-canary' >"${SYMLINK_RENDERER_ESCAPE}" ln -s -- "${SYMLINK_RENDERER_ESCAPE}" \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/lib/render-kubeconfig.awk" chmod 0700 -- "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/bin/redis-lab" reset_case assert_fails run_lab \ "${SYMLINK_RENDERER_REPOSITORY}/infra/redis-lab/bin/redis-lab" preflight assert_count 0 \ '^(multipass|kubectl|ip|sha256sum|openssl|timeout|curl|uname) ' \ "${FAKE_LOG}" assert_file_contains "${SYMLINK_RENDERER_ESCAPE}" 'renderer-canary' [[ ! -e "${SYMLINK_RENDERER_REPOSITORY}/src/build/redis-lab" ]] || fail 'symlink renderer published runtime state' SYMLINK_REPOSITORY="${FIXTURE_ROOT}/symlink-repository" SYMLINK_ESCAPE="${FIXTURE_ROOT}/symlink-escape" rm -rf -- "${SYMLINK_REPOSITORY}" "${SYMLINK_ESCAPE}" mkdir -p -- \ "${SYMLINK_REPOSITORY}/infra/redis-lab/bin" \ "${SYMLINK_REPOSITORY}/infra/redis-lab/cloud-init" \ "${SYMLINK_REPOSITORY}/infra/redis-lab/lib" \ "${SYMLINK_REPOSITORY}/src" \ "${SYMLINK_ESCAPE}" cp -- "${LAB_SCRIPT}" "${SYMLINK_REPOSITORY}/infra/redis-lab/bin/redis-lab" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" \ "${SYMLINK_REPOSITORY}/infra/redis-lab/cloud-init/node.yaml" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk" \ "${SYMLINK_REPOSITORY}/infra/redis-lab/lib/render-kubeconfig.awk" cp -- "${SOURCE_REPOSITORY_ROOT}/infra/redis-lab/versions.env" \ "${SYMLINK_REPOSITORY}/infra/redis-lab/versions.env" ln -s -- "${SYMLINK_ESCAPE}" "${SYMLINK_REPOSITORY}/src/build" chmod 0700 -- "${SYMLINK_REPOSITORY}/infra/redis-lab/bin/redis-lab" reset_case assert_fails run_lab "${SYMLINK_REPOSITORY}/infra/redis-lab/bin/redis-lab" preflight assert_count 0 '^(multipass|kubectl|ip|sha256sum|openssl|timeout) ' "${FAKE_LOG}" [[ -z "$(find "${SYMLINK_ESCAPE}" -mindepth 1 -print -quit)" ]] || fail 'runtime path validation wrote through an ancestor symlink' reset_case CHILD_ESCAPE="${FIXTURE_ROOT}/child-symlink-escape" printf '%s\n' 'child-canary' >"${CHILD_ESCAPE}" mkdir -p -- "${RUNTIME_ROOT}" ln -s -- "${CHILD_ESCAPE}" "${EXPECTED_STATE}" assert_fails run_lab "${LAB_SCRIPT}" preflight assert_count 0 '^(multipass|kubectl|ip|sha256sum|openssl|timeout) ' "${FAKE_LOG}" assert_file_contains "${CHILD_ESCAPE}" 'child-canary' reset_case KUBECONFIG_NEXT_ESCAPE="${FIXTURE_ROOT}/kubeconfig-next-child-escape" printf '%s\n' 'kubeconfig-next-canary' >"${KUBECONFIG_NEXT_ESCAPE}" mkdir -p -- "${RUNTIME_ROOT}" ln -s -- "${KUBECONFIG_NEXT_ESCAPE}" "${EXPECTED_KUBECONFIG}.next" assert_fails run_lab "${LAB_SCRIPT}" preflight assert_count 0 \ '^(multipass|kubectl|ip|sha256sum|openssl|timeout|curl|uname) ' \ "${FAKE_LOG}" assert_file_contains "${KUBECONFIG_NEXT_ESCAPE}" 'kubeconfig-next-canary' [[ ! -e "${EXPECTED_KUBECONFIG}" ]] || fail 'kubeconfig next symlink published a destination' rm -f -- "${EXPECTED_KUBECONFIG}.next" reset_case run_lab "${LAB_SCRIPT}" preflight OBSERVATION_ESCAPE="${FIXTURE_ROOT}/observation-child-escape" printf '%s\n' 'observation-canary' >"${OBSERVATION_ESCAPE}" ln -s -- "${OBSERVATION_ESCAPE}" \ "${RUNTIME_ROOT}/observations/fingerprint-after.raw" assert_fails run_lab "${LAB_SCRIPT}" postflight assert_file_contains "${OBSERVATION_ESCAPE}" 'observation-canary' rm -f -- "${RUNTIME_ROOT}/observations/fingerprint-after.raw" run_lab "${LAB_SCRIPT}" down reset_case run_lab "${LAB_SCRIPT}" preflight RENDERED_CLOUD_INIT_ESCAPE="${FIXTURE_ROOT}/rendered-cloud-init-escape" printf '%s\n' 'rendered-cloud-init-canary' >"${RENDERED_CLOUD_INIT_ESCAPE}" mkdir -p -- "${EXPECTED_CLOUD_INIT_ROOT}" ln -s -- "${RENDERED_CLOUD_INIT_ESCAPE}" \ "${EXPECTED_CLOUD_INIT_ROOT}/ca-redis-lab-server.yaml" : >"${FAKE_LOG}" assert_fails run_lab "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" assert_file_contains "${RENDERED_CLOUD_INIT_ESCAPE}" 'rendered-cloud-init-canary' rm -f -- "${EXPECTED_CLOUD_INIT_ROOT}/ca-redis-lab-server.yaml" run_lab "${LAB_SCRIPT}" down reset_case CONCURRENT_BARRIER="${FIXTURE_ROOT}/concurrent-barrier" rm -f -- "${CONCURRENT_BARRIER}.started" "${CONCURRENT_BARRIER}.release" run_lab env REDIS_LAB_FAKE_LAUNCH_BARRIER="${CONCURRENT_BARRIER}" \ "${LAB_SCRIPT}" up >"${FIXTURE_ROOT}/concurrent-first.out" 2>&1 & CONCURRENT_PID=$! for barrier_attempt in {1..100}; do [[ -f "${CONCURRENT_BARRIER}.started" ]] && break /bin/sleep 0.02 done [[ -f "${CONCURRENT_BARRIER}.started" ]] || fail 'concurrent launch barrier was not reached' cp -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/concurrent-state.before" if run_lab "${LAB_SCRIPT}" preflight; then : >"${CONCURRENT_BARRIER}.release" wait "${CONCURRENT_PID}" || true fail 'concurrent lifecycle invocation acquired an active run' fi cmp -s -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/concurrent-state.before" || fail 'concurrent lifecycle invocation changed active run state' : >"${CONCURRENT_BARRIER}.release" wait "${CONCURRENT_PID}" || fail 'first concurrent lifecycle did not finish' run_lab "${LAB_SCRIPT}" down reset_case DIRECT_TOOL_BACKGROUND_PID_FILE="${FIXTURE_ROOT}/direct-tool-background-child.pid" rm -f -- "${DIRECT_TOOL_BACKGROUND_PID_FILE}" run_lab env \ REDIS_LAB_FAKE_DIRECT_TOOL_BACKGROUND_PID_FILE="${DIRECT_TOOL_BACKGROUND_PID_FILE}" \ "${LAB_SCRIPT}" preflight [[ -s "${DIRECT_TOOL_BACKGROUND_PID_FILE}" ]] || fail 'direct-tool background child pid was not recorded' DIRECT_TOOL_BACKGROUND_PID="$(cat "${DIRECT_TOOL_BACKGROUND_PID_FILE}")" if ! run_lab "${LAB_SCRIPT}" preflight; then kill "${DIRECT_TOOL_BACKGROUND_PID}" 2>/dev/null || true fail 'detached direct-tool child inherited the lifecycle lock' fi kill "${DIRECT_TOOL_BACKGROUND_PID}" 2>/dev/null || true run_lab "${LAB_SCRIPT}" down reset_case INFRA_BACKGROUND_PID_FILE="${FIXTURE_ROOT}/infra-background-child.pid" rm -f -- "${INFRA_BACKGROUND_PID_FILE}" run_lab env REDIS_LAB_FAKE_INFRA_BACKGROUND_PID_FILE="${INFRA_BACKGROUND_PID_FILE}" \ "${LAB_SCRIPT}" preflight [[ -s "${INFRA_BACKGROUND_PID_FILE}" ]] || fail 'infra background child pid was not recorded' INFRA_BACKGROUND_PID="$(cat "${INFRA_BACKGROUND_PID_FILE}")" if ! run_lab "${LAB_SCRIPT}" preflight; then kill "${INFRA_BACKGROUND_PID}" 2>/dev/null || true fail 'detached infra child inherited the lifecycle lock' fi kill "${INFRA_BACKGROUND_PID}" 2>/dev/null || true run_lab "${LAB_SCRIPT}" down reset_case SIGNAL_BARRIER="${FIXTURE_ROOT}/signal-barrier" rm -f -- "${SIGNAL_BARRIER}.started" "${SIGNAL_BARRIER}.release" printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env \ REDIS_LAB_FAKE_LAUNCH_SIGNAL_PARENT=1 \ REDIS_LAB_FAKE_LAUNCH_BARRIER="${SIGNAL_BARRIER}" \ "${LAB_SCRIPT}" up [[ -f "${SIGNAL_BARRIER}.started" ]] || fail 'signal launch barrier was not reached' assert_count 1 '^multipass <--purge> $' "${FAKE_LOG}" assert_count 1 '^unrelated-instance$' "${FAKE_INVENTORY}" reset_case RUN_SIGNAL_BARRIER="${FIXTURE_ROOT}/run-signal-barrier" rm -f -- "${RUN_SIGNAL_BARRIER}.started" "${RUN_SIGNAL_BARRIER}.release" printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env \ REDIS_LAB_FAKE_LAUNCH_SIGNAL_PARENT=1 \ REDIS_LAB_FAKE_LAUNCH_BARRIER="${RUN_SIGNAL_BARRIER}" \ "${LAB_SCRIPT}" run -- scenario-success [[ -f "${RUN_SIGNAL_BARRIER}.started" ]] || fail 'run-flow launch barrier was not reached' assert_count 1 '^multipass <--purge> $' "${FAKE_LOG}" assert_count 1 '^unrelated-instance$' "${FAKE_INVENTORY}" reset_case RUN_HANDOFF_SCENARIO_MARKER="${FIXTURE_ROOT}/run-handoff-scenario.started" rm -f -- "${RUN_HANDOFF_SCENARIO_MARKER}" printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env \ REDIS_LAB_FAKE_RUN_HANDOFF_SIGNAL=1 \ REDIS_LAB_FAKE_SCENARIO_MARKER="${RUN_HANDOFF_SCENARIO_MARKER}" \ "${LAB_SCRIPT}" run -- scenario-success [[ -f "${EXPECTED_RUN_HANDOFF_MARKER}" ]] || fail 'run-flow post-up handoff seam was not reached' [[ ! -e "${RUN_HANDOFF_SCENARIO_MARKER}" ]] || fail 'run-flow executed the user command after handoff TERM' assert_count 3 '^multipass <--purge> "${FAKE_INVENTORY}" printf '%s\n' \ 'ca-redis-lab-server|run-701-702-703|ca-redis-lab-server' >"${FAKE_OWNERSHIP}" printf '%s\n' \ 'RUN|run-701-702-703' \ 'CREATED|run-701-702-703|ca-redis-lab-server' >"${EXPECTED_STATE}" cp -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/prior-created-state.before" assert_fails run_lab "${LAB_SCRIPT}" run -- scenario-success assert_count 0 '^multipass ' "${FAKE_LOG}" cmp -s -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/prior-created-state.before" || fail 'rejected run changed a prior CREATED state' assert_count 1 '^ca-redis-lab-server$' "${FAKE_INVENTORY}" assert_count 1 '^unrelated-instance$' "${FAKE_INVENTORY}" reset_case printf '%s\n' \ 'ca-redis-lab-server' \ 'unrelated-instance' >"${FAKE_INVENTORY}" printf '%s\n' \ 'ca-redis-lab-server|run-704-705-706|ca-redis-lab-server' >"${FAKE_OWNERSHIP}" printf '%s\n' \ 'RUN|run-704-705-706' \ 'RECONCILE|run-704-705-706|ca-redis-lab-server' >"${EXPECTED_STATE}" cp -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/prior-reconcile-state.before" assert_fails run_lab "${LAB_SCRIPT}" run -- scenario-success assert_count 0 '^multipass ' "${FAKE_LOG}" cmp -s -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/prior-reconcile-state.before" || fail 'rejected run changed a prior RECONCILE state' assert_count 1 '^ca-redis-lab-server$' "${FAKE_INVENTORY}" assert_count 1 '^unrelated-instance$' "${FAKE_INVENTORY}" reset_case assert_fails run_lab env REDIS_LAB_FAKE_LAUNCH_MARKER_MODE=missing "${LAB_SCRIPT}" up assert_count 1 '^multipass .*' "${FAKE_LOG}" assert_count 0 '^multipass .*' "${FAKE_LOG}" assert_count 1 '^RECONCILE\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-server$' \ "${EXPECTED_STATE}" assert_file_contains "${FAKE_INVENTORY}" 'ca-redis-lab-server' reset_case assert_fails run_lab env REDIS_LAB_FAKE_LAUNCH_MARKER_MODE=foreign "${LAB_SCRIPT}" up assert_count 1 '^multipass .*' "${FAKE_LOG}" assert_count 0 '^multipass .*' "${FAKE_LOG}" assert_count 1 '^RECONCILE\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-server$' \ "${EXPECTED_STATE}" assert_file_contains "${FAKE_INVENTORY}" 'ca-redis-lab-server' reset_case run_lab "${LAB_SCRIPT}" up assert_count 3 '^multipass ' "${FAKE_LOG}" assert_file_contains "${FAKE_LOG}" \ "multipass <--name> <--cpus> <2> <--memory> <3G> <--disk> <12G> <--cloud-init> <${EXPECTED_CLOUD_INIT_ROOT}/ca-redis-lab-server.yaml> <24.04>" assert_file_contains "${FAKE_LOG}" \ "multipass <--name> <--cpus> <2> <--memory> <2560M> <--disk> <12G> <--cloud-init> <${EXPECTED_CLOUD_INIT_ROOT}/ca-redis-lab-agent-1.yaml> <24.04>" assert_file_contains "${FAKE_LOG}" \ "multipass <--name> <--cpus> <2> <--memory> <2560M> <--disk> <12G> <--cloud-init> <${EXPECTED_CLOUD_INIT_ROOT}/ca-redis-lab-agent-2.yaml> <24.04>" assert_file_contains "${FAKE_LOG}" \ 'curl <--fail> <--location> <--silent> <--show-error> <--output> <'"${RUNTIME_ROOT}"'/k3s-amd64> ' assert_count 3 '^multipass ' "${FAKE_LOG}" assert_count 1 '^multipass .* <--cluster-cidr=10\.52\.0\.0/16> <--service-cidr=10\.53\.0\.0/16> <--disable=traefik> <--disable=servicelb> <--write-kubeconfig-mode=0600>$' \ "${FAKE_LOG}" assert_count 2 '^multipass .* $' \ "${FAKE_LOG}" assert_count 1 "^kubectl <--kubeconfig> <${EXPECTED_KUBECONFIG}> <--context> " \ "${FAKE_LOG}" assert_count 5 "^kubectl <--kubeconfig> <${EXPECTED_HOST_KUBECONFIG}> <--context> " \ "${FAKE_LOG}" cmp -s -- "${TEST_HOME_ROOT}/.kube/config" "${DEFAULT_KUBECONFIG_SNAPSHOT}" || fail 'default kubeconfig was mutated' [[ ! -e "${EXPECTED_HOST_KUBECONFIG}" ]] || fail 'run-scoped host kubeconfig was retained' assert_file_contains "${RUNTIME_ROOT}/fingerprint.before" 'lab-resource-count|0' assert_count 3 '^CREATED\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-' "${EXPECTED_STATE}" SUCCESS_RUN_ID="$(awk -F'|' '$1 == "RUN" {print $2}' "${EXPECTED_STATE}")" for expected_name in \ ca-redis-lab-server \ ca-redis-lab-agent-1 \ ca-redis-lab-agent-2; do [[ "$(/usr/bin/stat -c '%a' "${EXPECTED_CLOUD_INIT_ROOT}/${expected_name}.yaml")" == 600 ]] || fail 'rendered cloud-init permissions are not 0600' assert_file_contains "${EXPECTED_CLOUD_INIT_ROOT}/${expected_name}.yaml" \ " ${SUCCESS_RUN_ID}|${expected_name}" if grep -Fq '0123456789abcdef0123456789abcdef' \ "${EXPECTED_CLOUD_INIT_ROOT}/${expected_name}.yaml"; then fail 'rendered ownership marker contains secret material' fi done assert_file_contains "${EXPECTED_KUBECONFIG}" ' server: https://192.0.2.10:6443' assert_count 2 '^ name: ca-redis-lab$' "${EXPECTED_KUBECONFIG}" assert_file_contains "${EXPECTED_KUBECONFIG}" ' cluster: ca-redis-lab' assert_file_contains "${EXPECTED_KUBECONFIG}" ' user: ca-redis-lab' assert_file_contains "${EXPECTED_KUBECONFIG}" 'current-context: ca-redis-lab' assert_file_contains "${EXPECTED_KUBECONFIG}" '- name: ca-redis-lab' assert_file_contains "${EXPECTED_KUBECONFIG}" ' namespace: team-default' assert_file_contains "${EXPECTED_KUBECONFIG}" \ ' certificate-authority-data: preserve-default-ca-canary' assert_file_contains "${EXPECTED_KUBECONFIG}" \ ' client-certificate-data: preserve-default-client-cert-canary' assert_file_contains "${EXPECTED_KUBECONFIG}" \ ' client-key-data: preserve-default-client-key-canary' cmp -s -- "${EXPECTED_KUBECONFIG}" "${KUBECONFIG_WITH_NAMESPACE_GOLDEN}" || fail 'namespace-present kubeconfig output changed' [[ "$(/usr/bin/stat -c '%a' "${EXPECTED_KUBECONFIG}")" == 600 ]] || fail 'lab kubeconfig permissions are not 0600' [[ ! -s "${FAKE_VIOLATIONS}" ]] || fail 'exact launch/state contract violation' run_lab "${LAB_SCRIPT}" down assert_count 3 '^multipass <--purge> <--> $' \ "${FAKE_LOG}" [[ ! -s "${FAKE_INVENTORY}" ]] || fail 'down left lab instances' assert_created_cleanup_reconciles() { local mode="$1" local created_run_id local expected_server_delete_count=0 reset_case run_lab "${LAB_SCRIPT}" up created_run_id="$(awk -F'|' '$1 == "RUN" {print $2}' "${EXPECTED_STATE}")" case "${mode}" in missing) printf '%s\n' \ "ca-redis-lab-agent-1|${created_run_id}|ca-redis-lab-agent-1" \ "ca-redis-lab-agent-2|${created_run_id}|ca-redis-lab-agent-2" \ >"${FAKE_OWNERSHIP}" ;; foreign) printf '%s\n' \ 'ca-redis-lab-server|run-999-999-999|ca-redis-lab-server' \ "ca-redis-lab-agent-1|${created_run_id}|ca-redis-lab-agent-1" \ "ca-redis-lab-agent-2|${created_run_id}|ca-redis-lab-agent-2" \ >"${FAKE_OWNERSHIP}" ;; delete-failure) expected_server_delete_count=1 ;; *) fail 'invalid CREATED cleanup test mode' ;; esac : >"${FAKE_LOG}" if [[ "${mode}" == delete-failure ]]; then assert_fails run_lab env \ REDIS_LAB_FAKE_DELETE_FAILURE_NAME=ca-redis-lab-server \ "${LAB_SCRIPT}" down else assert_fails run_lab "${LAB_SCRIPT}" down fi assert_count "${expected_server_delete_count}" \ '^multipass <--purge> $' "${FAKE_LOG}" assert_count 2 '^multipass <--purge> "${FAKE_OWNERSHIP}" run_lab "${LAB_SCRIPT}" down } assert_created_cleanup_reconciles missing assert_created_cleanup_reconciles foreign assert_created_cleanup_reconciles delete-failure reset_case printf '%s\n' 'ca-redis-lab-agent-1' >"${FAKE_INVENTORY}" assert_fails run_lab "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" assert_file_contains "${FAKE_INVENTORY}" 'ca-redis-lab-agent-1' reset_case assert_fails env REDIS_LAB_FAIL_LAUNCH_NAME='ca-redis-lab-agent-2' \ PATH="${FAKE_BIN}" \ REDIS_LAB_CONTRACT_TEST=1 \ REDIS_LAB_HOST_SERVICE_CIDRS='10.81.0.0/16' \ REDIS_LAB_HOME_DIR="${TEST_HOME_ROOT}" \ REDIS_LAB_FAKE_LOG="${FAKE_LOG}" \ REDIS_LAB_FAKE_INVENTORY="${FAKE_INVENTORY}" \ REDIS_LAB_FAKE_VIOLATIONS="${FAKE_VIOLATIONS}" \ REDIS_LAB_EXPECTED_STATE="${EXPECTED_STATE}" \ REDIS_LAB_EXPECTED_KUBECONFIG="${EXPECTED_KUBECONFIG}" \ REDIS_LAB_EXPECTED_HOST_KUBECONFIG="${EXPECTED_HOST_KUBECONFIG}" \ "${LAB_SCRIPT}" up assert_count 3 '^multipass ' "${FAKE_LOG}" assert_file_contains "${FAKE_LOG}" \ 'multipass <--purge> ' assert_file_contains "${FAKE_LOG}" \ 'multipass <--purge> ' assert_count 0 '^multipass <--purge> $' "${FAKE_LOG}" assert_count 1 '^RECONCILE\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-agent-2$' \ "${EXPECTED_STATE}" [[ ! -s "${FAKE_INVENTORY}" ]] || fail 'partial failure cleanup was incomplete' [[ ! -s "${FAKE_VIOLATIONS}" ]] || fail 'immediate state recording failed' reset_case printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env REDIS_LAB_FAIL_STATE_RESERVATION=1 "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" assert_file_contains "${FAKE_INVENTORY}" 'unrelated-instance' reset_case printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env \ REDIS_LAB_FAKE_TIMEOUT_COMMAND=multipass-launch \ REDIS_LAB_FAKE_LATE_CREATE_MODE=matching \ "${LAB_SCRIPT}" up assert_count 1 '^timeout .* .*' "${FAKE_LOG}" assert_count 1 '^multipass <--purge> $' "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" assert_file_contains "${FAKE_INVENTORY}" 'unrelated-instance' [[ ! -e "${EXPECTED_STATE}" ]] || fail 'matching late create retained run state' reset_case printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env \ REDIS_LAB_FAKE_TIMEOUT_COMMAND=multipass-launch \ REDIS_LAB_FAKE_LATE_CREATE_MODE=absent \ "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 1 '^RECONCILE\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-server$' \ "${EXPECTED_STATE}" cp -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/absent-reconcile-state.before" : >"${FAKE_LOG}" assert_fails run_lab "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" cmp -s -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/absent-reconcile-state.before" || fail 'new up overwrote absent reconciliation state' reset_case printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env \ REDIS_LAB_FAKE_TIMEOUT_COMMAND=multipass-launch \ REDIS_LAB_FAKE_LATE_CREATE_MODE=foreign \ "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 1 '^RECONCILE\|run-[0-9]+-[0-9]+-[0-9]+\|ca-redis-lab-server$' \ "${EXPECTED_STATE}" assert_file_contains "${FAKE_INVENTORY}" 'ca-redis-lab-server' cp -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/foreign-reconcile-state.before" : >"${FAKE_LOG}" assert_fails run_lab "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" cmp -s -- "${EXPECTED_STATE}" "${FIXTURE_ROOT}/foreign-reconcile-state.before" || fail 'new up overwrote foreign reconciliation state' reset_case printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" assert_fails run_lab env REDIS_LAB_FAIL_STATE_PROMOTION=1 "${LAB_SCRIPT}" up assert_count 1 '^multipass .*' "${FAKE_LOG}" assert_count 1 '^multipass <--purge> $' "${FAKE_LOG}" assert_count 0 '^multipass .*"${FAKE_INVENTORY}" assert_fails run_lab env REDIS_LAB_FAIL_STATE_CHMOD=1 "${LAB_SCRIPT}" up assert_count 1 '^multipass .*' "${FAKE_LOG}" assert_count 1 '^multipass <--purge> $' "${FAKE_LOG}" assert_count 0 '^multipass .*$' \ "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 3 '^multipass <--purge> ' "${FAKE_LOG}" assert_count 0 '^vm-install ' "${FAKE_LOG}" assert_count 3 '^multipass <--purge> <${EXPECTED_KUBECONFIG}> <--context> " \ "${FAKE_LOG}" for expected_name in \ ca-redis-lab-server \ ca-redis-lab-agent-1 \ ca-redis-lab-agent-2; do assert_count 2 \ "^multipass <${expected_name}> <--> $" \ "${FAKE_LOG}" assert_count 1 \ "^multipass <--purge> <${expected_name}>$" \ "${FAKE_LOG}" done [[ ! -e "${EXPECTED_KUBECONFIG}" ]] || fail "${label} retained the kubeconfig destination" [[ ! -e "${EXPECTED_KUBECONFIG}.next" ]] || fail "${label} retained the kubeconfig next file" [[ ! -e "${EXPECTED_STATE}" ]] || fail "${label} retained current-run state after exact cleanup" [[ ! -s "${FAKE_INVENTORY}" ]] || fail "${label} left a current-run instance" assert_file_not_contains_pattern "${FAKE_LOG}" '^multipass ' assert_file_not_contains_pattern "${FAKE_LOG}" '^multipass .*<--all>' assert_file_not_contains_pattern "${FAKE_LOG}" '^multipass .*<[?*]>' [[ ! -s "${FAKE_VIOLATIONS}" ]] || fail "${label} used a forbidden fake invocation" } KUBECONFIG_MUTATIONS=( missing-server duplicate-server reordered-kind-preferences unknown-cluster-key whitespace-before-colon quoted-key tagged-key explicit-key anchor alias merge flow-map flow-sequence tab-indentation crlf document-start document-end trailing-content sibling-flow-cluster sibling-flow-context ) for kubeconfig_mutation in "${KUBECONFIG_MUTATIONS[@]}"; do assert_kubeconfig_render_failure \ "kubeconfig mutation ${kubeconfig_mutation}" \ "REDIS_LAB_FAKE_KUBECONFIG_SCHEMA_VARIANT=${kubeconfig_mutation}" done assert_kubeconfig_render_failure \ 'kubeconfig chmod failure' \ REDIS_LAB_FAIL_KUBECONFIG_CHMOD=1 assert_kubeconfig_render_failure \ 'kubeconfig mv failure' \ REDIS_LAB_FAIL_KUBECONFIG_MV=1 reset_case assert_fails run_lab env REDIS_LAB_FAKE_BAD_SERVER_IP=1 "${LAB_SCRIPT}" up assert_count 3 '^multipass <--purge> <--purge> ' "${FAKE_LOG}" reset_case assert_fails run_lab env REDIS_LAB_HOST_SERVICE_CIDRS='10.53.8.0/24' "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" reset_case assert_fails run_lab env REDIS_LAB_HOST_SERVICE_CIDRS='10.81.1.2/16' "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" reset_case printf '%s\n' 'unrelated-instance' >"${FAKE_INVENTORY}" run_lab "${LAB_SCRIPT}" run -- scenario-success assert_count 3 '^multipass <--purge> <--purge> /dev/null || true fail 'user background child inherited the lifecycle lock' fi kill "${BACKGROUND_PID}" 2>/dev/null || true run_lab "${LAB_SCRIPT}" down reset_case assert_fails run_lab "${LAB_SCRIPT}" run --retain-on-failure -- scenario-failure assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 3 '^ca-redis-lab-' "${FAKE_INVENTORY}" run_lab "${LAB_SCRIPT}" down reset_case assert_fails env CI=true \ PATH="${FAKE_BIN}" \ REDIS_LAB_CONTRACT_TEST=1 \ REDIS_LAB_HOST_SERVICE_CIDRS='10.81.0.0/16' \ REDIS_LAB_HOME_DIR="${TEST_HOME_ROOT}" \ REDIS_LAB_FAKE_LOG="${FAKE_LOG}" \ REDIS_LAB_FAKE_INVENTORY="${FAKE_INVENTORY}" \ REDIS_LAB_FAKE_VIOLATIONS="${FAKE_VIOLATIONS}" \ REDIS_LAB_EXPECTED_STATE="${EXPECTED_STATE}" \ REDIS_LAB_EXPECTED_KUBECONFIG="${EXPECTED_KUBECONFIG}" \ REDIS_LAB_EXPECTED_HOST_KUBECONFIG="${EXPECTED_HOST_KUBECONFIG}" \ "${LAB_SCRIPT}" run --retain-on-failure -- scenario-failure assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" reset_case assert_fails env REDIS_LAB_FAKE_FINGERPRINT_MISMATCH=1 \ PATH="${FAKE_BIN}" \ REDIS_LAB_CONTRACT_TEST=1 \ REDIS_LAB_HOST_SERVICE_CIDRS='10.81.0.0/16' \ REDIS_LAB_HOME_DIR="${TEST_HOME_ROOT}" \ REDIS_LAB_FAKE_LOG="${FAKE_LOG}" \ REDIS_LAB_FAKE_INVENTORY="${FAKE_INVENTORY}" \ REDIS_LAB_FAKE_VIOLATIONS="${FAKE_VIOLATIONS}" \ REDIS_LAB_EXPECTED_STATE="${EXPECTED_STATE}" \ REDIS_LAB_EXPECTED_KUBECONFIG="${EXPECTED_KUBECONFIG}" \ REDIS_LAB_EXPECTED_HOST_KUBECONFIG="${EXPECTED_HOST_KUBECONFIG}" \ "${LAB_SCRIPT}" run -- scenario-success assert_count 3 '^multipass <--purge> ' \ "${FAKE_LOG}" reset_case assert_fails env REDIS_LAB_FAKE_CIDR_OVERLAP=1 \ PATH="${FAKE_BIN}" \ REDIS_LAB_CONTRACT_TEST=1 \ REDIS_LAB_HOST_SERVICE_CIDRS='10.81.0.0/16' \ REDIS_LAB_HOME_DIR="${TEST_HOME_ROOT}" \ REDIS_LAB_FAKE_LOG="${FAKE_LOG}" \ REDIS_LAB_FAKE_INVENTORY="${FAKE_INVENTORY}" \ REDIS_LAB_FAKE_VIOLATIONS="${FAKE_VIOLATIONS}" \ REDIS_LAB_EXPECTED_STATE="${EXPECTED_STATE}" \ REDIS_LAB_EXPECTED_KUBECONFIG="${EXPECTED_KUBECONFIG}" \ REDIS_LAB_EXPECTED_HOST_KUBECONFIG="${EXPECTED_HOST_KUBECONFIG}" \ "${LAB_SCRIPT}" up assert_count 0 '^multipass ' "${FAKE_LOG}" assert_count 0 '^multipass ' "${FAKE_LOG}" assert_file_not_contains_pattern "${FAKE_LOG}" '^multipass ' assert_file_not_contains_pattern "${FAKE_LOG}" '^multipass .*<--all>' assert_file_not_contains_pattern "${FAKE_LOG}" '^multipass .*<[?*]>' assert_file_not_contains_pattern "${FAKE_LOG}" \ '^kubectl .*<(apply|create|delete|edit|patch|replace|rollout|scale|taint)>' [[ ! -s "${FAKE_VIOLATIONS}" ]] || fail 'a fake command rejected an unsafe invocation' snapshot_actual_runtime "${ACTUAL_RUNTIME_AFTER}" cmp -s -- "${ACTUAL_RUNTIME_BEFORE}" "${ACTUAL_RUNTIME_AFTER}" || fail 'contract modified the actual repository runtime' printf '%s\n' 'redis-lab-contract: PASS'