#!/usr/bin/env bash
set -Eeuo pipefail
umask 077

SCRIPT_PATH="${BASH_SOURCE[0]}"
if [[ "${SCRIPT_PATH}" != /* ]]; then
    SCRIPT_PATH="${PWD}/${SCRIPT_PATH}"
fi
SCRIPT_DIR="$(cd -P -- "${SCRIPT_PATH%/*}" && pwd -P)"
REPOSITORY_ROOT="$(cd -P -- "${SCRIPT_DIR}/../../.." && pwd -P)"
SRC_ROOT="${REPOSITORY_ROOT}/src"
BUILD_ROOT="${SRC_ROOT}/build"
LAB_ROOT="${BUILD_ROOT}/redis-lab"
OBSERVATION_ROOT="${LAB_ROOT}/observations"
CLOUD_INIT_ROOT="${LAB_ROOT}/cloud-init"
HOST_KUBECONFIG="${OBSERVATION_ROOT}/host-kubeconfig"
LAB_KUBECONFIG="${LAB_ROOT}/kubeconfig"
STATE_FILE="${LAB_ROOT}/run.state"
LOCK_FILE="${LAB_ROOT}/lifecycle.lock"
TOKEN_FILE="${LAB_ROOT}/k3s-token"
K3S_BINARY="${LAB_ROOT}/k3s-amd64"
RUN_HANDOFF_MARKER="${LAB_ROOT}/run-handoff.started"
VERSIONS_FILE="${REPOSITORY_ROOT}/infra/redis-lab/versions.env"
CLOUD_INIT_FILE="${REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml"
KUBECONFIG_RENDERER="${REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk"
DEFAULT_KUBECONFIG_ROOT="${REDIS_LAB_HOME_DIR:-${HOME}}"
DEFAULT_KUBECONFIG="${DEFAULT_KUBECONFIG_ROOT}/.kube/config"
CONTEXT_NAME='ca-redis-lab'
POD_CIDR='10.52.0.0/16'
SERVICE_CIDR='10.53.0.0/16'
SERVER_NAME='ca-redis-lab-server'
AGENT_ONE_NAME='ca-redis-lab-agent-1'
AGENT_TWO_NAME='ca-redis-lab-agent-2'
OWNERSHIP_MARKER_PATH='/var/lib/ca-redis-lab/ownership'
MULTIPASS_LIST_TIMEOUT_SECONDS=30
MULTIPASS_INFO_TIMEOUT_SECONDS=30
MULTIPASS_LAUNCH_TIMEOUT_SECONDS=300
MULTIPASS_EXEC_TIMEOUT_SECONDS=300
KUBECTL_TIMEOUT_SECONDS=20
RECONCILE_ATTEMPTS=3
RECONCILE_INTERVAL_SECONDS=1
READY_ATTEMPTS=9
READY_INTERVAL_SECONDS=2
RUN_ID=''
RUN_OWNERSHIP_ACTIVE=0
TRACKED_INPUTS_VALIDATED=0
K3S_VERSION=''
MULTIPASS_IMAGE=''
K3S_AMD64_URL=''
K3S_AMD64_SHA256=''

fail() {
    printf 'redis-lab: %s\n' "$1" >&2
    return 1
}

usage() {
    printf '%s\n' \
        'usage: redis-lab preflight|up|down|postflight|run [--retain-on-failure] -- command [args...]' \
        >&2
    return 64
}

validate_tracked_input_paths() {
    local tracked_parent
    local canonical_parent

    for tracked_parent in \
        "${REPOSITORY_ROOT}/infra" \
        "${REPOSITORY_ROOT}/infra/redis-lab" \
        "${REPOSITORY_ROOT}/infra/redis-lab/cloud-init" \
        "${REPOSITORY_ROOT}/infra/redis-lab/lib"; do
        if [[ -L "${tracked_parent}" || ! -d "${tracked_parent}" ]]; then
            return 1
        fi
        if ! canonical_parent="$(cd -P -- "${tracked_parent}" && pwd -P)" ||
            [[ "${canonical_parent}" != "${tracked_parent}" ]]; then
            return 1
        fi
    done
    if [[ "${VERSIONS_FILE}" != "${REPOSITORY_ROOT}/infra/redis-lab/versions.env" ||
        -L "${VERSIONS_FILE}" || ! -f "${VERSIONS_FILE}" || ! -r "${VERSIONS_FILE}" ||
        "${CLOUD_INIT_FILE}" != "${REPOSITORY_ROOT}/infra/redis-lab/cloud-init/node.yaml" ||
        -L "${CLOUD_INIT_FILE}" || ! -f "${CLOUD_INIT_FILE}" || ! -r "${CLOUD_INIT_FILE}" ||
        "${KUBECONFIG_RENDERER}" != "${REPOSITORY_ROOT}/infra/redis-lab/lib/render-kubeconfig.awk" ||
        -L "${KUBECONFIG_RENDERER}" || ! -f "${KUBECONFIG_RENDERER}" ||
        ! -r "${KUBECONFIG_RENDERER}" ]]; then
        return 1
    fi
}

load_tracked_versions() {
    local line
    local key
    local value
    local line_count=0
    local parsed_k3s_version=''
    local parsed_multipass_image=''
    local parsed_k3s_amd64_url=''
    local parsed_k3s_amd64_sha256=''
    local -A seen_keys=()

    while IFS= read -r line || [[ -n "${line}" ]]; do
        ((line_count += 1))
        if ((line_count > 4)) ||
            [[ ! "${line}" =~ ^([A-Z0-9_]+)=([^[:space:]=]+)$ ]]; then
            fail 'tracked lab input unavailable'
            return 1
        fi
        key="${BASH_REMATCH[1]}"
        value="${BASH_REMATCH[2]}"
        case "${key}" in
            K3S_VERSION)
                [[ ! -v 'seen_keys[K3S_VERSION]' ]] || {
                    fail 'tracked lab input unavailable'
                    return 1
                }
                parsed_k3s_version="${value}"
                ;;
            MULTIPASS_IMAGE)
                [[ ! -v 'seen_keys[MULTIPASS_IMAGE]' ]] || {
                    fail 'tracked lab input unavailable'
                    return 1
                }
                parsed_multipass_image="${value}"
                ;;
            K3S_AMD64_URL)
                [[ ! -v 'seen_keys[K3S_AMD64_URL]' ]] || {
                    fail 'tracked lab input unavailable'
                    return 1
                }
                parsed_k3s_amd64_url="${value}"
                ;;
            K3S_AMD64_SHA256)
                [[ ! -v 'seen_keys[K3S_AMD64_SHA256]' ]] || {
                    fail 'tracked lab input unavailable'
                    return 1
                }
                parsed_k3s_amd64_sha256="${value}"
                ;;
            *)
                fail 'tracked lab input unavailable'
                return 1
                ;;
        esac
        seen_keys["${key}"]=1
    done <"${VERSIONS_FILE}"
    if ((line_count != 4 || ${#seen_keys[@]} != 4)); then
        fail 'tracked lab input unavailable'
        return 1
    fi
    if [[ ! "${parsed_k3s_version}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]]; then
        fail 'invalid pinned version'
        return 1
    fi
    if [[ "${parsed_multipass_image}" != '24.04' ]]; then
        fail 'invalid pinned image'
        return 1
    fi
    if [[ "${parsed_k3s_amd64_url}" != 'https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' ||
        "${parsed_k3s_amd64_sha256}" != 'f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' ]]; then
        fail 'invalid pinned artifact'
        return 1
    fi
    K3S_VERSION="${parsed_k3s_version}"
    MULTIPASS_IMAGE="${parsed_multipass_image}"
    K3S_AMD64_URL="${parsed_k3s_amd64_url}"
    K3S_AMD64_SHA256="${parsed_k3s_amd64_sha256}"
}

validate_and_load_tracked_inputs() {
    if ! validate_tracked_input_paths; then
        fail 'tracked lab input unavailable'
        return 1
    fi
    if ! load_tracked_versions; then
        return 1
    fi
    TRACKED_INPUTS_VALIDATED=1
}

run_child() {
    "$@" 9>&-
}

bounded() {
    local seconds="$1"
    shift
    timeout --signal=TERM --kill-after=5s "${seconds}s" "$@" 9>&-
}

bounded_keep_lock() {
    local seconds="$1"
    shift
    timeout --signal=TERM --kill-after=5s "${seconds}s" "$@"
}

is_allowlisted_name() {
    case "$1" in
        "${SERVER_NAME}" | "${AGENT_ONE_NAME}" | "${AGENT_TWO_NAME}")
            return 0
            ;;
        *)
            return 1
            ;;
    esac
}

validate_static_contract() {
    local child_path
    if [[ "${LAB_ROOT}" != "${REPOSITORY_ROOT}/src/build/redis-lab" ]]; then
        fail 'invalid transient path'
        return 1
    fi
    if [[ -L "${LAB_ROOT}" || -L "${OBSERVATION_ROOT}" ]]; then
        fail 'invalid transient path'
        return 1
    fi
    if ! validate_tracked_input_paths; then
        fail 'tracked lab input unavailable'
        return 1
    fi
    if [[ ! "${K3S_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]]; then
        fail 'invalid pinned version'
        return 1
    fi
    if [[ "${MULTIPASS_IMAGE}" != '24.04' ]]; then
        fail 'invalid pinned image'
        return 1
    fi
    if [[ "${K3S_AMD64_URL}" != 'https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s' ||
        "${K3S_AMD64_SHA256}" != 'f03cad6610cf5b2903d8a9ac3d6716690e53dab461b09c07b0c913a262166abc' ]]; then
        fail 'invalid pinned artifact'
        return 1
    fi
    if [[ -L "${SRC_ROOT}" || ! -d "${SRC_ROOT}" ||
        "$(cd -- "${SRC_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src" ]]; then
        fail 'invalid transient path'
        return 1
    fi
    if [[ -e "${BUILD_ROOT}" || -L "${BUILD_ROOT}" ]]; then
        if [[ -L "${BUILD_ROOT}" || ! -d "${BUILD_ROOT}" ||
            "$(cd -- "${BUILD_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src/build" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    fi
    if [[ -e "${LAB_ROOT}" || -L "${LAB_ROOT}" ]]; then
        if [[ -L "${LAB_ROOT}" || ! -d "${LAB_ROOT}" ||
            "$(cd -- "${LAB_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src/build/redis-lab" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    fi
    if [[ -e "${OBSERVATION_ROOT}" || -L "${OBSERVATION_ROOT}" ]]; then
        if [[ -L "${OBSERVATION_ROOT}" || ! -d "${OBSERVATION_ROOT}" ||
            "$(cd -- "${OBSERVATION_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src/build/redis-lab/observations" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    fi
    if [[ -e "${CLOUD_INIT_ROOT}" || -L "${CLOUD_INIT_ROOT}" ]]; then
        if [[ -L "${CLOUD_INIT_ROOT}" || ! -d "${CLOUD_INIT_ROOT}" ||
            "$(cd -- "${CLOUD_INIT_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src/build/redis-lab/cloud-init" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    fi
    for child_path in \
        "${STATE_FILE}" \
        "${STATE_FILE}.next" \
        "${LOCK_FILE}" \
        "${TOKEN_FILE}" \
        "${K3S_BINARY}" \
        "${RUN_HANDOFF_MARKER}" \
        "${LAB_KUBECONFIG}" \
        "${LAB_KUBECONFIG}.next" \
        "${LAB_ROOT}/kubeconfig.rendered" \
        "${LAB_ROOT}/fingerprint.before" \
        "${LAB_ROOT}/fingerprint.after" \
        "${LAB_ROOT}/existing-inventory.csv"; do
        if [[ -L "${child_path}" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    done
    for child_path in "${LAB_ROOT}"/* "${OBSERVATION_ROOT}"/* "${CLOUD_INIT_ROOT}"/*; do
        if [[ -L "${child_path}" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    done
    for child_path in \
        "${OBSERVATION_ROOT}/host-kubeconfig" \
        "${OBSERVATION_ROOT}/host-service-cidrs" \
        "${OBSERVATION_ROOT}/cidr-inputs.raw" \
        "${OBSERVATION_ROOT}/fingerprint-before.raw" \
        "${OBSERVATION_ROOT}/fingerprint-after.raw" \
        "${OBSERVATION_ROOT}/multipass-before.csv" \
        "${OBSERVATION_ROOT}/multipass-after.csv" \
        "${OBSERVATION_ROOT}/lab-node-readiness.raw" \
        "${OBSERVATION_ROOT}/lab-node-readiness.sorted" \
        "${OBSERVATION_ROOT}/lab-node-readiness.expected"; do
        if [[ -L "${child_path}" ]]; then
            fail 'invalid transient path'
            return 1
        fi
    done
}

acquire_lifecycle_lock() {
    if ! validate_static_contract; then
        return 1
    fi
    if ! run_child mkdir -p -- "${BUILD_ROOT}" "${LAB_ROOT}"; then
        fail 'transient path unavailable'
        return 1
    fi
    if ! validate_static_contract; then
        return 1
    fi
    if ! run_child chmod 0700 -- "${LAB_ROOT}"; then
        fail 'transient permissions unavailable'
        return 1
    fi
    if ! exec 9>>"${LOCK_FILE}"; then
        fail 'lifecycle lock unavailable'
        return 1
    fi
    if ! bounded_keep_lock 5 flock -n 9; then
        fail 'lifecycle already active'
        return 1
    fi
}

prepare_runtime() {
    if ! validate_static_contract; then
        return 1
    fi
    if ! run_child mkdir -p -- "${BUILD_ROOT}"; then
        fail 'transient path unavailable'
        return 1
    fi
    if [[ -L "${BUILD_ROOT}" ||
        "$(cd -- "${BUILD_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src/build" ]]; then
        fail 'invalid transient path'
        return 1
    fi
    if ! run_child mkdir -p -- "${LAB_ROOT}" "${OBSERVATION_ROOT}"; then
        fail 'transient path unavailable'
        return 1
    fi
    if ! validate_static_contract; then
        return 1
    fi
    if ! run_child chmod 0700 -- "${LAB_ROOT}" "${OBSERVATION_ROOT}"; then
        fail 'transient permissions unavailable'
        return 1
    fi
}

start_new_run() {
    if [[ -v run_state_owned ]]; then
        run_state_owned=1
    fi
    if ! run_child rm -rf -- "${OBSERVATION_ROOT}" "${CLOUD_INIT_ROOT}"; then
        fail 'transient path unavailable'
        return 1
    fi
    if ! run_child rm -f -- \
        "${LAB_KUBECONFIG}" \
        "${TOKEN_FILE}" \
        "${K3S_BINARY}" \
        "${RUN_HANDOFF_MARKER}" \
        "${STATE_FILE}.next"; then
        fail 'transient path unavailable'
        return 1
    fi
    if ! run_child mkdir -p -- "${OBSERVATION_ROOT}" ||
        ! run_child chmod 0700 -- "${OBSERVATION_ROOT}"; then
        fail 'transient path unavailable'
        return 1
    fi
    RUN_ID="run-${BASHPID}-${RANDOM}-${RANDOM}"
    if [[ ! "${RUN_ID}" =~ ^run-[0-9]+-[0-9]+-[0-9]+$ ]]; then
        fail 'run identity unavailable'
        return 1
    fi
    if ! printf 'RUN|%s\n' "${RUN_ID}" >"${STATE_FILE}.next"; then
        fail 'run state unavailable'
        return 1
    fi
    if ! install_state_next; then
        return 1
    fi
}

cleanup_host_kubeconfig() {
    if [[ -L "${BUILD_ROOT}" || -L "${LAB_ROOT}" || -L "${OBSERVATION_ROOT}" ||
        ! -d "${OBSERVATION_ROOT}" ]]; then
        return 0
    fi
    if [[ "$(cd -- "${OBSERVATION_ROOT}" && pwd -P)" != "${REPOSITORY_ROOT}/src/build/redis-lab/observations" ]]; then
        return 0
    fi
    run_child rm -f -- "${HOST_KUBECONFIG}"
}

install_state_next() {
    local state_next="${STATE_FILE}.next"

    if ! run_child chmod 0600 -- "${state_next}"; then
        run_child rm -f -- "${state_next}"
        fail 'run state unavailable'
        return 1
    fi
    if ! run_child mv -- "${state_next}" "${STATE_FILE}"; then
        run_child rm -f -- "${state_next}"
        fail 'run state unavailable'
        return 1
    fi
}

reserve_attempt_name() {
    local name="$1"
    local state_next="${STATE_FILE}.next"

    if ! is_allowlisted_name "${name}"; then
        fail 'invalid lab instance name'
        return 1
    fi
    if ! run_child cp -- "${STATE_FILE}" "${state_next}"; then
        fail 'run state unavailable'
        return 1
    fi
    if [[ -z "${RUN_ID}" ]]; then
        fail 'run identity unavailable'
        return 1
    fi
    if ! printf 'PENDING|%s|%s\n' "${RUN_ID}" "${name}" >>"${state_next}"; then
        run_child rm -f -- "${state_next}"
        fail 'run state unavailable'
        return 1
    fi
    install_state_next
}

promote_attempt_name() {
    local name="$1"
    local state_next="${STATE_FILE}.next"

    if ! run_child awk -F'|' -v owner="${RUN_ID}" -v target="${name}" '
        $1 == "PENDING" && $2 == owner && $3 == target {
            print "CREATED|" owner "|" target
            promoted += 1
            next
        }
        {
            print
        }
        END {
            if (promoted != 1) {
                exit 1
            }
        }
    ' "${STATE_FILE}" >"${state_next}"; then
        run_child rm -f -- "${state_next}"
        fail 'run state unavailable'
        return 1
    fi
    install_state_next
}

mark_attempt_reconcile() {
    local name="$1"
    local state_next="${STATE_FILE}.next"

    if ! run_child awk -F'|' -v owner="${RUN_ID}" -v target="${name}" '
        ($1 == "PENDING" || $1 == "CREATED") && $2 == owner && $3 == target {
            print "RECONCILE|" owner "|" target
            changed += 1
            next
        }
        $1 == "RECONCILE" && $2 == owner && $3 == target {
            print
            changed += 1
            next
        }
        {
            print
        }
        END {
            if (changed != 1) {
                exit 1
            }
        }
    ' "${STATE_FILE}" >"${state_next}"; then
        run_child rm -f -- "${state_next}"
        fail 'run state unavailable'
        return 1
    fi
    install_state_next
}

remove_attempt_name() {
    local name="$1"
    local state_next="${STATE_FILE}.next"

    if ! run_child awk -F'|' -v owner="${RUN_ID}" -v target="${name}" '
        $1 != "RUN" && $2 == owner && $3 == target {
            removed += 1
            next
        }
        {
            print
        }
        END {
            if (removed != 1) {
                exit 1
            }
        }
    ' "${STATE_FILE}" >"${state_next}"; then
        run_child rm -f -- "${state_next}"
        fail 'run state unavailable'
        return 1
    fi
    install_state_next
}

validate_recorded_state() {
    local status owner name extra
    local line_number=0
    local count=0
    local seen_server=0
    local seen_agent_one=0
    local seen_agent_two=0

    [[ -f "${STATE_FILE}" ]] || return 0
    while IFS='|' read -r status owner name extra; do
        [[ -n "${status}" ]] || continue
        ((line_number += 1))
        if ((line_number == 1)); then
            if [[ "${status}" != RUN || -z "${owner}" || -n "${name}" || -n "${extra}" ||
                ! "${owner}" =~ ^run-[0-9]+-[0-9]+-[0-9]+$ ]]; then
                fail 'invalid run state'
                return 1
            fi
            if [[ -n "${RUN_ID}" && "${RUN_ID}" != "${owner}" ]]; then
                fail 'invalid run state'
                return 1
            fi
            RUN_ID="${owner}"
            continue
        fi
        if [[ -n "${extra}" || ! "${status}" =~ ^(PENDING|CREATED|RECONCILE)$ ||
            "${owner}" != "${RUN_ID}" ]]; then
            fail 'invalid run state'
            return 1
        fi
        if ! is_allowlisted_name "${name}"; then
            fail 'invalid run state'
            return 1
        fi
        case "${name}" in
            "${SERVER_NAME}")
                ((seen_server += 1))
                if [[ "${seen_server}" != 1 ]]; then
                    fail 'invalid run state'
                    return 1
                fi
                ;;
            "${AGENT_ONE_NAME}")
                ((seen_agent_one += 1))
                if [[ "${seen_agent_one}" != 1 ]]; then
                    fail 'invalid run state'
                    return 1
                fi
                ;;
            "${AGENT_TWO_NAME}")
                ((seen_agent_two += 1))
                if [[ "${seen_agent_two}" != 1 ]]; then
                    fail 'invalid run state'
                    return 1
                fi
                ;;
        esac
        ((count += 1))
        if ((count > 3)); then
            fail 'invalid run state'
            return 1
        fi
    done <"${STATE_FILE}"
    if ((line_number == 0)); then
        fail 'invalid run state'
        return 1
    fi
}

recorded_state_has_instances() {
    local status owner name extra

    [[ -f "${STATE_FILE}" ]] || return 1
    while IFS='|' read -r status owner name extra; do
        if [[ -n "${status}" && "${status}" != RUN ]]; then
            return 0
        fi
    done <"${STATE_FILE}"
    return 1
}

wait_for_owned_instance() {
    local owner="$1"
    local name="$2"
    local marker=''
    local attempt

    for ((attempt = 1; attempt <= RECONCILE_ATTEMPTS; attempt += 1)); do
        if bounded "${MULTIPASS_INFO_TIMEOUT_SECONDS}" \
            multipass info --format csv "${name}" >/dev/null 2>&1; then
            marker=''
            if marker="$(bounded "${MULTIPASS_INFO_TIMEOUT_SECONDS}" \
                multipass exec "${name}" -- sudo cat "${OWNERSHIP_MARKER_PATH}")" &&
                [[ "${marker}" == "${owner}|${name}" ]]; then
                return 0
            fi
        fi
        if ((attempt < RECONCILE_ATTEMPTS)); then
            bounded 5 sleep "${RECONCILE_INTERVAL_SECONDS}" || true
        fi
    done
    return 1
}

delete_if_owned() {
    local owner="$1"
    local name="$2"

    if ! wait_for_owned_instance "${owner}" "${name}"; then
        fail 'lab ownership unresolved'
        return 1
    fi
    if ! bounded "${MULTIPASS_INFO_TIMEOUT_SECONDS}" \
        multipass delete --purge "${name}"; then
        fail 'lab cleanup failed'
        return 1
    fi
    return 0
}

cleanup_recorded() {
    local status owner name extra
    local index
    local cleanup_status=0
    local -a statuses=()
    local -a owners=()
    local -a names=()

    validate_recorded_state || return 1
    [[ -f "${STATE_FILE}" ]] || return 0
    while IFS='|' read -r status owner name extra; do
        [[ -n "${status}" && "${status}" != RUN ]] || continue
        statuses+=("${status}")
        owners+=("${owner}")
        names+=("${name}")
    done <"${STATE_FILE}"

    for index in "${!names[@]}"; do
        status="${statuses[${index}]}"
        owner="${owners[${index}]}"
        name="${names[${index}]}"
        if [[ "${status}" != RECONCILE ]]; then
            if ! mark_attempt_reconcile "${name}"; then
                cleanup_status=1
                continue
            fi
        fi
        if delete_if_owned "${owner}" "${name}"; then
            if ! remove_attempt_name "${name}"; then
                cleanup_status=1
            fi
        else
            cleanup_status=1
        fi
    done

    if [[ -f "${STATE_FILE}" ]] && ! recorded_state_has_instances; then
        if ! run_child rm -f -- \
            "${STATE_FILE}" \
            "${TOKEN_FILE}" \
            "${LAB_KUBECONFIG}" \
            "${K3S_BINARY}" ||
            ! run_child rm -rf -- "${CLOUD_INIT_ROOT}"; then
            cleanup_status=1
        fi
    fi
    return "${cleanup_status}"
}

cleanup_failed_attempt() {
    local name="$1"

    if ! mark_attempt_reconcile "${name}"; then
        return 1
    fi
    cleanup_recorded
}

capture_multipass_inventory() {
    local destination="$1"
    if ! bounded "${MULTIPASS_LIST_TIMEOUT_SECONDS}" \
        multipass list --format csv >"${destination}"; then
        fail 'multipass inventory unavailable'
        return 1
    fi
    run_child chmod 0600 -- "${destination}"
}

reject_existing_names() {
    local inventory_file="${LAB_ROOT}/existing-inventory.csv"
    local name
    local ignored

    capture_multipass_inventory "${inventory_file}" || return 1
    while IFS=, read -r name ignored; do
        [[ "${name}" != 'Name' ]] || continue
        if is_allowlisted_name "${name}"; then
            fail 'lab instance name already exists'
            return 1
        fi
    done <"${inventory_file}"
    run_child rm -f -- "${inventory_file}"
}

host_context_from_copy() {
    run_child awk '
        $1 == "current-context:" {
            print $2
            found = 1
            exit
        }
        END {
            if (!found) {
                exit 1
            }
        }
    ' "${HOST_KUBECONFIG}"
}

validate_context_name() {
    if [[ ! "$1" =~ ^[A-Za-z0-9._@:/-]+$ ]]; then
        fail 'invalid host context'
        return 1
    fi
}

host_kubectl() {
    local host_context="$1"
    shift
    bounded "${KUBECTL_TIMEOUT_SECONDS}" kubectl \
        --kubeconfig "${HOST_KUBECONFIG}" \
        --context "${host_context}" \
        "$@"
}

lab_kubectl() {
    bounded "${KUBECTL_TIMEOUT_SECONDS}" kubectl \
        --kubeconfig "${LAB_KUBECONFIG}" \
        --context "${CONTEXT_NAME}" \
        "$@"
}

append_lab_inventory_projection() {
    local inventory_file="$1"
    local projection_file="$2"
    local name
    local ignored
    local count=0

    while IFS=, read -r name ignored; do
        [[ "${name}" != 'Name' ]] || continue
        if is_allowlisted_name "${name}"; then
            ((count += 1))
            printf 'lab-instance|%s\n' "${name}" >>"${projection_file}"
            if ! bounded "${MULTIPASS_INFO_TIMEOUT_SECONDS}" \
                multipass info --format csv "${name}" |
                LC_ALL=C run_child sort |
                run_child awk -v instance="${name}" '{print "lab-resource|" instance "|" $0}' \
                    >>"${projection_file}"; then
                fail 'lab resource observation unavailable'
                return 1
            fi
        fi
    done <"${inventory_file}"
    printf 'lab-resource-count|%s\n' "${count}" >>"${projection_file}"
}

capture_host_fingerprint() {
    local phase="$1"
    local raw_file="${OBSERVATION_ROOT}/fingerprint-${phase}.raw"
    local inventory_file="${OBSERVATION_ROOT}/multipass-${phase}.csv"
    local destination="${LAB_ROOT}/fingerprint.${phase}"
    local host_context=''

    : >"${raw_file}"
    run_child chmod 0600 -- "${raw_file}"

    if [[ -f "${DEFAULT_KUBECONFIG}" ]]; then
        if ! run_child cp -- "${DEFAULT_KUBECONFIG}" "${HOST_KUBECONFIG}"; then
            fail 'host kubeconfig observation unavailable'
            return 1
        fi
        run_child chmod 0600 -- "${HOST_KUBECONFIG}"
        if ! host_context="$(host_context_from_copy)"; then
            fail 'host kubeconfig observation unavailable'
            return 1
        fi
        validate_context_name "${host_context}" || return 1
        if ! REDIS_LAB_CAPTURE_PHASE="${phase}" \
            bounded 30 sha256sum "${DEFAULT_KUBECONFIG}" |
            run_child awk '{print "default-kubeconfig-sha256|" $1}' >>"${raw_file}"; then
            fail 'host kubeconfig observation unavailable'
            return 1
        fi
        printf 'host-current-context|%s\n' "${host_context}" >>"${raw_file}"
        if ! host_kubectl "${host_context}" config view --minify \
            -o 'jsonpath={.clusters[0].cluster.server}' |
            run_child awk '{print "host-api|" $0}' >>"${raw_file}"; then
            fail 'host kube API observation unavailable'
            return 1
        fi
        if ! host_kubectl "${host_context}" get nodes \
            -o 'jsonpath={range .items[*]}{.metadata.name}{"|"}{.spec.providerID}{"|"}{range .spec.podCIDRs[*]}{.}{","}{end}{"\n"}{end}' |
            LC_ALL=C run_child sort |
            run_child awk '{print "host-node|" $0}' >>"${raw_file}"; then
            fail 'host node observation unavailable'
            return 1
        fi
        if ! host_kubectl "${host_context}" get deployments,statefulsets,daemonsets \
            --all-namespaces \
            -o 'jsonpath={range .items[*]}{.metadata.namespace}{"|"}{.kind}{"|"}{.metadata.name}{"|"}{.spec.replicas}{"\n"}{end}' |
            LC_ALL=C run_child sort |
            run_child awk '{print "host-controller|" $0}' >>"${raw_file}"; then
            fail 'host controller observation unavailable'
            return 1
        fi
        if ! host_kubectl "${host_context}" get services --all-namespaces \
            -o 'jsonpath={range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.spec.clusterIP}{"|"}{range .spec.ports[*]}{.nodePort}{","}{end}{"\n"}{end}' |
            LC_ALL=C run_child sort |
            run_child awk '{print "host-service|" $0}' >>"${raw_file}"; then
            fail 'host service observation unavailable'
            return 1
        fi
        if ! resolve_host_service_cidrs "${OBSERVATION_ROOT}/host-service-cidrs"; then
            return 1
        fi
        run_child awk '{print "host-service-cidr|" $0}' \
            "${OBSERVATION_ROOT}/host-service-cidrs" >>"${raw_file}"
    else
        printf '%s\n' \
            'default-kubeconfig-sha256|ABSENT' \
            'host-current-context|ABSENT' \
            'host-api|ABSENT' \
            'host-node|ABSENT' \
            'host-controller|ABSENT' \
            'host-service|ABSENT' \
            'host-service-cidr|ABSENT' >>"${raw_file}"
    fi

    if ! bounded 10 ip -o -4 addr show |
        LC_ALL=C run_child sort |
        run_child awk '{print "host-interface|" $0}' >>"${raw_file}"; then
        fail 'host interface observation unavailable'
        return 1
    fi
    if ! bounded 10 ip -4 route show table all |
        LC_ALL=C run_child sort |
        run_child awk '{print "host-route|" $0}' >>"${raw_file}"; then
        fail 'host route observation unavailable'
        return 1
    fi
    capture_multipass_inventory "${inventory_file}" || return 1
    LC_ALL=C run_child sort "${inventory_file}" |
        run_child awk '{print "multipass-inventory|" $0}' >>"${raw_file}"
    append_lab_inventory_projection "${inventory_file}" "${raw_file}" || return 1
    LC_ALL=C run_child sort "${raw_file}" >"${destination}"
    run_child chmod 0600 -- "${destination}"
}

ipv4_to_integer() {
    local address="$1"
    local first second third fourth
    IFS=. read -r first second third fourth <<<"${address}"
    [[ "${first}" =~ ^[0-9]+$ && "${second}" =~ ^[0-9]+$ &&
        "${third}" =~ ^[0-9]+$ && "${fourth}" =~ ^[0-9]+$ ]] || return 1
    ((first <= 255 && second <= 255 && third <= 255 && fourth <= 255)) || return 1
    printf '%u\n' "$(((first << 24) | (second << 16) | (third << 8) | fourth))"
}

resolve_host_service_cidrs() {
    local destination="$1"
    local configured="${REDIS_LAB_HOST_SERVICE_CIDRS:-}"
    local cidr address prefix address_integer mask
    local count=0

    if [[ -z "${configured}" ||
        ! "${configured}" =~ ^[0-9.,/[:space:]]+$ ]]; then
        fail 'host service CIDRs required'
        return 1
    fi
    if ! run_child tr ', ' '\n\n' <<<"${configured}" |
        run_child awk 'NF {print}' |
        LC_ALL=C run_child sort -u >"${destination}"; then
        fail 'host service CIDRs invalid'
        return 1
    fi
    while IFS= read -r cidr; do
        ((count += 1))
        if ((count > 16)) || [[ "${cidr}" != */* ]]; then
            fail 'host service CIDRs invalid'
            return 1
        fi
        address="${cidr%/*}"
        prefix="${cidr#*/}"
        if [[ ! "${prefix}" =~ ^[0-9]+$ ]] ||
            ((prefix < 1 || prefix > 32)); then
            fail 'host service CIDRs invalid'
            return 1
        fi
        if ! address_integer="$(ipv4_to_integer "${address}")"; then
            fail 'host service CIDRs invalid'
            return 1
        fi
        mask=$(((0xFFFFFFFF << (32 - prefix)) & 0xFFFFFFFF))
        if ((address_integer != (address_integer & mask))); then
            fail 'host service CIDRs invalid'
            return 1
        fi
    done <"${destination}"
    if ((count == 0)); then
        fail 'host service CIDRs required'
        return 1
    fi
    run_child chmod 0600 -- "${destination}"
}

cidr_overlaps() {
    local candidate="$1"
    local lab_cidr="$2"
    local candidate_address="${candidate%/*}"
    local candidate_prefix=32
    local lab_address="${lab_cidr%/*}"
    local lab_prefix="${lab_cidr#*/}"
    local candidate_integer lab_integer candidate_mask lab_mask
    local candidate_start candidate_end lab_start lab_end

    [[ "${candidate}" == */* ]] && candidate_prefix="${candidate#*/}"
    [[ "${candidate_prefix}" =~ ^[0-9]+$ && "${lab_prefix}" =~ ^[0-9]+$ ]] || return 1
    ((candidate_prefix >= 1 && candidate_prefix <= 32)) || return 1
    candidate_integer="$(ipv4_to_integer "${candidate_address}")" || return 1
    lab_integer="$(ipv4_to_integer "${lab_address}")" || return 1
    candidate_mask=$(((0xFFFFFFFF << (32 - candidate_prefix)) & 0xFFFFFFFF))
    lab_mask=$(((0xFFFFFFFF << (32 - lab_prefix)) & 0xFFFFFFFF))
    candidate_start=$((candidate_integer & candidate_mask))
    candidate_end=$((candidate_start | (0xFFFFFFFF ^ candidate_mask)))
    lab_start=$((lab_integer & lab_mask))
    lab_end=$((lab_start | (0xFFFFFFFF ^ lab_mask)))
    ((candidate_start <= lab_end && lab_start <= candidate_end))
}

reject_cidr_overlap() {
    local cidr_inputs="${OBSERVATION_ROOT}/cidr-inputs.raw"
    local host_context=''
    local candidate

    : >"${cidr_inputs}"
    run_child chmod 0600 -- "${cidr_inputs}"
    if ! bounded 10 ip -o -4 addr show >>"${cidr_inputs}"; then
        fail 'host interface observation unavailable'
        return 1
    fi
    if ! bounded 10 ip -4 route show table all >>"${cidr_inputs}"; then
        fail 'host route observation unavailable'
        return 1
    fi
    if ! bounded "${MULTIPASS_LIST_TIMEOUT_SECONDS}" \
        multipass list --format csv >>"${cidr_inputs}"; then
        fail 'multipass inventory unavailable'
        return 1
    fi
    if [[ -f "${HOST_KUBECONFIG}" ]]; then
        if ! host_context="$(host_context_from_copy)"; then
            fail 'host kubeconfig observation unavailable'
            return 1
        fi
        validate_context_name "${host_context}" || return 1
        if ! host_kubectl "${host_context}" get nodes \
            -o 'jsonpath={range .items[*]}{range .spec.podCIDRs[*]}{.}{"\n"}{end}{end}' \
            >>"${cidr_inputs}"; then
            fail 'host pod CIDR observation unavailable'
            return 1
        fi
        if ! resolve_host_service_cidrs "${OBSERVATION_ROOT}/host-service-cidrs"; then
            return 1
        fi
        run_child cat "${OBSERVATION_ROOT}/host-service-cidrs" >>"${cidr_inputs}"
    fi

    while IFS= read -r candidate; do
        if cidr_overlaps "${candidate}" "${POD_CIDR}" ||
            cidr_overlaps "${candidate}" "${SERVICE_CIDR}"; then
            fail 'host CIDR overlap'
            return 1
        fi
    done < <(
        run_child grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?' "${cidr_inputs}" |
            LC_ALL=C run_child sort -u || true
    )
}

preflight() {
    if ! prepare_runtime; then
        return 1
    fi
    if [[ -f "${STATE_FILE}" ]]; then
        if ! validate_recorded_state; then
            return 1
        fi
        if recorded_state_has_instances; then
            fail 'existing run state requires down'
            return 1
        fi
    fi
    if ! reject_existing_names; then
        return 1
    fi
    if ! start_new_run; then
        cleanup_host_kubeconfig
        return 1
    fi
    if ! capture_host_fingerprint before; then
        cleanup_host_kubeconfig
        return 1
    fi
    if ! reject_cidr_overlap; then
        cleanup_host_kubeconfig
        return 1
    fi
    cleanup_host_kubeconfig
}

render_cloud_init() {
    local name="$1"
    local destination="${CLOUD_INIT_ROOT}/${name}.yaml"
    local destination_next="${destination}.next"

    if ! is_allowlisted_name "${name}" || [[ -z "${RUN_ID}" ]]; then
        fail 'run identity unavailable'
        return 1
    fi
    if ! validate_static_contract ||
        ! run_child mkdir -p -- "${CLOUD_INIT_ROOT}" ||
        ! run_child chmod 0700 -- "${CLOUD_INIT_ROOT}" ||
        ! validate_static_contract; then
        fail 'rendered cloud-init unavailable'
        return 1
    fi
    if [[ -L "${destination}" || -L "${destination_next}" ]]; then
        fail 'invalid transient path'
        return 1
    fi
    if ! run_child awk -v ownership="${RUN_ID}|${name}" '
        $0 == "runcmd:" {
            print "  - path: /var/lib/ca-redis-lab/ownership"
            print "    owner: root:root"
            print "    permissions: \"0600\""
            print "    content: |"
            print "      " ownership
            inserted += 1
        }
        {
            print
        }
        END {
            if (inserted != 1) {
                exit 1
            }
        }
    ' "${CLOUD_INIT_FILE}" >"${destination_next}"; then
        run_child rm -f -- "${destination_next}"
        fail 'rendered cloud-init unavailable'
        return 1
    fi
    if ! run_child chmod 0600 -- "${destination_next}" ||
        ! run_child mv -- "${destination_next}" "${destination}" ||
        ! validate_static_contract; then
        run_child rm -f -- "${destination_next}"
        fail 'rendered cloud-init unavailable'
        return 1
    fi
    printf '%s\n' "${destination}"
}

launch_one() {
    local name="$1"
    local memory="$2"
    local rendered_cloud_init

    if ! is_allowlisted_name "${name}"; then
        fail 'invalid lab instance name'
        return 1
    fi
    if ! rendered_cloud_init="$(render_cloud_init "${name}" 9>&-)"; then
        return 1
    fi
    if ! reserve_attempt_name "${name}"; then
        return 1
    fi
    RUN_OWNERSHIP_ACTIVE=1
    if ! bounded "${MULTIPASS_LAUNCH_TIMEOUT_SECONDS}" multipass launch \
        --name "${name}" \
        --cpus 2 \
        --memory "${memory}" \
        --disk 12G \
        --cloud-init "${rendered_cloud_init}" \
        "${MULTIPASS_IMAGE}"; then
        fail 'lab instance launch failed'
        cleanup_failed_attempt "${name}" || true
        return 1
    fi
    if ! wait_for_owned_instance "${RUN_ID}" "${name}"; then
        fail 'lab instance ownership unavailable'
        cleanup_failed_attempt "${name}" || true
        return 1
    fi
    if ! promote_attempt_name "${name}"; then
        cleanup_failed_attempt "${name}" || true
        return 1
    fi
}

server_ipv4() {
    bounded "${MULTIPASS_INFO_TIMEOUT_SECONDS}" \
        multipass info --format csv "${SERVER_NAME}" |
        run_child awk -F, 'NR == 2 {print $3; found = 1; exit} END {if (!found) exit 1}'
}

prepare_k3s_binary() {
    local architecture
    local actual_sha256
    local name

    if ! architecture="$(bounded 5 uname -m)" || [[ "${architecture}" != x86_64 ]]; then
        fail 'unsupported lab architecture'
        return 1
    fi
    if ! bounded 120 curl \
        --fail \
        --location \
        --silent \
        --show-error \
        --output "${K3S_BINARY}" \
        "${K3S_AMD64_URL}"; then
        fail 'pinned artifact download failed'
        return 1
    fi
    if ! actual_sha256="$(
        bounded 30 sha256sum "${K3S_BINARY}" |
            run_child awk '{print $1}'
    )"; then
        fail 'pinned artifact verification failed'
        return 1
    fi
    if [[ "${actual_sha256}" != "${K3S_AMD64_SHA256}" ]]; then
        run_child rm -f -- "${K3S_BINARY}"
        fail 'pinned artifact verification failed'
        return 1
    fi
    if ! run_child chmod 0700 -- "${K3S_BINARY}"; then
        fail 'pinned artifact unavailable'
        return 1
    fi
    for name in "${SERVER_NAME}" "${AGENT_ONE_NAME}" "${AGENT_TWO_NAME}"; do
        if ! bounded 60 multipass transfer \
            "${K3S_BINARY}" "${name}:/home/ubuntu/ca-redis-lab-k3s"; then
            fail 'pinned artifact transfer failed'
            return 1
        fi
    done
}

render_lab_kubeconfig() {
    local source_file="$1"
    local destination_file="$2"
    local server_address="$3"
    local render_next="${destination_file}.next"

    if ! run_child rm -f -- "${render_next}"; then
        run_child rm -f -- "${render_next}" "${destination_file}" || true
        fail 'lab kubeconfig invalid'
        return 1
    fi
    if ! run_child awk -v address="${server_address}" -v target="${CONTEXT_NAME}" \
        -f "${KUBECONFIG_RENDERER}" "${source_file}" >"${render_next}"; then
        run_child rm -f -- "${render_next}" "${destination_file}" || true
        fail 'lab kubeconfig invalid'
        return 1
    fi
    if ! run_child chmod 0600 -- "${render_next}"; then
        run_child rm -f -- "${render_next}" "${destination_file}" || true
        fail 'lab kubeconfig invalid'
        return 1
    fi
    if ! run_child mv -f -- "${render_next}" "${destination_file}"; then
        run_child rm -f -- "${render_next}" "${destination_file}" || true
        fail 'lab kubeconfig invalid'
        return 1
    fi
}

configure_k3s() {
    local server_address
    local rendered_kubeconfig="${LAB_ROOT}/kubeconfig.rendered"
    local server_exec="server --cluster-cidr=${POD_CIDR} --service-cidr=${SERVICE_CIDR} --disable=traefik --disable=servicelb --write-kubeconfig-mode=0600"
    local agent_exec='agent'

    if ! prepare_k3s_binary; then
        return 1
    fi
    if ! bounded 10 openssl rand -hex 32 >"${TOKEN_FILE}"; then
        fail 'lab token generation failed'
        return 1
    fi
    run_child chmod 0600 -- "${TOKEN_FILE}"
    if ! server_address="$(server_ipv4)"; then
        fail 'lab server address unavailable'
        return 1
    fi
    if [[ ! "${server_address}" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
        fail 'lab server address unavailable'
        return 1
    fi
    if ! ipv4_to_integer "${server_address}" >/dev/null; then
        fail 'lab server address unavailable'
        return 1
    fi

    if ! bounded "${MULTIPASS_EXEC_TIMEOUT_SECONDS}" \
        multipass exec "${SERVER_NAME}" -- sh -ceu \
        'IFS= read -r K3S_TOKEN; expected_sha="$1"; expected_version="$2"; shift 2; actual_sha="$(sha256sum /home/ubuntu/ca-redis-lab-k3s | awk '"'"'{print $1}'"'"')"; test "${actual_sha}" = "${expected_sha}"; sudo install -m 0755 /home/ubuntu/ca-redis-lab-k3s /usr/local/bin/k3s; actual_version="$(sudo /usr/local/bin/k3s --version | awk '"'"'NR == 1 {print $3}'"'"')"; test "${actual_version}" = "${expected_version}"; export K3S_TOKEN; sudo -E sh -ceu '"'"'nohup /usr/local/bin/k3s "$@" >/var/log/ca-redis-lab-k3s.log 2>&1 &'"'"' sh "$@"' \
        install-k3s-server "${K3S_AMD64_SHA256}" "${K3S_VERSION}" ${server_exec} \
        <"${TOKEN_FILE}"; then
        fail 'k3s server setup failed'
        return 1
    fi
    if ! bounded "${MULTIPASS_EXEC_TIMEOUT_SECONDS}" \
        multipass exec "${AGENT_ONE_NAME}" -- sh -ceu \
        'IFS= read -r K3S_TOKEN; K3S_URL="$1"; expected_sha="$2"; expected_version="$3"; shift 3; actual_sha="$(sha256sum /home/ubuntu/ca-redis-lab-k3s | awk '"'"'{print $1}'"'"')"; test "${actual_sha}" = "${expected_sha}"; sudo install -m 0755 /home/ubuntu/ca-redis-lab-k3s /usr/local/bin/k3s; actual_version="$(sudo /usr/local/bin/k3s --version | awk '"'"'NR == 1 {print $3}'"'"')"; test "${actual_version}" = "${expected_version}"; export K3S_TOKEN K3S_URL; sudo -E sh -ceu '"'"'nohup /usr/local/bin/k3s "$@" >/var/log/ca-redis-lab-k3s.log 2>&1 &'"'"' sh "$@"' \
        install-k3s-agent "https://${server_address}:6443" "${K3S_AMD64_SHA256}" \
        "${K3S_VERSION}" ${agent_exec} <"${TOKEN_FILE}"; then
        fail 'k3s agent setup failed'
        return 1
    fi
    if ! bounded "${MULTIPASS_EXEC_TIMEOUT_SECONDS}" \
        multipass exec "${AGENT_TWO_NAME}" -- sh -ceu \
        'IFS= read -r K3S_TOKEN; K3S_URL="$1"; expected_sha="$2"; expected_version="$3"; shift 3; actual_sha="$(sha256sum /home/ubuntu/ca-redis-lab-k3s | awk '"'"'{print $1}'"'"')"; test "${actual_sha}" = "${expected_sha}"; sudo install -m 0755 /home/ubuntu/ca-redis-lab-k3s /usr/local/bin/k3s; actual_version="$(sudo /usr/local/bin/k3s --version | awk '"'"'NR == 1 {print $3}'"'"')"; test "${actual_version}" = "${expected_version}"; export K3S_TOKEN K3S_URL; sudo -E sh -ceu '"'"'nohup /usr/local/bin/k3s "$@" >/var/log/ca-redis-lab-k3s.log 2>&1 &'"'"' sh "$@"' \
        install-k3s-agent "https://${server_address}:6443" "${K3S_AMD64_SHA256}" \
        "${K3S_VERSION}" ${agent_exec} <"${TOKEN_FILE}"; then
        fail 'k3s agent setup failed'
        return 1
    fi
    if ! bounded "${MULTIPASS_EXEC_TIMEOUT_SECONDS}" \
        multipass exec "${SERVER_NAME}" -- sh -ceu \
        'sudo cat /etc/rancher/k3s/k3s.yaml' >"${rendered_kubeconfig}"; then
        fail 'lab kubeconfig unavailable'
        return 1
    fi
    if [[ ! -s "${rendered_kubeconfig}" ]]; then
        fail 'lab kubeconfig unavailable'
        return 1
    fi
    if ! render_lab_kubeconfig \
        "${rendered_kubeconfig}" "${LAB_KUBECONFIG}" "${server_address}"; then
        return 1
    fi
    run_child rm -f -- "${rendered_kubeconfig}"
    wait_for_exact_nodes_ready
}

wait_for_exact_nodes_ready() {
    local attempts="${READY_ATTEMPTS}"
    local interval="${READY_INTERVAL_SECONDS}"
    local attempt
    local observed="${OBSERVATION_ROOT}/lab-node-readiness.raw"
    local sorted_observed="${OBSERVATION_ROOT}/lab-node-readiness.sorted"
    local expected="${OBSERVATION_ROOT}/lab-node-readiness.expected"

    if [[ "${REDIS_LAB_CONTRACT_TEST:-0}" == 1 ]]; then
        attempts="${REDIS_LAB_TEST_READY_ATTEMPTS:-3}"
        interval=0
        if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]] ||
            ((attempts > 10)); then
            fail 'invalid readiness test seam'
            return 1
        fi
    fi
    printf '%s\n' \
        "${SERVER_NAME}|True" \
        "${AGENT_ONE_NAME}|True" \
        "${AGENT_TWO_NAME}|True" |
        LC_ALL=C run_child sort >"${expected}"
    run_child chmod 0600 -- "${expected}"

    for ((attempt = 1; attempt <= attempts; attempt += 1)); do
        if lab_kubectl get nodes \
            -o 'jsonpath={range .items[*]}{.metadata.name}{"|"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \
            >"${observed}"; then
            LC_ALL=C run_child sort -u "${observed}" >"${sorted_observed}"
            if run_child cmp -s -- "${expected}" "${sorted_observed}"; then
                return 0
            fi
        fi
        run_child sleep "${interval}"
    done
    fail 'lab nodes not ready'
}

up() {
    preflight || return 1
    if ! launch_one "${SERVER_NAME}" 3G; then
        cleanup_recorded || true
        return 1
    fi
    if ! launch_one "${AGENT_ONE_NAME}" 2560M; then
        cleanup_recorded || true
        return 1
    fi
    if ! launch_one "${AGENT_TWO_NAME}" 2560M; then
        cleanup_recorded || true
        return 1
    fi
    if ! configure_k3s; then
        cleanup_recorded || true
        return 1
    fi
    RUN_OWNERSHIP_ACTIVE=0
}

down() {
    if ! prepare_runtime; then
        return 1
    fi
    if ! cleanup_recorded; then
        fail 'lab cleanup failed'
        return 1
    fi
}

postflight() {
    if ! prepare_runtime; then
        return 1
    fi
    if [[ ! -f "${LAB_ROOT}/fingerprint.before" ]]; then
        fail 'preflight fingerprint unavailable'
        return 1
    fi
    if ! capture_host_fingerprint after; then
        cleanup_host_kubeconfig
        return 1
    fi
    cleanup_host_kubeconfig
    if ! run_child cmp -s -- \
        "${LAB_ROOT}/fingerprint.before" "${LAB_ROOT}/fingerprint.after"; then
        fail 'host fingerprint mismatch'
        return 1
    fi
}

signal_run_handoff_for_contract() {
    if [[ "${REDIS_LAB_CONTRACT_TEST:-0}" != 1 ||
        "${REDIS_LAB_FAKE_RUN_HANDOFF_SIGNAL:-0}" != 1 ]]; then
        return 0
    fi
    if [[ -L "${RUN_HANDOFF_MARKER}" ]] ||
        ! printf '%s\n' 'post-up-pre-command' >"${RUN_HANDOFF_MARKER}" ||
        ! run_child chmod 0600 -- "${RUN_HANDOFF_MARKER}"; then
        fail 'run handoff test seam unavailable'
        return 1
    fi
    kill -TERM "${BASHPID}"
    return 143
}

run_flow() (
    local retain_on_failure=0
    local run_status=0
    local cleanup_status=0
    local postflight_status=0
    local cleanup_required=0
    local run_state_owned=0

    if [[ "${1:-}" == '--retain-on-failure' ]]; then
        retain_on_failure=1
        shift
    fi
    if [[ "${1:-}" != '--' ]]; then
        usage
        return $?
    fi
    shift
    if (($# == 0)); then
        usage
        return $?
    fi
    if ((retain_on_failure == 1)) && [[ "${CI:-false}" == 'true' ]]; then
        fail 'retain-on-failure is forbidden in CI'
        return 1
    fi

    emergency_cleanup() {
        local observed_status=$?
        local original_status="${1:-${observed_status}}"
        trap - EXIT HUP INT TERM
        cleanup_host_kubeconfig
        if ((cleanup_required == 1 && run_state_owned == 1)); then
            cleanup_recorded || true
        fi
        exit "${original_status}"
    }
    trap 'emergency_cleanup $?' EXIT
    trap 'emergency_cleanup 129' HUP
    trap 'emergency_cleanup 130' INT
    trap 'emergency_cleanup 143' TERM

    cleanup_required=1
    up || return 1
    signal_run_handoff_for_contract || return $?
    RUN_OWNERSHIP_ACTIVE=1
    if "$@" 9>&-; then
        run_status=0
    else
        run_status=$?
    fi

    if ((run_status != 0 && retain_on_failure == 1)); then
        RUN_OWNERSHIP_ACTIVE=0
        cleanup_required=0
        return "${run_status}"
    fi
    cleanup_recorded || cleanup_status=$?
    if ((cleanup_status == 0)); then
        cleanup_required=0
    fi
    RUN_OWNERSHIP_ACTIVE=0
    capture_host_fingerprint after || postflight_status=$?
    cleanup_host_kubeconfig
    if ((postflight_status == 0)); then
        run_child cmp -s -- \
            "${LAB_ROOT}/fingerprint.before" "${LAB_ROOT}/fingerprint.after" ||
            postflight_status=1
    fi
    if ((cleanup_status != 0)); then
        fail 'lab cleanup failed'
        return 1
    fi
    if ((postflight_status != 0)); then
        fail 'host fingerprint mismatch'
        return 1
    fi
    return "${run_status}"
)

main() {
    local command_name="${1:-}"
    if (($# == 0)); then
        usage
        return $?
    fi
    if ! validate_and_load_tracked_inputs; then
        return 1
    fi
    if ! acquire_lifecycle_lock; then
        return 1
    fi
    shift
    case "${command_name}" in
        preflight)
            if (($# != 0)); then
                usage
                return $?
            fi
            preflight
            ;;
        up)
            if (($# != 0)); then
                usage
                return $?
            fi
            up
            ;;
        down)
            if (($# != 0)); then
                usage
                return $?
            fi
            down
            ;;
        postflight)
            if (($# != 0)); then
                usage
                return $?
            fi
            postflight
            ;;
        run)
            run_flow "$@"
            ;;
        *)
            usage
            ;;
    esac
}

emergency_exit() {
    local observed_status=$?
    local original_status="${1:-${observed_status}}"
    trap - EXIT HUP INT TERM
    if ((TRACKED_INPUTS_VALIDATED == 1)); then
        cleanup_host_kubeconfig
        if ((RUN_OWNERSHIP_ACTIVE == 1)); then
            cleanup_recorded || true
        fi
    fi
    exit "${original_status}"
}

trap 'emergency_exit $?' EXIT
trap 'emergency_exit 129' HUP
trap 'emergency_exit 130' INT
trap 'emergency_exit 143' TERM
main "$@"
