#!/usr/bin/env bash
# The dynamic half of the Compose contract: the lanes actually run.
#
# One entry point, because the order below is what makes a lane's result mean anything and every
# step in it exists because skipping it produced a false green somewhere:
#
#   1. a unique project and a fresh evidence directory, so two runs cannot read each other's results
#      and a stale directory cannot be mistaken for this run's;
#   2. the static contract, then `config`, then `create` — a stack that cannot render has no partial
#      failure mode, and finding that out after `up` costs a teardown;
#   3. `up --wait` for long-running services only, then health and the resolved activation report,
#      which is the application's answer rather than the flags this script passed in;
#   4. every one-shot the lane declares, each of which must exit zero — a missing or skipped
#      required client is a lane failure, not a lane that had nothing to check;
#   5. sanitized evidence;
#   6. teardown scoped to this project alone, on success and failure alike, after the logs are
#      collected rather than before.
#
# CI calls this script. Inlining a subset of these commands into a workflow is how a lane ends up
# running without its one-shots and reporting green.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CONTRACTS="${REPO_ROOT}/src/config/runtime/compose-profile-contracts.json"
EVIDENCE_ROOT="${REPO_ROOT}/src/app-bootstrap/build/evidence/runtime-smoke"
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
# Compose project names accept lowercase alphanumerics, hyphens and underscores only, so the run id
# is lowercased for the project while the evidence directory keeps the readable timestamp.
PROJECT_RUN_ID="$(tr '[:upper:]' '[:lower:]' <<<"${RUN_ID}")"

MODE=""
SELECTED_LANE=""
usage() {
  cat >&2 <<'USAGE'
usage:
  run-compose-runtime-smoke.sh --matrix <contracts.json>   run every blocking lane, zero-skip
  run-compose-runtime-smoke.sh --lane <id>                 reproduce one lane; not a matrix run

--lane exists to reproduce a failure, and a green single lane is not evidence that the matrix
passes. Only --matrix is.
USAGE
  exit 64
}
while [[ $# -gt 0 ]]; do
  case "$1" in
    --matrix) MODE="matrix"; CONTRACTS="$2"; shift 2 ;;
    --lane)   MODE="lane";   SELECTED_LANE="$2"; shift 2 ;;
    *) usage ;;
  esac
done
[[ -n "${MODE}" ]] || usage

for tool in jq docker; do
  command -v "${tool}" >/dev/null 2>&1 || { echo "${tool} is required" >&2; exit 78; }
done
[[ -r "${CONTRACTS}" ]] || { echo "missing ${CONTRACTS}" >&2; exit 78; }

# ---- per-lane state, cleaned up by the trap ---------------------------------
PROJECT=""
LANE_TMP=""
LANE_EVIDENCE=""
LANE_ARGS=()

cleanup_lane() {
  local status=$?
  if [[ -n "${PROJECT}" ]]; then
    # Logs first. A teardown that runs before the logs are collected destroys the only description
    # of why the lane failed.
    if [[ -n "${LANE_EVIDENCE}" ]]; then
      docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color >"${LANE_EVIDENCE}/compose.log" 2>&1 || true
      docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps --format json >"${LANE_EVIDENCE}/containers.json" 2>&1 || true
      redact_in_place "${LANE_EVIDENCE}/compose.log"
      # The warning summary belongs here, after the log it summarises exists.
      #
      # It used to run at the end of run_lane, which is before this trap collects compose.log — so it
      # grepped a file that had not been written yet and produced an empty warnings.log for every
      # lane. Fifteen lanes reported zero warnings while their logs held up to three each, and the
      # one artifact Wave 4's gate reads was the one that could not see them.
      grep -E ' (WARN|ERROR) ' "${LANE_EVIDENCE}/compose.log" 2>/dev/null \
        >"${LANE_EVIDENCE}/warnings.log" || true
    fi
    # This project only. Never a bare `down`, which would take out whatever else the developer has
    # running, and never a volume outside it.
    docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
  fi
  [[ -n "${LANE_TMP}" && -d "${LANE_TMP}" ]] && rm -rf "${LANE_TMP}"
  # .env.lane is gone — the lane's values live in the generated overlay now — but a file left by an
  # older revision of this script would still be read by the local overlay's env_file list.
  rm -f "${REPO_ROOT}/src/.env.lane" "${REPO_ROOT}/infra/keycloak/secrets/graphql-smoke-client-secret"
  # The lane keypair, including the CA's private half. A certificate that outlives the run it was
  # made for is a credential nobody is tracking.
  rm -f "${REPO_ROOT}"/infra/postgres/tls/ca.key "${REPO_ROOT}"/infra/postgres/tls/ca.crt \
        "${REPO_ROOT}"/infra/postgres/tls/ca.srl "${REPO_ROOT}"/infra/postgres/tls/server.key \
        "${REPO_ROOT}"/infra/postgres/tls/server.crt "${REPO_ROOT}"/infra/postgres/tls/server.csr \
        "${REPO_ROOT}"/infra/mailpit/tls/server.key "${REPO_ROOT}"/infra/mailpit/tls/server.crt
  return "${status}"
}

# Secrets must not survive into an artifact. The generated values are known here, so they are
# replaced by name rather than by guessing at what a secret looks like.
redact_in_place() {
  local file="$1"
  [[ -r "${file}" ]] || return 0
  local value
  for value in "${GENERATED_SECRETS[@]:-}"; do
    [[ -n "${value}" ]] && sed -i "s|${value}|<redacted>|g" "${file}"
  done
}

random_secret() { head -c 32 /dev/urandom | base64 | tr -d '=+/' | cut -c1-32; }

# Key material, which is not the same thing as a password. The notification platform base64-decodes
# each of its eight keys and requires at least 32 bytes, so random_secret's alphanumeric 32
# characters decode to 24 and are rejected. This keeps the padding and the full alphabet.
random_key() { head -c 32 /dev/urandom | base64 | tr -d '\n'; }

run_lane() {
  local lane_json="$1"
  local id profile runtime
  id="$(jq -r '.id' <<<"${lane_json}")"
  profile="$(jq -r '.composeProfile // empty' <<<"${lane_json}")"
  runtime="$(jq -r '.springRuntime' <<<"${lane_json}")"

  PROJECT="casmoke-${id}-${PROJECT_RUN_ID}"
  LANE_EVIDENCE="${EVIDENCE_ROOT}/${id}/${RUN_ID}"
  if [[ -e "${LANE_EVIDENCE}" ]]; then
    echo "${id}: ${LANE_EVIDENCE} already exists; refusing to write into a previous run's evidence" >&2
    return 1
  fi
  mkdir -p "${LANE_EVIDENCE}"
  LANE_TMP="$(mktemp -d)"
  chmod 700 "${LANE_TMP}"

  LANE_ARGS=()
  local key file
  while read -r key; do
    file="$(jq -r --arg k "${key}" '.composeFiles[$k]' "${CONTRACTS}")"
    LANE_ARGS+=(-f "${REPO_ROOT}/${file}")
  done < <(jq -r '.files[]' <<<"${lane_json}")
  [[ -n "${profile}" ]] && LANE_ARGS+=(--profile "${profile}")

  # A fresh client secret per run, mode 0600, removed on teardown. The realm references it by name.
  local secret_dir="${REPO_ROOT}/infra/keycloak/secrets"
  local secret_file="${secret_dir}/graphql-smoke-client-secret"
  mkdir -p "${secret_dir}"
  local client_secret; client_secret="$(random_secret)"
  GENERATED_SECRETS+=("${client_secret}")
  (umask 077 && printf '%s' "${client_secret}" >"${secret_file}")

  # A CA and a server certificate for the host name `db`, when the lane brings the TLS overlay.
  #
  # Generated per run and removed on teardown, like the realm secret: a committed test certificate is
  # a private key in Git. The host name matters — the prod runtime connects with sslmode=verify-full,
  # which checks the certificate against the name it dialled, so a certificate for anything but `db`
  # fails exactly as a redirected connection would. That is the check, working.
  local tls_dir="${REPO_ROOT}/infra/postgres/tls"
  if jq -e '.files | index("tls")' <<<"${lane_json}" >/dev/null; then
    mkdir -p "${tls_dir}"
    (
      umask 077
      openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \
        -keyout "${tls_dir}/ca.key" -out "${tls_dir}/ca.crt" \
        -subj "/CN=ca-skeleton-lane-ca" >/dev/null 2>&1
      openssl req -newkey rsa:2048 -nodes \
        -keyout "${tls_dir}/server.key" -out "${tls_dir}/server.csr" \
        -subj "/CN=db" >/dev/null 2>&1
      openssl x509 -req -in "${tls_dir}/server.csr" -sha256 -days 1 \
        -CA "${tls_dir}/ca.crt" -CAkey "${tls_dir}/ca.key" -CAcreateserial \
        -extfile <(printf 'subjectAltName=DNS:db\nextendedKeyUsage=serverAuth\n') \
        -out "${tls_dir}/server.crt" >/dev/null 2>&1
    )
    # The CA certificate is public and is mounted as a Compose secret, which preserves the source
    # file's mode; 0600 would be unreadable to the application's non-root user.
    chmod 0644 "${tls_dir}/ca.crt"
    rm -f "${tls_dir}/server.csr"
    [[ -s "${tls_dir}/server.crt" ]] || { echo "${id}: could not generate the lane certificate" >&2; return 1; }
  fi

  # A certificate for the host name `mailpit`, when the lane brings the reference SMTP relay.
  #
  # Same rule as the PostgreSQL one above and for the same reason: the notification platform's
  # transport type has no plaintext member, so a relay without a certificate is one the platform
  # cannot be configured to talk to. Generated per run, removed on teardown.
  local mail_tls="${REPO_ROOT}/infra/mailpit/tls"
  if jq -e '.services | index("mailpit")' <<<"${lane_json}" >/dev/null; then
    mkdir -p "${mail_tls}"
    (
      umask 077
      openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \
        -keyout "${mail_tls}/server.key" -out "${mail_tls}/server.crt" \
        -subj "/CN=mailpit" \
        -addext "subjectAltName=DNS:mailpit" \
        -addext "extendedKeyUsage=serverAuth" >/dev/null 2>&1
    )
    # Mailpit runs as a non-root user and reads both halves; 0600 would be unreadable to it.
    chmod 0644 "${mail_tls}/server.crt" "${mail_tls}/server.key"
    [[ -s "${mail_tls}/server.crt" ]] || { echo "${id}: could not generate the Mailpit certificate" >&2; return 1; }
  fi

  # The values a deployment must supply: the seven with no inline default, and the secrets the prod
  # env validator requires to be non-blank. A lane supplies them explicitly rather than inheriting a
  # developer's src/.env, which is the difference between a lane that reproduces anywhere and one
  # that reproduces on the machine it was written on. All generated per run and removed on teardown.
  #
  # The database credentials are additionally *exported*, because two different mechanisms have to
  # agree on them. The application reads this file; the `db` service takes its POSTGRES_USER and
  # POSTGRES_PASSWORD from Compose interpolation — `${APP_DATASOURCE_PASSWORD:-ca_skeleton}` — and
  # interpolation reads the process environment and the project .env file, never a service's
  # env_file. So the application got the generated password and the database got the literal
  # default, and no lane whose overlay did not happen to restate the value could ever authenticate.
  # shared-infra-local passed only because the local overlay restates it, which made the dev lane's
  # failure look like a dev-specific problem rather than the general one it is.
  export POSTGRES_DB=ca_skeleton
  export APP_DATASOURCE_USERNAME=ca_skeleton
  export APP_DATASOURCE_PASSWORD="${client_secret}"

  # Which phase the notification client runs. Exported rather than defaulted in Compose, because a
  # default made local-notification-serving run the ingest assertions and pass without ever checking
  # a delivery.
  export NOTIFICATION_SMOKE_PHASE
  NOTIFICATION_SMOKE_PHASE="$(jq -r '.notificationSmokePhase // ""' <<<"${lane_json}")"

  # The capability schema streams this lane installs, exported for the same reason as the credentials
  # above: the migration containers take them through Compose interpolation, which reads the process
  # environment and never a lane overlay. They are deliberately not part of activationEnv — the
  # application does not read this value, the two one-shots that install and promote the schema do,
  # and putting an operator input in the application's environment block is how it would come to look
  # like a switch the application honours.
  export CAPABILITY_STREAMS
  CAPABILITY_STREAMS="$(jq -r '.capabilityStreams // [] | join(" ")' <<<"${lane_json}")"

  # The public health address, which is not the same in every runtime: application-local.yml pins
  # presentation.api-base-path to /api while the shipped default is /v1. The smoke client used to
  # hardcode /api, so it verified the local runtime and 404'd against a dev application that had
  # started perfectly.
  if [[ "${runtime}" == "local" ]]; then
    export APP_HEALTH_PATH=/api/healthcheck
  else
    export APP_HEALTH_PATH=/v1/healthcheck
  fi

  # The notification platform's eight purpose-scoped keys.
  #
  # All eight, always, for every lane — not only the notification ones. They cost nothing when the
  # capability is off, because the platform binds nothing at all then, and a lane that supplies only
  # the keys it currently needs is a lane that breaks the moment a switch is added to it.
  #
  # Distinct by construction: the platform refuses to start if two purposes carry the same material,
  # which is the check that stops one leaked key from being all eight. Each is redacted from the
  # evidence like every other generated value.
  local -a notification_keys=()
  local purpose
  for purpose in CONTACT_ENCRYPTION CONTACT_LOOKUP_HMAC CALLBACK_SIGNING PROVIDER_CREDENTIAL \
                 PAYLOAD_ENCRYPTION VAPID_SIGNING PROVIDER_REQUEST_LOOKUP_HMAC \
                 CALLBACK_FINGERPRINT_HMAC; do
    local value; value="$(random_key)"
    GENERATED_SECRETS+=("${value}")
    notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY=${value}")
    # The id, which is not secret and is deliberately tied to this run. A lane that reused a fixed
    # id across runs would be asserting the one thing the platform refuses to assume: that material
    # and id change together.
    notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY_ID=lane-${PROJECT_RUN_ID}-$(echo "${purpose}" | tr '[:upper:]_' '[:lower:]-')")
  done

  # Every deployment-supplied value the lane owns, collected here and written into the generated
  # overlay's `environment:` block below rather than into an env file.
  #
  # These lived in src/.env.lane, and an env file cannot win. env_file lists merge across overlays
  # and the later one takes precedence, so the developer's optional src/.env — declared by the local
  # overlay, after the base — silently replaced them. That is not hypothetical: it pinned the JWT
  # issuer to http://localhost:8081, which inside the app container is the application itself, so the
  # application fetched JWKS from its own port and answered AUTH_JWKS_UNAVAILABLE to every
  # authenticated request while the lane had supplied the correct issuer all along.
  #
  # An `environment:` block outranks every env_file regardless of order, which is the same fix the
  # lane's activation switches already use. The cost is that these values appear in `docker inspect`
  # for this run's own throwaway project; they are generated per run, removed on teardown, and
  # redacted from every evidence file by name. A lane whose settings are silently discarded is worse.
  local -a lane_environment=(
    "APP_NAME=ca-skeleton-${id}"
    # APP_DATASOURCE_URL is deliberately absent. The environment overlays own it and they do not
    # agree by accident: the local one is a plain jdbc:postgresql URL and the prod-smoke one carries
    # sslmode=verify-full and a CA path, because the prod profile refuses to start without them.
    # Setting it here overrode both — this overlay is appended last — and off-prod, shared-infra-dev
    # and prod-smoke failed the profile check on a TLS requirement the stack had satisfied.
    "APP_DATASOURCE_USERNAME=${APP_DATASOURCE_USERNAME}"
    "APP_DATASOURCE_PASSWORD=${APP_DATASOURCE_PASSWORD}"
    "APP_SECURITY_JWT_ISSUER=http://keycloak:8080/realms/ca-skeleton"
    "APP_SECURITY_JWT_AUDIENCE=ca-skeleton-api"
    "SPRING_PROFILES_ACTIVE=${runtime}"
    "${notification_keys[@]}"
  )
  local generated
  for generated in APP_SECURITY_JWT_SIGNING_KEY APP_SECURITY_OAUTH_CLIENT_SECRET \
                   APP_EXTERNAL_API_KEY APP_PRIVACY_PSEUDONYMIZATION_SALT; do
    local secret; secret="$(random_secret)"
    GENERATED_SECRETS+=("${secret}")
    lane_environment+=("${generated}=${secret}")
  done
  # The lane's own settings go into a generated overlay rather than into that file.
  #
  # env_file lists merge across overlays and the later file wins, so the developer's optional
  # src/.env — declared by the local overlay, after the base — silently overrode the lane's values.
  # local-messaging started Kafka, set APP_MESSAGING_BROKER=kafka, and was refused by the dependency
  # validator for a broker it had supplied, because an empty value from a file nobody mentioned won
  # the merge. An `environment:` block beats every env_file regardless of order, so the lane's
  # settings stop depending on where a file happens to sit in the stack.
  #
  # The overlay adds no service, so the set the static verifier checks is unchanged.
  # Written as a function, because a handoff lane rewrites it between phases: the same project and
  # the same database, with the application recreated under a different activation environment.
  local lane_overlay="${LANE_TMP}/lane-overrides.yml"
  write_lane_overlay() {
    local phase_override="${1:-{\}}"
    (
      umask 077
      {
        echo "services:"
        echo "  app:"
        echo "    environment:"
        local entry
        for entry in "${lane_environment[@]}"; do
          printf '      %s: "%s"\n' "${entry%%=*}" "${entry#*=}"
        done
        jq -r --argjson override "${phase_override}" \
          '.activationEnv + $override | to_entries[] | "      \(.key): \"\(.value)\""' \
          <<<"${lane_json}"
      } >"${lane_overlay}"
    )
  }
  write_lane_overlay "$(jq -c '.handoff.firstPhaseEnv // {}' <<<"${lane_json}")"
  LANE_ARGS+=(-f "${lane_overlay}")
  GENERATED_SECRETS+=("${client_secret}")

  echo "== ${id} (project ${PROJECT}, runtime ${runtime})"

  # 2. static contract, then render, then create
  "${REPO_ROOT}/scripts/verify-compose-profile-contracts.sh" >"${LANE_EVIDENCE}/contract.log" 2>&1
  ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" config >"${LANE_EVIDENCE}/rendered.yml" )
  redact_in_place "${LANE_EVIDENCE}/rendered.yml"
  # Build before create. The app service declares both `build:` and `image:`, so Compose reuses a
  # matching tag if one is lying around — and the first run of this lane did exactly that, starting
  # a jar built from a different state of the repository and failing on a class that no longer
  # exists in the tree. A lane that runs a stale image produces evidence about code nobody changed.
  ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" build app \
      >"${LANE_EVIDENCE}/build.log" 2>&1 )
  ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" create >/dev/null )

  # 3a. Anything that has to happen before the application starts, in the order the lane lists it.
  #
  # A capability schema stream is the case this exists for. Each stream keeps its own Flyway history
  # table and is promoted to ACTIVE deliberately after installation, and the application refuses to
  # start until that promotion has happened — so running it with the other one-shots, after `up`,
  # would run it after the thing it is a precondition for. The distinction is the lane's, not this
  # script's guess: preStartServices run here, oneShotServices run after the readiness poll.
  #
  # Read on fd 4 with stdin closed, for the reason the lane loop at the bottom of this file already
  # records: `docker compose run` consumes stdin, so a plain `while read` loop over several services
  # runs the first one and then finds its input exhausted. That is not hypothetical here — it is how
  # this lane ran its migration and silently skipped its promotion, and how shared-infra-local,
  # shared-infra-dev and prod-smoke each ran auth-smoke and skipped minio-init and
  # object-storage-smoke while reporting green. The evidence directories showed one log where the
  # contract named three.
  local pre_start
  while read -r -u 4 pre_start; do
    [[ -z "${pre_start}" ]] && continue
    echo "-- ${id}: ${pre_start} (before start)"
    if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${pre_start}" </dev/null ) \
        >"${LANE_EVIDENCE}/${pre_start}.log" 2>&1; then
      redact_in_place "${LANE_EVIDENCE}/${pre_start}.log"
      echo "${id}: ${pre_start} did not succeed; see ${pre_start}.log" >&2
      return 1
    fi
    redact_in_place "${LANE_EVIDENCE}/${pre_start}.log"
  done 4< <(jq -r '.preStartServices[]?' <<<"${lane_json}")

  # 3. long-running services only. A one-shot is not a --wait target: it is meant to exit.
  local long_running
  long_running="$(jq -r '.longRunningServices[]' <<<"${lane_json}")"
  # shellcheck disable=SC2086
  ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" up --wait --wait-timeout 300 ${long_running} )

  await_application_report() {
  # The application's own answer, not this script's input.
  #
  # The management port, not the application port. Actuator runs on its own connector here so the
  # management plane is not published on the public one, and fetching 8080/actuator returned an
  # empty file that looked exactly like a failed assertion about the profile.
  local management_port
  management_port="$(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
    sh -c 'echo "${MANAGEMENT_SERVER_PORT:-9001}"' 2>/dev/null | tr -d "\r\n")"
  management_port="${management_port:-9001}"
  # Poll rather than trust `up --wait`. Not every overlay defines a healthcheck — the dev one does
  # not — and where there is none `--wait` returns as soon as the container is created, so the first
  # fetch landed before the application had finished starting and produced an empty file that read
  # exactly like a failed assertion about the profile.
  local attempt
  for attempt in $(seq 1 60); do
    if docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
        wget -qO- "http://localhost:${management_port}/actuator/adapteractivation" \
        >"${LANE_EVIDENCE}/activation.json" 2>/dev/null \
        && [[ -s "${LANE_EVIDENCE}/activation.json" ]]; then
      break
    fi
    sleep 5
  done
  docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
    wget -qO- "http://localhost:${management_port}/actuator/health" \
    >"${LANE_EVIDENCE}/health.json" 2>/dev/null || true

  local reported
  reported="$(jq -r '.activeProfile // empty' "${LANE_EVIDENCE}/activation.json" 2>/dev/null || true)"
  if [[ "${reported}" != "${runtime}" ]]; then
    echo "${id}: the application reports profile '${reported}', the contract says '${runtime}'" >&2
    return 1
  fi

  # The switches the application says are on must be exactly the ones the lane asked for. Without
  # this a lane called local-jpa that ran with JPA off would render the right services, start
  # cleanly, and prove nothing — which is the failure mode a green lane is least able to reveal.
  local expected_on actual_on
  expected_on="$(jq -r '.expectedSwitchesOn | sort | join(",")' <<<"${lane_json}")"
  actual_on="$(jq -r '.switches | to_entries | map(select(.value)) | map(.key) | sort | join(",")' \
    "${LANE_EVIDENCE}/activation.json")"
  if [[ "${expected_on}" != "${actual_on}" ]]; then
    echo "${id}: the application reports [${actual_on}] on, the contract expects [${expected_on}]" >&2
    return 1
  fi

  # The vendor the application resolved, not the one the lane started. These differ exactly when a
  # profile pins a datasource that outranks the lane's environment, and every other field in the
  # report looks correct while it happens.
  local expected_vendor actual_vendor
  expected_vendor="$(jq -r '.expectedPersistenceVendor' <<<"${lane_json}")"
  actual_vendor="$(jq -r '.persistenceVendor // "absent"' "${LANE_EVIDENCE}/activation.json")"
  if [[ "${expected_vendor}" != "${actual_vendor}" ]]; then
    echo "${id}: the application resolved vendor '${actual_vendor}', the contract expects '${expected_vendor}'" >&2
    return 1
  fi

  }
  await_application_report || return 1

  # 3b. The handoff: one project, one database, the application recreated under the second phase.
  #
  # Not two lanes glued together. The evidence a handoff needs is that a request accepted while the
  # platform was INGEST_ONLY is delivered exactly once after a restart into SERVING — on the route
  # frozen at accept — and that is only evidence if the row is the same row. Tearing the volume down
  # between phases, or copying rows into a second project, proves nothing about a handoff.
  if jq -e '.handoff' <<<"${lane_json}" >/dev/null; then
    local first_smoke second_smoke
    first_smoke="$(jq -r '.handoff.firstPhaseSmoke' <<<"${lane_json}")"
    second_smoke="$(jq -r '.handoff.secondPhaseSmoke' <<<"${lane_json}")"

    echo "-- ${id}: ${first_smoke} (handoff phase 1)"
    if ! ( cd "${REPO_ROOT}" && NOTIFICATION_SMOKE_PHASE="${first_smoke}" \
             docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm notification-smoke </dev/null ) \
        >"${LANE_EVIDENCE}/notification-smoke-phase1.log" 2>&1; then
      redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log"
      echo "${id}: the first handoff phase did not succeed" >&2
      return 1
    fi
    redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log"
    cp "${LANE_EVIDENCE}/activation.json" "${LANE_EVIDENCE}/activation-phase1.json"

    # The application only. The database and its volume stay up, which is the whole point.
    echo "-- ${id}: restarting the application into the second phase"
    ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" stop app >/dev/null 2>&1 )

    write_lane_overlay '{}'
    ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" \
        up -d --force-recreate --wait --wait-timeout 300 app )
    await_application_report || return 1
  fi

  # 3c. The startup log carries no WARN and no ERROR.
  #
  # Wave 4's condition is about startup specifically, and a summary of the whole lane cannot express
  # it: the lanes that exercise authentication end with a deliberate malformed-token probe, and the
  # application answering it with a WARN is the security control working. Splitting at the line where
  # the application reports it has started is what makes "startup is silent" checkable instead of
  # argued — and it is checked here, per lane, rather than read off an artifact afterwards.
  #
  # StartupWarningZeroTest asserts the same property in-process for the all-off composition. This is
  # the same assertion against a real container with the lane's switches actually on.
  local startup_log="${LANE_EVIDENCE}/startup.log"
  docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color app \
    >"${startup_log}" 2>&1 || true
  redact_in_place "${startup_log}"
  local startup_noise
  startup_noise="$(awk '/Started CaSkeletonApplication/{exit} /(WARN|ERROR)/{print}' "${startup_log}")"
  if [[ -n "${startup_noise}" ]]; then
    echo "${id}: the startup log is not silent:" >&2
    echo "${startup_noise}" | head -20 >&2
    return 1
  fi

  # 4. every declared one-shot, each of which must exit zero
  #
  # fd 4 and a closed stdin, same reason as the pre-start loop above. And the count is checked rather
  # than trusted: this loop ran exactly one one-shot per lane for its whole life, so the three lanes
  # that declare three clients verified object storage in none of their runs and said so in neither
  # their output nor their exit status. A loop that silently does less than the contract asks is the
  # one failure a green lane cannot reveal, so the arithmetic is now part of the lane.
  local one_shot ran_one_shots=0 declared_one_shots
  declared_one_shots="$(jq -r '.oneShotServices // [] | length' <<<"${lane_json}")"
  while read -r -u 4 one_shot; do
    [[ -z "${one_shot}" ]] && continue
    echo "-- ${id}: ${one_shot}"
    if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${one_shot}" </dev/null ) \
        >"${LANE_EVIDENCE}/${one_shot}.log" 2>&1; then
      redact_in_place "${LANE_EVIDENCE}/${one_shot}.log"
      echo "${id}: ${one_shot} did not exit zero; see ${one_shot}.log" >&2
      return 1
    fi
    redact_in_place "${LANE_EVIDENCE}/${one_shot}.log"
    ran_one_shots=$((ran_one_shots + 1))
  done 4< <(jq -r '.oneShotServices[]?' <<<"${lane_json}")
  if [[ "${ran_one_shots}" -ne "${declared_one_shots}" ]]; then
    echo "${id}: ran ${ran_one_shots} of ${declared_one_shots} one-shots; a partial lane is not a pass" >&2
    return 1
  fi

  # 5. the manifest ties the evidence to the exact stack that produced it
  jq -n \
    --arg lane "${id}" --arg run "${RUN_ID}" --arg project "${PROJECT}" \
    --arg runtime "${runtime}" --arg profile "${profile}" \
    --argjson services "$(jq '.services' <<<"${lane_json}")" \
    '{lane:$lane, runId:$run, project:$project, springRuntime:$runtime,
      composeProfile:(if $profile=="" then null else $profile end), services:$services}' \
    >"${LANE_EVIDENCE}/manifest.json"

  echo "== ${id}: passed"
}

GENERATED_SECRETS=()
FAILED_LANES=()

if [[ "${MODE}" == "lane" ]]; then
  lane_json="$(jq -c --arg id "${SELECTED_LANE}" '.lanes[] | select(.id == $id)' "${CONTRACTS}")"
  [[ -n "${lane_json}" ]] || { echo "no lane '${SELECTED_LANE}' in ${CONTRACTS}" >&2; exit 64; }
  trap cleanup_lane EXIT
  run_lane "${lane_json}"
  echo "one lane reproduced. A green single lane is not evidence that the matrix passes."
  exit 0
fi

blocking="$(jq -c '.lanes[] | select(.blocking == true)' "${CONTRACTS}")"
total="$(wc -l <<<"${blocking}" | tr -d ' ')"
echo "running ${total} blocking lane(s), zero-skip"
# Read on fd 3, not stdin. `docker compose exec` and friends consume stdin, and inside a plain
# `while read` loop they ate the remaining lanes — so the first lane ran, the loop ended, and the
# script reported that all six had passed. A wrapper whose own success message is a false green is
# worse than no wrapper, and nothing else here would have caught it: the count came from the
# contract and the exit status from the one lane that did run.
RAN_LANES=0
while read -r -u 3 lane_json; do
  [[ -z "${lane_json}" ]] && continue
  RAN_LANES=$((RAN_LANES + 1))
  if ( trap cleanup_lane EXIT; run_lane "${lane_json}" ); then :; else
    FAILED_LANES+=("$(jq -r '.id' <<<"${lane_json}")")
  fi
done 3<<<"${blocking}"

if [[ "${RAN_LANES}" -ne "${total}" ]]; then
  echo "run-compose-runtime-smoke: ran ${RAN_LANES} of ${total} lanes; a partial matrix is not a pass" >&2
  exit 1
fi

if [[ ${#FAILED_LANES[@]} -gt 0 ]]; then
  echo "run-compose-runtime-smoke: ${#FAILED_LANES[@]} lane(s) failed: ${FAILED_LANES[*]}" >&2
  exit 1
fi
echo "run-compose-runtime-smoke: all ${total} blocking lanes passed; evidence under ${EVIDENCE_ROOT}"