#!/usr/bin/env bash # The dynamic half of the Compose contract: the lanes actually run. # # One entry point, because the order below is what makes a lane's result mean anything and every # step in it exists because skipping it produced a false green somewhere: # # 1. a unique project and a fresh evidence directory, so two runs cannot read each other's results # and a stale directory cannot be mistaken for this run's; # 2. the static contract, then `config`, then `create` — a stack that cannot render has no partial # failure mode, and finding that out after `up` costs a teardown; # 3. `up --wait` for long-running services only, then the application's own health answer and # its /actuator/info report of which capabilities are on and which database vendor it resolved, # compared with the lane's expectedCapabilitiesOn and expectedPersistenceVendor; # 4. every one-shot the lane declares, each of which must exit zero — a missing or skipped # required client is a lane failure, not a lane that had nothing to check; # 4a. no container of the lane restarted or was killed for memory — a container that died and came # back between the checks above would otherwise leave every one of them green; # 5. sanitized evidence; # 6. teardown scoped to this project alone, on success and failure alike, after the logs are # collected rather than before. # # CI calls this script. Inlining a subset of these commands into a workflow is how a lane ends up # running without its one-shots and reporting green. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CONTRACTS="${REPO_ROOT}/src/bootstrap/app/compose-lanes/compose-profile-contracts.json" EVIDENCE_ROOT="${REPO_ROOT}/src/bootstrap/app/build/evidence/runtime-smoke" RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" # Compose project names accept lowercase alphanumerics, hyphens and underscores only, so the run id # is lowercased for the project while the evidence directory keeps the readable timestamp. PROJECT_RUN_ID="$(tr '[:upper:]' '[:lower:]' <<<"${RUN_ID}")" # The lanes boot the qualification image: the application with every capability module on the # classpath (Dockerfile COMPOSITION, app-bootstrap qualificationBootJar). The default image is # the composition a service ships and carries none of them, so a lane that switches JPA or Kafka on # would have nothing to switch. compose.yaml passes this to the build and puts it in the tag. export COMPOSITION=qualification # Every capability is on while its module is present, so each lane starts from all of them off and # its activationEnv turns on what it is about. The off-* lanes therefore prove that a present but # switched-off capability needs no infrastructure at all. CAPABILITIES_OFF='{ "APP_PERSISTENCE_JPA_ENABLED": "false", "APP_REDIS_ENABLED": "false", "APP_MESSAGING_ENABLED": "false", "APP_PERSISTENCE_MONGO_ENABLED": "false", "APP_GRAPHQL_ENABLED": "false", "APP_HTTPCLIENT_ENABLED": "false", "APP_OBJECTSTORAGE_ENABLED": "false", "APP_WEBSOCKET_ENABLED": "false", "APP_GRPC_SERVER_ENABLED": "false", "APP_FILESERVER_PLATFORM_ENABLED": "false", "APP_NOTIFICATION_PLATFORM_ENABLED": "false" }' MODE="" SELECTED_LANE="" usage() { cat >&2 <<'USAGE' usage: run-compose-runtime-smoke.sh --matrix run every blocking lane, zero-skip run-compose-runtime-smoke.sh --lane reproduce one lane; not a matrix run --lane exists to reproduce a failure, and a green single lane is not evidence that the matrix passes. Only --matrix is. USAGE exit 64 } while [[ $# -gt 0 ]]; do case "$1" in --matrix) MODE="matrix"; CONTRACTS="$2"; shift 2 ;; --lane) MODE="lane"; SELECTED_LANE="$2"; shift 2 ;; *) usage ;; esac done [[ -n "${MODE}" ]] || usage for tool in jq docker; do command -v "${tool}" >/dev/null 2>&1 || { echo "${tool} is required" >&2; exit 78; } done [[ -r "${CONTRACTS}" ]] || { echo "missing ${CONTRACTS}" >&2; exit 78; } # The lane-only TLS overlay's keypair directory (src/bootstrap/app/compose-lanes/tls.yaml). LANE_POSTGRES_TLS="${REPO_ROOT}/src/bootstrap/app/compose-lanes/postgres/tls" # The user quay.io/keycloak/keycloak runs as (the image's USER), which reads the realm's client secret. KEYCLOAK_IMAGE_UID=1000 # ---- per-lane state, cleaned up by the trap --------------------------------- PROJECT="" LANE_TMP="" LANE_EVIDENCE="" LANE_ARGS=() cleanup_lane() { local status=$? if [[ -n "${PROJECT}" ]]; then # Logs first. A teardown that runs before the logs are collected destroys the only description # of why the lane failed. if [[ -n "${LANE_EVIDENCE}" ]]; then docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color >"${LANE_EVIDENCE}/compose.log" 2>&1 || true docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps --format json >"${LANE_EVIDENCE}/containers.json" 2>&1 || true redact_in_place "${LANE_EVIDENCE}/compose.log" # The warning summary belongs here, after the log it summarises exists. # # It used to run at the end of run_lane, which is before this trap collects compose.log — so it # grepped a file that had not been written yet and produced an empty warnings.log for every # lane. Fifteen lanes reported zero warnings while their logs held up to three each, and the # one artifact Wave 4's gate reads was the one that could not see them. grep -E ' (WARN|ERROR) ' "${LANE_EVIDENCE}/compose.log" 2>/dev/null \ >"${LANE_EVIDENCE}/warnings.log" || true fi # This project only. Never a bare `down`, which would take out whatever else the developer has # running, and never a volume outside it. docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true fi [[ -n "${LANE_TMP}" && -d "${LANE_TMP}" ]] && rm -rf "${LANE_TMP}" # .env.lane is gone — the lane's values live in the generated overlay now — but a file left by an # older revision of this script would still be read by an env_file list of an older stack. rm -f "${REPO_ROOT}/.env.lane" "${REPO_ROOT}/compose/keycloak/secrets/graphql-smoke-client-secret" # The lane keypair, including the CA's private half. A certificate that outlives the run it was # made for is a credential nobody is tracking. rm -f "${LANE_POSTGRES_TLS}"/ca.key "${LANE_POSTGRES_TLS}"/ca.crt \ "${LANE_POSTGRES_TLS}"/ca.srl "${LANE_POSTGRES_TLS}"/server.key \ "${LANE_POSTGRES_TLS}"/server.crt "${LANE_POSTGRES_TLS}"/server.csr \ "${REPO_ROOT}"/compose/mailpit/tls/server.key "${REPO_ROOT}"/compose/mailpit/tls/server.crt return "${status}" } # Secrets must not survive into an artifact. The generated values are known here, so they are # replaced by name rather than by guessing at what a secret looks like. redact_in_place() { local file="$1" [[ -r "${file}" ]] || return 0 local value for value in "${GENERATED_SECRETS[@]:-}"; do [[ -n "${value}" ]] && sed -i "s|${value}||g" "${file}" done } random_secret() { head -c 32 /dev/urandom | base64 | tr -d '=+/' | cut -c1-32; } # Key material, which is not the same thing as a password. The notification platform base64-decodes # each of its seven keys and requires at least 32 bytes, so random_secret's alphanumeric 32 # characters decode to 24 and are rejected. This keeps the padding and the full alphabet. random_key() { head -c 32 /dev/urandom | base64 | tr -d '\n'; } # Appends one line per container of the lane's project to restarts.txt, prefixed with the phase. # Fails when there is no container to inspect: an empty file used to pass the restart check, so a # lane whose project name drifted reported "no restarts" over containers it never looked at. The # callers run it as `record_restarts ... || return 1`, which turns `set -e` off inside, so a failed # inspect returns on its own line instead of counting a container whose line was never written. record_restarts() { local phase="$1" container inspected=0 while read -r container; do [[ -z "${container}" ]] && continue docker inspect --format "${phase} {{.Name}} restarts={{.RestartCount}} oomKilled={{.State.OOMKilled}}" \ "${container}" >>"${LANE_EVIDENCE}/restarts.txt" || return 1 inspected=$((inspected + 1)) done < <(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps -a -q) if [[ ${inspected} -eq 0 ]]; then echo "${id}: no container of project ${PROJECT} to inspect for restarts (${phase})" >&2 return 1 fi } run_lane() { local lane_json="$1" local id runtime profile id="$(jq -r '.id' <<<"${lane_json}")" runtime="$(jq -r '.springRuntime' <<<"${lane_json}")" PROJECT="casmoke-${id}-${PROJECT_RUN_ID}" LANE_EVIDENCE="${EVIDENCE_ROOT}/${id}/${RUN_ID}" if [[ -e "${LANE_EVIDENCE}" ]]; then echo "${id}: ${LANE_EVIDENCE} already exists; refusing to write into a previous run's evidence" >&2 return 1 fi mkdir -p "${LANE_EVIDENCE}" LANE_TMP="$(mktemp -d)" chmod 700 "${LANE_TMP}" LANE_ARGS=() local key file while read -r key; do file="$(jq -r --arg k "${key}" '.composeFiles[$k]' "${CONTRACTS}")" LANE_ARGS+=(-f "${REPO_ROOT}/${file}") done < <(jq -r '.files[]' <<<"${lane_json}") # The development stack's backing services carry capability profiles and the lane-only services # their own, so a lane enables several. The list is the lane's, never a guess from its name. while read -r profile; do [[ -n "${profile}" ]] && LANE_ARGS+=(--profile "${profile}") done < <(jq -r '.composeProfiles[]' <<<"${lane_json}") # A fresh client secret per run, mode 0600, removed on teardown. The realm references it by name. local secret_dir="${REPO_ROOT}/compose/keycloak/secrets" local secret_file="${secret_dir}/graphql-smoke-client-secret" mkdir -p "${secret_dir}" local client_secret; client_secret="$(random_secret)" GENERATED_SECRETS+=("${client_secret}") (umask 077 && printf '%s' "${client_secret}" >"${secret_file}") # Compose mounts the file with its owner and mode, and Keycloak reads it as its image user, uid # 1000. A developer with that uid owns the file already. A CI job runs as root, so the file is # handed to uid 1000 and stays 0600. Anyone else gets 0644, as the Mailpit key below does: the # value is random, lives for one run and sits in a Git-ignored directory removed on teardown. if [[ "$(id -u)" == 0 ]]; then chown "${KEYCLOAK_IMAGE_UID}:0" "${secret_file}" elif [[ "$(id -u)" != "${KEYCLOAK_IMAGE_UID}" ]]; then chmod 0644 "${secret_file}" fi # A CA and a server certificate for the host name `db`, when the lane brings the TLS overlay. # # Generated per run and removed on teardown, like the realm secret: a committed test certificate is # a private key in Git. The host name matters — the prod runtime connects with sslmode=verify-full, # which checks the certificate against the name it dialled, so a certificate for anything but `db` # fails exactly as a redirected connection would. That is the check, working. local tls_dir="${LANE_POSTGRES_TLS}" if jq -e '.files | index("tls")' <<<"${lane_json}" >/dev/null; then mkdir -p "${tls_dir}" ( umask 077 openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \ -keyout "${tls_dir}/ca.key" -out "${tls_dir}/ca.crt" \ -subj "/CN=ca-skeleton-lane-ca" >/dev/null 2>&1 openssl req -newkey rsa:2048 -nodes \ -keyout "${tls_dir}/server.key" -out "${tls_dir}/server.csr" \ -subj "/CN=db" >/dev/null 2>&1 openssl x509 -req -in "${tls_dir}/server.csr" -sha256 -days 1 \ -CA "${tls_dir}/ca.crt" -CAkey "${tls_dir}/ca.key" -CAcreateserial \ -extfile <(printf 'subjectAltName=DNS:db\nextendedKeyUsage=serverAuth\n') \ -out "${tls_dir}/server.crt" >/dev/null 2>&1 ) # The CA certificate is public and is mounted as a Compose secret, which preserves the source # file's mode; 0600 would be unreadable to the application's non-root user. chmod 0644 "${tls_dir}/ca.crt" rm -f "${tls_dir}/server.csr" [[ -s "${tls_dir}/server.crt" ]] || { echo "${id}: could not generate the lane certificate" >&2; return 1; } fi # A certificate for the host name `mailpit`, when the lane brings the reference SMTP relay. # # Same rule as the PostgreSQL one above and for the same reason: the notification platform's # transport type has no plaintext member, so a relay without a certificate is one the platform # cannot be configured to talk to. Generated per run, removed on teardown. local mail_tls="${REPO_ROOT}/compose/mailpit/tls" if jq -e '.services | index("mailpit")' <<<"${lane_json}" >/dev/null; then mkdir -p "${mail_tls}" ( umask 077 openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \ -keyout "${mail_tls}/server.key" -out "${mail_tls}/server.crt" \ -subj "/CN=mailpit" \ -addext "subjectAltName=DNS:mailpit" \ -addext "extendedKeyUsage=serverAuth" >/dev/null 2>&1 ) # Mailpit runs as a non-root user and reads both halves; 0600 would be unreadable to it. chmod 0644 "${mail_tls}/server.crt" "${mail_tls}/server.key" [[ -s "${mail_tls}/server.crt" ]] || { echo "${id}: could not generate the Mailpit certificate" >&2; return 1; } fi # The values a deployment must supply: the seven with no inline default, and the secrets the prod # env validator requires to be non-blank. A lane supplies them explicitly rather than inheriting a # developer's .env, which is the difference between a lane that reproduces anywhere and one # that reproduces on the machine it was written on. All generated per run and removed on teardown. # # The database credentials are additionally *exported*, because two different mechanisms have to # agree on them. The application reads this file; the `db` service takes its POSTGRES_USER and # POSTGRES_PASSWORD from Compose interpolation — `${APP_DATASOURCE_PASSWORD:-ca_skeleton}` — and # interpolation reads the process environment and the project .env file, never a service's # env_file. So the application got the generated password and the database got the literal # default, and no lane whose overlay did not happen to restate the value could ever authenticate. # shared-infra-local passed only because the local overlay (now compose.yaml) restated it, which made the dev lane's # failure look like a dev-specific problem rather than the general one it is. export POSTGRES_DB=ca_skeleton export APP_DATASOURCE_USERNAME=ca_skeleton export APP_DATASOURCE_PASSWORD="${client_secret}" # Which phase the notification client runs. Exported rather than defaulted in Compose, because a # default made local-notification-serving run the ingest assertions and pass without ever checking # a delivery. export NOTIFICATION_SMOKE_PHASE NOTIFICATION_SMOKE_PHASE="$(jq -r '.notificationSmokePhase // ""' <<<"${lane_json}")" # The capability schema streams this lane installs, exported for the same reason as the credentials # above: the migration containers take them through Compose interpolation, which reads the process # environment and never a lane overlay. They are deliberately not part of activationEnv — the # application does not read this value, the two one-shots that install and promote the schema do, # and putting an operator input in the application's environment block is how it would come to look # like a switch the application honours. export CAPABILITY_STREAMS CAPABILITY_STREAMS="$(jq -r '.capabilityStreams // [] | join(" ")' <<<"${lane_json}")" # The public health address, which is not the same in every runtime: application-local.yml pins # presentation.api-base-path to /api while the shipped default is /v1. The smoke client used to # hardcode /api, so it verified the local runtime and 404'd against a dev application that had # started perfectly. if [[ "${runtime}" == "local" ]]; then export APP_HEALTH_PATH=/api/healthcheck else export APP_HEALTH_PATH=/v1/healthcheck fi # The notification platform's seven purpose-scoped keys. # # All seven, always, for every lane — not only the notification ones. They cost nothing when the # capability is off, because the platform binds nothing at all then, and a lane that supplies only # the keys it currently needs is a lane that breaks the moment a switch is added to it. # # Distinct by construction: the platform refuses to start if two purposes carry the same material, # which is the check that stops one leaked key from being all seven. Each is redacted from the # evidence like every other generated value. local -a notification_keys=() local purpose for purpose in CONTACT_ENCRYPTION CONTACT_LOOKUP_HMAC CALLBACK_SIGNING PROVIDER_CREDENTIAL \ PAYLOAD_ENCRYPTION PROVIDER_REQUEST_LOOKUP_HMAC \ CALLBACK_FINGERPRINT_HMAC; do local value; value="$(random_key)" GENERATED_SECRETS+=("${value}") notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY=${value}") # The id, which is not secret and is deliberately tied to this run. A lane that reused a fixed # id across runs would be asserting the one thing the platform refuses to assume: that material # and id change together. notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY_ID=lane-${PROJECT_RUN_ID}-$(echo "${purpose}" | tr '[:upper:]_' '[:lower:]-')") done # Every deployment-supplied value the lane owns, collected here and written into the generated # overlay's `environment:` block below rather than into an env file. # # These lived in .env.lane, and an env file cannot win. env_file lists merge across overlays # and the later one takes precedence, so the developer's optional .env — declared by the local # overlay, after the base — silently replaced them. That is not hypothetical: it pinned the JWT # issuer to http://localhost:8081, which inside the app container is the application itself, so the # application fetched JWKS from its own port and answered AUTH_JWKS_UNAVAILABLE to every # authenticated request while the lane had supplied the correct issuer all along. # # An `environment:` block outranks every env_file regardless of order, which is the same fix the # lane's activation switches already use. The cost is that these values appear in `docker inspect` # for this run's own throwaway project; they are generated per run, removed on teardown, and # redacted from every evidence file by name. A lane whose settings are silently discarded is worse. local -a lane_environment=( "APP_NAME=ca-skeleton-${id}" # APP_DATASOURCE_URL is deliberately absent. The environment overlays own it and they do not # agree by accident: the local one is a plain jdbc:postgresql URL and the prod-smoke one carries # sslmode=verify-full and a CA path, because the prod profile refuses to start without them. # Setting it here overrode both — this overlay is appended last — and off-prod, shared-infra-dev # and prod-smoke failed the profile check on a TLS requirement the stack had satisfied. "APP_DATASOURCE_USERNAME=${APP_DATASOURCE_USERNAME}" "APP_DATASOURCE_PASSWORD=${APP_DATASOURCE_PASSWORD}" "APP_SECURITY_JWT_ISSUER=http://keycloak:8080/realms/ca-skeleton" "APP_SECURITY_JWT_AUDIENCE=ca-skeleton-api" "SPRING_PROFILES_ACTIVE=${runtime}" "${notification_keys[@]}" ) local generated for generated in APP_PRIVACY_PSEUDONYMIZATION_SALT; do local secret; secret="$(random_secret)" GENERATED_SECRETS+=("${secret}") lane_environment+=("${generated}=${secret}") done # The lane's own settings go into a generated overlay rather than into that file. # # env_file lists merge across overlays and the later file wins, so the developer's optional # .env — declared by the local overlay (now compose.yaml) — silently overrode the lane's values. # local-messaging started Kafka, set APP_MESSAGING_BROKER=kafka, and was refused by the dependency # validator for a broker it had supplied, because an empty value from a file nobody mentioned won # the merge. An `environment:` block beats every env_file regardless of order, so the lane's # settings stop depending on where a file happens to sit in the stack. # # The overlay adds no service, so the set the static verifier checks is unchanged. # Written as a function, because a handoff lane rewrites it between phases: the same project and # the same database, with the application recreated under a different activation environment. local lane_overlay="${LANE_TMP}/lane-overrides.yml" # The application's memory limit, when the lane needs more than the development stack's 512m. # Only the lane that boots every capability at once does (the contract's appMemoryLimit says why). local memory_limit memory_limit="$(jq -r '.appMemoryLimit // empty' <<<"${lane_json}")" write_lane_overlay() { local phase_override="${1:-{\}}" ( umask 077 { echo "services:" echo " app:" if [[ -n "${memory_limit}" ]]; then echo " mem_limit: ${memory_limit}" echo " memswap_limit: ${memory_limit}" fi echo " environment:" local entry for entry in "${lane_environment[@]}"; do printf ' %s: "%s"\n' "${entry%%=*}" "${entry#*=}" done jq -r --argjson off "${CAPABILITIES_OFF}" --argjson override "${phase_override}" \ '$off + .activationEnv + $override | to_entries[] | " \(.key): \"\(.value)\""' \ <<<"${lane_json}" } >"${lane_overlay}" ) } write_lane_overlay "$(jq -c '.handoff.firstPhaseEnv // {}' <<<"${lane_json}")" LANE_ARGS+=(-f "${lane_overlay}") GENERATED_SECRETS+=("${client_secret}") echo "== ${id} (project ${PROJECT}, runtime ${runtime})" # 2. render, then create. The static contract ran once for the whole run, before the first lane # (it renders every lane itself, so running it per lane rendered the matrix once per lane). ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" config >"${LANE_EVIDENCE}/rendered.yml" ) redact_in_place "${LANE_EVIDENCE}/rendered.yml" # Build before create. The app service declares both `build:` and `image:`, so Compose reuses a # matching tag if one is lying around — and the first run of this lane did exactly that, starting # a jar built from a different state of the repository and failing on a class that no longer # exists in the tree. A lane that runs a stale image produces evidence about code nobody changed. ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" build app \ >"${LANE_EVIDENCE}/build.log" 2>&1 ) ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" create >/dev/null ) # 3a. Anything that has to happen before the application starts, in the order the lane lists it. # # A capability schema stream is the case this exists for. Each stream keeps its own Flyway history # table and is promoted to ACTIVE deliberately after installation, and the application refuses to # start until that promotion has happened — so running it with the other one-shots, after `up`, # would run it after the thing it is a precondition for. The distinction is the lane's, not this # script's guess: preStartServices run here, oneShotServices run after the readiness poll. # # Read on fd 4 with stdin closed, for the reason the lane loop at the bottom of this file already # records: `docker compose run` consumes stdin, so a plain `while read` loop over several services # runs the first one and then finds its input exhausted. That is not hypothetical here — it is how # this lane ran its migration and silently skipped its promotion, and how shared-infra-local, # shared-infra-dev and prod-smoke each ran auth-smoke and skipped minio-init and # object-storage-smoke while reporting green. The evidence directories showed one log where the # contract named three. local pre_start while read -r -u 4 pre_start; do [[ -z "${pre_start}" ]] && continue echo "-- ${id}: ${pre_start} (before start)" if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${pre_start}" "${LANE_EVIDENCE}/${pre_start}.log" 2>&1; then redact_in_place "${LANE_EVIDENCE}/${pre_start}.log" echo "${id}: ${pre_start} did not succeed; see ${pre_start}.log" >&2 return 1 fi redact_in_place "${LANE_EVIDENCE}/${pre_start}.log" done 4< <(jq -r '.preStartServices[]?' <<<"${lane_json}") # 3. long-running services only. A one-shot is not a --wait target: it is meant to exit. local long_running backing long_running="$(jq -r '.longRunningServices[]' <<<"${lane_json}")" # The backing services first, until healthy, then the application. Started together, the # application met a Kafka broker that did not accept connections yet, its producer logged the # retries, and the silent-startup check below failed on a race rather than a defect (local-messaging, # main run 59, 2026-11-11; the same lane passed in the run before and on the desktop). backing="$(grep -vx app <<<"${long_running}" || true)" if [[ -n "${backing}" ]]; then # shellcheck disable=SC2086 ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" up --wait --wait-timeout 300 ${backing} ) fi # shellcheck disable=SC2086 ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" up --wait --wait-timeout 300 ${long_running} ) await_application_report() { # The application's own answer, not this script's input. # # The management port, not the application port. Actuator runs on its own connector here so the # management plane is not published on the public one, and fetching 8080/actuator returned an # empty file that looked exactly like a failed assertion. local management_port management_port="$(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \ sh -c 'echo "${MANAGEMENT_SERVER_PORT:-9001}"' 2>/dev/null | tr -d "\r\n")" management_port="${management_port:-9001}" # Poll rather than trust `up --wait`. Not every overlay defines a healthcheck — the dev one does # not — and where there is none `--wait` returns as soon as the container is created. local attempt answered="" for attempt in $(seq 1 60); do if docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \ wget -qO- "http://localhost:${management_port}/actuator/health" \ >"${LANE_EVIDENCE}/health.json" 2>/dev/null \ && [[ -s "${LANE_EVIDENCE}/health.json" ]]; then answered=yes break fi sleep 5 done if [[ -z "${answered}" ]]; then echo "${id}: the application never answered /actuator/health on port ${management_port}" >&2 return 1 fi # Which capabilities the application turned on, and the relational vendor it resolved. Without # this a lane called local-jpa that ran with JPA off would render the right services, start # cleanly, and prove nothing. Each capability module ships info.capabilities. beside its # switch (management.info.env), so the application must name every capability the contract # catalogues — true or false — and a name that is missing is a module that is missing. if ! docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \ wget -qO- "http://localhost:${management_port}/actuator/info" \ >"${LANE_EVIDENCE}/info.json" 2>/dev/null \ || ! jq -e 'type == "object"' "${LANE_EVIDENCE}/info.json" >/dev/null 2>&1; then echo "${id}: the application did not answer /actuator/info with a JSON object" >&2 return 1 fi local catalogued reported catalogued="$(jq -r '.capabilities | sort | join(",")' "${CONTRACTS}")" reported="$(jq -r '.capabilities // {} | keys | sort | join(",")' "${LANE_EVIDENCE}/info.json")" if [[ "${catalogued}" != "${reported}" ]]; then echo "${id}: the application reports capabilities [${reported}], the contract catalogues [${catalogued}]" >&2 return 1 fi local expected_on actual_on expected_on="$(jq -r '.expectedCapabilitiesOn | sort | join(",")' <<<"${lane_json}")" actual_on="$(jq -r '[.capabilities | to_entries[] | select((.value.enabled | tostring) == "true") | .key] | sort | join(",")' \ "${LANE_EVIDENCE}/info.json")" if [[ "${expected_on}" != "${actual_on}" ]]; then echo "${id}: the application reports [${actual_on}] on, the contract expects [${expected_on}]" >&2 return 1 fi local expected_vendor actual_vendor expected_vendor="$(jq -r '.expectedPersistenceVendor' <<<"${lane_json}")" actual_vendor="$(jq -r '.capabilities["persistence-jpa"] | if (.enabled | tostring) == "true" then .vendor else "none" end' \ "${LANE_EVIDENCE}/info.json")" if [[ "${expected_vendor}" != "${actual_vendor}" ]]; then echo "${id}: the application resolved vendor '${actual_vendor}', the contract expects '${expected_vendor}'" >&2 return 1 fi } await_application_report || return 1 # 3b. The handoff: one project, one database, the application recreated under the second phase. # # Not two lanes glued together. The evidence a handoff needs is that a request accepted while the # platform was INGEST_ONLY is delivered exactly once after a restart into SERVING — on the route # frozen at accept — and that is only evidence if the row is the same row. Tearing the volume down # between phases, or copying rows into a second project, proves nothing about a handoff. if jq -e '.handoff' <<<"${lane_json}" >/dev/null; then local first_smoke second_smoke first_smoke="$(jq -r '.handoff.firstPhaseSmoke' <<<"${lane_json}")" second_smoke="$(jq -r '.handoff.secondPhaseSmoke' <<<"${lane_json}")" echo "-- ${id}: ${first_smoke} (handoff phase 1)" if ! ( cd "${REPO_ROOT}" && NOTIFICATION_SMOKE_PHASE="${first_smoke}" \ docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm notification-smoke "${LANE_EVIDENCE}/notification-smoke-phase1.log" 2>&1; then redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log" echo "${id}: the first handoff phase did not succeed" >&2 return 1 fi redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log" cp "${LANE_EVIDENCE}/health.json" "${LANE_EVIDENCE}/health-phase1.json" cp "${LANE_EVIDENCE}/info.json" "${LANE_EVIDENCE}/info-phase1.json" # Recorded before the recreate: --force-recreate makes a new container whose restart count # starts at zero, so a kill during the first phase would otherwise leave no trace. record_restarts phase1 || return 1 # The application only. The database and its volume stay up, which is the whole point. echo "-- ${id}: restarting the application into the second phase" ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" stop app >/dev/null 2>&1 ) write_lane_overlay '{}' ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" \ up -d --force-recreate --wait --wait-timeout 300 app ) await_application_report || return 1 fi # 3c. The startup log carries no WARN and no ERROR. # # Wave 4's condition is about startup specifically, and a summary of the whole lane cannot express # it: the lanes that exercise authentication end with a deliberate malformed-token probe, and the # application answering it with a WARN is the security control working. Splitting at the line where # the application reports it has started is what makes "startup is silent" checkable instead of # argued — and it is checked here, per lane, rather than read off an artifact afterwards. # # StartupWarningZeroTest asserts the same property in-process for the all-off composition. This is # the same assertion against a real container with the lane's switches actually on. local startup_log="${LANE_EVIDENCE}/startup.log" docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color app \ >"${startup_log}" 2>&1 || true redact_in_place "${startup_log}" local startup_noise startup_noise="$(awk '/Started CaSkeletonApplication/{exit} /(WARN|ERROR)/{print}' "${startup_log}")" if [[ -n "${startup_noise}" ]]; then echo "${id}: the startup log is not silent:" >&2 echo "${startup_noise}" | head -20 >&2 return 1 fi # 4. every declared one-shot, each of which must exit zero # # fd 4 and a closed stdin, same reason as the pre-start loop above. And the count is checked rather # than trusted: this loop ran exactly one one-shot per lane for its whole life, so the three lanes # that declare three clients verified object storage in none of their runs and said so in neither # their output nor their exit status. A loop that silently does less than the contract asks is the # one failure a green lane cannot reveal, so the arithmetic is now part of the lane. local one_shot ran_one_shots=0 declared_one_shots declared_one_shots="$(jq -r '.oneShotServices // [] | length' <<<"${lane_json}")" while read -r -u 4 one_shot; do [[ -z "${one_shot}" ]] && continue echo "-- ${id}: ${one_shot}" if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${one_shot}" "${LANE_EVIDENCE}/${one_shot}.log" 2>&1; then redact_in_place "${LANE_EVIDENCE}/${one_shot}.log" echo "${id}: ${one_shot} did not exit zero; see ${one_shot}.log" >&2 return 1 fi redact_in_place "${LANE_EVIDENCE}/${one_shot}.log" ran_one_shots=$((ran_one_shots + 1)) done 4< <(jq -r '.oneShotServices[]?' <<<"${lane_json}") if [[ "${ran_one_shots}" -ne "${declared_one_shots}" ]]; then echo "${id}: ran ${ran_one_shots} of ${declared_one_shots} one-shots; a partial lane is not a pass" >&2 return 1 fi # 4a. No container of the lane restarted and none was killed for memory. # # The application runs with `restart: unless-stopped`, so a container the kernel killed for # exceeding its memory limit came back on its own. all-adapters did exactly that for a long time # and failed only when the kill happened to land on a smoke request; every other run reported # green over a container that had died and restarted under it (integration-r9, 2026-11-07). The # checks above all ask the application a question, and a restarted application answers them, so # the restart itself has to be the failure. The peak memory is recorded beside it, so a lane that # is close to its limit is visible before it crosses it. local restarted="" docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app cat /sys/fs/cgroup/memory.peak \ >"${LANE_EVIDENCE}/app-memory-peak-bytes.txt" 2>/dev/null || true record_restarts final || return 1 restarted="$(grep -v ' restarts=0 oomKilled=false$' "${LANE_EVIDENCE}/restarts.txt" || true)" if [[ -n "${restarted}" ]]; then echo "${id}: a container restarted or was killed for memory during the lane:" >&2 echo "${restarted}" >&2 return 1 fi # 5. the manifest ties the evidence to the exact stack that produced it jq -n \ --arg lane "${id}" --arg run "${RUN_ID}" --arg project "${PROJECT}" \ --arg runtime "${runtime}" \ --argjson profiles "$(jq '.composeProfiles' <<<"${lane_json}")" \ --argjson services "$(jq '.services' <<<"${lane_json}")" \ '{lane:$lane, runId:$run, project:$project, springRuntime:$runtime, composeProfiles:$profiles, services:$services}' \ >"${LANE_EVIDENCE}/manifest.json" echo "== ${id}: passed" } GENERATED_SECRETS=() FAILED_LANES=() # The static contract once for the run, in both modes: it renders every lane by itself, so running it # inside each lane rendered the whole matrix once per lane. mkdir -p "${EVIDENCE_ROOT}" CONTRACT_LOG="${EVIDENCE_ROOT}/contract-${RUN_ID}.log" "${REPO_ROOT}/scripts/verify-compose-profile-contracts.sh" >"${CONTRACT_LOG}" 2>&1 \ || { echo "run-compose-runtime-smoke: the static Compose contract failed; see ${CONTRACT_LOG}" >&2; exit 1; } if [[ "${MODE}" == "lane" ]]; then lane_json="$(jq -c --arg id "${SELECTED_LANE}" '.lanes[] | select(.id == $id)' "${CONTRACTS}")" [[ -n "${lane_json}" ]] || { echo "no lane '${SELECTED_LANE}' in ${CONTRACTS}" >&2; exit 64; } trap cleanup_lane EXIT run_lane "${lane_json}" echo "one lane reproduced. A green single lane is not evidence that the matrix passes." exit 0 fi blocking="$(jq -c '.lanes[] | select(.blocking == true)' "${CONTRACTS}")" total="$(wc -l <<<"${blocking}" | tr -d ' ')" echo "running ${total} blocking lane(s), zero-skip" # Read on fd 3, not stdin. `docker compose exec` and friends consume stdin, and inside a plain # `while read` loop they ate the remaining lanes — so the first lane ran, the loop ended, and the # script reported that all six had passed. A wrapper whose own success message is a false green is # worse than no wrapper, and nothing else here would have caught it: the count came from the # contract and the exit status from the one lane that did run. # # Each lane runs in a subshell with `set -e` on and its status read afterwards, never as an `if` # condition. Bash ignores `set -e` for every command run as part of an `if` condition, functions and # subshells included, so `if ( run_lane ... )` turned every step that relied on it into a step whose # failure was ignored. The image build was one: a build that failed left the lane to start the # previous run's image, and all eighteen lanes reported green on code the build had refused # (2026-11-07). --lane mode calls run_lane directly and was not affected. RAN_LANES=0 while read -r -u 3 lane_json; do [[ -z "${lane_json}" ]] && continue RAN_LANES=$((RAN_LANES + 1)) set +e ( set -e; trap cleanup_lane EXIT; run_lane "${lane_json}" ) lane_status=$? set -e if [[ ${lane_status} -ne 0 ]]; then FAILED_LANES+=("$(jq -r '.id' <<<"${lane_json}")") fi done 3<<<"${blocking}" if [[ "${RAN_LANES}" -ne "${total}" ]]; then echo "run-compose-runtime-smoke: ran ${RAN_LANES} of ${total} lanes; a partial matrix is not a pass" >&2 exit 1 fi if [[ ${#FAILED_LANES[@]} -gt 0 ]]; then echo "run-compose-runtime-smoke: ${#FAILED_LANES[@]} lane(s) failed: ${FAILED_LANES[*]}" >&2 exit 1 fi echo "run-compose-runtime-smoke: all ${total} blocking lanes passed; evidence under ${EVIDENCE_ROOT}"