Files
clean-architecture-backend-…/scripts/run-compose-runtime-smoke.sh
T
DongHyeonka 6325b5183f fix(compose): root로 도는 CI에서도 Keycloak이 smoke lane secret을 읽는다
문제: compose smoke는 실행마다 Keycloak client secret을 0600 파일로 만든다.
Compose는 파일을 소유자·권한 그대로 붙이고 Keycloak은 이미지 사용자 uid
1000으로 읽는다. runner job은 root라 파일이 root 소유가 되고, Keycloak
entrypoint가 읽지 못해 exit 78로 끝난다. auth profile을 쓰는 8 lane이
run 52·63에서 모두 이것으로 실패했다. desktop은 만든 사람이 uid 1000이라
통과했다.

고른 방법: root면 파일을 uid 1000에게 넘기고 0600을 지킨다. uid 1000도
root도 아니면 같은 스크립트의 Mailpit 키처럼 0644로 둔다(값은 실행마다
새로 만들고, Git이 무시하는 폴더에 있고, 끝나면 지운다).

위험·한계: Keycloak 이미지의 사용자가 1000이 아니게 바뀌면 상수를 고쳐야
한다. 이미지 태그(26.0)를 올릴 때 확인할 것으로 주석에 적었다.

검증: runner main 수동 실행의 compose-smoke(이 PR 본문에 적는다).
2026-11-12 01:16:57 +09:00

667 lines
36 KiB
Bash
Executable File

#!/usr/bin/env bash
# The dynamic half of the Compose contract: the lanes actually run.
#
# One entry point, because the order below is what makes a lane's result mean anything and every
# step in it exists because skipping it produced a false green somewhere:
#
# 1. a unique project and a fresh evidence directory, so two runs cannot read each other's results
# and a stale directory cannot be mistaken for this run's;
# 2. the static contract, then `config`, then `create` — a stack that cannot render has no partial
# failure mode, and finding that out after `up` costs a teardown;
# 3. `up --wait` for long-running services only, then the application's own health answer and
# its /actuator/info report of which capabilities are on and which database vendor it resolved,
# compared with the lane's expectedCapabilitiesOn and expectedPersistenceVendor;
# 4. every one-shot the lane declares, each of which must exit zero — a missing or skipped
# required client is a lane failure, not a lane that had nothing to check;
# 4a. no container of the lane restarted or was killed for memory — a container that died and came
# back between the checks above would otherwise leave every one of them green;
# 5. sanitized evidence;
# 6. teardown scoped to this project alone, on success and failure alike, after the logs are
# collected rather than before.
#
# CI calls this script. Inlining a subset of these commands into a workflow is how a lane ends up
# running without its one-shots and reporting green.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CONTRACTS="${REPO_ROOT}/src/bootstrap/app/compose-lanes/compose-profile-contracts.json"
EVIDENCE_ROOT="${REPO_ROOT}/src/bootstrap/app/build/evidence/runtime-smoke"
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
# Compose project names accept lowercase alphanumerics, hyphens and underscores only, so the run id
# is lowercased for the project while the evidence directory keeps the readable timestamp.
PROJECT_RUN_ID="$(tr '[:upper:]' '[:lower:]' <<<"${RUN_ID}")"
# The lanes boot the qualification image: the application with every capability module on the
# classpath (Dockerfile COMPOSITION, app-bootstrap qualificationBootJar). The default image is
# the composition a service ships and carries none of them, so a lane that switches JPA or Kafka on
# would have nothing to switch. compose.yaml passes this to the build and puts it in the tag.
export COMPOSITION=qualification
# Every capability is on while its module is present, so each lane starts from all of them off and
# its activationEnv turns on what it is about. The off-* lanes therefore prove that a present but
# switched-off capability needs no infrastructure at all.
CAPABILITIES_OFF='{
"APP_PERSISTENCE_JPA_ENABLED": "false",
"APP_REDIS_ENABLED": "false",
"APP_MESSAGING_ENABLED": "false",
"APP_PERSISTENCE_MONGO_ENABLED": "false",
"APP_GRAPHQL_ENABLED": "false",
"APP_HTTPCLIENT_ENABLED": "false",
"APP_OBJECTSTORAGE_ENABLED": "false",
"APP_WEBSOCKET_ENABLED": "false",
"APP_GRPC_SERVER_ENABLED": "false",
"APP_FILESERVER_PLATFORM_ENABLED": "false",
"APP_NOTIFICATION_PLATFORM_ENABLED": "false"
}'
MODE=""
SELECTED_LANE=""
usage() {
cat >&2 <<'USAGE'
usage:
run-compose-runtime-smoke.sh --matrix <contracts.json> run every blocking lane, zero-skip
run-compose-runtime-smoke.sh --lane <id> reproduce one lane; not a matrix run
--lane exists to reproduce a failure, and a green single lane is not evidence that the matrix
passes. Only --matrix is.
USAGE
exit 64
}
while [[ $# -gt 0 ]]; do
case "$1" in
--matrix) MODE="matrix"; CONTRACTS="$2"; shift 2 ;;
--lane) MODE="lane"; SELECTED_LANE="$2"; shift 2 ;;
*) usage ;;
esac
done
[[ -n "${MODE}" ]] || usage
for tool in jq docker; do
command -v "${tool}" >/dev/null 2>&1 || { echo "${tool} is required" >&2; exit 78; }
done
[[ -r "${CONTRACTS}" ]] || { echo "missing ${CONTRACTS}" >&2; exit 78; }
# The lane-only TLS overlay's keypair directory (src/bootstrap/app/compose-lanes/tls.yaml).
LANE_POSTGRES_TLS="${REPO_ROOT}/src/bootstrap/app/compose-lanes/postgres/tls"
# The user quay.io/keycloak/keycloak runs as (the image's USER), which reads the realm's client secret.
KEYCLOAK_IMAGE_UID=1000
# ---- per-lane state, cleaned up by the trap ---------------------------------
PROJECT=""
LANE_TMP=""
LANE_EVIDENCE=""
LANE_ARGS=()
cleanup_lane() {
local status=$?
if [[ -n "${PROJECT}" ]]; then
# Logs first. A teardown that runs before the logs are collected destroys the only description
# of why the lane failed.
if [[ -n "${LANE_EVIDENCE}" ]]; then
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color >"${LANE_EVIDENCE}/compose.log" 2>&1 || true
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps --format json >"${LANE_EVIDENCE}/containers.json" 2>&1 || true
redact_in_place "${LANE_EVIDENCE}/compose.log"
# The warning summary belongs here, after the log it summarises exists.
#
# It used to run at the end of run_lane, which is before this trap collects compose.log — so it
# grepped a file that had not been written yet and produced an empty warnings.log for every
# lane. Fifteen lanes reported zero warnings while their logs held up to three each, and the
# one artifact Wave 4's gate reads was the one that could not see them.
grep -E ' (WARN|ERROR) ' "${LANE_EVIDENCE}/compose.log" 2>/dev/null \
>"${LANE_EVIDENCE}/warnings.log" || true
fi
# This project only. Never a bare `down`, which would take out whatever else the developer has
# running, and never a volume outside it.
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
fi
[[ -n "${LANE_TMP}" && -d "${LANE_TMP}" ]] && rm -rf "${LANE_TMP}"
# .env.lane is gone — the lane's values live in the generated overlay now — but a file left by an
# older revision of this script would still be read by an env_file list of an older stack.
rm -f "${REPO_ROOT}/.env.lane" "${REPO_ROOT}/compose/keycloak/secrets/graphql-smoke-client-secret"
# The lane keypair, including the CA's private half. A certificate that outlives the run it was
# made for is a credential nobody is tracking.
rm -f "${LANE_POSTGRES_TLS}"/ca.key "${LANE_POSTGRES_TLS}"/ca.crt \
"${LANE_POSTGRES_TLS}"/ca.srl "${LANE_POSTGRES_TLS}"/server.key \
"${LANE_POSTGRES_TLS}"/server.crt "${LANE_POSTGRES_TLS}"/server.csr \
"${REPO_ROOT}"/compose/mailpit/tls/server.key "${REPO_ROOT}"/compose/mailpit/tls/server.crt
return "${status}"
}
# Secrets must not survive into an artifact. The generated values are known here, so they are
# replaced by name rather than by guessing at what a secret looks like.
redact_in_place() {
local file="$1"
[[ -r "${file}" ]] || return 0
local value
for value in "${GENERATED_SECRETS[@]:-}"; do
[[ -n "${value}" ]] && sed -i "s|${value}|<redacted>|g" "${file}"
done
}
random_secret() { head -c 32 /dev/urandom | base64 | tr -d '=+/' | cut -c1-32; }
# Key material, which is not the same thing as a password. The notification platform base64-decodes
# each of its seven keys and requires at least 32 bytes, so random_secret's alphanumeric 32
# characters decode to 24 and are rejected. This keeps the padding and the full alphabet.
random_key() { head -c 32 /dev/urandom | base64 | tr -d '\n'; }
run_lane() {
local lane_json="$1"
local id runtime profile
id="$(jq -r '.id' <<<"${lane_json}")"
runtime="$(jq -r '.springRuntime' <<<"${lane_json}")"
PROJECT="casmoke-${id}-${PROJECT_RUN_ID}"
LANE_EVIDENCE="${EVIDENCE_ROOT}/${id}/${RUN_ID}"
if [[ -e "${LANE_EVIDENCE}" ]]; then
echo "${id}: ${LANE_EVIDENCE} already exists; refusing to write into a previous run's evidence" >&2
return 1
fi
mkdir -p "${LANE_EVIDENCE}"
LANE_TMP="$(mktemp -d)"
chmod 700 "${LANE_TMP}"
LANE_ARGS=()
local key file
while read -r key; do
file="$(jq -r --arg k "${key}" '.composeFiles[$k]' "${CONTRACTS}")"
LANE_ARGS+=(-f "${REPO_ROOT}/${file}")
done < <(jq -r '.files[]' <<<"${lane_json}")
# The development stack's backing services carry capability profiles and the lane-only services
# their own, so a lane enables several. The list is the lane's, never a guess from its name.
while read -r profile; do
[[ -n "${profile}" ]] && LANE_ARGS+=(--profile "${profile}")
done < <(jq -r '.composeProfiles[]' <<<"${lane_json}")
# A fresh client secret per run, mode 0600, removed on teardown. The realm references it by name.
local secret_dir="${REPO_ROOT}/compose/keycloak/secrets"
local secret_file="${secret_dir}/graphql-smoke-client-secret"
mkdir -p "${secret_dir}"
local client_secret; client_secret="$(random_secret)"
GENERATED_SECRETS+=("${client_secret}")
(umask 077 && printf '%s' "${client_secret}" >"${secret_file}")
# Compose mounts the file with its owner and mode, and Keycloak reads it as its image user, uid
# 1000. A developer with that uid owns the file already. A CI job runs as root, so the file is
# handed to uid 1000 and stays 0600. Anyone else gets 0644, as the Mailpit key below does: the
# value is random, lives for one run and sits in a Git-ignored directory removed on teardown.
if [[ "$(id -u)" == 0 ]]; then
chown "${KEYCLOAK_IMAGE_UID}:0" "${secret_file}"
elif [[ "$(id -u)" != "${KEYCLOAK_IMAGE_UID}" ]]; then
chmod 0644 "${secret_file}"
fi
# A CA and a server certificate for the host name `db`, when the lane brings the TLS overlay.
#
# Generated per run and removed on teardown, like the realm secret: a committed test certificate is
# a private key in Git. The host name matters — the prod runtime connects with sslmode=verify-full,
# which checks the certificate against the name it dialled, so a certificate for anything but `db`
# fails exactly as a redirected connection would. That is the check, working.
local tls_dir="${LANE_POSTGRES_TLS}"
if jq -e '.files | index("tls")' <<<"${lane_json}" >/dev/null; then
mkdir -p "${tls_dir}"
(
umask 077
openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \
-keyout "${tls_dir}/ca.key" -out "${tls_dir}/ca.crt" \
-subj "/CN=ca-skeleton-lane-ca" >/dev/null 2>&1
openssl req -newkey rsa:2048 -nodes \
-keyout "${tls_dir}/server.key" -out "${tls_dir}/server.csr" \
-subj "/CN=db" >/dev/null 2>&1
openssl x509 -req -in "${tls_dir}/server.csr" -sha256 -days 1 \
-CA "${tls_dir}/ca.crt" -CAkey "${tls_dir}/ca.key" -CAcreateserial \
-extfile <(printf 'subjectAltName=DNS:db\nextendedKeyUsage=serverAuth\n') \
-out "${tls_dir}/server.crt" >/dev/null 2>&1
)
# The CA certificate is public and is mounted as a Compose secret, which preserves the source
# file's mode; 0600 would be unreadable to the application's non-root user.
chmod 0644 "${tls_dir}/ca.crt"
rm -f "${tls_dir}/server.csr"
[[ -s "${tls_dir}/server.crt" ]] || { echo "${id}: could not generate the lane certificate" >&2; return 1; }
fi
# A certificate for the host name `mailpit`, when the lane brings the reference SMTP relay.
#
# Same rule as the PostgreSQL one above and for the same reason: the notification platform's
# transport type has no plaintext member, so a relay without a certificate is one the platform
# cannot be configured to talk to. Generated per run, removed on teardown.
local mail_tls="${REPO_ROOT}/compose/mailpit/tls"
if jq -e '.services | index("mailpit")' <<<"${lane_json}" >/dev/null; then
mkdir -p "${mail_tls}"
(
umask 077
openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \
-keyout "${mail_tls}/server.key" -out "${mail_tls}/server.crt" \
-subj "/CN=mailpit" \
-addext "subjectAltName=DNS:mailpit" \
-addext "extendedKeyUsage=serverAuth" >/dev/null 2>&1
)
# Mailpit runs as a non-root user and reads both halves; 0600 would be unreadable to it.
chmod 0644 "${mail_tls}/server.crt" "${mail_tls}/server.key"
[[ -s "${mail_tls}/server.crt" ]] || { echo "${id}: could not generate the Mailpit certificate" >&2; return 1; }
fi
# The values a deployment must supply: the seven with no inline default, and the secrets the prod
# env validator requires to be non-blank. A lane supplies them explicitly rather than inheriting a
# developer's .env, which is the difference between a lane that reproduces anywhere and one
# that reproduces on the machine it was written on. All generated per run and removed on teardown.
#
# The database credentials are additionally *exported*, because two different mechanisms have to
# agree on them. The application reads this file; the `db` service takes its POSTGRES_USER and
# POSTGRES_PASSWORD from Compose interpolation — `${APP_DATASOURCE_PASSWORD:-ca_skeleton}` — and
# interpolation reads the process environment and the project .env file, never a service's
# env_file. So the application got the generated password and the database got the literal
# default, and no lane whose overlay did not happen to restate the value could ever authenticate.
# shared-infra-local passed only because the local overlay (now compose.yaml) restated it, which made the dev lane's
# failure look like a dev-specific problem rather than the general one it is.
export POSTGRES_DB=ca_skeleton
export APP_DATASOURCE_USERNAME=ca_skeleton
export APP_DATASOURCE_PASSWORD="${client_secret}"
# Which phase the notification client runs. Exported rather than defaulted in Compose, because a
# default made local-notification-serving run the ingest assertions and pass without ever checking
# a delivery.
export NOTIFICATION_SMOKE_PHASE
NOTIFICATION_SMOKE_PHASE="$(jq -r '.notificationSmokePhase // ""' <<<"${lane_json}")"
# The capability schema streams this lane installs, exported for the same reason as the credentials
# above: the migration containers take them through Compose interpolation, which reads the process
# environment and never a lane overlay. They are deliberately not part of activationEnv — the
# application does not read this value, the two one-shots that install and promote the schema do,
# and putting an operator input in the application's environment block is how it would come to look
# like a switch the application honours.
export CAPABILITY_STREAMS
CAPABILITY_STREAMS="$(jq -r '.capabilityStreams // [] | join(" ")' <<<"${lane_json}")"
# The public health address, which is not the same in every runtime: application-local.yml pins
# presentation.api-base-path to /api while the shipped default is /v1. The smoke client used to
# hardcode /api, so it verified the local runtime and 404'd against a dev application that had
# started perfectly.
if [[ "${runtime}" == "local" ]]; then
export APP_HEALTH_PATH=/api/healthcheck
else
export APP_HEALTH_PATH=/v1/healthcheck
fi
# The notification platform's seven purpose-scoped keys.
#
# All seven, always, for every lane — not only the notification ones. They cost nothing when the
# capability is off, because the platform binds nothing at all then, and a lane that supplies only
# the keys it currently needs is a lane that breaks the moment a switch is added to it.
#
# Distinct by construction: the platform refuses to start if two purposes carry the same material,
# which is the check that stops one leaked key from being all seven. Each is redacted from the
# evidence like every other generated value.
local -a notification_keys=()
local purpose
for purpose in CONTACT_ENCRYPTION CONTACT_LOOKUP_HMAC CALLBACK_SIGNING PROVIDER_CREDENTIAL \
PAYLOAD_ENCRYPTION PROVIDER_REQUEST_LOOKUP_HMAC \
CALLBACK_FINGERPRINT_HMAC; do
local value; value="$(random_key)"
GENERATED_SECRETS+=("${value}")
notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY=${value}")
# The id, which is not secret and is deliberately tied to this run. A lane that reused a fixed
# id across runs would be asserting the one thing the platform refuses to assume: that material
# and id change together.
notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY_ID=lane-${PROJECT_RUN_ID}-$(echo "${purpose}" | tr '[:upper:]_' '[:lower:]-')")
done
# Every deployment-supplied value the lane owns, collected here and written into the generated
# overlay's `environment:` block below rather than into an env file.
#
# These lived in .env.lane, and an env file cannot win. env_file lists merge across overlays
# and the later one takes precedence, so the developer's optional .env — declared by the local
# overlay, after the base — silently replaced them. That is not hypothetical: it pinned the JWT
# issuer to http://localhost:8081, which inside the app container is the application itself, so the
# application fetched JWKS from its own port and answered AUTH_JWKS_UNAVAILABLE to every
# authenticated request while the lane had supplied the correct issuer all along.
#
# An `environment:` block outranks every env_file regardless of order, which is the same fix the
# lane's activation switches already use. The cost is that these values appear in `docker inspect`
# for this run's own throwaway project; they are generated per run, removed on teardown, and
# redacted from every evidence file by name. A lane whose settings are silently discarded is worse.
local -a lane_environment=(
"APP_NAME=ca-skeleton-${id}"
# APP_DATASOURCE_URL is deliberately absent. The environment overlays own it and they do not
# agree by accident: the local one is a plain jdbc:postgresql URL and the prod-smoke one carries
# sslmode=verify-full and a CA path, because the prod profile refuses to start without them.
# Setting it here overrode both — this overlay is appended last — and off-prod, shared-infra-dev
# and prod-smoke failed the profile check on a TLS requirement the stack had satisfied.
"APP_DATASOURCE_USERNAME=${APP_DATASOURCE_USERNAME}"
"APP_DATASOURCE_PASSWORD=${APP_DATASOURCE_PASSWORD}"
"APP_SECURITY_JWT_ISSUER=http://keycloak:8080/realms/ca-skeleton"
"APP_SECURITY_JWT_AUDIENCE=ca-skeleton-api"
"SPRING_PROFILES_ACTIVE=${runtime}"
"${notification_keys[@]}"
)
local generated
for generated in APP_PRIVACY_PSEUDONYMIZATION_SALT; do
local secret; secret="$(random_secret)"
GENERATED_SECRETS+=("${secret}")
lane_environment+=("${generated}=${secret}")
done
# The lane's own settings go into a generated overlay rather than into that file.
#
# env_file lists merge across overlays and the later file wins, so the developer's optional
# .env — declared by the local overlay (now compose.yaml) — silently overrode the lane's values.
# local-messaging started Kafka, set APP_MESSAGING_BROKER=kafka, and was refused by the dependency
# validator for a broker it had supplied, because an empty value from a file nobody mentioned won
# the merge. An `environment:` block beats every env_file regardless of order, so the lane's
# settings stop depending on where a file happens to sit in the stack.
#
# The overlay adds no service, so the set the static verifier checks is unchanged.
# Written as a function, because a handoff lane rewrites it between phases: the same project and
# the same database, with the application recreated under a different activation environment.
local lane_overlay="${LANE_TMP}/lane-overrides.yml"
# The application's memory limit, when the lane needs more than the development stack's 512m.
# Only the lane that boots every capability at once does (the contract's appMemoryLimit says why).
local memory_limit
memory_limit="$(jq -r '.appMemoryLimit // empty' <<<"${lane_json}")"
write_lane_overlay() {
local phase_override="${1:-{\}}"
(
umask 077
{
echo "services:"
echo " app:"
if [[ -n "${memory_limit}" ]]; then
echo " mem_limit: ${memory_limit}"
echo " memswap_limit: ${memory_limit}"
fi
echo " environment:"
local entry
for entry in "${lane_environment[@]}"; do
printf ' %s: "%s"\n' "${entry%%=*}" "${entry#*=}"
done
jq -r --argjson off "${CAPABILITIES_OFF}" --argjson override "${phase_override}" \
'$off + .activationEnv + $override | to_entries[] | " \(.key): \"\(.value)\""' \
<<<"${lane_json}"
} >"${lane_overlay}"
)
}
write_lane_overlay "$(jq -c '.handoff.firstPhaseEnv // {}' <<<"${lane_json}")"
LANE_ARGS+=(-f "${lane_overlay}")
GENERATED_SECRETS+=("${client_secret}")
echo "== ${id} (project ${PROJECT}, runtime ${runtime})"
# 2. static contract, then render, then create
"${REPO_ROOT}/scripts/verify-compose-profile-contracts.sh" >"${LANE_EVIDENCE}/contract.log" 2>&1
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" config >"${LANE_EVIDENCE}/rendered.yml" )
redact_in_place "${LANE_EVIDENCE}/rendered.yml"
# Build before create. The app service declares both `build:` and `image:`, so Compose reuses a
# matching tag if one is lying around — and the first run of this lane did exactly that, starting
# a jar built from a different state of the repository and failing on a class that no longer
# exists in the tree. A lane that runs a stale image produces evidence about code nobody changed.
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" build app \
>"${LANE_EVIDENCE}/build.log" 2>&1 )
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" create >/dev/null )
# 3a. Anything that has to happen before the application starts, in the order the lane lists it.
#
# A capability schema stream is the case this exists for. Each stream keeps its own Flyway history
# table and is promoted to ACTIVE deliberately after installation, and the application refuses to
# start until that promotion has happened — so running it with the other one-shots, after `up`,
# would run it after the thing it is a precondition for. The distinction is the lane's, not this
# script's guess: preStartServices run here, oneShotServices run after the readiness poll.
#
# Read on fd 4 with stdin closed, for the reason the lane loop at the bottom of this file already
# records: `docker compose run` consumes stdin, so a plain `while read` loop over several services
# runs the first one and then finds its input exhausted. That is not hypothetical here — it is how
# this lane ran its migration and silently skipped its promotion, and how shared-infra-local,
# shared-infra-dev and prod-smoke each ran auth-smoke and skipped minio-init and
# object-storage-smoke while reporting green. The evidence directories showed one log where the
# contract named three.
local pre_start
while read -r -u 4 pre_start; do
[[ -z "${pre_start}" ]] && continue
echo "-- ${id}: ${pre_start} (before start)"
if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${pre_start}" </dev/null ) \
>"${LANE_EVIDENCE}/${pre_start}.log" 2>&1; then
redact_in_place "${LANE_EVIDENCE}/${pre_start}.log"
echo "${id}: ${pre_start} did not succeed; see ${pre_start}.log" >&2
return 1
fi
redact_in_place "${LANE_EVIDENCE}/${pre_start}.log"
done 4< <(jq -r '.preStartServices[]?' <<<"${lane_json}")
# 3. long-running services only. A one-shot is not a --wait target: it is meant to exit.
local long_running
long_running="$(jq -r '.longRunningServices[]' <<<"${lane_json}")"
# shellcheck disable=SC2086
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" up --wait --wait-timeout 300 ${long_running} )
await_application_report() {
# The application's own answer, not this script's input.
#
# The management port, not the application port. Actuator runs on its own connector here so the
# management plane is not published on the public one, and fetching 8080/actuator returned an
# empty file that looked exactly like a failed assertion.
local management_port
management_port="$(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
sh -c 'echo "${MANAGEMENT_SERVER_PORT:-9001}"' 2>/dev/null | tr -d "\r\n")"
management_port="${management_port:-9001}"
# Poll rather than trust `up --wait`. Not every overlay defines a healthcheck — the dev one does
# not — and where there is none `--wait` returns as soon as the container is created.
local attempt answered=""
for attempt in $(seq 1 60); do
if docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
wget -qO- "http://localhost:${management_port}/actuator/health" \
>"${LANE_EVIDENCE}/health.json" 2>/dev/null \
&& [[ -s "${LANE_EVIDENCE}/health.json" ]]; then
answered=yes
break
fi
sleep 5
done
if [[ -z "${answered}" ]]; then
echo "${id}: the application never answered /actuator/health on port ${management_port}" >&2
return 1
fi
# Which capabilities the application turned on, and the relational vendor it resolved. Without
# this a lane called local-jpa that ran with JPA off would render the right services, start
# cleanly, and prove nothing. Each capability module ships info.capabilities.<name> beside its
# switch (management.info.env), so the application must name every capability the contract
# catalogues — true or false — and a name that is missing is a module that is missing.
if ! docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
wget -qO- "http://localhost:${management_port}/actuator/info" \
>"${LANE_EVIDENCE}/info.json" 2>/dev/null \
|| ! jq -e 'type == "object"' "${LANE_EVIDENCE}/info.json" >/dev/null 2>&1; then
echo "${id}: the application did not answer /actuator/info with a JSON object" >&2
return 1
fi
local catalogued reported
catalogued="$(jq -r '.capabilities | sort | join(",")' "${CONTRACTS}")"
reported="$(jq -r '.capabilities // {} | keys | sort | join(",")' "${LANE_EVIDENCE}/info.json")"
if [[ "${catalogued}" != "${reported}" ]]; then
echo "${id}: the application reports capabilities [${reported}], the contract catalogues [${catalogued}]" >&2
return 1
fi
local expected_on actual_on
expected_on="$(jq -r '.expectedCapabilitiesOn | sort | join(",")' <<<"${lane_json}")"
actual_on="$(jq -r '[.capabilities | to_entries[] | select((.value.enabled | tostring) == "true") | .key] | sort | join(",")' \
"${LANE_EVIDENCE}/info.json")"
if [[ "${expected_on}" != "${actual_on}" ]]; then
echo "${id}: the application reports [${actual_on}] on, the contract expects [${expected_on}]" >&2
return 1
fi
local expected_vendor actual_vendor
expected_vendor="$(jq -r '.expectedPersistenceVendor' <<<"${lane_json}")"
actual_vendor="$(jq -r '.capabilities["persistence-jpa"] | if (.enabled | tostring) == "true" then .vendor else "none" end' \
"${LANE_EVIDENCE}/info.json")"
if [[ "${expected_vendor}" != "${actual_vendor}" ]]; then
echo "${id}: the application resolved vendor '${actual_vendor}', the contract expects '${expected_vendor}'" >&2
return 1
fi
}
await_application_report || return 1
# 3b. The handoff: one project, one database, the application recreated under the second phase.
#
# Not two lanes glued together. The evidence a handoff needs is that a request accepted while the
# platform was INGEST_ONLY is delivered exactly once after a restart into SERVING — on the route
# frozen at accept — and that is only evidence if the row is the same row. Tearing the volume down
# between phases, or copying rows into a second project, proves nothing about a handoff.
if jq -e '.handoff' <<<"${lane_json}" >/dev/null; then
local first_smoke second_smoke
first_smoke="$(jq -r '.handoff.firstPhaseSmoke' <<<"${lane_json}")"
second_smoke="$(jq -r '.handoff.secondPhaseSmoke' <<<"${lane_json}")"
echo "-- ${id}: ${first_smoke} (handoff phase 1)"
if ! ( cd "${REPO_ROOT}" && NOTIFICATION_SMOKE_PHASE="${first_smoke}" \
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm notification-smoke </dev/null ) \
>"${LANE_EVIDENCE}/notification-smoke-phase1.log" 2>&1; then
redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log"
echo "${id}: the first handoff phase did not succeed" >&2
return 1
fi
redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log"
cp "${LANE_EVIDENCE}/health.json" "${LANE_EVIDENCE}/health-phase1.json"
cp "${LANE_EVIDENCE}/info.json" "${LANE_EVIDENCE}/info-phase1.json"
# The application only. The database and its volume stay up, which is the whole point.
echo "-- ${id}: restarting the application into the second phase"
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" stop app >/dev/null 2>&1 )
write_lane_overlay '{}'
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" \
up -d --force-recreate --wait --wait-timeout 300 app )
await_application_report || return 1
fi
# 3c. The startup log carries no WARN and no ERROR.
#
# Wave 4's condition is about startup specifically, and a summary of the whole lane cannot express
# it: the lanes that exercise authentication end with a deliberate malformed-token probe, and the
# application answering it with a WARN is the security control working. Splitting at the line where
# the application reports it has started is what makes "startup is silent" checkable instead of
# argued — and it is checked here, per lane, rather than read off an artifact afterwards.
#
# StartupWarningZeroTest asserts the same property in-process for the all-off composition. This is
# the same assertion against a real container with the lane's switches actually on.
local startup_log="${LANE_EVIDENCE}/startup.log"
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color app \
>"${startup_log}" 2>&1 || true
redact_in_place "${startup_log}"
local startup_noise
startup_noise="$(awk '/Started CaSkeletonApplication/{exit} /(WARN|ERROR)/{print}' "${startup_log}")"
if [[ -n "${startup_noise}" ]]; then
echo "${id}: the startup log is not silent:" >&2
echo "${startup_noise}" | head -20 >&2
return 1
fi
# 4. every declared one-shot, each of which must exit zero
#
# fd 4 and a closed stdin, same reason as the pre-start loop above. And the count is checked rather
# than trusted: this loop ran exactly one one-shot per lane for its whole life, so the three lanes
# that declare three clients verified object storage in none of their runs and said so in neither
# their output nor their exit status. A loop that silently does less than the contract asks is the
# one failure a green lane cannot reveal, so the arithmetic is now part of the lane.
local one_shot ran_one_shots=0 declared_one_shots
declared_one_shots="$(jq -r '.oneShotServices // [] | length' <<<"${lane_json}")"
while read -r -u 4 one_shot; do
[[ -z "${one_shot}" ]] && continue
echo "-- ${id}: ${one_shot}"
if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${one_shot}" </dev/null ) \
>"${LANE_EVIDENCE}/${one_shot}.log" 2>&1; then
redact_in_place "${LANE_EVIDENCE}/${one_shot}.log"
echo "${id}: ${one_shot} did not exit zero; see ${one_shot}.log" >&2
return 1
fi
redact_in_place "${LANE_EVIDENCE}/${one_shot}.log"
ran_one_shots=$((ran_one_shots + 1))
done 4< <(jq -r '.oneShotServices[]?' <<<"${lane_json}")
if [[ "${ran_one_shots}" -ne "${declared_one_shots}" ]]; then
echo "${id}: ran ${ran_one_shots} of ${declared_one_shots} one-shots; a partial lane is not a pass" >&2
return 1
fi
# 4a. No container of the lane restarted and none was killed for memory.
#
# The application runs with `restart: unless-stopped`, so a container the kernel killed for
# exceeding its memory limit came back on its own. all-adapters did exactly that for a long time
# and failed only when the kill happened to land on a smoke request; every other run reported
# green over a container that had died and restarted under it (integration-r9, 2026-11-07). The
# checks above all ask the application a question, and a restarted application answers them, so
# the restart itself has to be the failure. The peak memory is recorded beside it, so a lane that
# is close to its limit is visible before it crosses it.
local container restarted=""
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app cat /sys/fs/cgroup/memory.peak \
>"${LANE_EVIDENCE}/app-memory-peak-bytes.txt" 2>/dev/null || true
while read -r container; do
[[ -z "${container}" ]] && continue
docker inspect --format '{{.Name}} restarts={{.RestartCount}} oomKilled={{.State.OOMKilled}}' \
"${container}" >>"${LANE_EVIDENCE}/restarts.txt"
done < <(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps -a -q)
restarted="$(grep -v ' restarts=0 oomKilled=false$' "${LANE_EVIDENCE}/restarts.txt" || true)"
if [[ -n "${restarted}" ]]; then
echo "${id}: a container restarted or was killed for memory during the lane:" >&2
echo "${restarted}" >&2
return 1
fi
# 5. the manifest ties the evidence to the exact stack that produced it
jq -n \
--arg lane "${id}" --arg run "${RUN_ID}" --arg project "${PROJECT}" \
--arg runtime "${runtime}" \
--argjson profiles "$(jq '.composeProfiles' <<<"${lane_json}")" \
--argjson services "$(jq '.services' <<<"${lane_json}")" \
'{lane:$lane, runId:$run, project:$project, springRuntime:$runtime,
composeProfiles:$profiles, services:$services}' \
>"${LANE_EVIDENCE}/manifest.json"
echo "== ${id}: passed"
}
GENERATED_SECRETS=()
FAILED_LANES=()
if [[ "${MODE}" == "lane" ]]; then
lane_json="$(jq -c --arg id "${SELECTED_LANE}" '.lanes[] | select(.id == $id)' "${CONTRACTS}")"
[[ -n "${lane_json}" ]] || { echo "no lane '${SELECTED_LANE}' in ${CONTRACTS}" >&2; exit 64; }
trap cleanup_lane EXIT
run_lane "${lane_json}"
echo "one lane reproduced. A green single lane is not evidence that the matrix passes."
exit 0
fi
blocking="$(jq -c '.lanes[] | select(.blocking == true)' "${CONTRACTS}")"
total="$(wc -l <<<"${blocking}" | tr -d ' ')"
echo "running ${total} blocking lane(s), zero-skip"
# Read on fd 3, not stdin. `docker compose exec` and friends consume stdin, and inside a plain
# `while read` loop they ate the remaining lanes — so the first lane ran, the loop ended, and the
# script reported that all six had passed. A wrapper whose own success message is a false green is
# worse than no wrapper, and nothing else here would have caught it: the count came from the
# contract and the exit status from the one lane that did run.
#
# Each lane runs in a subshell with `set -e` on and its status read afterwards, never as an `if`
# condition. Bash ignores `set -e` for every command run as part of an `if` condition, functions and
# subshells included, so `if ( run_lane ... )` turned every step that relied on it into a step whose
# failure was ignored. The image build was one: a build that failed left the lane to start the
# previous run's image, and all eighteen lanes reported green on code the build had refused
# (2026-11-07). --lane mode calls run_lane directly and was not affected.
RAN_LANES=0
while read -r -u 3 lane_json; do
[[ -z "${lane_json}" ]] && continue
RAN_LANES=$((RAN_LANES + 1))
set +e
( set -e; trap cleanup_lane EXIT; run_lane "${lane_json}" )
lane_status=$?
set -e
if [[ ${lane_status} -ne 0 ]]; then
FAILED_LANES+=("$(jq -r '.id' <<<"${lane_json}")")
fi
done 3<<<"${blocking}"
if [[ "${RAN_LANES}" -ne "${total}" ]]; then
echo "run-compose-runtime-smoke: ran ${RAN_LANES} of ${total} lanes; a partial matrix is not a pass" >&2
exit 1
fi
if [[ ${#FAILED_LANES[@]} -gt 0 ]]; then
echo "run-compose-runtime-smoke: ${#FAILED_LANES[@]} lane(s) failed: ${FAILED_LANES[*]}" >&2
exit 1
fi
echo "run-compose-runtime-smoke: all ${total} blocking lanes passed; evidence under ${EVIDENCE_ROOT}"