13개 작업 묶음(P1–P13)의 W2 커밋을 합치고 전체 게이트로 검증했다. - 묶음별 정합성 수정(notification dispatch·recovery, object storage staged publication, fileserver 복구·정리, gRPC 플랫폼, Redis 분류·lease TTL, JPA outbox claim 순서·durable operation HTTP, web 요청 id·category 필수, graphql 에러 mapper, Mongo profile 바인딩, build 소스 보존 게이트·compose smoke info 확인) - 통합 때 드러난 3건 수정: outbox claim 테스트 fixture의 가짜 envelope hash, sample scan이 새 operation controller를 등록하던 문제, SpotBugs aux classpath 파일이 truncate되지 않아 로컬에서 ".jar.jar"로 실패하던 문제 - 현황표: 묶음 결과 반영, 소유자 결정 21–27, 후속 발견 결함 47행, 요약 갱신 - docs/capability-status.md 「제거된 코드」에 85행 추가 - 묶음 결과 파일·에이전트 지시·스크립트를 docs로 옮기고 desktop 재개 프롬프트 추가 검증: check, ci verifyDependencyLocks, mainCheck, architecture/public path/env keys/config contract, bootstrap·sample integration/system, build-logic·build-tools test, optional-platforms ci, qualificationBootJar 통과. integrationCheck는 이번 작업 전에도 실패하던 lane(Redis topology 모드 인자, httpclient failure injection, AWS 보호 lane)만 실패. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
604 lines
32 KiB
Bash
Executable File
604 lines
32 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The dynamic half of the Compose contract: the lanes actually run.
|
|
#
|
|
# One entry point, because the order below is what makes a lane's result mean anything and every
|
|
# step in it exists because skipping it produced a false green somewhere:
|
|
#
|
|
# 1. a unique project and a fresh evidence directory, so two runs cannot read each other's results
|
|
# and a stale directory cannot be mistaken for this run's;
|
|
# 2. the static contract, then `config`, then `create` — a stack that cannot render has no partial
|
|
# failure mode, and finding that out after `up` costs a teardown;
|
|
# 3. `up --wait` for long-running services only, then the application's own health answer and
|
|
# its /actuator/info report of which capabilities are on and which database vendor it resolved,
|
|
# compared with the lane's expectedCapabilitiesOn and expectedPersistenceVendor;
|
|
# 4. every one-shot the lane declares, each of which must exit zero — a missing or skipped
|
|
# required client is a lane failure, not a lane that had nothing to check;
|
|
# 5. sanitized evidence;
|
|
# 6. teardown scoped to this project alone, on success and failure alike, after the logs are
|
|
# collected rather than before.
|
|
#
|
|
# CI calls this script. Inlining a subset of these commands into a workflow is how a lane ends up
|
|
# running without its one-shots and reporting green.
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
CONTRACTS="${REPO_ROOT}/src/config/runtime/compose-profile-contracts.json"
|
|
EVIDENCE_ROOT="${REPO_ROOT}/src/app-bootstrap/build/evidence/runtime-smoke"
|
|
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
|
# Compose project names accept lowercase alphanumerics, hyphens and underscores only, so the run id
|
|
# is lowercased for the project while the evidence directory keeps the readable timestamp.
|
|
PROJECT_RUN_ID="$(tr '[:upper:]' '[:lower:]' <<<"${RUN_ID}")"
|
|
|
|
# The lanes boot the qualification image: the application with every capability module on the
|
|
# classpath (src/Dockerfile COMPOSITION, app-bootstrap qualificationBootJar). The default image is
|
|
# the composition a service ships and carries none of them, so a lane that switches JPA or Kafka on
|
|
# would have nothing to switch. docker-compose.yml passes this to the build and puts it in the tag.
|
|
export COMPOSITION=qualification
|
|
|
|
# Every capability is on while its module is present, so each lane starts from all of them off and
|
|
# its activationEnv turns on what it is about. The off-* lanes therefore prove that a present but
|
|
# switched-off capability needs no infrastructure at all.
|
|
CAPABILITIES_OFF='{
|
|
"APP_PERSISTENCE_JPA_ENABLED": "false",
|
|
"APP_REDIS_ENABLED": "false",
|
|
"APP_MESSAGING_ENABLED": "false",
|
|
"APP_PERSISTENCE_MONGO_ENABLED": "false",
|
|
"APP_GRAPHQL_ENABLED": "false",
|
|
"APP_HTTPCLIENT_ENABLED": "false",
|
|
"APP_OBJECTSTORAGE_ENABLED": "false",
|
|
"APP_WEBSOCKET_ENABLED": "false",
|
|
"CASKELETON_GRPC_ENABLED": "false",
|
|
"APP_FILESERVER_PLATFORM_ENABLED": "false",
|
|
"APP_NOTIFICATION_PLATFORM_ENABLED": "false"
|
|
}'
|
|
|
|
MODE=""
|
|
SELECTED_LANE=""
|
|
usage() {
|
|
cat >&2 <<'USAGE'
|
|
usage:
|
|
run-compose-runtime-smoke.sh --matrix <contracts.json> run every blocking lane, zero-skip
|
|
run-compose-runtime-smoke.sh --lane <id> reproduce one lane; not a matrix run
|
|
|
|
--lane exists to reproduce a failure, and a green single lane is not evidence that the matrix
|
|
passes. Only --matrix is.
|
|
USAGE
|
|
exit 64
|
|
}
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--matrix) MODE="matrix"; CONTRACTS="$2"; shift 2 ;;
|
|
--lane) MODE="lane"; SELECTED_LANE="$2"; shift 2 ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
[[ -n "${MODE}" ]] || usage
|
|
|
|
for tool in jq docker; do
|
|
command -v "${tool}" >/dev/null 2>&1 || { echo "${tool} is required" >&2; exit 78; }
|
|
done
|
|
[[ -r "${CONTRACTS}" ]] || { echo "missing ${CONTRACTS}" >&2; exit 78; }
|
|
|
|
# ---- per-lane state, cleaned up by the trap ---------------------------------
|
|
PROJECT=""
|
|
LANE_TMP=""
|
|
LANE_EVIDENCE=""
|
|
LANE_ARGS=()
|
|
|
|
cleanup_lane() {
|
|
local status=$?
|
|
if [[ -n "${PROJECT}" ]]; then
|
|
# Logs first. A teardown that runs before the logs are collected destroys the only description
|
|
# of why the lane failed.
|
|
if [[ -n "${LANE_EVIDENCE}" ]]; then
|
|
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color >"${LANE_EVIDENCE}/compose.log" 2>&1 || true
|
|
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps --format json >"${LANE_EVIDENCE}/containers.json" 2>&1 || true
|
|
redact_in_place "${LANE_EVIDENCE}/compose.log"
|
|
# The warning summary belongs here, after the log it summarises exists.
|
|
#
|
|
# It used to run at the end of run_lane, which is before this trap collects compose.log — so it
|
|
# grepped a file that had not been written yet and produced an empty warnings.log for every
|
|
# lane. Fifteen lanes reported zero warnings while their logs held up to three each, and the
|
|
# one artifact Wave 4's gate reads was the one that could not see them.
|
|
grep -E ' (WARN|ERROR) ' "${LANE_EVIDENCE}/compose.log" 2>/dev/null \
|
|
>"${LANE_EVIDENCE}/warnings.log" || true
|
|
fi
|
|
# This project only. Never a bare `down`, which would take out whatever else the developer has
|
|
# running, and never a volume outside it.
|
|
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
fi
|
|
[[ -n "${LANE_TMP}" && -d "${LANE_TMP}" ]] && rm -rf "${LANE_TMP}"
|
|
# .env.lane is gone — the lane's values live in the generated overlay now — but a file left by an
|
|
# older revision of this script would still be read by the local overlay's env_file list.
|
|
rm -f "${REPO_ROOT}/src/.env.lane" "${REPO_ROOT}/infra/keycloak/secrets/graphql-smoke-client-secret"
|
|
# The lane keypair, including the CA's private half. A certificate that outlives the run it was
|
|
# made for is a credential nobody is tracking.
|
|
rm -f "${REPO_ROOT}"/infra/postgres/tls/ca.key "${REPO_ROOT}"/infra/postgres/tls/ca.crt \
|
|
"${REPO_ROOT}"/infra/postgres/tls/ca.srl "${REPO_ROOT}"/infra/postgres/tls/server.key \
|
|
"${REPO_ROOT}"/infra/postgres/tls/server.crt "${REPO_ROOT}"/infra/postgres/tls/server.csr \
|
|
"${REPO_ROOT}"/infra/mailpit/tls/server.key "${REPO_ROOT}"/infra/mailpit/tls/server.crt
|
|
return "${status}"
|
|
}
|
|
|
|
# Secrets must not survive into an artifact. The generated values are known here, so they are
|
|
# replaced by name rather than by guessing at what a secret looks like.
|
|
redact_in_place() {
|
|
local file="$1"
|
|
[[ -r "${file}" ]] || return 0
|
|
local value
|
|
for value in "${GENERATED_SECRETS[@]:-}"; do
|
|
[[ -n "${value}" ]] && sed -i "s|${value}|<redacted>|g" "${file}"
|
|
done
|
|
}
|
|
|
|
random_secret() { head -c 32 /dev/urandom | base64 | tr -d '=+/' | cut -c1-32; }
|
|
|
|
# Key material, which is not the same thing as a password. The notification platform base64-decodes
|
|
# each of its seven keys and requires at least 32 bytes, so random_secret's alphanumeric 32
|
|
# characters decode to 24 and are rejected. This keeps the padding and the full alphabet.
|
|
random_key() { head -c 32 /dev/urandom | base64 | tr -d '\n'; }
|
|
|
|
run_lane() {
|
|
local lane_json="$1"
|
|
local id profile runtime
|
|
id="$(jq -r '.id' <<<"${lane_json}")"
|
|
profile="$(jq -r '.composeProfile // empty' <<<"${lane_json}")"
|
|
runtime="$(jq -r '.springRuntime' <<<"${lane_json}")"
|
|
|
|
PROJECT="casmoke-${id}-${PROJECT_RUN_ID}"
|
|
LANE_EVIDENCE="${EVIDENCE_ROOT}/${id}/${RUN_ID}"
|
|
if [[ -e "${LANE_EVIDENCE}" ]]; then
|
|
echo "${id}: ${LANE_EVIDENCE} already exists; refusing to write into a previous run's evidence" >&2
|
|
return 1
|
|
fi
|
|
mkdir -p "${LANE_EVIDENCE}"
|
|
LANE_TMP="$(mktemp -d)"
|
|
chmod 700 "${LANE_TMP}"
|
|
|
|
LANE_ARGS=()
|
|
local key file
|
|
while read -r key; do
|
|
file="$(jq -r --arg k "${key}" '.composeFiles[$k]' "${CONTRACTS}")"
|
|
LANE_ARGS+=(-f "${REPO_ROOT}/${file}")
|
|
done < <(jq -r '.files[]' <<<"${lane_json}")
|
|
[[ -n "${profile}" ]] && LANE_ARGS+=(--profile "${profile}")
|
|
|
|
# A fresh client secret per run, mode 0600, removed on teardown. The realm references it by name.
|
|
local secret_dir="${REPO_ROOT}/infra/keycloak/secrets"
|
|
local secret_file="${secret_dir}/graphql-smoke-client-secret"
|
|
mkdir -p "${secret_dir}"
|
|
local client_secret; client_secret="$(random_secret)"
|
|
GENERATED_SECRETS+=("${client_secret}")
|
|
(umask 077 && printf '%s' "${client_secret}" >"${secret_file}")
|
|
|
|
# A CA and a server certificate for the host name `db`, when the lane brings the TLS overlay.
|
|
#
|
|
# Generated per run and removed on teardown, like the realm secret: a committed test certificate is
|
|
# a private key in Git. The host name matters — the prod runtime connects with sslmode=verify-full,
|
|
# which checks the certificate against the name it dialled, so a certificate for anything but `db`
|
|
# fails exactly as a redirected connection would. That is the check, working.
|
|
local tls_dir="${REPO_ROOT}/infra/postgres/tls"
|
|
if jq -e '.files | index("tls")' <<<"${lane_json}" >/dev/null; then
|
|
mkdir -p "${tls_dir}"
|
|
(
|
|
umask 077
|
|
openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \
|
|
-keyout "${tls_dir}/ca.key" -out "${tls_dir}/ca.crt" \
|
|
-subj "/CN=ca-skeleton-lane-ca" >/dev/null 2>&1
|
|
openssl req -newkey rsa:2048 -nodes \
|
|
-keyout "${tls_dir}/server.key" -out "${tls_dir}/server.csr" \
|
|
-subj "/CN=db" >/dev/null 2>&1
|
|
openssl x509 -req -in "${tls_dir}/server.csr" -sha256 -days 1 \
|
|
-CA "${tls_dir}/ca.crt" -CAkey "${tls_dir}/ca.key" -CAcreateserial \
|
|
-extfile <(printf 'subjectAltName=DNS:db\nextendedKeyUsage=serverAuth\n') \
|
|
-out "${tls_dir}/server.crt" >/dev/null 2>&1
|
|
)
|
|
# The CA certificate is public and is mounted as a Compose secret, which preserves the source
|
|
# file's mode; 0600 would be unreadable to the application's non-root user.
|
|
chmod 0644 "${tls_dir}/ca.crt"
|
|
rm -f "${tls_dir}/server.csr"
|
|
[[ -s "${tls_dir}/server.crt" ]] || { echo "${id}: could not generate the lane certificate" >&2; return 1; }
|
|
fi
|
|
|
|
# A certificate for the host name `mailpit`, when the lane brings the reference SMTP relay.
|
|
#
|
|
# Same rule as the PostgreSQL one above and for the same reason: the notification platform's
|
|
# transport type has no plaintext member, so a relay without a certificate is one the platform
|
|
# cannot be configured to talk to. Generated per run, removed on teardown.
|
|
local mail_tls="${REPO_ROOT}/infra/mailpit/tls"
|
|
if jq -e '.services | index("mailpit")' <<<"${lane_json}" >/dev/null; then
|
|
mkdir -p "${mail_tls}"
|
|
(
|
|
umask 077
|
|
openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \
|
|
-keyout "${mail_tls}/server.key" -out "${mail_tls}/server.crt" \
|
|
-subj "/CN=mailpit" \
|
|
-addext "subjectAltName=DNS:mailpit" \
|
|
-addext "extendedKeyUsage=serverAuth" >/dev/null 2>&1
|
|
)
|
|
# Mailpit runs as a non-root user and reads both halves; 0600 would be unreadable to it.
|
|
chmod 0644 "${mail_tls}/server.crt" "${mail_tls}/server.key"
|
|
[[ -s "${mail_tls}/server.crt" ]] || { echo "${id}: could not generate the Mailpit certificate" >&2; return 1; }
|
|
fi
|
|
|
|
# The values a deployment must supply: the seven with no inline default, and the secrets the prod
|
|
# env validator requires to be non-blank. A lane supplies them explicitly rather than inheriting a
|
|
# developer's src/.env, which is the difference between a lane that reproduces anywhere and one
|
|
# that reproduces on the machine it was written on. All generated per run and removed on teardown.
|
|
#
|
|
# The database credentials are additionally *exported*, because two different mechanisms have to
|
|
# agree on them. The application reads this file; the `db` service takes its POSTGRES_USER and
|
|
# POSTGRES_PASSWORD from Compose interpolation — `${APP_DATASOURCE_PASSWORD:-ca_skeleton}` — and
|
|
# interpolation reads the process environment and the project .env file, never a service's
|
|
# env_file. So the application got the generated password and the database got the literal
|
|
# default, and no lane whose overlay did not happen to restate the value could ever authenticate.
|
|
# shared-infra-local passed only because the local overlay restates it, which made the dev lane's
|
|
# failure look like a dev-specific problem rather than the general one it is.
|
|
export POSTGRES_DB=ca_skeleton
|
|
export APP_DATASOURCE_USERNAME=ca_skeleton
|
|
export APP_DATASOURCE_PASSWORD="${client_secret}"
|
|
|
|
# Which phase the notification client runs. Exported rather than defaulted in Compose, because a
|
|
# default made local-notification-serving run the ingest assertions and pass without ever checking
|
|
# a delivery.
|
|
export NOTIFICATION_SMOKE_PHASE
|
|
NOTIFICATION_SMOKE_PHASE="$(jq -r '.notificationSmokePhase // ""' <<<"${lane_json}")"
|
|
|
|
# The capability schema streams this lane installs, exported for the same reason as the credentials
|
|
# above: the migration containers take them through Compose interpolation, which reads the process
|
|
# environment and never a lane overlay. They are deliberately not part of activationEnv — the
|
|
# application does not read this value, the two one-shots that install and promote the schema do,
|
|
# and putting an operator input in the application's environment block is how it would come to look
|
|
# like a switch the application honours.
|
|
export CAPABILITY_STREAMS
|
|
CAPABILITY_STREAMS="$(jq -r '.capabilityStreams // [] | join(" ")' <<<"${lane_json}")"
|
|
|
|
# The public health address, which is not the same in every runtime: application-local.yml pins
|
|
# presentation.api-base-path to /api while the shipped default is /v1. The smoke client used to
|
|
# hardcode /api, so it verified the local runtime and 404'd against a dev application that had
|
|
# started perfectly.
|
|
if [[ "${runtime}" == "local" ]]; then
|
|
export APP_HEALTH_PATH=/api/healthcheck
|
|
else
|
|
export APP_HEALTH_PATH=/v1/healthcheck
|
|
fi
|
|
|
|
# The notification platform's seven purpose-scoped keys.
|
|
#
|
|
# All seven, always, for every lane — not only the notification ones. They cost nothing when the
|
|
# capability is off, because the platform binds nothing at all then, and a lane that supplies only
|
|
# the keys it currently needs is a lane that breaks the moment a switch is added to it.
|
|
#
|
|
# Distinct by construction: the platform refuses to start if two purposes carry the same material,
|
|
# which is the check that stops one leaked key from being all seven. Each is redacted from the
|
|
# evidence like every other generated value.
|
|
local -a notification_keys=()
|
|
local purpose
|
|
for purpose in CONTACT_ENCRYPTION CONTACT_LOOKUP_HMAC CALLBACK_SIGNING PROVIDER_CREDENTIAL \
|
|
PAYLOAD_ENCRYPTION PROVIDER_REQUEST_LOOKUP_HMAC \
|
|
CALLBACK_FINGERPRINT_HMAC; do
|
|
local value; value="$(random_key)"
|
|
GENERATED_SECRETS+=("${value}")
|
|
notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY=${value}")
|
|
# The id, which is not secret and is deliberately tied to this run. A lane that reused a fixed
|
|
# id across runs would be asserting the one thing the platform refuses to assume: that material
|
|
# and id change together.
|
|
notification_keys+=("APP_NOTIFICATION_PLATFORM_${purpose}_KEY_ID=lane-${PROJECT_RUN_ID}-$(echo "${purpose}" | tr '[:upper:]_' '[:lower:]-')")
|
|
done
|
|
|
|
# Every deployment-supplied value the lane owns, collected here and written into the generated
|
|
# overlay's `environment:` block below rather than into an env file.
|
|
#
|
|
# These lived in src/.env.lane, and an env file cannot win. env_file lists merge across overlays
|
|
# and the later one takes precedence, so the developer's optional src/.env — declared by the local
|
|
# overlay, after the base — silently replaced them. That is not hypothetical: it pinned the JWT
|
|
# issuer to http://localhost:8081, which inside the app container is the application itself, so the
|
|
# application fetched JWKS from its own port and answered AUTH_JWKS_UNAVAILABLE to every
|
|
# authenticated request while the lane had supplied the correct issuer all along.
|
|
#
|
|
# An `environment:` block outranks every env_file regardless of order, which is the same fix the
|
|
# lane's activation switches already use. The cost is that these values appear in `docker inspect`
|
|
# for this run's own throwaway project; they are generated per run, removed on teardown, and
|
|
# redacted from every evidence file by name. A lane whose settings are silently discarded is worse.
|
|
local -a lane_environment=(
|
|
"APP_NAME=ca-skeleton-${id}"
|
|
# APP_DATASOURCE_URL is deliberately absent. The environment overlays own it and they do not
|
|
# agree by accident: the local one is a plain jdbc:postgresql URL and the prod-smoke one carries
|
|
# sslmode=verify-full and a CA path, because the prod profile refuses to start without them.
|
|
# Setting it here overrode both — this overlay is appended last — and off-prod, shared-infra-dev
|
|
# and prod-smoke failed the profile check on a TLS requirement the stack had satisfied.
|
|
"APP_DATASOURCE_USERNAME=${APP_DATASOURCE_USERNAME}"
|
|
"APP_DATASOURCE_PASSWORD=${APP_DATASOURCE_PASSWORD}"
|
|
"APP_SECURITY_JWT_ISSUER=http://keycloak:8080/realms/ca-skeleton"
|
|
"APP_SECURITY_JWT_AUDIENCE=ca-skeleton-api"
|
|
"SPRING_PROFILES_ACTIVE=${runtime}"
|
|
"${notification_keys[@]}"
|
|
)
|
|
local generated
|
|
for generated in APP_PRIVACY_PSEUDONYMIZATION_SALT; do
|
|
local secret; secret="$(random_secret)"
|
|
GENERATED_SECRETS+=("${secret}")
|
|
lane_environment+=("${generated}=${secret}")
|
|
done
|
|
# The lane's own settings go into a generated overlay rather than into that file.
|
|
#
|
|
# env_file lists merge across overlays and the later file wins, so the developer's optional
|
|
# src/.env — declared by the local overlay, after the base — silently overrode the lane's values.
|
|
# local-messaging started Kafka, set APP_MESSAGING_BROKER=kafka, and was refused by the dependency
|
|
# validator for a broker it had supplied, because an empty value from a file nobody mentioned won
|
|
# the merge. An `environment:` block beats every env_file regardless of order, so the lane's
|
|
# settings stop depending on where a file happens to sit in the stack.
|
|
#
|
|
# The overlay adds no service, so the set the static verifier checks is unchanged.
|
|
# Written as a function, because a handoff lane rewrites it between phases: the same project and
|
|
# the same database, with the application recreated under a different activation environment.
|
|
local lane_overlay="${LANE_TMP}/lane-overrides.yml"
|
|
write_lane_overlay() {
|
|
local phase_override="${1:-{\}}"
|
|
(
|
|
umask 077
|
|
{
|
|
echo "services:"
|
|
echo " app:"
|
|
echo " environment:"
|
|
local entry
|
|
for entry in "${lane_environment[@]}"; do
|
|
printf ' %s: "%s"\n' "${entry%%=*}" "${entry#*=}"
|
|
done
|
|
jq -r --argjson off "${CAPABILITIES_OFF}" --argjson override "${phase_override}" \
|
|
'$off + .activationEnv + $override | to_entries[] | " \(.key): \"\(.value)\""' \
|
|
<<<"${lane_json}"
|
|
} >"${lane_overlay}"
|
|
)
|
|
}
|
|
write_lane_overlay "$(jq -c '.handoff.firstPhaseEnv // {}' <<<"${lane_json}")"
|
|
LANE_ARGS+=(-f "${lane_overlay}")
|
|
GENERATED_SECRETS+=("${client_secret}")
|
|
|
|
echo "== ${id} (project ${PROJECT}, runtime ${runtime})"
|
|
|
|
# 2. static contract, then render, then create
|
|
"${REPO_ROOT}/scripts/verify-compose-profile-contracts.sh" >"${LANE_EVIDENCE}/contract.log" 2>&1
|
|
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" config >"${LANE_EVIDENCE}/rendered.yml" )
|
|
redact_in_place "${LANE_EVIDENCE}/rendered.yml"
|
|
# Build before create. The app service declares both `build:` and `image:`, so Compose reuses a
|
|
# matching tag if one is lying around — and the first run of this lane did exactly that, starting
|
|
# a jar built from a different state of the repository and failing on a class that no longer
|
|
# exists in the tree. A lane that runs a stale image produces evidence about code nobody changed.
|
|
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" build app \
|
|
>"${LANE_EVIDENCE}/build.log" 2>&1 )
|
|
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" create >/dev/null )
|
|
|
|
# 3a. Anything that has to happen before the application starts, in the order the lane lists it.
|
|
#
|
|
# A capability schema stream is the case this exists for. Each stream keeps its own Flyway history
|
|
# table and is promoted to ACTIVE deliberately after installation, and the application refuses to
|
|
# start until that promotion has happened — so running it with the other one-shots, after `up`,
|
|
# would run it after the thing it is a precondition for. The distinction is the lane's, not this
|
|
# script's guess: preStartServices run here, oneShotServices run after the readiness poll.
|
|
#
|
|
# Read on fd 4 with stdin closed, for the reason the lane loop at the bottom of this file already
|
|
# records: `docker compose run` consumes stdin, so a plain `while read` loop over several services
|
|
# runs the first one and then finds its input exhausted. That is not hypothetical here — it is how
|
|
# this lane ran its migration and silently skipped its promotion, and how shared-infra-local,
|
|
# shared-infra-dev and prod-smoke each ran auth-smoke and skipped minio-init and
|
|
# object-storage-smoke while reporting green. The evidence directories showed one log where the
|
|
# contract named three.
|
|
local pre_start
|
|
while read -r -u 4 pre_start; do
|
|
[[ -z "${pre_start}" ]] && continue
|
|
echo "-- ${id}: ${pre_start} (before start)"
|
|
if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${pre_start}" </dev/null ) \
|
|
>"${LANE_EVIDENCE}/${pre_start}.log" 2>&1; then
|
|
redact_in_place "${LANE_EVIDENCE}/${pre_start}.log"
|
|
echo "${id}: ${pre_start} did not succeed; see ${pre_start}.log" >&2
|
|
return 1
|
|
fi
|
|
redact_in_place "${LANE_EVIDENCE}/${pre_start}.log"
|
|
done 4< <(jq -r '.preStartServices[]?' <<<"${lane_json}")
|
|
|
|
# 3. long-running services only. A one-shot is not a --wait target: it is meant to exit.
|
|
local long_running
|
|
long_running="$(jq -r '.longRunningServices[]' <<<"${lane_json}")"
|
|
# shellcheck disable=SC2086
|
|
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" up --wait --wait-timeout 300 ${long_running} )
|
|
|
|
await_application_report() {
|
|
# The application's own answer, not this script's input.
|
|
#
|
|
# The management port, not the application port. Actuator runs on its own connector here so the
|
|
# management plane is not published on the public one, and fetching 8080/actuator returned an
|
|
# empty file that looked exactly like a failed assertion.
|
|
local management_port
|
|
management_port="$(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
|
|
sh -c 'echo "${MANAGEMENT_SERVER_PORT:-9001}"' 2>/dev/null | tr -d "\r\n")"
|
|
management_port="${management_port:-9001}"
|
|
# Poll rather than trust `up --wait`. Not every overlay defines a healthcheck — the dev one does
|
|
# not — and where there is none `--wait` returns as soon as the container is created.
|
|
local attempt answered=""
|
|
for attempt in $(seq 1 60); do
|
|
if docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
|
|
wget -qO- "http://localhost:${management_port}/actuator/health" \
|
|
>"${LANE_EVIDENCE}/health.json" 2>/dev/null \
|
|
&& [[ -s "${LANE_EVIDENCE}/health.json" ]]; then
|
|
answered=yes
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
if [[ -z "${answered}" ]]; then
|
|
echo "${id}: the application never answered /actuator/health on port ${management_port}" >&2
|
|
return 1
|
|
fi
|
|
|
|
# Which capabilities the application turned on, and the relational vendor it resolved. Without
|
|
# this a lane called local-jpa that ran with JPA off would render the right services, start
|
|
# cleanly, and prove nothing. Each capability module ships info.capabilities.<name> beside its
|
|
# switch (management.info.env), so the application must name every capability the contract
|
|
# catalogues — true or false — and a name that is missing is a module that is missing.
|
|
if ! docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app \
|
|
wget -qO- "http://localhost:${management_port}/actuator/info" \
|
|
>"${LANE_EVIDENCE}/info.json" 2>/dev/null \
|
|
|| ! jq -e 'type == "object"' "${LANE_EVIDENCE}/info.json" >/dev/null 2>&1; then
|
|
echo "${id}: the application did not answer /actuator/info with a JSON object" >&2
|
|
return 1
|
|
fi
|
|
local catalogued reported
|
|
catalogued="$(jq -r '.capabilities | sort | join(",")' "${CONTRACTS}")"
|
|
reported="$(jq -r '.capabilities // {} | keys | sort | join(",")' "${LANE_EVIDENCE}/info.json")"
|
|
if [[ "${catalogued}" != "${reported}" ]]; then
|
|
echo "${id}: the application reports capabilities [${reported}], the contract catalogues [${catalogued}]" >&2
|
|
return 1
|
|
fi
|
|
local expected_on actual_on
|
|
expected_on="$(jq -r '.expectedCapabilitiesOn | sort | join(",")' <<<"${lane_json}")"
|
|
actual_on="$(jq -r '[.capabilities | to_entries[] | select((.value.enabled | tostring) == "true") | .key] | sort | join(",")' \
|
|
"${LANE_EVIDENCE}/info.json")"
|
|
if [[ "${expected_on}" != "${actual_on}" ]]; then
|
|
echo "${id}: the application reports [${actual_on}] on, the contract expects [${expected_on}]" >&2
|
|
return 1
|
|
fi
|
|
local expected_vendor actual_vendor
|
|
expected_vendor="$(jq -r '.expectedPersistenceVendor' <<<"${lane_json}")"
|
|
actual_vendor="$(jq -r '.capabilities["persistence-jpa"] | if (.enabled | tostring) == "true" then .vendor else "none" end' \
|
|
"${LANE_EVIDENCE}/info.json")"
|
|
if [[ "${expected_vendor}" != "${actual_vendor}" ]]; then
|
|
echo "${id}: the application resolved vendor '${actual_vendor}', the contract expects '${expected_vendor}'" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
await_application_report || return 1
|
|
|
|
# 3b. The handoff: one project, one database, the application recreated under the second phase.
|
|
#
|
|
# Not two lanes glued together. The evidence a handoff needs is that a request accepted while the
|
|
# platform was INGEST_ONLY is delivered exactly once after a restart into SERVING — on the route
|
|
# frozen at accept — and that is only evidence if the row is the same row. Tearing the volume down
|
|
# between phases, or copying rows into a second project, proves nothing about a handoff.
|
|
if jq -e '.handoff' <<<"${lane_json}" >/dev/null; then
|
|
local first_smoke second_smoke
|
|
first_smoke="$(jq -r '.handoff.firstPhaseSmoke' <<<"${lane_json}")"
|
|
second_smoke="$(jq -r '.handoff.secondPhaseSmoke' <<<"${lane_json}")"
|
|
|
|
echo "-- ${id}: ${first_smoke} (handoff phase 1)"
|
|
if ! ( cd "${REPO_ROOT}" && NOTIFICATION_SMOKE_PHASE="${first_smoke}" \
|
|
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm notification-smoke </dev/null ) \
|
|
>"${LANE_EVIDENCE}/notification-smoke-phase1.log" 2>&1; then
|
|
redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log"
|
|
echo "${id}: the first handoff phase did not succeed" >&2
|
|
return 1
|
|
fi
|
|
redact_in_place "${LANE_EVIDENCE}/notification-smoke-phase1.log"
|
|
cp "${LANE_EVIDENCE}/health.json" "${LANE_EVIDENCE}/health-phase1.json"
|
|
cp "${LANE_EVIDENCE}/info.json" "${LANE_EVIDENCE}/info-phase1.json"
|
|
|
|
# The application only. The database and its volume stay up, which is the whole point.
|
|
echo "-- ${id}: restarting the application into the second phase"
|
|
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" stop app >/dev/null 2>&1 )
|
|
|
|
write_lane_overlay '{}'
|
|
( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" \
|
|
up -d --force-recreate --wait --wait-timeout 300 app )
|
|
await_application_report || return 1
|
|
fi
|
|
|
|
# 3c. The startup log carries no WARN and no ERROR.
|
|
#
|
|
# Wave 4's condition is about startup specifically, and a summary of the whole lane cannot express
|
|
# it: the lanes that exercise authentication end with a deliberate malformed-token probe, and the
|
|
# application answering it with a WARN is the security control working. Splitting at the line where
|
|
# the application reports it has started is what makes "startup is silent" checkable instead of
|
|
# argued — and it is checked here, per lane, rather than read off an artifact afterwards.
|
|
#
|
|
# StartupWarningZeroTest asserts the same property in-process for the all-off composition. This is
|
|
# the same assertion against a real container with the lane's switches actually on.
|
|
local startup_log="${LANE_EVIDENCE}/startup.log"
|
|
docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color app \
|
|
>"${startup_log}" 2>&1 || true
|
|
redact_in_place "${startup_log}"
|
|
local startup_noise
|
|
startup_noise="$(awk '/Started CaSkeletonApplication/{exit} /(WARN|ERROR)/{print}' "${startup_log}")"
|
|
if [[ -n "${startup_noise}" ]]; then
|
|
echo "${id}: the startup log is not silent:" >&2
|
|
echo "${startup_noise}" | head -20 >&2
|
|
return 1
|
|
fi
|
|
|
|
# 4. every declared one-shot, each of which must exit zero
|
|
#
|
|
# fd 4 and a closed stdin, same reason as the pre-start loop above. And the count is checked rather
|
|
# than trusted: this loop ran exactly one one-shot per lane for its whole life, so the three lanes
|
|
# that declare three clients verified object storage in none of their runs and said so in neither
|
|
# their output nor their exit status. A loop that silently does less than the contract asks is the
|
|
# one failure a green lane cannot reveal, so the arithmetic is now part of the lane.
|
|
local one_shot ran_one_shots=0 declared_one_shots
|
|
declared_one_shots="$(jq -r '.oneShotServices // [] | length' <<<"${lane_json}")"
|
|
while read -r -u 4 one_shot; do
|
|
[[ -z "${one_shot}" ]] && continue
|
|
echo "-- ${id}: ${one_shot}"
|
|
if ! ( cd "${REPO_ROOT}" && docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" run --rm "${one_shot}" </dev/null ) \
|
|
>"${LANE_EVIDENCE}/${one_shot}.log" 2>&1; then
|
|
redact_in_place "${LANE_EVIDENCE}/${one_shot}.log"
|
|
echo "${id}: ${one_shot} did not exit zero; see ${one_shot}.log" >&2
|
|
return 1
|
|
fi
|
|
redact_in_place "${LANE_EVIDENCE}/${one_shot}.log"
|
|
ran_one_shots=$((ran_one_shots + 1))
|
|
done 4< <(jq -r '.oneShotServices[]?' <<<"${lane_json}")
|
|
if [[ "${ran_one_shots}" -ne "${declared_one_shots}" ]]; then
|
|
echo "${id}: ran ${ran_one_shots} of ${declared_one_shots} one-shots; a partial lane is not a pass" >&2
|
|
return 1
|
|
fi
|
|
|
|
# 5. the manifest ties the evidence to the exact stack that produced it
|
|
jq -n \
|
|
--arg lane "${id}" --arg run "${RUN_ID}" --arg project "${PROJECT}" \
|
|
--arg runtime "${runtime}" --arg profile "${profile}" \
|
|
--argjson services "$(jq '.services' <<<"${lane_json}")" \
|
|
'{lane:$lane, runId:$run, project:$project, springRuntime:$runtime,
|
|
composeProfile:(if $profile=="" then null else $profile end), services:$services}' \
|
|
>"${LANE_EVIDENCE}/manifest.json"
|
|
|
|
echo "== ${id}: passed"
|
|
}
|
|
|
|
GENERATED_SECRETS=()
|
|
FAILED_LANES=()
|
|
|
|
if [[ "${MODE}" == "lane" ]]; then
|
|
lane_json="$(jq -c --arg id "${SELECTED_LANE}" '.lanes[] | select(.id == $id)' "${CONTRACTS}")"
|
|
[[ -n "${lane_json}" ]] || { echo "no lane '${SELECTED_LANE}' in ${CONTRACTS}" >&2; exit 64; }
|
|
trap cleanup_lane EXIT
|
|
run_lane "${lane_json}"
|
|
echo "one lane reproduced. A green single lane is not evidence that the matrix passes."
|
|
exit 0
|
|
fi
|
|
|
|
blocking="$(jq -c '.lanes[] | select(.blocking == true)' "${CONTRACTS}")"
|
|
total="$(wc -l <<<"${blocking}" | tr -d ' ')"
|
|
echo "running ${total} blocking lane(s), zero-skip"
|
|
# Read on fd 3, not stdin. `docker compose exec` and friends consume stdin, and inside a plain
|
|
# `while read` loop they ate the remaining lanes — so the first lane ran, the loop ended, and the
|
|
# script reported that all six had passed. A wrapper whose own success message is a false green is
|
|
# worse than no wrapper, and nothing else here would have caught it: the count came from the
|
|
# contract and the exit status from the one lane that did run.
|
|
RAN_LANES=0
|
|
while read -r -u 3 lane_json; do
|
|
[[ -z "${lane_json}" ]] && continue
|
|
RAN_LANES=$((RAN_LANES + 1))
|
|
if ( trap cleanup_lane EXIT; run_lane "${lane_json}" ); then :; else
|
|
FAILED_LANES+=("$(jq -r '.id' <<<"${lane_json}")")
|
|
fi
|
|
done 3<<<"${blocking}"
|
|
|
|
if [[ "${RAN_LANES}" -ne "${total}" ]]; then
|
|
echo "run-compose-runtime-smoke: ran ${RAN_LANES} of ${total} lanes; a partial matrix is not a pass" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ${#FAILED_LANES[@]} -gt 0 ]]; then
|
|
echo "run-compose-runtime-smoke: ${#FAILED_LANES[@]} lane(s) failed: ${FAILED_LANES[*]}" >&2
|
|
exit 1
|
|
fi
|
|
echo "run-compose-runtime-smoke: all ${total} blocking lanes passed; evidence under ${EVIDENCE_ROOT}"
|