diff --git a/scripts/compose-smoke-verdict.py b/scripts/compose-smoke-verdict.py new file mode 100644 index 00000000..67c28769 --- /dev/null +++ b/scripts/compose-smoke-verdict.py @@ -0,0 +1,142 @@ +"""Judges the evidence a Compose runtime smoke lane leaves behind. + +Called by scripts/run-compose-runtime-smoke.sh, which keeps the docker calls and writes their output +to files; this script reads those files and says pass or fail. The judgements live here rather than +in the shell because each of them once passed on input that should have failed it, and a shell +branch cannot be tried on a broken input without starting a lane (scripts/tests has the cases): + +- startup-log: the application's log up to the line where it reports it has started carries no WARN + and no ERROR. A log without that line fails: an empty log, or one that `docker compose logs` never + wrote, has no WARN in it either, and passed as silent. +- restarts: every phase of the lane recorded at least one container, and none of them restarted or + was killed for memory. A phase with no line at all is a `ps` that found nothing, which left the + file empty and the check green (2026-11-13). +- matrix: every blocking lane reported exactly one status and every status is zero. A lane missing + from the list ran nothing. What this cannot see is a status the shell turned into zero before + writing it — all eighteen lanes reported green on code the build had refused because `set -e` was + off inside `if ( run_lane )` (2026-11-07) — so the shell reads each lane's status outside any + condition, and scripts/tests checks that it writes that status and not another. + +Exit status 0 is a pass, 1 a failure with its reasons on stderr, 2 a call this script does not know. + + python3 scripts/compose-smoke-verdict.py [--label ] startup-log + python3 scripts/compose-smoke-verdict.py [--label ] restarts ... + python3 scripts/compose-smoke-verdict.py [--label ] matrix ... +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +# Spring Boot's own line once the context is up: "Started CaSkeletonApplication in 4.2 seconds (process +# running for 5.0)". Any application name, so a service that renames the class keeps the check. +STARTED = re.compile(r"Started \S+ in \d+(\.\d+)? seconds") +NOISE = re.compile(r"WARN|ERROR") +# One line per container, written by `docker inspect --format` in record_restarts. +RESTART_LINE = re.compile(r"(?P\S+) (?P\S+) restarts=(?P\d+) oomKilled=(?Ptrue|false)") +STATUS_LINE = re.compile(r"(?P\S+) (?P\d+)") +SHOWN = 20 + + +def startup_log(text: str) -> list[str]: + """The reasons the startup log is not silent; empty when it is.""" + noise: list[str] = [] + for line in text.splitlines(): + if STARTED.search(line): + break + if NOISE.search(line): + noise.append(line) + else: + return [ + "the log never reaches Spring Boot's 'Started in seconds' line, " + "so it does not show the startup it is meant to check" + ] + if noise: + return ["the startup log is not silent:", *noise[:SHOWN]] + return [] + + +def restarts(text: str, phases: list[str]) -> list[str]: + """The reasons the lane's containers did not stay up; empty when every phase recorded a clean one.""" + problems: list[str] = [] + recorded: dict[str, int] = {} + for line in text.splitlines(): + match = RESTART_LINE.fullmatch(line) + if match is None: + problems.append(f"unreadable line: {line!r}") + continue + recorded[match["phase"]] = recorded.get(match["phase"], 0) + 1 + if match["restarts"] != "0" or match["oom"] != "false": + problems.append(f"a container restarted or was killed for memory: {line}") + for phase in phases: + if recorded.get(phase, 0) == 0: + problems.append(f"no container was recorded for phase '{phase}'") + return problems + + +def matrix(text: str, lanes: list[str]) -> list[str]: + """The reasons the matrix did not pass; empty when every lane reported zero exactly once.""" + problems: list[str] = [] + statuses: dict[str, list[int]] = {} + for line in text.splitlines(): + match = STATUS_LINE.fullmatch(line) + if match is None: + problems.append(f"unreadable line: {line!r}") + continue + statuses.setdefault(match["lane"], []).append(int(match["rc"])) + missing = [lane for lane in lanes if lane not in statuses] + if missing: + problems.append( + f"{len(lanes) - len(missing)} of {len(lanes)} lanes reported; a partial matrix is not a pass. " + f"Missing: {' '.join(missing)}" + ) + unexpected = [lane for lane in statuses if lane not in lanes] + if unexpected: + problems.append(f"lanes that are not blocking lanes of the contract reported: {' '.join(unexpected)}") + repeated = [lane for lane in lanes if len(statuses.get(lane, [])) > 1] + if repeated: + problems.append(f"lanes that reported more than once: {' '.join(repeated)}") + failed = [lane for lane in lanes if any(rc != 0 for rc in statuses.get(lane, []))] + if failed: + problems.append(f"{len(failed)} lane(s) failed: {' '.join(failed)}") + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--label", default="", help="put in front of every reason, such as the lane id") + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("startup-log").add_argument("log", type=Path) + restart_command = commands.add_parser("restarts") + restart_command.add_argument("record", type=Path) + restart_command.add_argument("phases", nargs="+") + matrix_command = commands.add_parser("matrix") + matrix_command.add_argument("statuses", type=Path) + matrix_command.add_argument("lanes", nargs="+") + args = parser.parse_args(argv) + + # A file the shell did not write is read as empty, which every judgement fails on its own terms. + if args.command == "startup-log": + problems = startup_log(read(args.log)) + elif args.command == "restarts": + problems = restarts(read(args.record), args.phases) + else: + problems = matrix(read(args.statuses), args.lanes) + prefix = f"{args.label}: " if args.label else "" + for problem in problems: + print(f"{prefix}{problem}", file=sys.stderr) + return 1 if problems else 0 + + +def read(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except FileNotFoundError: + return "" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run-compose-runtime-smoke.sh b/scripts/run-compose-runtime-smoke.sh index c944db8d..13a02f19 100755 --- a/scripts/run-compose-runtime-smoke.sh +++ b/scripts/run-compose-runtime-smoke.sh @@ -21,10 +21,15 @@ # # CI calls this script. Inlining a subset of these commands into a workflow is how a lane ends up # running without its one-shots and reporting green. +# +# This script runs docker and writes what it answered to the lane's evidence. What that evidence +# means — the startup log is silent, no container restarted, every lane of the matrix passed — is +# scripts/compose-smoke-verdict.py, whose cases in scripts/tests try each judgement on broken input. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CONTRACTS="${REPO_ROOT}/src/bootstrap/app/compose-lanes/compose-profile-contracts.json" +VERDICT="${REPO_ROOT}/scripts/compose-smoke-verdict.py" EVIDENCE_ROOT="${REPO_ROOT}/src/bootstrap/app/build/evidence/runtime-smoke" RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" # Compose project names accept lowercase alphanumerics, hyphens and underscores only, so the run id @@ -76,7 +81,7 @@ while [[ $# -gt 0 ]]; do done [[ -n "${MODE}" ]] || usage -for tool in jq docker; do +for tool in jq docker python3; do command -v "${tool}" >/dev/null 2>&1 || { echo "${tool} is required" >&2; exit 78; } done [[ -r "${CONTRACTS}" ]] || { echo "missing ${CONTRACTS}" >&2; exit 78; } @@ -179,23 +184,19 @@ random_secret() { head -c 32 /dev/urandom | base64 | tr -d '=+/' | cut -c1-32; } # characters decode to 24 and are rejected. This keeps the padding and the full alphabet. random_key() { head -c 32 /dev/urandom | base64 | tr -d '\n'; } -# Appends one line per container of the lane's project to restarts.txt, prefixed with the phase. -# Fails when there is no container to inspect: an empty file used to pass the restart check, so a -# lane whose project name drifted reported "no restarts" over containers it never looked at. The -# callers run it as `record_restarts ... || return 1`, which turns `set -e` off inside, so a failed -# inspect returns on its own line instead of counting a container whose line was never written. +# One line per container of the lane's project in restarts.txt, prefixed with the phase. +# `compose-smoke-verdict.py restarts` judges the file, and a phase that left no line there fails it: +# an empty file used to pass, so a lane whose project name drifted reported "no restarts" over +# containers it never looked at. Callers run this as +# `record_restarts ... || return 1`, which turns `set -e` off inside, so a failed inspect returns on +# its own line instead of leaving its container out of the file. record_restarts() { - local phase="$1" container inspected=0 + local phase="$1" container while read -r container; do [[ -z "${container}" ]] && continue docker inspect --format "${phase} {{.Name}} restarts={{.RestartCount}} oomKilled={{.State.OOMKilled}}" \ "${container}" >>"${LANE_EVIDENCE}/restarts.txt" || return 1 - inspected=$((inspected + 1)) done < <(docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" ps -a -q) - if [[ ${inspected} -eq 0 ]]; then - echo "${id}: no container of project ${PROJECT} to inspect for restarts (${phase})" >&2 - return 1 - fi } run_lane() { @@ -571,6 +572,7 @@ run_lane() { # platform was INGEST_ONLY is delivered exactly once after a restart into SERVING — on the route # frozen at accept — and that is only evidence if the row is the same row. Tearing the volume down # between phases, or copying rows into a second project, proves nothing about a handoff. + local restart_phases=(final) if jq -e '.handoff' <<<"${lane_json}" >/dev/null; then local first_smoke second_smoke first_smoke="$(jq -r '.handoff.firstPhaseSmoke' <<<"${lane_json}")" @@ -593,6 +595,8 @@ run_lane() { # starts at zero, so a kill during the first phase would otherwise leave no trace. lane_step "handoff restart into the second phase" record_restarts phase1 || return 1 + python3 "${VERDICT}" --label "${id} (${PROJECT})" restarts "${LANE_EVIDENCE}/restarts.txt" phase1 || return 1 + restart_phases=(phase1 final) # The application only. The database and its volume stay up, which is the whole point. echo "-- ${id}: restarting the application into the second phase" @@ -615,18 +619,20 @@ run_lane() { # # StartupWarningZeroTest asserts the same property in-process for the all-off composition. This is # the same assertion against a real container with the lane's switches actually on. + # + # A log that could not be read is a failure, not a silent one. This call used to end in `|| true`: + # a `logs` that failed left only its own error message in startup.log, that message held no WARN, + # and the step passed without having seen a line of the application. The verdict fails a log that + # never reaches the started line for the same reason, so an empty log cannot pass either. local startup_log="${LANE_EVIDENCE}/startup.log" lane_step "startup log is silent" "${startup_log}" - docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color app \ - >"${startup_log}" 2>&1 || true - redact_in_place "${startup_log}" - local startup_noise - startup_noise="$(awk '/Started CaSkeletonApplication/{exit} /(WARN|ERROR)/{print}' "${startup_log}")" - if [[ -n "${startup_noise}" ]]; then - echo "${id}: the startup log is not silent:" >&2 - echo "${startup_noise}" | head -20 >&2 + if ! docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" logs --no-color app >"${startup_log}" 2>&1; then + redact_in_place "${startup_log}" + echo "${id}: docker compose logs app failed, so there is no startup log to check" >&2 return 1 fi + redact_in_place "${startup_log}" + python3 "${VERDICT}" --label "${id}" startup-log "${startup_log}" || return 1 # 4. every declared one-shot, each of which must exit zero # @@ -665,17 +671,11 @@ run_lane() { # checks above all ask the application a question, and a restarted application answers them, so # the restart itself has to be the failure. The peak memory is recorded beside it, so a lane that # is close to its limit is visible before it crosses it. - local restarted="" lane_step "no restart, no OOM kill" "${LANE_EVIDENCE}/restarts.txt" docker compose -p "${PROJECT}" "${LANE_ARGS[@]}" exec -T app cat /sys/fs/cgroup/memory.peak \ >"${LANE_EVIDENCE}/app-memory-peak-bytes.txt" 2>/dev/null || true record_restarts final || return 1 - restarted="$(grep -v ' restarts=0 oomKilled=false$' "${LANE_EVIDENCE}/restarts.txt" || true)" - if [[ -n "${restarted}" ]]; then - echo "${id}: a container restarted or was killed for memory during the lane:" >&2 - echo "${restarted}" >&2 - return 1 - fi + python3 "${VERDICT}" --label "${id} (${PROJECT})" restarts "${LANE_EVIDENCE}/restarts.txt" "${restart_phases[@]}" || return 1 # 5. the manifest ties the evidence to the exact stack that produced it jq -n \ @@ -691,7 +691,6 @@ run_lane() { } GENERATED_SECRETS=() -FAILED_LANES=() # The static contract once for the run, in both modes: it renders every lane by itself, so running it # inside each lane rendered the whole matrix once per lane. @@ -726,26 +725,20 @@ echo "running ${total} blocking lane(s), zero-skip" # failure was ignored. The image build was one: a build that failed left the lane to start the # previous run's image, and all eighteen lanes reported green on code the build had refused # (2026-11-07). --lane mode calls run_lane directly and was not affected. -RAN_LANES=0 +# +# Each lane's status goes to a file, one line per lane, and compose-smoke-verdict.py reads it against +# the contract's blocking lanes: a lane that is missing, repeated or not zero fails the matrix. +LANE_STATUSES="${EVIDENCE_ROOT}/lanes-${RUN_ID}.txt" +: >"${LANE_STATUSES}" while read -r -u 3 lane_json; do [[ -z "${lane_json}" ]] && continue - RAN_LANES=$((RAN_LANES + 1)) set +e ( set -e; trap cleanup_lane EXIT; run_lane "${lane_json}" ) lane_status=$? set -e - if [[ ${lane_status} -ne 0 ]]; then - FAILED_LANES+=("$(jq -r '.id' <<<"${lane_json}")") - fi + printf '%s %s\n' "$(jq -r '.id' <<<"${lane_json}")" "${lane_status}" >>"${LANE_STATUSES}" done 3<<<"${blocking}" -if [[ "${RAN_LANES}" -ne "${total}" ]]; then - echo "run-compose-runtime-smoke: ran ${RAN_LANES} of ${total} lanes; a partial matrix is not a pass" >&2 - exit 1 -fi - -if [[ ${#FAILED_LANES[@]} -gt 0 ]]; then - echo "run-compose-runtime-smoke: ${#FAILED_LANES[@]} lane(s) failed: ${FAILED_LANES[*]}" >&2 - exit 1 -fi +mapfile -t blocking_ids < <(jq -r '.lanes[] | select(.blocking == true) | .id' "${CONTRACTS}") +python3 "${VERDICT}" --label run-compose-runtime-smoke matrix "${LANE_STATUSES}" "${blocking_ids[@]}" || exit 1 echo "run-compose-runtime-smoke: all ${total} blocking lanes passed; evidence under ${EVIDENCE_ROOT}" diff --git a/scripts/tests/test_compose_smoke_verdict.py b/scripts/tests/test_compose_smoke_verdict.py new file mode 100644 index 00000000..65766011 --- /dev/null +++ b/scripts/tests/test_compose_smoke_verdict.py @@ -0,0 +1,365 @@ +"""scripts/compose-smoke-verdict.py의 시험. smoke lane이 남긴 증거를 판정이 고장 난 입력에서 실패로 보는지 본다: +빈 시작 로그나 시작 줄이 없는 로그, 어느 단계에서 컨테이너를 하나도 적지 못한 재시작 기록(`ps`가 빈 결과), +0이 아닌 lane rc나 빠진 lane. 정상 입력은 통과하는지도 같이 본다. 마지막으로 +scripts/run-compose-runtime-smoke.sh가 세 판정을 이 스크립트에 맡기고 `logs` 실패를 삼키지 않는지 본다. + + python3 -m unittest discover -s scripts/tests +""" + +from __future__ import annotations + +import importlib.util +import io +import re +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path + +SCRIPTS = Path(__file__).resolve().parent.parent +VERDICT = SCRIPTS / "compose-smoke-verdict.py" +SMOKE = SCRIPTS / "run-compose-runtime-smoke.sh" +_spec = importlib.util.spec_from_file_location("compose_smoke_verdict", VERDICT) +verdict = importlib.util.module_from_spec(_spec) +sys.modules["compose_smoke_verdict"] = verdict +_spec.loader.exec_module(verdict) + +STARTED = "app-1 | 2026-11-21T10:00:05Z INFO [main] d.c.b.CaSkeletonApplication : Started CaSkeletonApplication in 4.2 seconds" +BOOTING = "app-1 | 2026-11-21T10:00:01Z INFO [main] d.c.b.CaSkeletonApplication : Starting CaSkeletonApplication" + + +def lines(*items: str) -> str: + return "".join(f"{item}\n" for item in items) + + +class StartupLogTest(unittest.TestCase): + + def test_a_log_without_noise_before_the_started_line_passes(self) -> None: + # given + log = lines(BOOTING, STARTED) + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(problems, []) + + def test_a_warn_before_the_started_line_fails_naming_the_line(self) -> None: + # given + warn = "app-1 | 2026-11-21T10:00:03Z WARN [main] o.a.k.c.NetworkClient : Connection refused" + log = lines(BOOTING, warn, STARTED) + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(problems, ["the startup log is not silent:", warn]) + + def test_the_started_line_of_a_renamed_application_ends_the_startup(self) -> None: + # given + started = "app-1 | {\"level\":\"INFO\",\"message\":\"Started TechLogApplication in 6 seconds (process running for 7.1)\"}" + late = "app-1 | {\"level\":\"WARN\",\"message\":\"rejected token\"}" + log = lines(BOOTING, started, late) + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(problems, []) + + def test_an_error_before_the_started_line_fails(self) -> None: + # given + error = "app-1 | 2026-11-21T10:00:03Z ERROR [main] o.s.b.SpringApplication : Application run failed" + log = lines(BOOTING, error, STARTED) + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(problems, ["the startup log is not silent:", error]) + + def test_a_warn_after_the_started_line_is_not_startup_noise(self) -> None: + # given: the lanes that exercise authentication end with a malformed-token probe the application warns about + probe = "app-1 | 2026-11-21T10:01:00Z WARN [http-nio-8080-exec-1] d.c.a.i.w.a.BearerTokenFilter : rejected token" + log = lines(BOOTING, STARTED, probe) + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(problems, []) + + def test_an_empty_log_fails_because_it_never_reaches_the_started_line(self) -> None: + # given + log = "" + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(len(problems), 1) + self.assertIn("never reaches Spring Boot's 'Started in seconds' line", problems[0]) + + def test_the_error_message_of_a_failed_logs_call_fails(self) -> None: + # given: what `docker compose logs --no-color app-missing` writes instead of the application's log + log = lines("no such service: app-missing") + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(len(problems), 1) + self.assertIn("never reaches", problems[0]) + + def test_a_log_cut_off_before_the_started_line_fails_even_without_noise(self) -> None: + # given + log = lines(BOOTING) + + # when + problems = verdict.startup_log(log) + + # then + self.assertEqual(len(problems), 1) + self.assertIn("never reaches", problems[0]) + + +class RestartsTest(unittest.TestCase): + + def test_clean_containers_in_every_phase_pass(self) -> None: + # given + record = lines( + "phase1 /lane-app-1 restarts=0 oomKilled=false", + "phase1 /lane-postgres-1 restarts=0 oomKilled=false", + "final /lane-app-1 restarts=0 oomKilled=false", + "final /lane-postgres-1 restarts=0 oomKilled=false", + ) + + # when + problems = verdict.restarts(record, ["phase1", "final"]) + + # then + self.assertEqual(problems, []) + + def test_an_empty_record_fails_because_ps_found_no_container(self) -> None: + # given + record = "" + + # when + problems = verdict.restarts(record, ["final"]) + + # then + self.assertEqual(problems, ["no container was recorded for phase 'final'"]) + + def test_a_phase_without_a_line_fails_when_another_phase_has_lines(self) -> None: + # given + record = lines("phase1 /lane-app-1 restarts=0 oomKilled=false") + + # when + problems = verdict.restarts(record, ["phase1", "final"]) + + # then + self.assertEqual(problems, ["no container was recorded for phase 'final'"]) + + def test_a_restarted_container_fails_naming_it(self) -> None: + # given + restarted = "final /lane-app-1 restarts=1 oomKilled=false" + record = lines("final /lane-postgres-1 restarts=0 oomKilled=false", restarted) + + # when + problems = verdict.restarts(record, ["final"]) + + # then + self.assertEqual(problems, [f"a container restarted or was killed for memory: {restarted}"]) + + def test_a_container_killed_for_memory_fails(self) -> None: + # given + killed = "final /lane-app-1 restarts=0 oomKilled=true" + + # when + problems = verdict.restarts(lines(killed), ["final"]) + + # then + self.assertEqual(problems, [f"a container restarted or was killed for memory: {killed}"]) + + def test_a_line_docker_did_not_write_in_the_recorded_form_fails(self) -> None: + # given + record = lines("final /lane-app-1 restarts=0 oomKilled=false", "final /lane-app-1 restarts=") + + # when + problems = verdict.restarts(record, ["final"]) + + # then + self.assertEqual(problems, ["unreadable line: 'final /lane-app-1 restarts='"]) + + +class MatrixTest(unittest.TestCase): + LANES = ["off-local", "local-jpa", "all-adapters"] + + def test_every_lane_reporting_zero_once_passes(self) -> None: + # given + statuses = lines("off-local 0", "local-jpa 0", "all-adapters 0") + + # when + problems = verdict.matrix(statuses, self.LANES) + + # then + self.assertEqual(problems, []) + + def test_a_lane_whose_step_failed_fails_the_matrix_naming_it(self) -> None: + # given + statuses = lines("off-local 0", "local-jpa 1", "all-adapters 0") + + # when + problems = verdict.matrix(statuses, self.LANES) + + # then + self.assertEqual(problems, ["1 lane(s) failed: local-jpa"]) + + def test_a_lane_that_did_not_report_fails_the_matrix(self) -> None: + # given + statuses = lines("off-local 0") + + # when + problems = verdict.matrix(statuses, self.LANES) + + # then + self.assertEqual( + problems, + ["1 of 3 lanes reported; a partial matrix is not a pass. Missing: local-jpa all-adapters"], + ) + + def test_no_status_at_all_fails_the_matrix(self) -> None: + # given + statuses = "" + + # when + problems = verdict.matrix(statuses, self.LANES) + + # then + self.assertEqual(len(problems), 1) + self.assertIn("0 of 3 lanes reported", problems[0]) + + def test_a_lane_reporting_twice_fails_even_when_both_are_zero(self) -> None: + # given + statuses = lines("off-local 0", "off-local 0", "local-jpa 0", "all-adapters 0") + + # when + problems = verdict.matrix(statuses, self.LANES) + + # then + self.assertEqual(problems, ["lanes that reported more than once: off-local"]) + + def test_a_lane_the_contract_does_not_block_on_fails(self) -> None: + # given + statuses = lines("off-local 0", "local-jpa 0", "all-adapters 0", "experimental 0") + + # when + problems = verdict.matrix(statuses, self.LANES) + + # then + self.assertEqual(problems, ["lanes that are not blocking lanes of the contract reported: experimental"]) + + +class CommandLineTest(unittest.TestCase): + + def test_a_failure_exits_one_with_each_reason_after_the_label(self) -> None: + # given + with tempfile.TemporaryDirectory() as directory: + record = Path(directory, "restarts.txt") + record.write_text("", encoding="utf-8") + stderr = io.StringIO() + + # when + with redirect_stderr(stderr): + rc = verdict.main(["--label", "off-local", "restarts", str(record), "final"]) + + # then + self.assertEqual(rc, 1) + self.assertEqual(stderr.getvalue(), "off-local: no container was recorded for phase 'final'\n") + + def test_a_file_the_shell_never_wrote_fails_rather_than_passing_as_empty(self) -> None: + # given + with tempfile.TemporaryDirectory() as directory: + missing = Path(directory, "startup.log") + stderr = io.StringIO() + + # when + with redirect_stderr(stderr): + rc = verdict.main(["startup-log", str(missing)]) + + # then + self.assertEqual(rc, 1) + self.assertIn("never reaches", stderr.getvalue()) + + def test_the_script_run_as_the_smoke_script_runs_it_exits_with_the_verdict(self) -> None: + # given + with tempfile.TemporaryDirectory() as directory: + statuses = Path(directory, "lanes.txt") + statuses.write_text(lines("off-local 0", "off-dev 2"), encoding="utf-8") + + # when + failed = subprocess.run( + [sys.executable, str(VERDICT), "matrix", str(statuses), "off-local", "off-dev"], + capture_output=True, text=True, check=False, + ) + statuses.write_text(lines("off-local 0", "off-dev 0"), encoding="utf-8") + passed = subprocess.run( + [sys.executable, str(VERDICT), "matrix", str(statuses), "off-local", "off-dev"], + capture_output=True, text=True, check=False, + ) + + # then + self.assertEqual((failed.returncode, failed.stderr), (1, "1 lane(s) failed: off-dev\n")) + self.assertEqual((passed.returncode, passed.stderr), (0, "")) + + +class SmokeScriptWiringTest(unittest.TestCase): + # The judgements above are only a gate while the smoke script calls them and fails on their status. + + def setUp(self) -> None: + self.script = SMOKE.read_text(encoding="utf-8") + + def test_the_smoke_script_hands_each_judgement_to_the_verdict_and_stops_on_its_failure(self) -> None: + # Each call site on its own: the restart check runs after phase1 of a handoff and at the end of + # every lane, and only the second exists in a lane without a handoff. + verdict_call = r'python3 "\$\{VERDICT\}" --label (?:"[^"\n]*"|[^\s"]+)' + sites = { + "startup log": rf'\n {verdict_call} startup-log "\$\{{startup_log\}}" \|\| return 1\n', + "restarts after phase1": ( + r'\n record_restarts phase1 \|\| return 1\n' + rf' {verdict_call} restarts "\$\{{LANE_EVIDENCE\}}/restarts.txt" phase1 \|\| return 1\n' + ), + "restarts at the end": ( + r'\n record_restarts final \|\| return 1\n' + rf' {verdict_call} restarts "\$\{{LANE_EVIDENCE\}}/restarts.txt" "\$\{{restart_phases\[@\]\}}" \|\| return 1\n' + ), + "matrix": rf'\n{verdict_call} matrix "\$\{{LANE_STATUSES\}}" "\$\{{blocking_ids\[@\]\}}" \|\| exit 1\n', + } + for site, pattern in sites.items(): + with self.subTest(site=site): + self.assertRegex(self.script, pattern) + + def test_the_matrix_writes_the_status_each_lane_exited_with(self) -> None: + # The matrix verdict reads these lines and cannot tell a real zero from one the shell wrote. + self.assertRegex( + self.script, + r'\n \( set -e; trap cleanup_lane EXIT; run_lane "\$\{lane_json\}" \)\n' + r' lane_status=\$\?\n' + r' set -e\n' + r' printf \'%s %s\\n\' "\$\(jq -r \'\.id\' <<<"\$\{lane_json\}"\)" "\$\{lane_status\}" >>"\$\{LANE_STATUSES\}"\n', + ) + + def test_a_failed_logs_call_for_the_startup_log_is_not_swallowed(self) -> None: + # given: the call, with any line continuations + calls = re.findall(r"logs --no-color app\b(?:[^\n]|\\\n)*", self.script) + + # then + self.assertEqual(len(calls), 1, calls) + self.assertNotIn("|| true", calls[0]) + self.assertTrue(re.search(r"if ! docker compose [^\n]*logs --no-color app ", self.script)) + + +if __name__ == "__main__": + unittest.main()