Files
clean-architecture-backend-…/scripts/tests/test_compose_smoke_verdict.py
DongHyeonka 720cfc3116
pr / done (pull_request) Successful in 3m24s
fix(compose): smoke 판정을 시험 있는 스크립트로 옮기고 빈 시작 로그를 막는다
scripts/run-compose-runtime-smoke.sh는 nightly 게이트인데 판정이 셸 분기라
시험이 없다. 「startup log is silent」 단계는 `docker compose logs app`을
`|| true`로 받고 시작 줄 앞의 WARN·ERROR만 찾는다. 그래서 `logs`가 실패해
오류 문구만 남거나 로그가 비어도 WARN이 없으니 통과한다. 같은 스크립트에서
빈 `ps`(d904d680)와 무시된 단계 실패(2dd4acc6)가 이미 같은 꼴로 초록을 냈다.

판정 셋을 scripts/compose-smoke-verdict.py로 옮긴다. 셸은 docker를 부르고 그
답을 증거 파일에 쓰기만 하고, 판정은 그 파일을 읽어 rc로 답한다.
- startup-log: Spring Boot의 시작 줄(Started <앱> in <n> seconds)이 없으면
  실패, 그 앞의 WARN·ERROR도 실패. `logs` 호출이 실패하면 판정 전에 lane이
  실패한다.
- restarts: 기록한 phase마다 컨테이너 줄이 하나 이상이고 재시작 0·OOM 아님.
  handoff lane은 1단계를 기록한 직후에도 본다.
- matrix: 계약의 blocking lane마다 상태 줄이 정확히 하나이고 0. 빠지거나
  겹치거나 계약 밖 lane이면 실패.
scripts/tests/test_compose_smoke_verdict.py가 각 판정을 고장 난 입력과 정상
입력으로 보고, smoke 스크립트가 세 판정을 부르고 그 실패에서 멈추는지,
`logs` 실패를 삼키지 않는지 본다. prCheck의 testDocumentChecks가 돌린다.

버린 안: bats는 desktop과 runner 둘 다에 새 도구를 깔아야 한다. 가짜 docker로
셸 전체를 돌리는 시험은 정적 계약 스크립트가 실제 렌더 결과를 요구해서,
게이트 스크립트에 시험용 분기를 넣어야 한다.

위험·한계: lane rc가 셸 안에서 0으로 바뀌는 일(2dd4acc6의 `if ( run_lane )`)은
matrix 판정이 볼 수 없다. 그 자리는 전처럼 subshell rc를 조건 밖에서 읽는다.
시작 줄 꼴은 Spring Boot 문구에 기댄다. 문구가 바뀌면 모든 lane이 시작 줄 없음으로
실패한다(조용히 통과하지 않는다).

검증: 노트북 `python3 -m unittest discover -s scripts/tests` 220개 통과. 판정의
시작 줄 검사·phase 검사·lane rc 검사를 지우거나 셸의 `logs`에 `|| true`를 되돌리거나
판정 호출을 `|| true`로 바꾸거나, 판정 호출을 자리마다 지우거나, lane rc 대신 0을
쓰거나 rc를 `set -e` 뒤에 읽으면 새 시험이 실패한다(변이 8개 모두). desktop RED·GREEN
lane 변이와 nightly smoke matrix 결과는 PR 본문에 있다.

Refs: REV-BE-0016
2026-11-21 16:43:44 +09:00

366 lines
13 KiB
Python

"""scripts/compose-smoke-verdict.py의 시험. smoke lane이 남긴 증거를 판정이 고장 난 입력에서 실패로 보는지 본다:
빈 시작 로그나 시작 줄이 없는 로그, 어느 단계에서 컨테이너를 하나도 적지 못한 재시작 기록(`ps`가 빈 결과),
0이 아닌 lane rc나 빠진 lane. 정상 입력은 통과하는지도 같이 본다. 마지막으로
scripts/run-compose-runtime-smoke.sh가 세 판정을 이 스크립트에 맡기고 `logs` 실패를 삼키지 않는지 본다.
python3 -m unittest discover -s scripts/tests
"""
from __future__ import annotations
import importlib.util
import io
import re
import subprocess
import sys
import tempfile
import unittest
from contextlib import redirect_stderr
from pathlib import Path
SCRIPTS = Path(__file__).resolve().parent.parent
VERDICT = SCRIPTS / "compose-smoke-verdict.py"
SMOKE = SCRIPTS / "run-compose-runtime-smoke.sh"
_spec = importlib.util.spec_from_file_location("compose_smoke_verdict", VERDICT)
verdict = importlib.util.module_from_spec(_spec)
sys.modules["compose_smoke_verdict"] = verdict
_spec.loader.exec_module(verdict)
STARTED = "app-1 | 2026-11-21T10:00:05Z INFO [main] d.c.b.CaSkeletonApplication : Started CaSkeletonApplication in 4.2 seconds"
BOOTING = "app-1 | 2026-11-21T10:00:01Z INFO [main] d.c.b.CaSkeletonApplication : Starting CaSkeletonApplication"
def lines(*items: str) -> str:
return "".join(f"{item}\n" for item in items)
class StartupLogTest(unittest.TestCase):
def test_a_log_without_noise_before_the_started_line_passes(self) -> None:
# given
log = lines(BOOTING, STARTED)
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(problems, [])
def test_a_warn_before_the_started_line_fails_naming_the_line(self) -> None:
# given
warn = "app-1 | 2026-11-21T10:00:03Z WARN [main] o.a.k.c.NetworkClient : Connection refused"
log = lines(BOOTING, warn, STARTED)
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(problems, ["the startup log is not silent:", warn])
def test_the_started_line_of_a_renamed_application_ends_the_startup(self) -> None:
# given
started = "app-1 | {\"level\":\"INFO\",\"message\":\"Started TechLogApplication in 6 seconds (process running for 7.1)\"}"
late = "app-1 | {\"level\":\"WARN\",\"message\":\"rejected token\"}"
log = lines(BOOTING, started, late)
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(problems, [])
def test_an_error_before_the_started_line_fails(self) -> None:
# given
error = "app-1 | 2026-11-21T10:00:03Z ERROR [main] o.s.b.SpringApplication : Application run failed"
log = lines(BOOTING, error, STARTED)
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(problems, ["the startup log is not silent:", error])
def test_a_warn_after_the_started_line_is_not_startup_noise(self) -> None:
# given: the lanes that exercise authentication end with a malformed-token probe the application warns about
probe = "app-1 | 2026-11-21T10:01:00Z WARN [http-nio-8080-exec-1] d.c.a.i.w.a.BearerTokenFilter : rejected token"
log = lines(BOOTING, STARTED, probe)
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(problems, [])
def test_an_empty_log_fails_because_it_never_reaches_the_started_line(self) -> None:
# given
log = ""
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(len(problems), 1)
self.assertIn("never reaches Spring Boot's 'Started <application> in <n> seconds' line", problems[0])
def test_the_error_message_of_a_failed_logs_call_fails(self) -> None:
# given: what `docker compose logs --no-color app-missing` writes instead of the application's log
log = lines("no such service: app-missing")
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(len(problems), 1)
self.assertIn("never reaches", problems[0])
def test_a_log_cut_off_before_the_started_line_fails_even_without_noise(self) -> None:
# given
log = lines(BOOTING)
# when
problems = verdict.startup_log(log)
# then
self.assertEqual(len(problems), 1)
self.assertIn("never reaches", problems[0])
class RestartsTest(unittest.TestCase):
def test_clean_containers_in_every_phase_pass(self) -> None:
# given
record = lines(
"phase1 /lane-app-1 restarts=0 oomKilled=false",
"phase1 /lane-postgres-1 restarts=0 oomKilled=false",
"final /lane-app-1 restarts=0 oomKilled=false",
"final /lane-postgres-1 restarts=0 oomKilled=false",
)
# when
problems = verdict.restarts(record, ["phase1", "final"])
# then
self.assertEqual(problems, [])
def test_an_empty_record_fails_because_ps_found_no_container(self) -> None:
# given
record = ""
# when
problems = verdict.restarts(record, ["final"])
# then
self.assertEqual(problems, ["no container was recorded for phase 'final'"])
def test_a_phase_without_a_line_fails_when_another_phase_has_lines(self) -> None:
# given
record = lines("phase1 /lane-app-1 restarts=0 oomKilled=false")
# when
problems = verdict.restarts(record, ["phase1", "final"])
# then
self.assertEqual(problems, ["no container was recorded for phase 'final'"])
def test_a_restarted_container_fails_naming_it(self) -> None:
# given
restarted = "final /lane-app-1 restarts=1 oomKilled=false"
record = lines("final /lane-postgres-1 restarts=0 oomKilled=false", restarted)
# when
problems = verdict.restarts(record, ["final"])
# then
self.assertEqual(problems, [f"a container restarted or was killed for memory: {restarted}"])
def test_a_container_killed_for_memory_fails(self) -> None:
# given
killed = "final /lane-app-1 restarts=0 oomKilled=true"
# when
problems = verdict.restarts(lines(killed), ["final"])
# then
self.assertEqual(problems, [f"a container restarted or was killed for memory: {killed}"])
def test_a_line_docker_did_not_write_in_the_recorded_form_fails(self) -> None:
# given
record = lines("final /lane-app-1 restarts=0 oomKilled=false", "final /lane-app-1 restarts=<no value>")
# when
problems = verdict.restarts(record, ["final"])
# then
self.assertEqual(problems, ["unreadable line: 'final /lane-app-1 restarts=<no value>'"])
class MatrixTest(unittest.TestCase):
LANES = ["off-local", "local-jpa", "all-adapters"]
def test_every_lane_reporting_zero_once_passes(self) -> None:
# given
statuses = lines("off-local 0", "local-jpa 0", "all-adapters 0")
# when
problems = verdict.matrix(statuses, self.LANES)
# then
self.assertEqual(problems, [])
def test_a_lane_whose_step_failed_fails_the_matrix_naming_it(self) -> None:
# given
statuses = lines("off-local 0", "local-jpa 1", "all-adapters 0")
# when
problems = verdict.matrix(statuses, self.LANES)
# then
self.assertEqual(problems, ["1 lane(s) failed: local-jpa"])
def test_a_lane_that_did_not_report_fails_the_matrix(self) -> None:
# given
statuses = lines("off-local 0")
# when
problems = verdict.matrix(statuses, self.LANES)
# then
self.assertEqual(
problems,
["1 of 3 lanes reported; a partial matrix is not a pass. Missing: local-jpa all-adapters"],
)
def test_no_status_at_all_fails_the_matrix(self) -> None:
# given
statuses = ""
# when
problems = verdict.matrix(statuses, self.LANES)
# then
self.assertEqual(len(problems), 1)
self.assertIn("0 of 3 lanes reported", problems[0])
def test_a_lane_reporting_twice_fails_even_when_both_are_zero(self) -> None:
# given
statuses = lines("off-local 0", "off-local 0", "local-jpa 0", "all-adapters 0")
# when
problems = verdict.matrix(statuses, self.LANES)
# then
self.assertEqual(problems, ["lanes that reported more than once: off-local"])
def test_a_lane_the_contract_does_not_block_on_fails(self) -> None:
# given
statuses = lines("off-local 0", "local-jpa 0", "all-adapters 0", "experimental 0")
# when
problems = verdict.matrix(statuses, self.LANES)
# then
self.assertEqual(problems, ["lanes that are not blocking lanes of the contract reported: experimental"])
class CommandLineTest(unittest.TestCase):
def test_a_failure_exits_one_with_each_reason_after_the_label(self) -> None:
# given
with tempfile.TemporaryDirectory() as directory:
record = Path(directory, "restarts.txt")
record.write_text("", encoding="utf-8")
stderr = io.StringIO()
# when
with redirect_stderr(stderr):
rc = verdict.main(["--label", "off-local", "restarts", str(record), "final"])
# then
self.assertEqual(rc, 1)
self.assertEqual(stderr.getvalue(), "off-local: no container was recorded for phase 'final'\n")
def test_a_file_the_shell_never_wrote_fails_rather_than_passing_as_empty(self) -> None:
# given
with tempfile.TemporaryDirectory() as directory:
missing = Path(directory, "startup.log")
stderr = io.StringIO()
# when
with redirect_stderr(stderr):
rc = verdict.main(["startup-log", str(missing)])
# then
self.assertEqual(rc, 1)
self.assertIn("never reaches", stderr.getvalue())
def test_the_script_run_as_the_smoke_script_runs_it_exits_with_the_verdict(self) -> None:
# given
with tempfile.TemporaryDirectory() as directory:
statuses = Path(directory, "lanes.txt")
statuses.write_text(lines("off-local 0", "off-dev 2"), encoding="utf-8")
# when
failed = subprocess.run(
[sys.executable, str(VERDICT), "matrix", str(statuses), "off-local", "off-dev"],
capture_output=True, text=True, check=False,
)
statuses.write_text(lines("off-local 0", "off-dev 0"), encoding="utf-8")
passed = subprocess.run(
[sys.executable, str(VERDICT), "matrix", str(statuses), "off-local", "off-dev"],
capture_output=True, text=True, check=False,
)
# then
self.assertEqual((failed.returncode, failed.stderr), (1, "1 lane(s) failed: off-dev\n"))
self.assertEqual((passed.returncode, passed.stderr), (0, ""))
class SmokeScriptWiringTest(unittest.TestCase):
# The judgements above are only a gate while the smoke script calls them and fails on their status.
def setUp(self) -> None:
self.script = SMOKE.read_text(encoding="utf-8")
def test_the_smoke_script_hands_each_judgement_to_the_verdict_and_stops_on_its_failure(self) -> None:
# Each call site on its own: the restart check runs after phase1 of a handoff and at the end of
# every lane, and only the second exists in a lane without a handoff.
verdict_call = r'python3 "\$\{VERDICT\}" --label (?:"[^"\n]*"|[^\s"]+)'
sites = {
"startup log": rf'\n {verdict_call} startup-log "\$\{{startup_log\}}" \|\| return 1\n',
"restarts after phase1": (
r'\n record_restarts phase1 \|\| return 1\n'
rf' {verdict_call} restarts "\$\{{LANE_EVIDENCE\}}/restarts.txt" phase1 \|\| return 1\n'
),
"restarts at the end": (
r'\n record_restarts final \|\| return 1\n'
rf' {verdict_call} restarts "\$\{{LANE_EVIDENCE\}}/restarts.txt" "\$\{{restart_phases\[@\]\}}" \|\| return 1\n'
),
"matrix": rf'\n{verdict_call} matrix "\$\{{LANE_STATUSES\}}" "\$\{{blocking_ids\[@\]\}}" \|\| exit 1\n',
}
for site, pattern in sites.items():
with self.subTest(site=site):
self.assertRegex(self.script, pattern)
def test_the_matrix_writes_the_status_each_lane_exited_with(self) -> None:
# The matrix verdict reads these lines and cannot tell a real zero from one the shell wrote.
self.assertRegex(
self.script,
r'\n \( set -e; trap cleanup_lane EXIT; run_lane "\$\{lane_json\}" \)\n'
r' lane_status=\$\?\n'
r' set -e\n'
r' printf \'%s %s\\n\' "\$\(jq -r \'\.id\' <<<"\$\{lane_json\}"\)" "\$\{lane_status\}" >>"\$\{LANE_STATUSES\}"\n',
)
def test_a_failed_logs_call_for_the_startup_log_is_not_swallowed(self) -> None:
# given: the call, with any line continuations
calls = re.findall(r"logs --no-color app\b(?:[^\n]|\\\n)*", self.script)
# then
self.assertEqual(len(calls), 1, calls)
self.assertNotIn("|| true", calls[0])
self.assertTrue(re.search(r"if ! docker compose [^\n]*logs --no-color app ", self.script))
if __name__ == "__main__":
unittest.main()