pr / done (pull_request) Successful in 3m24s
scripts/run-compose-runtime-smoke.sh는 nightly 게이트인데 판정이 셸 분기라 시험이 없다. 「startup log is silent」 단계는 `docker compose logs app`을 `|| true`로 받고 시작 줄 앞의 WARN·ERROR만 찾는다. 그래서 `logs`가 실패해 오류 문구만 남거나 로그가 비어도 WARN이 없으니 통과한다. 같은 스크립트에서 빈 `ps`(d904d680)와 무시된 단계 실패(2dd4acc6)가 이미 같은 꼴로 초록을 냈다. 판정 셋을 scripts/compose-smoke-verdict.py로 옮긴다. 셸은 docker를 부르고 그 답을 증거 파일에 쓰기만 하고, 판정은 그 파일을 읽어 rc로 답한다. - startup-log: Spring Boot의 시작 줄(Started <앱> in <n> seconds)이 없으면 실패, 그 앞의 WARN·ERROR도 실패. `logs` 호출이 실패하면 판정 전에 lane이 실패한다. - restarts: 기록한 phase마다 컨테이너 줄이 하나 이상이고 재시작 0·OOM 아님. handoff lane은 1단계를 기록한 직후에도 본다. - matrix: 계약의 blocking lane마다 상태 줄이 정확히 하나이고 0. 빠지거나 겹치거나 계약 밖 lane이면 실패. scripts/tests/test_compose_smoke_verdict.py가 각 판정을 고장 난 입력과 정상 입력으로 보고, smoke 스크립트가 세 판정을 부르고 그 실패에서 멈추는지, `logs` 실패를 삼키지 않는지 본다. prCheck의 testDocumentChecks가 돌린다. 버린 안: bats는 desktop과 runner 둘 다에 새 도구를 깔아야 한다. 가짜 docker로 셸 전체를 돌리는 시험은 정적 계약 스크립트가 실제 렌더 결과를 요구해서, 게이트 스크립트에 시험용 분기를 넣어야 한다. 위험·한계: lane rc가 셸 안에서 0으로 바뀌는 일(2dd4acc6의 `if ( run_lane )`)은 matrix 판정이 볼 수 없다. 그 자리는 전처럼 subshell rc를 조건 밖에서 읽는다. 시작 줄 꼴은 Spring Boot 문구에 기댄다. 문구가 바뀌면 모든 lane이 시작 줄 없음으로 실패한다(조용히 통과하지 않는다). 검증: 노트북 `python3 -m unittest discover -s scripts/tests` 220개 통과. 판정의 시작 줄 검사·phase 검사·lane rc 검사를 지우거나 셸의 `logs`에 `|| true`를 되돌리거나 판정 호출을 `|| true`로 바꾸거나, 판정 호출을 자리마다 지우거나, lane rc 대신 0을 쓰거나 rc를 `set -e` 뒤에 읽으면 새 시험이 실패한다(변이 8개 모두). desktop RED·GREEN lane 변이와 nightly smoke matrix 결과는 PR 본문에 있다. Refs: REV-BE-0016
366 lines
13 KiB
Python
366 lines
13 KiB
Python
"""scripts/compose-smoke-verdict.py의 시험. smoke lane이 남긴 증거를 판정이 고장 난 입력에서 실패로 보는지 본다:
|
|
빈 시작 로그나 시작 줄이 없는 로그, 어느 단계에서 컨테이너를 하나도 적지 못한 재시작 기록(`ps`가 빈 결과),
|
|
0이 아닌 lane rc나 빠진 lane. 정상 입력은 통과하는지도 같이 본다. 마지막으로
|
|
scripts/run-compose-runtime-smoke.sh가 세 판정을 이 스크립트에 맡기고 `logs` 실패를 삼키지 않는지 본다.
|
|
|
|
python3 -m unittest discover -s scripts/tests
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import redirect_stderr
|
|
from pathlib import Path
|
|
|
|
SCRIPTS = Path(__file__).resolve().parent.parent
|
|
VERDICT = SCRIPTS / "compose-smoke-verdict.py"
|
|
SMOKE = SCRIPTS / "run-compose-runtime-smoke.sh"
|
|
_spec = importlib.util.spec_from_file_location("compose_smoke_verdict", VERDICT)
|
|
verdict = importlib.util.module_from_spec(_spec)
|
|
sys.modules["compose_smoke_verdict"] = verdict
|
|
_spec.loader.exec_module(verdict)
|
|
|
|
STARTED = "app-1 | 2026-11-21T10:00:05Z INFO [main] d.c.b.CaSkeletonApplication : Started CaSkeletonApplication in 4.2 seconds"
|
|
BOOTING = "app-1 | 2026-11-21T10:00:01Z INFO [main] d.c.b.CaSkeletonApplication : Starting CaSkeletonApplication"
|
|
|
|
|
|
def lines(*items: str) -> str:
|
|
return "".join(f"{item}\n" for item in items)
|
|
|
|
|
|
class StartupLogTest(unittest.TestCase):
|
|
|
|
def test_a_log_without_noise_before_the_started_line_passes(self) -> None:
|
|
# given
|
|
log = lines(BOOTING, STARTED)
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_a_warn_before_the_started_line_fails_naming_the_line(self) -> None:
|
|
# given
|
|
warn = "app-1 | 2026-11-21T10:00:03Z WARN [main] o.a.k.c.NetworkClient : Connection refused"
|
|
log = lines(BOOTING, warn, STARTED)
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(problems, ["the startup log is not silent:", warn])
|
|
|
|
def test_the_started_line_of_a_renamed_application_ends_the_startup(self) -> None:
|
|
# given
|
|
started = "app-1 | {\"level\":\"INFO\",\"message\":\"Started TechLogApplication in 6 seconds (process running for 7.1)\"}"
|
|
late = "app-1 | {\"level\":\"WARN\",\"message\":\"rejected token\"}"
|
|
log = lines(BOOTING, started, late)
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_an_error_before_the_started_line_fails(self) -> None:
|
|
# given
|
|
error = "app-1 | 2026-11-21T10:00:03Z ERROR [main] o.s.b.SpringApplication : Application run failed"
|
|
log = lines(BOOTING, error, STARTED)
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(problems, ["the startup log is not silent:", error])
|
|
|
|
def test_a_warn_after_the_started_line_is_not_startup_noise(self) -> None:
|
|
# given: the lanes that exercise authentication end with a malformed-token probe the application warns about
|
|
probe = "app-1 | 2026-11-21T10:01:00Z WARN [http-nio-8080-exec-1] d.c.a.i.w.a.BearerTokenFilter : rejected token"
|
|
log = lines(BOOTING, STARTED, probe)
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_an_empty_log_fails_because_it_never_reaches_the_started_line(self) -> None:
|
|
# given
|
|
log = ""
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(len(problems), 1)
|
|
self.assertIn("never reaches Spring Boot's 'Started <application> in <n> seconds' line", problems[0])
|
|
|
|
def test_the_error_message_of_a_failed_logs_call_fails(self) -> None:
|
|
# given: what `docker compose logs --no-color app-missing` writes instead of the application's log
|
|
log = lines("no such service: app-missing")
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(len(problems), 1)
|
|
self.assertIn("never reaches", problems[0])
|
|
|
|
def test_a_log_cut_off_before_the_started_line_fails_even_without_noise(self) -> None:
|
|
# given
|
|
log = lines(BOOTING)
|
|
|
|
# when
|
|
problems = verdict.startup_log(log)
|
|
|
|
# then
|
|
self.assertEqual(len(problems), 1)
|
|
self.assertIn("never reaches", problems[0])
|
|
|
|
|
|
class RestartsTest(unittest.TestCase):
|
|
|
|
def test_clean_containers_in_every_phase_pass(self) -> None:
|
|
# given
|
|
record = lines(
|
|
"phase1 /lane-app-1 restarts=0 oomKilled=false",
|
|
"phase1 /lane-postgres-1 restarts=0 oomKilled=false",
|
|
"final /lane-app-1 restarts=0 oomKilled=false",
|
|
"final /lane-postgres-1 restarts=0 oomKilled=false",
|
|
)
|
|
|
|
# when
|
|
problems = verdict.restarts(record, ["phase1", "final"])
|
|
|
|
# then
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_an_empty_record_fails_because_ps_found_no_container(self) -> None:
|
|
# given
|
|
record = ""
|
|
|
|
# when
|
|
problems = verdict.restarts(record, ["final"])
|
|
|
|
# then
|
|
self.assertEqual(problems, ["no container was recorded for phase 'final'"])
|
|
|
|
def test_a_phase_without_a_line_fails_when_another_phase_has_lines(self) -> None:
|
|
# given
|
|
record = lines("phase1 /lane-app-1 restarts=0 oomKilled=false")
|
|
|
|
# when
|
|
problems = verdict.restarts(record, ["phase1", "final"])
|
|
|
|
# then
|
|
self.assertEqual(problems, ["no container was recorded for phase 'final'"])
|
|
|
|
def test_a_restarted_container_fails_naming_it(self) -> None:
|
|
# given
|
|
restarted = "final /lane-app-1 restarts=1 oomKilled=false"
|
|
record = lines("final /lane-postgres-1 restarts=0 oomKilled=false", restarted)
|
|
|
|
# when
|
|
problems = verdict.restarts(record, ["final"])
|
|
|
|
# then
|
|
self.assertEqual(problems, [f"a container restarted or was killed for memory: {restarted}"])
|
|
|
|
def test_a_container_killed_for_memory_fails(self) -> None:
|
|
# given
|
|
killed = "final /lane-app-1 restarts=0 oomKilled=true"
|
|
|
|
# when
|
|
problems = verdict.restarts(lines(killed), ["final"])
|
|
|
|
# then
|
|
self.assertEqual(problems, [f"a container restarted or was killed for memory: {killed}"])
|
|
|
|
def test_a_line_docker_did_not_write_in_the_recorded_form_fails(self) -> None:
|
|
# given
|
|
record = lines("final /lane-app-1 restarts=0 oomKilled=false", "final /lane-app-1 restarts=<no value>")
|
|
|
|
# when
|
|
problems = verdict.restarts(record, ["final"])
|
|
|
|
# then
|
|
self.assertEqual(problems, ["unreadable line: 'final /lane-app-1 restarts=<no value>'"])
|
|
|
|
|
|
class MatrixTest(unittest.TestCase):
|
|
LANES = ["off-local", "local-jpa", "all-adapters"]
|
|
|
|
def test_every_lane_reporting_zero_once_passes(self) -> None:
|
|
# given
|
|
statuses = lines("off-local 0", "local-jpa 0", "all-adapters 0")
|
|
|
|
# when
|
|
problems = verdict.matrix(statuses, self.LANES)
|
|
|
|
# then
|
|
self.assertEqual(problems, [])
|
|
|
|
def test_a_lane_whose_step_failed_fails_the_matrix_naming_it(self) -> None:
|
|
# given
|
|
statuses = lines("off-local 0", "local-jpa 1", "all-adapters 0")
|
|
|
|
# when
|
|
problems = verdict.matrix(statuses, self.LANES)
|
|
|
|
# then
|
|
self.assertEqual(problems, ["1 lane(s) failed: local-jpa"])
|
|
|
|
def test_a_lane_that_did_not_report_fails_the_matrix(self) -> None:
|
|
# given
|
|
statuses = lines("off-local 0")
|
|
|
|
# when
|
|
problems = verdict.matrix(statuses, self.LANES)
|
|
|
|
# then
|
|
self.assertEqual(
|
|
problems,
|
|
["1 of 3 lanes reported; a partial matrix is not a pass. Missing: local-jpa all-adapters"],
|
|
)
|
|
|
|
def test_no_status_at_all_fails_the_matrix(self) -> None:
|
|
# given
|
|
statuses = ""
|
|
|
|
# when
|
|
problems = verdict.matrix(statuses, self.LANES)
|
|
|
|
# then
|
|
self.assertEqual(len(problems), 1)
|
|
self.assertIn("0 of 3 lanes reported", problems[0])
|
|
|
|
def test_a_lane_reporting_twice_fails_even_when_both_are_zero(self) -> None:
|
|
# given
|
|
statuses = lines("off-local 0", "off-local 0", "local-jpa 0", "all-adapters 0")
|
|
|
|
# when
|
|
problems = verdict.matrix(statuses, self.LANES)
|
|
|
|
# then
|
|
self.assertEqual(problems, ["lanes that reported more than once: off-local"])
|
|
|
|
def test_a_lane_the_contract_does_not_block_on_fails(self) -> None:
|
|
# given
|
|
statuses = lines("off-local 0", "local-jpa 0", "all-adapters 0", "experimental 0")
|
|
|
|
# when
|
|
problems = verdict.matrix(statuses, self.LANES)
|
|
|
|
# then
|
|
self.assertEqual(problems, ["lanes that are not blocking lanes of the contract reported: experimental"])
|
|
|
|
|
|
class CommandLineTest(unittest.TestCase):
|
|
|
|
def test_a_failure_exits_one_with_each_reason_after_the_label(self) -> None:
|
|
# given
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
record = Path(directory, "restarts.txt")
|
|
record.write_text("", encoding="utf-8")
|
|
stderr = io.StringIO()
|
|
|
|
# when
|
|
with redirect_stderr(stderr):
|
|
rc = verdict.main(["--label", "off-local", "restarts", str(record), "final"])
|
|
|
|
# then
|
|
self.assertEqual(rc, 1)
|
|
self.assertEqual(stderr.getvalue(), "off-local: no container was recorded for phase 'final'\n")
|
|
|
|
def test_a_file_the_shell_never_wrote_fails_rather_than_passing_as_empty(self) -> None:
|
|
# given
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
missing = Path(directory, "startup.log")
|
|
stderr = io.StringIO()
|
|
|
|
# when
|
|
with redirect_stderr(stderr):
|
|
rc = verdict.main(["startup-log", str(missing)])
|
|
|
|
# then
|
|
self.assertEqual(rc, 1)
|
|
self.assertIn("never reaches", stderr.getvalue())
|
|
|
|
def test_the_script_run_as_the_smoke_script_runs_it_exits_with_the_verdict(self) -> None:
|
|
# given
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
statuses = Path(directory, "lanes.txt")
|
|
statuses.write_text(lines("off-local 0", "off-dev 2"), encoding="utf-8")
|
|
|
|
# when
|
|
failed = subprocess.run(
|
|
[sys.executable, str(VERDICT), "matrix", str(statuses), "off-local", "off-dev"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
statuses.write_text(lines("off-local 0", "off-dev 0"), encoding="utf-8")
|
|
passed = subprocess.run(
|
|
[sys.executable, str(VERDICT), "matrix", str(statuses), "off-local", "off-dev"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
|
|
# then
|
|
self.assertEqual((failed.returncode, failed.stderr), (1, "1 lane(s) failed: off-dev\n"))
|
|
self.assertEqual((passed.returncode, passed.stderr), (0, ""))
|
|
|
|
|
|
class SmokeScriptWiringTest(unittest.TestCase):
|
|
# The judgements above are only a gate while the smoke script calls them and fails on their status.
|
|
|
|
def setUp(self) -> None:
|
|
self.script = SMOKE.read_text(encoding="utf-8")
|
|
|
|
def test_the_smoke_script_hands_each_judgement_to_the_verdict_and_stops_on_its_failure(self) -> None:
|
|
# Each call site on its own: the restart check runs after phase1 of a handoff and at the end of
|
|
# every lane, and only the second exists in a lane without a handoff.
|
|
verdict_call = r'python3 "\$\{VERDICT\}" --label (?:"[^"\n]*"|[^\s"]+)'
|
|
sites = {
|
|
"startup log": rf'\n {verdict_call} startup-log "\$\{{startup_log\}}" \|\| return 1\n',
|
|
"restarts after phase1": (
|
|
r'\n record_restarts phase1 \|\| return 1\n'
|
|
rf' {verdict_call} restarts "\$\{{LANE_EVIDENCE\}}/restarts.txt" phase1 \|\| return 1\n'
|
|
),
|
|
"restarts at the end": (
|
|
r'\n record_restarts final \|\| return 1\n'
|
|
rf' {verdict_call} restarts "\$\{{LANE_EVIDENCE\}}/restarts.txt" "\$\{{restart_phases\[@\]\}}" \|\| return 1\n'
|
|
),
|
|
"matrix": rf'\n{verdict_call} matrix "\$\{{LANE_STATUSES\}}" "\$\{{blocking_ids\[@\]\}}" \|\| exit 1\n',
|
|
}
|
|
for site, pattern in sites.items():
|
|
with self.subTest(site=site):
|
|
self.assertRegex(self.script, pattern)
|
|
|
|
def test_the_matrix_writes_the_status_each_lane_exited_with(self) -> None:
|
|
# The matrix verdict reads these lines and cannot tell a real zero from one the shell wrote.
|
|
self.assertRegex(
|
|
self.script,
|
|
r'\n \( set -e; trap cleanup_lane EXIT; run_lane "\$\{lane_json\}" \)\n'
|
|
r' lane_status=\$\?\n'
|
|
r' set -e\n'
|
|
r' printf \'%s %s\\n\' "\$\(jq -r \'\.id\' <<<"\$\{lane_json\}"\)" "\$\{lane_status\}" >>"\$\{LANE_STATUSES\}"\n',
|
|
)
|
|
|
|
def test_a_failed_logs_call_for_the_startup_log_is_not_swallowed(self) -> None:
|
|
# given: the call, with any line continuations
|
|
calls = re.findall(r"logs --no-color app\b(?:[^\n]|\\\n)*", self.script)
|
|
|
|
# then
|
|
self.assertEqual(len(calls), 1, calls)
|
|
self.assertNotIn("|| true", calls[0])
|
|
self.assertTrue(re.search(r"if ! docker compose [^\n]*logs --no-color app ", self.script))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|