Files
clean-architecture-backend-…/scripts/tests/test_junit_summary.py
DongHyeonka 9873fc06af
pr / done (pull_request) Successful in 1m51s
fix(verify): --on 단계가 worktree를 지우기 전에 lane별 시험 클래스 목록을 남긴다
verify.sh --on <호스트>는 단계를 돌린 뒤 새 worktree의 JUnit XML을
읽어 합계 한 줄만 로그에 쓰고, trap이 worktree와 XML을 지운다. Gradle
--console=plain 로그에는 성공한 시험의 수도 클래스도 없어, 표준
진입점으로 돌리면 「고친 시험이 그 lane에서 돌았다」를 남길 곳이 없다.
requires lane은 빌드가 단언하지만 tag lane과 기본 test는 비어 있고,
세션마다 XML을 손으로 복사해 왔다.

합계를 새 scripts/junit-summary.py로 옮기고, 같은 스크립트가 XML마다
leaf·태스크·클래스·숫자 한 줄을 ~/<로그 이름>-logs/<단계>-classes.tsv에
남긴다. 단계 전에 지난 목록을 지워, 단계가 합계 전에 끊겨도 옛 목록이
새 로그 옆에 남지 않는다. 합계 줄은 전 awk와 같은 꼴이고, 읽는 범위도
전 find와 같다(build/test-results/ 아래 TEST-*.xml). 다른 점: 루트
testsuite만 세서 시험이 찍은 "<testsuite " 글자를 더 세지 않고, 열지
못하거나 끝까지 읽지 못하거나 숫자가 아닌 XML은 stderr에 이름을 찍고
뺀다. rc는 전처럼 단계의 rc다. 버린 안: heredoc 안 셸로 목록 만들기
(이스케이프가 겹쳐 시험하기 어렵다), XML 폴더 통째 복사(pr 단계 약
1천 개가 실행마다 쌓인다). gates.md 4절·operate.md의 손 복사 절차는
--on here나 손으로 돌린 worktree일 때만으로 좁히고, 같은 절차를 말하던
docs/work/README.md와 스킬 셋(be-testing·be-build-ci·desktop-gates)의
한 줄도 맞춘다.

위험: 원격 worktree에 python3이 있어야 한다(desktop 3.12.3). 합계
스크립트가 죽어도 단계 rc는 바뀌지 않는다. 캐시에서 되살린 태스크의
XML도 목록에 줄이 되므로 「돌았다」의 증거는 --no-cache 실행의 목록만
쓴다고 gates.md에 적었다. 잠금·정리·트리 비교·rc 처리·workflow는
그대로다.

검증(트리 fb7606c4): scripts 시험 전체 332개 통과, 새 시험 9개는 base
트리에서 모두 실패(rc=1). 변이 9개를 모두 시험이 잡는다. desktop
scripts/verify.sh pr --no-cache rc=0(tests=7353, 목록 1059줄, TSV 합
7353), leaf :adapter:inbound:grpc rc=0(tests=85, TSV 합 85,
GrpcP1BoundaryWireTest 줄), docs rc=0.

Refs: CI-BE-0006
2026-11-22 06:23:40 +09:00

264 lines
12 KiB
Python

"""scripts/junit-summary.py의 시험. JUnit XML마다 TSV 한 줄(leaf, 태스크, 클래스, 숫자)을 남기는지, 합계 줄이
scripts/verify.sh가 전에 awk로 찍던 줄과 글자 하나까지 같은지, 읽지 못한 XML을 stderr에 이름으로 찍고 합계에서
빼며 rc를 0으로 두는지 본다. 마지막으로 verify.sh `--on <호스트>`가 원격에 보내는 스크립트가 단계 전에 지난 목록을
지우고, rc를 받은 뒤 worktree를 지우기 전에 이 스크립트를 부르며, 그 줄이 로그 폴더에 두 파일을 남기는지 본다(ssh는
가짜라 Gradle·네트워크를 쓰지 않는다).
python3 -m unittest discover -s scripts/tests
"""
from __future__ import annotations
import os
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
SCRIPTS = Path(__file__).resolve().parent.parent
SUMMARY = SCRIPTS / "junit-summary.py"
VERIFY = SCRIPTS / "verify.sh"
# The total line scripts/verify.sh printed before it handed the count to junit-summary.py: its heredoc at
# 3bfefe69 with the heredoc escapes undone (\\ -> \, \$ -> $). The new line must stay the same for the same tree.
FORMER_AWK_TOTAL = r"""find . -path '*/build/test-results/*' -name 'TEST-*.xml' -print0 | xargs -0 -r cat \
| grep -o '<testsuite [^>]*' \
| awk '{ for (i = 1; i <= NF; i++) if (split($i, kv, "=") == 2) { gsub(/"/, "", kv[2]); n[kv[1]] += kv[2] } }
END { printf "verify: 테스트 tests=%d failures=%d errors=%d skipped=%d\n", n["tests"], n["failures"], n["errors"], n["skipped"] }'
"""
def suite(name: str, tests: int, skipped: int = 0, failures: int = 0, errors: int = 0, time: str = "0.100",
out: str = "") -> str:
"""A result file in the form Gradle writes it, one testsuite per class."""
return (
'<?xml version="1.0" encoding="UTF-8"?>\n'
f'<testsuite name="{name}" tests="{tests}" skipped="{skipped}" failures="{failures}" errors="{errors}" '
f'timestamp="2026-11-22T03:00:00.000Z" hostname="desktop" time="{time}">\n'
' <properties/>\n'
f' <testcase name="works()" classname="{name}" time="{time}"/>\n'
f' <system-out><![CDATA[{out}]]></system-out>\n'
' <system-err><![CDATA[]]></system-err>\n'
'</testsuite>\n'
)
GRPC = "src/adapter/inbound/grpc/build/test-results"
APP = "src/bootstrap/app/build/test-results"
# Two leaves, two tasks, three result files: one with a failure, one with two skips, one with an error. No
# system-out holds "<testsuite ", which the former awk counted wherever it appeared.
RESULTS = {
f"{GRPC}/test/TEST-dev.caskeleton.adapter.inbound.grpc.WireTest.xml":
suite("dev.caskeleton.adapter.inbound.grpc.WireTest", tests=3, failures=1, time="0.412"),
f"{GRPC}/grpcContractTest/TEST-dev.caskeleton.adapter.inbound.grpc.ContractTest.xml":
suite("dev.caskeleton.adapter.inbound.grpc.ContractTest", tests=2, skipped=2, time="0.003"),
f"{APP}/test/TEST-dev.caskeleton.bootstrap.AppTest.xml":
suite("dev.caskeleton.bootstrap.AppTest", tests=4, errors=1, time="1.250"),
# Not results, so neither the former find nor the new script reads them: one outside build/test-results,
# one inside it without the TEST- name.
"src/bootstrap/app/src/test/resources/TEST-dev.caskeleton.bootstrap.Fixture.xml":
suite("dev.caskeleton.bootstrap.Fixture", tests=9),
f"{APP}/test/results.xml": suite("dev.caskeleton.bootstrap.NotAResult", tests=5),
}
ROWS = (
"src/adapter/inbound/grpc\tgrpcContractTest\tdev.caskeleton.adapter.inbound.grpc.ContractTest\t2\t2\t0\t0\t0.003\n"
"src/adapter/inbound/grpc\ttest\tdev.caskeleton.adapter.inbound.grpc.WireTest\t3\t0\t1\t0\t0.412\n"
"src/bootstrap/app\ttest\tdev.caskeleton.bootstrap.AppTest\t4\t0\t0\t1\t1.250\n"
)
TOTAL = "verify: 테스트 tests=9 failures=1 errors=1 skipped=2\n"
class TreeTest(unittest.TestCase):
def setUp(self) -> None:
directory = tempfile.TemporaryDirectory()
self.addCleanup(directory.cleanup)
self.root = Path(directory.name) / "tree"
self.root.mkdir()
self.classes = Path(directory.name) / "pr-classes.tsv"
def write(self, files: dict[str, str]) -> None:
for path, text in files.items():
target = self.root / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(text, encoding="utf-8")
def summary(self, *extra: str) -> subprocess.CompletedProcess[str]:
# As verify.sh calls it: from the worktree root, with the root as ".".
return subprocess.run([sys.executable, str(SUMMARY), ".", *extra], cwd=self.root, capture_output=True,
text=True, check=False, timeout=60)
class ClassListTest(TreeTest):
def test_each_result_file_becomes_one_row_with_its_leaf_task_class_and_counts(self) -> None:
# given
self.write(RESULTS)
# when
self.summary("--classes", str(self.classes))
# then
self.assertEqual(self.classes.read_text(encoding="utf-8"), ROWS)
def test_a_result_of_the_root_project_is_listed_under_dot(self) -> None:
# given
self.write({"build/test-results/test/TEST-dev.caskeleton.RootTest.xml": suite("dev.caskeleton.RootTest", 1)})
# when
self.summary("--classes", str(self.classes))
# then
self.assertEqual(self.classes.read_text(encoding="utf-8"), ".\ttest\tdev.caskeleton.RootTest\t1\t0\t0\t0\t0.100\n")
def test_the_run_prints_the_total_and_where_the_class_list_went(self) -> None:
# given
self.write(RESULTS)
# when
result = self.summary("--classes", str(self.classes))
# then
self.assertEqual((result.returncode, result.stdout, result.stderr),
(0, f"{TOTAL}verify: 클래스 목록 3줄 → {self.classes}\n", ""))
class TotalLineTest(TreeTest):
def test_the_total_line_is_the_line_the_former_awk_pipeline_printed_for_the_same_tree(self) -> None:
for name, files in {"three results and a non-result": RESULTS, "no result at all": {}}.items():
with self.subTest(tree=name):
# given
shutil.rmtree(self.root)
self.root.mkdir()
self.write(files)
former = subprocess.run(["bash", "-c", FORMER_AWK_TOTAL], cwd=self.root, capture_output=True,
text=True, check=False, timeout=60).stdout
# when
result = self.summary()
# then
self.assertEqual(result.stdout, former)
def test_a_testsuite_tag_a_test_printed_is_not_counted(self) -> None:
# given: the former awk would have counted this output as a second suite of five tests
self.write({f"{APP}/test/TEST-dev.caskeleton.bootstrap.PrintsTest.xml": suite(
"dev.caskeleton.bootstrap.PrintsTest", tests=1, out='<testsuite name="Other" tests="5" failures="5"')})
# when
result = self.summary()
# then
self.assertEqual(result.stdout, "verify: 테스트 tests=1 failures=0 errors=0 skipped=0\n")
def test_an_unreadable_file_is_named_on_stderr_and_left_out_while_the_exit_stays_zero(self) -> None:
# given: one broken inside the root start tag, one cut off after the root started, one whose count
# is not a number, one that cannot be opened (a link to nothing)
broken = f"{APP}/test/TEST-dev.caskeleton.bootstrap.BrokenTest.xml"
cut = f"{APP}/test/TEST-dev.caskeleton.bootstrap.CutTest.xml"
gone = f"{APP}/test/TEST-dev.caskeleton.bootstrap.GoneTest.xml"
wrong = f"{GRPC}/test/TEST-dev.caskeleton.adapter.inbound.grpc.WrongTest.xml"
self.write({
**RESULTS,
broken: '<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="dev.caskeleton.bootstrap.BrokenTest" tests=',
cut: suite("dev.caskeleton.bootstrap.CutTest", tests=7).split(" <system-out>")[0],
wrong: suite("dev.caskeleton.adapter.inbound.grpc.WrongTest", tests=1).replace('tests="1"', 'tests="many"'),
})
(self.root / gone).symlink_to(self.root / "nowhere.xml")
# when
result = self.summary("--classes", str(self.classes))
# then
unreadable = "".join(f"verify: 읽지 못한 XML {path}\n" for path in (wrong, broken, cut, gone))
self.assertEqual((result.returncode, result.stdout, result.stderr),
(0, f"{TOTAL}verify: 클래스 목록 3줄 → {self.classes}\n", unreadable))
# verify.sh --on <host> pipes its remote script into `ssh <host> bash -s`. This ssh keeps that script and
# drops the patch, so the script can be read as verify.sh expanded it.
FAKE_SSH = """#!/bin/sh
case "$2" in
"bash") cat > "$FAKE_SSH_SCRIPT" ;;
*) cat > /dev/null ;;
esac
"""
GIT_ENV = {
**os.environ,
"GIT_CONFIG_GLOBAL": os.devnull,
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_AUTHOR_NAME": "test",
"GIT_AUTHOR_EMAIL": "test@example.invalid",
"GIT_COMMITTER_NAME": "test",
"GIT_COMMITTER_EMAIL": "test@example.invalid",
}
# After the stage's rc is kept and before the script exits (its EXIT trap removes the worktree and the XML).
SUMMARY_STEP = re.compile(
r"\nrc=\$\{PIPESTATUS\[0\]\}\n(?:#[^\n]*\n)*"
r"(python3 scripts/junit-summary\.py \. --classes ~/wiring-logs/pr-classes\.tsv \| tee -a ~/wiring-logs/pr\.log)\n"
r'echo "verify: rc=\$rc" \| tee -a ~/wiring-logs/pr\.log\n'
r"exit \$rc\n"
)
class RemoteScriptWiringTest(TreeTest):
def setUp(self) -> None:
super().setUp()
work = self.root.parent
repo = work / "repo"
(repo / "scripts").mkdir(parents=True)
shutil.copy2(VERIFY, repo / "scripts/verify.sh")
for args in (["init", "-q", "-b", "main"], ["add", "-A"], ["commit", "-q", "-m", "first"]):
subprocess.run(["git", *args], cwd=repo, env=GIT_ENV, check=True, capture_output=True)
ssh = work / "ssh"
ssh.write_text(FAKE_SSH, encoding="utf-8")
ssh.chmod(0o755)
self.remote_script = work / "remote.sh"
env = {**GIT_ENV, "VERIFY_SSH": str(ssh), "VERIFY_BASE_REF": "HEAD", "FAKE_SSH_SCRIPT": str(self.remote_script)}
subprocess.run(["bash", "scripts/verify.sh", "pr", "--on", "fake", "--log", "wiring"], cwd=repo, env=env,
check=True, capture_output=True, timeout=60)
def test_the_remote_script_summarises_after_keeping_the_rc_and_before_it_exits(self) -> None:
self.assertRegex(self.remote_script.read_text(encoding="utf-8"), SUMMARY_STEP)
def test_the_remote_script_drops_the_last_class_list_before_the_stage_runs(self) -> None:
# A stage cut off before the summary must not leave the last run's list beside its new log.
self.assertRegex(
self.remote_script.read_text(encoding="utf-8"),
r"\nmkdir -p [^\n]* ~/wiring-logs\nrm -f ~/wiring-logs/pr-classes\.tsv(?: #[^\n]*)?\n(?:[^\n]*\n)*?"
r"VERIFY_LOCKED=1 flock ",
)
def test_the_summary_line_it_sends_leaves_the_class_list_and_the_total_in_the_log_folder(self) -> None:
# given: the line as sent, run from a worktree that holds the results, with a home of its own
line = SUMMARY_STEP.search(self.remote_script.read_text(encoding="utf-8")).group(1)
self.write(RESULTS)
(self.root / "scripts").mkdir()
shutil.copy2(SUMMARY, self.root / "scripts/junit-summary.py")
home = self.root.parent / "home"
(home / "wiring-logs").mkdir(parents=True)
# when
subprocess.run(["bash", "-c", line], cwd=self.root, env={**os.environ, "HOME": str(home)}, check=True,
capture_output=True, timeout=60)
# then
logs = home / "wiring-logs"
self.assertEqual(
((logs / "pr-classes.tsv").read_text(encoding="utf-8"), (logs / "pr.log").read_text(encoding="utf-8")),
(ROWS, f"{TOTAL}verify: 클래스 목록 3줄 → {logs / 'pr-classes.tsv'}\n"),
)
if __name__ == "__main__":
unittest.main()