frontend-quality-gates / merge / static checks (pull_request) Canceled after 0s
frontend-quality-gates / merge / tests (pull_request) Canceled after 0s
frontend-quality-gates / merge / browser tests (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-014 / config-compatibility (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-026 / lab-performance (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-028 / manual-accessibility-review (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-015 / immutable-release-candidate (pull_request) Canceled after 0s
frontend-quality-gates / external-vulnerability-provider (pull_request) Canceled after 0s
frontend-quality-gates / external-provenance-provider (pull_request) Canceled after 0s
frontend-quality-gates / promote-verified-immutable-candidate (pull_request) Canceled after 0s
frontend-quality-gates / deploy-promoted-release (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-019 / hosting-header (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-021 / runbook-boot-config (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-022 / runbook-chunk-mismatch (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-023 / runbook-api-degradation (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-024 / runbook-telemetry (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-025 / runbook-release-rollback (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-018 / field-web-vitals (pull_request) Canceled after 0s
frontend-quality-gates / FE-GATE-017 / diagram-review (pull_request) Canceled after 0s
frontend-quality-gates / main / self-tests and removal drills (pull_request) Canceled after 0s
문제: `scripts/verify.sh parts <ID>`는 expect=fail 명령(검사기의 음성 fixture)을 종료 코드만으로 판정한다. `ci:gate`(run-ci-gate.ts의 classifyGateStepResult)는 정해진 종료 코드와 정해진 진단 문구 (`expectedDiagnosticId`)가 둘 다 있어야 통과시킨다. 그래서 검사기가 깨져 엉뚱한 원인으로 exit 1(TypeError 등)을 내도 parts는 그 음성 fixture를 통과로 센다. 문서가 검사기를 고친 뒤 `parts FE-GATE-029`를 돌리라고 하는 바로 그 경로다. args를 버리던 앞 절반은 79a225b6(#19)이 고쳤다. 고른 방법: 계약에서 `expectedDiagnosticId`를 함께 읽고, expect=fail 명령의 출력을 따로 잡아(`STEP_CAPTURE`) 진단 문구가 있는지 본다. 판정은 classifyGateStepResult와 같은 규칙(종료 코드가 같고, 출력에 문구가 들어 있음)이다. 어긋나면 무엇이 어긋났는지 한 줄을 stderr에 낸다. 위험·한계: 문구가 빈 음성 fixture는 예전처럼 종료 코드만 본다(ci:gate도 빈 문구는 늘 맞는다). 시스템 컨테이너 접두어가 붙은 게이트는 래퍼 출력까지 함께 잡는다. 검증: scripts/tests/test_verify.py에 가짜 corepack 시험을 더했다. 음성 fixture가 맞는 종료 코드에 엉뚱한 문구를 내면 parts FE-GATE-029가 rc=1이어야 하는데, 고치기 전 rc=0으로 실패했다(RED). 고친 뒤 9개 시험 통과. desktop 실제 검사기로 `scripts/verify.sh parts FE-GATE-029` rc=0(음성 fixture 9개가 등록된 문구를 냄), 로그 /tmp/claude-gates/w2fe2-gd01/. `scripts/verify.sh docs` OK. Refs: 1112/08-01 (GD-01)
229 lines
14 KiB
Python
229 lines
14 KiB
Python
"""scripts/verify.sh의 시험. 단계를 계약 파일에서 읽는지, 잘못된 인자를 원격에 보내기 전에 거절하는지,
|
|
지금 트리를 원격 worktree로 옮겨 같은 트리에서 돌리고 지우는지를 표준 라이브러리와 git·bash·flock으로 본다.
|
|
원격은 가짜 ssh(호스트를 무시하고 이 기계에서 실행)다. pnpm은 돌리지 않는다.
|
|
|
|
python3 -m unittest discover -s scripts/tests
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent.parent
|
|
VERIFY = REPO / "scripts/verify.sh"
|
|
GIT_ENV = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@example.com", "GIT_COMMITTER_NAME": "t",
|
|
"GIT_COMMITTER_EMAIL": "t@example.com", "GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.hooksPath",
|
|
"GIT_CONFIG_VALUE_0": "/dev/null"}
|
|
|
|
|
|
def sh(args: list[str], cwd: Path, env: dict | None = None, check: bool = True) -> subprocess.CompletedProcess:
|
|
result = subprocess.run(args, cwd=cwd, env={**os.environ, **GIT_ENV, **(env or {})}, capture_output=True,
|
|
text=True, timeout=60)
|
|
if check and result.returncode != 0:
|
|
raise AssertionError(f"{args} → {result.returncode}\n{result.stdout}\n{result.stderr}")
|
|
return result
|
|
|
|
|
|
def gates() -> dict:
|
|
return json.loads((REPO / "config/ci/gates.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
class StagePlanTest(unittest.TestCase):
|
|
def test_a_stage_runs_every_gate_the_contract_lists(self) -> None:
|
|
for stage in gates()["stages"]:
|
|
with self.subTest(stage=stage["id"]):
|
|
result = sh([str(VERIFY), stage["id"], "--dry-run"], cwd=REPO)
|
|
planned = [line.split(":")[0] for line in result.stdout.splitlines()]
|
|
self.assertEqual(planned, stage["gateIds"])
|
|
self.assertIn(f"ci:gate {stage['gateIds'][0]}", result.stdout)
|
|
|
|
def test_parts_runs_each_command_of_the_gate(self) -> None:
|
|
contract = gates()
|
|
gate = next(g for g in contract["gates"] if g["id"] == "FE-GATE-005")
|
|
scripts = {c["id"]: c["script"] for c in contract["commands"]}
|
|
result = sh([str(VERIFY), "parts", "FE-GATE-005", "--dry-run"], cwd=REPO)
|
|
self.assertEqual([line.split(": ", 1)[1] for line in result.stdout.splitlines()],
|
|
[f"corepack pnpm -s {scripts[c]}" for c in gate["commandIds"]])
|
|
|
|
def test_a_gate_with_a_host_environment_runs_in_it_as_in_the_workflow(self) -> None:
|
|
# FE-GATE-013 needs systemd and bubblewrap; the workflow and verify.sh both run it through
|
|
# scripts/run-in-system-container.sh (config/ci/gates.json hostEnvironment).
|
|
contract = gates()
|
|
hosted = [g for g in contract["gates"] if g.get("hostEnvironment") == "system-container"]
|
|
self.assertEqual([g["id"] for g in hosted], ["FE-GATE-013"])
|
|
scripts = {c["id"]: c["script"] for c in contract["commands"]}
|
|
wrapper = "scripts/run-in-system-container.sh"
|
|
result = sh([str(VERIFY), "gate", "FE-GATE-013", "FE-GATE-005", "--dry-run"], cwd=REPO)
|
|
self.assertEqual(result.stdout.splitlines(), [f"FE-GATE-013: {wrapper} corepack pnpm -s ci:gate FE-GATE-013",
|
|
"FE-GATE-005: corepack pnpm -s ci:gate FE-GATE-005"])
|
|
result = sh([str(VERIFY), "parts", "FE-GATE-013", "--dry-run"], cwd=REPO)
|
|
self.assertEqual([line.split(": ", 1)[1] for line in result.stdout.splitlines()],
|
|
[f"{wrapper} corepack pnpm -s {scripts[c]}" for c in hosted[0]["commandIds"]])
|
|
workflow = (REPO / ".gitea/workflows/quality-gates.yml").read_text(encoding="utf-8")
|
|
self.assertIn(f"run: {wrapper} corepack pnpm ci:gate -- FE-GATE-013", workflow)
|
|
# A host that cannot start the container runs the gate here (gates.md 3절).
|
|
result = sh([str(VERIFY), "gate", "FE-GATE-013", "--dry-run"], cwd=REPO, env={"VERIFY_HOST_ENVIRONMENT": "off"})
|
|
self.assertEqual(result.stdout.splitlines(), ["FE-GATE-013: corepack pnpm -s ci:gate FE-GATE-013"])
|
|
|
|
def test_parts_passes_each_command_its_arguments_as_ci_gate_does(self) -> None:
|
|
# run-ci-gate.ts runs `pnpm <script> ...args`. Without the args FE-GATE-009's
|
|
# `review:a11y-manual -- --format-only` checked the release evidence instead and failed.
|
|
contract = gates()
|
|
commands = {c["id"]: c for c in contract["commands"]}
|
|
with_args = [g for g in contract["gates"]
|
|
if any(commands[c].get("args") for c in g["commandIds"])]
|
|
self.assertTrue(with_args, "the contract has no command with arguments to check")
|
|
for gate in with_args:
|
|
with self.subTest(gate=gate["id"]):
|
|
result = sh([str(VERIFY), "parts", gate["id"], "--dry-run"], cwd=REPO)
|
|
expected = []
|
|
prefix = ["scripts/run-in-system-container.sh"] if gate.get("hostEnvironment") else []
|
|
for command_id in gate["commandIds"]:
|
|
command = commands[command_id]
|
|
words = [*prefix, "corepack", "pnpm", "-s", command["script"], *command.get("args", [])]
|
|
expected.append(" ".join(words))
|
|
self.assertEqual([line.split(": ", 1)[1] for line in result.stdout.splitlines()], expected)
|
|
|
|
def test_parts_passes_a_negative_fixture_only_with_its_registered_diagnostic(self) -> None:
|
|
# run-ci-gate.ts passes an expect=fail command only when it exits with its code and prints its
|
|
# diagnostic. parts saw the exit code alone, so a checker broken into an unrelated exit 1 (a
|
|
# TypeError) still counted as its negative fixture passing.
|
|
contract = gates()
|
|
commands = {c["id"]: c for c in contract["commands"]}
|
|
gate = next(g for g in contract["gates"] if g["id"] == "FE-GATE-029")
|
|
negatives = [commands[c] for c in gate["commandIds"] if commands[c].get("expect") == "fail"]
|
|
self.assertTrue(any(c.get("expectedDiagnosticId") for c in negatives))
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fake = Path(tmp) / "corepack"
|
|
# Answers `corepack pnpm -s <script> ...args` as the contract registers that command:
|
|
# pass commands exit 0, negative fixtures exit with their code and print DIAGNOSTIC_MODE.
|
|
fake.write_text(f"""#!/usr/bin/env python3
|
|
import json, os, sys
|
|
script, args = sys.argv[3], sys.argv[4:]
|
|
for command in json.load(open({str(REPO / "config/ci/gates.json")!r}))["commands"]:
|
|
if command["script"] == script and command.get("args", []) == args:
|
|
break
|
|
else:
|
|
sys.exit(97)
|
|
if command.get("expect") != "fail":
|
|
sys.exit(0)
|
|
if os.environ["DIAGNOSTIC_MODE"] == "registered":
|
|
print("checker refused the fixture:", command.get("expectedDiagnosticId", ""))
|
|
else:
|
|
print("TypeError: Cannot read properties of undefined")
|
|
sys.exit(command.get("expectedExitCode", 1))
|
|
""", encoding="utf-8")
|
|
fake.chmod(0o755)
|
|
env = {"PATH": f"{tmp}:{os.environ['PATH']}"}
|
|
wrong = sh([str(VERIFY), "parts", "FE-GATE-029", "--on", "here"], cwd=REPO, check=False,
|
|
env={**env, "DIAGNOSTIC_MODE": "unrelated"})
|
|
self.assertEqual(wrong.returncode, 1, wrong.stdout + wrong.stderr)
|
|
self.assertIn("diagnostic", wrong.stderr)
|
|
right = sh([str(VERIFY), "parts", "FE-GATE-029", "--on", "here"], cwd=REPO, check=False,
|
|
env={**env, "DIAGNOSTIC_MODE": "registered"})
|
|
self.assertEqual(right.returncode, 0, right.stdout + right.stderr)
|
|
|
|
def test_arguments_after_the_separator_go_to_the_one_script_or_command_as_given(self) -> None:
|
|
result = sh([str(VERIFY), "script", "test:sample-removal", "--dry-run", "--", "--feature=invoice-feature"],
|
|
cwd=REPO)
|
|
self.assertEqual(result.stdout.strip(),
|
|
"test:sample-removal: corepack pnpm -s test:sample-removal --feature=invoice-feature")
|
|
result = sh([str(VERIFY), "exec", "--dry-run", "--", "node", "scripts/run-vitest.ts", "run", "tests/a b.test.ts"],
|
|
cwd=REPO)
|
|
self.assertEqual(result.stdout.strip(), r"exec node: node scripts/run-vitest.ts run tests/a\ b.test.ts")
|
|
|
|
def test_wrong_arguments_are_refused_here_before_anything_is_sent(self) -> None:
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
calls = Path(folder) / "calls"
|
|
fake_ssh = Path(folder) / "ssh"
|
|
fake_ssh.write_text(f'#!/bin/sh\necho "$@" >> {calls}\n', encoding="utf-8")
|
|
fake_ssh.chmod(0o755)
|
|
for args in (["nope"], ["gate", "FE-GATE-999"], ["parts"], ["gate"], ["merge", "extra"], ["docs", "x"],
|
|
["script", "a", "b", "--", "--x"], ["exec"], ["exec", "--"], ["script", "--", "x"]):
|
|
with self.subTest(args=args):
|
|
result = sh([str(VERIFY), *args], cwd=REPO, env={"VERIFY_SSH": str(fake_ssh)}, check=False)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertFalse(calls.exists(), "a refused command must not reach the remote host")
|
|
|
|
|
|
class CiEnvironmentTest(unittest.TestCase):
|
|
def test_the_ci_env_names_the_head_commit_and_its_time(self) -> None:
|
|
# parts and script run the gate commands without ci:gate, which would otherwise set SOURCE_DATE_EPOCH.
|
|
script = (f'eval "$(sed -n "/^ci_env()/,/^}}/p" {VERIFY})"; ci_env; '
|
|
'echo "$CI $VITE_COMMIT_SHA $SOURCE_DATE_EPOCH $CI_RUNNER_IMAGE"')
|
|
ci, sha, epoch, image = sh(["bash", "-c", script], cwd=REPO).stdout.split()
|
|
self.assertEqual(ci, "true")
|
|
self.assertEqual(sha, sh(["git", "rev-parse", "HEAD"], cwd=REPO).stdout.strip())
|
|
self.assertEqual(epoch, sh(["git", "log", "-1", "--format=%ct"], cwd=REPO).stdout.strip())
|
|
self.assertRegex(image, r"@sha256:[0-9a-f]{64}$")
|
|
|
|
|
|
class RemoteTransportTest(unittest.TestCase):
|
|
"""--on <호스트>: the patch lands in a fresh worktree, the tree matches, the run is logged, the worktree goes."""
|
|
|
|
def test_the_working_tree_is_run_remotely_on_the_same_tree_and_cleaned_up(self) -> None:
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
base = Path(folder)
|
|
project, bare, remote, logs = (base / name for name in ("project", "origin.git", "remote", "logs"))
|
|
(project / "scripts").mkdir(parents=True)
|
|
(project / "config/ci").mkdir(parents=True)
|
|
shutil.copy(VERIFY, project / "scripts/verify.sh")
|
|
shutil.copy(REPO / "config/ci/gates.json", project / "config/ci/gates.json")
|
|
shutil.copy(REPO / ".nvmrc", project / ".nvmrc")
|
|
(project / "scripts/check-work-docs.py").write_text("print('docs checked')\n", encoding="utf-8")
|
|
sh(["git", "init", "-q", "-b", "main"], cwd=project)
|
|
sh(["git", "add", "-A"], cwd=project)
|
|
sh(["git", "commit", "-q", "-m", "init"], cwd=project)
|
|
sh(["git", "clone", "-q", "--bare", str(project), str(bare)], cwd=base)
|
|
sh(["git", "remote", "add", "origin", str(bare)], cwd=project)
|
|
sh(["git", "fetch", "-q", "origin"], cwd=project)
|
|
sh(["git", "clone", "-q", str(bare), str(remote)], cwd=base)
|
|
(project / "notes.md").write_text("커밋 안 한 새 파일\n", encoding="utf-8")
|
|
with (project / "scripts/check-work-docs.py").open("a", encoding="utf-8") as handle:
|
|
handle.write("print('changed here')\n")
|
|
fake_ssh = base / "fake-ssh"
|
|
fake_ssh.write_text('#!/bin/sh\nshift\nexec bash -c "$*"\n', encoding="utf-8")
|
|
fake_ssh.chmod(0o755)
|
|
env = {"VERIFY_SSH": str(fake_ssh), "VERIFY_REMOTE_REPO": str(remote), "VERIFY_LOCK": str(base / "lock"),
|
|
"VERIFY_LOG_ROOT": str(logs), "VERIFY_NODE_BIN": "/nonexistent"}
|
|
temp_index = base / "index"
|
|
shutil.copy(project / ".git/index", temp_index)
|
|
sh(["git", "add", "-A"], cwd=project, env={"GIT_INDEX_FILE": str(temp_index)})
|
|
here = sh(["git", "write-tree"], cwd=project, env={"GIT_INDEX_FILE": str(temp_index)}).stdout.strip()
|
|
|
|
result = sh(["scripts/verify.sh", "docs", "--on", "build-host", "--log", "t"], cwd=project, env=env,
|
|
check=False)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
summary = (logs / "t/summary.txt").read_text(encoding="utf-8")
|
|
self.assertTrue(summary.startswith(f"tree {here} commit "), summary)
|
|
self.assertIn("rc=0", summary)
|
|
self.assertIn("changed here", (logs / "t/run.log").read_text(encoding="utf-8"))
|
|
self.assertFalse((logs / "t/install.log").exists(), "a laptop stage installs nothing")
|
|
worktrees = sh(["git", "worktree", "list"], cwd=remote).stdout.strip().splitlines()
|
|
self.assertEqual(len(worktrees), 1, worktrees)
|
|
self.assertEqual(sh(["git", "status", "--porcelain"], cwd=remote).stdout, "")
|
|
self.assertEqual((logs / "t/changes.patch").read_text(encoding="utf-8"), "")
|
|
|
|
made = sh(["scripts/verify.sh", "exec", "--on", "build-host", "--log", "t2", "--apply-back", "--", "bash", "-c",
|
|
"echo made > made.txt && echo more >> notes.md"], cwd=project,
|
|
env={**env, "VERIFY_INSTALL": "0"}, check=False)
|
|
self.assertEqual(made.returncode, 0, made.stdout + made.stderr)
|
|
self.assertEqual((project / "made.txt").read_text(encoding="utf-8"), "made\n")
|
|
self.assertEqual((project / "notes.md").read_text(encoding="utf-8"), "커밋 안 한 새 파일\nmore\n")
|
|
self.assertIn("2개를 이 작업 트리에 얹었다", made.stderr)
|
|
|
|
missing = sh(["scripts/verify.sh", "docs", "--on", "build-host"], cwd=project,
|
|
env={**env, "VERIFY_REMOTE_REPO": str(base / "missing")}, check=False)
|
|
self.assertEqual(missing.returncode, 4)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|