diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a50ebc0..e203972f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,8 @@ Gitea runner에서 merge 단계를 초록으로 만들고, worker를 늘리지 | 위험 커버리지가 `"use strict"` 밖의 지시문(`"use client"` 등)을 문장으로 센다. V8 coverage가 그렇게 센다(`check:v8-coverage-counter-semantics` fixture에 지시문 모듈 둘) | `scripts/lib/risk-coverage.ts`, `scripts/lib/v8-coverage-counter-semantics.ts` | `0c3c9d33` | | e2e fixture build를 브라우저 project마다가 아니라 run에 한 번 만든다. global setup이 공유 폴더를 만들고 처음 닿은 worker만 build한다. 자기 spec에서 `prepareServiceWorkerBuilds`를 부르면 `key`를 준다 | `playwright.config.ts`, `tests/e2e/support/shared-fixture-builds.ts`, `tests/e2e/support/fixture-builds.ts`, `tests/e2e/support/service-worker-builds.ts` | `c6d7f43a` | | FE-GATE-013의 시스템 컨테이너가 systemd 전에 root cgroup 제어기를 켠다. 부팅 중 `docker exec`가 root cgroup에 남으면 systemd가 user manager에 제어기를 넘기지 못해 provider 시험 18개가 exit 125로 떨어졌다(desktop 4회 중 1회). 사전 점검이 user manager의 cpu·memory·pids 제어기를 보고, provider 실패 메시지에 우리 단계가 쓴 고정 이유 줄을 남긴다 | `scripts/run-in-system-container.sh`, `scripts/check-system-test-prerequisites.ts`, `scripts/lib/provider-sandbox-diagnostics.ts`, `scripts/run-and-validate-provider.ts`, `scripts/lib/provider-scope-wrapper.ts` | `6d6fd61a`, `4e1a385f`, `13b40934` | +| `ci:gate`가 실패하면 job 로그의 끝 줄 앞에 실패한 명령의 첫 실패 블록을 「first failure in the log」로 싣는다. Playwright는 번호 붙은 실패 줄부터, vitest는 `Failed Suites`·`Failed Tests`·`Unhandled Errors` 줄부터 싣고, 길이 상한은 `FIRST_FAILURE_MAX_LINES`다. 끝 줄만 남던 때는 단언의 기대·받은 값이 밀려나 artifact를 받아야 원인이 보였다. 블록을 못 찾으면 지금처럼 끝 줄만 보인다 | `scripts/lib/ci-gate-failure.ts`, `tests/unit/ci-gate-failure.test.ts` | `829733cf` | +| 시험 증거 전체 검사(FE-GATE-008)가 checkout에 없는 Playwright 설정에서 ENOENT 스택으로 죽지 않고, 그 파일과 출처(`config/repository-layout.json`의 `playwright-configs` 항목, 또는 `--playwright-config`)를 대는 실패 줄을 보고서와 로그에 남긴다. 릴리스 증거 토큰을 읽는 `playwright.config.ts`가 없을 때도 같다. 게이트 판정은 그대로 실패다. capability를 지우며 설정 파일만 지우고 등록부 항목을 남기면 이 실패가 난다. 매뉴얼 03 E절이 루트 항목을 지울 때 등록부 항목과 secret-scan `trackedRoots`도 지우라고 적는다 | `scripts/check-test-evidence.ts`, `tests/unit/test-evidence-repository-configs.test.ts`, `docs/manual/03-outside-feature-folder.md` | `439d3271` | ## 미출시 — `8327e221` → 하네스와 작업 매뉴얼 diff --git a/docs/manual/03-outside-feature-folder.md b/docs/manual/03-outside-feature-folder.md index c8b939d9..0d64202d 100644 --- a/docs/manual/03-outside-feature-folder.md +++ b/docs/manual/03-outside-feature-folder.md @@ -249,6 +249,8 @@ src/features/product-capabilities.ts (SERVICE_WORKER = ACTIVE) 3. `secret-scan` 집합을 바꿨으면 `config/security/secret-scan-policy.json`의 `trackedRoots`도 같게 고친다. 4. 새 tsconfig라면 R9를 먼저 본다. 다른 tsconfig와 옵션이 같으면 만들지 않는다. +루트 항목을 지울 때(capability와 그 시험 설정을 함께 지울 때 등)는 `config/repository-layout.json`의 그 항목도 지운다. `secret-scan` 집합에 있던 항목이면 `config/security/secret-scan-policy.json`의 `trackedRoots`에서도 뺀다. 항목을 남기면 그 집합을 읽는 검사가 실패한다. 예를 들어 `playwright-configs` 집합의 설정 파일만 지우면 FE-GATE-008의 `scripts/check-test-evidence.ts`가 그 파일과 등록부를 이름으로 대며 실패한다. + ## 기능을 지울 때 `corepack pnpm test:sample-removal -- --feature=`는 복사본에서 지워 보고 게이트를 돈다. 실제 저장소에서 지울 때는 아래를 함께 지운다. diff --git a/scripts/check-test-evidence.ts b/scripts/check-test-evidence.ts index f7a5357b..5caa0068 100644 --- a/scripts/check-test-evidence.ts +++ b/scripts/check-test-evidence.ts @@ -12,7 +12,7 @@ import { type HttpScenarioExpectation, type HttpScenarioReceipt, } from "./lib/http-scenario-evidence.ts"; -import { selectRepositoryPaths } from "./lib/repository-layout.ts"; +import { REPOSITORY_LAYOUT_FILE, selectRepositoryPaths } from "./lib/repository-layout.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts"; @@ -372,14 +372,35 @@ if (parsedPolicy) { } } -async function missingForbidOnly(configs: readonly string[]): Promise { - const missing: string[] = []; +/** + * A config this checkout lacks fails by name, with where it was named. The + * full check used to end on ENOENT before it wrote its report: a product that + * deletes playwright.capabilities.config.ts with its capability and keeps the + * registry entry saw a stack trace in FE-GATE-008 (fe-ci-next (3)). + */ +async function playwrightConfigFailures( + configs: readonly string[], + namedBy: "registry" | "argument", +): Promise { + const failures: string[] = []; for (const config of configs) { - if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) { - missing.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`); + let source: string; + try { + source = await readFile(config, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + failures.push( + namedBy === "registry" + ? `${config}: ${REPOSITORY_LAYOUT_FILE} lists it in playwright-configs, but this checkout has no such file; restore it or remove its entry` + : `${config}: --playwright-config names it, but this checkout has no such file`, + ); + continue; + } + if (!source.includes("forbidOnly: true")) { + failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`); } } - return missing; + return failures; } // The configs the run names, in every mode but the scenario check; without @@ -395,11 +416,29 @@ if (!scenarioOnly) { : fixtureMode ? [] : [...selectRepositoryPaths("playwright-configs")]; - failures.push(...(await missingForbidOnly(facts.playwrightConfigs))); + failures.push( + ...(await playwrightConfigFailures( + facts.playwrightConfigs, + explicitPlaywrightConfigs.length > 0 ? "argument" : "registry", + )), + ); } if (!fixtureMode && !scenarioOnly) { - const e2eConfig = await readFile("playwright.config.ts", "utf8"); + // Without the e2e config there are no tokens to read; the config check + // above has named it when the run's configs include it (from the registry or + // --playwright-config), and this names it when they do not. + let e2eConfig: string | undefined; + try { + e2eConfig = await readFile("playwright.config.ts", "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (!facts.playwrightConfigs.includes("playwright.config.ts")) { + failures.push( + "playwright.config.ts: the full check reads its release evidence tokens, but this checkout has no such file", + ); + } + } for (const token of [ "pnpm build", "pnpm preview", @@ -410,7 +449,7 @@ if (!fixtureMode && !scenarioOnly) { '"firefox"', '"webkit"', ]) { - if (!e2eConfig.includes(token)) { + if (e2eConfig !== undefined && !e2eConfig.includes(token)) { failures.push(`playwright.config.ts missing release evidence token ${token}`); } } diff --git a/tests/unit/test-evidence-repository-configs.test.ts b/tests/unit/test-evidence-repository-configs.test.ts index 267861f7..d86dcd16 100644 --- a/tests/unit/test-evidence-repository-configs.test.ts +++ b/tests/unit/test-evidence-repository-configs.test.ts @@ -1,11 +1,40 @@ import { spawnSync } from "node:child_process"; -import { readFile, rm } from "node:fs/promises"; +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts"; +import { REPOSITORY_LAYOUT_FILE, selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts"; import { childProcessTimeout } from "../helpers/child-process-timeout.ts"; const ARTIFACT = "artifacts/quality/test-evidence-repository-configs.json"; +const CHECKER = path.resolve("scripts/check-test-evidence.ts"); +/** + * Longest desktop run of a test in this file: the first one, in FE-GATE-005's + * test:unit at load 14 (2026-11-20); its test:coverage run there was shorter. + * The ones that run in a small checkout took 131 to 233 ms alone. + */ +const CHECK_MS = 746; + +type Report = { facts: { playwrightConfigs?: string[] }; failures: string[] }; + +/** + * The full check (no --source-root) in a checkout that has only what the + * test copied into it and a policy with no scenario catalog or source + * contract, so every other failure it reports names a missing file without + * reading one. + */ +async function fullCheckIn(root: string, ...options: string[]) { + const policy = path.join(root, "test-evidence-policy.json"); + await writeFile(policy, JSON.stringify({ schemaVersion: 2, scenarioCatalogs: [], sourceContracts: [] })); + const artifact = path.join(root, "report.json"); + const result = spawnSync( + process.execPath, + [CHECKER, "--source-only", "--policy", policy, "--artifact", artifact, ...options], + { cwd: root, encoding: "utf8" }, + ); + return { result, report: async () => JSON.parse(await readFile(artifact, "utf8")) as Report }; +} // The full check reads every Playwright config of the registry for forbidOnly; // a fixture run reads only the configs it is given @@ -21,10 +50,91 @@ describe("test evidence full check", () => { { encoding: "utf8" }, ); expect(result.status, result.stderr).toBe(0); - const report = JSON.parse(await readFile(ARTIFACT, "utf8")) as { - facts: { playwrightConfigs?: string[] }; - }; + const report = JSON.parse(await readFile(ARTIFACT, "utf8")) as Report; expect(report.facts.playwrightConfigs).toEqual([...selectRepositoryPaths("playwright-configs")]); expect(report.facts.playwrightConfigs?.length).toBeGreaterThan(0); - }, childProcessTimeout(232)); + }, childProcessTimeout(CHECK_MS)); + + // fe-ci-next (3). A product that deletes a Playwright config and keeps its + // registry entry (playwright.capabilities.config.ts with its capability) + // ended the full check on ENOENT, before it wrote its report, so FE-GATE-008 + // showed a stack trace instead of what to fix. Now it fails by name: the + // config, the registry, and the two ways out. Each run is in a checkout that + // has every other registered config and nothing more, so the check reports + // what that checkout lacks instead of reading this repository. + const configs = [...selectRepositoryPaths("playwright-configs")]; + it.each(configs)("names %s when the checkout lacks it, with the registry entry to remove", async (lacking) => { + const root = await mkdtemp(path.join(tmpdir(), "test-evidence-registry-")); + try { + for (const config of configs.filter((config) => config !== lacking)) { + await mkdir(path.dirname(path.join(root, config)), { recursive: true }); + await copyFile(config, path.join(root, config)); + } + const { result, report: read } = await fullCheckIn(root); + + expect(result.status).toBe(1); + expect(result.stderr).not.toMatch(/ENOENT|\n {4}at /u); + const report = await read(); + expect(report.facts.playwrightConfigs).toEqual(configs); + const named = `${lacking}: ${REPOSITORY_LAYOUT_FILE} lists it in playwright-configs, but this checkout has no such file; restore it or remove its entry`; + expect(report.failures.filter((failure) => configs.some((config) => failure.startsWith(`${config}:`)))).toEqual([ + named, + ]); + expect(result.stderr).toContain(named); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, childProcessTimeout(CHECK_MS)); + + // With configs given, the registry's are not read, so the e2e config the + // full check reads for its release evidence tokens is named on its own. + it("names a config the run gives and the e2e config when the checkout lacks them", async () => { + const root = await mkdtemp(path.join(tmpdir(), "test-evidence-registry-")); + try { + const { result, report: read } = await fullCheckIn(root, "--playwright-config", "absent.config.ts"); + + expect(result.status).toBe(1); + expect(result.stderr).not.toMatch(/ENOENT|\n {4}at /u); + const report = await read(); + expect(report.facts.playwrightConfigs).toEqual(["absent.config.ts"]); + expect( + report.failures.filter((failure) => /^(?:absent|playwright)\.config\.ts:/u.test(failure)), + ).toEqual([ + "absent.config.ts: --playwright-config names it, but this checkout has no such file", + "playwright.config.ts: the full check reads its release evidence tokens, but this checkout has no such file", + ]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, childProcessTimeout(CHECK_MS)); + + // Only a missing file becomes a named failure. Another read error (a folder + // where the config should be, no permission) is not one this check can + // explain, so the check still ends on it, in each place it reads a config. + // The folder stands where only that place reads: a registered config other + // than the e2e one (the token read after the config check then succeeds), or + // the e2e config in a run that gives a readable config instead. + const readErrorRuns: [string, string, string[]][] = [ + ["the config check", configs.find((config) => config !== "playwright.config.ts") ?? "", []], + ["the release evidence token read", "playwright.config.ts", ["--playwright-config", "given.config.ts"]], + ]; + it.each(readErrorRuns)("ends on a read error other than a missing file in %s", async (_, folder, options) => { + expect(configs).toContain(folder); + const root = await mkdtemp(path.join(tmpdir(), "test-evidence-registry-")); + try { + for (const config of configs.filter((config) => config !== folder)) { + await mkdir(path.dirname(path.join(root, config)), { recursive: true }); + await copyFile(config, path.join(root, config)); + } + await mkdir(path.join(root, folder), { recursive: true }); + await writeFile(path.join(root, "given.config.ts"), "export default { forbidOnly: true };\n"); + const { result } = await fullCheckIn(root, ...options); + + expect(result.status).not.toBe(0); + expect(result.stderr).toMatch(/EISDIR/u); + expect(result.stderr).not.toContain("this checkout has no such file"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, childProcessTimeout(CHECK_MS)); });