fix(checkers): 시험 증거 검사가 checkout에 없는 Playwright 설정을 이름으로 댄다 #69
@@ -53,6 +53,8 @@ Gitea runner에서 merge 단계를 초록으로 만들고, worker를 늘리지
|
||||
| 위험 커버리지가 `"use strict"` 밖의 지시문(`"use client"` 등)을 문장으로 센다. V8 coverage가 그렇게 센다(`check:v8-coverage-counter-semantics` fixture에 지시문 모듈 둘) | `scripts/lib/risk-coverage.ts`, `scripts/lib/v8-coverage-counter-semantics.ts` | `0c3c9d33` |
|
||||
| e2e fixture build를 브라우저 project마다가 아니라 run에 한 번 만든다. global setup이 공유 폴더를 만들고 처음 닿은 worker만 build한다. 자기 spec에서 `prepareServiceWorkerBuilds`를 부르면 `key`를 준다 | `playwright.config.ts`, `tests/e2e/support/shared-fixture-builds.ts`, `tests/e2e/support/fixture-builds.ts`, `tests/e2e/support/service-worker-builds.ts` | `c6d7f43a` |
|
||||
| FE-GATE-013의 시스템 컨테이너가 systemd 전에 root cgroup 제어기를 켠다. 부팅 중 `docker exec`가 root cgroup에 남으면 systemd가 user manager에 제어기를 넘기지 못해 provider 시험 18개가 exit 125로 떨어졌다(desktop 4회 중 1회). 사전 점검이 user manager의 cpu·memory·pids 제어기를 보고, provider 실패 메시지에 우리 단계가 쓴 고정 이유 줄을 남긴다 | `scripts/run-in-system-container.sh`, `scripts/check-system-test-prerequisites.ts`, `scripts/lib/provider-sandbox-diagnostics.ts`, `scripts/run-and-validate-provider.ts`, `scripts/lib/provider-scope-wrapper.ts` | `6d6fd61a`, `4e1a385f`, `13b40934` |
|
||||
| `ci:gate`가 실패하면 job 로그의 끝 줄 앞에 실패한 명령의 첫 실패 블록을 「first failure in the log」로 싣는다. Playwright는 번호 붙은 실패 줄부터, vitest는 `Failed Suites`·`Failed Tests`·`Unhandled Errors` 줄부터 싣고, 길이 상한은 `FIRST_FAILURE_MAX_LINES`다. 끝 줄만 남던 때는 단언의 기대·받은 값이 밀려나 artifact를 받아야 원인이 보였다. 블록을 못 찾으면 지금처럼 끝 줄만 보인다 | `scripts/lib/ci-gate-failure.ts`, `tests/unit/ci-gate-failure.test.ts` | `829733cf` |
|
||||
| 시험 증거 전체 검사(FE-GATE-008)가 checkout에 없는 Playwright 설정에서 ENOENT 스택으로 죽지 않고, 그 파일과 출처(`config/repository-layout.json`의 `playwright-configs` 항목, 또는 `--playwright-config`)를 대는 실패 줄을 보고서와 로그에 남긴다. 릴리스 증거 토큰을 읽는 `playwright.config.ts`가 없을 때도 같다. 게이트 판정은 그대로 실패다. capability를 지우며 설정 파일만 지우고 등록부 항목을 남기면 이 실패가 난다. 매뉴얼 03 E절이 루트 항목을 지울 때 등록부 항목과 secret-scan `trackedRoots`도 지우라고 적는다 | `scripts/check-test-evidence.ts`, `tests/unit/test-evidence-repository-configs.test.ts`, `docs/manual/03-outside-feature-folder.md` | `439d3271` |
|
||||
|
||||
## 미출시 — `8327e221` → 하네스와 작업 매뉴얼
|
||||
|
||||
|
||||
@@ -249,6 +249,8 @@ src/features/product-capabilities.ts (SERVICE_WORKER = ACTIVE)
|
||||
3. `secret-scan` 집합을 바꿨으면 `config/security/secret-scan-policy.json`의 `trackedRoots`도 같게 고친다.
|
||||
4. 새 tsconfig라면 R9를 먼저 본다. 다른 tsconfig와 옵션이 같으면 만들지 않는다.
|
||||
|
||||
루트 항목을 지울 때(capability와 그 시험 설정을 함께 지울 때 등)는 `config/repository-layout.json`의 그 항목도 지운다. `secret-scan` 집합에 있던 항목이면 `config/security/secret-scan-policy.json`의 `trackedRoots`에서도 뺀다. 항목을 남기면 그 집합을 읽는 검사가 실패한다. 예를 들어 `playwright-configs` 집합의 설정 파일만 지우면 FE-GATE-008의 `scripts/check-test-evidence.ts`가 그 파일과 등록부를 이름으로 대며 실패한다.
|
||||
|
||||
## 기능을 지울 때
|
||||
|
||||
`corepack pnpm test:sample-removal -- --feature=<id>`는 복사본에서 지워 보고 게이트를 돈다. 실제 저장소에서 지울 때는 아래를 함께 지운다.
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
type HttpScenarioExpectation,
|
||||
type HttpScenarioReceipt,
|
||||
} from "./lib/http-scenario-evidence.ts";
|
||||
import { selectRepositoryPaths } from "./lib/repository-layout.ts";
|
||||
import { REPOSITORY_LAYOUT_FILE, selectRepositoryPaths } from "./lib/repository-layout.ts";
|
||||
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
||||
import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts";
|
||||
|
||||
@@ -372,14 +372,35 @@ if (parsedPolicy) {
|
||||
}
|
||||
}
|
||||
|
||||
async function missingForbidOnly(configs: readonly string[]): Promise<string[]> {
|
||||
const missing: string[] = [];
|
||||
/**
|
||||
* A config this checkout lacks fails by name, with where it was named. The
|
||||
* full check used to end on ENOENT before it wrote its report: a product that
|
||||
* deletes playwright.capabilities.config.ts with its capability and keeps the
|
||||
* registry entry saw a stack trace in FE-GATE-008 (fe-ci-next (3)).
|
||||
*/
|
||||
async function playwrightConfigFailures(
|
||||
configs: readonly string[],
|
||||
namedBy: "registry" | "argument",
|
||||
): Promise<string[]> {
|
||||
const failures: string[] = [];
|
||||
for (const config of configs) {
|
||||
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
||||
missing.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(config, "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
failures.push(
|
||||
namedBy === "registry"
|
||||
? `${config}: ${REPOSITORY_LAYOUT_FILE} lists it in playwright-configs, but this checkout has no such file; restore it or remove its entry`
|
||||
: `${config}: --playwright-config names it, but this checkout has no such file`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!source.includes("forbidOnly: true")) {
|
||||
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
return failures;
|
||||
}
|
||||
|
||||
// The configs the run names, in every mode but the scenario check; without
|
||||
@@ -395,11 +416,29 @@ if (!scenarioOnly) {
|
||||
: fixtureMode
|
||||
? []
|
||||
: [...selectRepositoryPaths("playwright-configs")];
|
||||
failures.push(...(await missingForbidOnly(facts.playwrightConfigs)));
|
||||
failures.push(
|
||||
...(await playwrightConfigFailures(
|
||||
facts.playwrightConfigs,
|
||||
explicitPlaywrightConfigs.length > 0 ? "argument" : "registry",
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
if (!fixtureMode && !scenarioOnly) {
|
||||
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
||||
// Without the e2e config there are no tokens to read; the config check
|
||||
// above has named it when the run's configs include it (from the registry or
|
||||
// --playwright-config), and this names it when they do not.
|
||||
let e2eConfig: string | undefined;
|
||||
try {
|
||||
e2eConfig = await readFile("playwright.config.ts", "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
if (!facts.playwrightConfigs.includes("playwright.config.ts")) {
|
||||
failures.push(
|
||||
"playwright.config.ts: the full check reads its release evidence tokens, but this checkout has no such file",
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const token of [
|
||||
"pnpm build",
|
||||
"pnpm preview",
|
||||
@@ -410,7 +449,7 @@ if (!fixtureMode && !scenarioOnly) {
|
||||
'"firefox"',
|
||||
'"webkit"',
|
||||
]) {
|
||||
if (!e2eConfig.includes(token)) {
|
||||
if (e2eConfig !== undefined && !e2eConfig.includes(token)) {
|
||||
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,40 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts";
|
||||
import { REPOSITORY_LAYOUT_FILE, selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts";
|
||||
import { childProcessTimeout } from "../helpers/child-process-timeout.ts";
|
||||
|
||||
const ARTIFACT = "artifacts/quality/test-evidence-repository-configs.json";
|
||||
const CHECKER = path.resolve("scripts/check-test-evidence.ts");
|
||||
/**
|
||||
* Longest desktop run of a test in this file: the first one, in FE-GATE-005's
|
||||
* test:unit at load 14 (2026-11-20); its test:coverage run there was shorter.
|
||||
* The ones that run in a small checkout took 131 to 233 ms alone.
|
||||
*/
|
||||
const CHECK_MS = 746;
|
||||
|
||||
type Report = { facts: { playwrightConfigs?: string[] }; failures: string[] };
|
||||
|
||||
/**
|
||||
* The full check (no --source-root) in a checkout that has only what the
|
||||
* test copied into it and a policy with no scenario catalog or source
|
||||
* contract, so every other failure it reports names a missing file without
|
||||
* reading one.
|
||||
*/
|
||||
async function fullCheckIn(root: string, ...options: string[]) {
|
||||
const policy = path.join(root, "test-evidence-policy.json");
|
||||
await writeFile(policy, JSON.stringify({ schemaVersion: 2, scenarioCatalogs: [], sourceContracts: [] }));
|
||||
const artifact = path.join(root, "report.json");
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[CHECKER, "--source-only", "--policy", policy, "--artifact", artifact, ...options],
|
||||
{ cwd: root, encoding: "utf8" },
|
||||
);
|
||||
return { result, report: async () => JSON.parse(await readFile(artifact, "utf8")) as Report };
|
||||
}
|
||||
|
||||
// The full check reads every Playwright config of the registry for forbidOnly;
|
||||
// a fixture run reads only the configs it is given
|
||||
@@ -21,10 +50,91 @@ describe("test evidence full check", () => {
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const report = JSON.parse(await readFile(ARTIFACT, "utf8")) as {
|
||||
facts: { playwrightConfigs?: string[] };
|
||||
};
|
||||
const report = JSON.parse(await readFile(ARTIFACT, "utf8")) as Report;
|
||||
expect(report.facts.playwrightConfigs).toEqual([...selectRepositoryPaths("playwright-configs")]);
|
||||
expect(report.facts.playwrightConfigs?.length).toBeGreaterThan(0);
|
||||
}, childProcessTimeout(232));
|
||||
}, childProcessTimeout(CHECK_MS));
|
||||
|
||||
// fe-ci-next (3). A product that deletes a Playwright config and keeps its
|
||||
// registry entry (playwright.capabilities.config.ts with its capability)
|
||||
// ended the full check on ENOENT, before it wrote its report, so FE-GATE-008
|
||||
// showed a stack trace instead of what to fix. Now it fails by name: the
|
||||
// config, the registry, and the two ways out. Each run is in a checkout that
|
||||
// has every other registered config and nothing more, so the check reports
|
||||
// what that checkout lacks instead of reading this repository.
|
||||
const configs = [...selectRepositoryPaths("playwright-configs")];
|
||||
it.each(configs)("names %s when the checkout lacks it, with the registry entry to remove", async (lacking) => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-registry-"));
|
||||
try {
|
||||
for (const config of configs.filter((config) => config !== lacking)) {
|
||||
await mkdir(path.dirname(path.join(root, config)), { recursive: true });
|
||||
await copyFile(config, path.join(root, config));
|
||||
}
|
||||
const { result, report: read } = await fullCheckIn(root);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).not.toMatch(/ENOENT|\n {4}at /u);
|
||||
const report = await read();
|
||||
expect(report.facts.playwrightConfigs).toEqual(configs);
|
||||
const named = `${lacking}: ${REPOSITORY_LAYOUT_FILE} lists it in playwright-configs, but this checkout has no such file; restore it or remove its entry`;
|
||||
expect(report.failures.filter((failure) => configs.some((config) => failure.startsWith(`${config}:`)))).toEqual([
|
||||
named,
|
||||
]);
|
||||
expect(result.stderr).toContain(named);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, childProcessTimeout(CHECK_MS));
|
||||
|
||||
// With configs given, the registry's are not read, so the e2e config the
|
||||
// full check reads for its release evidence tokens is named on its own.
|
||||
it("names a config the run gives and the e2e config when the checkout lacks them", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-registry-"));
|
||||
try {
|
||||
const { result, report: read } = await fullCheckIn(root, "--playwright-config", "absent.config.ts");
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).not.toMatch(/ENOENT|\n {4}at /u);
|
||||
const report = await read();
|
||||
expect(report.facts.playwrightConfigs).toEqual(["absent.config.ts"]);
|
||||
expect(
|
||||
report.failures.filter((failure) => /^(?:absent|playwright)\.config\.ts:/u.test(failure)),
|
||||
).toEqual([
|
||||
"absent.config.ts: --playwright-config names it, but this checkout has no such file",
|
||||
"playwright.config.ts: the full check reads its release evidence tokens, but this checkout has no such file",
|
||||
]);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, childProcessTimeout(CHECK_MS));
|
||||
|
||||
// Only a missing file becomes a named failure. Another read error (a folder
|
||||
// where the config should be, no permission) is not one this check can
|
||||
// explain, so the check still ends on it, in each place it reads a config.
|
||||
// The folder stands where only that place reads: a registered config other
|
||||
// than the e2e one (the token read after the config check then succeeds), or
|
||||
// the e2e config in a run that gives a readable config instead.
|
||||
const readErrorRuns: [string, string, string[]][] = [
|
||||
["the config check", configs.find((config) => config !== "playwright.config.ts") ?? "", []],
|
||||
["the release evidence token read", "playwright.config.ts", ["--playwright-config", "given.config.ts"]],
|
||||
];
|
||||
it.each(readErrorRuns)("ends on a read error other than a missing file in %s", async (_, folder, options) => {
|
||||
expect(configs).toContain(folder);
|
||||
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-registry-"));
|
||||
try {
|
||||
for (const config of configs.filter((config) => config !== folder)) {
|
||||
await mkdir(path.dirname(path.join(root, config)), { recursive: true });
|
||||
await copyFile(config, path.join(root, config));
|
||||
}
|
||||
await mkdir(path.join(root, folder), { recursive: true });
|
||||
await writeFile(path.join(root, "given.config.ts"), "export default { forbidOnly: true };\n");
|
||||
const { result } = await fullCheckIn(root, ...options);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toMatch(/EISDIR/u);
|
||||
expect(result.stderr).not.toContain("this checkout has no such file");
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, childProcessTimeout(CHECK_MS));
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user