fix(test-evidence): 검사 규칙마다 혼자 도는 거절 시험을 두고 strict import를 lint로 본다

문제: `check-test-evidence.ts`의 세 규칙에 혼자 도는 거절 시험이 없었다.
화면 전체 mask 규칙은 fixture만 있고 시험이 없었고, FE-GATE-029의 디렉터리
실행은 공통 접두어만 기대해 다른 fixture가 실패하면 통과했다. Playwright
설정의 forbidOnly 확인과 strict fixture 우회 확인은 전체 실행에서만 돌아
fixture로 시험할 수 없었다. strict fixture 확인은 `tests/e2e/*.spec.ts`
본문에 경로 글자가 있는지만 보아, 주석에만 적어도 통과했고 다른 브라우저
폴더 셋은 보지 않았다(1112/03-04).

고친 것:
- mask fixture 하나만 돌리는 시험을 더한다.
- forbidOnly 규칙이 시나리오 검사 밖의 모든 실행에서 돌고, 읽은 설정 목록을
  보고의 `facts.playwrightConfigs`에 남긴다. `--playwright-config <파일>`로
  다른 설정을 줄 수 있다. 거절 fixture(forbidOnly 없는 설정)와, 플래그가
  없으면 저장소 설정 전부를 읽는지 보는 시험을 더한다.
- strict fixture 확인은 ESLint `no-restricted-imports`로 옮긴다.
  `tests/{e2e,browser-capabilities,visual,storybook}/**/*.spec.ts`가
  `@playwright/test`의 `test`(이름·기본 import)나 strict fixture가 넓히는
  base test를 import하면 오류다. 일부러 plain test를 쓰는 둘
  (`preload-use.spec.ts`, `chunk-load-error.spec.ts`)만 예외다. 문자열 검사는
  지운다. 네 폴더 각각의 lint fixture를 시험이 lint한다.

위험·한계: w2-fe-2 PR #47이 두 예외 spec의 import를
`tests/support/browser/browser-test.ts`로 바꾼다. 그 경로의 `test`도 이
규칙이 막으므로 예외 둘은 그대로 필요하다.

검증: desktop 변이 셋 — mask 블록 삭제, forbidOnly 호출 삭제, ESLint 규칙
삭제 — 에서 각각 해당 시험이 실패하고(1·1·5건), 고친 코드에서 51건 통과.
`corepack pnpm lint` rc=0.

Refs: 1112/03-03, 1112/03-04
This commit is contained in:
DongHyeonka
2026-11-13 23:29:13 +09:00
parent dec8a9db32
commit 34a7627d36
6 changed files with 175 additions and 18 deletions
+42
View File
@@ -937,6 +937,48 @@ export default [
"react-hooks/rules-of-hooks": "off",
},
},
{
// 1112/03-04. A browser spec takes `test` from the strict fixture, which
// fails it on console errors, page errors, failed requests and unused
// preloads. FE-GATE-008 used to look for the fixture's path as text in
// tests/e2e only, so a path in a comment passed and the other browser
// folders were not looked at. tests/unit/browser-spec-imports.test.ts
// lints a fixture spec in each folder against this entry.
files: [`tests/{e2e,browser-capabilities,visual,storybook}/**/*.spec.ts`],
rules: {
"no-restricted-imports": [
"error",
{
paths: [
{
name: "@playwright/test",
importNames: ["test", "default"],
message:
"import test from tests/support/browser/strict-browser-test.ts; the strict fixture fails a spec on browser errors (.claude/skills/fe-qa).",
},
],
patterns: [
...deprecatedAliasImports,
{
group: ["**/support/browser/browser-test.ts", "**/support/browser/browser-test"],
importNames: ["test"],
message:
"import test from tests/support/browser/strict-browser-test.ts; the base test does not check browser errors.",
},
],
},
],
},
},
{
// 1112/03-04. The two specs that need the plain test on purpose:
// preload-use proves the strict fixture's own record, and chunk-load-error
// provokes the 404 a missing chunk gives to classify the engine's wording.
files: ["tests/e2e/preload-use.spec.ts", "tests/browser-capabilities/chunk-load-error.spec.ts"],
rules: {
"no-restricted-imports": ["error", { patterns: deprecatedAliasImports }],
},
},
{
files: [
`tests/fixtures/browser-file-storage-boundaries/forbidden/**/*.${sourceExtensions}`,
+30 -16
View File
@@ -64,6 +64,12 @@ const policyPath = argumentValue(
"config/testing/test-evidence.json",
);
const explicitCatalogPath = optionalArgumentValue("--catalog");
// 1112/03-03. The forbidOnly rule read only the repository's configs and ran
// only in the full check, so no fixture could show it still fires.
// `--playwright-config <file>` (repeatable) checks those files in any mode.
const explicitPlaywrightConfigs = process.argv.flatMap((argument, index) =>
argument === "--playwright-config" && process.argv[index + 1] ? [process.argv[index + 1]!] : [],
);
const explicitReceiptPath = optionalArgumentValue("--receipt");
const fixtureMode = sourceRoot !== "tests";
const sourceOnly = process.argv.includes("--source-only");
@@ -78,6 +84,7 @@ const facts = {
sharedScenarios: 0,
declaredScenarioExecutions: 0,
executedScenarioExecutions: 0,
playwrightConfigs: [] as string[],
};
async function filesBelow(target: string): Promise<string[]> {
@@ -365,6 +372,26 @@ if (parsedPolicy) {
}
}
async function missingForbidOnly(configs: readonly string[]): Promise<string[]> {
const missing: string[] = [];
for (const config of configs) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
missing.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
return missing;
}
// The repository's configs unless the run names others; every mode but the
// scenario check reads them, and the report lists what it read.
if (!scenarioOnly) {
// playwright.dev.config.ts spreads playwright.config.ts.
facts.playwrightConfigs = explicitPlaywrightConfigs.length > 0
? explicitPlaywrightConfigs
: [...selectRepositoryPaths("playwright-configs")];
failures.push(...(await missingForbidOnly(facts.playwrightConfigs)));
}
if (!fixtureMode && !scenarioOnly) {
const e2eConfig = await readFile("playwright.config.ts", "utf8");
for (const token of [
@@ -381,22 +408,9 @@ if (!fixtureMode && !scenarioOnly) {
failures.push(`playwright.config.ts missing release evidence token ${token}`);
}
}
// playwright.dev.config.ts spreads playwright.config.ts.
for (const config of selectRepositoryPaths("playwright-configs")) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
/\.spec\.ts$/.test(file),
);
for (const file of e2eFiles) {
const source = await readFile(file, "utf8");
if (!source.includes("support/browser/strict-browser-test")) {
failures.push(`${file}: bypasses strict browser fixture`);
}
}
// 1112/03-04. That a browser spec takes the strict fixture's `test` is an
// ESLint rule now (eslint.config.ts, tests/unit/browser-spec-imports.test.ts),
// which reads the import instead of looking for the path as text.
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
(file) => file.endsWith(".png"),
+2
View File
@@ -12,6 +12,8 @@ export const testEvidenceReportSchema = z
sharedScenarios: z.number().int().nonnegative(),
declaredScenarioExecutions: z.number().int().nonnegative(),
executedScenarioExecutions: z.number().int().nonnegative(),
// 1112/03-03. The Playwright configs the forbidOnly rule read.
playwrightConfigs: z.array(z.string().min(1)).optional(),
})
.strict(),
failures: z.array(z.string()),
@@ -0,0 +1,8 @@
// A Playwright config that would let a committed focused test run alone:
// check-test-evidence.ts --playwright-config must reject it (1112/03-03).
import { defineConfig } from "@playwright/test";
export default defineConfig({
testDir: "tests/e2e",
reporter: [["junit", { outputFile: "artifacts/tests/e2e/results.xml" }]],
});
+51
View File
@@ -0,0 +1,51 @@
import path from "node:path";
import { ESLint } from "eslint";
import { describe, expect, it } from "vitest";
import { childProcessTimeout } from "../helpers/child-process-timeout.ts";
const FOLDERS = ["e2e", "browser-capabilities", "visual", "storybook"] as const;
const BARE = 'import { expect, test } from "@playwright/test";\n\ntest("probe", async () => { expect(1).toBe(1); });\n';
const STRICT =
'import { expect, test } from "../support/browser/strict-browser-test.ts";\n' +
'import type { Page } from "@playwright/test";\n\n' +
'test("probe", async ({ page }: { page: Page }) => { expect(page).toBeTruthy(); });\n';
const DEFAULT_IMPORT = 'import test from "@playwright/test";\n\ntest("probe", async () => {});\n';
const BASE_TEST = 'import { test } from "../support/browser/browser-test.ts";\n\ntest("probe", async () => {});\n';
async function restrictedImports(code: string, file: string): Promise<string[]> {
const linter = new ESLint({ cwd: process.cwd() });
const [result] = await linter.lintText(code, { filePath: path.join(process.cwd(), file), warnIgnored: true });
return (result?.messages ?? [])
.filter((message) => message.ruleId === "no-restricted-imports")
.map((message) => message.message);
}
// 1112/03-04. Strict fixture use was a text search for its path in
// tests/e2e/*.spec.ts: a path in a comment passed, a file could mix in the
// bare test, and tests/browser-capabilities, tests/visual and tests/storybook
// were not looked at. It is an import rule in each of the four folders now.
describe("browser specs take test from the strict fixture (1112/03-04)", () => {
it.each(FOLDERS)("rejects the bare Playwright test in tests/%s", async (folder) => {
const messages = await restrictedImports(BARE, `tests/${folder}/lint-probe.spec.ts`);
expect(messages).toEqual([expect.stringContaining("tests/support/browser/strict-browser-test.ts")]);
}, childProcessTimeout(683));
it("rejects the default import, the base test the strict fixture extends, and accepts the strict test", async () => {
expect(await restrictedImports(DEFAULT_IMPORT, "tests/visual/lint-probe.spec.ts")).toEqual([
expect.stringContaining("tests/support/browser/strict-browser-test.ts"),
]);
expect(await restrictedImports(BASE_TEST, "tests/e2e/lint-probe.spec.ts")).toEqual([
expect.stringContaining("the base test does not check browser errors"),
]);
expect(await restrictedImports(STRICT, "tests/browser-capabilities/lint-probe.spec.ts")).toEqual([]);
}, childProcessTimeout(683));
it("lets only the two specs that need the plain test import it", async () => {
for (const file of ["tests/e2e/preload-use.spec.ts", "tests/browser-capabilities/chunk-load-error.spec.ts"]) {
expect(await restrictedImports(BARE, file), file).toEqual([]);
}
}, childProcessTimeout(683));
});
+42 -2
View File
@@ -1,9 +1,10 @@
import { spawnSync } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts";
import { childProcessTimeout } from "../helpers/child-process-timeout.ts";
const FORBIDDEN = "tests/fixtures/test-evidence/forbidden";
@@ -13,7 +14,7 @@ afterAll(async () => {
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })));
});
function checkSource(sourceRoot: string) {
function checkSource(sourceRoot: string, extra: readonly string[] = []) {
const result = spawnSync(
process.execPath,
[
@@ -23,6 +24,7 @@ function checkSource(sourceRoot: string) {
sourceRoot,
"--artifact",
"artifacts/quality/test-evidence-source-rules.json",
...extra,
],
{ encoding: "utf8" },
);
@@ -44,6 +46,44 @@ describe("test evidence source rules", () => {
expect(result.stderr).toContain("quarantine expired on 2020-01-01");
}, childProcessTimeout(739));
// 1112/03-03. The mask fixture had no test of its own: the directory run
// expects only the common prefix, so this rule could stop firing unseen.
it("rejects a screenshot mask over the whole UI", () => {
const result = checkSource(`${FORBIDDEN}/full-screen-mask.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("screenshot mask may not cover the whole UI");
}, childProcessTimeout(141));
// 1112/03-03. The forbidOnly rule ran only in the full check, on the
// repository's own configs, so nothing showed it still rejects a config.
it("rejects a Playwright config without forbidOnly and accepts the repository's", async () => {
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-config-"));
roots.push(root);
await writeFile(path.join(root, "clean.fixture"), "export {};\n");
const missing = checkSource(root, [
"--playwright-config",
"tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture",
]);
expect(missing.status).toBe(1);
expect(missing.stderr).toContain(
"tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture: missing forbidOnly: true",
);
const present = checkSource(root, ["--playwright-config", "playwright.config.ts"]);
expect(present.stderr).toBe("");
expect(present.status).toBe(0);
// Without the flag the rule reads the repository's configs, every one.
const repository = checkSource(root);
expect(repository.status, repository.stderr).toBe(0);
const report = JSON.parse(await readFile("artifacts/quality/test-evidence-source-rules.json", "utf8")) as {
facts: { playwrightConfigs?: string[] };
};
expect(report.facts.playwrightConfigs).toEqual([...selectRepositoryPaths("playwright-configs")]);
expect(report.facts.playwrightConfigs?.length).toBeGreaterThan(0);
}, childProcessTimeout(296));
it("rejects a skip without a quarantine", () => {
const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`);
expect(result.status).toBe(1);