From 34a7627d362823b712fd5b019eb9169e1c368392 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 13 Nov 2026 19:06:48 +0900 Subject: [PATCH] =?UTF-8?q?fix(test-evidence):=20=EA=B2=80=EC=82=AC=20?= =?UTF-8?q?=EA=B7=9C=EC=B9=99=EB=A7=88=EB=8B=A4=20=ED=98=BC=EC=9E=90=20?= =?UTF-8?q?=EB=8F=84=EB=8A=94=20=EA=B1=B0=EC=A0=88=20=EC=8B=9C=ED=97=98?= =?UTF-8?q?=EC=9D=84=20=EB=91=90=EA=B3=A0=20strict=20import=EB=A5=BC=20lin?= =?UTF-8?q?t=EB=A1=9C=20=EB=B3=B8=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 문제: `check-test-evidence.ts`의 세 규칙에 혼자 도는 거절 시험이 없었다. 화면 전체 mask 규칙은 fixture만 있고 시험이 없었고, FE-GATE-029의 디렉터리 실행은 공통 접두어만 기대해 다른 fixture가 실패하면 통과했다. Playwright 설정의 forbidOnly 확인과 strict fixture 우회 확인은 전체 실행에서만 돌아 fixture로 시험할 수 없었다. strict fixture 확인은 `tests/e2e/*.spec.ts` 본문에 경로 글자가 있는지만 보아, 주석에만 적어도 통과했고 다른 브라우저 폴더 셋은 보지 않았다(1112/03-04). 고친 것: - mask fixture 하나만 돌리는 시험을 더한다. - forbidOnly 규칙이 시나리오 검사 밖의 모든 실행에서 돌고, 읽은 설정 목록을 보고의 `facts.playwrightConfigs`에 남긴다. `--playwright-config <파일>`로 다른 설정을 줄 수 있다. 거절 fixture(forbidOnly 없는 설정)와, 플래그가 없으면 저장소 설정 전부를 읽는지 보는 시험을 더한다. - strict fixture 확인은 ESLint `no-restricted-imports`로 옮긴다. `tests/{e2e,browser-capabilities,visual,storybook}/**/*.spec.ts`가 `@playwright/test`의 `test`(이름·기본 import)나 strict fixture가 넓히는 base test를 import하면 오류다. 일부러 plain test를 쓰는 둘 (`preload-use.spec.ts`, `chunk-load-error.spec.ts`)만 예외다. 문자열 검사는 지운다. 네 폴더 각각의 lint fixture를 시험이 lint한다. 위험·한계: w2-fe-2 PR #47이 두 예외 spec의 import를 `tests/support/browser/browser-test.ts`로 바꾼다. 그 경로의 `test`도 이 규칙이 막으므로 예외 둘은 그대로 필요하다. 검증: desktop 변이 셋 — mask 블록 삭제, forbidOnly 호출 삭제, ESLint 규칙 삭제 — 에서 각각 해당 시험이 실패하고(1·1·5건), 고친 코드에서 51건 통과. `corepack pnpm lint` rc=0. Refs: 1112/03-03, 1112/03-04 --- eslint.config.ts | 42 +++++++++++++++ scripts/check-test-evidence.ts | 46 +++++++++++------ scripts/lib/test-evidence-artifact.ts | 2 + .../missing-forbid-only.fixture | 8 +++ tests/unit/browser-spec-imports.test.ts | 51 +++++++++++++++++++ tests/unit/test-evidence-source-rules.test.ts | 44 +++++++++++++++- 6 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture create mode 100644 tests/unit/browser-spec-imports.test.ts diff --git a/eslint.config.ts b/eslint.config.ts index fbdcd82a..f0079032 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -937,6 +937,48 @@ export default [ "react-hooks/rules-of-hooks": "off", }, }, + { + // 1112/03-04. A browser spec takes `test` from the strict fixture, which + // fails it on console errors, page errors, failed requests and unused + // preloads. FE-GATE-008 used to look for the fixture's path as text in + // tests/e2e only, so a path in a comment passed and the other browser + // folders were not looked at. tests/unit/browser-spec-imports.test.ts + // lints a fixture spec in each folder against this entry. + files: [`tests/{e2e,browser-capabilities,visual,storybook}/**/*.spec.ts`], + rules: { + "no-restricted-imports": [ + "error", + { + paths: [ + { + name: "@playwright/test", + importNames: ["test", "default"], + message: + "import test from tests/support/browser/strict-browser-test.ts; the strict fixture fails a spec on browser errors (.claude/skills/fe-qa).", + }, + ], + patterns: [ + ...deprecatedAliasImports, + { + group: ["**/support/browser/browser-test.ts", "**/support/browser/browser-test"], + importNames: ["test"], + message: + "import test from tests/support/browser/strict-browser-test.ts; the base test does not check browser errors.", + }, + ], + }, + ], + }, + }, + { + // 1112/03-04. The two specs that need the plain test on purpose: + // preload-use proves the strict fixture's own record, and chunk-load-error + // provokes the 404 a missing chunk gives to classify the engine's wording. + files: ["tests/e2e/preload-use.spec.ts", "tests/browser-capabilities/chunk-load-error.spec.ts"], + rules: { + "no-restricted-imports": ["error", { patterns: deprecatedAliasImports }], + }, + }, { files: [ `tests/fixtures/browser-file-storage-boundaries/forbidden/**/*.${sourceExtensions}`, diff --git a/scripts/check-test-evidence.ts b/scripts/check-test-evidence.ts index 6b16b483..4c9002b2 100644 --- a/scripts/check-test-evidence.ts +++ b/scripts/check-test-evidence.ts @@ -64,6 +64,12 @@ const policyPath = argumentValue( "config/testing/test-evidence.json", ); const explicitCatalogPath = optionalArgumentValue("--catalog"); +// 1112/03-03. The forbidOnly rule read only the repository's configs and ran +// only in the full check, so no fixture could show it still fires. +// `--playwright-config ` (repeatable) checks those files in any mode. +const explicitPlaywrightConfigs = process.argv.flatMap((argument, index) => + argument === "--playwright-config" && process.argv[index + 1] ? [process.argv[index + 1]!] : [], +); const explicitReceiptPath = optionalArgumentValue("--receipt"); const fixtureMode = sourceRoot !== "tests"; const sourceOnly = process.argv.includes("--source-only"); @@ -78,6 +84,7 @@ const facts = { sharedScenarios: 0, declaredScenarioExecutions: 0, executedScenarioExecutions: 0, + playwrightConfigs: [] as string[], }; async function filesBelow(target: string): Promise { @@ -365,6 +372,26 @@ if (parsedPolicy) { } } +async function missingForbidOnly(configs: readonly string[]): Promise { + const missing: string[] = []; + for (const config of configs) { + if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) { + missing.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`); + } + } + return missing; +} + +// The repository's configs unless the run names others; every mode but the +// scenario check reads them, and the report lists what it read. +if (!scenarioOnly) { + // playwright.dev.config.ts spreads playwright.config.ts. + facts.playwrightConfigs = explicitPlaywrightConfigs.length > 0 + ? explicitPlaywrightConfigs + : [...selectRepositoryPaths("playwright-configs")]; + failures.push(...(await missingForbidOnly(facts.playwrightConfigs))); +} + if (!fixtureMode && !scenarioOnly) { const e2eConfig = await readFile("playwright.config.ts", "utf8"); for (const token of [ @@ -381,22 +408,9 @@ if (!fixtureMode && !scenarioOnly) { failures.push(`playwright.config.ts missing release evidence token ${token}`); } } - // playwright.dev.config.ts spreads playwright.config.ts. - for (const config of selectRepositoryPaths("playwright-configs")) { - if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) { - failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`); - } - } - - const e2eFiles = (await filesBelow("tests/e2e")).filter((file) => - /\.spec\.ts$/.test(file), - ); - for (const file of e2eFiles) { - const source = await readFile(file, "utf8"); - if (!source.includes("support/browser/strict-browser-test")) { - failures.push(`${file}: bypasses strict browser fixture`); - } - } + // 1112/03-04. That a browser spec takes the strict fixture's `test` is an + // ESLint rule now (eslint.config.ts, tests/unit/browser-spec-imports.test.ts), + // which reads the import instead of looking for the path as text. const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter( (file) => file.endsWith(".png"), diff --git a/scripts/lib/test-evidence-artifact.ts b/scripts/lib/test-evidence-artifact.ts index 1f86f3f7..dcc1de8c 100644 --- a/scripts/lib/test-evidence-artifact.ts +++ b/scripts/lib/test-evidence-artifact.ts @@ -12,6 +12,8 @@ export const testEvidenceReportSchema = z sharedScenarios: z.number().int().nonnegative(), declaredScenarioExecutions: z.number().int().nonnegative(), executedScenarioExecutions: z.number().int().nonnegative(), + // 1112/03-03. The Playwright configs the forbidOnly rule read. + playwrightConfigs: z.array(z.string().min(1)).optional(), }) .strict(), failures: z.array(z.string()), diff --git a/tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture b/tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture new file mode 100644 index 00000000..ce5ed861 --- /dev/null +++ b/tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture @@ -0,0 +1,8 @@ +// A Playwright config that would let a committed focused test run alone: +// check-test-evidence.ts --playwright-config must reject it (1112/03-03). +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: "tests/e2e", + reporter: [["junit", { outputFile: "artifacts/tests/e2e/results.xml" }]], +}); diff --git a/tests/unit/browser-spec-imports.test.ts b/tests/unit/browser-spec-imports.test.ts new file mode 100644 index 00000000..596db74f --- /dev/null +++ b/tests/unit/browser-spec-imports.test.ts @@ -0,0 +1,51 @@ +import path from "node:path"; + +import { ESLint } from "eslint"; +import { describe, expect, it } from "vitest"; + +import { childProcessTimeout } from "../helpers/child-process-timeout.ts"; + +const FOLDERS = ["e2e", "browser-capabilities", "visual", "storybook"] as const; + +const BARE = 'import { expect, test } from "@playwright/test";\n\ntest("probe", async () => { expect(1).toBe(1); });\n'; +const STRICT = + 'import { expect, test } from "../support/browser/strict-browser-test.ts";\n' + + 'import type { Page } from "@playwright/test";\n\n' + + 'test("probe", async ({ page }: { page: Page }) => { expect(page).toBeTruthy(); });\n'; +const DEFAULT_IMPORT = 'import test from "@playwright/test";\n\ntest("probe", async () => {});\n'; +const BASE_TEST = 'import { test } from "../support/browser/browser-test.ts";\n\ntest("probe", async () => {});\n'; + +async function restrictedImports(code: string, file: string): Promise { + const linter = new ESLint({ cwd: process.cwd() }); + const [result] = await linter.lintText(code, { filePath: path.join(process.cwd(), file), warnIgnored: true }); + return (result?.messages ?? []) + .filter((message) => message.ruleId === "no-restricted-imports") + .map((message) => message.message); +} + +// 1112/03-04. Strict fixture use was a text search for its path in +// tests/e2e/*.spec.ts: a path in a comment passed, a file could mix in the +// bare test, and tests/browser-capabilities, tests/visual and tests/storybook +// were not looked at. It is an import rule in each of the four folders now. +describe("browser specs take test from the strict fixture (1112/03-04)", () => { + it.each(FOLDERS)("rejects the bare Playwright test in tests/%s", async (folder) => { + const messages = await restrictedImports(BARE, `tests/${folder}/lint-probe.spec.ts`); + expect(messages).toEqual([expect.stringContaining("tests/support/browser/strict-browser-test.ts")]); + }, childProcessTimeout(683)); + + it("rejects the default import, the base test the strict fixture extends, and accepts the strict test", async () => { + expect(await restrictedImports(DEFAULT_IMPORT, "tests/visual/lint-probe.spec.ts")).toEqual([ + expect.stringContaining("tests/support/browser/strict-browser-test.ts"), + ]); + expect(await restrictedImports(BASE_TEST, "tests/e2e/lint-probe.spec.ts")).toEqual([ + expect.stringContaining("the base test does not check browser errors"), + ]); + expect(await restrictedImports(STRICT, "tests/browser-capabilities/lint-probe.spec.ts")).toEqual([]); + }, childProcessTimeout(683)); + + it("lets only the two specs that need the plain test import it", async () => { + for (const file of ["tests/e2e/preload-use.spec.ts", "tests/browser-capabilities/chunk-load-error.spec.ts"]) { + expect(await restrictedImports(BARE, file), file).toEqual([]); + } + }, childProcessTimeout(683)); +}); diff --git a/tests/unit/test-evidence-source-rules.test.ts b/tests/unit/test-evidence-source-rules.test.ts index 074cb705..53a13f59 100644 --- a/tests/unit/test-evidence-source-rules.test.ts +++ b/tests/unit/test-evidence-source-rules.test.ts @@ -1,9 +1,10 @@ import { spawnSync } from "node:child_process"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterAll, describe, expect, it } from "vitest"; +import { selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts"; import { childProcessTimeout } from "../helpers/child-process-timeout.ts"; const FORBIDDEN = "tests/fixtures/test-evidence/forbidden"; @@ -13,7 +14,7 @@ afterAll(async () => { await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))); }); -function checkSource(sourceRoot: string) { +function checkSource(sourceRoot: string, extra: readonly string[] = []) { const result = spawnSync( process.execPath, [ @@ -23,6 +24,7 @@ function checkSource(sourceRoot: string) { sourceRoot, "--artifact", "artifacts/quality/test-evidence-source-rules.json", + ...extra, ], { encoding: "utf8" }, ); @@ -44,6 +46,44 @@ describe("test evidence source rules", () => { expect(result.stderr).toContain("quarantine expired on 2020-01-01"); }, childProcessTimeout(739)); + // 1112/03-03. The mask fixture had no test of its own: the directory run + // expects only the common prefix, so this rule could stop firing unseen. + it("rejects a screenshot mask over the whole UI", () => { + const result = checkSource(`${FORBIDDEN}/full-screen-mask.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("screenshot mask may not cover the whole UI"); + }, childProcessTimeout(141)); + + // 1112/03-03. The forbidOnly rule ran only in the full check, on the + // repository's own configs, so nothing showed it still rejects a config. + it("rejects a Playwright config without forbidOnly and accepts the repository's", async () => { + const root = await mkdtemp(path.join(tmpdir(), "test-evidence-config-")); + roots.push(root); + await writeFile(path.join(root, "clean.fixture"), "export {};\n"); + + const missing = checkSource(root, [ + "--playwright-config", + "tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture", + ]); + expect(missing.status).toBe(1); + expect(missing.stderr).toContain( + "tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture: missing forbidOnly: true", + ); + + const present = checkSource(root, ["--playwright-config", "playwright.config.ts"]); + expect(present.stderr).toBe(""); + expect(present.status).toBe(0); + + // Without the flag the rule reads the repository's configs, every one. + const repository = checkSource(root); + expect(repository.status, repository.stderr).toBe(0); + const report = JSON.parse(await readFile("artifacts/quality/test-evidence-source-rules.json", "utf8")) as { + facts: { playwrightConfigs?: string[] }; + }; + expect(report.facts.playwrightConfigs).toEqual([...selectRepositoryPaths("playwright-configs")]); + expect(report.facts.playwrightConfigs?.length).toBeGreaterThan(0); + }, childProcessTimeout(296)); + it("rejects a skip without a quarantine", () => { const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`); expect(result.status).toBe(1);