fix(test-evidence): 검사 규칙마다 혼자 도는 거절 시험을 두고 strict import를 lint로 본다
문제: `check-test-evidence.ts`의 세 규칙에 혼자 도는 거절 시험이 없었다.
화면 전체 mask 규칙은 fixture만 있고 시험이 없었고, FE-GATE-029의 디렉터리
실행은 공통 접두어만 기대해 다른 fixture가 실패하면 통과했다. Playwright
설정의 forbidOnly 확인과 strict fixture 우회 확인은 전체 실행에서만 돌아
fixture로 시험할 수 없었다. strict fixture 확인은 `tests/e2e/*.spec.ts`
본문에 경로 글자가 있는지만 보아, 주석에만 적어도 통과했고 다른 브라우저
폴더 셋은 보지 않았다(1112/03-04).
고친 것:
- mask fixture 하나만 돌리는 시험을 더한다.
- forbidOnly 규칙이 시나리오 검사 밖의 모든 실행에서 돌고, 읽은 설정 목록을
보고의 `facts.playwrightConfigs`에 남긴다. `--playwright-config <파일>`로
다른 설정을 줄 수 있다. 거절 fixture(forbidOnly 없는 설정)와, 플래그가
없으면 저장소 설정 전부를 읽는지 보는 시험을 더한다.
- strict fixture 확인은 ESLint `no-restricted-imports`로 옮긴다.
`tests/{e2e,browser-capabilities,visual,storybook}/**/*.spec.ts`가
`@playwright/test`의 `test`(이름·기본 import)나 strict fixture가 넓히는
base test를 import하면 오류다. 일부러 plain test를 쓰는 둘
(`preload-use.spec.ts`, `chunk-load-error.spec.ts`)만 예외다. 문자열 검사는
지운다. 네 폴더 각각의 lint fixture를 시험이 lint한다.
위험·한계: w2-fe-2 PR #47이 두 예외 spec의 import를
`tests/support/browser/browser-test.ts`로 바꾼다. 그 경로의 `test`도 이
규칙이 막으므로 예외 둘은 그대로 필요하다.
검증: desktop 변이 셋 — mask 블록 삭제, forbidOnly 호출 삭제, ESLint 규칙
삭제 — 에서 각각 해당 시험이 실패하고(1·1·5건), 고친 코드에서 51건 통과.
`corepack pnpm lint` rc=0.
Refs: 1112/03-03, 1112/03-04
This commit is contained in:
@@ -937,6 +937,48 @@ export default [
|
||||
"react-hooks/rules-of-hooks": "off",
|
||||
},
|
||||
},
|
||||
{
|
||||
// 1112/03-04. A browser spec takes `test` from the strict fixture, which
|
||||
// fails it on console errors, page errors, failed requests and unused
|
||||
// preloads. FE-GATE-008 used to look for the fixture's path as text in
|
||||
// tests/e2e only, so a path in a comment passed and the other browser
|
||||
// folders were not looked at. tests/unit/browser-spec-imports.test.ts
|
||||
// lints a fixture spec in each folder against this entry.
|
||||
files: [`tests/{e2e,browser-capabilities,visual,storybook}/**/*.spec.ts`],
|
||||
rules: {
|
||||
"no-restricted-imports": [
|
||||
"error",
|
||||
{
|
||||
paths: [
|
||||
{
|
||||
name: "@playwright/test",
|
||||
importNames: ["test", "default"],
|
||||
message:
|
||||
"import test from tests/support/browser/strict-browser-test.ts; the strict fixture fails a spec on browser errors (.claude/skills/fe-qa).",
|
||||
},
|
||||
],
|
||||
patterns: [
|
||||
...deprecatedAliasImports,
|
||||
{
|
||||
group: ["**/support/browser/browser-test.ts", "**/support/browser/browser-test"],
|
||||
importNames: ["test"],
|
||||
message:
|
||||
"import test from tests/support/browser/strict-browser-test.ts; the base test does not check browser errors.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
// 1112/03-04. The two specs that need the plain test on purpose:
|
||||
// preload-use proves the strict fixture's own record, and chunk-load-error
|
||||
// provokes the 404 a missing chunk gives to classify the engine's wording.
|
||||
files: ["tests/e2e/preload-use.spec.ts", "tests/browser-capabilities/chunk-load-error.spec.ts"],
|
||||
rules: {
|
||||
"no-restricted-imports": ["error", { patterns: deprecatedAliasImports }],
|
||||
},
|
||||
},
|
||||
{
|
||||
files: [
|
||||
`tests/fixtures/browser-file-storage-boundaries/forbidden/**/*.${sourceExtensions}`,
|
||||
|
||||
@@ -64,6 +64,12 @@ const policyPath = argumentValue(
|
||||
"config/testing/test-evidence.json",
|
||||
);
|
||||
const explicitCatalogPath = optionalArgumentValue("--catalog");
|
||||
// 1112/03-03. The forbidOnly rule read only the repository's configs and ran
|
||||
// only in the full check, so no fixture could show it still fires.
|
||||
// `--playwright-config <file>` (repeatable) checks those files in any mode.
|
||||
const explicitPlaywrightConfigs = process.argv.flatMap((argument, index) =>
|
||||
argument === "--playwright-config" && process.argv[index + 1] ? [process.argv[index + 1]!] : [],
|
||||
);
|
||||
const explicitReceiptPath = optionalArgumentValue("--receipt");
|
||||
const fixtureMode = sourceRoot !== "tests";
|
||||
const sourceOnly = process.argv.includes("--source-only");
|
||||
@@ -78,6 +84,7 @@ const facts = {
|
||||
sharedScenarios: 0,
|
||||
declaredScenarioExecutions: 0,
|
||||
executedScenarioExecutions: 0,
|
||||
playwrightConfigs: [] as string[],
|
||||
};
|
||||
|
||||
async function filesBelow(target: string): Promise<string[]> {
|
||||
@@ -365,6 +372,26 @@ if (parsedPolicy) {
|
||||
}
|
||||
}
|
||||
|
||||
async function missingForbidOnly(configs: readonly string[]): Promise<string[]> {
|
||||
const missing: string[] = [];
|
||||
for (const config of configs) {
|
||||
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
||||
missing.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
// The repository's configs unless the run names others; every mode but the
|
||||
// scenario check reads them, and the report lists what it read.
|
||||
if (!scenarioOnly) {
|
||||
// playwright.dev.config.ts spreads playwright.config.ts.
|
||||
facts.playwrightConfigs = explicitPlaywrightConfigs.length > 0
|
||||
? explicitPlaywrightConfigs
|
||||
: [...selectRepositoryPaths("playwright-configs")];
|
||||
failures.push(...(await missingForbidOnly(facts.playwrightConfigs)));
|
||||
}
|
||||
|
||||
if (!fixtureMode && !scenarioOnly) {
|
||||
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
||||
for (const token of [
|
||||
@@ -381,22 +408,9 @@ if (!fixtureMode && !scenarioOnly) {
|
||||
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
||||
}
|
||||
}
|
||||
// playwright.dev.config.ts spreads playwright.config.ts.
|
||||
for (const config of selectRepositoryPaths("playwright-configs")) {
|
||||
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
||||
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
}
|
||||
}
|
||||
|
||||
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
||||
/\.spec\.ts$/.test(file),
|
||||
);
|
||||
for (const file of e2eFiles) {
|
||||
const source = await readFile(file, "utf8");
|
||||
if (!source.includes("support/browser/strict-browser-test")) {
|
||||
failures.push(`${file}: bypasses strict browser fixture`);
|
||||
}
|
||||
}
|
||||
// 1112/03-04. That a browser spec takes the strict fixture's `test` is an
|
||||
// ESLint rule now (eslint.config.ts, tests/unit/browser-spec-imports.test.ts),
|
||||
// which reads the import instead of looking for the path as text.
|
||||
|
||||
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
|
||||
(file) => file.endsWith(".png"),
|
||||
|
||||
@@ -12,6 +12,8 @@ export const testEvidenceReportSchema = z
|
||||
sharedScenarios: z.number().int().nonnegative(),
|
||||
declaredScenarioExecutions: z.number().int().nonnegative(),
|
||||
executedScenarioExecutions: z.number().int().nonnegative(),
|
||||
// 1112/03-03. The Playwright configs the forbidOnly rule read.
|
||||
playwrightConfigs: z.array(z.string().min(1)).optional(),
|
||||
})
|
||||
.strict(),
|
||||
failures: z.array(z.string()),
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// A Playwright config that would let a committed focused test run alone:
|
||||
// check-test-evidence.ts --playwright-config must reject it (1112/03-03).
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "tests/e2e",
|
||||
reporter: [["junit", { outputFile: "artifacts/tests/e2e/results.xml" }]],
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import path from "node:path";
|
||||
|
||||
import { ESLint } from "eslint";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { childProcessTimeout } from "../helpers/child-process-timeout.ts";
|
||||
|
||||
const FOLDERS = ["e2e", "browser-capabilities", "visual", "storybook"] as const;
|
||||
|
||||
const BARE = 'import { expect, test } from "@playwright/test";\n\ntest("probe", async () => { expect(1).toBe(1); });\n';
|
||||
const STRICT =
|
||||
'import { expect, test } from "../support/browser/strict-browser-test.ts";\n' +
|
||||
'import type { Page } from "@playwright/test";\n\n' +
|
||||
'test("probe", async ({ page }: { page: Page }) => { expect(page).toBeTruthy(); });\n';
|
||||
const DEFAULT_IMPORT = 'import test from "@playwright/test";\n\ntest("probe", async () => {});\n';
|
||||
const BASE_TEST = 'import { test } from "../support/browser/browser-test.ts";\n\ntest("probe", async () => {});\n';
|
||||
|
||||
async function restrictedImports(code: string, file: string): Promise<string[]> {
|
||||
const linter = new ESLint({ cwd: process.cwd() });
|
||||
const [result] = await linter.lintText(code, { filePath: path.join(process.cwd(), file), warnIgnored: true });
|
||||
return (result?.messages ?? [])
|
||||
.filter((message) => message.ruleId === "no-restricted-imports")
|
||||
.map((message) => message.message);
|
||||
}
|
||||
|
||||
// 1112/03-04. Strict fixture use was a text search for its path in
|
||||
// tests/e2e/*.spec.ts: a path in a comment passed, a file could mix in the
|
||||
// bare test, and tests/browser-capabilities, tests/visual and tests/storybook
|
||||
// were not looked at. It is an import rule in each of the four folders now.
|
||||
describe("browser specs take test from the strict fixture (1112/03-04)", () => {
|
||||
it.each(FOLDERS)("rejects the bare Playwright test in tests/%s", async (folder) => {
|
||||
const messages = await restrictedImports(BARE, `tests/${folder}/lint-probe.spec.ts`);
|
||||
expect(messages).toEqual([expect.stringContaining("tests/support/browser/strict-browser-test.ts")]);
|
||||
}, childProcessTimeout(683));
|
||||
|
||||
it("rejects the default import, the base test the strict fixture extends, and accepts the strict test", async () => {
|
||||
expect(await restrictedImports(DEFAULT_IMPORT, "tests/visual/lint-probe.spec.ts")).toEqual([
|
||||
expect.stringContaining("tests/support/browser/strict-browser-test.ts"),
|
||||
]);
|
||||
expect(await restrictedImports(BASE_TEST, "tests/e2e/lint-probe.spec.ts")).toEqual([
|
||||
expect.stringContaining("the base test does not check browser errors"),
|
||||
]);
|
||||
expect(await restrictedImports(STRICT, "tests/browser-capabilities/lint-probe.spec.ts")).toEqual([]);
|
||||
}, childProcessTimeout(683));
|
||||
|
||||
it("lets only the two specs that need the plain test import it", async () => {
|
||||
for (const file of ["tests/e2e/preload-use.spec.ts", "tests/browser-capabilities/chunk-load-error.spec.ts"]) {
|
||||
expect(await restrictedImports(BARE, file), file).toEqual([]);
|
||||
}
|
||||
}, childProcessTimeout(683));
|
||||
});
|
||||
@@ -1,9 +1,10 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import { selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts";
|
||||
import { childProcessTimeout } from "../helpers/child-process-timeout.ts";
|
||||
|
||||
const FORBIDDEN = "tests/fixtures/test-evidence/forbidden";
|
||||
@@ -13,7 +14,7 @@ afterAll(async () => {
|
||||
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
function checkSource(sourceRoot: string) {
|
||||
function checkSource(sourceRoot: string, extra: readonly string[] = []) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
@@ -23,6 +24,7 @@ function checkSource(sourceRoot: string) {
|
||||
sourceRoot,
|
||||
"--artifact",
|
||||
"artifacts/quality/test-evidence-source-rules.json",
|
||||
...extra,
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
@@ -44,6 +46,44 @@ describe("test evidence source rules", () => {
|
||||
expect(result.stderr).toContain("quarantine expired on 2020-01-01");
|
||||
}, childProcessTimeout(739));
|
||||
|
||||
// 1112/03-03. The mask fixture had no test of its own: the directory run
|
||||
// expects only the common prefix, so this rule could stop firing unseen.
|
||||
it("rejects a screenshot mask over the whole UI", () => {
|
||||
const result = checkSource(`${FORBIDDEN}/full-screen-mask.fixture`);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("screenshot mask may not cover the whole UI");
|
||||
}, childProcessTimeout(141));
|
||||
|
||||
// 1112/03-03. The forbidOnly rule ran only in the full check, on the
|
||||
// repository's own configs, so nothing showed it still rejects a config.
|
||||
it("rejects a Playwright config without forbidOnly and accepts the repository's", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-config-"));
|
||||
roots.push(root);
|
||||
await writeFile(path.join(root, "clean.fixture"), "export {};\n");
|
||||
|
||||
const missing = checkSource(root, [
|
||||
"--playwright-config",
|
||||
"tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture",
|
||||
]);
|
||||
expect(missing.status).toBe(1);
|
||||
expect(missing.stderr).toContain(
|
||||
"tests/fixtures/test-evidence/playwright-config/missing-forbid-only.fixture: missing forbidOnly: true",
|
||||
);
|
||||
|
||||
const present = checkSource(root, ["--playwright-config", "playwright.config.ts"]);
|
||||
expect(present.stderr).toBe("");
|
||||
expect(present.status).toBe(0);
|
||||
|
||||
// Without the flag the rule reads the repository's configs, every one.
|
||||
const repository = checkSource(root);
|
||||
expect(repository.status, repository.stderr).toBe(0);
|
||||
const report = JSON.parse(await readFile("artifacts/quality/test-evidence-source-rules.json", "utf8")) as {
|
||||
facts: { playwrightConfigs?: string[] };
|
||||
};
|
||||
expect(report.facts.playwrightConfigs).toEqual([...selectRepositoryPaths("playwright-configs")]);
|
||||
expect(report.facts.playwrightConfigs?.length).toBeGreaterThan(0);
|
||||
}, childProcessTimeout(296));
|
||||
|
||||
it("rejects a skip without a quarantine", () => {
|
||||
const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`);
|
||||
expect(result.status).toBe(1);
|
||||
|
||||
Reference in New Issue
Block a user