diff --git a/playwright.config.ts b/playwright.config.ts index 83fe0819..b6523844 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./tests/e2e", + forbidOnly: true, outputDir: "./artifacts/tests/e2e/results", reporter: [ ["list"], diff --git a/playwright.storybook.config.ts b/playwright.storybook.config.ts index 4c2fac04..75abbe7c 100644 --- a/playwright.storybook.config.ts +++ b/playwright.storybook.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./tests/storybook", + forbidOnly: true, outputDir: "./artifacts/tests/storybook/results", reporter: [ ["list"], diff --git a/playwright.visual.config.ts b/playwright.visual.config.ts index f66f9967..36528303 100644 --- a/playwright.visual.config.ts +++ b/playwright.visual.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./tests/visual", + forbidOnly: true, snapshotDir: "./tests/visual/__snapshots__", outputDir: "./artifacts/tests/visual/results", reporter: [ diff --git a/scripts/check-test-evidence.ts b/scripts/check-test-evidence.ts index 9158bde6..ab3e0dc6 100644 --- a/scripts/check-test-evidence.ts +++ b/scripts/check-test-evidence.ts @@ -281,17 +281,27 @@ if (!scenarioOnly) { const sourceFiles = (await filesBelow(sourceRoot)).filter( (file) => fixtureMode || !file.split(path.sep).includes("fixtures"), ); + const today = new Date().toISOString().slice(0, 10); for (const file of sourceFiles) { if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue; const source = await readFile(file, "utf8"); facts.scannedFiles += 1; + if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) { + failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`); + } const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g; if (skipPattern.test(source)) { - const quarantine = - /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/; - if (!quarantine.test(source)) { + const quarantines = [ + ...source.matchAll( + /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g, + ), + ]; + if (quarantines.length === 0) { failures.push(`${file}: skip/fixme lacks owned expiring quarantine`); } + for (const [, expires = ""] of quarantines) { + if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`); + } } const wholeUiMask = /\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i; @@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) { failures.push(`playwright.config.ts missing release evidence token ${token}`); } } + // playwright.dev.config.ts spreads playwright.config.ts. + for (const config of [ + "playwright.config.ts", + "playwright.visual.config.ts", + "playwright.storybook.config.ts", + "playwright.capabilities.config.ts", + ]) { + if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) { + failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`); + } + } const e2eFiles = (await filesBelow("tests/e2e")).filter((file) => /\.spec\.ts$/.test(file), diff --git a/tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture b/tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture new file mode 100644 index 00000000..5fb8dab3 --- /dev/null +++ b/tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture @@ -0,0 +1,2 @@ +// quarantine(owner=frontend-platform, defect=expired-fixture, expires=2020-01-01) +test.skip("quarantine whose expiry date has passed", async () => {}); diff --git a/tests/fixtures/test-evidence/forbidden/focused-test.fixture b/tests/fixtures/test-evidence/forbidden/focused-test.fixture new file mode 100644 index 00000000..f3481dc1 --- /dev/null +++ b/tests/fixtures/test-evidence/forbidden/focused-test.fixture @@ -0,0 +1 @@ +test.only("focused test that would pass alone", async () => {}); diff --git a/tests/unit/test-evidence-source-rules.test.ts b/tests/unit/test-evidence-source-rules.test.ts new file mode 100644 index 00000000..abd36c8f --- /dev/null +++ b/tests/unit/test-evidence-source-rules.test.ts @@ -0,0 +1,70 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterAll, describe, expect, it } from "vitest"; + +const FORBIDDEN = "tests/fixtures/test-evidence/forbidden"; +const roots: string[] = []; + +afterAll(async () => { + await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))); +}); + +function checkSource(sourceRoot: string) { + const result = spawnSync( + process.execPath, + [ + "scripts/check-test-evidence.ts", + "--source-only", + "--source-root", + sourceRoot, + "--artifact", + "artifacts/quality/test-evidence-source-rules.json", + ], + { encoding: "utf8" }, + ); + return { status: result.status, stderr: result.stderr }; +} + +// 11-08. Each forbidden fixture is checked alone, so a rule that stops firing +// is not hidden by another fixture in the same directory still failing. +describe("test evidence source rules", () => { + it("rejects a focused test", () => { + const result = checkSource(`${FORBIDDEN}/focused-test.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("focused test"); + }, 30_000); + + it("rejects a quarantine whose expiry date has passed", () => { + const result = checkSource(`${FORBIDDEN}/expired-quarantine.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("quarantine expired on 2020-01-01"); + }, 30_000); + + it("rejects a skip without a quarantine", () => { + const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("lacks owned expiring quarantine"); + }, 30_000); + + it("accepts a skip under an unexpired quarantine", async () => { + const root = await mkdtemp(path.join(tmpdir(), "test-evidence-quarantine-")); + roots.push(root); + const file = path.join(root, "quarantined.fixture"); + // Built from parts so this file is not itself a skip the checker scans. + const skip = ["test", "skip"].join("."); + await writeFile( + file, + [ + `// ${"quarantine"}(owner=frontend-platform, defect=open-fixture, expires=2999-12-31)`, + `${skip}("quarantined", async () => {});`, + "", + ].join("\n"), + ); + const result = checkSource(file); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + }, 30_000); +});