test(evidence): forbid focused tests and expired quarantines (11-08)

Only the capabilities config set forbidOnly, so a committed test.only ran one
e2e, visual or storybook test and passed. The skip check accepted any
quarantine marker without comparing its expiry date to today.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-26 01:01:49 +09:00
co-authored by Claude Opus 5.5
parent 84f2ab929f
commit 7870c48b5f
7 changed files with 100 additions and 3 deletions
+1
View File
@@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
export default defineConfig({
testDir: "./tests/e2e",
forbidOnly: true,
outputDir: "./artifacts/tests/e2e/results",
reporter: [
["list"],
+1
View File
@@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
export default defineConfig({
testDir: "./tests/storybook",
forbidOnly: true,
outputDir: "./artifacts/tests/storybook/results",
reporter: [
["list"],
+1
View File
@@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
export default defineConfig({
testDir: "./tests/visual",
forbidOnly: true,
snapshotDir: "./tests/visual/__snapshots__",
outputDir: "./artifacts/tests/visual/results",
reporter: [
+24 -3
View File
@@ -281,17 +281,27 @@ if (!scenarioOnly) {
const sourceFiles = (await filesBelow(sourceRoot)).filter(
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
);
const today = new Date().toISOString().slice(0, 10);
for (const file of sourceFiles) {
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
const source = await readFile(file, "utf8");
facts.scannedFiles += 1;
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
}
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
if (skipPattern.test(source)) {
const quarantine =
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
if (!quarantine.test(source)) {
const quarantines = [
...source.matchAll(
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
),
];
if (quarantines.length === 0) {
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
}
for (const [, expires = ""] of quarantines) {
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
}
}
const wholeUiMask =
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
@@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) {
failures.push(`playwright.config.ts missing release evidence token ${token}`);
}
}
// playwright.dev.config.ts spreads playwright.config.ts.
for (const config of [
"playwright.config.ts",
"playwright.visual.config.ts",
"playwright.storybook.config.ts",
"playwright.capabilities.config.ts",
]) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
/\.spec\.ts$/.test(file),
@@ -0,0 +1,2 @@
// quarantine(owner=frontend-platform, defect=expired-fixture, expires=2020-01-01)
test.skip("quarantine whose expiry date has passed", async () => {});
@@ -0,0 +1 @@
test.only("focused test that would pass alone", async () => {});
@@ -0,0 +1,70 @@
import { spawnSync } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterAll, describe, expect, it } from "vitest";
const FORBIDDEN = "tests/fixtures/test-evidence/forbidden";
const roots: string[] = [];
afterAll(async () => {
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })));
});
function checkSource(sourceRoot: string) {
const result = spawnSync(
process.execPath,
[
"scripts/check-test-evidence.ts",
"--source-only",
"--source-root",
sourceRoot,
"--artifact",
"artifacts/quality/test-evidence-source-rules.json",
],
{ encoding: "utf8" },
);
return { status: result.status, stderr: result.stderr };
}
// 11-08. Each forbidden fixture is checked alone, so a rule that stops firing
// is not hidden by another fixture in the same directory still failing.
describe("test evidence source rules", () => {
it("rejects a focused test", () => {
const result = checkSource(`${FORBIDDEN}/focused-test.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("focused test");
}, 30_000);
it("rejects a quarantine whose expiry date has passed", () => {
const result = checkSource(`${FORBIDDEN}/expired-quarantine.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("quarantine expired on 2020-01-01");
}, 30_000);
it("rejects a skip without a quarantine", () => {
const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`);
expect(result.status).toBe(1);
expect(result.stderr).toContain("lacks owned expiring quarantine");
}, 30_000);
it("accepts a skip under an unexpired quarantine", async () => {
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-quarantine-"));
roots.push(root);
const file = path.join(root, "quarantined.fixture");
// Built from parts so this file is not itself a skip the checker scans.
const skip = ["test", "skip"].join(".");
await writeFile(
file,
[
`// ${"quarantine"}(owner=frontend-platform, defect=open-fixture, expires=2999-12-31)`,
`${skip}("quarantined", async () => {});`,
"",
].join("\n"),
);
const result = checkSource(file);
expect(result.stderr).toBe("");
expect(result.status).toBe(0);
}, 30_000);
});