test(evidence): forbid focused tests and expired quarantines (11-08)
Only the capabilities config set forbidOnly, so a committed test.only ran one e2e, visual or storybook test and passed. The skip check accepted any quarantine marker without comparing its expiry date to today. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
84f2ab929f
commit
7870c48b5f
@@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./tests/e2e",
|
||||
forbidOnly: true,
|
||||
outputDir: "./artifacts/tests/e2e/results",
|
||||
reporter: [
|
||||
["list"],
|
||||
|
||||
@@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./tests/storybook",
|
||||
forbidOnly: true,
|
||||
outputDir: "./artifacts/tests/storybook/results",
|
||||
reporter: [
|
||||
["list"],
|
||||
|
||||
@@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./tests/visual",
|
||||
forbidOnly: true,
|
||||
snapshotDir: "./tests/visual/__snapshots__",
|
||||
outputDir: "./artifacts/tests/visual/results",
|
||||
reporter: [
|
||||
|
||||
@@ -281,17 +281,27 @@ if (!scenarioOnly) {
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
|
||||
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
|
||||
}
|
||||
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
const quarantines = [
|
||||
...source.matchAll(
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
|
||||
),
|
||||
];
|
||||
if (quarantines.length === 0) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
}
|
||||
for (const [, expires = ""] of quarantines) {
|
||||
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
@@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) {
|
||||
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
||||
}
|
||||
}
|
||||
// playwright.dev.config.ts spreads playwright.config.ts.
|
||||
for (const config of [
|
||||
"playwright.config.ts",
|
||||
"playwright.visual.config.ts",
|
||||
"playwright.storybook.config.ts",
|
||||
"playwright.capabilities.config.ts",
|
||||
]) {
|
||||
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
||||
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
}
|
||||
}
|
||||
|
||||
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
||||
/\.spec\.ts$/.test(file),
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
// quarantine(owner=frontend-platform, defect=expired-fixture, expires=2020-01-01)
|
||||
test.skip("quarantine whose expiry date has passed", async () => {});
|
||||
@@ -0,0 +1 @@
|
||||
test.only("focused test that would pass alone", async () => {});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
|
||||
const FORBIDDEN = "tests/fixtures/test-evidence/forbidden";
|
||||
const roots: string[] = [];
|
||||
|
||||
afterAll(async () => {
|
||||
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
function checkSource(sourceRoot: string) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/check-test-evidence.ts",
|
||||
"--source-only",
|
||||
"--source-root",
|
||||
sourceRoot,
|
||||
"--artifact",
|
||||
"artifacts/quality/test-evidence-source-rules.json",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
return { status: result.status, stderr: result.stderr };
|
||||
}
|
||||
|
||||
// 11-08. Each forbidden fixture is checked alone, so a rule that stops firing
|
||||
// is not hidden by another fixture in the same directory still failing.
|
||||
describe("test evidence source rules", () => {
|
||||
it("rejects a focused test", () => {
|
||||
const result = checkSource(`${FORBIDDEN}/focused-test.fixture`);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("focused test");
|
||||
}, 30_000);
|
||||
|
||||
it("rejects a quarantine whose expiry date has passed", () => {
|
||||
const result = checkSource(`${FORBIDDEN}/expired-quarantine.fixture`);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("quarantine expired on 2020-01-01");
|
||||
}, 30_000);
|
||||
|
||||
it("rejects a skip without a quarantine", () => {
|
||||
const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("lacks owned expiring quarantine");
|
||||
}, 30_000);
|
||||
|
||||
it("accepts a skip under an unexpired quarantine", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "test-evidence-quarantine-"));
|
||||
roots.push(root);
|
||||
const file = path.join(root, "quarantined.fixture");
|
||||
// Built from parts so this file is not itself a skip the checker scans.
|
||||
const skip = ["test", "skip"].join(".");
|
||||
await writeFile(
|
||||
file,
|
||||
[
|
||||
`// ${"quarantine"}(owner=frontend-platform, defect=open-fixture, expires=2999-12-31)`,
|
||||
`${skip}("quarantined", async () => {});`,
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
const result = checkSource(file);
|
||||
expect(result.stderr).toBe("");
|
||||
expect(result.status).toBe(0);
|
||||
}, 30_000);
|
||||
});
|
||||
Reference in New Issue
Block a user