test(evidence): forbid focused tests and expired quarantines (11-08)

Only the capabilities config set forbidOnly, so a committed test.only ran one
e2e, visual or storybook test and passed. The skip check accepted any
quarantine marker without comparing its expiry date to today.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-26 01:01:49 +09:00
co-authored by Claude Opus 5.5
parent 84f2ab929f
commit 7870c48b5f
7 changed files with 100 additions and 3 deletions
+24 -3
View File
@@ -281,17 +281,27 @@ if (!scenarioOnly) {
const sourceFiles = (await filesBelow(sourceRoot)).filter(
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
);
const today = new Date().toISOString().slice(0, 10);
for (const file of sourceFiles) {
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
const source = await readFile(file, "utf8");
facts.scannedFiles += 1;
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
}
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
if (skipPattern.test(source)) {
const quarantine =
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
if (!quarantine.test(source)) {
const quarantines = [
...source.matchAll(
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
),
];
if (quarantines.length === 0) {
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
}
for (const [, expires = ""] of quarantines) {
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
}
}
const wholeUiMask =
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
@@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) {
failures.push(`playwright.config.ts missing release evidence token ${token}`);
}
}
// playwright.dev.config.ts spreads playwright.config.ts.
for (const config of [
"playwright.config.ts",
"playwright.visual.config.ts",
"playwright.storybook.config.ts",
"playwright.capabilities.config.ts",
]) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
/\.spec\.ts$/.test(file),