test(evidence): forbid focused tests and expired quarantines (11-08)
Only the capabilities config set forbidOnly, so a committed test.only ran one e2e, visual or storybook test and passed. The skip check accepted any quarantine marker without comparing its expiry date to today. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
84f2ab929f
commit
7870c48b5f
@@ -281,17 +281,27 @@ if (!scenarioOnly) {
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
|
||||
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
|
||||
}
|
||||
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
const quarantines = [
|
||||
...source.matchAll(
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
|
||||
),
|
||||
];
|
||||
if (quarantines.length === 0) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
}
|
||||
for (const [, expires = ""] of quarantines) {
|
||||
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
@@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) {
|
||||
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
||||
}
|
||||
}
|
||||
// playwright.dev.config.ts spreads playwright.config.ts.
|
||||
for (const config of [
|
||||
"playwright.config.ts",
|
||||
"playwright.visual.config.ts",
|
||||
"playwright.storybook.config.ts",
|
||||
"playwright.capabilities.config.ts",
|
||||
]) {
|
||||
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
||||
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
}
|
||||
}
|
||||
|
||||
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
||||
/\.spec\.ts$/.test(file),
|
||||
|
||||
Reference in New Issue
Block a user