No CI job ran any scale tier, and every tier past pr compares timings only on the baseline CPU. The pr tier checks counts, bounds and growth shape; it now includes S2 composition, since S0/S1 stay under the noise floor and let a quadratic accumulation through. The ungated-test check reads tier paths. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
109 lines
4.2 KiB
TypeScript
109 lines
4.2 KiB
TypeScript
import { readdir } from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
import { z } from "zod";
|
|
|
|
import { SCALE_TIERS, type ScaleTier } from "../scale/run-scale-profile.ts";
|
|
import { reachablePackageScripts } from "./package-script-graph.ts";
|
|
|
|
/**
|
|
* 12-04. A test file protects nothing unless a CI gate runs it. The gated set
|
|
* is read from the package scripts the CI contract's commands reach: Vitest
|
|
* path filters (`run-vitest.ts run <paths>`), the Vitest steps of a scale tier
|
|
* (`run-scale-profile.ts <tier>`) and Playwright config `testDir`s.
|
|
* A test outside that set needs a reasoned entry in config/ci/ungated-tests.json.
|
|
*/
|
|
export const UNGATED_TESTS_PATH = "config/ci/ungated-tests.json";
|
|
|
|
const TEST_FILE = /\.(?:test|spec)\.tsx?$/u;
|
|
|
|
const ungatedTestsSchema = z
|
|
.object({
|
|
schemaVersion: z.literal(1),
|
|
entries: z.array(
|
|
z
|
|
.object({
|
|
path: z.string().regex(/^tests\/\S+$/u),
|
|
reason: z.string().trim().min(1, "exclusion reason is required"),
|
|
})
|
|
.strict(),
|
|
),
|
|
})
|
|
.strict();
|
|
|
|
export type UngatedTestExclusion = Readonly<{ path: string; reason: string }>;
|
|
|
|
export function parseUngatedTestExclusions(value: unknown): readonly UngatedTestExclusion[] {
|
|
return ungatedTestsSchema.parse(value).entries;
|
|
}
|
|
|
|
/** Repository-relative test files below tests/, fixtures excluded. */
|
|
export async function listTestFiles(root: string): Promise<string[]> {
|
|
const entries = await readdir(path.join(root, "tests"), { recursive: true, withFileTypes: true });
|
|
return entries
|
|
.filter((entry) => entry.isFile() && TEST_FILE.test(entry.name))
|
|
.map((entry) =>
|
|
path.relative(root, path.join(entry.parentPath, entry.name)).split(path.sep).join("/"),
|
|
)
|
|
.filter((file) => !file.startsWith("tests/fixtures/"))
|
|
.sort();
|
|
}
|
|
|
|
export function gatedTestPaths(
|
|
scripts: Readonly<Record<string, string>>,
|
|
entryScripts: readonly string[],
|
|
readConfig: (file: string) => string,
|
|
): string[] {
|
|
const paths = new Set<string>();
|
|
const addVitestPaths = (tokens: readonly string[]) => {
|
|
const vitestRun = tokens.findIndex(
|
|
(token, index) => token === "run" && /(?:run-vitest\.ts|vitest)$/u.test(tokens[index - 1] ?? ""),
|
|
);
|
|
if (vitestRun < 0) return;
|
|
for (const token of tokens.slice(vitestRun + 1)) {
|
|
if (!token.startsWith("-")) paths.add(token.replace(/\/$/u, ""));
|
|
}
|
|
};
|
|
for (const script of reachablePackageScripts(scripts, entryScripts)) {
|
|
for (const segment of scripts[script]!.split("&&")) {
|
|
const tokens = segment.trim().split(/\s+/u);
|
|
addVitestPaths(tokens);
|
|
const scaleProfile = tokens.findIndex((token) => /run-scale-profile\.ts$/u.test(token));
|
|
if (scaleProfile >= 0) {
|
|
for (const step of SCALE_TIERS[tokens[scaleProfile + 1] as ScaleTier] ?? []) addVitestPaths(step.command);
|
|
}
|
|
const playwrightTest = tokens.findIndex(
|
|
(token, index) => token === "test" && tokens[index - 1] === "playwright",
|
|
);
|
|
if (playwrightTest >= 0) {
|
|
const configFlag = tokens.indexOf("--config");
|
|
const config = configFlag >= 0 ? tokens[configFlag + 1]! : "playwright.config.ts";
|
|
const testDir = /testDir:\s*["']\.?\/?([^"']+)["']/u.exec(readConfig(config))?.[1];
|
|
if (testDir) paths.add(testDir.replace(/\/$/u, ""));
|
|
}
|
|
}
|
|
}
|
|
return [...paths].sort();
|
|
}
|
|
|
|
export function ungatedTestFailures(input: Readonly<{
|
|
testFiles: readonly string[];
|
|
gatedPaths: readonly string[];
|
|
exclusions: readonly UngatedTestExclusion[];
|
|
}>): string[] {
|
|
const gated = (file: string) =>
|
|
input.gatedPaths.some((gatedPath) => file === gatedPath || file.startsWith(`${gatedPath}/`));
|
|
const excluded = new Set(input.exclusions.map(({ path: file }) => file));
|
|
const failures = input.testFiles
|
|
.filter((file) => !gated(file) && !excluded.has(file))
|
|
.map((file) =>
|
|
`test file is not run by any CI gate command and is not in ${UNGATED_TESTS_PATH}: ${file}`
|
|
);
|
|
// An entry for a file a removal deleted is harmless; one for a file a gate
|
|
// now runs would hide the next regression, so it has to go.
|
|
for (const { path: file } of input.exclusions) {
|
|
if (gated(file)) failures.push(`ungated test exclusion is stale; a CI gate runs it: ${file}`);
|
|
}
|
|
return failures;
|
|
}
|