diff --git a/scripts/lib/feature-scaffold-drill.ts b/scripts/lib/feature-scaffold-drill.ts index e499337c..a27acd2f 100644 --- a/scripts/lib/feature-scaffold-drill.ts +++ b/scripts/lib/feature-scaffold-drill.ts @@ -22,6 +22,29 @@ const BROWSER = const DRILL = "a removal drill copies the repository again. The drill runs the generated " + "feature's own removal instead (feature-removal)."; +/** + * The drill runs FE-GATE-005's test:coverage in the same copy, and its paths + * hold every path of these suites (tests/unit/removal-drill-test-paths.test.ts). + * Running them plain as well found nothing more and cost about 80 s a drill. + */ +const COVERED = + "test:coverage runs the same test files in this copy " + + "(tests/unit/removal-drill-test-paths.test.ts checks the paths)."; +export const SCAFFOLD_DRILL_COVERED_SUITES: readonly string[] = Object.freeze([ + "test:runtime-schema", + "test:unit", + "test:contract", + "test:component", + "test:integration", +]); +/** + * What a removal drill runs next to test:coverage so that every test:all path + * still runs once in the copy (scripts/test-sample-removal.ts). The rest of + * test:all, check:release-coherence included, is inside coverage's paths. + */ +export const REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE: readonly string[] = Object.freeze([ + "test:recipes", +]); export const SCAFFOLD_DRILL_EXCLUDED_SCRIPTS: Readonly> = Object.freeze({ "verify:lockfile": @@ -38,6 +61,7 @@ export const SCAFFOLD_DRILL_EXCLUDED_SCRIPTS: Readonly> = "needs the CI runner's bubblewrap, systemd and cgroup v2 " + "(scripts/check-system-test-prerequisites.ts). It tests the provider " + "sandbox, which a feature does not reach.", + ...Object.fromEntries(SCAFFOLD_DRILL_COVERED_SUITES.map((script) => [script, COVERED])), "test:sample-removal": DRILL, "test:feature-scaffold": "is this drill.", "test:optional-recipe-removal": DRILL, @@ -65,6 +89,14 @@ export function scaffoldDrillCommands( const mergeScripts = new Set(); const seen = new Set(); const selected: ScaffoldDrillCommand[] = []; + // A covered suite is left out only while the drill runs test:coverage. A + // copy whose merge stage lost it runs the suites plain again. + const runsCoverage = !Object.hasOwn(excluded, "test:coverage") && merge.gateIds.some((gateId) => + contract.gates.find(({ id }) => id === gateId)?.commandIds.some((commandId) => + commands.get(commandId)?.script === "test:coverage")); + const leftOut = (script: string) => + Object.hasOwn(excluded, script) && + (runsCoverage || !SCAFFOLD_DRILL_COVERED_SUITES.includes(script) || excluded[script] !== COVERED); for (const gateId of merge.gateIds) { const gate = contract.gates.find(({ id }) => id === gateId); if (!gate) throw new TypeError(`the merge stage names an undefined gate: ${gateId}`); @@ -72,7 +104,7 @@ export function scaffoldDrillCommands( const command = commands.get(commandId); if (!command) throw new TypeError(`${gateId} names an undefined command: ${commandId}`); mergeScripts.add(command.script); - if (seen.has(commandId) || Object.hasOwn(excluded, command.script)) continue; + if (seen.has(commandId) || leftOut(command.script)) continue; seen.add(commandId); selected.push(Object.freeze({ name: `${gateId}:${commandId}`, diff --git a/scripts/test-sample-removal.ts b/scripts/test-sample-removal.ts index e60d011d..7437e854 100644 --- a/scripts/test-sample-removal.ts +++ b/scripts/test-sample-removal.ts @@ -31,6 +31,7 @@ import { sealRemovalFixtureRepository, type RemovalCheck, } from "./lib/removal-fixture.ts"; +import { REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE } from "./lib/feature-scaffold-drill.ts"; import { REMOVABLE_TEMPLATE_FEATURES } from "./lib/removable-template-features.ts"; /** @@ -476,8 +477,13 @@ try { check(`${prefix}design-system`, "check:design-system"), check(`${prefix}architecture`, "check:architecture"), check(`${prefix}registries`, "check:registries"), - check(`${prefix}unit-integration`, "test:all"), + // test:all ran here before coverage and ran its files twice. coverage + // holds every test:all path but the optional recipes' + // (tests/unit/removal-drill-test-paths.test.ts), so those run on their own. check(`${prefix}coverage`, "test:coverage"), + ...REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE + .filter((script) => Object.hasOwn(fixturePackage.scripts, script)) + .map((script) => check(`${prefix}${script.replace(/^test:/u, "")}`, script)), check(`${prefix}test-evidence-source`, "check:test-evidence:source"), check(`${prefix}ci-contract`, "check:ci"), check(`${prefix}home-smoke`, "exec", [ diff --git a/tests/unit/example-feature-rules.test.ts b/tests/unit/example-feature-rules.test.ts index 880abcfe..1795cc33 100644 --- a/tests/unit/example-feature-rules.test.ts +++ b/tests/unit/example-feature-rules.test.ts @@ -95,7 +95,10 @@ describe("sample features (R4)", () => { // It builds and tests the copy; a documents-only check does not prove removal. const drill = await readFile("scripts/test-sample-removal.ts", "utf8"); expect(drill).toMatch(/check\(`\$\{prefix\}build`, "build"\)/u); - expect(drill).toMatch(/check\(`\$\{prefix\}unit-integration`, "test:all"\)/u); + // Every test:all path runs once: coverage, then what coverage leaves out + // (tests/unit/removal-drill-test-paths.test.ts). + expect(drill).toMatch(/check\(`\$\{prefix\}coverage`, "test:coverage"\)/u); + expect(drill).toMatch(/\.\.\.REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE/u); }); }); diff --git a/tests/unit/feature-scaffold-drill.test.ts b/tests/unit/feature-scaffold-drill.test.ts index 7d52ae70..2037ea13 100644 --- a/tests/unit/feature-scaffold-drill.test.ts +++ b/tests/unit/feature-scaffold-drill.test.ts @@ -27,9 +27,11 @@ describe("feature scaffold drill commands", () => { commandId, ).toBe(true); } + // test:unit, test:contract and the other plain suites run inside + // test:coverage (tests/unit/removal-drill-test-paths.test.ts). expect(commands.map(({ script }) => script)).toEqual(expect.arrayContaining([ - "check:types", "lint", "test:unit", "test:coverage", "build", "check:bundle", - "test:contract", "test:runtime-schema", "check:request-fanout", "build:release-candidate", + "check:types", "lint", "test:coverage", "build", "check:bundle", + "test:recipes", "check:request-fanout", "build:release-candidate", ])); for (const reason of Object.values(SCAFFOLD_DRILL_EXCLUDED_SCRIPTS)) { expect(reason.length).toBeGreaterThan(10); diff --git a/tests/unit/removal-drill-test-paths.test.ts b/tests/unit/removal-drill-test-paths.test.ts new file mode 100644 index 00000000..8b3f5188 --- /dev/null +++ b/tests/unit/removal-drill-test-paths.test.ts @@ -0,0 +1,70 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { loadCiGateContract } from "../../scripts/contracts/ci-gates.ts"; +import { + REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE, + SCAFFOLD_DRILL_COVERED_SUITES, + scaffoldDrillCommands, +} from "../../scripts/lib/feature-scaffold-drill.ts"; +import { gatedTestPaths } from "../../scripts/lib/ungated-tests.ts"; + +/** + * FE-GATE-020's drills ran each test file twice in one copy: the plain suites + * and then test:coverage over the same paths (feature scaffold), test:all and + * then test:coverage (feature removal). Each runs once now. These tests keep + * the claim true: a path added to a plain suite or to test:all that coverage + * does not run fails here instead of silently leaving the drill. + */ +const scripts = ( + JSON.parse(readFileSync("package.json", "utf8")) as { scripts: Record } +).scripts; +const readConfig = (file: string) => readFileSync(file, "utf8"); +const pathsOf = (entries: readonly string[]) => gatedTestPaths(scripts, entries, readConfig); +const within = (target: string, roots: readonly string[]) => + roots.some((root) => target === root || target.startsWith(`${root}/`)); + +describe("removal drill test paths", () => { + const coverage = pathsOf(["test:coverage"]); + + it("leave out of the scaffold drill only suites test:coverage runs whole", () => { + expect(coverage.length).toBeGreaterThan(0); + for (const suite of SCAFFOLD_DRILL_COVERED_SUITES) { + const paths = pathsOf([suite]); + expect(paths.length, suite).toBeGreaterThan(0); + expect(paths.filter((target) => !within(target, coverage)), suite).toEqual([]); + } + }); + + it("run every test:all path in the feature removal: coverage and what it leaves out", () => { + const run = [...coverage, ...pathsOf(REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE)]; + expect(pathsOf(["test:all"]).filter((target) => !within(target, run))).toEqual([]); + }); + + it("name as outside coverage only suites coverage does not already run", () => { + for (const suite of REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE) { + expect(pathsOf([suite]).some((target) => !within(target, coverage)), suite).toBe(true); + } + }); + + it("run the covered suites plain again when the copy's merge stage has no test:coverage", async () => { + const contract = await loadCiGateContract(process.cwd()); + const coverageIds = new Set( + contract.commands.filter(({ script }) => script === "test:coverage").map(({ id }) => id), + ); + const withoutCoverage = { + ...contract, + gates: contract.gates.map((gate) => ({ + ...gate, + commandIds: gate.commandIds.filter((id) => !coverageIds.has(id)), + })), + }; + const withCoverage = scaffoldDrillCommands(contract).map(({ script }) => script); + const without = scaffoldDrillCommands(withoutCoverage).map(({ script }) => script); + for (const suite of SCAFFOLD_DRILL_COVERED_SUITES) { + expect(withCoverage, suite).not.toContain(suite); + expect(without, suite).toContain(suite); + } + }); +});