perf(scripts): FE-GATE-020 드릴이 같은 시험 파일을 한 복사본에서 두 번 돌지 않는다
문제: FE-GATE-020이 desktop merge 단계 27.4분 중 17.2분이다. 그 안에서
같은 시험 파일이 한 복사본에서 두 번 돈다.
- feature-scaffold 드릴은 merge 단계 명령을 차례로 돌며 test:runtime-schema
·test:unit·test:contract·test:component·test:integration을 돌고, 바로
FE-GATE-005의 test:coverage가 같은 경로를 다시 돈다(약 80초).
- feature-removal 확인은 test:all을 돈 뒤 test:coverage를 돈다. test:all의
경로는 tests/recipes 말고 모두 coverage 안에 있다(약 110초).
고른 방법:
- scaffold 드릴 제외 목록에 다섯 suite를 넣는다. 그 복사본의 merge 단계에
test:coverage가 있을 때만 뺀다. 없으면 plain으로 다시 돈다.
- feature-removal은 test:coverage 다음에 coverage 밖에 남는 suite
(REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE = test:recipes)만 돈다.
- tests/unit/removal-drill-test-paths.test.ts가 경로를 package.json에서
읽어 「뺀 suite의 경로 ⊆ coverage 경로」, 「test:all 경로 ⊆ coverage +
남긴 suite」, coverage가 없으면 다시 도는 것을 지킨다.
- 버린 대안: FE-GATE-005 자체의 plain test:unit·test:contract를 빼고
coverage 실행에서 junit을 나누기. 그 script가 어느 게이트에서도 안 돌게
되어 ungated 목록을 늘려야 한다(게이트 기준을 낮추는 일).
위험·한계: plain 실행만의 차이(coverage 계측 없음, worker 수 기본값,
tests/platform을 test:platform처럼 파일 차례로 도는 것)는 드릴에서 더
보지 않는다. merge 단계의 FE-GATE-005가 coverage로 같은 파일을 병렬로
돌고 있어 그 모양이 이미 게이트다. 선택 기능 제거 드릴 셋의 test:all은
그대로다(그 드릴은 coverage를 돌지 않는다).
검증(desktop, 38c0dc7d 위): removal-drill-test-paths·feature-scaffold-drill
·example-feature-rules·test-all-paths·fixture-node-modules 20개 통과,
check:types·eslint rc=0. 변이 두 개(남긴 suite 목록 비우기, test:unit에
tests/recipes 더하기)에서 각각 해당 시험이 실패.
This commit is contained in:
@@ -22,6 +22,29 @@ const BROWSER =
|
||||
const DRILL =
|
||||
"a removal drill copies the repository again. The drill runs the generated " +
|
||||
"feature's own removal instead (feature-removal).";
|
||||
/**
|
||||
* The drill runs FE-GATE-005's test:coverage in the same copy, and its paths
|
||||
* hold every path of these suites (tests/unit/removal-drill-test-paths.test.ts).
|
||||
* Running them plain as well found nothing more and cost about 80 s a drill.
|
||||
*/
|
||||
const COVERED =
|
||||
"test:coverage runs the same test files in this copy " +
|
||||
"(tests/unit/removal-drill-test-paths.test.ts checks the paths).";
|
||||
export const SCAFFOLD_DRILL_COVERED_SUITES: readonly string[] = Object.freeze([
|
||||
"test:runtime-schema",
|
||||
"test:unit",
|
||||
"test:contract",
|
||||
"test:component",
|
||||
"test:integration",
|
||||
]);
|
||||
/**
|
||||
* What a removal drill runs next to test:coverage so that every test:all path
|
||||
* still runs once in the copy (scripts/test-sample-removal.ts). The rest of
|
||||
* test:all, check:release-coherence included, is inside coverage's paths.
|
||||
*/
|
||||
export const REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE: readonly string[] = Object.freeze([
|
||||
"test:recipes",
|
||||
]);
|
||||
|
||||
export const SCAFFOLD_DRILL_EXCLUDED_SCRIPTS: Readonly<Record<string, string>> = Object.freeze({
|
||||
"verify:lockfile":
|
||||
@@ -38,6 +61,7 @@ export const SCAFFOLD_DRILL_EXCLUDED_SCRIPTS: Readonly<Record<string, string>> =
|
||||
"needs the CI runner's bubblewrap, systemd and cgroup v2 " +
|
||||
"(scripts/check-system-test-prerequisites.ts). It tests the provider " +
|
||||
"sandbox, which a feature does not reach.",
|
||||
...Object.fromEntries(SCAFFOLD_DRILL_COVERED_SUITES.map((script) => [script, COVERED])),
|
||||
"test:sample-removal": DRILL,
|
||||
"test:feature-scaffold": "is this drill.",
|
||||
"test:optional-recipe-removal": DRILL,
|
||||
@@ -65,6 +89,14 @@ export function scaffoldDrillCommands(
|
||||
const mergeScripts = new Set<string>();
|
||||
const seen = new Set<string>();
|
||||
const selected: ScaffoldDrillCommand[] = [];
|
||||
// A covered suite is left out only while the drill runs test:coverage. A
|
||||
// copy whose merge stage lost it runs the suites plain again.
|
||||
const runsCoverage = !Object.hasOwn(excluded, "test:coverage") && merge.gateIds.some((gateId) =>
|
||||
contract.gates.find(({ id }) => id === gateId)?.commandIds.some((commandId) =>
|
||||
commands.get(commandId)?.script === "test:coverage"));
|
||||
const leftOut = (script: string) =>
|
||||
Object.hasOwn(excluded, script) &&
|
||||
(runsCoverage || !SCAFFOLD_DRILL_COVERED_SUITES.includes(script) || excluded[script] !== COVERED);
|
||||
for (const gateId of merge.gateIds) {
|
||||
const gate = contract.gates.find(({ id }) => id === gateId);
|
||||
if (!gate) throw new TypeError(`the merge stage names an undefined gate: ${gateId}`);
|
||||
@@ -72,7 +104,7 @@ export function scaffoldDrillCommands(
|
||||
const command = commands.get(commandId);
|
||||
if (!command) throw new TypeError(`${gateId} names an undefined command: ${commandId}`);
|
||||
mergeScripts.add(command.script);
|
||||
if (seen.has(commandId) || Object.hasOwn(excluded, command.script)) continue;
|
||||
if (seen.has(commandId) || leftOut(command.script)) continue;
|
||||
seen.add(commandId);
|
||||
selected.push(Object.freeze({
|
||||
name: `${gateId}:${commandId}`,
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
sealRemovalFixtureRepository,
|
||||
type RemovalCheck,
|
||||
} from "./lib/removal-fixture.ts";
|
||||
import { REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE } from "./lib/feature-scaffold-drill.ts";
|
||||
import { REMOVABLE_TEMPLATE_FEATURES } from "./lib/removable-template-features.ts";
|
||||
|
||||
/**
|
||||
@@ -476,8 +477,13 @@ try {
|
||||
check(`${prefix}design-system`, "check:design-system"),
|
||||
check(`${prefix}architecture`, "check:architecture"),
|
||||
check(`${prefix}registries`, "check:registries"),
|
||||
check(`${prefix}unit-integration`, "test:all"),
|
||||
// test:all ran here before coverage and ran its files twice. coverage
|
||||
// holds every test:all path but the optional recipes'
|
||||
// (tests/unit/removal-drill-test-paths.test.ts), so those run on their own.
|
||||
check(`${prefix}coverage`, "test:coverage"),
|
||||
...REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE
|
||||
.filter((script) => Object.hasOwn(fixturePackage.scripts, script))
|
||||
.map((script) => check(`${prefix}${script.replace(/^test:/u, "")}`, script)),
|
||||
check(`${prefix}test-evidence-source`, "check:test-evidence:source"),
|
||||
check(`${prefix}ci-contract`, "check:ci"),
|
||||
check(`${prefix}home-smoke`, "exec", [
|
||||
|
||||
@@ -95,7 +95,10 @@ describe("sample features (R4)", () => {
|
||||
// It builds and tests the copy; a documents-only check does not prove removal.
|
||||
const drill = await readFile("scripts/test-sample-removal.ts", "utf8");
|
||||
expect(drill).toMatch(/check\(`\$\{prefix\}build`, "build"\)/u);
|
||||
expect(drill).toMatch(/check\(`\$\{prefix\}unit-integration`, "test:all"\)/u);
|
||||
// Every test:all path runs once: coverage, then what coverage leaves out
|
||||
// (tests/unit/removal-drill-test-paths.test.ts).
|
||||
expect(drill).toMatch(/check\(`\$\{prefix\}coverage`, "test:coverage"\)/u);
|
||||
expect(drill).toMatch(/\.\.\.REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE/u);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -27,9 +27,11 @@ describe("feature scaffold drill commands", () => {
|
||||
commandId,
|
||||
).toBe(true);
|
||||
}
|
||||
// test:unit, test:contract and the other plain suites run inside
|
||||
// test:coverage (tests/unit/removal-drill-test-paths.test.ts).
|
||||
expect(commands.map(({ script }) => script)).toEqual(expect.arrayContaining([
|
||||
"check:types", "lint", "test:unit", "test:coverage", "build", "check:bundle",
|
||||
"test:contract", "test:runtime-schema", "check:request-fanout", "build:release-candidate",
|
||||
"check:types", "lint", "test:coverage", "build", "check:bundle",
|
||||
"test:recipes", "check:request-fanout", "build:release-candidate",
|
||||
]));
|
||||
for (const reason of Object.values(SCAFFOLD_DRILL_EXCLUDED_SCRIPTS)) {
|
||||
expect(reason.length).toBeGreaterThan(10);
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { loadCiGateContract } from "../../scripts/contracts/ci-gates.ts";
|
||||
import {
|
||||
REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE,
|
||||
SCAFFOLD_DRILL_COVERED_SUITES,
|
||||
scaffoldDrillCommands,
|
||||
} from "../../scripts/lib/feature-scaffold-drill.ts";
|
||||
import { gatedTestPaths } from "../../scripts/lib/ungated-tests.ts";
|
||||
|
||||
/**
|
||||
* FE-GATE-020's drills ran each test file twice in one copy: the plain suites
|
||||
* and then test:coverage over the same paths (feature scaffold), test:all and
|
||||
* then test:coverage (feature removal). Each runs once now. These tests keep
|
||||
* the claim true: a path added to a plain suite or to test:all that coverage
|
||||
* does not run fails here instead of silently leaving the drill.
|
||||
*/
|
||||
const scripts = (
|
||||
JSON.parse(readFileSync("package.json", "utf8")) as { scripts: Record<string, string> }
|
||||
).scripts;
|
||||
const readConfig = (file: string) => readFileSync(file, "utf8");
|
||||
const pathsOf = (entries: readonly string[]) => gatedTestPaths(scripts, entries, readConfig);
|
||||
const within = (target: string, roots: readonly string[]) =>
|
||||
roots.some((root) => target === root || target.startsWith(`${root}/`));
|
||||
|
||||
describe("removal drill test paths", () => {
|
||||
const coverage = pathsOf(["test:coverage"]);
|
||||
|
||||
it("leave out of the scaffold drill only suites test:coverage runs whole", () => {
|
||||
expect(coverage.length).toBeGreaterThan(0);
|
||||
for (const suite of SCAFFOLD_DRILL_COVERED_SUITES) {
|
||||
const paths = pathsOf([suite]);
|
||||
expect(paths.length, suite).toBeGreaterThan(0);
|
||||
expect(paths.filter((target) => !within(target, coverage)), suite).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("run every test:all path in the feature removal: coverage and what it leaves out", () => {
|
||||
const run = [...coverage, ...pathsOf(REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE)];
|
||||
expect(pathsOf(["test:all"]).filter((target) => !within(target, run))).toEqual([]);
|
||||
});
|
||||
|
||||
it("name as outside coverage only suites coverage does not already run", () => {
|
||||
for (const suite of REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE) {
|
||||
expect(pathsOf([suite]).some((target) => !within(target, coverage)), suite).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("run the covered suites plain again when the copy's merge stage has no test:coverage", async () => {
|
||||
const contract = await loadCiGateContract(process.cwd());
|
||||
const coverageIds = new Set(
|
||||
contract.commands.filter(({ script }) => script === "test:coverage").map(({ id }) => id),
|
||||
);
|
||||
const withoutCoverage = {
|
||||
...contract,
|
||||
gates: contract.gates.map((gate) => ({
|
||||
...gate,
|
||||
commandIds: gate.commandIds.filter((id) => !coverageIds.has(id)),
|
||||
})),
|
||||
};
|
||||
const withCoverage = scaffoldDrillCommands(contract).map(({ script }) => script);
|
||||
const without = scaffoldDrillCommands(withoutCoverage).map(({ script }) => script);
|
||||
for (const suite of SCAFFOLD_DRILL_COVERED_SUITES) {
|
||||
expect(withCoverage, suite).not.toContain(suite);
|
||||
expect(without, suite).toContain(suite);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user