문제: FE-GATE-020이 desktop merge 단계 27.4분 중 17.2분이다. 그 안에서
같은 시험 파일이 한 복사본에서 두 번 돈다.
- feature-scaffold 드릴은 merge 단계 명령을 차례로 돌며 test:runtime-schema
·test:unit·test:contract·test:component·test:integration을 돌고, 바로
FE-GATE-005의 test:coverage가 같은 경로를 다시 돈다(약 80초).
- feature-removal 확인은 test:all을 돈 뒤 test:coverage를 돈다. test:all의
경로는 tests/recipes 말고 모두 coverage 안에 있다(약 110초).
고른 방법:
- scaffold 드릴 제외 목록에 다섯 suite를 넣는다. 그 복사본의 merge 단계에
test:coverage가 있을 때만 뺀다. 없으면 plain으로 다시 돈다.
- feature-removal은 test:coverage 다음에 coverage 밖에 남는 suite
(REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE = test:recipes)만 돈다.
- tests/unit/removal-drill-test-paths.test.ts가 경로를 package.json에서
읽어 「뺀 suite의 경로 ⊆ coverage 경로」, 「test:all 경로 ⊆ coverage +
남긴 suite」, coverage가 없으면 다시 도는 것을 지킨다.
- 버린 대안: FE-GATE-005 자체의 plain test:unit·test:contract를 빼고
coverage 실행에서 junit을 나누기. 그 script가 어느 게이트에서도 안 돌게
되어 ungated 목록을 늘려야 한다(게이트 기준을 낮추는 일).
위험·한계: plain 실행만의 차이(coverage 계측 없음, worker 수 기본값,
tests/platform을 test:platform처럼 파일 차례로 도는 것)는 드릴에서 더
보지 않는다. merge 단계의 FE-GATE-005가 coverage로 같은 파일을 병렬로
돌고 있어 그 모양이 이미 게이트다. 선택 기능 제거 드릴 셋의 test:all은
그대로다(그 드릴은 coverage를 돌지 않는다).
검증(desktop, 38c0dc7d 위): removal-drill-test-paths·feature-scaffold-drill
·example-feature-rules·test-all-paths·fixture-node-modules 20개 통과,
check:types·eslint rc=0. 변이 두 개(남긴 suite 목록 비우기, test:unit에
tests/recipes 더하기)에서 각각 해당 시험이 실패.
176 lines
7.7 KiB
TypeScript
176 lines
7.7 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
|
|
import type { CiGateContract } from "../contracts/ci-gates.ts";
|
|
import { classifyGateStepResult, type GateStepExpectation } from "./ci-step-result.ts";
|
|
import { FIXTURE_PNPM_ENVIRONMENT } from "./fixture-node-modules.ts";
|
|
import type { RemovalCheck } from "./removal-fixture.ts";
|
|
|
|
/**
|
|
* 1103/12-02. test:feature-scaffold proves that a feature from
|
|
* `create:feature` passes the merge gates. It ran a hand-kept list of 15
|
|
* commands while the merge stage has 88, so a template change that broke a
|
|
* generated feature in any other merge command passed FE-GATE-020.
|
|
*
|
|
* The drill now runs the merge stage of the copy's config/ci/gates.json, in
|
|
* stage order, each command once. A command added to the merge stage runs here
|
|
* unless this list names it, with the reason it cannot run in the drill.
|
|
*/
|
|
const BROWSER =
|
|
"needs Playwright browsers. FE-GATE-020 runs in a merge job without them " +
|
|
"(merge_test installs none). The generated routes in the browser " +
|
|
"gates are reviewed as 1103/12-01 and 12-03.";
|
|
const DRILL =
|
|
"a removal drill copies the repository again. The drill runs the generated " +
|
|
"feature's own removal instead (feature-removal).";
|
|
/**
|
|
* The drill runs FE-GATE-005's test:coverage in the same copy, and its paths
|
|
* hold every path of these suites (tests/unit/removal-drill-test-paths.test.ts).
|
|
* Running them plain as well found nothing more and cost about 80 s a drill.
|
|
*/
|
|
const COVERED =
|
|
"test:coverage runs the same test files in this copy " +
|
|
"(tests/unit/removal-drill-test-paths.test.ts checks the paths).";
|
|
export const SCAFFOLD_DRILL_COVERED_SUITES: readonly string[] = Object.freeze([
|
|
"test:runtime-schema",
|
|
"test:unit",
|
|
"test:contract",
|
|
"test:component",
|
|
"test:integration",
|
|
]);
|
|
/**
|
|
* What a removal drill runs next to test:coverage so that every test:all path
|
|
* still runs once in the copy (scripts/test-sample-removal.ts). The rest of
|
|
* test:all, check:release-coherence included, is inside coverage's paths.
|
|
*/
|
|
export const REMOVAL_DRILL_SUITES_OUTSIDE_COVERAGE: readonly string[] = Object.freeze([
|
|
"test:recipes",
|
|
]);
|
|
|
|
export const SCAFFOLD_DRILL_EXCLUDED_SCRIPTS: Readonly<Record<string, string>> = Object.freeze({
|
|
"verify:lockfile":
|
|
"runs pnpm install, which writes into the node_modules the copy links from " +
|
|
"the repository (scripts/lib/fixture-node-modules.ts).",
|
|
"test:e2e": BROWSER,
|
|
"test:browser-capabilities": BROWSER,
|
|
"verify:browser-capability-evidence": `reads the test:browser-capabilities results; ${BROWSER}`,
|
|
"test:storybook": BROWSER,
|
|
"test:visual": BROWSER,
|
|
"check:test-evidence:browser": `reads the browser gates' results; ${BROWSER}`,
|
|
"test:a11y": BROWSER,
|
|
"test:system":
|
|
"needs the CI runner's bubblewrap, systemd and cgroup v2 " +
|
|
"(scripts/check-system-test-prerequisites.ts). It tests the provider " +
|
|
"sandbox, which a feature does not reach.",
|
|
...Object.fromEntries(SCAFFOLD_DRILL_COVERED_SUITES.map((script) => [script, COVERED])),
|
|
"test:sample-removal": DRILL,
|
|
"test:feature-scaffold": "is this drill.",
|
|
"test:optional-recipe-removal": DRILL,
|
|
"test:browser-file-storage-removal": DRILL,
|
|
"test:realtime-removal": DRILL,
|
|
});
|
|
|
|
export type ScaffoldDrillCommand = Readonly<{
|
|
/** `<gate>:<command id>`, the check's name in the drill report. */
|
|
name: string;
|
|
script: string;
|
|
args: readonly string[];
|
|
expectation: GateStepExpectation;
|
|
}>;
|
|
|
|
type ContractShape = Pick<CiGateContract, "commands" | "gates" | "stages">;
|
|
|
|
export function scaffoldDrillCommands(
|
|
contract: ContractShape,
|
|
excluded: Readonly<Record<string, string>> = SCAFFOLD_DRILL_EXCLUDED_SCRIPTS,
|
|
): readonly ScaffoldDrillCommand[] {
|
|
const merge = contract.stages.find(({ id }) => id === "merge");
|
|
if (!merge) throw new TypeError("the CI contract has no merge stage");
|
|
const commands = new Map(contract.commands.map((command) => [command.id, command]));
|
|
const mergeScripts = new Set<string>();
|
|
const seen = new Set<string>();
|
|
const selected: ScaffoldDrillCommand[] = [];
|
|
// A covered suite is left out only while the drill runs test:coverage. A
|
|
// copy whose merge stage lost it runs the suites plain again.
|
|
const runsCoverage = !Object.hasOwn(excluded, "test:coverage") && merge.gateIds.some((gateId) =>
|
|
contract.gates.find(({ id }) => id === gateId)?.commandIds.some((commandId) =>
|
|
commands.get(commandId)?.script === "test:coverage"));
|
|
const leftOut = (script: string) =>
|
|
Object.hasOwn(excluded, script) &&
|
|
(runsCoverage || !SCAFFOLD_DRILL_COVERED_SUITES.includes(script) || excluded[script] !== COVERED);
|
|
for (const gateId of merge.gateIds) {
|
|
const gate = contract.gates.find(({ id }) => id === gateId);
|
|
if (!gate) throw new TypeError(`the merge stage names an undefined gate: ${gateId}`);
|
|
for (const commandId of gate.commandIds) {
|
|
const command = commands.get(commandId);
|
|
if (!command) throw new TypeError(`${gateId} names an undefined command: ${commandId}`);
|
|
mergeScripts.add(command.script);
|
|
if (seen.has(commandId) || leftOut(command.script)) continue;
|
|
seen.add(commandId);
|
|
selected.push(Object.freeze({
|
|
name: `${gateId}:${commandId}`,
|
|
script: command.script,
|
|
args: Object.freeze([...(command.args ?? [])]),
|
|
expectation: command.expect === "pass"
|
|
? Object.freeze({ kind: "pass" as const })
|
|
: Object.freeze({
|
|
kind: "fail" as const,
|
|
expectedExitCode: command.expectedExitCode ?? 1,
|
|
expectedDiagnosticId: command.expectedDiagnosticId ?? "",
|
|
}),
|
|
}));
|
|
}
|
|
}
|
|
// An exclusion for a command that moved out of the merge stage would hide
|
|
// nothing today and silently drop it if it came back. A command the contract
|
|
// no longer has at all (a removal drill pruned its capability) is not stale.
|
|
const contractScripts = new Set(contract.commands.map(({ script }) => script));
|
|
const stale = Object.keys(excluded).filter(
|
|
(script) => contractScripts.has(script) && !mergeScripts.has(script),
|
|
);
|
|
if (stale.length > 0) {
|
|
throw new TypeError(
|
|
`the scaffold drill excludes scripts the contract runs outside the merge stage: ${stale.join(", ")}`,
|
|
);
|
|
}
|
|
return Object.freeze(selected);
|
|
}
|
|
|
|
/**
|
|
* Runs one merge command in the fixture. A negative fixture command passes
|
|
* only with its registered exit code and diagnostic, as in run-ci-gate.ts.
|
|
*/
|
|
export function runScaffoldDrillCommand(
|
|
command: ScaffoldDrillCommand,
|
|
root: string,
|
|
pnpmCli: string,
|
|
): RemovalCheck {
|
|
const capture = command.expectation.kind === "fail";
|
|
const result = spawnSync(process.execPath, [pnpmCli, command.script, ...command.args], {
|
|
cwd: root,
|
|
encoding: "utf8",
|
|
stdio: capture ? "pipe" : "inherit",
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT },
|
|
});
|
|
if (capture) {
|
|
process.stdout.write(result.stdout ?? "");
|
|
process.stderr.write(result.stderr ?? "");
|
|
}
|
|
const classification = classifyGateStepResult(command.expectation, {
|
|
status: result.status,
|
|
signal: result.signal,
|
|
stdout: result.stdout ?? "",
|
|
stderr: result.stderr ?? "",
|
|
...(result.error ? { error: { code: (result.error as NodeJS.ErrnoException).code } } : {}),
|
|
});
|
|
if (classification.expectationMet) return { name: command.name, passed: true };
|
|
const invocation = `corepack pnpm ${[command.script, ...command.args].join(" ")}`;
|
|
return {
|
|
name: command.name,
|
|
passed: false,
|
|
failure: command.expectation.kind === "fail"
|
|
? `${invocation}: ${classification.kind}, expected exit ${command.expectation.expectedExitCode} with ${JSON.stringify(command.expectation.expectedDiagnosticId)}, got exit ${result.status ?? result.signal}`
|
|
: `${invocation} exited ${result.status ?? result.signal ?? "on a spawn error"}`,
|
|
};
|
|
}
|