pnpm verifies the install before a script, and a fixture running it with CI=true against an install made without CI saw other settings and installed through the linked .pnpm and .bin, rewriting the shared lock.yaml and shims. Every fixture pnpm run now turns that check off; a test pins it for all. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
473 lines
18 KiB
TypeScript
473 lines
18 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import {
|
|
cp,
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
stat,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
import {
|
|
loadCiGateContract,
|
|
parseCiGateContract,
|
|
} from "../contracts/ci-gates.ts";
|
|
import {
|
|
FIXTURE_PNPM_ENVIRONMENT,
|
|
linkFixtureNodeModules,
|
|
} from "./fixture-node-modules.ts";
|
|
import { generateCiWorkflow } from "../generate-ci-workflow.ts";
|
|
import { updateCiContractBaseline } from "../update-ci-contract-baseline.ts";
|
|
|
|
export const REMOVAL_FIXTURE_COPY_TARGETS = Object.freeze([
|
|
"src", "tests", "recipes", "scripts", "schemas", "config", "public",
|
|
".gitea", ".storybook", "index.html", "package.json", "tsconfig.base.json",
|
|
"tsconfig.json", "tsconfig.app.json", "tsconfig.node.json", "tsconfig.test.json",
|
|
"tsconfig.recipes.json", "tsconfig.web-worker.json", "tsconfig.service-worker.json",
|
|
"vite.service-worker.config.ts", "vite.config.ts", "vitest.config.ts",
|
|
"playwright.config.ts", "playwright.capabilities.config.ts", "playwright.dev.config.ts",
|
|
"playwright.storybook.config.ts", "playwright.visual.config.ts", "eslint.config.ts",
|
|
".dependency-cruiser.json", ".nvmrc", ".gitignore",
|
|
// Install and workspace identity. Without these the fixture is not the same
|
|
// project: `corepack pnpm` resolves a different store, and the provider
|
|
// suites — which build a release candidate containing `pnpm-lock.yaml` —
|
|
// cannot assemble their fixture at all.
|
|
".npmrc", "pnpm-lock.yaml", "pnpm-workspace.yaml",
|
|
] as const);
|
|
|
|
/**
|
|
* This is the only copy-target list. Each removal script used to keep its own,
|
|
* and they drifted: the reference-feature fixture omitted
|
|
* `playwright.capabilities.config.ts`, which the repository file inventory
|
|
* requires, so supply-chain generation failed inside the fixture and took every
|
|
* provider suite down with it — twenty-odd failures with one cause.
|
|
*/
|
|
|
|
/**
|
|
* Regenerated result trees under `artifacts/`: traces, coverage HTML, recorded
|
|
* videos and Storybook bundles. They are tens of megabytes and mean nothing to
|
|
* a fixture. Everything else under `artifacts/` is release evidence a candidate
|
|
* is assembled from — and most of it is git-ignored too, so "is it tracked?"
|
|
* cannot be used to tell the two apart. `keepsReleaseEvidence` in
|
|
* tests/unit/removal-fixture.test.ts pins both halves of this split.
|
|
*/
|
|
const REGENERATED_ARTIFACT_TREES: readonly string[] = Object.freeze([
|
|
"artifacts/storybook",
|
|
"artifacts/tests/browser-capabilities",
|
|
"artifacts/tests/coverage",
|
|
"artifacts/tests/e2e",
|
|
"artifacts/tests/storybook",
|
|
"artifacts/tests/visual",
|
|
]);
|
|
|
|
/**
|
|
* Copies the release evidence a candidate build needs into a fixture root.
|
|
*
|
|
* A fixture that omits it cannot assemble a candidate archive at all, so every
|
|
* provider suite fails while constructing its own fixture — long before it
|
|
* reaches an assertion, and with an error that says nothing about the
|
|
* capability under test.
|
|
*/
|
|
export async function copyReleaseEvidenceTree(
|
|
sourceRoot: string,
|
|
destinationRoot: string,
|
|
): Promise<void> {
|
|
const source = path.join(sourceRoot, "artifacts");
|
|
try {
|
|
await stat(source);
|
|
} catch {
|
|
return;
|
|
}
|
|
await cp(source, path.join(destinationRoot, "artifacts"), {
|
|
recursive: true,
|
|
filter: (candidate) => {
|
|
const relative = path.relative(sourceRoot, candidate).split(path.sep).join("/");
|
|
return !REGENERATED_ARTIFACT_TREES.some(
|
|
(tree) => relative === tree || relative.startsWith(`${tree}/`),
|
|
);
|
|
},
|
|
});
|
|
// The result directories still have to exist: several are tracked through a
|
|
// `.gitkeep` the repository inventory expects to find.
|
|
for (const tree of REGENERATED_ARTIFACT_TREES) {
|
|
await mkdir(path.join(destinationRoot, tree), { recursive: true });
|
|
}
|
|
}
|
|
|
|
export const RELEASE_EVIDENCE_REGENERATED_TREES = REGENERATED_ARTIFACT_TREES;
|
|
|
|
export function requireRemovalFixtureEnvironment(name: string): string {
|
|
const value = process.env[name];
|
|
if (!value) throw new Error(`${name} is required for removal verification`);
|
|
return value;
|
|
}
|
|
|
|
export async function prepareRemovalFixture(
|
|
root: string,
|
|
copyTargets: readonly string[] = REMOVAL_FIXTURE_COPY_TARGETS,
|
|
): Promise<void> {
|
|
await rm(root, { recursive: true, force: true });
|
|
await mkdir(root, { recursive: true });
|
|
for (const target of copyTargets) {
|
|
await cp(target, path.join(root, target), { recursive: true });
|
|
}
|
|
await copyReleaseEvidenceTree(process.cwd(), root);
|
|
runFixtureGit(root, ["init", "--quiet", "--initial-branch=fixture"]);
|
|
await linkFixtureNodeModules(root);
|
|
}
|
|
|
|
/**
|
|
* Records the fixture's post-removal contents as its repository state.
|
|
*
|
|
* The release candidate path asks `git ls-files` what the repository contains —
|
|
* the supply-chain inventory is defined as the tracked file set, not as
|
|
* whatever happens to be on disk. A fixture without a repository cannot answer
|
|
* that, so supply-chain generation failed and took every provider suite down
|
|
* with it; the claim "this build still produces a release candidate after the
|
|
* capability is removed" was never actually being tested.
|
|
*
|
|
* It runs after the removal, not during preparation: an index recorded before
|
|
* the deletions still lists the removed files, and the inventory then demands
|
|
* files the fixture exists to prove are gone.
|
|
*/
|
|
export function sealRemovalFixtureRepository(root: string): void {
|
|
// `.gitignore` travels with the fixture, so the tracked set it records is the
|
|
// same tracked set the real repository has. Without it every generated
|
|
// artifact and every linked module landed in the index, and the supply-chain
|
|
// inventory refused the fixture for having tracked and generated paths
|
|
// collide — the fixture disagreed with the repository it was copied from.
|
|
runFixtureGit(root, ["add", "--all"]);
|
|
runFixtureGit(root, ["commit", "--quiet", "--no-gpg-sign", "-m", "removal fixture"]);
|
|
}
|
|
|
|
function runFixtureGit(root: string, argv: readonly string[]): void {
|
|
const result = spawnSync("git", [...argv], {
|
|
cwd: root,
|
|
encoding: "utf8",
|
|
env: {
|
|
...process.env,
|
|
GIT_AUTHOR_NAME: "removal-fixture",
|
|
GIT_AUTHOR_EMAIL: "removal-fixture@localhost",
|
|
GIT_COMMITTER_NAME: "removal-fixture",
|
|
GIT_COMMITTER_EMAIL: "removal-fixture@localhost",
|
|
},
|
|
});
|
|
if (result.error || result.status !== 0) {
|
|
throw new Error(
|
|
`removal fixture repository setup failed at git ${argv[0]}: ${
|
|
result.stderr || result.error?.message || `exit ${result.status}`
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 14-08. A fresh fixture directory per run: two runs, or a run after one that
|
|
* crashed, no longer share (and overwrite) one fixed path. Callers remove it
|
|
* in `finally`.
|
|
*/
|
|
export async function createRemovalFixtureRoot(name: string): Promise<string> {
|
|
const parent = path.resolve(".tmp");
|
|
await mkdir(parent, { recursive: true });
|
|
return mkdtemp(path.join(parent, `${name}-`));
|
|
}
|
|
|
|
export type RemovalCheck = Readonly<{
|
|
name: string;
|
|
passed: boolean;
|
|
/** Why this check failed; each check names its own cause. */
|
|
failure?: string;
|
|
}>;
|
|
|
|
export function runRemovalFixtureCheck(
|
|
name: string,
|
|
root: string,
|
|
pnpmCli: string,
|
|
script: string,
|
|
extra: readonly string[] = [],
|
|
): RemovalCheck {
|
|
const status = spawnSync(process.execPath, [pnpmCli, script, ...extra], {
|
|
cwd: root,
|
|
stdio: "inherit",
|
|
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT },
|
|
}).status;
|
|
return status === 0
|
|
? { name, passed: true }
|
|
: {
|
|
name,
|
|
passed: false,
|
|
failure: `corepack pnpm ${[script, ...extra].join(" ")} exited ${status ?? "on a signal"}`,
|
|
};
|
|
}
|
|
|
|
function escapeXml(value: string): string {
|
|
return value
|
|
.replaceAll("&", "&")
|
|
.replaceAll("<", "<")
|
|
.replaceAll(">", ">")
|
|
.replaceAll('"', """);
|
|
}
|
|
|
|
/**
|
|
* 14-08. The report used to say `failures="1"` whatever failed, and put the
|
|
* residue list into every failed testcase, so a type error read as residue.
|
|
*/
|
|
export function renderRemovalJunit(
|
|
suite: string,
|
|
checks: readonly RemovalCheck[],
|
|
): string {
|
|
const failures = checks.filter((check) => !check.passed).length;
|
|
const cases = checks
|
|
.map((check) =>
|
|
`<testcase name="${escapeXml(check.name)}">${
|
|
check.passed
|
|
? ""
|
|
: `<failure message="${escapeXml(check.failure ?? "failed")}" />`
|
|
}</testcase>`,
|
|
)
|
|
.join("");
|
|
return (
|
|
`<?xml version="1.0" encoding="UTF-8"?>\n` +
|
|
`<testsuite name="${escapeXml(suite)}" tests="${checks.length}" failures="${failures}">` +
|
|
`${cases}</testsuite>\n`
|
|
);
|
|
}
|
|
|
|
export function runRemovalFixturePnpm(
|
|
root: string,
|
|
pnpmCli: string,
|
|
script: string,
|
|
extra: readonly string[] = [],
|
|
env: Readonly<Record<string, string>> = {},
|
|
): boolean {
|
|
return spawnSync(process.execPath, [pnpmCli, script, ...extra], {
|
|
cwd: root,
|
|
stdio: "inherit",
|
|
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, ...env },
|
|
}).status === 0;
|
|
}
|
|
|
|
export async function filesBelow(directory: string): Promise<string[]> {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
return (await Promise.all(entries.map((entry) => {
|
|
const target = path.join(directory, entry.name);
|
|
return entry.isDirectory() ? filesBelow(target) : [target];
|
|
}))).flat();
|
|
}
|
|
|
|
function isWithin(target: string, root: string): boolean {
|
|
const relative = path.relative(root, target);
|
|
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
|
}
|
|
|
|
export async function runtimeImportGraph(
|
|
root: string,
|
|
runtimeSourceRoots: readonly string[],
|
|
): Promise<Readonly<{ dependentTests: readonly string[]; importingFiles: readonly string[] }>> {
|
|
const files = (await filesBelow(root))
|
|
.filter((file) => /\.(?:[cm]?ts|tsx)$/u.test(file))
|
|
.map((file) => path.resolve(file));
|
|
if (files.length === 0) {
|
|
throw new Error("removal fixture scanned module universe is empty");
|
|
}
|
|
const sourceSet = new Set(files);
|
|
const runtimeRoots = runtimeSourceRoots.map((entry) => path.resolve(root, entry));
|
|
const imports = new Map<string, readonly string[]>();
|
|
for (const file of files) {
|
|
const source = await readFile(file, "utf8");
|
|
const specifiers = [...source.matchAll(/(?:from\s*|import\s*\(\s*|import\s*)["']([^"']+)["']/gu)]
|
|
.map((match) => match[1])
|
|
.filter((specifier): specifier is string => typeof specifier === "string" && specifier.startsWith("."));
|
|
imports.set(file, specifiers.map((specifier) => {
|
|
const base = path.resolve(path.dirname(file), specifier);
|
|
return [base, `${base}.ts`, `${base}.tsx`, `${base}.mts`, `${base}.cts`, path.join(base, "index.ts"), path.join(base, "index.tsx")]
|
|
.find((candidate) => sourceSet.has(candidate)) ?? base;
|
|
}));
|
|
}
|
|
const memo = new Map<string, boolean>();
|
|
const reachesRuntime = (file: string, visiting = new Set<string>()): boolean => {
|
|
if (runtimeRoots.some((runtimeRoot) => isWithin(file, runtimeRoot))) return true;
|
|
const known = memo.get(file);
|
|
if (known !== undefined) return known;
|
|
if (visiting.has(file)) return false;
|
|
visiting.add(file);
|
|
const reaches = (imports.get(file) ?? []).some((dependency) =>
|
|
runtimeRoots.some((runtimeRoot) => isWithin(dependency, runtimeRoot)) ||
|
|
(sourceSet.has(dependency) && reachesRuntime(dependency, visiting))
|
|
);
|
|
visiting.delete(file);
|
|
memo.set(file, reaches);
|
|
return reaches;
|
|
};
|
|
const testsRoot = path.resolve(root, "tests");
|
|
return Object.freeze({
|
|
dependentTests: Object.freeze(files.filter((file) => isWithin(file, testsRoot) && reachesRuntime(file))),
|
|
importingFiles: Object.freeze(files.filter((file) =>
|
|
!runtimeRoots.some((runtimeRoot) => isWithin(file, runtimeRoot)) &&
|
|
(imports.get(file) ?? []).some((dependency) =>
|
|
runtimeRoots.some((runtimeRoot) => isWithin(dependency, runtimeRoot))
|
|
)
|
|
)),
|
|
});
|
|
}
|
|
|
|
export async function removeRuntimeDependentTests(
|
|
root: string,
|
|
runtimeSourceRoots: readonly string[],
|
|
): Promise<number> {
|
|
const graph = await runtimeImportGraph(root, runtimeSourceRoots);
|
|
if (graph.dependentTests.length === 0) {
|
|
throw new Error("removal fixture: no runtime-dependent tests discovered");
|
|
}
|
|
await Promise.all(graph.dependentTests.map((file) => rm(file, { force: true })));
|
|
return graph.dependentTests.length;
|
|
}
|
|
|
|
export async function assertNoRuntimeImports(
|
|
root: string,
|
|
runtimeSourceRoots: readonly string[],
|
|
capability: string,
|
|
): Promise<void> {
|
|
const graph = await runtimeImportGraph(root, runtimeSourceRoots);
|
|
if (graph.importingFiles.length > 0) {
|
|
throw new Error(`Removed ${capability} runtime is still imported by: ${graph.importingFiles.map((file) => path.relative(root, file)).join(", ")}`);
|
|
}
|
|
}
|
|
|
|
export function pruneScriptOrchestration(
|
|
scripts: Record<string, string>,
|
|
orchestrationScript: string,
|
|
removedScripts: ReadonlySet<string>,
|
|
): void {
|
|
const command = scripts[orchestrationScript];
|
|
if (!command) return;
|
|
scripts[orchestrationScript] = command.split(" && ").filter((segment) =>
|
|
![...removedScripts].some((removed) =>
|
|
new RegExp(`(?:^|\\s)(?:corepack\\s+)?pnpm\\s+${removed.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")}(?:\\s|$)`, "u").test(segment)
|
|
)
|
|
).join(" && ");
|
|
}
|
|
|
|
export async function regenerateRemovalFixtureWorkflow(root: string): Promise<void> {
|
|
const contract = await loadCiGateContract(root);
|
|
await generateCiWorkflow({ root, contract, check: false });
|
|
// Every removal script calls this once, after it has finished mutating the
|
|
// tree, so it is the one place where the fixture's contents are final.
|
|
await pruneRemovalFixtureInventoryRoots(root);
|
|
sealRemovalFixtureRepository(root);
|
|
}
|
|
|
|
/**
|
|
* Drops repository roots the removal deleted from the supply-chain inventory
|
|
* policy.
|
|
*
|
|
* The policy lists `recipes` as a required tracked root, and removing an
|
|
* optional recipe deletes exactly that directory. Supply-chain generation then
|
|
* refused the fixture for missing a root the removal was supposed to remove, so
|
|
* the capability could never be shown to be removable. A root that is not on
|
|
* disk after the removal is not required of the result.
|
|
*/
|
|
async function pruneRemovalFixtureInventoryRoots(root: string): Promise<void> {
|
|
const policyPath = path.join(root, "config/security/secret-scan-policy.json");
|
|
let policy: Record<string, unknown>;
|
|
try {
|
|
policy = JSON.parse(await readFile(policyPath, "utf8")) as Record<string, unknown>;
|
|
} catch {
|
|
return;
|
|
}
|
|
const tracked = policy["trackedRoots"];
|
|
if (!Array.isArray(tracked)) return;
|
|
const surviving: string[] = [];
|
|
for (const entry of tracked) {
|
|
if (typeof entry !== "string") continue;
|
|
try {
|
|
await stat(path.join(root, entry));
|
|
surviving.push(entry);
|
|
} catch {
|
|
// Deleted by the removal under test.
|
|
}
|
|
}
|
|
if (surviving.length === tracked.length) return;
|
|
policy["trackedRoots"] = surviving;
|
|
await writeFile(policyPath, `${JSON.stringify(policy, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
export async function pruneRemovalFixtureCiContract(options: Readonly<{
|
|
root: string;
|
|
removedScripts: ReadonlySet<string>;
|
|
removedEvidencePathFragments: readonly string[];
|
|
}>): Promise<void> {
|
|
const packagePath = path.join(options.root, "package.json");
|
|
const gatesPath = path.join(options.root, "config/ci/gates.json");
|
|
const packageDocument = JSON.parse(await readFile(packagePath, "utf8")) as {
|
|
scripts: Record<string, string>;
|
|
};
|
|
for (const script of options.removedScripts) delete packageDocument.scripts[script];
|
|
|
|
const contract = structuredClone(
|
|
parseCiGateContract(JSON.parse(await readFile(gatesPath, "utf8"))),
|
|
);
|
|
const removedCommandIds = new Set(
|
|
contract.commands
|
|
.filter(({ script }) => options.removedScripts.has(script))
|
|
.map(({ id }) => id),
|
|
);
|
|
const missing = [...options.removedScripts].filter(
|
|
(script) => !contract.commands.some((command) => command.script === script),
|
|
);
|
|
if (missing.length > 0) {
|
|
throw new Error(`removal fixture CI command set is incomplete: ${missing.join(", ")}`);
|
|
}
|
|
const removedArtifactIds = new Set(
|
|
contract.artifacts
|
|
.filter(({ path: artifactPath }) =>
|
|
options.removedEvidencePathFragments.some((fragment) => artifactPath.includes(fragment))
|
|
)
|
|
.map(({ id }) => id),
|
|
);
|
|
for (const fragment of options.removedEvidencePathFragments) {
|
|
if (!contract.artifacts.some(({ path: artifactPath }) => artifactPath.includes(fragment))) {
|
|
throw new Error(`removal fixture CI evidence is missing: ${fragment}`);
|
|
}
|
|
}
|
|
contract.commands = contract.commands.filter(({ id }) => !removedCommandIds.has(id));
|
|
contract.artifacts = contract.artifacts
|
|
.filter(({ id }) => !removedArtifactIds.has(id))
|
|
.map((artifact) => artifact.production === "command-generated"
|
|
? {
|
|
...artifact,
|
|
producerCommandIds: artifact.producerCommandIds.filter(
|
|
(commandId) => !removedCommandIds.has(commandId),
|
|
),
|
|
}
|
|
: artifact)
|
|
.filter((artifact) =>
|
|
artifact.production !== "command-generated" || artifact.producerCommandIds.length > 0
|
|
);
|
|
const retainedArtifactIds = new Set(contract.artifacts.map(({ id }) => id));
|
|
for (const gate of contract.gates) {
|
|
gate.commandIds = gate.commandIds.filter((commandId) => !removedCommandIds.has(commandId));
|
|
gate.evidenceArtifactIds = gate.evidenceArtifactIds.filter((artifactId) =>
|
|
retainedArtifactIds.has(artifactId)
|
|
);
|
|
}
|
|
const referencedSchemaIds = new Set(contract.artifacts.map(({ schemaId }) => schemaId));
|
|
contract.artifactSchemas = contract.artifactSchemas.filter(({ id }) =>
|
|
referencedSchemaIds.has(id)
|
|
);
|
|
const validated = parseCiGateContract(contract);
|
|
await Promise.all([
|
|
writeFile(packagePath, `${JSON.stringify(packageDocument, null, 2)}\n`),
|
|
writeFile(gatesPath, `${JSON.stringify(validated, null, 2)}\n`),
|
|
]);
|
|
// D-13: the reduced contract is a new baseline, recorded the way a project
|
|
// that removes the capability records it; canonical check:ci then applies.
|
|
await updateCiContractBaseline(options.root, {
|
|
owner: "removal-fixture",
|
|
reason: `removal fixture pruned ${[...options.removedScripts].join(", ")}`,
|
|
});
|
|
}
|