fix: measure repository-wide risk coverage
This commit is contained in:
DongHyeonka
@@ -0,0 +1,449 @@
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildProductionModuleInventory,
|
||||
evaluateRiskCoverage,
|
||||
parseRepositoryRiskCoveragePolicy,
|
||||
parseRiskCoveragePolicy,
|
||||
} from "../../scripts/lib/risk-coverage.ts";
|
||||
|
||||
const roots: string[] = [];
|
||||
const now = Date.parse("2026-08-02T00:00:00.000Z");
|
||||
const fullMetrics = {
|
||||
lines: { pct: 100 },
|
||||
statements: { pct: 100 },
|
||||
functions: { pct: 100 },
|
||||
branches: { pct: 100 },
|
||||
};
|
||||
|
||||
async function repositoryFixture(): Promise<string> {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "risk-coverage-"));
|
||||
roots.push(root);
|
||||
await mkdir(path.join(root, "src/nested"), { recursive: true });
|
||||
await Promise.all([
|
||||
writeFile(path.join(root, "src/a.ts"), "export const a = 1;\n"),
|
||||
writeFile(path.join(root, "src/nested/b.tsx"), "export const b = 2;\n"),
|
||||
writeFile(path.join(root, "src/types.d.ts"), "declare const value: 1;\n"),
|
||||
writeFile(path.join(root, "src/example.stories.tsx"), "export {};\n"),
|
||||
writeFile(path.join(root, "src/generated.ts"), "export const generated = true;\n"),
|
||||
writeFile(path.join(root, "src/ignored.js"), "export const ignored = true;\n"),
|
||||
]);
|
||||
return root;
|
||||
}
|
||||
|
||||
function policy(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
repositoryBaseline: 2,
|
||||
generatedPaths: ["src/generated.ts"],
|
||||
summary: {
|
||||
lines: 80,
|
||||
statements: 78,
|
||||
functions: 85,
|
||||
branches: 68,
|
||||
},
|
||||
criticalModules: [
|
||||
{
|
||||
path: "src/a.ts",
|
||||
owner: "platform-runtime",
|
||||
minimum: {
|
||||
lines: 80,
|
||||
statements: 78,
|
||||
functions: 85,
|
||||
branches: 68,
|
||||
},
|
||||
},
|
||||
],
|
||||
highRiskPaths: ["src/a.ts"],
|
||||
waivers: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("repository-aware risk coverage", () => {
|
||||
it("rejects the historical 14-file summary against the full repository", async () => {
|
||||
const repositoryRoot = process.cwd();
|
||||
const rawPolicy = JSON.parse(
|
||||
await readFile("config/testing/risk-coverage.json", "utf8"),
|
||||
) as unknown;
|
||||
const summary = JSON.parse(
|
||||
await readFile("tests/fixtures/coverage/repository-omission.json", "utf8"),
|
||||
) as unknown;
|
||||
const parsedPolicy = parseRepositoryRiskCoveragePolicy(rawPolicy, { now });
|
||||
const inventory = await buildProductionModuleInventory({
|
||||
repositoryRoot,
|
||||
generatedPaths: parsedPolicy.generatedPaths,
|
||||
});
|
||||
const result = evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parsedPolicy,
|
||||
summary,
|
||||
changedFiles: [],
|
||||
now,
|
||||
});
|
||||
|
||||
expect(result.selectedTotal).toBe(14);
|
||||
expect(result.repositoryTotal).toBeGreaterThan(result.selectedTotal);
|
||||
expect(result.uncoveredModules).toHaveLength(
|
||||
result.repositoryTotal - result.selectedTotal,
|
||||
);
|
||||
expect(result.status).toBe("FAIL");
|
||||
});
|
||||
|
||||
it("reports the exact selected and repository totals plus sorted omissions", async () => {
|
||||
const repositoryRoot = await repositoryFixture();
|
||||
const inventory = await buildProductionModuleInventory({
|
||||
repositoryRoot,
|
||||
generatedPaths: ["src/generated.ts"],
|
||||
});
|
||||
const result = evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parseRiskCoveragePolicy(policy(), { now }),
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
[path.join(repositoryRoot, "src/a.ts")]: fullMetrics,
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
});
|
||||
|
||||
expect(inventory).toEqual(["src/a.ts", "src/nested/b.tsx"]);
|
||||
expect(result).toMatchObject({
|
||||
status: "FAIL",
|
||||
selectedTotal: 1,
|
||||
repositoryTotal: 2,
|
||||
uncoveredModules: ["src/nested/b.tsx"],
|
||||
});
|
||||
expect(result.failures).toContain(
|
||||
"production module missing from coverage: src/nested/b.tsx",
|
||||
);
|
||||
});
|
||||
|
||||
it("maps only exact repository-relative POSIX coverage paths", async () => {
|
||||
const repositoryRoot = await repositoryFixture();
|
||||
const inventory = await buildProductionModuleInventory({
|
||||
repositoryRoot,
|
||||
generatedPaths: ["src/generated.ts"],
|
||||
});
|
||||
const parsedPolicy = parseRiskCoveragePolicy(policy(), { now });
|
||||
|
||||
expect(
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parsedPolicy,
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
"src/a.ts": fullMetrics,
|
||||
"src/nested/b.tsx": fullMetrics,
|
||||
"tests/unit/a.test.ts": fullMetrics,
|
||||
"src/generated.ts": fullMetrics,
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toMatchObject({
|
||||
status: "PASS",
|
||||
selectedTotal: 2,
|
||||
ignoredCoveragePaths: ["src/generated.ts", "tests/unit/a.test.ts"],
|
||||
});
|
||||
expect(() =>
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parsedPolicy,
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
[path.join(tmpdir(), "other/src/a.ts")]: fullMetrics,
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toThrow(/outside repository/u);
|
||||
expect(() =>
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parsedPolicy,
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
"src/a.ts": fullMetrics,
|
||||
[path.join(repositoryRoot, "src/a.ts")]: fullMetrics,
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toThrow(/duplicate coverage path/u);
|
||||
});
|
||||
|
||||
it("accepts and validates Vitest's branchesTrue total without evaluating it", () => {
|
||||
const parsedPolicy = parseRiskCoveragePolicy(policy(), { now });
|
||||
const branchesTrue = { total: 0, covered: 0, skipped: 0, pct: 100 };
|
||||
|
||||
expect(
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot: "/repository",
|
||||
inventory: ["src/a.ts"],
|
||||
policy: { ...parsedPolicy, repositoryBaseline: 1 },
|
||||
summary: {
|
||||
total: { ...fullMetrics, branchesTrue },
|
||||
"src/a.ts": fullMetrics,
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toMatchObject({ status: "PASS", selectedTotal: 1 });
|
||||
expect(() =>
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot: "/repository",
|
||||
inventory: ["src/a.ts"],
|
||||
policy: { ...parsedPolicy, repositoryBaseline: 1 },
|
||||
summary: {
|
||||
total: {
|
||||
...fullMetrics,
|
||||
branchesTrue: { ...branchesTrue, pct: Number.NaN },
|
||||
},
|
||||
"src/a.ts": fullMetrics,
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toThrow(/branchesTrue.*finite/u);
|
||||
});
|
||||
|
||||
it("fails closed on an empty, unreadable, traversing, or symlinked inventory", async () => {
|
||||
const repositoryRoot = await repositoryFixture();
|
||||
const emptyRoot = await mkdtemp(path.join(tmpdir(), "risk-coverage-empty-"));
|
||||
roots.push(emptyRoot);
|
||||
await mkdir(path.join(emptyRoot, "src"));
|
||||
|
||||
await expect(
|
||||
buildProductionModuleInventory({ repositoryRoot: emptyRoot }),
|
||||
).rejects.toThrow(/inventory is empty/u);
|
||||
await expect(
|
||||
buildProductionModuleInventory({
|
||||
repositoryRoot,
|
||||
assertReadable: async (target) => {
|
||||
if (target.endsWith("src/a.ts")) throw new Error("denied");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(/unreadable.*src\/a\.ts/u);
|
||||
await expect(
|
||||
buildProductionModuleInventory({
|
||||
repositoryRoot,
|
||||
generatedPaths: ["../escape.ts"],
|
||||
}),
|
||||
).rejects.toThrow(/repository-relative POSIX/u);
|
||||
|
||||
const outside = await mkdtemp(path.join(tmpdir(), "risk-coverage-outside-"));
|
||||
roots.push(outside);
|
||||
await writeFile(path.join(outside, "linked.ts"), "export {};\n");
|
||||
await symlink(path.join(outside, "linked.ts"), path.join(repositoryRoot, "src/link.ts"));
|
||||
await expect(
|
||||
buildProductionModuleInventory({ repositoryRoot }),
|
||||
).rejects.toThrow(/symlink.*src\/link\.ts/u);
|
||||
});
|
||||
|
||||
it("rejects malformed totals, duplicate paths, invalid minimums, and weak ownership", () => {
|
||||
expect(() => parseRiskCoveragePolicy(null, { now })).toThrow(/policy/u);
|
||||
expect(() =>
|
||||
parseRiskCoveragePolicy(policy({ repositoryBaseline: 0 }), { now }),
|
||||
).toThrow(/repositoryBaseline/u);
|
||||
expect(() =>
|
||||
parseRiskCoveragePolicy(
|
||||
policy({
|
||||
criticalModules: [
|
||||
{ path: "src/a.ts", owner: "team", minimum: { lines: 101 } },
|
||||
],
|
||||
}),
|
||||
{ now },
|
||||
),
|
||||
).toThrow(/minimum/u);
|
||||
expect(() =>
|
||||
parseRiskCoveragePolicy(
|
||||
policy({
|
||||
criticalModules: [
|
||||
{ path: "src/a.ts", owner: " ", minimum: { lines: 80 } },
|
||||
],
|
||||
}),
|
||||
{ now },
|
||||
),
|
||||
).toThrow(/owner/u);
|
||||
expect(() =>
|
||||
parseRiskCoveragePolicy(
|
||||
policy({ highRiskPaths: ["src/a.ts", "src/a.ts"] }),
|
||||
{ now },
|
||||
),
|
||||
).toThrow(/duplicate/u);
|
||||
|
||||
const parsedPolicy = parseRiskCoveragePolicy(policy(), { now });
|
||||
expect(() =>
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot: "/repository",
|
||||
inventory: ["src/a.ts"],
|
||||
policy: parsedPolicy,
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
"src/a.ts": { ...fullMetrics, lines: { pct: Number.NaN } },
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toThrow(/finite/u);
|
||||
expect(() =>
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot: "/repository",
|
||||
inventory: ["src/a.ts"],
|
||||
policy: parsedPolicy,
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
"src/a.ts": {
|
||||
...fullMetrics,
|
||||
lines: {
|
||||
total: Number.NaN,
|
||||
covered: 1,
|
||||
skipped: 0,
|
||||
pct: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toThrow(/lines\.total.*nonnegative/u);
|
||||
expect(() =>
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot: "/repository",
|
||||
inventory: ["src/a.ts"],
|
||||
policy: parsedPolicy,
|
||||
summary: {
|
||||
total: fullMetrics,
|
||||
"src/a.ts": { ...fullMetrics, conditions: { pct: 100 } },
|
||||
},
|
||||
changedFiles: [],
|
||||
now,
|
||||
}),
|
||||
).toThrow(/unknown.*metric/u);
|
||||
});
|
||||
|
||||
it("requires inventory-owned critical rows and exact owned future waivers", async () => {
|
||||
const repositoryRoot = await repositoryFixture();
|
||||
const inventory = await buildProductionModuleInventory({
|
||||
repositoryRoot,
|
||||
generatedPaths: ["src/generated.ts"],
|
||||
});
|
||||
const summary = {
|
||||
total: fullMetrics,
|
||||
"src/a.ts": fullMetrics,
|
||||
"src/nested/b.tsx": fullMetrics,
|
||||
};
|
||||
const waiverPolicy = parseRiskCoveragePolicy(
|
||||
policy({
|
||||
highRiskPaths: ["src/a.ts", "src/nested/b.tsx"],
|
||||
waivers: [
|
||||
{
|
||||
path: "src/nested/b.tsx",
|
||||
owner: "runtime-security",
|
||||
reason: "Temporary branch instrumentation gap",
|
||||
expiresAt: "2026-08-03T00:00:00.000Z",
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ now },
|
||||
);
|
||||
|
||||
expect(
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: waiverPolicy,
|
||||
summary,
|
||||
changedFiles: ["src/nested/b.tsx"],
|
||||
now,
|
||||
}),
|
||||
).toMatchObject({ status: "PASS" });
|
||||
expect(
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parseRiskCoveragePolicy(
|
||||
policy({ highRiskPaths: ["src/a.ts", "src/nested/b.tsx"] }),
|
||||
{ now },
|
||||
),
|
||||
summary,
|
||||
changedFiles: ["src/nested/b.tsx"],
|
||||
now,
|
||||
}).failures,
|
||||
).toContain("changed high-risk module has no owner or waiver: src/nested/b.tsx");
|
||||
expect(
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory,
|
||||
policy: parseRiskCoveragePolicy(
|
||||
policy({ highRiskPaths: ["src/a.ts", "src/nested/b.tsx"] }),
|
||||
{ now },
|
||||
),
|
||||
summary,
|
||||
changedFiles: [],
|
||||
now,
|
||||
}).failures,
|
||||
).toContain("high-risk module has no owner or waiver: src/nested/b.tsx");
|
||||
expect(
|
||||
evaluateRiskCoverage({
|
||||
repositoryRoot,
|
||||
inventory: ["src/nested/b.tsx"],
|
||||
policy: parseRiskCoveragePolicy(policy(), { now }),
|
||||
summary,
|
||||
changedFiles: [],
|
||||
now,
|
||||
}).failures,
|
||||
).toContain("critical module is outside production inventory: src/a.ts");
|
||||
});
|
||||
|
||||
it("does not let the repository policy delete a required high-risk path", async () => {
|
||||
const rawPolicy = JSON.parse(
|
||||
await readFile("config/testing/risk-coverage.json", "utf8"),
|
||||
) as Record<string, unknown>;
|
||||
rawPolicy.highRiskPaths = (
|
||||
rawPolicy.highRiskPaths as string[]
|
||||
).filter((modulePath) => modulePath !== "src/adapters/http/http-execution-v3.ts");
|
||||
|
||||
expect(() => parseRepositoryRiskCoveragePolicy(rawPolicy, { now })).toThrow(
|
||||
/required high-risk path.*http-execution-v3/u,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[{ path: "src/*.ts", owner: "team", reason: "reason", expiresAt: "2026-08-03T00:00:00.000Z" }, /POSIX/u],
|
||||
[{ path: "src/a.ts", owner: "", reason: "reason", expiresAt: "2026-08-03T00:00:00.000Z" }, /owner/u],
|
||||
[{ path: "src/a.ts", owner: "team", reason: "", expiresAt: "2026-08-03T00:00:00.000Z" }, /reason/u],
|
||||
[{ path: "src/a.ts", owner: "team", reason: "reason", expiresAt: "2026-08-01T00:00:00.000Z" }, /expired/u],
|
||||
[{ path: "src/stale.ts", owner: "team", reason: "reason", expiresAt: "2026-08-03T00:00:00.000Z" }, /stale/u],
|
||||
] as const)("rejects invalid exact-path waivers %#", (waiver, message) => {
|
||||
expect(() =>
|
||||
parseRiskCoveragePolicy(
|
||||
policy({ highRiskPaths: ["src/a.ts"], waivers: [waiver] }),
|
||||
{ now },
|
||||
),
|
||||
).toThrow(message);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user