diff --git a/scripts/lib/ci-artifact-validator.ts b/scripts/lib/ci-artifact-validator.ts index 2c309466..c39cd78e 100644 --- a/scripts/lib/ci-artifact-validator.ts +++ b/scripts/lib/ci-artifact-validator.ts @@ -9,6 +9,10 @@ import type { CiGateArtifactSchema, } from "../contracts/ci-gates.ts"; import { ciContractReportSchema } from "./ci-contract-report.ts"; +import { + COVERAGE_SUMMARY_METRICS, + COVERAGE_SUMMARY_OPTIONAL_METRICS, +} from "./coverage-summary-metrics.ts"; import { architectureDependencyReportArtifactSchema, automatedA11yArtifactSchema, @@ -76,16 +80,17 @@ const coverageCounterSchema = z context.addIssue({ code: "custom", path: ["pct"], message: "coverage pct is not exact" }); } }); +// 1103/12-09. The key list is shared with the risk coverage gate, which reads +// the same file; this schema used to refuse the `branchesTrue` the reporter +// always writes. const coverageSummarySchema = z .record( z.string(), z - .object({ - lines: coverageCounterSchema, - statements: coverageCounterSchema, - functions: coverageCounterSchema, - branches: coverageCounterSchema, - }) + .object(Object.fromEntries([ + ...COVERAGE_SUMMARY_METRICS.map((metric) => [metric, coverageCounterSchema]), + ...COVERAGE_SUMMARY_OPTIONAL_METRICS.map((metric) => [metric, coverageCounterSchema.optional()]), + ])) .strict(), ) .refine((value) => "total" in value, "coverage summary lacks total"); diff --git a/scripts/lib/coverage-summary-metrics.ts b/scripts/lib/coverage-summary-metrics.ts new file mode 100644 index 00000000..9579bb33 --- /dev/null +++ b/scripts/lib/coverage-summary-metrics.ts @@ -0,0 +1,24 @@ +/** + * The keys of one entry in Vitest's `coverage-summary.json`. Two readers check + * that file: the CI evidence validator (ci-artifact-validator.ts) and the risk + * coverage gate (risk-coverage.ts). They kept separate lists, and the + * validator refused `branchesTrue` while the gate accepted it, so FE-GATE-005 + * failed at its evidence step after every test had passed (1103/12-09). + */ + +/** The four metrics both readers evaluate. */ +export const COVERAGE_SUMMARY_METRICS = Object.freeze([ + "lines", + "statements", + "functions", + "branches", +] as const); + +/** + * Keys the reporter may add. istanbul-lib-coverage 3.2.2 puts `branchesTrue` + * in every blank summary, so the total always carries it. Each is still + * checked as a counter; neither reader evaluates it. + */ +export const COVERAGE_SUMMARY_OPTIONAL_METRICS = Object.freeze([ + "branchesTrue", +] as const); diff --git a/scripts/lib/risk-coverage.ts b/scripts/lib/risk-coverage.ts index 0d951f22..9a12908a 100644 --- a/scripts/lib/risk-coverage.ts +++ b/scripts/lib/risk-coverage.ts @@ -8,14 +8,12 @@ import { import path from "node:path"; import babelParser from "@babel/eslint-parser"; +import { + COVERAGE_SUMMARY_METRICS as coverageMetrics, + COVERAGE_SUMMARY_OPTIONAL_METRICS, +} from "./coverage-summary-metrics.ts"; import { normalizeRepositoryRelativePath } from "./repository-file-inventory.ts"; -const coverageMetrics = [ - "lines", - "statements", - "functions", - "branches", -] as const; const teamIdPattern = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u; const maximumWaiverDurationMs = 90 * 24 * 60 * 60 * 1_000; @@ -641,16 +639,20 @@ function parseCoverageCounter(value: unknown, label: string): CoverageCounter { function parseCoverageMetrics(value: unknown, label: string): CoverageMetrics { if (!isRecord(value)) throw new TypeError(`${label} must be an object`); + const optionalMetrics: readonly string[] = COVERAGE_SUMMARY_OPTIONAL_METRICS; const unknownMetrics = Object.keys(value).filter( - (metric) => metric !== "branchesTrue" && !coverageMetrics.includes(metric as CoverageMetric), + (metric) => + !optionalMetrics.includes(metric) && !coverageMetrics.includes(metric as CoverageMetric), ); if (unknownMetrics.length > 0) { throw new TypeError( `${label} has unknown coverage metric keys: ${unknownMetrics.sort().join(", ")}`, ); } - if (value.branchesTrue !== undefined) { - parseCoverageCounter(value.branchesTrue, `${label}.branchesTrue`); + for (const metric of optionalMetrics) { + if (value[metric] !== undefined) { + parseCoverageCounter(value[metric], `${label}.${metric}`); + } } const parsed = {} as Record; for (const metric of coverageMetrics) { diff --git a/tests/unit/coverage-summary-artifact.test.ts b/tests/unit/coverage-summary-artifact.test.ts new file mode 100644 index 00000000..960c3c5f --- /dev/null +++ b/tests/unit/coverage-summary-artifact.test.ts @@ -0,0 +1,66 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { loadCiGateContract } from "../../scripts/contracts/ci-gates.ts"; +import { validateCiArtifact } from "../../scripts/lib/ci-artifact-validator.ts"; + +const SUMMARY_PATH = "artifacts/tests/coverage/coverage-summary.json"; +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +const counter = (total: number, covered = total) => ({ + total, + covered, + skipped: 0, + pct: total === 0 ? 100 : Math.floor((covered / total) * 10_000) / 100, +}); +const metrics = { + lines: counter(4), + statements: counter(4), + functions: counter(1), + branches: counter(2), +}; + +async function validateSummary(summary: unknown): Promise { + const contract = await loadCiGateContract(process.cwd()); + const artifact = contract.artifacts.find((entry) => entry.path === SUMMARY_PATH); + const schema = contract.artifactSchemas.find((entry) => entry.id === artifact?.schemaId); + if (!artifact || !schema) throw new Error(`the CI contract no longer declares ${SUMMARY_PATH}`); + const root = await mkdtemp(path.join(tmpdir(), "coverage-summary-artifact-")); + roots.push(root); + await mkdir(path.join(root, path.dirname(SUMMARY_PATH)), { recursive: true }); + await writeFile(path.join(root, SUMMARY_PATH), `${JSON.stringify(summary)}\n`); + await validateCiArtifact({ root, artifact, schema }); +} + +// 1103/12-09. istanbul-lib-coverage 3.2.2 puts `branchesTrue` in every blank +// summary, so the json-summary total always carries it. The CI evidence check +// refused the key that the risk-coverage gate reading the same file accepts, +// and FE-GATE-005 failed after every test had passed. +describe("coverage summary evidence (1103/12-09)", () => { + it("accepts the summary the coverage reporter writes, with branchesTrue", async () => { + await expect(validateSummary({ + total: { ...metrics, branchesTrue: counter(0) }, + "src/a.ts": metrics, + })).resolves.toBeUndefined(); + }); + + it("accepts a summary without branchesTrue", async () => { + await expect(validateSummary({ total: metrics, "src/a.ts": metrics })) + .resolves.toBeUndefined(); + }); + + it("still validates branchesTrue as a counter and refuses other keys", async () => { + await expect(validateSummary({ + total: { ...metrics, branchesTrue: { ...counter(0), pct: 0 } }, + })).rejects.toThrow(/pct/u); + await expect(validateSummary({ total: { ...metrics, branchesFalse: counter(0) } })) + .rejects.toThrow(/branchesFalse/u); + }); +});