From 76bf9f1aa3862a5095a3dc285be10416cbca3751 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sun, 2 Aug 2026 10:03:11 +0900 Subject: [PATCH] test: lock V8 coverage counter semantics --- ...2026-08-02-v8-coverage-counter-contract.md | 176 +++++++++++ .../frontend-platform-testing-strategy.md | 7 +- package.json | 3 +- scripts/check-risk-coverage.ts | 2 +- .../check-v8-coverage-counter-semantics.ts | 12 + scripts/lib/v8-coverage-counter-semantics.ts | 294 ++++++++++++++++++ .../counter-semantics.fixture.ts | 19 ++ .../src/import-empty.ts | 1 + .../src/import-side-effect.ts | 1 + .../src/import-type-empty.ts | 1 + .../src/import-value.ts | 3 + .../src/reexport-named.ts | 1 + .../src/reexport-star.ts | 1 + .../src/runtime.ts | 1 + .../src/type-only.ts | 5 + tests/unit/risk-coverage.test.ts | 83 +++++ .../v8-coverage-counter-semantics.test.ts | 214 +++++++++++++ vitest.config.ts | 6 +- 18 files changed, 826 insertions(+), 4 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-02-v8-coverage-counter-contract.md create mode 100644 scripts/check-v8-coverage-counter-semantics.ts create mode 100644 scripts/lib/v8-coverage-counter-semantics.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/counter-semantics.fixture.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/import-empty.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/import-side-effect.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/import-type-empty.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/import-value.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/reexport-named.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/reexport-star.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/runtime.ts create mode 100644 tests/fixtures/v8-coverage-counter-semantics/src/type-only.ts create mode 100644 tests/unit/v8-coverage-counter-semantics.test.ts diff --git a/docs/superpowers/plans/2026-08-02-v8-coverage-counter-contract.md b/docs/superpowers/plans/2026-08-02-v8-coverage-counter-contract.md new file mode 100644 index 0000000..0ceb963 --- /dev/null +++ b/docs/superpowers/plans/2026-08-02-v8-coverage-counter-contract.md @@ -0,0 +1,176 @@ +# V8 Coverage Counter Contract Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish risk-coverage artifact schema version 3 and continuously verify the installed Vitest/V8 producer's counter-bearing/counterless row semantics in an isolated child run. + +**Architecture:** A real CLI contract test owns the serialized artifact assertion. A standalone producer checker copies fixed source templates into one OS-temp root, creates its child config and report there, validates an exact JSON summary, bounds child diagnostics, and removes the owned root in `finally`. `test:coverage` invokes the checker before repository coverage, which also carries it into CI and sample removal. + +**Tech Stack:** TypeScript 7, Node.js 24 child processes and filesystem APIs, Vitest 4, V8 coverage. + +## Global Constraints + +- Policy schema remains version 2; serialized risk-coverage artifact schema becomes version 3. +- Child root, config, and reports directory are all below one owned OS temporary directory. +- Main Vitest must not discover the child `.fixture.ts` file. +- Child exit, summary absence, malformed/missing/additional rows, counterless nonzero drift, and runtime all-zero drift fail closed. +- Child stdout/stderr included in diagnostics is bounded. +- Cleanup uses `finally` and targets only the exact owned temporary root. +- Source edits use `apply_patch`; behavior changes follow RED-GREEN TDD. + +--- + +### Task 1: Version the real serialized artifact + +**Files:** +- Modify: `tests/unit/risk-coverage.test.ts` +- Modify: `scripts/check-risk-coverage.ts` + +**Interfaces:** +- Consumes: the real `check-risk-coverage.ts` CLI, current policy structure, and an exact temporary coverage summary. +- Produces: serialized artifact schema version 3 with `counterBearingTotal`, `instrumentedCounterBearingTotal`, `counterlessTotal`, and `counterlessModules` only. + +- [x] **Step 1: Add the actual CLI serialization contract test.** + +Create a temporary repository with the 19 current policy paths, write each as `export const covered = true`, set the cloned policy baseline to 19, write one full counter row per module, run the CLI with `process.execPath`, and assert: + +```ts +expect(artifact).toMatchObject({ + schemaVersion: 3, + counterBearingTotal: 19, + instrumentedCounterBearingTotal: 19, + counterlessTotal: 0, + counterlessModules: [], +}); +expect(artifact).not.toHaveProperty("executableTotal"); +expect(artifact).not.toHaveProperty("instrumentedExecutableTotal"); +expect(artifact).not.toHaveProperty("nonExecutableTotal"); +expect(artifact).not.toHaveProperty("nonExecutableModules"); +``` + +- [x] **Step 2: Run the single test and verify RED.** + +Run: `./node_modules/.bin/vitest run tests/unit/risk-coverage.test.ts -t "publishes artifact schema version 3" --reporter=dot` + +Expected: FAIL because the actual artifact contains `schemaVersion: 2`. + +- [x] **Step 3: Change only the serialized envelope to version 3.** + +Change `schemaVersion: 2` to `schemaVersion: 3` in the value passed to `writeRiskCoverageArtifactAtomic`; do not change policy parsing. + +- [x] **Step 4: Re-run the single test and verify GREEN.** + +Run the Step 2 command and expect one passing test. + +### Task 2: Lock actual Vitest/V8 counter semantics + +**Files:** +- Create: `tests/fixtures/v8-coverage-counter-semantics/counter-semantics.fixture.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/runtime.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/import-type-empty.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/import-empty.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/import-side-effect.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/import-value.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/reexport-named.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/reexport-star.ts` +- Create: `tests/fixtures/v8-coverage-counter-semantics/src/type-only.ts` +- Create: `tests/unit/v8-coverage-counter-semantics.test.ts` +- Create: `scripts/lib/v8-coverage-counter-semantics.ts` +- Create: `scripts/check-v8-coverage-counter-semantics.ts` +- Modify: `vitest.config.ts` + +**Interfaces:** +- Produces: `assertV8CoverageCounterSemantics(summary, fixtureRoot)` and `checkV8CoverageCounterSemantics(options?)`. +- Consumes: fixed fixture templates, owned temp paths, a shell-free Vitest child result, and `coverage-summary.json`. + +- [x] **Step 1: Add fixture templates and failing checker tests.** + +The fixture test imports the seven counterless modules and observes the direct/named/star runtime values. The unit tests use literal summaries to require exact rows and mutate them for missing row, extra row, counterless nonzero, and runtime all-zero failures. Runner tests inject child exit and successful-without-summary results and require bounded diagnostics plus removal of the owned root. + +- [x] **Step 2: Run the new unit file and verify RED.** + +Run: `./node_modules/.bin/vitest run tests/unit/v8-coverage-counter-semantics.test.ts --reporter=dot` + +Expected: FAIL because `scripts/lib/v8-coverage-counter-semantics.ts` does not exist. + +- [x] **Step 3: Implement exact summary validation and owned child execution.** + +The default runner executes: + +```ts +execFile(process.execPath, [ + path.join(repositoryRoot, "node_modules/vitest/vitest.mjs"), + "run", + "--config", + configPath, + "--coverage", + "--reporter=dot", + "--no-color", +], { cwd: ownedRoot, timeout: 30_000, maxBuffer: 256 * 1024 }); +``` + +The generated config has `root`, `include`, `coverage.reportsDirectory`, and `coverage.include` paths inside the owned root. Always remove the root in `finally`. + +- [x] **Step 4: Add a behavioral main-discovery assertion.** + +Run main `vitest list` filtered to the fixture directory with `--filesOnly --passWithNoTests`; require empty stdout. Add an explicit fixture-directory exclude in `vitest.config.ts`. + +- [x] **Step 5: Run the new unit file and standalone checker for GREEN.** + +Run: + +```sh +./node_modules/.bin/vitest run tests/unit/v8-coverage-counter-semantics.test.ts --reporter=dot +node scripts/check-v8-coverage-counter-semantics.ts +``` + +Expected checker output: `V8 coverage counter semantics: PASS (1 counter-bearing, 7 counterless)`. + +### Task 3: Wire coverage/CI and refresh documentation + +**Files:** +- Modify: `package.json` +- Modify: `docs/testing/frontend-platform-testing-strategy.md` +- Modify: `.superpowers/sdd/2026-08-01-quality-architecture-remediation/task-1-report.md` +- Modify: `.superpowers/sdd/2026-08-01-quality-architecture-remediation/progress.md` + +**Interfaces:** +- Consumes: `check:v8-coverage-counter-semantics` and existing FE-GATE-005 `test:coverage` step. +- Produces: package/CI/sample-removal execution and current 19-module/80-threshold documentation. + +- [x] **Step 1: Add the package checker and prepend it to `test:coverage`.** + +```json +"check:v8-coverage-counter-semantics": "node scripts/check-v8-coverage-counter-semantics.ts", +"test:coverage": "corepack pnpm check:v8-coverage-counter-semantics && vitest run ..." +``` + +- [x] **Step 2: Synchronize documentation.** + +Replace stale `12개 high-risk module` and `52개 scoped threshold` with `19개` and `80개`; document artifact schema 3 and policy schema 2 separately. + +- [x] **Step 3: Run full relevant verification.** + +```sh +./node_modules/.bin/vitest run tests/unit/risk-coverage.test.ts tests/unit/risk-coverage-files.test.ts tests/unit/v8-coverage-counter-semantics.test.ts tests/unit/bounded-body-reader.test.ts --reporter=dot +./node_modules/.bin/tsc --noEmit -p tsconfig.node.json +./node_modules/.bin/tsc --noEmit -p tsconfig.test.json +./node_modules/.bin/eslint scripts/check-risk-coverage.ts scripts/check-v8-coverage-counter-semantics.ts scripts/lib/v8-coverage-counter-semantics.ts tests/unit/risk-coverage.test.ts tests/unit/v8-coverage-counter-semantics.test.ts vitest.config.ts --max-warnings=0 +corepack pnpm check:v8-coverage-counter-semantics +node scripts/check-risk-coverage.ts +corepack pnpm test:sample-removal +git diff --check +``` + +- [x] **Step 4: Commit the verified closeout.** + +```sh +git add package.json vitest.config.ts scripts/check-risk-coverage.ts scripts/check-v8-coverage-counter-semantics.ts scripts/lib/v8-coverage-counter-semantics.ts tests/fixtures/v8-coverage-counter-semantics tests/unit/risk-coverage.test.ts tests/unit/v8-coverage-counter-semantics.test.ts docs/testing/frontend-platform-testing-strategy.md docs/superpowers/plans/2026-08-02-v8-coverage-counter-contract.md +git commit -m "test: lock V8 coverage counter semantics" +``` + +## Self-review + +- Spec coverage: artifact versioning, actual producer rows, discovery isolation, every fail-closed path, bounded diagnostics, cleanup, coverage/CI linkage, sample-removal preservation, and documentation counts are assigned. +- Placeholder scan: no deferred implementation remains. +- Type consistency: parser and runner names match in tests, script, and plan. diff --git a/docs/testing/frontend-platform-testing-strategy.md b/docs/testing/frontend-platform-testing-strategy.md index 2319fec..6003f36 100644 --- a/docs/testing/frontend-platform-testing-strategy.md +++ b/docs/testing/frontend-platform-testing-strategy.md @@ -129,7 +129,7 @@ Chromium, Firefox, WebKit과 compact project로 실행한다. 빠른 Vite 개발 V8 text/JSON/LCOV를 생성하고 전체 기준과 retry/storage/telemetry/application composition/compatibility/performance/promotion/chunk/diagnostics/reference HTTP -operation/query-mutation/registry compatibility 12개 high-risk module에 52개 +operation/query-mutation/registry compatibility 19개 high-risk module에 80개 scoped threshold를 적용한다. critical module 누락 또는 threshold 미달 fixture는 merge gate를 실패시킨다. @@ -144,6 +144,11 @@ re-export만 있는 모듈은 모두 exact all-zero row였고, 선언/초기화 static counterless 집합과 exact all-zero row 집합의 일치를 요구하고 critical/high-risk policy module이 counterless이면 실패시킨다. +Coverage policy 입력은 schema version 2를 유지한다. 반면 위 counter-bearing 필드를 +직렬화하는 risk-coverage 결과 artifact는 schema version 3이다. 두 version은 서로 +독립적인 계약이며, 실제 CLI contract test가 artifact의 새 필드와 legacy 필드 부재를 +검증한다. + 남은 범위는 실제 device/browser farm, cloud visual approval, 외부 인증·telemetry provider와 production field data다. 이 증거가 없을 때 저장소 내부 test를 `PRODUCTION_READY`의 대체물로 사용하지 않는다. diff --git a/package.json b/package.json index ae1f067..49f1247 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,8 @@ "test:browser-file-storage-removal": "node scripts/test-browser-file-storage-runtime-removal.ts", "test:realtime-removal": "node scripts/test-realtime-runtime-removal.ts", "test:reference-feature": "vitest run tests/features/reference-feature --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/reference-feature.xml --passWithNoTests", - "test:coverage": "vitest run tests/runtime-schema tests/unit tests/component tests/integration tests/features/reference-feature --coverage --maxWorkers=4 --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/coverage.xml && node scripts/check-risk-coverage.ts", + "check:v8-coverage-counter-semantics": "node scripts/check-v8-coverage-counter-semantics.ts", + "test:coverage": "corepack pnpm check:v8-coverage-counter-semantics && vitest run tests/runtime-schema tests/unit tests/component tests/integration tests/features/reference-feature --coverage --maxWorkers=4 --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/coverage.xml && node scripts/check-risk-coverage.ts", "check:coverage:fixture": "node scripts/check-risk-coverage.ts --summary tests/fixtures/coverage/below-threshold.json --artifact artifacts/quality/risk-coverage-fixture.json", "test:all": "corepack pnpm test:runtime-schema && corepack pnpm test:unit && corepack pnpm test:component && corepack pnpm test:integration && corepack pnpm test:reference-feature && corepack pnpm test:recipes", "verify:lockfile": "corepack pnpm install --frozen-lockfile", diff --git a/scripts/check-risk-coverage.ts b/scripts/check-risk-coverage.ts index 6233641..2575f18 100644 --- a/scripts/check-risk-coverage.ts +++ b/scripts/check-risk-coverage.ts @@ -62,7 +62,7 @@ await writeRiskCoverageArtifactAtomic({ relativePath: artifactPath, inputPaths: [policyInput.relativePath, summaryInput.relativePath], value: { - schemaVersion: 2, + schemaVersion: 3, policy: policyInput.relativePath, summary: summaryInput.relativePath, ...result, diff --git a/scripts/check-v8-coverage-counter-semantics.ts b/scripts/check-v8-coverage-counter-semantics.ts new file mode 100644 index 0000000..cf449d0 --- /dev/null +++ b/scripts/check-v8-coverage-counter-semantics.ts @@ -0,0 +1,12 @@ +import { checkV8CoverageCounterSemantics } from "./lib/v8-coverage-counter-semantics.ts"; + +try { + const result = await checkV8CoverageCounterSemantics(); + process.stdout.write( + `V8 coverage counter semantics: PASS (${result.counterBearingModules.length} counter-bearing, ${result.counterlessModules.length} counterless)\n`, + ); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`V8 coverage counter semantics failed: ${message}\n`); + process.exitCode = 1; +} diff --git a/scripts/lib/v8-coverage-counter-semantics.ts b/scripts/lib/v8-coverage-counter-semantics.ts new file mode 100644 index 0000000..1859f13 --- /dev/null +++ b/scripts/lib/v8-coverage-counter-semantics.ts @@ -0,0 +1,294 @@ +import { execFile } from "node:child_process"; +import { + cp, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const coverageMetrics = [ + "lines", + "statements", + "functions", + "branches", +] as const; +const runtimeModule = "src/runtime.ts"; +const counterlessModules = Object.freeze([ + "src/import-empty.ts", + "src/import-side-effect.ts", + "src/import-type-empty.ts", + "src/import-value.ts", + "src/reexport-named.ts", + "src/reexport-star.ts", + "src/type-only.ts", +] as const); +const expectedModules = Object.freeze( + [runtimeModule, ...counterlessModules].sort(), +); +const childOutputLimit = 1_800; + +type ChildOutput = Readonly<{ stdout: string; stderr: string }>; +type ChildInput = Readonly<{ + repositoryRoot: string; + ownedRoot: string; + configPath: string; +}>; +type CheckerOptions = Readonly<{ + repositoryRoot?: string; + createOwnedRoot?: () => Promise; + runVitest?: (input: ChildInput) => Promise; +}>; + +export type V8CoverageCounterSemantics = Readonly<{ + counterBearingModules: readonly string[]; + counterlessModules: readonly string[]; +}>; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function hasErrorCode(error: unknown, code: string): boolean { + return isRecord(error) && error.code === code; +} + +function normalizeSummaryPath(value: string, fixtureRoot: string): string { + const relative = path.isAbsolute(value) ? path.relative(fixtureRoot, value) : value; + const normalized = relative.split(path.sep).join("/"); + if ( + normalized.length === 0 || + normalized === ".." || + normalized.startsWith("../") || + path.posix.isAbsolute(normalized) + ) { + throw new TypeError(`V8 coverage row is outside the owned fixture: ${value}`); + } + return normalized; +} + +function coverageCounter( + value: unknown, + label: string, +): Readonly<{ total: number; covered: number; skipped: number; pct: number }> { + if (!isRecord(value)) throw new TypeError(`${label} must be an object`); + const { total, covered, skipped, pct } = value; + if ( + typeof total !== "number" || + !Number.isSafeInteger(total) || + typeof covered !== "number" || + !Number.isSafeInteger(covered) || + typeof skipped !== "number" || + !Number.isSafeInteger(skipped) || + typeof pct !== "number" || + !Number.isFinite(pct) || + total < 0 || + covered < 0 || + skipped < 0 || + covered + skipped > total + ) { + throw new TypeError(`${label} contains invalid coverage counters`); + } + const expectedPercentage = + total === 0 ? 100 : Math.floor((covered / total) * 10_000) / 100; + if (pct !== expectedPercentage) { + throw new TypeError(`${label} contains invalid coverage counters`); + } + return { total, covered, skipped, pct }; +} + +function coverageRow( + value: unknown, + label: string, +): Readonly>> { + if (!isRecord(value)) throw new TypeError(`${label} must be an object`); + return Object.fromEntries( + coverageMetrics.map((metric) => [ + metric, + coverageCounter(value[metric], `${label}.${metric}`), + ]), + ) as Readonly< + Record<(typeof coverageMetrics)[number], ReturnType> + >; +} + +export function assertV8CoverageCounterSemantics( + value: unknown, + fixtureRoot: string, +): V8CoverageCounterSemantics { + if (!isRecord(value)) throw new TypeError("V8 coverage summary must be an object"); + coverageRow(value.total, "V8 coverage total"); + const rows = new Map>(); + for (const [producerPath, producerRow] of Object.entries(value)) { + if (producerPath === "total") continue; + const normalizedPath = normalizeSummaryPath(producerPath, fixtureRoot); + if (rows.has(normalizedPath)) { + throw new TypeError(`V8 coverage row is duplicated: ${normalizedPath}`); + } + rows.set( + normalizedPath, + coverageRow(producerRow, `V8 coverage row ${normalizedPath}`), + ); + } + const actualModules = [...rows.keys()].sort(); + const missing = expectedModules.filter((modulePath) => !rows.has(modulePath)); + const additional = actualModules.filter( + (modulePath) => !expectedModules.includes(modulePath), + ); + if (missing.length > 0 || additional.length > 0) { + throw new Error( + `V8 coverage row set mismatch; missing: ${missing.join(", ") || "none"}; additional: ${additional.join(", ") || "none"}`, + ); + } + const runtimeRow = rows.get(runtimeModule)!; + if (!coverageMetrics.some((metric) => runtimeRow[metric].total > 0)) { + throw new Error(`V8 runtime module is not counter-bearing: ${runtimeModule}`); + } + for (const modulePath of counterlessModules) { + const row = rows.get(modulePath)!; + const exactAllZero = coverageMetrics.every((metric) => { + const counter = row[metric]; + return ( + counter.total === 0 && + counter.covered === 0 && + counter.skipped === 0 && + counter.pct === 100 + ); + }); + if (!exactAllZero) { + throw new Error( + `V8 counterless module must have exact all-zero counters: ${modulePath}`, + ); + } + } + return Object.freeze({ + counterBearingModules: Object.freeze([runtimeModule]), + counterlessModules: Object.freeze([...counterlessModules]), + }); +} + +function boundedChildOutput(value: unknown): string { + const output = + typeof value === "string" + ? value + : value instanceof Uint8Array + ? new TextDecoder().decode(value) + : ""; + if (output.length <= childOutputLimit) return output; + return `${output.slice(0, childOutputLimit)}\n[truncated ${output.length - childOutputLimit} characters]`; +} + +async function defaultRunVitest(input: ChildInput): Promise { + const result = await execFileAsync( + process.execPath, + [ + path.join(input.repositoryRoot, "node_modules/vitest/vitest.mjs"), + "run", + "--config", + input.configPath, + "--coverage", + "--reporter=dot", + "--no-color", + ], + { + cwd: input.ownedRoot, + encoding: "utf8", + timeout: 30_000, + maxBuffer: 256 * 1024, + }, + ); + return { stdout: result.stdout, stderr: result.stderr }; +} + +function assertOwnedTemporaryRoot(value: string): string { + const temporaryRoot = path.resolve(tmpdir()); + const ownedRoot = path.resolve(value); + const relative = path.relative(temporaryRoot, ownedRoot); + if ( + relative.length === 0 || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new TypeError(`V8 coverage fixture root is not an owned temp path: ${ownedRoot}`); + } + return ownedRoot; +} + +export async function checkV8CoverageCounterSemantics( + options: CheckerOptions = {}, +): Promise { + const repositoryRoot = path.resolve(options.repositoryRoot ?? process.cwd()); + const createOwnedRoot = + options.createOwnedRoot ?? + (() => mkdtemp(path.join(tmpdir(), "v8-counter-semantics-"))); + const runVitest = options.runVitest ?? defaultRunVitest; + const ownedRoot = assertOwnedTemporaryRoot(await createOwnedRoot()); + const fixtureSource = path.join( + repositoryRoot, + "tests/fixtures/v8-coverage-counter-semantics", + ); + const configPath = path.join(ownedRoot, "vitest.config.mjs"); + const reportsDirectory = path.join(ownedRoot, "coverage"); + try { + await cp(fixtureSource, ownedRoot, { recursive: true }); + await writeFile( + configPath, + `export default ${JSON.stringify( + { + root: ownedRoot, + test: { + globals: true, + include: ["counter-semantics.fixture.ts"], + setupFiles: [], + coverage: { + provider: "v8", + reportsDirectory, + reporter: ["json-summary"], + include: ["src/**/*.ts"], + }, + }, + }, + null, + 2, + )};\n`, + "utf8", + ); + try { + await runVitest({ repositoryRoot, ownedRoot, configPath }); + } catch (error) { + const record = isRecord(error) ? error : {}; + throw new Error( + `V8 coverage counter semantics child Vitest failed\nstdout:\n${boundedChildOutput(record.stdout)}\nstderr:\n${boundedChildOutput(record.stderr)}`, + { cause: error }, + ); + } + const summaryPath = path.join(reportsDirectory, "coverage-summary.json"); + let summaryText: string; + try { + summaryText = await readFile(summaryPath, "utf8"); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) { + throw new Error("V8 coverage counter semantics coverage summary is missing", { + cause: error, + }); + } + throw error; + } + let summary: unknown; + try { + summary = JSON.parse(summaryText) as unknown; + } catch (error) { + throw new TypeError("V8 coverage counter semantics summary is invalid JSON", { + cause: error, + }); + } + return assertV8CoverageCounterSemantics(summary, ownedRoot); + } finally { + await rm(ownedRoot, { recursive: true, force: true }); + } +} diff --git a/tests/fixtures/v8-coverage-counter-semantics/counter-semantics.fixture.ts b/tests/fixtures/v8-coverage-counter-semantics/counter-semantics.fixture.ts new file mode 100644 index 0000000..9b572dc --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/counter-semantics.fixture.ts @@ -0,0 +1,19 @@ +/// + +import "./src/import-empty.ts"; +import "./src/import-side-effect.ts"; +import "./src/import-type-empty.ts"; +import { importedRuntimeValue } from "./src/import-value.ts"; +import { runtimeValue as namedRuntimeValue } from "./src/reexport-named.ts"; +import { runtimeValue } from "./src/runtime.ts"; +import { runtimeValue as starRuntimeValue } from "./src/reexport-star.ts"; +import type {} from "./src/type-only.ts"; + +test("loads every counter semantics module", () => { + expect([ + runtimeValue, + importedRuntimeValue, + namedRuntimeValue, + starRuntimeValue, + ]).toEqual([1, 1, 1, 1]); +}); diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/import-empty.ts b/tests/fixtures/v8-coverage-counter-semantics/src/import-empty.ts new file mode 100644 index 0000000..4324d9f --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/import-empty.ts @@ -0,0 +1 @@ +import {} from "./runtime.ts"; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/import-side-effect.ts b/tests/fixtures/v8-coverage-counter-semantics/src/import-side-effect.ts new file mode 100644 index 0000000..1519184 --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/import-side-effect.ts @@ -0,0 +1 @@ +import "./runtime.ts"; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/import-type-empty.ts b/tests/fixtures/v8-coverage-counter-semantics/src/import-type-empty.ts new file mode 100644 index 0000000..0256720 --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/import-type-empty.ts @@ -0,0 +1 @@ +import type {} from "./runtime.ts"; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/import-value.ts b/tests/fixtures/v8-coverage-counter-semantics/src/import-value.ts new file mode 100644 index 0000000..df5b0b6 --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/import-value.ts @@ -0,0 +1,3 @@ +import { runtimeValue } from "./runtime.ts"; + +export { runtimeValue as importedRuntimeValue }; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/reexport-named.ts b/tests/fixtures/v8-coverage-counter-semantics/src/reexport-named.ts new file mode 100644 index 0000000..9c3a075 --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/reexport-named.ts @@ -0,0 +1 @@ +export { runtimeValue } from "./runtime.ts"; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/reexport-star.ts b/tests/fixtures/v8-coverage-counter-semantics/src/reexport-star.ts new file mode 100644 index 0000000..48277aa --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/reexport-star.ts @@ -0,0 +1 @@ +export * from "./runtime.ts"; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/runtime.ts b/tests/fixtures/v8-coverage-counter-semantics/src/runtime.ts new file mode 100644 index 0000000..b600af5 --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/runtime.ts @@ -0,0 +1 @@ +export const runtimeValue = 1; diff --git a/tests/fixtures/v8-coverage-counter-semantics/src/type-only.ts b/tests/fixtures/v8-coverage-counter-semantics/src/type-only.ts new file mode 100644 index 0000000..740dc83 --- /dev/null +++ b/tests/fixtures/v8-coverage-counter-semantics/src/type-only.ts @@ -0,0 +1,5 @@ +export interface CounterFixtureShape { + readonly value: number; +} + +export type CounterFixtureValue = CounterFixtureShape["value"]; diff --git a/tests/unit/risk-coverage.test.ts b/tests/unit/risk-coverage.test.ts index 3bc567d..4137915 100644 --- a/tests/unit/risk-coverage.test.ts +++ b/tests/unit/risk-coverage.test.ts @@ -1,3 +1,4 @@ +import { execFile } from "node:child_process"; import { constants } from "node:fs"; import { mkdir, @@ -10,6 +11,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; @@ -24,6 +26,7 @@ import { const roots: string[] = []; const now = Date.parse("2026-08-02T00:00:00.000Z"); +const execFileAsync = promisify(execFile); function counter(total = 1, covered = total, skipped = 0) { return { @@ -174,6 +177,86 @@ describe("repository-aware risk coverage", () => { }); }); + it("publishes artifact schema version 3 with exact counter-bearing fields", async () => { + const repositoryRoot = await mkdtemp(path.join(tmpdir(), "risk-coverage-cli-")); + roots.push(repositoryRoot); + const rawPolicy = JSON.parse( + await readFile("config/testing/risk-coverage.json", "utf8"), + ) as Record; + const modulePaths = rawPolicy.highRiskPaths as string[]; + const cliPolicy = { + ...rawPolicy, + repositoryBaseline: modulePaths.length, + }; + await Promise.all([ + mkdir(path.join(repositoryRoot, "config/testing"), { recursive: true }), + mkdir(path.join(repositoryRoot, "artifacts/tests/coverage"), { + recursive: true, + }), + ...modulePaths.map(async (modulePath) => { + const absolutePath = path.join(repositoryRoot, modulePath); + await mkdir(path.dirname(absolutePath), { recursive: true }); + await writeFile(absolutePath, "export const covered = true;\n"); + }), + ]); + const summary = Object.fromEntries([ + ["total", metrics(modulePaths.length)], + ...modulePaths.map((modulePath) => [modulePath, fullMetrics]), + ]); + await Promise.all([ + writeFile( + path.join(repositoryRoot, "config/testing/policy.json"), + `${JSON.stringify(cliPolicy)}\n`, + ), + writeFile( + path.join(repositoryRoot, "artifacts/tests/coverage/summary.json"), + `${JSON.stringify(summary)}\n`, + ), + ]); + + await execFileAsync( + process.execPath, + [ + "scripts/check-risk-coverage.ts", + "--repository-root", + repositoryRoot, + "--policy", + "config/testing/policy.json", + "--summary", + "artifacts/tests/coverage/summary.json", + "--artifact", + "artifacts/quality/risk-coverage.json", + ], + { + cwd: process.cwd(), + encoding: "utf8", + timeout: 30_000, + maxBuffer: 256 * 1024, + }, + ); + const artifact = JSON.parse( + await readFile( + path.join(repositoryRoot, "artifacts/quality/risk-coverage.json"), + "utf8", + ), + ) as Record; + + expect(rawPolicy["schemaVersion"]).toBe(2); + expect(artifact).toMatchObject({ + schemaVersion: 3, + policy: "config/testing/policy.json", + summary: "artifacts/tests/coverage/summary.json", + counterBearingTotal: modulePaths.length, + instrumentedCounterBearingTotal: modulePaths.length, + counterlessTotal: 0, + counterlessModules: [], + }); + expect(artifact).not.toHaveProperty("executableTotal"); + expect(artifact).not.toHaveProperty("instrumentedExecutableTotal"); + expect(artifact).not.toHaveProperty("nonExecutableTotal"); + expect(artifact).not.toHaveProperty("nonExecutableModules"); + }); + it("rejects arbitrary coverage paths instead of silently allowing inflation", () => { const parsedPolicy = parseRiskCoveragePolicy( policy({ repositoryBaseline: 1, generatedPaths: [] }), diff --git a/tests/unit/v8-coverage-counter-semantics.test.ts b/tests/unit/v8-coverage-counter-semantics.test.ts new file mode 100644 index 0000000..5d61754 --- /dev/null +++ b/tests/unit/v8-coverage-counter-semantics.test.ts @@ -0,0 +1,214 @@ +import { execFile } from "node:child_process"; +import { + access, + mkdtemp, + readFile, + rm, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + assertV8CoverageCounterSemantics, + checkV8CoverageCounterSemantics, +} from "../../scripts/lib/v8-coverage-counter-semantics.ts"; + +const roots: string[] = []; +const execFileAsync = promisify(execFile); +const zeroCounter = Object.freeze({ + total: 0, + covered: 0, + skipped: 0, + pct: 100, +}); +const zeroMetrics = Object.freeze({ + lines: zeroCounter, + statements: zeroCounter, + functions: zeroCounter, + branches: zeroCounter, +}); +const counterlessPaths = Object.freeze([ + "src/import-empty.ts", + "src/import-side-effect.ts", + "src/import-type-empty.ts", + "src/import-value.ts", + "src/reexport-named.ts", + "src/reexport-star.ts", + "src/type-only.ts", +]); + +function fixtureSummary(root: string): Record { + return Object.fromEntries([ + [ + "total", + { + lines: { total: 1, covered: 1, skipped: 0, pct: 100 }, + statements: { total: 1, covered: 1, skipped: 0, pct: 100 }, + functions: zeroCounter, + branches: zeroCounter, + }, + ], + [ + path.join(root, "src/runtime.ts"), + { + lines: { total: 1, covered: 1, skipped: 0, pct: 100 }, + statements: { total: 1, covered: 1, skipped: 0, pct: 100 }, + functions: zeroCounter, + branches: zeroCounter, + }, + ], + ...counterlessPaths.map((modulePath) => [path.join(root, modulePath), zeroMetrics]), + ]); +} + +async function captureFailure(operation: Promise): Promise { + try { + await operation; + } catch (error) { + if (error instanceof Error) return error; + throw error; + } + throw new Error("expected operation to fail"); +} + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +describe("V8 coverage counter semantics", () => { + it("accepts one counter-bearing row and seven exact counterless rows", () => { + const root = "/owned-fixture"; + + expect(assertV8CoverageCounterSemantics(fixtureSummary(root), root)).toEqual({ + counterBearingModules: ["src/runtime.ts"], + counterlessModules: counterlessPaths, + }); + }); + + it("rejects a missing or additional producer row", () => { + const root = "/owned-fixture"; + const missing = fixtureSummary(root); + delete missing[path.join(root, "src/import-empty.ts")]; + expect(() => assertV8CoverageCounterSemantics(missing, root)).toThrow( + /row set.*missing.*import-empty/u, + ); + + const additional = fixtureSummary(root); + additional[path.join(root, "src/unexpected.ts")] = zeroMetrics; + expect(() => assertV8CoverageCounterSemantics(additional, root)).toThrow( + /row set.*additional.*unexpected/u, + ); + }); + + it("rejects counterless nonzero and runtime all-zero drift", () => { + const root = "/owned-fixture"; + const counterlessDrift = fixtureSummary(root); + counterlessDrift[path.join(root, "src/import-value.ts")] = { + ...zeroMetrics, + lines: { total: 1, covered: 1, skipped: 0, pct: 100 }, + }; + expect(() => + assertV8CoverageCounterSemantics(counterlessDrift, root), + ).toThrow(/counterless.*exact all-zero.*import-value/u); + + const runtimeDrift = fixtureSummary(root); + runtimeDrift[path.join(root, "src/runtime.ts")] = zeroMetrics; + expect(() => assertV8CoverageCounterSemantics(runtimeDrift, root)).toThrow( + /runtime.*counter-bearing/u, + ); + }); + + it("rejects producer percentages that do not exactly match their counts", () => { + const root = "/owned-fixture"; + const totalDrift = fixtureSummary(root); + totalDrift["total"] = { + ...totalDrift["total"] as Record, + lines: { total: 1, covered: 1, skipped: 0, pct: -500 }, + }; + expect(() => assertV8CoverageCounterSemantics(totalDrift, root)).toThrow( + /total\.lines.*invalid coverage counters/u, + ); + + const runtimeDrift = fixtureSummary(root); + runtimeDrift[path.join(root, "src/runtime.ts")] = { + ...runtimeDrift[path.join(root, "src/runtime.ts")] as Record< + string, + unknown + >, + lines: { total: 1, covered: 1, skipped: 0, pct: 99.99 }, + }; + expect(() => assertV8CoverageCounterSemantics(runtimeDrift, root)).toThrow( + /runtime.*lines.*invalid coverage counters/u, + ); + }); + + it("bounds child diagnostics and cleans its owned root on child exit", async () => { + const ownedRoot = await mkdtemp(path.join(tmpdir(), "v8-counter-exit-")); + roots.push(ownedRoot); + const childFailure = Object.assign(new Error("child exit 1"), { + stdout: "o".repeat(20_000), + stderr: "e".repeat(20_000), + }); + + const failure = await captureFailure( + checkV8CoverageCounterSemantics({ + repositoryRoot: process.cwd(), + createOwnedRoot: async () => ownedRoot, + runVitest: async () => Promise.reject(childFailure), + }), + ); + + expect(failure.message).toMatch(/child Vitest failed/u); + expect(failure.message).toMatch(/truncated/u); + expect(failure.message.length).toBeLessThan(5_000); + await expect(access(ownedRoot)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("fails closed on a missing summary and cleans its owned root", async () => { + const ownedRoot = await mkdtemp(path.join(tmpdir(), "v8-counter-summary-")); + roots.push(ownedRoot); + + await expect( + checkV8CoverageCounterSemantics({ + repositoryRoot: process.cwd(), + createOwnedRoot: async () => ownedRoot, + runVitest: async () => ({ stdout: "", stderr: "" }), + }), + ).rejects.toThrow(/coverage summary is missing/u); + await expect(access(ownedRoot)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("keeps the child fixture outside main Vitest discovery", async () => { + const outputRoot = await mkdtemp(path.join(tmpdir(), "v8-counter-list-")); + roots.push(outputRoot); + const outputPath = path.join(outputRoot, "listed.json"); + await execFileAsync( + process.execPath, + [ + path.join(process.cwd(), "node_modules/vitest/vitest.mjs"), + "list", + "tests/fixtures/v8-coverage-counter-semantics", + "--config", + path.join(process.cwd(), "vitest.config.ts"), + "--filesOnly", + "--passWithNoTests", + "--json", + outputPath, + "--no-color", + ], + { + cwd: process.cwd(), + encoding: "utf8", + timeout: 30_000, + maxBuffer: 256 * 1024, + }, + ); + + expect(JSON.parse(await readFile(outputPath, "utf8"))).toEqual([]); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 3f4e639..406932a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,7 +8,11 @@ export default defineConfig({ clearMocks: true, mockReset: true, testTimeout: 10_000, - exclude: [...configDefaults.exclude, ".tmp/**"], + exclude: [ + ...configDefaults.exclude, + ".tmp/**", + "tests/fixtures/v8-coverage-counter-semantics/**", + ], coverage: { provider: "v8", reportsDirectory: "artifacts/tests/coverage",