import { spawnSync } from "node:child_process"; import { mkdir, mkdtemp, readFile, rm, writeFile, } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { createBoundedChildInvocation, createBoundedPnpmScriptInvocation, formatPnpmScriptFailure, } from "../../scripts/lib/bounded-pnpm-script.ts"; import { computeHttpScenarioCatalogDigest, httpScenarioExpectationSchema, httpScenarioReceiptSchema, type HttpScenarioAssertionGroups, type HttpScenarioExpectation, } from "../../scripts/lib/http-scenario-evidence.ts"; type MutableReceipt = { schemaVersion: number; catalogDigest: string; catalogTotal: number; executedIds: string[]; rows: Array<{ executionId: string; expected: HttpScenarioAssertionGroups; observed: HttpScenarioAssertionGroups; testDeadlineOverrideMs: number | null; }>; }; const temporaryDirectories: string[] = []; afterEach(async () => { await Promise.all( temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }), ), ); }); async function validFixture() { const source = JSON.parse( await readFile( "tests/fixtures/test-evidence/scenarios/catalog.json", "utf8", ), ) as Readonly>; const expectations = httpScenarioExpectationSchema .array() .parse(source.HTTP_SCENARIO_EXPECTATIONS) as HttpScenarioExpectation[]; const rows = expectations .map((entry) => ({ executionId: entry.executionId, expected: structuredClone(entry.expected), observed: structuredClone(entry.expected), testDeadlineOverrideMs: entry.testDeadlineOverrideMs, })) .sort((left, right) => left.executionId.localeCompare(right.executionId)); const receipt: MutableReceipt = { schemaVersion: 1, catalogDigest: computeHttpScenarioCatalogDigest(1, expectations), catalogTotal: expectations.length, executedIds: rows.map((row) => row.executionId), rows, }; return { expectations, receipt }; } async function runFixture( expectations: readonly HttpScenarioExpectation[], receipt: MutableReceipt, ) { await mkdir(".tmp", { recursive: true }); const directory = await mkdtemp(path.resolve(".tmp/http-scenario-evidence-")); temporaryDirectories.push(directory); const sourceRoot = path.join(directory, "source"); const catalogPath = path.join(directory, "catalog.json"); const receiptPath = path.join(directory, "receipt.json"); const policyPath = path.join(directory, "policy.json"); const artifactPath = path.join(directory, "report.json"); await mkdir(sourceRoot); await writeFile( catalogPath, `${JSON.stringify({ HTTP_SCENARIO_EXPECTATIONS: expectations }, null, 2)}\n`, ); await writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`); await writeFile( policyPath, `${JSON.stringify( { schemaVersion: 2, scenarioCatalogs: [ { owner: "mutation-fixture", path: catalogPath, expectationExport: "HTTP_SCENARIO_EXPECTATIONS", receiptPath, receiptSchemaVersion: 1, }, ], sourceContracts: [], }, null, 2, )}\n`, ); const invocation = createBoundedChildInvocation({ command: process.execPath, arguments: [ "scripts/check-test-evidence.ts", "--scenario-only", "--source-root", sourceRoot, "--policy", policyPath, "--catalog", catalogPath, "--receipt", receiptPath, "--artifact", artifactPath, ], environment: process.env, cwd: process.cwd(), }); return spawnSync( invocation.command, [...invocation.arguments], invocation.options, ); } describe("HTTP scenario evidence receipt schema", () => { it("bounds checker fixture children with the shared process policy", () => { const environment = { CI: "true" }; expect( createBoundedChildInvocation({ command: "/runtime/node", arguments: ["scripts/check-test-evidence.ts", "--scenario-only"], environment, cwd: "/workspace", }), ).toEqual({ command: "/runtime/node", arguments: ["scripts/check-test-evidence.ts", "--scenario-only"], options: { cwd: "/workspace", encoding: "utf8", env: environment, killSignal: "SIGTERM", maxBuffer: 16 * 1024 * 1024, timeout: 60_000, }, }); }); it("bounds every orchestration child by time and captured output", () => { const environment = { CI: "true" }; expect( createBoundedPnpmScriptInvocation({ nodePath: "/runtime/node", pnpmCli: "/runtime/pnpm.cjs", script: "test:http-scenario-catalog", environment, }), ).toEqual({ command: "/runtime/node", arguments: [ "/runtime/pnpm.cjs", "run", "test:http-scenario-catalog", ], options: { encoding: "utf8", env: environment, killSignal: "SIGTERM", maxBuffer: 16 * 1024 * 1024, timeout: 60_000, }, }); }); it("preserves timeout code and termination signal in child diagnostics", () => { const error = Object.assign(new Error("spawn timed out"), { code: "ETIMEDOUT", }); expect( formatPnpmScriptFailure("test:http-scenario-catalog", { status: null, signal: "SIGTERM", error, }), ).toBe( "test:http-scenario-catalog failed: exit=null, signal=SIGTERM, error=ETIMEDOUT: spawn timed out", ); }); it("preserves output overflow codes in child diagnostics", () => { const error = Object.assign(new Error("stdout maxBuffer exceeded"), { code: "ENOBUFS", }); expect( formatPnpmScriptFailure("check:http-scenario-evidence", { status: null, signal: null, error, }), ).toBe( "check:http-scenario-evidence failed: exit=null, signal=none, error=ENOBUFS: stdout maxBuffer exceeded", ); }); it("rejects internally inconsistent assertion groups as schema drift", async () => { const receipt = JSON.parse( await readFile( "tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json", "utf8", ), ); const result = httpScenarioReceiptSchema.safeParse(receipt); expect(result.success).toBe(false); if (!result.success) { expect(result.error.issues).toEqual( expect.arrayContaining([ expect.objectContaining({ path: ["rows", 0, "expected", "fetch", "agrees"], }), expect.objectContaining({ path: ["rows", 0, "observed", "fetch", "agrees"], }), expect.objectContaining({ path: ["rows", 0, "expected", "retry", "count"], }), expect.objectContaining({ path: ["rows", 0, "expected", "retry", "reasons", 0], }), expect.objectContaining({ path: ["rows", 0, "observed", "retry", "count"], }), expect.objectContaining({ path: ["rows", 0, "observed", "retry", "reasons", 0], }), ]), ); } }); it.each([ { label: "zero declarations and executions", diagnostic: "[TEST-EVIDENCE-CATALOG-ZERO]", mutate(expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { expectations.splice(0); receipt.catalogTotal = 0; receipt.executedIds.splice(0); receipt.rows.splice(0); }, }, { label: "a missing execution", diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-MISSING]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { receipt.executedIds.pop(); receipt.rows.pop(); }, }, { label: "an extra execution", diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-EXTRA]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { const extra = structuredClone(receipt.rows.at(-1)!); extra.executionId = "FIXTURE_OPERATION::zz-extra"; receipt.executedIds.push(extra.executionId); receipt.rows.push(extra); }, }, { label: "a duplicate execution", diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { receipt.executedIds.splice(1, 0, receipt.executedIds[0]!); receipt.rows.splice(1, 0, structuredClone(receipt.rows[0]!)); }, }, { label: "digest drift", diagnostic: "[TEST-EVIDENCE-CATALOG-DIGEST]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { receipt.catalogDigest = `sha256:${"0".repeat(64)}`; }, preserveDigest: true, }, { label: "unsorted execution rows", diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-UNSORTED]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { receipt.executedIds.reverse(); receipt.rows.reverse(); }, }, { label: "expected value drift", diagnostic: "[TEST-EVIDENCE-EXPECTED-DRIFT]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { const changed = { ...receipt.rows[0]!.expected, outcome: { kind: "FORBIDDEN", detail: null }, }; receipt.rows[0]!.expected = changed; receipt.rows[0]!.observed = structuredClone(changed); }, }, { label: "observed value mismatch", diagnostic: "[TEST-EVIDENCE-OBSERVATION-MISMATCH]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { receipt.rows[0]!.observed = { ...receipt.rows[0]!.observed, outcome: { kind: "FORBIDDEN", detail: null }, }; }, }, { label: "a deadline override drift", diagnostic: "[TEST-EVIDENCE-DEADLINE-DRIFT]", mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { receipt.rows[0]!.testDeadlineOverrideMs = 500; }, }, ])("rejects $label with its stable diagnostic", async (fixture) => { const { expectations, receipt } = await validFixture(); fixture.mutate(expectations, receipt); if (!fixture.preserveDigest) { receipt.catalogDigest = computeHttpScenarioCatalogDigest( receipt.schemaVersion, expectations, ); } const result = await runFixture(expectations, receipt); expect(result.error).toBeUndefined(); expect(result.status).toBe(1); expect(`${result.stdout}\n${result.stderr}`).toContain(fixture.diagnostic); }); });