361 lines
11 KiB
TypeScript
361 lines
11 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import {
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
rm,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
createBoundedChildInvocation,
|
|
createBoundedPnpmScriptInvocation,
|
|
formatPnpmScriptFailure,
|
|
} from "../../scripts/lib/bounded-pnpm-script.ts";
|
|
import {
|
|
computeHttpScenarioCatalogDigest,
|
|
httpScenarioExpectationSchema,
|
|
httpScenarioReceiptSchema,
|
|
type HttpScenarioAssertionGroups,
|
|
type HttpScenarioExpectation,
|
|
} from "../../scripts/lib/http-scenario-evidence.ts";
|
|
|
|
type MutableReceipt = {
|
|
schemaVersion: number;
|
|
catalogDigest: string;
|
|
catalogTotal: number;
|
|
executedIds: string[];
|
|
rows: Array<{
|
|
executionId: string;
|
|
expected: HttpScenarioAssertionGroups;
|
|
observed: HttpScenarioAssertionGroups;
|
|
testDeadlineOverrideMs: number | null;
|
|
}>;
|
|
};
|
|
|
|
const temporaryDirectories: string[] = [];
|
|
|
|
afterEach(async () => {
|
|
await Promise.all(
|
|
temporaryDirectories.splice(0).map((directory) =>
|
|
rm(directory, { recursive: true, force: true }),
|
|
),
|
|
);
|
|
});
|
|
|
|
async function validFixture() {
|
|
const source = JSON.parse(
|
|
await readFile(
|
|
"tests/fixtures/test-evidence/scenarios/catalog.json",
|
|
"utf8",
|
|
),
|
|
) as Readonly<Record<string, unknown>>;
|
|
const expectations = httpScenarioExpectationSchema
|
|
.array()
|
|
.parse(source.HTTP_SCENARIO_EXPECTATIONS) as HttpScenarioExpectation[];
|
|
const rows = expectations
|
|
.map((entry) => ({
|
|
executionId: entry.executionId,
|
|
expected: structuredClone(entry.expected),
|
|
observed: structuredClone(entry.expected),
|
|
testDeadlineOverrideMs: entry.testDeadlineOverrideMs,
|
|
}))
|
|
.sort((left, right) => left.executionId.localeCompare(right.executionId));
|
|
const receipt: MutableReceipt = {
|
|
schemaVersion: 1,
|
|
catalogDigest: computeHttpScenarioCatalogDigest(1, expectations),
|
|
catalogTotal: expectations.length,
|
|
executedIds: rows.map((row) => row.executionId),
|
|
rows,
|
|
};
|
|
return { expectations, receipt };
|
|
}
|
|
|
|
async function runFixture(
|
|
expectations: readonly HttpScenarioExpectation[],
|
|
receipt: MutableReceipt,
|
|
) {
|
|
await mkdir(".tmp", { recursive: true });
|
|
const directory = await mkdtemp(path.resolve(".tmp/http-scenario-evidence-"));
|
|
temporaryDirectories.push(directory);
|
|
const sourceRoot = path.join(directory, "source");
|
|
const catalogPath = path.join(directory, "catalog.json");
|
|
const receiptPath = path.join(directory, "receipt.json");
|
|
const policyPath = path.join(directory, "policy.json");
|
|
const artifactPath = path.join(directory, "report.json");
|
|
await mkdir(sourceRoot);
|
|
await writeFile(
|
|
catalogPath,
|
|
`${JSON.stringify({ HTTP_SCENARIO_EXPECTATIONS: expectations }, null, 2)}\n`,
|
|
);
|
|
await writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`);
|
|
await writeFile(
|
|
policyPath,
|
|
`${JSON.stringify(
|
|
{
|
|
schemaVersion: 2,
|
|
scenarioCatalogs: [
|
|
{
|
|
owner: "mutation-fixture",
|
|
path: catalogPath,
|
|
expectationExport: "HTTP_SCENARIO_EXPECTATIONS",
|
|
receiptPath,
|
|
receiptSchemaVersion: 1,
|
|
},
|
|
],
|
|
sourceContracts: [],
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
const invocation = createBoundedChildInvocation({
|
|
command: process.execPath,
|
|
arguments: [
|
|
"scripts/check-test-evidence.ts",
|
|
"--scenario-only",
|
|
"--source-root",
|
|
sourceRoot,
|
|
"--policy",
|
|
policyPath,
|
|
"--catalog",
|
|
catalogPath,
|
|
"--receipt",
|
|
receiptPath,
|
|
"--artifact",
|
|
artifactPath,
|
|
],
|
|
environment: process.env,
|
|
cwd: process.cwd(),
|
|
});
|
|
return spawnSync(
|
|
invocation.command,
|
|
[...invocation.arguments],
|
|
invocation.options,
|
|
);
|
|
}
|
|
|
|
describe("HTTP scenario evidence receipt schema", () => {
|
|
it("bounds checker fixture children with the shared process policy", () => {
|
|
const environment = { CI: "true" };
|
|
|
|
expect(
|
|
createBoundedChildInvocation({
|
|
command: "/runtime/node",
|
|
arguments: ["scripts/check-test-evidence.ts", "--scenario-only"],
|
|
environment,
|
|
cwd: "/workspace",
|
|
}),
|
|
).toEqual({
|
|
command: "/runtime/node",
|
|
arguments: ["scripts/check-test-evidence.ts", "--scenario-only"],
|
|
options: {
|
|
cwd: "/workspace",
|
|
encoding: "utf8",
|
|
env: environment,
|
|
killSignal: "SIGTERM",
|
|
maxBuffer: 16 * 1024 * 1024,
|
|
timeout: 60_000,
|
|
},
|
|
});
|
|
});
|
|
|
|
it("bounds every orchestration child by time and captured output", () => {
|
|
const environment = { CI: "true" };
|
|
|
|
expect(
|
|
createBoundedPnpmScriptInvocation({
|
|
nodePath: "/runtime/node",
|
|
pnpmCli: "/runtime/pnpm.cjs",
|
|
script: "test:http-scenario-catalog",
|
|
environment,
|
|
}),
|
|
).toEqual({
|
|
command: "/runtime/node",
|
|
arguments: [
|
|
"/runtime/pnpm.cjs",
|
|
"run",
|
|
"test:http-scenario-catalog",
|
|
],
|
|
options: {
|
|
encoding: "utf8",
|
|
env: environment,
|
|
killSignal: "SIGTERM",
|
|
maxBuffer: 16 * 1024 * 1024,
|
|
timeout: 60_000,
|
|
},
|
|
});
|
|
});
|
|
|
|
it("preserves timeout code and termination signal in child diagnostics", () => {
|
|
const error = Object.assign(new Error("spawn timed out"), {
|
|
code: "ETIMEDOUT",
|
|
});
|
|
|
|
expect(
|
|
formatPnpmScriptFailure("test:http-scenario-catalog", {
|
|
status: null,
|
|
signal: "SIGTERM",
|
|
error,
|
|
}),
|
|
).toBe(
|
|
"test:http-scenario-catalog failed: exit=null, signal=SIGTERM, error=ETIMEDOUT: spawn timed out",
|
|
);
|
|
});
|
|
|
|
it("preserves output overflow codes in child diagnostics", () => {
|
|
const error = Object.assign(new Error("stdout maxBuffer exceeded"), {
|
|
code: "ENOBUFS",
|
|
});
|
|
|
|
expect(
|
|
formatPnpmScriptFailure("check:http-scenario-evidence", {
|
|
status: null,
|
|
signal: null,
|
|
error,
|
|
}),
|
|
).toBe(
|
|
"check:http-scenario-evidence failed: exit=null, signal=none, error=ENOBUFS: stdout maxBuffer exceeded",
|
|
);
|
|
});
|
|
|
|
it("rejects internally inconsistent assertion groups as schema drift", async () => {
|
|
const receipt = JSON.parse(
|
|
await readFile(
|
|
"tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json",
|
|
"utf8",
|
|
),
|
|
);
|
|
|
|
const result = httpScenarioReceiptSchema.safeParse(receipt);
|
|
|
|
expect(result.success).toBe(false);
|
|
if (!result.success) {
|
|
expect(result.error.issues).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
path: ["rows", 0, "expected", "fetch", "agrees"],
|
|
}),
|
|
expect.objectContaining({
|
|
path: ["rows", 0, "observed", "fetch", "agrees"],
|
|
}),
|
|
expect.objectContaining({
|
|
path: ["rows", 0, "expected", "retry", "count"],
|
|
}),
|
|
expect.objectContaining({
|
|
path: ["rows", 0, "expected", "retry", "reasons", 0],
|
|
}),
|
|
expect.objectContaining({
|
|
path: ["rows", 0, "observed", "retry", "count"],
|
|
}),
|
|
expect.objectContaining({
|
|
path: ["rows", 0, "observed", "retry", "reasons", 0],
|
|
}),
|
|
]),
|
|
);
|
|
}
|
|
});
|
|
|
|
it.each([
|
|
{
|
|
label: "zero declarations and executions",
|
|
diagnostic: "[TEST-EVIDENCE-CATALOG-ZERO]",
|
|
mutate(expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
expectations.splice(0);
|
|
receipt.catalogTotal = 0;
|
|
receipt.executedIds.splice(0);
|
|
receipt.rows.splice(0);
|
|
},
|
|
},
|
|
{
|
|
label: "a missing execution",
|
|
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-MISSING]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
receipt.executedIds.pop();
|
|
receipt.rows.pop();
|
|
},
|
|
},
|
|
{
|
|
label: "an extra execution",
|
|
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-EXTRA]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
const extra = structuredClone(receipt.rows.at(-1)!);
|
|
extra.executionId = "FIXTURE_OPERATION::zz-extra";
|
|
receipt.executedIds.push(extra.executionId);
|
|
receipt.rows.push(extra);
|
|
},
|
|
},
|
|
{
|
|
label: "a duplicate execution",
|
|
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
receipt.executedIds.splice(1, 0, receipt.executedIds[0]!);
|
|
receipt.rows.splice(1, 0, structuredClone(receipt.rows[0]!));
|
|
},
|
|
},
|
|
{
|
|
label: "digest drift",
|
|
diagnostic: "[TEST-EVIDENCE-CATALOG-DIGEST]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
receipt.catalogDigest = `sha256:${"0".repeat(64)}`;
|
|
},
|
|
preserveDigest: true,
|
|
},
|
|
{
|
|
label: "unsorted execution rows",
|
|
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-UNSORTED]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
receipt.executedIds.reverse();
|
|
receipt.rows.reverse();
|
|
},
|
|
},
|
|
{
|
|
label: "expected value drift",
|
|
diagnostic: "[TEST-EVIDENCE-EXPECTED-DRIFT]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
const changed = {
|
|
...receipt.rows[0]!.expected,
|
|
outcome: { kind: "FORBIDDEN", detail: null },
|
|
};
|
|
receipt.rows[0]!.expected = changed;
|
|
receipt.rows[0]!.observed = structuredClone(changed);
|
|
},
|
|
},
|
|
{
|
|
label: "observed value mismatch",
|
|
diagnostic: "[TEST-EVIDENCE-OBSERVATION-MISMATCH]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
receipt.rows[0]!.observed = {
|
|
...receipt.rows[0]!.observed,
|
|
outcome: { kind: "FORBIDDEN", detail: null },
|
|
};
|
|
},
|
|
},
|
|
{
|
|
label: "a deadline override drift",
|
|
diagnostic: "[TEST-EVIDENCE-DEADLINE-DRIFT]",
|
|
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
|
receipt.rows[0]!.testDeadlineOverrideMs = 500;
|
|
},
|
|
},
|
|
])("rejects $label with its stable diagnostic", async (fixture) => {
|
|
const { expectations, receipt } = await validFixture();
|
|
fixture.mutate(expectations, receipt);
|
|
if (!fixture.preserveDigest) {
|
|
receipt.catalogDigest = computeHttpScenarioCatalogDigest(
|
|
receipt.schemaVersion,
|
|
expectations,
|
|
);
|
|
}
|
|
|
|
const result = await runFixture(expectations, receipt);
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.status).toBe(1);
|
|
expect(`${result.stdout}\n${result.stderr}`).toContain(fixture.diagnostic);
|
|
});
|
|
});
|