test: execute the HTTP scenario catalog
This commit is contained in:
DongHyeonka
@@ -0,0 +1,294 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createBoundedPnpmScriptInvocation,
|
||||
formatPnpmScriptFailure,
|
||||
} from "../../scripts/lib/bounded-pnpm-script.ts";
|
||||
import {
|
||||
computeHttpScenarioCatalogDigest,
|
||||
httpScenarioExpectationSchema,
|
||||
httpScenarioReceiptSchema,
|
||||
type HttpScenarioAssertionGroups,
|
||||
type HttpScenarioExpectation,
|
||||
} from "../../scripts/lib/http-scenario-evidence.ts";
|
||||
|
||||
type MutableReceipt = {
|
||||
schemaVersion: number;
|
||||
catalogDigest: string;
|
||||
catalogTotal: number;
|
||||
executedIds: string[];
|
||||
rows: Array<{
|
||||
executionId: string;
|
||||
expected: HttpScenarioAssertionGroups;
|
||||
observed: HttpScenarioAssertionGroups;
|
||||
testDeadlineOverrideMs: number | null;
|
||||
}>;
|
||||
};
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories.splice(0).map((directory) =>
|
||||
rm(directory, { recursive: true, force: true }),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
async function validFixture() {
|
||||
const source = JSON.parse(
|
||||
await readFile(
|
||||
"tests/fixtures/test-evidence/scenarios/catalog.json",
|
||||
"utf8",
|
||||
),
|
||||
) as Readonly<Record<string, unknown>>;
|
||||
const expectations = httpScenarioExpectationSchema
|
||||
.array()
|
||||
.parse(source.HTTP_SCENARIO_EXPECTATIONS) as HttpScenarioExpectation[];
|
||||
const rows = expectations
|
||||
.map((entry) => ({
|
||||
executionId: entry.executionId,
|
||||
expected: structuredClone(entry.expected),
|
||||
observed: structuredClone(entry.expected),
|
||||
testDeadlineOverrideMs: entry.testDeadlineOverrideMs,
|
||||
}))
|
||||
.sort((left, right) => left.executionId.localeCompare(right.executionId));
|
||||
const receipt: MutableReceipt = {
|
||||
schemaVersion: 1,
|
||||
catalogDigest: computeHttpScenarioCatalogDigest(1, expectations),
|
||||
catalogTotal: expectations.length,
|
||||
executedIds: rows.map((row) => row.executionId),
|
||||
rows,
|
||||
};
|
||||
return { expectations, receipt };
|
||||
}
|
||||
|
||||
async function runFixture(
|
||||
expectations: readonly HttpScenarioExpectation[],
|
||||
receipt: MutableReceipt,
|
||||
) {
|
||||
await mkdir(".tmp", { recursive: true });
|
||||
const directory = await mkdtemp(path.resolve(".tmp/http-scenario-evidence-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const sourceRoot = path.join(directory, "source");
|
||||
const catalogPath = path.join(directory, "catalog.json");
|
||||
const receiptPath = path.join(directory, "receipt.json");
|
||||
const policyPath = path.join(directory, "policy.json");
|
||||
const artifactPath = path.join(directory, "report.json");
|
||||
await mkdir(sourceRoot);
|
||||
await writeFile(
|
||||
catalogPath,
|
||||
`${JSON.stringify({ HTTP_SCENARIO_EXPECTATIONS: expectations }, null, 2)}\n`,
|
||||
);
|
||||
await writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||
await writeFile(
|
||||
policyPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 2,
|
||||
scenarioCatalogs: [
|
||||
{
|
||||
owner: "mutation-fixture",
|
||||
path: catalogPath,
|
||||
expectationExport: "HTTP_SCENARIO_EXPECTATIONS",
|
||||
receiptPath,
|
||||
receiptSchemaVersion: 1,
|
||||
},
|
||||
],
|
||||
sourceContracts: [],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
return spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/check-test-evidence.ts",
|
||||
"--scenario-only",
|
||||
"--source-root",
|
||||
sourceRoot,
|
||||
"--policy",
|
||||
policyPath,
|
||||
"--catalog",
|
||||
catalogPath,
|
||||
"--receipt",
|
||||
receiptPath,
|
||||
"--artifact",
|
||||
artifactPath,
|
||||
],
|
||||
{ encoding: "utf8", cwd: process.cwd() },
|
||||
);
|
||||
}
|
||||
|
||||
describe("HTTP scenario evidence receipt schema", () => {
|
||||
it("bounds every orchestration child by time and captured output", () => {
|
||||
const environment = { CI: "true" };
|
||||
|
||||
expect(
|
||||
createBoundedPnpmScriptInvocation({
|
||||
nodePath: "/runtime/node",
|
||||
pnpmCli: "/runtime/pnpm.cjs",
|
||||
script: "test:http-scenario-catalog",
|
||||
environment,
|
||||
}),
|
||||
).toEqual({
|
||||
command: "/runtime/node",
|
||||
arguments: [
|
||||
"/runtime/pnpm.cjs",
|
||||
"run",
|
||||
"test:http-scenario-catalog",
|
||||
],
|
||||
options: {
|
||||
encoding: "utf8",
|
||||
env: environment,
|
||||
killSignal: "SIGTERM",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: 60_000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves timeout code and termination signal in child diagnostics", () => {
|
||||
const error = Object.assign(new Error("spawn timed out"), {
|
||||
code: "ETIMEDOUT",
|
||||
});
|
||||
|
||||
expect(
|
||||
formatPnpmScriptFailure("test:http-scenario-catalog", {
|
||||
status: null,
|
||||
signal: "SIGTERM",
|
||||
error,
|
||||
}),
|
||||
).toBe(
|
||||
"test:http-scenario-catalog failed: exit=null, signal=SIGTERM, error=ETIMEDOUT: spawn timed out",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects internally inconsistent assertion groups as schema drift", async () => {
|
||||
const receipt = JSON.parse(
|
||||
await readFile(
|
||||
"tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json",
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
|
||||
const result = httpScenarioReceiptSchema.safeParse(receipt);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
if (!result.success) {
|
||||
expect(result.error.issues).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
path: ["rows", 0, "expected", "fetch", "agrees"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
path: ["rows", 0, "observed", "fetch", "agrees"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "zero declarations and executions",
|
||||
diagnostic: "[TEST-EVIDENCE-CATALOG-ZERO]",
|
||||
mutate(expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
expectations.splice(0);
|
||||
receipt.catalogTotal = 0;
|
||||
receipt.executedIds.splice(0);
|
||||
receipt.rows.splice(0);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "a missing execution",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-MISSING]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.executedIds.pop();
|
||||
receipt.rows.pop();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "an extra execution",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-EXTRA]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
const extra = structuredClone(receipt.rows.at(-1)!);
|
||||
extra.executionId = "FIXTURE_OPERATION::zz-extra";
|
||||
receipt.executedIds.push(extra.executionId);
|
||||
receipt.rows.push(extra);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "a duplicate execution",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.executedIds.splice(1, 0, receipt.executedIds[0]!);
|
||||
receipt.rows.splice(1, 0, structuredClone(receipt.rows[0]!));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "digest drift",
|
||||
diagnostic: "[TEST-EVIDENCE-CATALOG-DIGEST]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.catalogDigest = `sha256:${"0".repeat(64)}`;
|
||||
},
|
||||
preserveDigest: true,
|
||||
},
|
||||
{
|
||||
label: "unsorted execution rows",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-UNSORTED]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.executedIds.reverse();
|
||||
receipt.rows.reverse();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "expected value drift",
|
||||
diagnostic: "[TEST-EVIDENCE-EXPECTED-DRIFT]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
const changed = {
|
||||
...receipt.rows[0]!.expected,
|
||||
outcome: { kind: "FORBIDDEN", detail: null },
|
||||
};
|
||||
receipt.rows[0]!.expected = changed;
|
||||
receipt.rows[0]!.observed = structuredClone(changed);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "observed value mismatch",
|
||||
diagnostic: "[TEST-EVIDENCE-OBSERVATION-MISMATCH]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.rows[0]!.observed = {
|
||||
...receipt.rows[0]!.observed,
|
||||
outcome: { kind: "FORBIDDEN", detail: null },
|
||||
};
|
||||
},
|
||||
},
|
||||
])("rejects $label with its stable diagnostic", async (fixture) => {
|
||||
const { expectations, receipt } = await validFixture();
|
||||
fixture.mutate(expectations, receipt);
|
||||
if (!fixture.preserveDigest) {
|
||||
receipt.catalogDigest = computeHttpScenarioCatalogDigest(
|
||||
receipt.schemaVersion,
|
||||
expectations,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await runFixture(expectations, receipt);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).toBe(1);
|
||||
expect(`${result.stdout}\n${result.stderr}`).toContain(fixture.diagnostic);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user