test: harden HTTP scenario execution evidence

This commit is contained in:
DongHyeonka
2026-08-02 11:36:27 +09:00
parent e08d8c2dd8
commit d2eb320936
6 changed files with 143 additions and 28 deletions
@@ -13,7 +13,7 @@
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"retry": { "count": 1, "reasons": ["HTTP_503"] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": false },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
@@ -23,7 +23,7 @@
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"retry": { "count": 1, "reasons": ["HTTP_503"] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": false },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
+16 -15
View File
@@ -3,9 +3,9 @@ import path from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { readBoundedBytes } from "../../src/adapters/http/bounded-body-reader.ts";
import { createContractHttpExecutor } from "../../src/adapters/http/http-execution-v3.ts";
import {
HTTP_EXECUTION_CEILINGS,
type InstalledHttpContract,
} from "../../src/contracts/external-contract-runtime.ts";
import type { CacheScopeSnapshot } from "../../src/contracts/server-state-scope.ts";
@@ -42,6 +42,7 @@ type AttemptTrace = {
status: AttemptStatus;
media: string | null;
pulledBytes: number;
appliedCeiling: number | null;
completed: boolean;
cancelled: boolean;
};
@@ -73,6 +74,7 @@ function tracingFetcher(attempts: AttemptTrace[]): typeof fetch {
status: "PENDING_ABORT",
media: null,
pulledBytes: 0,
appliedCeiling: null,
completed: false,
cancelled: false,
};
@@ -109,6 +111,9 @@ function tracingFetcher(attempts: AttemptTrace[]): typeof fetch {
},
cancel(reason) {
trace.cancelled = true;
// MSW-transferred oversized bodies leave the original cancel promise
// permanently pending. Record and request cancellation without making
// the proxy response less responsive than the real executor contract.
void reader.cancel(reason).catch(() => {});
},
},
@@ -161,19 +166,6 @@ function bodyDisposition(
throw new Error(`Response body was neither consumed nor cancelled: ${trace.status}`);
}
function appliedBodyCeiling(
trace: AttemptTrace,
operation: InstalledHttpContract<unknown, unknown, unknown>,
): number {
if (typeof trace.status !== "number") return 0;
if (operation.contract.acceptedStatuses.includes(trace.status)) {
return operation.frontend.responseByteLimit;
}
return [404, 409, 422, 503].includes(trace.status)
? HTTP_EXECUTION_CEILINGS.problemResponseBytes
: 0;
}
async function waitForHandler(
ready: Promise<void>,
executionId: string,
@@ -240,6 +232,15 @@ async function executeScenario(
}),
fetcher,
random: () => 0,
readBoundedResponseBytes: async (response, maximumBytes) => {
const trace = physicalAttempts.at(-1);
if (!trace) throw new Error("Body reader ran before a physical attempt");
if (trace.appliedCeiling !== null) {
throw new Error("Body reader ran more than once for one attempt");
}
trace.appliedCeiling = maximumBytes;
return readBoundedBytes(response, maximumBytes);
},
sleep: async () => {
const prior = physicalAttempts.at(-1);
if (!prior) throw new Error("Retry sleep occurred before an attempt");
@@ -331,7 +332,7 @@ async function executeScenario(
Object.freeze({
disposition: bodyDisposition(attempt, outcome),
pulledBytes: attempt.pulledBytes,
ceiling: appliedBodyCeiling(attempt, operation),
ceiling: attempt.appliedCeiling ?? 0,
}),
),
),
+70 -4
View File
@@ -11,6 +11,7 @@ import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
createBoundedChildInvocation,
createBoundedPnpmScriptInvocation,
formatPnpmScriptFailure,
} from "../../scripts/lib/bounded-pnpm-script.ts";
@@ -111,9 +112,9 @@ async function runFixture(
2,
)}\n`,
);
return spawnSync(
process.execPath,
[
const invocation = createBoundedChildInvocation({
command: process.execPath,
arguments: [
"scripts/check-test-evidence.ts",
"--scenario-only",
"--source-root",
@@ -127,11 +128,41 @@ async function runFixture(
"--artifact",
artifactPath,
],
{ encoding: "utf8", cwd: process.cwd() },
environment: process.env,
cwd: process.cwd(),
});
return spawnSync(
invocation.command,
[...invocation.arguments],
invocation.options,
);
}
describe("HTTP scenario evidence receipt schema", () => {
it("bounds checker fixture children with the shared process policy", () => {
const environment = { CI: "true" };
expect(
createBoundedChildInvocation({
command: "/runtime/node",
arguments: ["scripts/check-test-evidence.ts", "--scenario-only"],
environment,
cwd: "/workspace",
}),
).toEqual({
command: "/runtime/node",
arguments: ["scripts/check-test-evidence.ts", "--scenario-only"],
options: {
cwd: "/workspace",
encoding: "utf8",
env: environment,
killSignal: "SIGTERM",
maxBuffer: 16 * 1024 * 1024,
timeout: 60_000,
},
});
});
it("bounds every orchestration child by time and captured output", () => {
const environment = { CI: "true" };
@@ -175,6 +206,22 @@ describe("HTTP scenario evidence receipt schema", () => {
);
});
it("preserves output overflow codes in child diagnostics", () => {
const error = Object.assign(new Error("stdout maxBuffer exceeded"), {
code: "ENOBUFS",
});
expect(
formatPnpmScriptFailure("check:http-scenario-evidence", {
status: null,
signal: null,
error,
}),
).toBe(
"check:http-scenario-evidence failed: exit=null, signal=none, error=ENOBUFS: stdout maxBuffer exceeded",
);
});
it("rejects internally inconsistent assertion groups as schema drift", async () => {
const receipt = JSON.parse(
await readFile(
@@ -195,6 +242,18 @@ describe("HTTP scenario evidence receipt schema", () => {
expect.objectContaining({
path: ["rows", 0, "observed", "fetch", "agrees"],
}),
expect.objectContaining({
path: ["rows", 0, "expected", "retry", "count"],
}),
expect.objectContaining({
path: ["rows", 0, "expected", "retry", "reasons", 0],
}),
expect.objectContaining({
path: ["rows", 0, "observed", "retry", "count"],
}),
expect.objectContaining({
path: ["rows", 0, "observed", "retry", "reasons", 0],
}),
]),
);
}
@@ -275,6 +334,13 @@ describe("HTTP scenario evidence receipt schema", () => {
};
},
},
{
label: "a deadline override drift",
diagnostic: "[TEST-EVIDENCE-DEADLINE-DRIFT]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
receipt.rows[0]!.testDeadlineOverrideMs = 500;
},
},
])("rejects $label with its stable diagnostic", async (fixture) => {
const { expectations, receipt } = await validFixture();
fixture.mutate(expectations, receipt);