test: execute the HTTP scenario catalog

This commit is contained in:
DongHyeonka
2026-08-02 11:17:28 +09:00
parent 76bf9f1aa3
commit abdd90ad5d
21 changed files with 2255 additions and 157 deletions
+36
View File
@@ -0,0 +1,36 @@
{
"HTTP_SCENARIO_EXPECTATIONS": [
{
"executionId": "FIXTURE_OPERATION::declared-and-executed",
"operationId": "FIXTURE_OPERATION",
"scenarioId": "declared-and-executed",
"expected": {
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": true },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"testDeadlineOverrideMs": null
},
{
"executionId": "FIXTURE_OPERATION::declared-but-unexecuted",
"operationId": "FIXTURE_OPERATION",
"scenarioId": "declared-but-unexecuted",
"expected": {
"status": { "attempts": [503], "final": 503 },
"outcome": { "kind": "PROBLEM", "detail": "503" },
"effect": { "outcome": "NOT_APPLIED", "observer": "NOT_STARTED" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": true },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 64, "ceiling": 65536 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"testDeadlineOverrideMs": null
}
]
}
+13
View File
@@ -0,0 +1,13 @@
{
"schemaVersion": 2,
"scenarioCatalogs": [
{
"owner": "fixture-owner",
"path": "tests/fixtures/test-evidence/scenarios/catalog.json",
"expectationExport": "HTTP_SCENARIO_EXPECTATIONS",
"receiptPath": "tests/fixtures/test-evidence/scenarios/receipt.json",
"receiptSchemaVersion": 1
}
],
"sourceContracts": []
}
@@ -0,0 +1,59 @@
{
"schemaVersion": 1,
"catalogDigest": "sha256:87779c47f0042025edf18e148f86c29e5d219f7010cf7effeb69d29059420841",
"catalogTotal": 2,
"executedIds": [
"FIXTURE_OPERATION::declared-and-executed",
"FIXTURE_OPERATION::declared-but-unexecuted"
],
"rows": [
{
"executionId": "FIXTURE_OPERATION::declared-and-executed",
"expected": {
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": false },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"observed": {
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": false },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"testDeadlineOverrideMs": null
},
{
"executionId": "FIXTURE_OPERATION::declared-but-unexecuted",
"expected": {
"status": { "attempts": [503], "final": 503 },
"outcome": { "kind": "PROBLEM", "detail": "503" },
"effect": { "outcome": "NOT_APPLIED", "observer": "NOT_STARTED" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": true },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 64, "ceiling": 65536 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"observed": {
"status": { "attempts": [503], "final": 503 },
"outcome": { "kind": "PROBLEM", "detail": "503" },
"effect": { "outcome": "NOT_APPLIED", "observer": "NOT_STARTED" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": true },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 64, "ceiling": 65536 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"testDeadlineOverrideMs": null
}
]
}
+32
View File
@@ -0,0 +1,32 @@
{
"schemaVersion": 1,
"catalogDigest": "sha256:87779c47f0042025edf18e148f86c29e5d219f7010cf7effeb69d29059420841",
"catalogTotal": 2,
"executedIds": ["FIXTURE_OPERATION::declared-and-executed"],
"rows": [
{
"executionId": "FIXTURE_OPERATION::declared-and-executed",
"expected": {
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": true },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"observed": {
"status": { "attempts": [200], "final": 200 },
"outcome": { "kind": "SUCCESS", "detail": null },
"effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" },
"retry": { "count": 0, "reasons": [] },
"fetch": { "count": 1, "observerAttempts": 1, "agrees": true },
"media": { "attempts": ["application/json"], "final": "application/json" },
"body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] },
"scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" }
},
"testDeadlineOverrideMs": null
}
]
}
@@ -0,0 +1,410 @@
import { mkdir, rm } from "node:fs/promises";
import path from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { createContractHttpExecutor } from "../../src/adapters/http/http-execution-v3.ts";
import {
HTTP_EXECUTION_CEILINGS,
type InstalledHttpContract,
} from "../../src/contracts/external-contract-runtime.ts";
import type { CacheScopeSnapshot } from "../../src/contracts/server-state-scope.ts";
import { REFERENCE_FEATURE_TEMPLATE_CONTRIBUTION } from "../../src/features/reference-feature/contracts/reference-feature-contract-contribution.ts";
import {
computeHttpScenarioCatalogDigest,
httpScenarioReceiptSchema,
type AttemptStatus,
type BodyDisposition,
type HttpScenarioAssertionGroups,
type RetryReason,
} from "../../scripts/lib/http-scenario-evidence.ts";
import { writeValidatedJsonArtifact } from "../../scripts/lib/validated-json-artifact.ts";
import { createReferenceScenarioHandlers } from "../mocks/handlers/reference-resources.ts";
import { createStrictMockServer } from "../mocks/server.ts";
import {
HTTP_SCENARIO_EXECUTION_IDS,
HTTP_SCENARIO_EXPECTATIONS,
HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION,
OPERATION_SCENARIO_CATALOG,
type HttpScenarioExpectation,
type HttpScenarioOperationId,
} from "../mocks/scenarios/catalog.ts";
const RECEIPT_PATH = path.resolve(
"artifacts/tests/http-scenario-executions.json",
);
const mockApi = createStrictMockServer();
beforeAll(mockApi.listen);
afterAll(mockApi.close);
type AttemptTrace = {
status: AttemptStatus;
media: string | null;
pulledBytes: number;
completed: boolean;
cancelled: boolean;
};
function operationFor(operationId: HttpScenarioOperationId) {
const operation = REFERENCE_FEATURE_TEMPLATE_CONTRIBUTION.http.find(
(candidate) => candidate.contract.operationId === operationId,
);
if (!operation) throw new Error(`Missing operation fixture: ${operationId}`);
return operation as InstalledHttpContract<unknown, unknown, unknown>;
}
function inputFor(operationId: HttpScenarioOperationId): unknown {
if (operationId === "LIST_REFERENCE_RESOURCES") return { limit: 20 };
if (operationId === "GET_REFERENCE_RESOURCE") {
return { resourceId: "reference-1" };
}
return { name: "Created" };
}
function normalizeMedia(response: Response): string | null {
const value = response.headers.get("content-type");
return value?.split(";", 1)[0]?.trim().toLowerCase() || null;
}
function tracingFetcher(attempts: AttemptTrace[]): typeof fetch {
return async (input, init) => {
const trace: AttemptTrace = {
status: "PENDING_ABORT",
media: null,
pulledBytes: 0,
completed: false,
cancelled: false,
};
attempts.push(trace);
let response: Response;
try {
response = await fetch(input, init);
} catch (error) {
trace.status = init?.signal?.aborted
? "PENDING_ABORT"
: "NETWORK_REJECTION";
throw error;
}
trace.status = response.status;
trace.media = normalizeMedia(response);
if (!response.body) return response;
const reader = response.body.getReader();
const proxy = new ReadableStream<Uint8Array>(
{
async pull(controller) {
try {
const chunk = await reader.read();
if (chunk.done) {
trace.completed = true;
controller.close();
return;
}
trace.pulledBytes += chunk.value.byteLength;
controller.enqueue(chunk.value);
} catch (error) {
controller.error(error);
}
},
cancel(reason) {
trace.cancelled = true;
void reader.cancel(reason).catch(() => {});
},
},
{ highWaterMark: 0 },
);
return new Response(proxy, {
status: response.status,
statusText: response.statusText,
headers: response.headers,
});
};
}
function outcomeDetail(outcome: Readonly<Record<string, unknown>>): string | null {
if (outcome.kind === "CONTRACT_VIOLATION") {
return String((outcome.violation as Readonly<{ kind: string }>).kind);
}
if (outcome.kind === "TRANSPORT_FAILURE") {
return String((outcome.failure as Readonly<{ kind: string }>).kind);
}
if (outcome.kind === "PROBLEM") {
return String((outcome.metadata as Readonly<{ status: number }>).status);
}
return null;
}
function retryReason(status: AttemptStatus): RetryReason {
if (status === "NETWORK_REJECTION") return "NETWORK_FAILURE";
if (status === 429) return "HTTP_429";
if (status === 503) return "HTTP_503";
throw new Error(`Sleep followed a non-retryable attempt: ${String(status)}`);
}
function bodyDisposition(
trace: AttemptTrace,
outcome: Readonly<Record<string, unknown>>,
): BodyDisposition {
if (trace.status === "NETWORK_REJECTION" || trace.status === "PENDING_ABORT") {
return "NO_RESPONSE";
}
if (
outcome.kind === "CONTRACT_VIOLATION" &&
outcomeDetail(outcome) === "RESPONSE_TOO_LARGE" &&
trace.cancelled
) {
return "REJECTED_LIMIT";
}
if (trace.completed) return "FULLY_READ_WITHIN_BOUND";
if (trace.cancelled) return "CANCELLED_WITHOUT_READ";
throw new Error(`Response body was neither consumed nor cancelled: ${trace.status}`);
}
function appliedBodyCeiling(
trace: AttemptTrace,
operation: InstalledHttpContract<unknown, unknown, unknown>,
): number {
if (typeof trace.status !== "number") return 0;
if (operation.contract.acceptedStatuses.includes(trace.status)) {
return operation.frontend.responseByteLimit;
}
return [404, 409, 422, 503].includes(trace.status)
? HTTP_EXECUTION_CEILINGS.problemResponseBytes
: 0;
}
async function waitForHandler(
ready: Promise<void>,
executionId: string,
): Promise<void> {
let watchdog: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([
ready,
new Promise<never>((_resolve, reject) => {
watchdog = setTimeout(
() => reject(new Error(`Handler readiness timed out: ${executionId}`)),
2_000,
);
}),
]);
} finally {
if (watchdog !== undefined) clearTimeout(watchdog);
}
}
async function executeScenario(
entry: HttpScenarioExpectation,
): Promise<HttpScenarioAssertionGroups> {
const physicalAttempts: AttemptTrace[] = [];
const sleeps: RetryReason[] = [];
const observations: Array<Readonly<{
outcome: string;
attempts: number;
certainty: string;
}>> = [];
const caller = new AbortController();
const scopeLifetime = new AbortController();
let scopeCurrent = true;
let handlerReady!: () => void;
const ready = new Promise<void>((resolve) => {
handlerReady = resolve;
});
const scope: CacheScopeSnapshot = Object.freeze({
generation: 1,
fingerprint: "scenario-scope-1",
identities: Object.freeze({}) as CacheScopeSnapshot["identities"],
signal: scopeLifetime.signal,
isCurrent: () => scopeCurrent,
});
const baseOperation = operationFor(entry.operationId);
const operation =
entry.testDeadlineOverrideMs === null
? baseOperation
: Object.freeze({
...baseOperation,
frontend: Object.freeze({
...baseOperation.frontend,
totalDeadlineMs: entry.testDeadlineOverrideMs,
}),
});
const fetcher = tracingFetcher(physicalAttempts);
const executor = createContractHttpExecutor({
baseUrl: "https://api.test",
maxRetryAttempts: 2,
attachCredentials: () => ({
kind: "READY",
headers: {},
credentials: "omit",
}),
fetcher,
random: () => 0,
sleep: async () => {
const prior = physicalAttempts.at(-1);
if (!prior) throw new Error("Retry sleep occurred before an attempt");
sleeps.push(retryReason(prior.status));
},
observe: (observation) => observations.push(observation),
});
mockApi.server.use(
...createReferenceScenarioHandlers({
scenarios: { [entry.operationId]: entry.scenarioId },
onAttempt(attempt) {
if (
attempt.operationId === entry.operationId &&
attempt.scenarioId === entry.scenarioId
) {
handlerReady();
}
},
}),
);
try {
const execution = executor.execute(operation, inputFor(entry.operationId), {
scope,
signal: caller.signal,
...(entry.operationId === "CREATE_REFERENCE_RESOURCE"
? {
intent: Object.freeze({
intentId: `intent-${entry.scenarioId}`,
operationId: entry.operationId,
canonicalInputIdentity: "scenario-input",
idempotencyKey: `scenario-${entry.scenarioId}`,
createdAtMonotonicMs: 1,
}),
}
: {}),
});
if (entry.scenarioId === "timeout" || entry.scenarioId === "aborted") {
await waitForHandler(ready, entry.executionId);
if (entry.scenarioId === "aborted") {
if (entry.operationId === "LIST_REFERENCE_RESOURCES") {
caller.abort();
} else {
scopeCurrent = false;
scopeLifetime.abort();
}
}
}
const outcome = (await execution) as unknown as Readonly<Record<string, unknown>>;
expect(observations, `${entry.executionId} observer count`).toHaveLength(1);
const observation = observations[0]!;
const observedSignal = scopeLifetime.signal.aborted ? "ABORTED" : "ACTIVE";
const cancellationOwner =
observation.certainty === "TIMEOUT"
? "DEADLINE"
: caller.signal.aborted
? "CALLER"
: scopeLifetime.signal.aborted
? "SCOPE_FENCE"
: "NONE";
const observed: HttpScenarioAssertionGroups = Object.freeze({
status: Object.freeze({
attempts: Object.freeze(physicalAttempts.map((attempt) => attempt.status)),
final: physicalAttempts.at(-1)!.status,
}),
outcome: Object.freeze({
kind: String(outcome.kind),
detail: outcomeDetail(outcome),
}),
effect: Object.freeze({
outcome: String(outcome.effect),
observer: observation.certainty,
}),
retry: Object.freeze({ count: sleeps.length, reasons: Object.freeze(sleeps) }),
fetch: Object.freeze({
count: physicalAttempts.length,
observerAttempts: observation.attempts,
agrees: physicalAttempts.length === observation.attempts,
}),
media: Object.freeze({
attempts: Object.freeze(physicalAttempts.map((attempt) => attempt.media)),
final: physicalAttempts.at(-1)?.media ?? null,
}),
body: Object.freeze({
attempts: Object.freeze(
physicalAttempts.map((attempt) =>
Object.freeze({
disposition: bodyDisposition(attempt, outcome),
pulledBytes: attempt.pulledBytes,
ceiling: appliedBodyCeiling(attempt, operation),
}),
),
),
}),
scope: Object.freeze({
start: "CURRENT",
end: scopeCurrent ? "CURRENT" : "STALE",
signal: observedSignal,
cancellationOwner,
}),
});
return observed;
} finally {
caller.abort();
scopeLifetime.abort();
mockApi.reset();
}
}
describe("HTTP scenario catalog execution evidence", () => {
it("declares exactly the 54 behaviorally distinct operation scenarios", () => {
const executionIds = Object.entries(OPERATION_SCENARIO_CATALOG).flatMap(
([operationId, scenarioIds]) =>
scenarioIds.map((scenarioId) => `${operationId}::${scenarioId}`),
);
expect(OPERATION_SCENARIO_CATALOG.LIST_REFERENCE_RESOURCES).toHaveLength(19);
expect(OPERATION_SCENARIO_CATALOG.GET_REFERENCE_RESOURCE).toHaveLength(18);
expect(OPERATION_SCENARIO_CATALOG.CREATE_REFERENCE_RESOURCE).toHaveLength(17);
expect(executionIds).toHaveLength(54);
expect(new Set(executionIds)).toHaveLength(54);
expect(executionIds).toEqual(HTTP_SCENARIO_EXECUTION_IDS);
});
it("executes every declared scenario and publishes one complete receipt", async () => {
await rm(RECEIPT_PATH, { force: true });
const rows: Array<Readonly<{
executionId: string;
expected: HttpScenarioAssertionGroups;
observed: HttpScenarioAssertionGroups;
testDeadlineOverrideMs: number | null;
}>> = [];
for (const entry of HTTP_SCENARIO_EXPECTATIONS) {
const observed = await executeScenario(entry);
expect(observed, entry.executionId).toEqual(entry.expected);
rows.push(
Object.freeze({
executionId: entry.executionId,
expected: entry.expected,
observed,
testDeadlineOverrideMs: entry.testDeadlineOverrideMs,
}),
);
}
const sortedRows = [...rows].sort((left, right) =>
left.executionId.localeCompare(right.executionId),
);
await mkdir(path.dirname(RECEIPT_PATH), { recursive: true });
await writeValidatedJsonArtifact({
path: RECEIPT_PATH,
schema: httpScenarioReceiptSchema,
value: {
schemaVersion: HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION,
catalogDigest: computeHttpScenarioCatalogDigest(
HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION,
HTTP_SCENARIO_EXPECTATIONS,
),
catalogTotal: 54,
executedIds: sortedRows.map((row) => row.executionId),
rows: sortedRows,
},
});
}, 30_000);
});
+60 -8
View File
@@ -1,6 +1,9 @@
import { delay, http, HttpResponse, type JsonBodyType } from "msw";
import type { HttpScenarioId } from "../scenarios/catalog.ts";
import type {
HttpScenarioId,
HttpScenarioOperationId,
} from "../scenarios/catalog.ts";
import { assertOperationScenario } from "../scenarios/catalog.ts";
export type ReferenceResourceFixture = Readonly<{
@@ -20,6 +23,11 @@ type ScenarioOptions = Readonly<{
>
>;
resources?: ReferenceResourceFixture[];
onAttempt?(attempt: Readonly<{
operationId: HttpScenarioOperationId;
scenarioId: HttpScenarioId;
attempt: number;
}>): void;
onList?(search: string): void;
onCreate?(body: Readonly<Record<string, unknown>>): void;
}>;
@@ -31,12 +39,15 @@ const DEFAULT_RESOURCE = Object.freeze({
});
async function scenarioResponse(
operationId: HttpScenarioOperationId,
scenario: HttpScenarioId,
payload: JsonBodyType,
attempt: number,
) {
if (scenario === "slow") await delay(50);
if (scenario === "timeout") await delay(30_000);
if (scenario === "timeout" || scenario === "aborted") {
await delay("infinite");
}
if (scenario === "network-error") return HttpResponse.error();
if (scenario === "content-type-mismatch") {
return new HttpResponse("<html>not json</html>", {
@@ -55,8 +66,7 @@ async function scenarioResponse(
return HttpResponse.json({ unexpected: true });
}
if (
scenario === "auth-persistent-401" ||
(scenario === "auth-recover-once" && attempt === 1)
scenario === "unauthenticated-401"
) {
return HttpResponse.json(problem(401, "AUTH_REQUIRED"), {
status: 401,
@@ -84,13 +94,34 @@ async function scenarioResponse(
});
}
if (
scenario === "server-terminal-500" ||
scenario === "server-terminal-503" ||
(scenario === "server-retry-success" && attempt === 1)
) {
return HttpResponse.json(problem(503, "SERVER_FAILURE"), {
status: 503,
});
}
if (scenario === "response-too-large") {
const ceiling =
operationId === "LIST_REFERENCE_RESOURCES" ? 262_144 : 32_768;
const bytes = new Uint8Array(ceiling + 1).fill(0x20);
bytes[0] = 0x5b;
bytes[bytes.length - 1] = 0x5d;
return new HttpResponse(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(bytes);
controller.close();
},
}),
{
headers: {
"Content-Type": "application/json",
"Content-Length": String(bytes.byteLength),
},
},
);
}
return HttpResponse.json(payload);
}
@@ -125,10 +156,17 @@ export function createReferenceScenarioHandlers(options: ScenarioOptions = {}) {
options.onList?.(new URL(request.url).search);
const scenario = scenarioFor("LIST_REFERENCE_RESOURCES");
const payload = scenario === "empty" ? [] : resources;
const attempt = nextAttempt("LIST_REFERENCE_RESOURCES");
options.onAttempt?.({
operationId: "LIST_REFERENCE_RESOURCES",
scenarioId: scenario,
attempt,
});
return scenarioResponse(
"LIST_REFERENCE_RESOURCES",
scenario,
payload,
nextAttempt("LIST_REFERENCE_RESOURCES"),
attempt,
);
}),
http.post(
@@ -149,10 +187,17 @@ export function createReferenceScenarioHandlers(options: ScenarioOptions = {}) {
createdAt: "2026-07-26T00:00:00.000Z",
};
if (scenario === "success") resources.push(created);
const attempt = nextAttempt("CREATE_REFERENCE_RESOURCE");
options.onAttempt?.({
operationId: "CREATE_REFERENCE_RESOURCE",
scenarioId: scenario,
attempt,
});
return scenarioResponse(
"CREATE_REFERENCE_RESOURCE",
scenario,
created,
nextAttempt("CREATE_REFERENCE_RESOURCE"),
attempt,
);
},
),
@@ -163,10 +208,17 @@ export function createReferenceScenarioHandlers(options: ScenarioOptions = {}) {
const resource =
resources.find((entry) => entry.id === params.resourceId) ??
DEFAULT_RESOURCE;
const attempt = nextAttempt("GET_REFERENCE_RESOURCE");
options.onAttempt?.({
operationId: "GET_REFERENCE_RESOURCE",
scenarioId: scenario,
attempt,
});
return scenarioResponse(
"GET_REFERENCE_RESOURCE",
scenario,
resource,
nextAttempt("GET_REFERENCE_RESOURCE"),
attempt,
);
},
),
+401 -11
View File
@@ -1,3 +1,19 @@
import type {
AttemptStatus,
BodyDisposition,
HttpScenarioAssertionGroups,
RetryReason,
} from "../../../scripts/lib/http-scenario-evidence.ts";
export type {
AttemptStatus,
BodyDisposition,
HttpScenarioAssertionGroups,
RetryReason,
} from "../../../scripts/lib/http-scenario-evidence.ts";
export const HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION = 1 as const;
export const HTTP_SCENARIO_IDS = Object.freeze([
"success",
"empty",
@@ -9,37 +25,411 @@ export const HTTP_SCENARIO_IDS = Object.freeze([
"malformed-json",
"envelope-mismatch",
"schema-mismatch",
"auth-recover-once",
"auth-persistent-401",
"response-too-large",
"unauthenticated-401",
"forbidden-403",
"not-found-404",
"conflict-409",
"validation-422",
"rate-limited-429",
"server-retry-success",
"server-terminal-500",
"server-terminal-503",
] as const);
export type HttpScenarioId = (typeof HTTP_SCENARIO_IDS)[number];
export const OPERATION_SCENARIO_CATALOG = Object.freeze({
export const HTTP_SCENARIO_OPERATION_IDS = Object.freeze([
"LIST_REFERENCE_RESOURCES",
"GET_REFERENCE_RESOURCE",
"CREATE_REFERENCE_RESOURCE",
] as const);
export type HttpScenarioOperationId =
(typeof HTTP_SCENARIO_OPERATION_IDS)[number];
export type HttpScenarioExecutionId =
`${HttpScenarioOperationId}::${HttpScenarioId}`;
export type HttpScenarioExpectation = Readonly<{
executionId: HttpScenarioExecutionId;
operationId: HttpScenarioOperationId;
scenarioId: HttpScenarioId;
expected: HttpScenarioAssertionGroups;
testDeadlineOverrideMs: number | null;
}>;
const encoder = new TextEncoder();
const jsonBytes = (value: unknown) => encoder.encode(JSON.stringify(value)).byteLength;
const RESOURCE = Object.freeze({
id: "reference-1",
name: "Reference",
createdAt: "2026-07-26T00:00:00.000Z",
});
const CREATED = Object.freeze({
id: "reference-created",
name: "Created",
createdAt: "2026-07-26T00:00:00.000Z",
});
function operationFacts(operationId: HttpScenarioOperationId) {
const command = operationId === "CREATE_REFERENCE_RESOURCE";
return Object.freeze({
command,
ceiling:
operationId === "LIST_REFERENCE_RESOURCES" ? 262_144 : 32_768,
payload:
operationId === "LIST_REFERENCE_RESOURCES"
? [RESOURCE]
: operationId === "GET_REFERENCE_RESOURCE"
? RESOURCE
: CREATED,
});
}
function problem(status: number, code: string) {
return {
type: `https://api.test/problems/${code.toLowerCase()}`,
title: code,
status,
code,
};
}
function attemptBody(
disposition: BodyDisposition,
pulledBytes: number,
ceiling: number,
) {
return Object.freeze({ disposition, pulledBytes, ceiling });
}
function expectationFor(
operationId: HttpScenarioOperationId,
scenarioId: HttpScenarioId,
): HttpScenarioExpectation {
const { command, ceiling, payload } = operationFacts(operationId);
let statuses: AttemptStatus[] = [200];
let kind = "SUCCESS";
let detail: string | null = null;
let outcomeEffect = command ? "APPLIED_CONFIRMED" : "NOT_APPLICABLE";
let observer = outcomeEffect;
let retryReasons: RetryReason[] = [];
let media: (string | null)[] = ["application/json"];
let bodies = [attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes(payload), ceiling)];
let scopeEnd: "CURRENT" | "STALE" = "CURRENT";
let scopeSignal: "ACTIVE" | "ABORTED" = "ACTIVE";
let cancellationOwner: "NONE" | "CALLER" | "SCOPE_FENCE" | "DEADLINE" =
"NONE";
let testDeadlineOverrideMs: number | null = null;
switch (scenarioId) {
case "success":
case "slow":
break;
case "empty":
bodies = [attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes([]), ceiling)];
break;
case "network-error": {
const attemptCount = command ? 1 : 3;
statuses = Array.from({ length: attemptCount }, () => "NETWORK_REJECTION");
kind = "TRANSPORT_FAILURE";
detail = "NETWORK_FAILURE";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_STARTED";
observer = "NETWORK_FAILURE";
retryReasons = command ? [] : ["NETWORK_FAILURE", "NETWORK_FAILURE"];
media = Array.from({ length: attemptCount }, () => null);
bodies = Array.from({ length: attemptCount }, () =>
attemptBody("NO_RESPONSE", 0, 0),
);
break;
}
case "timeout":
statuses = ["PENDING_ABORT"];
kind = "TRANSPORT_FAILURE";
detail = "TIMEOUT";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_STARTED";
observer = "TIMEOUT";
media = [null];
bodies = [attemptBody("NO_RESPONSE", 0, 0)];
cancellationOwner = "DEADLINE";
testDeadlineOverrideMs = 500;
break;
case "aborted":
statuses = ["PENDING_ABORT"];
media = [null];
bodies = [attemptBody("NO_RESPONSE", 0, 0)];
if (operationId === "LIST_REFERENCE_RESOURCES") {
kind = "CANCELLED";
outcomeEffect = "NOT_STARTED";
observer = "CANCELLED";
cancellationOwner = "CALLER";
} else {
scopeSignal = "ABORTED";
kind = "TRANSPORT_FAILURE";
detail = "ABORTED_BY_SCOPE";
outcomeEffect = "NOT_STARTED";
observer = "SCOPE_FENCED";
scopeEnd = "STALE";
cancellationOwner = "SCOPE_FENCE";
}
break;
case "content-type-mismatch":
kind = "CONTRACT_VIOLATION";
detail = "CONTENT_TYPE_MISMATCH";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "CONTRACT_VIOLATION";
media = ["text/html"];
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
encoder.encode("<html>not json</html>").byteLength,
ceiling,
),
];
break;
case "malformed-json":
kind = "CONTRACT_VIOLATION";
detail = "JSON_INVALID";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "CONTRACT_VIOLATION";
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
encoder.encode("{invalid").byteLength,
ceiling,
),
];
break;
case "envelope-mismatch":
kind = "CONTRACT_VIOLATION";
detail = "SUCCESS_SCHEMA_INVALID";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "CONTRACT_VIOLATION";
bodies = [
attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes({ data: payload }), ceiling),
];
break;
case "schema-mismatch":
kind = "CONTRACT_VIOLATION";
detail = "SUCCESS_SCHEMA_INVALID";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "CONTRACT_VIOLATION";
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes({ unexpected: true }),
ceiling,
),
];
break;
case "response-too-large":
kind = "CONTRACT_VIOLATION";
detail = "RESPONSE_TOO_LARGE";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "RESPONSE_TOO_LARGE";
bodies = [attemptBody("REJECTED_LIMIT", 0, ceiling)];
break;
case "unauthenticated-401":
statuses = [401];
kind = "UNAUTHENTICATED";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "UNAUTHENTICATED";
bodies = [attemptBody("CANCELLED_WITHOUT_READ", 0, 0)];
break;
case "forbidden-403":
statuses = [403];
kind = "FORBIDDEN";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "FORBIDDEN";
bodies = [attemptBody("CANCELLED_WITHOUT_READ", 0, 0)];
break;
case "not-found-404":
statuses = [404];
kind = "PROBLEM";
detail = "404";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLIED";
observer = command ? "MAYBE_APPLIED" : "NOT_STARTED";
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes(problem(404, "NOT_FOUND")),
65_536,
),
];
break;
case "conflict-409":
statuses = [409];
kind = "PROBLEM";
detail = "409";
outcomeEffect = "NOT_APPLIED";
observer = command ? "NOT_APPLIED" : "NOT_STARTED";
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes(problem(409, "CONFLICT")),
65_536,
),
];
break;
case "validation-422":
statuses = [422];
kind = "PROBLEM";
detail = "422";
outcomeEffect = "NOT_APPLIED";
observer = command ? "NOT_APPLIED" : "NOT_STARTED";
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes(problem(422, "VALIDATION_REJECTED")),
65_536,
),
];
break;
case "rate-limited-429": {
const attemptCount = command ? 1 : 3;
statuses = Array.from({ length: attemptCount }, () => 429);
kind = "RATE_LIMITED";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE";
observer = "RATE_LIMITED";
retryReasons = command ? [] : ["HTTP_429", "HTTP_429"];
media = Array.from({ length: attemptCount }, () => "application/json");
bodies = Array.from({ length: attemptCount }, () =>
attemptBody("CANCELLED_WITHOUT_READ", 0, 0),
);
break;
}
case "server-retry-success":
if (command) {
statuses = [503];
kind = "PROBLEM";
detail = "503";
outcomeEffect = "MAYBE_APPLIED";
observer = "MAYBE_APPLIED";
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes(problem(503, "SERVER_FAILURE")),
65_536,
),
];
} else {
statuses = [503, 200];
retryReasons = ["HTTP_503"];
media = ["application/json", "application/json"];
bodies = [
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes(problem(503, "SERVER_FAILURE")),
65_536,
),
attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes(payload), ceiling),
];
}
break;
case "server-terminal-503": {
const attemptCount = command ? 1 : 3;
statuses = Array.from({ length: attemptCount }, () => 503);
kind = "PROBLEM";
detail = "503";
outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLIED";
observer = command ? "MAYBE_APPLIED" : "NOT_STARTED";
retryReasons = command ? [] : ["HTTP_503", "HTTP_503"];
media = Array.from({ length: attemptCount }, () => "application/json");
bodies = Array.from({ length: attemptCount }, () =>
attemptBody(
"FULLY_READ_WITHIN_BOUND",
jsonBytes(problem(503, "SERVER_FAILURE")),
65_536,
),
);
break;
}
}
const executionId = `${operationId}::${scenarioId}` as const;
return Object.freeze({
executionId,
operationId,
scenarioId,
expected: Object.freeze({
status: Object.freeze({
attempts: Object.freeze(statuses),
final: statuses.at(-1)!,
}),
outcome: Object.freeze({ kind, detail }),
effect: Object.freeze({ outcome: outcomeEffect, observer }),
retry: Object.freeze({
count: retryReasons.length,
reasons: Object.freeze(retryReasons),
}),
fetch: Object.freeze({
count: statuses.length,
observerAttempts: statuses.length,
agrees: true,
}),
media: Object.freeze({
attempts: Object.freeze(media),
final: media.at(-1) ?? null,
}),
body: Object.freeze({ attempts: Object.freeze(bodies) }),
scope: Object.freeze({
start: "CURRENT" as const,
end: scopeEnd,
signal: scopeSignal,
cancellationOwner,
}),
}),
testDeadlineOverrideMs,
});
}
const OPERATION_SCENARIOS = Object.freeze({
LIST_REFERENCE_RESOURCES: HTTP_SCENARIO_IDS,
GET_REFERENCE_RESOURCE: Object.freeze(
HTTP_SCENARIO_IDS.filter((scenario) => scenario !== "empty"),
),
CREATE_REFERENCE_RESOURCE: Object.freeze(
HTTP_SCENARIO_IDS.filter(
(scenario) => !["empty", "aborted"].includes(scenario),
(scenario) => scenario !== "empty" && scenario !== "aborted",
),
),
GET_REFERENCE_RESOURCE: HTTP_SCENARIO_IDS,
} satisfies Readonly<Record<string, readonly HttpScenarioId[]>>);
} satisfies Readonly<
Record<HttpScenarioOperationId, readonly HttpScenarioId[]>
>);
export const HTTP_SCENARIO_EXPECTATIONS = Object.freeze(
HTTP_SCENARIO_OPERATION_IDS.flatMap((operationId) =>
OPERATION_SCENARIOS[operationId].map((scenarioId) =>
expectationFor(operationId, scenarioId),
),
),
);
export const HTTP_SCENARIO_EXECUTION_IDS = Object.freeze(
HTTP_SCENARIO_EXPECTATIONS.map((expectation) => expectation.executionId),
);
export const OPERATION_SCENARIO_CATALOG = Object.freeze(
Object.fromEntries(
HTTP_SCENARIO_OPERATION_IDS.map((operationId) => [
operationId,
Object.freeze(
HTTP_SCENARIO_EXPECTATIONS.filter(
(expectation) => expectation.operationId === operationId,
).map((expectation) => expectation.scenarioId),
),
]),
) as Readonly<
Record<HttpScenarioOperationId, readonly HttpScenarioId[]>
>,
);
export function assertOperationScenario(
operationId: keyof typeof OPERATION_SCENARIO_CATALOG,
operationId: HttpScenarioOperationId,
scenario: HttpScenarioId,
) {
if (!OPERATION_SCENARIO_CATALOG[operationId].includes(scenario)) {
throw new Error(
`Scenario ${scenario} is not declared for ${operationId}`,
);
throw new Error(`Scenario ${scenario} is not declared for ${operationId}`);
}
return scenario;
}
+85
View File
@@ -65,6 +65,30 @@ async function flushMicrotasks(): Promise<void> {
}
describe("descriptor-driven HTTP execution lifetime", () => {
it("normalizes a read-side 429 to the non-applicable effect vocabulary", async () => {
const executor = createContractHttpExecutor({
baseUrl: "https://api.example/",
maxRetryAttempts: 0,
attachCredentials: () => ({
kind: "READY",
headers: {},
credentials: "omit",
}),
fetcher: async () =>
Response.json(
{ type: "about:blank", title: "limited", status: 429 },
{ status: 429 },
),
});
await expect(
executor.execute(installed, { limit: 20 }, { scope }),
).resolves.toMatchObject({
kind: "RATE_LIMITED",
effect: "NOT_APPLICABLE",
});
});
it.each([
["absent intent", () => undefined],
["empty key", () => mutationIntent({ idempotencyKey: "" })],
@@ -471,6 +495,67 @@ describe("descriptor-driven HTTP execution lifetime", () => {
vi.useRealTimers();
});
it("never retries a deadline-owned abort while the monotonic clock still has a sub-tick budget", async () => {
vi.useFakeTimers();
try {
const iterationCount = 100;
const sleep = vi.fn(async () => {
throw new Error("a deadline-owned abort must not enter retry sleep");
});
const fetcher = vi.fn(
(_input: RequestInfo | URL, init?: RequestInit) =>
new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener(
"abort",
() => reject(new DOMException("Aborted", "AbortError")),
{ once: true },
);
}),
);
const observe = vi.fn();
const executor = createContractHttpExecutor({
baseUrl: "https://api.example/",
maxRetryAttempts: 2,
attachCredentials: () => ({
kind: "READY",
headers: {},
credentials: "omit",
}),
fetcher,
monotonicNow: () => 0,
random: () => 0,
sleep,
observe,
});
for (let iteration = 0; iteration < iterationCount; iteration += 1) {
const result = executor.execute(
operation({ deadlineMs: 5 }),
{ limit: 20 },
{ scope },
);
await vi.advanceTimersByTimeAsync(5);
await flushMicrotasks();
await expect(result).resolves.toMatchObject({
kind: "TRANSPORT_FAILURE",
failure: { kind: "TIMEOUT" },
});
}
expect(fetcher).toHaveBeenCalledTimes(iterationCount);
expect(sleep).not.toHaveBeenCalled();
expect(observe).toHaveBeenCalledTimes(iterationCount);
for (const [observation] of observe.mock.calls) {
expect(observation).toEqual(
expect.objectContaining({ attempts: 1, certainty: "TIMEOUT" }),
);
}
} finally {
vi.useRealTimers();
}
});
it("cancels a non-cooperative retry sleep when the caller aborts", async () => {
const caller = new AbortController();
let sleepSignal: AbortSignal | undefined;
+294
View File
@@ -0,0 +1,294 @@
import { spawnSync } from "node:child_process";
import {
mkdir,
mkdtemp,
readFile,
rm,
writeFile,
} from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
createBoundedPnpmScriptInvocation,
formatPnpmScriptFailure,
} from "../../scripts/lib/bounded-pnpm-script.ts";
import {
computeHttpScenarioCatalogDigest,
httpScenarioExpectationSchema,
httpScenarioReceiptSchema,
type HttpScenarioAssertionGroups,
type HttpScenarioExpectation,
} from "../../scripts/lib/http-scenario-evidence.ts";
type MutableReceipt = {
schemaVersion: number;
catalogDigest: string;
catalogTotal: number;
executedIds: string[];
rows: Array<{
executionId: string;
expected: HttpScenarioAssertionGroups;
observed: HttpScenarioAssertionGroups;
testDeadlineOverrideMs: number | null;
}>;
};
const temporaryDirectories: string[] = [];
afterEach(async () => {
await Promise.all(
temporaryDirectories.splice(0).map((directory) =>
rm(directory, { recursive: true, force: true }),
),
);
});
async function validFixture() {
const source = JSON.parse(
await readFile(
"tests/fixtures/test-evidence/scenarios/catalog.json",
"utf8",
),
) as Readonly<Record<string, unknown>>;
const expectations = httpScenarioExpectationSchema
.array()
.parse(source.HTTP_SCENARIO_EXPECTATIONS) as HttpScenarioExpectation[];
const rows = expectations
.map((entry) => ({
executionId: entry.executionId,
expected: structuredClone(entry.expected),
observed: structuredClone(entry.expected),
testDeadlineOverrideMs: entry.testDeadlineOverrideMs,
}))
.sort((left, right) => left.executionId.localeCompare(right.executionId));
const receipt: MutableReceipt = {
schemaVersion: 1,
catalogDigest: computeHttpScenarioCatalogDigest(1, expectations),
catalogTotal: expectations.length,
executedIds: rows.map((row) => row.executionId),
rows,
};
return { expectations, receipt };
}
async function runFixture(
expectations: readonly HttpScenarioExpectation[],
receipt: MutableReceipt,
) {
await mkdir(".tmp", { recursive: true });
const directory = await mkdtemp(path.resolve(".tmp/http-scenario-evidence-"));
temporaryDirectories.push(directory);
const sourceRoot = path.join(directory, "source");
const catalogPath = path.join(directory, "catalog.json");
const receiptPath = path.join(directory, "receipt.json");
const policyPath = path.join(directory, "policy.json");
const artifactPath = path.join(directory, "report.json");
await mkdir(sourceRoot);
await writeFile(
catalogPath,
`${JSON.stringify({ HTTP_SCENARIO_EXPECTATIONS: expectations }, null, 2)}\n`,
);
await writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`);
await writeFile(
policyPath,
`${JSON.stringify(
{
schemaVersion: 2,
scenarioCatalogs: [
{
owner: "mutation-fixture",
path: catalogPath,
expectationExport: "HTTP_SCENARIO_EXPECTATIONS",
receiptPath,
receiptSchemaVersion: 1,
},
],
sourceContracts: [],
},
null,
2,
)}\n`,
);
return spawnSync(
process.execPath,
[
"scripts/check-test-evidence.ts",
"--scenario-only",
"--source-root",
sourceRoot,
"--policy",
policyPath,
"--catalog",
catalogPath,
"--receipt",
receiptPath,
"--artifact",
artifactPath,
],
{ encoding: "utf8", cwd: process.cwd() },
);
}
describe("HTTP scenario evidence receipt schema", () => {
it("bounds every orchestration child by time and captured output", () => {
const environment = { CI: "true" };
expect(
createBoundedPnpmScriptInvocation({
nodePath: "/runtime/node",
pnpmCli: "/runtime/pnpm.cjs",
script: "test:http-scenario-catalog",
environment,
}),
).toEqual({
command: "/runtime/node",
arguments: [
"/runtime/pnpm.cjs",
"run",
"test:http-scenario-catalog",
],
options: {
encoding: "utf8",
env: environment,
killSignal: "SIGTERM",
maxBuffer: 16 * 1024 * 1024,
timeout: 60_000,
},
});
});
it("preserves timeout code and termination signal in child diagnostics", () => {
const error = Object.assign(new Error("spawn timed out"), {
code: "ETIMEDOUT",
});
expect(
formatPnpmScriptFailure("test:http-scenario-catalog", {
status: null,
signal: "SIGTERM",
error,
}),
).toBe(
"test:http-scenario-catalog failed: exit=null, signal=SIGTERM, error=ETIMEDOUT: spawn timed out",
);
});
it("rejects internally inconsistent assertion groups as schema drift", async () => {
const receipt = JSON.parse(
await readFile(
"tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json",
"utf8",
),
);
const result = httpScenarioReceiptSchema.safeParse(receipt);
expect(result.success).toBe(false);
if (!result.success) {
expect(result.error.issues).toEqual(
expect.arrayContaining([
expect.objectContaining({
path: ["rows", 0, "expected", "fetch", "agrees"],
}),
expect.objectContaining({
path: ["rows", 0, "observed", "fetch", "agrees"],
}),
]),
);
}
});
it.each([
{
label: "zero declarations and executions",
diagnostic: "[TEST-EVIDENCE-CATALOG-ZERO]",
mutate(expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
expectations.splice(0);
receipt.catalogTotal = 0;
receipt.executedIds.splice(0);
receipt.rows.splice(0);
},
},
{
label: "a missing execution",
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-MISSING]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
receipt.executedIds.pop();
receipt.rows.pop();
},
},
{
label: "an extra execution",
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-EXTRA]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
const extra = structuredClone(receipt.rows.at(-1)!);
extra.executionId = "FIXTURE_OPERATION::zz-extra";
receipt.executedIds.push(extra.executionId);
receipt.rows.push(extra);
},
},
{
label: "a duplicate execution",
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
receipt.executedIds.splice(1, 0, receipt.executedIds[0]!);
receipt.rows.splice(1, 0, structuredClone(receipt.rows[0]!));
},
},
{
label: "digest drift",
diagnostic: "[TEST-EVIDENCE-CATALOG-DIGEST]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
receipt.catalogDigest = `sha256:${"0".repeat(64)}`;
},
preserveDigest: true,
},
{
label: "unsorted execution rows",
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-UNSORTED]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
receipt.executedIds.reverse();
receipt.rows.reverse();
},
},
{
label: "expected value drift",
diagnostic: "[TEST-EVIDENCE-EXPECTED-DRIFT]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
const changed = {
...receipt.rows[0]!.expected,
outcome: { kind: "FORBIDDEN", detail: null },
};
receipt.rows[0]!.expected = changed;
receipt.rows[0]!.observed = structuredClone(changed);
},
},
{
label: "observed value mismatch",
diagnostic: "[TEST-EVIDENCE-OBSERVATION-MISMATCH]",
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
receipt.rows[0]!.observed = {
...receipt.rows[0]!.observed,
outcome: { kind: "FORBIDDEN", detail: null },
};
},
},
])("rejects $label with its stable diagnostic", async (fixture) => {
const { expectations, receipt } = await validFixture();
fixture.mutate(expectations, receipt);
if (!fixture.preserveDigest) {
receipt.catalogDigest = computeHttpScenarioCatalogDigest(
receipt.schemaVersion,
expectations,
);
}
const result = await runFixture(expectations, receipt);
expect(result.error).toBeUndefined();
expect(result.status).toBe(1);
expect(`${result.stdout}\n${result.stderr}`).toContain(fixture.diagnostic);
});
});
@@ -7,6 +7,7 @@ import {
rm,
writeFile,
} from "node:fs/promises";
import { constants } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -41,6 +42,67 @@ afterEach(async () => {
});
describe("validated JSON artifact writer", () => {
it("syncs an O_NOFOLLOW exclusive temp and its directory around rename", async () => {
const events: string[] = [];
let openFlags = 0;
let openMode = 0;
const writer = createValidatedJsonArtifactWriter({
createNonce: () => "durable",
fileSystem: {
open: async (_target, flags, mode) => {
openFlags = flags;
openMode = mode;
events.push(`open:${flags}`);
return {
writeFile: async () => {
events.push("write");
},
sync: async () => {
events.push("file-sync");
},
close: async () => {
events.push("file-close");
},
};
},
openDirectory: async () => ({
sync: async () => {
events.push("directory-sync");
},
close: async () => {
events.push("directory-close");
},
}),
rename: async () => {
events.push("rename");
},
rm: async () => {},
},
});
await writer({
path: "/tmp/report.json",
schema: artifactSchema,
value: { schemaVersion: 1, name: "valid" },
});
expect(events).toHaveLength(7);
expect(events[0]).toMatch(/^open:\d+$/);
expect(openFlags & constants.O_WRONLY).toBe(constants.O_WRONLY);
expect(openFlags & constants.O_CREAT).toBe(constants.O_CREAT);
expect(openFlags & constants.O_EXCL).toBe(constants.O_EXCL);
expect(openFlags & constants.O_NOFOLLOW).toBe(constants.O_NOFOLLOW);
expect(openMode).toBe(0o600);
expect(events.slice(1)).toEqual([
"write",
"file-sync",
"file-close",
"rename",
"directory-sync",
"directory-close",
]);
});
it.each(["existing", "missing"] as const)(
"rejects invalid %s artifacts before changing destination state",
async (destinationState) => {
@@ -123,12 +185,20 @@ describe("validated JSON artifact writer", () => {
return {
writeFile: async (data, encoding) =>
handle.writeFile(data, encoding),
sync: async () => handle.sync(),
close: async () => {
await handle.close();
throw closeError;
},
};
},
openDirectory: async (target) => {
const handle = await open(target, constants.O_RDONLY);
return {
sync: async () => handle.sync(),
close: async () => handle.close(),
};
},
rename,
rm,
},
@@ -165,12 +235,20 @@ describe("validated JSON artifact writer", () => {
await handle.writeFile(data, encoding);
throw writeError;
},
sync: async () => handle.sync(),
close: async () => {
await handle.close();
throw closeError;
},
};
},
openDirectory: async (target) => {
const handle = await open(target, constants.O_RDONLY);
return {
sync: async () => handle.sync(),
close: async () => handle.close(),
};
},
rename,
rm,
},
@@ -244,6 +322,14 @@ describe("validated JSON artifact writer", () => {
throw new Error("injected write failure");
}
},
sync: async () => handle.sync(),
close: async () => handle.close(),
};
},
openDirectory: async (target) => {
const handle = await open(target, constants.O_RDONLY);
return {
sync: async () => handle.sync(),
close: async () => handle.close(),
};
},