test: execute the HTTP scenario catalog
This commit is contained in:
DongHyeonka
@@ -0,0 +1,410 @@
|
||||
import { mkdir, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
import { createContractHttpExecutor } from "../../src/adapters/http/http-execution-v3.ts";
|
||||
import {
|
||||
HTTP_EXECUTION_CEILINGS,
|
||||
type InstalledHttpContract,
|
||||
} from "../../src/contracts/external-contract-runtime.ts";
|
||||
import type { CacheScopeSnapshot } from "../../src/contracts/server-state-scope.ts";
|
||||
import { REFERENCE_FEATURE_TEMPLATE_CONTRIBUTION } from "../../src/features/reference-feature/contracts/reference-feature-contract-contribution.ts";
|
||||
import {
|
||||
computeHttpScenarioCatalogDigest,
|
||||
httpScenarioReceiptSchema,
|
||||
type AttemptStatus,
|
||||
type BodyDisposition,
|
||||
type HttpScenarioAssertionGroups,
|
||||
type RetryReason,
|
||||
} from "../../scripts/lib/http-scenario-evidence.ts";
|
||||
import { writeValidatedJsonArtifact } from "../../scripts/lib/validated-json-artifact.ts";
|
||||
import { createReferenceScenarioHandlers } from "../mocks/handlers/reference-resources.ts";
|
||||
import { createStrictMockServer } from "../mocks/server.ts";
|
||||
import {
|
||||
HTTP_SCENARIO_EXECUTION_IDS,
|
||||
HTTP_SCENARIO_EXPECTATIONS,
|
||||
HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION,
|
||||
OPERATION_SCENARIO_CATALOG,
|
||||
type HttpScenarioExpectation,
|
||||
type HttpScenarioOperationId,
|
||||
} from "../mocks/scenarios/catalog.ts";
|
||||
|
||||
const RECEIPT_PATH = path.resolve(
|
||||
"artifacts/tests/http-scenario-executions.json",
|
||||
);
|
||||
const mockApi = createStrictMockServer();
|
||||
|
||||
beforeAll(mockApi.listen);
|
||||
afterAll(mockApi.close);
|
||||
|
||||
type AttemptTrace = {
|
||||
status: AttemptStatus;
|
||||
media: string | null;
|
||||
pulledBytes: number;
|
||||
completed: boolean;
|
||||
cancelled: boolean;
|
||||
};
|
||||
|
||||
function operationFor(operationId: HttpScenarioOperationId) {
|
||||
const operation = REFERENCE_FEATURE_TEMPLATE_CONTRIBUTION.http.find(
|
||||
(candidate) => candidate.contract.operationId === operationId,
|
||||
);
|
||||
if (!operation) throw new Error(`Missing operation fixture: ${operationId}`);
|
||||
return operation as InstalledHttpContract<unknown, unknown, unknown>;
|
||||
}
|
||||
|
||||
function inputFor(operationId: HttpScenarioOperationId): unknown {
|
||||
if (operationId === "LIST_REFERENCE_RESOURCES") return { limit: 20 };
|
||||
if (operationId === "GET_REFERENCE_RESOURCE") {
|
||||
return { resourceId: "reference-1" };
|
||||
}
|
||||
return { name: "Created" };
|
||||
}
|
||||
|
||||
function normalizeMedia(response: Response): string | null {
|
||||
const value = response.headers.get("content-type");
|
||||
return value?.split(";", 1)[0]?.trim().toLowerCase() || null;
|
||||
}
|
||||
|
||||
function tracingFetcher(attempts: AttemptTrace[]): typeof fetch {
|
||||
return async (input, init) => {
|
||||
const trace: AttemptTrace = {
|
||||
status: "PENDING_ABORT",
|
||||
media: null,
|
||||
pulledBytes: 0,
|
||||
completed: false,
|
||||
cancelled: false,
|
||||
};
|
||||
attempts.push(trace);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(input, init);
|
||||
} catch (error) {
|
||||
trace.status = init?.signal?.aborted
|
||||
? "PENDING_ABORT"
|
||||
: "NETWORK_REJECTION";
|
||||
throw error;
|
||||
}
|
||||
trace.status = response.status;
|
||||
trace.media = normalizeMedia(response);
|
||||
if (!response.body) return response;
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const proxy = new ReadableStream<Uint8Array>(
|
||||
{
|
||||
async pull(controller) {
|
||||
try {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) {
|
||||
trace.completed = true;
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
trace.pulledBytes += chunk.value.byteLength;
|
||||
controller.enqueue(chunk.value);
|
||||
} catch (error) {
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
cancel(reason) {
|
||||
trace.cancelled = true;
|
||||
void reader.cancel(reason).catch(() => {});
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
);
|
||||
return new Response(proxy, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function outcomeDetail(outcome: Readonly<Record<string, unknown>>): string | null {
|
||||
if (outcome.kind === "CONTRACT_VIOLATION") {
|
||||
return String((outcome.violation as Readonly<{ kind: string }>).kind);
|
||||
}
|
||||
if (outcome.kind === "TRANSPORT_FAILURE") {
|
||||
return String((outcome.failure as Readonly<{ kind: string }>).kind);
|
||||
}
|
||||
if (outcome.kind === "PROBLEM") {
|
||||
return String((outcome.metadata as Readonly<{ status: number }>).status);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function retryReason(status: AttemptStatus): RetryReason {
|
||||
if (status === "NETWORK_REJECTION") return "NETWORK_FAILURE";
|
||||
if (status === 429) return "HTTP_429";
|
||||
if (status === 503) return "HTTP_503";
|
||||
throw new Error(`Sleep followed a non-retryable attempt: ${String(status)}`);
|
||||
}
|
||||
|
||||
function bodyDisposition(
|
||||
trace: AttemptTrace,
|
||||
outcome: Readonly<Record<string, unknown>>,
|
||||
): BodyDisposition {
|
||||
if (trace.status === "NETWORK_REJECTION" || trace.status === "PENDING_ABORT") {
|
||||
return "NO_RESPONSE";
|
||||
}
|
||||
if (
|
||||
outcome.kind === "CONTRACT_VIOLATION" &&
|
||||
outcomeDetail(outcome) === "RESPONSE_TOO_LARGE" &&
|
||||
trace.cancelled
|
||||
) {
|
||||
return "REJECTED_LIMIT";
|
||||
}
|
||||
if (trace.completed) return "FULLY_READ_WITHIN_BOUND";
|
||||
if (trace.cancelled) return "CANCELLED_WITHOUT_READ";
|
||||
throw new Error(`Response body was neither consumed nor cancelled: ${trace.status}`);
|
||||
}
|
||||
|
||||
function appliedBodyCeiling(
|
||||
trace: AttemptTrace,
|
||||
operation: InstalledHttpContract<unknown, unknown, unknown>,
|
||||
): number {
|
||||
if (typeof trace.status !== "number") return 0;
|
||||
if (operation.contract.acceptedStatuses.includes(trace.status)) {
|
||||
return operation.frontend.responseByteLimit;
|
||||
}
|
||||
return [404, 409, 422, 503].includes(trace.status)
|
||||
? HTTP_EXECUTION_CEILINGS.problemResponseBytes
|
||||
: 0;
|
||||
}
|
||||
|
||||
async function waitForHandler(
|
||||
ready: Promise<void>,
|
||||
executionId: string,
|
||||
): Promise<void> {
|
||||
let watchdog: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
ready,
|
||||
new Promise<never>((_resolve, reject) => {
|
||||
watchdog = setTimeout(
|
||||
() => reject(new Error(`Handler readiness timed out: ${executionId}`)),
|
||||
2_000,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (watchdog !== undefined) clearTimeout(watchdog);
|
||||
}
|
||||
}
|
||||
|
||||
async function executeScenario(
|
||||
entry: HttpScenarioExpectation,
|
||||
): Promise<HttpScenarioAssertionGroups> {
|
||||
const physicalAttempts: AttemptTrace[] = [];
|
||||
const sleeps: RetryReason[] = [];
|
||||
const observations: Array<Readonly<{
|
||||
outcome: string;
|
||||
attempts: number;
|
||||
certainty: string;
|
||||
}>> = [];
|
||||
const caller = new AbortController();
|
||||
const scopeLifetime = new AbortController();
|
||||
let scopeCurrent = true;
|
||||
let handlerReady!: () => void;
|
||||
const ready = new Promise<void>((resolve) => {
|
||||
handlerReady = resolve;
|
||||
});
|
||||
const scope: CacheScopeSnapshot = Object.freeze({
|
||||
generation: 1,
|
||||
fingerprint: "scenario-scope-1",
|
||||
identities: Object.freeze({}) as CacheScopeSnapshot["identities"],
|
||||
signal: scopeLifetime.signal,
|
||||
isCurrent: () => scopeCurrent,
|
||||
});
|
||||
const baseOperation = operationFor(entry.operationId);
|
||||
const operation =
|
||||
entry.testDeadlineOverrideMs === null
|
||||
? baseOperation
|
||||
: Object.freeze({
|
||||
...baseOperation,
|
||||
frontend: Object.freeze({
|
||||
...baseOperation.frontend,
|
||||
totalDeadlineMs: entry.testDeadlineOverrideMs,
|
||||
}),
|
||||
});
|
||||
const fetcher = tracingFetcher(physicalAttempts);
|
||||
const executor = createContractHttpExecutor({
|
||||
baseUrl: "https://api.test",
|
||||
maxRetryAttempts: 2,
|
||||
attachCredentials: () => ({
|
||||
kind: "READY",
|
||||
headers: {},
|
||||
credentials: "omit",
|
||||
}),
|
||||
fetcher,
|
||||
random: () => 0,
|
||||
sleep: async () => {
|
||||
const prior = physicalAttempts.at(-1);
|
||||
if (!prior) throw new Error("Retry sleep occurred before an attempt");
|
||||
sleeps.push(retryReason(prior.status));
|
||||
},
|
||||
observe: (observation) => observations.push(observation),
|
||||
});
|
||||
|
||||
mockApi.server.use(
|
||||
...createReferenceScenarioHandlers({
|
||||
scenarios: { [entry.operationId]: entry.scenarioId },
|
||||
onAttempt(attempt) {
|
||||
if (
|
||||
attempt.operationId === entry.operationId &&
|
||||
attempt.scenarioId === entry.scenarioId
|
||||
) {
|
||||
handlerReady();
|
||||
}
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
const execution = executor.execute(operation, inputFor(entry.operationId), {
|
||||
scope,
|
||||
signal: caller.signal,
|
||||
...(entry.operationId === "CREATE_REFERENCE_RESOURCE"
|
||||
? {
|
||||
intent: Object.freeze({
|
||||
intentId: `intent-${entry.scenarioId}`,
|
||||
operationId: entry.operationId,
|
||||
canonicalInputIdentity: "scenario-input",
|
||||
idempotencyKey: `scenario-${entry.scenarioId}`,
|
||||
createdAtMonotonicMs: 1,
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
if (entry.scenarioId === "timeout" || entry.scenarioId === "aborted") {
|
||||
await waitForHandler(ready, entry.executionId);
|
||||
if (entry.scenarioId === "aborted") {
|
||||
if (entry.operationId === "LIST_REFERENCE_RESOURCES") {
|
||||
caller.abort();
|
||||
} else {
|
||||
scopeCurrent = false;
|
||||
scopeLifetime.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
const outcome = (await execution) as unknown as Readonly<Record<string, unknown>>;
|
||||
expect(observations, `${entry.executionId} observer count`).toHaveLength(1);
|
||||
const observation = observations[0]!;
|
||||
const observedSignal = scopeLifetime.signal.aborted ? "ABORTED" : "ACTIVE";
|
||||
const cancellationOwner =
|
||||
observation.certainty === "TIMEOUT"
|
||||
? "DEADLINE"
|
||||
: caller.signal.aborted
|
||||
? "CALLER"
|
||||
: scopeLifetime.signal.aborted
|
||||
? "SCOPE_FENCE"
|
||||
: "NONE";
|
||||
const observed: HttpScenarioAssertionGroups = Object.freeze({
|
||||
status: Object.freeze({
|
||||
attempts: Object.freeze(physicalAttempts.map((attempt) => attempt.status)),
|
||||
final: physicalAttempts.at(-1)!.status,
|
||||
}),
|
||||
outcome: Object.freeze({
|
||||
kind: String(outcome.kind),
|
||||
detail: outcomeDetail(outcome),
|
||||
}),
|
||||
effect: Object.freeze({
|
||||
outcome: String(outcome.effect),
|
||||
observer: observation.certainty,
|
||||
}),
|
||||
retry: Object.freeze({ count: sleeps.length, reasons: Object.freeze(sleeps) }),
|
||||
fetch: Object.freeze({
|
||||
count: physicalAttempts.length,
|
||||
observerAttempts: observation.attempts,
|
||||
agrees: physicalAttempts.length === observation.attempts,
|
||||
}),
|
||||
media: Object.freeze({
|
||||
attempts: Object.freeze(physicalAttempts.map((attempt) => attempt.media)),
|
||||
final: physicalAttempts.at(-1)?.media ?? null,
|
||||
}),
|
||||
body: Object.freeze({
|
||||
attempts: Object.freeze(
|
||||
physicalAttempts.map((attempt) =>
|
||||
Object.freeze({
|
||||
disposition: bodyDisposition(attempt, outcome),
|
||||
pulledBytes: attempt.pulledBytes,
|
||||
ceiling: appliedBodyCeiling(attempt, operation),
|
||||
}),
|
||||
),
|
||||
),
|
||||
}),
|
||||
scope: Object.freeze({
|
||||
start: "CURRENT",
|
||||
end: scopeCurrent ? "CURRENT" : "STALE",
|
||||
signal: observedSignal,
|
||||
cancellationOwner,
|
||||
}),
|
||||
});
|
||||
return observed;
|
||||
} finally {
|
||||
caller.abort();
|
||||
scopeLifetime.abort();
|
||||
mockApi.reset();
|
||||
}
|
||||
}
|
||||
|
||||
describe("HTTP scenario catalog execution evidence", () => {
|
||||
it("declares exactly the 54 behaviorally distinct operation scenarios", () => {
|
||||
const executionIds = Object.entries(OPERATION_SCENARIO_CATALOG).flatMap(
|
||||
([operationId, scenarioIds]) =>
|
||||
scenarioIds.map((scenarioId) => `${operationId}::${scenarioId}`),
|
||||
);
|
||||
|
||||
expect(OPERATION_SCENARIO_CATALOG.LIST_REFERENCE_RESOURCES).toHaveLength(19);
|
||||
expect(OPERATION_SCENARIO_CATALOG.GET_REFERENCE_RESOURCE).toHaveLength(18);
|
||||
expect(OPERATION_SCENARIO_CATALOG.CREATE_REFERENCE_RESOURCE).toHaveLength(17);
|
||||
expect(executionIds).toHaveLength(54);
|
||||
expect(new Set(executionIds)).toHaveLength(54);
|
||||
expect(executionIds).toEqual(HTTP_SCENARIO_EXECUTION_IDS);
|
||||
});
|
||||
|
||||
it("executes every declared scenario and publishes one complete receipt", async () => {
|
||||
await rm(RECEIPT_PATH, { force: true });
|
||||
const rows: Array<Readonly<{
|
||||
executionId: string;
|
||||
expected: HttpScenarioAssertionGroups;
|
||||
observed: HttpScenarioAssertionGroups;
|
||||
testDeadlineOverrideMs: number | null;
|
||||
}>> = [];
|
||||
|
||||
for (const entry of HTTP_SCENARIO_EXPECTATIONS) {
|
||||
const observed = await executeScenario(entry);
|
||||
expect(observed, entry.executionId).toEqual(entry.expected);
|
||||
rows.push(
|
||||
Object.freeze({
|
||||
executionId: entry.executionId,
|
||||
expected: entry.expected,
|
||||
observed,
|
||||
testDeadlineOverrideMs: entry.testDeadlineOverrideMs,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const sortedRows = [...rows].sort((left, right) =>
|
||||
left.executionId.localeCompare(right.executionId),
|
||||
);
|
||||
await mkdir(path.dirname(RECEIPT_PATH), { recursive: true });
|
||||
await writeValidatedJsonArtifact({
|
||||
path: RECEIPT_PATH,
|
||||
schema: httpScenarioReceiptSchema,
|
||||
value: {
|
||||
schemaVersion: HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION,
|
||||
catalogDigest: computeHttpScenarioCatalogDigest(
|
||||
HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION,
|
||||
HTTP_SCENARIO_EXPECTATIONS,
|
||||
),
|
||||
catalogTotal: 54,
|
||||
executedIds: sortedRows.map((row) => row.executionId),
|
||||
rows: sortedRows,
|
||||
},
|
||||
});
|
||||
}, 30_000);
|
||||
});
|
||||
Reference in New Issue
Block a user