test: execute the HTTP scenario catalog
This commit is contained in:
DongHyeonka
@@ -65,6 +65,30 @@ async function flushMicrotasks(): Promise<void> {
|
||||
}
|
||||
|
||||
describe("descriptor-driven HTTP execution lifetime", () => {
|
||||
it("normalizes a read-side 429 to the non-applicable effect vocabulary", async () => {
|
||||
const executor = createContractHttpExecutor({
|
||||
baseUrl: "https://api.example/",
|
||||
maxRetryAttempts: 0,
|
||||
attachCredentials: () => ({
|
||||
kind: "READY",
|
||||
headers: {},
|
||||
credentials: "omit",
|
||||
}),
|
||||
fetcher: async () =>
|
||||
Response.json(
|
||||
{ type: "about:blank", title: "limited", status: 429 },
|
||||
{ status: 429 },
|
||||
),
|
||||
});
|
||||
|
||||
await expect(
|
||||
executor.execute(installed, { limit: 20 }, { scope }),
|
||||
).resolves.toMatchObject({
|
||||
kind: "RATE_LIMITED",
|
||||
effect: "NOT_APPLICABLE",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["absent intent", () => undefined],
|
||||
["empty key", () => mutationIntent({ idempotencyKey: "" })],
|
||||
@@ -471,6 +495,67 @@ describe("descriptor-driven HTTP execution lifetime", () => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("never retries a deadline-owned abort while the monotonic clock still has a sub-tick budget", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const iterationCount = 100;
|
||||
const sleep = vi.fn(async () => {
|
||||
throw new Error("a deadline-owned abort must not enter retry sleep");
|
||||
});
|
||||
const fetcher = vi.fn(
|
||||
(_input: RequestInfo | URL, init?: RequestInit) =>
|
||||
new Promise<Response>((_resolve, reject) => {
|
||||
init?.signal?.addEventListener(
|
||||
"abort",
|
||||
() => reject(new DOMException("Aborted", "AbortError")),
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
);
|
||||
const observe = vi.fn();
|
||||
const executor = createContractHttpExecutor({
|
||||
baseUrl: "https://api.example/",
|
||||
maxRetryAttempts: 2,
|
||||
attachCredentials: () => ({
|
||||
kind: "READY",
|
||||
headers: {},
|
||||
credentials: "omit",
|
||||
}),
|
||||
fetcher,
|
||||
monotonicNow: () => 0,
|
||||
random: () => 0,
|
||||
sleep,
|
||||
observe,
|
||||
});
|
||||
|
||||
for (let iteration = 0; iteration < iterationCount; iteration += 1) {
|
||||
const result = executor.execute(
|
||||
operation({ deadlineMs: 5 }),
|
||||
{ limit: 20 },
|
||||
{ scope },
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await flushMicrotasks();
|
||||
|
||||
await expect(result).resolves.toMatchObject({
|
||||
kind: "TRANSPORT_FAILURE",
|
||||
failure: { kind: "TIMEOUT" },
|
||||
});
|
||||
}
|
||||
|
||||
expect(fetcher).toHaveBeenCalledTimes(iterationCount);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
expect(observe).toHaveBeenCalledTimes(iterationCount);
|
||||
for (const [observation] of observe.mock.calls) {
|
||||
expect(observation).toEqual(
|
||||
expect.objectContaining({ attempts: 1, certainty: "TIMEOUT" }),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels a non-cooperative retry sleep when the caller aborts", async () => {
|
||||
const caller = new AbortController();
|
||||
let sleepSignal: AbortSignal | undefined;
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createBoundedPnpmScriptInvocation,
|
||||
formatPnpmScriptFailure,
|
||||
} from "../../scripts/lib/bounded-pnpm-script.ts";
|
||||
import {
|
||||
computeHttpScenarioCatalogDigest,
|
||||
httpScenarioExpectationSchema,
|
||||
httpScenarioReceiptSchema,
|
||||
type HttpScenarioAssertionGroups,
|
||||
type HttpScenarioExpectation,
|
||||
} from "../../scripts/lib/http-scenario-evidence.ts";
|
||||
|
||||
type MutableReceipt = {
|
||||
schemaVersion: number;
|
||||
catalogDigest: string;
|
||||
catalogTotal: number;
|
||||
executedIds: string[];
|
||||
rows: Array<{
|
||||
executionId: string;
|
||||
expected: HttpScenarioAssertionGroups;
|
||||
observed: HttpScenarioAssertionGroups;
|
||||
testDeadlineOverrideMs: number | null;
|
||||
}>;
|
||||
};
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories.splice(0).map((directory) =>
|
||||
rm(directory, { recursive: true, force: true }),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
async function validFixture() {
|
||||
const source = JSON.parse(
|
||||
await readFile(
|
||||
"tests/fixtures/test-evidence/scenarios/catalog.json",
|
||||
"utf8",
|
||||
),
|
||||
) as Readonly<Record<string, unknown>>;
|
||||
const expectations = httpScenarioExpectationSchema
|
||||
.array()
|
||||
.parse(source.HTTP_SCENARIO_EXPECTATIONS) as HttpScenarioExpectation[];
|
||||
const rows = expectations
|
||||
.map((entry) => ({
|
||||
executionId: entry.executionId,
|
||||
expected: structuredClone(entry.expected),
|
||||
observed: structuredClone(entry.expected),
|
||||
testDeadlineOverrideMs: entry.testDeadlineOverrideMs,
|
||||
}))
|
||||
.sort((left, right) => left.executionId.localeCompare(right.executionId));
|
||||
const receipt: MutableReceipt = {
|
||||
schemaVersion: 1,
|
||||
catalogDigest: computeHttpScenarioCatalogDigest(1, expectations),
|
||||
catalogTotal: expectations.length,
|
||||
executedIds: rows.map((row) => row.executionId),
|
||||
rows,
|
||||
};
|
||||
return { expectations, receipt };
|
||||
}
|
||||
|
||||
async function runFixture(
|
||||
expectations: readonly HttpScenarioExpectation[],
|
||||
receipt: MutableReceipt,
|
||||
) {
|
||||
await mkdir(".tmp", { recursive: true });
|
||||
const directory = await mkdtemp(path.resolve(".tmp/http-scenario-evidence-"));
|
||||
temporaryDirectories.push(directory);
|
||||
const sourceRoot = path.join(directory, "source");
|
||||
const catalogPath = path.join(directory, "catalog.json");
|
||||
const receiptPath = path.join(directory, "receipt.json");
|
||||
const policyPath = path.join(directory, "policy.json");
|
||||
const artifactPath = path.join(directory, "report.json");
|
||||
await mkdir(sourceRoot);
|
||||
await writeFile(
|
||||
catalogPath,
|
||||
`${JSON.stringify({ HTTP_SCENARIO_EXPECTATIONS: expectations }, null, 2)}\n`,
|
||||
);
|
||||
await writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||
await writeFile(
|
||||
policyPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 2,
|
||||
scenarioCatalogs: [
|
||||
{
|
||||
owner: "mutation-fixture",
|
||||
path: catalogPath,
|
||||
expectationExport: "HTTP_SCENARIO_EXPECTATIONS",
|
||||
receiptPath,
|
||||
receiptSchemaVersion: 1,
|
||||
},
|
||||
],
|
||||
sourceContracts: [],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
return spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/check-test-evidence.ts",
|
||||
"--scenario-only",
|
||||
"--source-root",
|
||||
sourceRoot,
|
||||
"--policy",
|
||||
policyPath,
|
||||
"--catalog",
|
||||
catalogPath,
|
||||
"--receipt",
|
||||
receiptPath,
|
||||
"--artifact",
|
||||
artifactPath,
|
||||
],
|
||||
{ encoding: "utf8", cwd: process.cwd() },
|
||||
);
|
||||
}
|
||||
|
||||
describe("HTTP scenario evidence receipt schema", () => {
|
||||
it("bounds every orchestration child by time and captured output", () => {
|
||||
const environment = { CI: "true" };
|
||||
|
||||
expect(
|
||||
createBoundedPnpmScriptInvocation({
|
||||
nodePath: "/runtime/node",
|
||||
pnpmCli: "/runtime/pnpm.cjs",
|
||||
script: "test:http-scenario-catalog",
|
||||
environment,
|
||||
}),
|
||||
).toEqual({
|
||||
command: "/runtime/node",
|
||||
arguments: [
|
||||
"/runtime/pnpm.cjs",
|
||||
"run",
|
||||
"test:http-scenario-catalog",
|
||||
],
|
||||
options: {
|
||||
encoding: "utf8",
|
||||
env: environment,
|
||||
killSignal: "SIGTERM",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: 60_000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves timeout code and termination signal in child diagnostics", () => {
|
||||
const error = Object.assign(new Error("spawn timed out"), {
|
||||
code: "ETIMEDOUT",
|
||||
});
|
||||
|
||||
expect(
|
||||
formatPnpmScriptFailure("test:http-scenario-catalog", {
|
||||
status: null,
|
||||
signal: "SIGTERM",
|
||||
error,
|
||||
}),
|
||||
).toBe(
|
||||
"test:http-scenario-catalog failed: exit=null, signal=SIGTERM, error=ETIMEDOUT: spawn timed out",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects internally inconsistent assertion groups as schema drift", async () => {
|
||||
const receipt = JSON.parse(
|
||||
await readFile(
|
||||
"tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json",
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
|
||||
const result = httpScenarioReceiptSchema.safeParse(receipt);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
if (!result.success) {
|
||||
expect(result.error.issues).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
path: ["rows", 0, "expected", "fetch", "agrees"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
path: ["rows", 0, "observed", "fetch", "agrees"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "zero declarations and executions",
|
||||
diagnostic: "[TEST-EVIDENCE-CATALOG-ZERO]",
|
||||
mutate(expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
expectations.splice(0);
|
||||
receipt.catalogTotal = 0;
|
||||
receipt.executedIds.splice(0);
|
||||
receipt.rows.splice(0);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "a missing execution",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-MISSING]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.executedIds.pop();
|
||||
receipt.rows.pop();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "an extra execution",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-EXTRA]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
const extra = structuredClone(receipt.rows.at(-1)!);
|
||||
extra.executionId = "FIXTURE_OPERATION::zz-extra";
|
||||
receipt.executedIds.push(extra.executionId);
|
||||
receipt.rows.push(extra);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "a duplicate execution",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.executedIds.splice(1, 0, receipt.executedIds[0]!);
|
||||
receipt.rows.splice(1, 0, structuredClone(receipt.rows[0]!));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "digest drift",
|
||||
diagnostic: "[TEST-EVIDENCE-CATALOG-DIGEST]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.catalogDigest = `sha256:${"0".repeat(64)}`;
|
||||
},
|
||||
preserveDigest: true,
|
||||
},
|
||||
{
|
||||
label: "unsorted execution rows",
|
||||
diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-UNSORTED]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.executedIds.reverse();
|
||||
receipt.rows.reverse();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "expected value drift",
|
||||
diagnostic: "[TEST-EVIDENCE-EXPECTED-DRIFT]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
const changed = {
|
||||
...receipt.rows[0]!.expected,
|
||||
outcome: { kind: "FORBIDDEN", detail: null },
|
||||
};
|
||||
receipt.rows[0]!.expected = changed;
|
||||
receipt.rows[0]!.observed = structuredClone(changed);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "observed value mismatch",
|
||||
diagnostic: "[TEST-EVIDENCE-OBSERVATION-MISMATCH]",
|
||||
mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) {
|
||||
receipt.rows[0]!.observed = {
|
||||
...receipt.rows[0]!.observed,
|
||||
outcome: { kind: "FORBIDDEN", detail: null },
|
||||
};
|
||||
},
|
||||
},
|
||||
])("rejects $label with its stable diagnostic", async (fixture) => {
|
||||
const { expectations, receipt } = await validFixture();
|
||||
fixture.mutate(expectations, receipt);
|
||||
if (!fixture.preserveDigest) {
|
||||
receipt.catalogDigest = computeHttpScenarioCatalogDigest(
|
||||
receipt.schemaVersion,
|
||||
expectations,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await runFixture(expectations, receipt);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).toBe(1);
|
||||
expect(`${result.stdout}\n${result.stderr}`).toContain(fixture.diagnostic);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { constants } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -41,6 +42,67 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe("validated JSON artifact writer", () => {
|
||||
it("syncs an O_NOFOLLOW exclusive temp and its directory around rename", async () => {
|
||||
const events: string[] = [];
|
||||
let openFlags = 0;
|
||||
let openMode = 0;
|
||||
const writer = createValidatedJsonArtifactWriter({
|
||||
createNonce: () => "durable",
|
||||
fileSystem: {
|
||||
open: async (_target, flags, mode) => {
|
||||
openFlags = flags;
|
||||
openMode = mode;
|
||||
events.push(`open:${flags}`);
|
||||
return {
|
||||
writeFile: async () => {
|
||||
events.push("write");
|
||||
},
|
||||
sync: async () => {
|
||||
events.push("file-sync");
|
||||
},
|
||||
close: async () => {
|
||||
events.push("file-close");
|
||||
},
|
||||
};
|
||||
},
|
||||
openDirectory: async () => ({
|
||||
sync: async () => {
|
||||
events.push("directory-sync");
|
||||
},
|
||||
close: async () => {
|
||||
events.push("directory-close");
|
||||
},
|
||||
}),
|
||||
rename: async () => {
|
||||
events.push("rename");
|
||||
},
|
||||
rm: async () => {},
|
||||
},
|
||||
});
|
||||
|
||||
await writer({
|
||||
path: "/tmp/report.json",
|
||||
schema: artifactSchema,
|
||||
value: { schemaVersion: 1, name: "valid" },
|
||||
});
|
||||
|
||||
expect(events).toHaveLength(7);
|
||||
expect(events[0]).toMatch(/^open:\d+$/);
|
||||
expect(openFlags & constants.O_WRONLY).toBe(constants.O_WRONLY);
|
||||
expect(openFlags & constants.O_CREAT).toBe(constants.O_CREAT);
|
||||
expect(openFlags & constants.O_EXCL).toBe(constants.O_EXCL);
|
||||
expect(openFlags & constants.O_NOFOLLOW).toBe(constants.O_NOFOLLOW);
|
||||
expect(openMode).toBe(0o600);
|
||||
expect(events.slice(1)).toEqual([
|
||||
"write",
|
||||
"file-sync",
|
||||
"file-close",
|
||||
"rename",
|
||||
"directory-sync",
|
||||
"directory-close",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(["existing", "missing"] as const)(
|
||||
"rejects invalid %s artifacts before changing destination state",
|
||||
async (destinationState) => {
|
||||
@@ -123,12 +185,20 @@ describe("validated JSON artifact writer", () => {
|
||||
return {
|
||||
writeFile: async (data, encoding) =>
|
||||
handle.writeFile(data, encoding),
|
||||
sync: async () => handle.sync(),
|
||||
close: async () => {
|
||||
await handle.close();
|
||||
throw closeError;
|
||||
},
|
||||
};
|
||||
},
|
||||
openDirectory: async (target) => {
|
||||
const handle = await open(target, constants.O_RDONLY);
|
||||
return {
|
||||
sync: async () => handle.sync(),
|
||||
close: async () => handle.close(),
|
||||
};
|
||||
},
|
||||
rename,
|
||||
rm,
|
||||
},
|
||||
@@ -165,12 +235,20 @@ describe("validated JSON artifact writer", () => {
|
||||
await handle.writeFile(data, encoding);
|
||||
throw writeError;
|
||||
},
|
||||
sync: async () => handle.sync(),
|
||||
close: async () => {
|
||||
await handle.close();
|
||||
throw closeError;
|
||||
},
|
||||
};
|
||||
},
|
||||
openDirectory: async (target) => {
|
||||
const handle = await open(target, constants.O_RDONLY);
|
||||
return {
|
||||
sync: async () => handle.sync(),
|
||||
close: async () => handle.close(),
|
||||
};
|
||||
},
|
||||
rename,
|
||||
rm,
|
||||
},
|
||||
@@ -244,6 +322,14 @@ describe("validated JSON artifact writer", () => {
|
||||
throw new Error("injected write failure");
|
||||
}
|
||||
},
|
||||
sync: async () => handle.sync(),
|
||||
close: async () => handle.close(),
|
||||
};
|
||||
},
|
||||
openDirectory: async (target) => {
|
||||
const handle = await open(target, constants.O_RDONLY);
|
||||
return {
|
||||
sync: async () => handle.sync(),
|
||||
close: async () => handle.close(),
|
||||
};
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user