diff --git a/config/ci/gates.json b/config/ci/gates.json index 7ed8574..5d3aa8f 100644 --- a/config/ci/gates.json +++ b/config/ci/gates.json @@ -133,12 +133,16 @@ "name": "integration", "steps": [ { "script": "test:integration", "expect": "pass" }, + { "script": "test:http-scenario-evidence", "expect": "pass" }, { "script": "test:reference-feature", "expect": "pass" }, { "script": "test:recipes", "expect": "pass" } ], "logPath": "artifacts/quality/gates/FE-GATE-007.txt", "evidence": [ "artifacts/tests/integration.xml", + "artifacts/tests/http-scenario-executions.json", + "artifacts/quality/http-scenario-evidence.json", + "artifacts/quality/http-scenario-evidence-fixture.json", "artifacts/tests/reference-feature.xml", "artifacts/tests/optional-recipes.xml" ], @@ -155,7 +159,7 @@ }, { "script": "test:storybook", "expect": "pass" }, { "script": "test:visual", "expect": "pass" }, - { "script": "check:test-evidence", "expect": "pass" }, + { "script": "check:test-evidence:browser", "expect": "pass" }, { "script": "check:test-evidence:fixture", "expect": "fail", "expectedExitCode": 1, "expectedDiagnosticId": "Test evidence failed:" } ], "logPath": "artifacts/quality/gates/FE-GATE-008.txt", diff --git a/config/testing/test-evidence.json b/config/testing/test-evidence.json index 4da088b..afa519f 100644 --- a/config/testing/test-evidence.json +++ b/config/testing/test-evidence.json @@ -1,11 +1,12 @@ { - "schemaVersion": 1, + "schemaVersion": 2, "scenarioCatalogs": [ { "owner": "reference-feature", "path": "tests/mocks/scenarios/catalog.ts", - "arrayExport": "HTTP_SCENARIO_IDS", - "minimumEntries": 19 + "expectationExport": "HTTP_SCENARIO_EXPECTATIONS", + "receiptPath": "artifacts/tests/http-scenario-executions.json", + "receiptSchemaVersion": 1 } ], "sourceContracts": [ diff --git a/package.json b/package.json index 49f1247..2d21720 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,8 @@ "test:unit": "vitest run tests/unit --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/unit.xml", "test:component": "vitest run tests/component --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/component.xml", "test:integration": "vitest run tests/integration --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/integration.xml", + "test:http-scenario-catalog": "vitest run tests/integration/http-scenario-catalog.test.ts --reporter=default --maxWorkers=1", + "test:http-scenario-evidence": "node scripts/run-http-scenario-evidence.ts", "test:recipes": "vitest run tests/recipes --reporter=default --reporter=junit --outputFile.junit=artifacts/tests/optional-recipes.xml --passWithNoTests", "test:e2e": "playwright test", "test:e2e:dev": "playwright test --config playwright.dev.config.ts", @@ -56,6 +58,9 @@ "test:visual": "playwright test --config playwright.visual.config.ts", "test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots", "check:test-evidence": "node scripts/check-test-evidence.ts", + "check:test-evidence:browser": "node scripts/check-test-evidence.ts --skip-scenario-executions", + "check:http-scenario-evidence": "node scripts/check-test-evidence.ts --scenario-only --artifact artifacts/quality/http-scenario-evidence.json", + "check:http-scenario-evidence:fixture": "node scripts/check-test-evidence.ts --scenario-only --source-root tests/fixtures/test-evidence/scenarios/source --policy tests/fixtures/test-evidence/scenarios/policy.json --catalog tests/fixtures/test-evidence/scenarios/catalog.json --receipt tests/fixtures/test-evidence/scenarios/receipt.json --artifact artifacts/quality/http-scenario-evidence-fixture.json", "check:test-evidence:source": "node scripts/check-test-evidence.ts --source-only --artifact artifacts/quality/test-evidence-source.json", "check:test-evidence:fixture": "node scripts/check-test-evidence.ts --source-root tests/fixtures/test-evidence/forbidden --artifact artifacts/quality/test-evidence-fixture.json", "test:a11y": "playwright test --grep @a11y && node scripts/write-a11y-report.ts", diff --git a/scripts/check-test-evidence.ts b/scripts/check-test-evidence.ts index 25d82d4..6377b7c 100644 --- a/scripts/check-test-evidence.ts +++ b/scripts/check-test-evidence.ts @@ -1,24 +1,62 @@ -import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, stat } from "node:fs/promises"; import path from "node:path"; +import { pathToFileURL } from "node:url"; -type ScenarioCatalogContribution = Readonly<{ - owner: string; - path: string; - arrayExport: string; - minimumEntries: number; -}>; +import { z } from "zod"; -type SourceContractContribution = Readonly<{ - owner: string; - path: string; - requiredTokens: readonly string[]; -}>; +import { + computeHttpScenarioCatalogDigest, + httpScenarioExpectationSchema, + httpScenarioReceiptSchema, + sameScenarioJson, + type HttpScenarioExpectation, + type HttpScenarioReceipt, +} from "./lib/http-scenario-evidence.ts"; +import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; -type TestEvidencePolicy = Readonly<{ - schemaVersion: number; - scenarioCatalogs: readonly unknown[]; - sourceContracts: readonly unknown[]; -}>; +const scenarioContributionSchema = z + .object({ + owner: z.string().min(1), + path: z.string().min(1), + expectationExport: z.string().min(1), + receiptPath: z.string().min(1), + receiptSchemaVersion: z.number().int().positive(), + }) + .strict(); + +const sourceContractSchema = z + .object({ + owner: z.string().min(1), + path: z.string().min(1), + requiredTokens: z.array(z.string().min(1)), + }) + .strict(); + +const policySchema = z + .object({ + schemaVersion: z.literal(2), + scenarioCatalogs: z.array(scenarioContributionSchema), + sourceContracts: z.array(sourceContractSchema), + }) + .strict(); + +const reportSchema = z + .object({ + schemaVersion: z.literal(2), + sourceRoot: z.string().min(1), + status: z.enum(["PASS", "FAIL"]), + facts: z + .object({ + scannedFiles: z.number().int().nonnegative(), + visualBaselines: z.number().int().nonnegative(), + sharedScenarios: z.number().int().nonnegative(), + declaredScenarioExecutions: z.number().int().nonnegative(), + executedScenarioExecutions: z.number().int().nonnegative(), + }) + .strict(), + failures: z.array(z.string()), + }) + .strict(); function argumentValue(name: string, fallback: string): string { const index = process.argv.indexOf(name); @@ -27,6 +65,11 @@ function argumentValue(name: string, fallback: string): string { : fallback; } +function optionalArgumentValue(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : undefined; +} + const sourceRoot = argumentValue("--source-root", "tests"); const artifactPath = argumentValue( "--artifact", @@ -36,13 +79,21 @@ const policyPath = argumentValue( "--policy", "config/testing/test-evidence.json", ); +const explicitCatalogPath = optionalArgumentValue("--catalog"); +const explicitReceiptPath = optionalArgumentValue("--receipt"); const fixtureMode = sourceRoot !== "tests"; const sourceOnly = process.argv.includes("--source-only"); +const scenarioOnly = process.argv.includes("--scenario-only"); +const skipScenarioExecutions = process.argv.includes( + "--skip-scenario-executions", +); const failures: string[] = []; const facts = { scannedFiles: 0, visualBaselines: 0, sharedScenarios: 0, + declaredScenarioExecutions: 0, + executedScenarioExecutions: 0, }; async function filesBelow(target: string): Promise { @@ -59,34 +110,268 @@ async function filesBelow(target: string): Promise { } } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +function duplicates(values: readonly string[]): string[] { + const seen = new Set(); + const duplicate = new Set(); + for (const value of values) { + if (seen.has(value)) duplicate.add(value); + seen.add(value); + } + return [...duplicate].sort(); } -const sourceFiles = (await filesBelow(sourceRoot)).filter( - (file) => fixtureMode || !file.split(path.sep).includes("fixtures"), -); -for (const file of sourceFiles) { - if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue; - const source = await readFile(file, "utf8"); - facts.scannedFiles += 1; - const skipPattern = - /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g; - if (skipPattern.test(source)) { - const quarantine = - /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/; - if (!quarantine.test(source)) { - failures.push(`${file}: skip/fixme lacks owned expiring quarantine`); +function isSorted(values: readonly string[]): boolean { + return values.every( + (value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0, + ); +} + +async function readJson(target: string): Promise { + return JSON.parse(await readFile(target, "utf8")); +} + +async function loadCatalogExport( + target: string, + exportName: string, +): Promise { + if (target.endsWith(".json")) { + const document = await readJson(target); + return (document as Readonly>)[exportName]; + } + const module = (await import( + `${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}` + )) as Readonly>; + return module[exportName]; +} + +async function checkScenarioContribution( + contribution: z.output, +): Promise { + const catalogPath = explicitCatalogPath ?? contribution.path; + const receiptPath = explicitReceiptPath ?? contribution.receiptPath; + let expectations: readonly HttpScenarioExpectation[]; + try { + expectations = z + .array(httpScenarioExpectationSchema) + .parse( + await loadCatalogExport(catalogPath, contribution.expectationExport), + ); + } catch (error) { + failures.push( + `[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + + facts.declaredScenarioExecutions += expectations.length; + facts.sharedScenarios += new Set( + expectations.map((entry) => entry.scenarioId), + ).size; + const declarationIds = expectations.map((entry) => entry.executionId); + if (declarationIds.length === 0) { + failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`); + } + const duplicateDeclarations = duplicates(declarationIds); + if (duplicateDeclarations.length > 0) { + failures.push( + `[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`, + ); + } + for (const entry of expectations) { + if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) { + failures.push( + `[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`, + ); } } - const wholeUiMask = - /\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i; - if (wholeUiMask.test(source)) { - failures.push(`${file}: screenshot mask may not cover the whole UI`); + + let receipt: HttpScenarioReceipt; + try { + const parsedReceipt = httpScenarioReceiptSchema.safeParse( + await readJson(receiptPath), + ); + if (!parsedReceipt.success) { + if ( + parsedReceipt.error.issues.some((issue) => + issue.message.includes("must be sorted by execution ID"), + ) + ) { + failures.push( + `[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`, + ); + } + if ( + parsedReceipt.error.issues.some((issue) => + issue.message.includes("row IDs must exactly equal executed IDs"), + ) + ) { + failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`); + } + throw parsedReceipt.error; + } + receipt = parsedReceipt.data; + } catch (error) { + failures.push( + `[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + facts.executedScenarioExecutions += receipt.rows.length; + if (receipt.schemaVersion !== contribution.receiptSchemaVersion) { + failures.push( + `[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`, + ); + } + if (receipt.rows.length === 0 || receipt.executedIds.length === 0) { + failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`); + } + if (receipt.catalogTotal !== declarationIds.length) { + failures.push( + `[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`, + ); + } + const duplicateExecutedIds = duplicates(receipt.executedIds); + const rowIds = receipt.rows.map((row) => row.executionId); + const duplicateRows = duplicates(rowIds); + if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) { + failures.push( + `[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`, + ); + } + if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) { + failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`); + } + if (!sameScenarioJson(receipt.executedIds, rowIds)) { + failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`); + } + + const declared = new Set(declarationIds); + const executed = new Set(receipt.executedIds); + const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort(); + const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort(); + if (missing.length > 0) { + failures.push( + `[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`, + ); + } + if (extra.length > 0) { + failures.push( + `[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`, + ); + } + + const expectedDigest = computeHttpScenarioCatalogDigest( + receipt.schemaVersion, + expectations, + ); + if (receipt.catalogDigest !== expectedDigest) { + failures.push( + `[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`, + ); + } + + const declarationsById = new Map( + expectations.map((entry) => [entry.executionId, entry] as const), + ); + for (const row of receipt.rows) { + const declaration = declarationsById.get(row.executionId); + if (!declaration) continue; + if (!sameScenarioJson(row.expected, declaration.expected)) { + failures.push( + `[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`, + ); + } + if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) { + failures.push( + `[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`, + ); + } + if (!sameScenarioJson(row.observed, row.expected)) { + failures.push( + `[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`, + ); + } } } -if (!fixtureMode) { +if (!scenarioOnly) { + const sourceFiles = (await filesBelow(sourceRoot)).filter( + (file) => fixtureMode || !file.split(path.sep).includes("fixtures"), + ); + for (const file of sourceFiles) { + if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue; + const source = await readFile(file, "utf8"); + facts.scannedFiles += 1; + const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g; + if (skipPattern.test(source)) { + const quarantine = + /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/; + if (!quarantine.test(source)) { + failures.push(`${file}: skip/fixme lacks owned expiring quarantine`); + } + } + const wholeUiMask = + /\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i; + if (wholeUiMask.test(source)) { + failures.push(`${file}: screenshot mask may not cover the whole UI`); + } + } +} + +let parsedPolicy: z.output | undefined; +if (!fixtureMode || scenarioOnly) { + try { + parsedPolicy = policySchema.parse(await readJson(policyPath)); + } catch (error) { + failures.push( + `[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +if (parsedPolicy) { + for (const contract of parsedPolicy.sourceContracts) { + let source: string; + try { + source = await readFile(contract.path, "utf8"); + } catch (error) { + failures.push( + `${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`, + ); + continue; + } + for (const token of contract.requiredTokens) { + if (!source.includes(token)) { + failures.push( + `${contract.path}: ${contract.owner} evidence contract is missing ${token}`, + ); + } + } + } + + const shouldCheckScenarioExecutions = + scenarioOnly || + (!sourceOnly && !skipScenarioExecutions && !fixtureMode); + if (shouldCheckScenarioExecutions) { + if (parsedPolicy.scenarioCatalogs.length === 0) { + failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs"); + } + if ( + (explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) && + parsedPolicy.scenarioCatalogs.length !== 1 + ) { + failures.push( + "[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution", + ); + } else { + for (const contribution of parsedPolicy.scenarioCatalogs) { + await checkScenarioContribution(contribution); + } + } + } +} + +if (!fixtureMode && !scenarioOnly) { const e2eConfig = await readFile("playwright.config.ts", "utf8"); for (const token of [ "pnpm build", @@ -113,89 +398,6 @@ if (!fixtureMode) { } } - let evidencePolicy: unknown; - try { - evidencePolicy = JSON.parse(await readFile(policyPath, "utf8")); - } catch (error) { - failures.push( - `${policyPath}: cannot read test evidence policy (${error instanceof Error ? error.message : String(error)})`, - ); - } - const policy = evidencePolicy as Partial | undefined; - if ( - policy?.schemaVersion !== 1 || - !Array.isArray(policy.scenarioCatalogs) || - !Array.isArray(policy.sourceContracts) - ) { - failures.push(`${policyPath}: invalid test evidence policy`); - } else { - for (const candidate of policy.scenarioCatalogs) { - const contribution = candidate as Partial; - const minimumEntries = contribution.minimumEntries; - if ( - typeof contribution?.owner !== "string" || - typeof contribution?.path !== "string" || - typeof contribution?.arrayExport !== "string" || - typeof minimumEntries !== "number" || - !Number.isInteger(minimumEntries) || - minimumEntries < 1 - ) { - failures.push(`${policyPath}: invalid scenario catalog contribution`); - continue; - } - let source: string; - try { - source = await readFile(contribution.path, "utf8"); - } catch (error) { - failures.push( - `${contribution.path}: cannot read scenario catalog (${error instanceof Error ? error.message : String(error)})`, - ); - continue; - } - const arrayPattern = new RegExp( - `${escapeRegExp(contribution.arrayExport)}\\s*=\\s*Object\\.freeze\\(\\[([\\s\\S]*?)\\]\\s*as const\\)`, - ); - const entryCount = ( - arrayPattern.exec(source)?.[1]?.match(/"[^"]+"/g) ?? [] - ).length; - facts.sharedScenarios += entryCount; - if (entryCount < minimumEntries) { - failures.push( - `${contribution.path}: ${contribution.owner} requires at least ${minimumEntries} shared scenarios`, - ); - } - } - for (const candidate of policy.sourceContracts) { - const contract = candidate as Partial; - const requiredTokens = contract.requiredTokens; - if ( - typeof contract?.owner !== "string" || - typeof contract?.path !== "string" || - !Array.isArray(requiredTokens) || - requiredTokens.some((token: unknown) => typeof token !== "string") - ) { - failures.push(`${policyPath}: invalid source contract contribution`); - continue; - } - let source: string; - try { - source = await readFile(contract.path, "utf8"); - } catch (error) { - failures.push( - `${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`, - ); - continue; - } - for (const token of requiredTokens as readonly string[]) { - if (!source.includes(token)) { - failures.push( - `${contract.path}: ${contract.owner} evidence contract is missing ${token}`, - ); - } - } - } - } - const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter( (file) => file.endsWith(".png"), ); @@ -223,14 +425,13 @@ if (!fixtureMode) { } } - const requiredBuiltFiles = [ + for (const required of [ "dist/index.html", "dist/config.json", "dist/release-manifest.json", "dist/runtime-config.schema.json", "dist/.vite/manifest.json", - ]; - for (const required of requiredBuiltFiles) { + ]) { if ((await filesBelow(required)).length === 0) { failures.push(`built-dist contract missing ${required}`); } @@ -245,18 +446,22 @@ if (!fixtureMode) { } const report = { - schemaVersion: 1, + schemaVersion: 2 as const, sourceRoot, - status: failures.length === 0 ? "PASS" : "FAIL", + status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const), facts, failures, }; await mkdir(path.dirname(artifactPath), { recursive: true }); -await writeFile(artifactPath, `${JSON.stringify(report, null, 2)}\n`); +await writeValidatedJsonArtifact({ + path: artifactPath, + schema: reportSchema, + value: report, +}); if (failures.length > 0) { process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`); process.exit(1); } process.stdout.write( - `Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.sharedScenarios} scenarios)\n`, + `Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`, ); diff --git a/scripts/lib/bounded-pnpm-script.ts b/scripts/lib/bounded-pnpm-script.ts new file mode 100644 index 0000000..75929be --- /dev/null +++ b/scripts/lib/bounded-pnpm-script.ts @@ -0,0 +1,52 @@ +import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; + +export const PNPM_SCRIPT_TIMEOUT_MS = 60_000; +export const PNPM_SCRIPT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; + +type BoundedPnpmScriptInvocation = Readonly<{ + command: string; + arguments: readonly string[]; + options: SpawnSyncOptionsWithStringEncoding; +}>; + +type PnpmScriptResult = Readonly<{ + status: number | null; + signal: NodeJS.Signals | null; + error?: Error; +}>; + +export function createBoundedPnpmScriptInvocation(input: Readonly<{ + nodePath: string; + pnpmCli: string; + script: string; + environment: NodeJS.ProcessEnv; +}>): BoundedPnpmScriptInvocation { + return Object.freeze({ + command: input.nodePath, + arguments: Object.freeze([input.pnpmCli, "run", input.script]), + options: Object.freeze({ + encoding: "utf8", + env: input.environment, + killSignal: "SIGTERM", + maxBuffer: PNPM_SCRIPT_MAX_OUTPUT_BYTES, + timeout: PNPM_SCRIPT_TIMEOUT_MS, + }), + }); +} + +export function formatPnpmScriptFailure( + script: string, + result: PnpmScriptResult, +): string { + const code = + result.error && "code" in result.error && + typeof result.error.code === "string" + ? result.error.code + : null; + const message = result.error?.message.trim().replace(/\s+/g, " ") ?? null; + const error = + result.error === undefined + ? "none" + : `${code ?? result.error.name}: ${message || "no message"}`; + return `${script} failed: exit=${String(result.status)}, signal=${result.signal ?? "none"}, error=${error}`; +} diff --git a/scripts/lib/http-scenario-evidence.ts b/scripts/lib/http-scenario-evidence.ts new file mode 100644 index 0000000..59ecbf6 --- /dev/null +++ b/scripts/lib/http-scenario-evidence.ts @@ -0,0 +1,261 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +export type AttemptStatus = number | "NETWORK_REJECTION" | "PENDING_ABORT"; +export type RetryReason = "NETWORK_FAILURE" | "HTTP_429" | "HTTP_503"; +export type BodyDisposition = + | "FULLY_READ_WITHIN_BOUND" + | "CANCELLED_WITHOUT_READ" + | "REJECTED_LIMIT" + | "NO_RESPONSE"; + +const attemptStatusSchema = z.union([ + z.number().int().min(100).max(599), + z.literal("NETWORK_REJECTION"), + z.literal("PENDING_ABORT"), +]); + +export const httpScenarioAssertionGroupsSchema = z + .object({ + status: z + .object({ + attempts: z.array(attemptStatusSchema).min(1), + final: attemptStatusSchema, + }) + .strict(), + outcome: z.object({ kind: z.string().min(1), detail: z.string().nullable() }).strict(), + effect: z.object({ outcome: z.string().min(1), observer: z.string().min(1) }).strict(), + retry: z + .object({ + count: z.number().int().nonnegative(), + reasons: z.array(z.enum(["NETWORK_FAILURE", "HTTP_429", "HTTP_503"])), + }) + .strict(), + fetch: z + .object({ + count: z.number().int().positive(), + observerAttempts: z.number().int().positive(), + agrees: z.boolean(), + }) + .strict(), + media: z.object({ attempts: z.array(z.string().nullable()).min(1), final: z.string().nullable() }).strict(), + body: z + .object({ + attempts: z + .array( + z + .object({ + disposition: z.enum([ + "FULLY_READ_WITHIN_BOUND", + "CANCELLED_WITHOUT_READ", + "REJECTED_LIMIT", + "NO_RESPONSE", + ]), + pulledBytes: z.number().int().nonnegative(), + ceiling: z.number().int().nonnegative(), + }) + .strict(), + ) + .min(1), + }) + .strict(), + scope: z + .object({ + start: z.literal("CURRENT"), + end: z.enum(["CURRENT", "STALE"]), + signal: z.enum(["ACTIVE", "ABORTED"]), + cancellationOwner: z.enum(["NONE", "CALLER", "SCOPE_FENCE", "DEADLINE"]), + }) + .strict(), + }) + .strict() + .superRefine((groups, context) => { + const fail = (path: (string | number)[], message: string) => { + context.addIssue({ code: "custom", path, message }); + }; + if (groups.status.final !== groups.status.attempts.at(-1)) { + fail(["status", "final"], "must equal the final attempt status"); + } + if (groups.retry.count !== groups.retry.reasons.length) { + fail(["retry", "count"], "must equal retry reasons length"); + } + if (groups.fetch.count !== groups.status.attempts.length) { + fail(["fetch", "count"], "must equal status attempts length"); + } + if (groups.fetch.count !== groups.media.attempts.length) { + fail(["media", "attempts"], "must equal fetch count"); + } + if (groups.fetch.count !== groups.body.attempts.length) { + fail(["body", "attempts"], "must equal fetch count"); + } + if (groups.fetch.observerAttempts !== groups.fetch.count) { + fail(["fetch", "observerAttempts"], "must equal physical fetch count"); + } + if (!groups.fetch.agrees) { + fail(["fetch", "agrees"], "must prove physical/observer agreement"); + } + if (groups.media.final !== groups.media.attempts.at(-1)) { + fail(["media", "final"], "must equal the final attempt media essence"); + } + }); + +export type HttpScenarioAssertionGroups = Readonly<{ + status: Readonly<{ attempts: readonly AttemptStatus[]; final: AttemptStatus }>; + outcome: Readonly<{ kind: string; detail: string | null }>; + effect: Readonly<{ outcome: string; observer: string }>; + retry: Readonly<{ count: number; reasons: readonly RetryReason[] }>; + fetch: Readonly<{ + count: number; + observerAttempts: number; + agrees: boolean; + }>; + media: Readonly<{ + attempts: readonly (string | null)[]; + final: string | null; + }>; + body: Readonly<{ + attempts: readonly Readonly<{ + disposition: BodyDisposition; + pulledBytes: number; + ceiling: number; + }>[]; + }>; + scope: Readonly<{ + start: "CURRENT"; + end: "CURRENT" | "STALE"; + signal: "ACTIVE" | "ABORTED"; + cancellationOwner: "NONE" | "CALLER" | "SCOPE_FENCE" | "DEADLINE"; + }>; +}>; + +export const httpScenarioExpectationSchema = z + .object({ + executionId: z.string().min(1), + operationId: z.string().min(1), + scenarioId: z.string().min(1), + expected: httpScenarioAssertionGroupsSchema, + testDeadlineOverrideMs: z.number().int().positive().nullable(), + }) + .strict() + .superRefine((entry, context) => { + if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) { + context.addIssue({ + code: "custom", + path: ["executionId"], + message: "must equal operationId::scenarioId", + }); + } + }); + +export type HttpScenarioExpectation = Readonly<{ + executionId: string; + operationId: string; + scenarioId: string; + expected: HttpScenarioAssertionGroups; + testDeadlineOverrideMs: number | null; +}>; + +export const httpScenarioReceiptRowSchema = z + .object({ + executionId: z.string().min(1), + expected: httpScenarioAssertionGroupsSchema, + observed: httpScenarioAssertionGroupsSchema, + testDeadlineOverrideMs: z.number().int().positive().nullable(), + }) + .strict(); + +export const httpScenarioReceiptSchema = z + .object({ + schemaVersion: z.number().int().positive(), + catalogDigest: z.string().regex(/^sha256:[0-9a-f]{64}$/), + catalogTotal: z.number().int().nonnegative(), + executedIds: z.array(z.string().min(1)), + rows: z.array(httpScenarioReceiptRowSchema), + }) + .strict() + .superRefine((receipt, context) => { + const rowIds = receipt.rows.map((row) => row.executionId); + const sortedIds = [...receipt.executedIds].sort((left, right) => + left.localeCompare(right), + ); + const sortedRowIds = [...rowIds].sort((left, right) => + left.localeCompare(right), + ); + if (!sameScenarioJson(receipt.executedIds, sortedIds)) { + context.addIssue({ + code: "custom", + path: ["executedIds"], + message: "must be sorted by execution ID", + }); + } + if (!sameScenarioJson(rowIds, sortedRowIds)) { + context.addIssue({ + code: "custom", + path: ["rows"], + message: "must be sorted by execution ID", + }); + } + if (!sameScenarioJson(receipt.executedIds, rowIds)) { + context.addIssue({ + code: "custom", + path: ["rows"], + message: "row IDs must exactly equal executed IDs", + }); + } + }); + +export type HttpScenarioReceipt = Readonly<{ + schemaVersion: number; + catalogDigest: string; + catalogTotal: number; + executedIds: readonly string[]; + rows: readonly Readonly<{ + executionId: string; + expected: HttpScenarioAssertionGroups; + observed: HttpScenarioAssertionGroups; + testDeadlineOverrideMs: number | null; + }>[]; +}>; + +export function stableScenarioJson(value: unknown): string { + const normalize = (candidate: unknown): unknown => { + if (Array.isArray(candidate)) return candidate.map(normalize); + if (candidate && typeof candidate === "object") { + return Object.fromEntries( + Object.entries(candidate as Readonly>) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, normalize(nested)]), + ); + } + return candidate; + }; + return JSON.stringify(normalize(value)); +} + +export function sameScenarioJson(left: unknown, right: unknown): boolean { + return stableScenarioJson(left) === stableScenarioJson(right); +} + +export function computeHttpScenarioCatalogDigest( + schemaVersion: number, + expectations: readonly HttpScenarioExpectation[], +): `sha256:${string}` { + const tuples = [...expectations] + .sort((left, right) => left.executionId.localeCompare(right.executionId)) + .map((entry) => [ + entry.executionId, + entry.expected.status, + entry.expected.outcome, + entry.expected.effect, + entry.expected.retry, + entry.expected.fetch, + entry.expected.media, + entry.expected.body, + entry.expected.scope, + entry.testDeadlineOverrideMs, + ]); + return `sha256:${createHash("sha256") + .update(stableScenarioJson([schemaVersion, ...tuples])) + .digest("hex")}`; +} diff --git a/scripts/lib/validated-json-artifact.ts b/scripts/lib/validated-json-artifact.ts index 68bd0e2..853879e 100644 --- a/scripts/lib/validated-json-artifact.ts +++ b/scripts/lib/validated-json-artifact.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; import { open as openFile, rename as renameFile, @@ -15,8 +16,13 @@ export type ValidatedJsonArtifactInput = Readonly<{ }>; export type ValidatedJsonArtifactFileSystem = Readonly<{ - open: (path: string, flags: "wx") => Promise<{ + open: (path: string, flags: number, mode: number) => Promise<{ writeFile(data: string, encoding: "utf8"): Promise; + sync(): Promise; + close(): Promise; + }>; + openDirectory: (path: string) => Promise<{ + sync(): Promise; close(): Promise; }>; rename: (source: string, destination: string) => Promise; @@ -29,11 +35,21 @@ type ValidatedJsonArtifactWriterDependencies = Readonly<{ }>; const defaultFileSystem: ValidatedJsonArtifactFileSystem = Object.freeze({ - open: async (target, flags) => openFile(target, flags), + open: async (target, flags, mode) => openFile(target, flags, mode), + openDirectory: async (target) => openFile(target, constants.O_RDONLY), rename: async (source, destination) => renameFile(source, destination), rm: async (target, options) => removeFile(target, options), }); +function hasErrorCode(error: unknown, code: string): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === code + ); +} + /** * Builds a writer whose only publish operation is an atomic sibling rename. * Dependency injection is limited to the file-system boundary so failure @@ -60,12 +76,20 @@ export function createValidatedJsonArtifactWriter( ); let ownsTemporaryFile = false; try { - const handle = await fileSystem.open(temporaryPath, "wx"); + const handle = await fileSystem.open( + temporaryPath, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); ownsTemporaryFile = true; let writeFailed = false; let writeFailure: unknown; try { await handle.writeFile(`${serialized}\n`, "utf8"); + await handle.sync(); } catch (error) { writeFailed = true; writeFailure = error; @@ -81,6 +105,21 @@ export function createValidatedJsonArtifactWriter( if (writeFailed) throw writeFailure; if (closeFailed) throw closeFailure; await fileSystem.rename(temporaryPath, input.path); + ownsTemporaryFile = false; + const directoryHandle = await fileSystem.openDirectory( + path.dirname(input.path), + ); + try { + try { + await directoryHandle.sync(); + } catch (error) { + if (!hasErrorCode(error, "EINVAL") && !hasErrorCode(error, "ENOTSUP")) { + throw error; + } + } + } finally { + await directoryHandle.close(); + } } catch (error) { if (ownsTemporaryFile) { try { diff --git a/scripts/run-http-scenario-evidence.ts b/scripts/run-http-scenario-evidence.ts new file mode 100644 index 0000000..eca3850 --- /dev/null +++ b/scripts/run-http-scenario-evidence.ts @@ -0,0 +1,72 @@ +import { spawnSync } from "node:child_process"; +import { access, rm } from "node:fs/promises"; + +import { + createBoundedPnpmScriptInvocation, + formatPnpmScriptFailure, +} from "./lib/bounded-pnpm-script.ts"; + +const receiptPath = "artifacts/tests/http-scenario-executions.json"; +const expectedNegativeDiagnostic = + "[TEST-EVIDENCE-EXECUTION-ID-MISSING]"; +const pnpmCli = process.env.npm_execpath; +if (!pnpmCli) throw new Error("npm_execpath is required"); + +await rm(receiptPath, { force: true }); +try { + await access(receiptPath); + throw new Error(`stale scenario receipt remains: ${receiptPath}`); +} catch (error) { + if ( + typeof error !== "object" || + error === null || + !("code" in error) || + error.code !== "ENOENT" + ) { + throw error; + } +} + +runExpectedPass("test:http-scenario-catalog"); +runExpectedPass("check:http-scenario-evidence"); +const negative = run("check:http-scenario-evidence:fixture"); +if ( + negative.status !== 1 || + !`${negative.stdout}\n${negative.stderr}`.includes(expectedNegativeDiagnostic) +) { + process.stderr.write(negative.stdout); + process.stderr.write(negative.stderr); + throw new Error( + `scenario negative fixture identity mismatch: ${formatPnpmScriptFailure( + "check:http-scenario-evidence:fixture", + negative, + )}`, + ); +} + +process.stdout.write( + "HTTP scenario evidence: PASS (stale receipt removed, 54 executed, missing-ID fixture rejected)\n", +); + +function run(script: string) { + const invocation = createBoundedPnpmScriptInvocation({ + nodePath: process.execPath, + pnpmCli: pnpmCli!, + script, + environment: process.env, + }); + return spawnSync( + invocation.command, + [...invocation.arguments], + invocation.options, + ); +} + +function runExpectedPass(script: string): void { + const result = run(script); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + if (result.status !== 0 || result.error !== undefined) { + throw new Error(formatPnpmScriptFailure(script, result)); + } +} diff --git a/scripts/test-sample-removal.ts b/scripts/test-sample-removal.ts index 40d2ee3..3e5d0d9 100644 --- a/scripts/test-sample-removal.ts +++ b/scripts/test-sample-removal.ts @@ -30,6 +30,7 @@ const commonTestPaths = [ const featureOwnedPaths = [ featureSource, featureTests, + "tests/integration/http-scenario-catalog.test.ts", "tests/e2e/reference-form.spec.ts", "tests/e2e/reference-route.spec.ts", "tests/mocks", diff --git a/src/adapters/http/http-execution-v3.ts b/src/adapters/http/http-execution-v3.ts index 190d642..9b53c5f 100644 --- a/src/adapters/http/http-execution-v3.ts +++ b/src/adapters/http/http-execution-v3.ts @@ -603,6 +603,7 @@ export function createContractHttpExecutor( } const kind = owner === "DEADLINE" ? "TIMEOUT" : "NETWORK_FAILURE"; if ( + owner === null && canRetryTransport(contract.retrySemantics, attemptState) && retryIndex < retryCeiling && remaining() > 0 @@ -766,10 +767,10 @@ async function admitResponse( result: Object.freeze({ kind: "RATE_LIMITED" as const, ...(retryAfterMs === null ? {} : { retryAfterMs }), - effect: certaintyForAbandonedAttempt( - attemptState, + effect: normalizeOptionalEffect( + certaintyForAbandonedAttempt(attemptState, isCommand), isCommand, - ) as "NOT_APPLICABLE" | "NOT_APPLIED" | "MAYBE_APPLIED", + ), }), certainty: "RATE_LIMITED", retryHint: true, diff --git a/tests/fixtures/test-evidence/scenarios/catalog.json b/tests/fixtures/test-evidence/scenarios/catalog.json new file mode 100644 index 0000000..73b2ab2 --- /dev/null +++ b/tests/fixtures/test-evidence/scenarios/catalog.json @@ -0,0 +1,36 @@ +{ + "HTTP_SCENARIO_EXPECTATIONS": [ + { + "executionId": "FIXTURE_OPERATION::declared-and-executed", + "operationId": "FIXTURE_OPERATION", + "scenarioId": "declared-and-executed", + "expected": { + "status": { "attempts": [200], "final": 200 }, + "outcome": { "kind": "SUCCESS", "detail": null }, + "effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": true }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "testDeadlineOverrideMs": null + }, + { + "executionId": "FIXTURE_OPERATION::declared-but-unexecuted", + "operationId": "FIXTURE_OPERATION", + "scenarioId": "declared-but-unexecuted", + "expected": { + "status": { "attempts": [503], "final": 503 }, + "outcome": { "kind": "PROBLEM", "detail": "503" }, + "effect": { "outcome": "NOT_APPLIED", "observer": "NOT_STARTED" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": true }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 64, "ceiling": 65536 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "testDeadlineOverrideMs": null + } + ] +} diff --git a/tests/fixtures/test-evidence/scenarios/policy.json b/tests/fixtures/test-evidence/scenarios/policy.json new file mode 100644 index 0000000..97a3d35 --- /dev/null +++ b/tests/fixtures/test-evidence/scenarios/policy.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "scenarioCatalogs": [ + { + "owner": "fixture-owner", + "path": "tests/fixtures/test-evidence/scenarios/catalog.json", + "expectationExport": "HTTP_SCENARIO_EXPECTATIONS", + "receiptPath": "tests/fixtures/test-evidence/scenarios/receipt.json", + "receiptSchemaVersion": 1 + } + ], + "sourceContracts": [] +} diff --git a/tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json b/tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json new file mode 100644 index 0000000..d5a055d --- /dev/null +++ b/tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "catalogDigest": "sha256:87779c47f0042025edf18e148f86c29e5d219f7010cf7effeb69d29059420841", + "catalogTotal": 2, + "executedIds": [ + "FIXTURE_OPERATION::declared-and-executed", + "FIXTURE_OPERATION::declared-but-unexecuted" + ], + "rows": [ + { + "executionId": "FIXTURE_OPERATION::declared-and-executed", + "expected": { + "status": { "attempts": [200], "final": 200 }, + "outcome": { "kind": "SUCCESS", "detail": null }, + "effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": false }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "observed": { + "status": { "attempts": [200], "final": 200 }, + "outcome": { "kind": "SUCCESS", "detail": null }, + "effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": false }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "testDeadlineOverrideMs": null + }, + { + "executionId": "FIXTURE_OPERATION::declared-but-unexecuted", + "expected": { + "status": { "attempts": [503], "final": 503 }, + "outcome": { "kind": "PROBLEM", "detail": "503" }, + "effect": { "outcome": "NOT_APPLIED", "observer": "NOT_STARTED" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": true }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 64, "ceiling": 65536 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "observed": { + "status": { "attempts": [503], "final": 503 }, + "outcome": { "kind": "PROBLEM", "detail": "503" }, + "effect": { "outcome": "NOT_APPLIED", "observer": "NOT_STARTED" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": true }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 64, "ceiling": 65536 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "testDeadlineOverrideMs": null + } + ] +} diff --git a/tests/fixtures/test-evidence/scenarios/receipt.json b/tests/fixtures/test-evidence/scenarios/receipt.json new file mode 100644 index 0000000..ba70d54 --- /dev/null +++ b/tests/fixtures/test-evidence/scenarios/receipt.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": 1, + "catalogDigest": "sha256:87779c47f0042025edf18e148f86c29e5d219f7010cf7effeb69d29059420841", + "catalogTotal": 2, + "executedIds": ["FIXTURE_OPERATION::declared-and-executed"], + "rows": [ + { + "executionId": "FIXTURE_OPERATION::declared-and-executed", + "expected": { + "status": { "attempts": [200], "final": 200 }, + "outcome": { "kind": "SUCCESS", "detail": null }, + "effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": true }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "observed": { + "status": { "attempts": [200], "final": 200 }, + "outcome": { "kind": "SUCCESS", "detail": null }, + "effect": { "outcome": "NOT_APPLICABLE", "observer": "NOT_APPLICABLE" }, + "retry": { "count": 0, "reasons": [] }, + "fetch": { "count": 1, "observerAttempts": 1, "agrees": true }, + "media": { "attempts": ["application/json"], "final": "application/json" }, + "body": { "attempts": [{ "disposition": "FULLY_READ_WITHIN_BOUND", "pulledBytes": 2, "ceiling": 16 }] }, + "scope": { "start": "CURRENT", "end": "CURRENT", "signal": "ACTIVE", "cancellationOwner": "NONE" } + }, + "testDeadlineOverrideMs": null + } + ] +} diff --git a/tests/fixtures/test-evidence/scenarios/source/.gitkeep b/tests/fixtures/test-evidence/scenarios/source/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/http-scenario-catalog.test.ts b/tests/integration/http-scenario-catalog.test.ts new file mode 100644 index 0000000..2c17fb5 --- /dev/null +++ b/tests/integration/http-scenario-catalog.test.ts @@ -0,0 +1,410 @@ +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { createContractHttpExecutor } from "../../src/adapters/http/http-execution-v3.ts"; +import { + HTTP_EXECUTION_CEILINGS, + type InstalledHttpContract, +} from "../../src/contracts/external-contract-runtime.ts"; +import type { CacheScopeSnapshot } from "../../src/contracts/server-state-scope.ts"; +import { REFERENCE_FEATURE_TEMPLATE_CONTRIBUTION } from "../../src/features/reference-feature/contracts/reference-feature-contract-contribution.ts"; +import { + computeHttpScenarioCatalogDigest, + httpScenarioReceiptSchema, + type AttemptStatus, + type BodyDisposition, + type HttpScenarioAssertionGroups, + type RetryReason, +} from "../../scripts/lib/http-scenario-evidence.ts"; +import { writeValidatedJsonArtifact } from "../../scripts/lib/validated-json-artifact.ts"; +import { createReferenceScenarioHandlers } from "../mocks/handlers/reference-resources.ts"; +import { createStrictMockServer } from "../mocks/server.ts"; +import { + HTTP_SCENARIO_EXECUTION_IDS, + HTTP_SCENARIO_EXPECTATIONS, + HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION, + OPERATION_SCENARIO_CATALOG, + type HttpScenarioExpectation, + type HttpScenarioOperationId, +} from "../mocks/scenarios/catalog.ts"; + +const RECEIPT_PATH = path.resolve( + "artifacts/tests/http-scenario-executions.json", +); +const mockApi = createStrictMockServer(); + +beforeAll(mockApi.listen); +afterAll(mockApi.close); + +type AttemptTrace = { + status: AttemptStatus; + media: string | null; + pulledBytes: number; + completed: boolean; + cancelled: boolean; +}; + +function operationFor(operationId: HttpScenarioOperationId) { + const operation = REFERENCE_FEATURE_TEMPLATE_CONTRIBUTION.http.find( + (candidate) => candidate.contract.operationId === operationId, + ); + if (!operation) throw new Error(`Missing operation fixture: ${operationId}`); + return operation as InstalledHttpContract; +} + +function inputFor(operationId: HttpScenarioOperationId): unknown { + if (operationId === "LIST_REFERENCE_RESOURCES") return { limit: 20 }; + if (operationId === "GET_REFERENCE_RESOURCE") { + return { resourceId: "reference-1" }; + } + return { name: "Created" }; +} + +function normalizeMedia(response: Response): string | null { + const value = response.headers.get("content-type"); + return value?.split(";", 1)[0]?.trim().toLowerCase() || null; +} + +function tracingFetcher(attempts: AttemptTrace[]): typeof fetch { + return async (input, init) => { + const trace: AttemptTrace = { + status: "PENDING_ABORT", + media: null, + pulledBytes: 0, + completed: false, + cancelled: false, + }; + attempts.push(trace); + let response: Response; + try { + response = await fetch(input, init); + } catch (error) { + trace.status = init?.signal?.aborted + ? "PENDING_ABORT" + : "NETWORK_REJECTION"; + throw error; + } + trace.status = response.status; + trace.media = normalizeMedia(response); + if (!response.body) return response; + + const reader = response.body.getReader(); + const proxy = new ReadableStream( + { + async pull(controller) { + try { + const chunk = await reader.read(); + if (chunk.done) { + trace.completed = true; + controller.close(); + return; + } + trace.pulledBytes += chunk.value.byteLength; + controller.enqueue(chunk.value); + } catch (error) { + controller.error(error); + } + }, + cancel(reason) { + trace.cancelled = true; + void reader.cancel(reason).catch(() => {}); + }, + }, + { highWaterMark: 0 }, + ); + return new Response(proxy, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + }; +} + +function outcomeDetail(outcome: Readonly>): string | null { + if (outcome.kind === "CONTRACT_VIOLATION") { + return String((outcome.violation as Readonly<{ kind: string }>).kind); + } + if (outcome.kind === "TRANSPORT_FAILURE") { + return String((outcome.failure as Readonly<{ kind: string }>).kind); + } + if (outcome.kind === "PROBLEM") { + return String((outcome.metadata as Readonly<{ status: number }>).status); + } + return null; +} + +function retryReason(status: AttemptStatus): RetryReason { + if (status === "NETWORK_REJECTION") return "NETWORK_FAILURE"; + if (status === 429) return "HTTP_429"; + if (status === 503) return "HTTP_503"; + throw new Error(`Sleep followed a non-retryable attempt: ${String(status)}`); +} + +function bodyDisposition( + trace: AttemptTrace, + outcome: Readonly>, +): BodyDisposition { + if (trace.status === "NETWORK_REJECTION" || trace.status === "PENDING_ABORT") { + return "NO_RESPONSE"; + } + if ( + outcome.kind === "CONTRACT_VIOLATION" && + outcomeDetail(outcome) === "RESPONSE_TOO_LARGE" && + trace.cancelled + ) { + return "REJECTED_LIMIT"; + } + if (trace.completed) return "FULLY_READ_WITHIN_BOUND"; + if (trace.cancelled) return "CANCELLED_WITHOUT_READ"; + throw new Error(`Response body was neither consumed nor cancelled: ${trace.status}`); +} + +function appliedBodyCeiling( + trace: AttemptTrace, + operation: InstalledHttpContract, +): number { + if (typeof trace.status !== "number") return 0; + if (operation.contract.acceptedStatuses.includes(trace.status)) { + return operation.frontend.responseByteLimit; + } + return [404, 409, 422, 503].includes(trace.status) + ? HTTP_EXECUTION_CEILINGS.problemResponseBytes + : 0; +} + +async function waitForHandler( + ready: Promise, + executionId: string, +): Promise { + let watchdog: ReturnType | undefined; + try { + await Promise.race([ + ready, + new Promise((_resolve, reject) => { + watchdog = setTimeout( + () => reject(new Error(`Handler readiness timed out: ${executionId}`)), + 2_000, + ); + }), + ]); + } finally { + if (watchdog !== undefined) clearTimeout(watchdog); + } +} + +async function executeScenario( + entry: HttpScenarioExpectation, +): Promise { + const physicalAttempts: AttemptTrace[] = []; + const sleeps: RetryReason[] = []; + const observations: Array> = []; + const caller = new AbortController(); + const scopeLifetime = new AbortController(); + let scopeCurrent = true; + let handlerReady!: () => void; + const ready = new Promise((resolve) => { + handlerReady = resolve; + }); + const scope: CacheScopeSnapshot = Object.freeze({ + generation: 1, + fingerprint: "scenario-scope-1", + identities: Object.freeze({}) as CacheScopeSnapshot["identities"], + signal: scopeLifetime.signal, + isCurrent: () => scopeCurrent, + }); + const baseOperation = operationFor(entry.operationId); + const operation = + entry.testDeadlineOverrideMs === null + ? baseOperation + : Object.freeze({ + ...baseOperation, + frontend: Object.freeze({ + ...baseOperation.frontend, + totalDeadlineMs: entry.testDeadlineOverrideMs, + }), + }); + const fetcher = tracingFetcher(physicalAttempts); + const executor = createContractHttpExecutor({ + baseUrl: "https://api.test", + maxRetryAttempts: 2, + attachCredentials: () => ({ + kind: "READY", + headers: {}, + credentials: "omit", + }), + fetcher, + random: () => 0, + sleep: async () => { + const prior = physicalAttempts.at(-1); + if (!prior) throw new Error("Retry sleep occurred before an attempt"); + sleeps.push(retryReason(prior.status)); + }, + observe: (observation) => observations.push(observation), + }); + + mockApi.server.use( + ...createReferenceScenarioHandlers({ + scenarios: { [entry.operationId]: entry.scenarioId }, + onAttempt(attempt) { + if ( + attempt.operationId === entry.operationId && + attempt.scenarioId === entry.scenarioId + ) { + handlerReady(); + } + }, + }), + ); + + try { + const execution = executor.execute(operation, inputFor(entry.operationId), { + scope, + signal: caller.signal, + ...(entry.operationId === "CREATE_REFERENCE_RESOURCE" + ? { + intent: Object.freeze({ + intentId: `intent-${entry.scenarioId}`, + operationId: entry.operationId, + canonicalInputIdentity: "scenario-input", + idempotencyKey: `scenario-${entry.scenarioId}`, + createdAtMonotonicMs: 1, + }), + } + : {}), + }); + + if (entry.scenarioId === "timeout" || entry.scenarioId === "aborted") { + await waitForHandler(ready, entry.executionId); + if (entry.scenarioId === "aborted") { + if (entry.operationId === "LIST_REFERENCE_RESOURCES") { + caller.abort(); + } else { + scopeCurrent = false; + scopeLifetime.abort(); + } + } + } + const outcome = (await execution) as unknown as Readonly>; + expect(observations, `${entry.executionId} observer count`).toHaveLength(1); + const observation = observations[0]!; + const observedSignal = scopeLifetime.signal.aborted ? "ABORTED" : "ACTIVE"; + const cancellationOwner = + observation.certainty === "TIMEOUT" + ? "DEADLINE" + : caller.signal.aborted + ? "CALLER" + : scopeLifetime.signal.aborted + ? "SCOPE_FENCE" + : "NONE"; + const observed: HttpScenarioAssertionGroups = Object.freeze({ + status: Object.freeze({ + attempts: Object.freeze(physicalAttempts.map((attempt) => attempt.status)), + final: physicalAttempts.at(-1)!.status, + }), + outcome: Object.freeze({ + kind: String(outcome.kind), + detail: outcomeDetail(outcome), + }), + effect: Object.freeze({ + outcome: String(outcome.effect), + observer: observation.certainty, + }), + retry: Object.freeze({ count: sleeps.length, reasons: Object.freeze(sleeps) }), + fetch: Object.freeze({ + count: physicalAttempts.length, + observerAttempts: observation.attempts, + agrees: physicalAttempts.length === observation.attempts, + }), + media: Object.freeze({ + attempts: Object.freeze(physicalAttempts.map((attempt) => attempt.media)), + final: physicalAttempts.at(-1)?.media ?? null, + }), + body: Object.freeze({ + attempts: Object.freeze( + physicalAttempts.map((attempt) => + Object.freeze({ + disposition: bodyDisposition(attempt, outcome), + pulledBytes: attempt.pulledBytes, + ceiling: appliedBodyCeiling(attempt, operation), + }), + ), + ), + }), + scope: Object.freeze({ + start: "CURRENT", + end: scopeCurrent ? "CURRENT" : "STALE", + signal: observedSignal, + cancellationOwner, + }), + }); + return observed; + } finally { + caller.abort(); + scopeLifetime.abort(); + mockApi.reset(); + } +} + +describe("HTTP scenario catalog execution evidence", () => { + it("declares exactly the 54 behaviorally distinct operation scenarios", () => { + const executionIds = Object.entries(OPERATION_SCENARIO_CATALOG).flatMap( + ([operationId, scenarioIds]) => + scenarioIds.map((scenarioId) => `${operationId}::${scenarioId}`), + ); + + expect(OPERATION_SCENARIO_CATALOG.LIST_REFERENCE_RESOURCES).toHaveLength(19); + expect(OPERATION_SCENARIO_CATALOG.GET_REFERENCE_RESOURCE).toHaveLength(18); + expect(OPERATION_SCENARIO_CATALOG.CREATE_REFERENCE_RESOURCE).toHaveLength(17); + expect(executionIds).toHaveLength(54); + expect(new Set(executionIds)).toHaveLength(54); + expect(executionIds).toEqual(HTTP_SCENARIO_EXECUTION_IDS); + }); + + it("executes every declared scenario and publishes one complete receipt", async () => { + await rm(RECEIPT_PATH, { force: true }); + const rows: Array> = []; + + for (const entry of HTTP_SCENARIO_EXPECTATIONS) { + const observed = await executeScenario(entry); + expect(observed, entry.executionId).toEqual(entry.expected); + rows.push( + Object.freeze({ + executionId: entry.executionId, + expected: entry.expected, + observed, + testDeadlineOverrideMs: entry.testDeadlineOverrideMs, + }), + ); + } + + const sortedRows = [...rows].sort((left, right) => + left.executionId.localeCompare(right.executionId), + ); + await mkdir(path.dirname(RECEIPT_PATH), { recursive: true }); + await writeValidatedJsonArtifact({ + path: RECEIPT_PATH, + schema: httpScenarioReceiptSchema, + value: { + schemaVersion: HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION, + catalogDigest: computeHttpScenarioCatalogDigest( + HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION, + HTTP_SCENARIO_EXPECTATIONS, + ), + catalogTotal: 54, + executedIds: sortedRows.map((row) => row.executionId), + rows: sortedRows, + }, + }); + }, 30_000); +}); diff --git a/tests/mocks/handlers/reference-resources.ts b/tests/mocks/handlers/reference-resources.ts index e8c3c12..48c960c 100644 --- a/tests/mocks/handlers/reference-resources.ts +++ b/tests/mocks/handlers/reference-resources.ts @@ -1,6 +1,9 @@ import { delay, http, HttpResponse, type JsonBodyType } from "msw"; -import type { HttpScenarioId } from "../scenarios/catalog.ts"; +import type { + HttpScenarioId, + HttpScenarioOperationId, +} from "../scenarios/catalog.ts"; import { assertOperationScenario } from "../scenarios/catalog.ts"; export type ReferenceResourceFixture = Readonly<{ @@ -20,6 +23,11 @@ type ScenarioOptions = Readonly<{ > >; resources?: ReferenceResourceFixture[]; + onAttempt?(attempt: Readonly<{ + operationId: HttpScenarioOperationId; + scenarioId: HttpScenarioId; + attempt: number; + }>): void; onList?(search: string): void; onCreate?(body: Readonly>): void; }>; @@ -31,12 +39,15 @@ const DEFAULT_RESOURCE = Object.freeze({ }); async function scenarioResponse( + operationId: HttpScenarioOperationId, scenario: HttpScenarioId, payload: JsonBodyType, attempt: number, ) { if (scenario === "slow") await delay(50); - if (scenario === "timeout") await delay(30_000); + if (scenario === "timeout" || scenario === "aborted") { + await delay("infinite"); + } if (scenario === "network-error") return HttpResponse.error(); if (scenario === "content-type-mismatch") { return new HttpResponse("not json", { @@ -55,8 +66,7 @@ async function scenarioResponse( return HttpResponse.json({ unexpected: true }); } if ( - scenario === "auth-persistent-401" || - (scenario === "auth-recover-once" && attempt === 1) + scenario === "unauthenticated-401" ) { return HttpResponse.json(problem(401, "AUTH_REQUIRED"), { status: 401, @@ -84,13 +94,34 @@ async function scenarioResponse( }); } if ( - scenario === "server-terminal-500" || + scenario === "server-terminal-503" || (scenario === "server-retry-success" && attempt === 1) ) { return HttpResponse.json(problem(503, "SERVER_FAILURE"), { status: 503, }); } + if (scenario === "response-too-large") { + const ceiling = + operationId === "LIST_REFERENCE_RESOURCES" ? 262_144 : 32_768; + const bytes = new Uint8Array(ceiling + 1).fill(0x20); + bytes[0] = 0x5b; + bytes[bytes.length - 1] = 0x5d; + return new HttpResponse( + new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }), + { + headers: { + "Content-Type": "application/json", + "Content-Length": String(bytes.byteLength), + }, + }, + ); + } return HttpResponse.json(payload); } @@ -125,10 +156,17 @@ export function createReferenceScenarioHandlers(options: ScenarioOptions = {}) { options.onList?.(new URL(request.url).search); const scenario = scenarioFor("LIST_REFERENCE_RESOURCES"); const payload = scenario === "empty" ? [] : resources; + const attempt = nextAttempt("LIST_REFERENCE_RESOURCES"); + options.onAttempt?.({ + operationId: "LIST_REFERENCE_RESOURCES", + scenarioId: scenario, + attempt, + }); return scenarioResponse( + "LIST_REFERENCE_RESOURCES", scenario, payload, - nextAttempt("LIST_REFERENCE_RESOURCES"), + attempt, ); }), http.post( @@ -149,10 +187,17 @@ export function createReferenceScenarioHandlers(options: ScenarioOptions = {}) { createdAt: "2026-07-26T00:00:00.000Z", }; if (scenario === "success") resources.push(created); + const attempt = nextAttempt("CREATE_REFERENCE_RESOURCE"); + options.onAttempt?.({ + operationId: "CREATE_REFERENCE_RESOURCE", + scenarioId: scenario, + attempt, + }); return scenarioResponse( + "CREATE_REFERENCE_RESOURCE", scenario, created, - nextAttempt("CREATE_REFERENCE_RESOURCE"), + attempt, ); }, ), @@ -163,10 +208,17 @@ export function createReferenceScenarioHandlers(options: ScenarioOptions = {}) { const resource = resources.find((entry) => entry.id === params.resourceId) ?? DEFAULT_RESOURCE; + const attempt = nextAttempt("GET_REFERENCE_RESOURCE"); + options.onAttempt?.({ + operationId: "GET_REFERENCE_RESOURCE", + scenarioId: scenario, + attempt, + }); return scenarioResponse( + "GET_REFERENCE_RESOURCE", scenario, resource, - nextAttempt("GET_REFERENCE_RESOURCE"), + attempt, ); }, ), diff --git a/tests/mocks/scenarios/catalog.ts b/tests/mocks/scenarios/catalog.ts index e9c3e58..3e8abaa 100644 --- a/tests/mocks/scenarios/catalog.ts +++ b/tests/mocks/scenarios/catalog.ts @@ -1,3 +1,19 @@ +import type { + AttemptStatus, + BodyDisposition, + HttpScenarioAssertionGroups, + RetryReason, +} from "../../../scripts/lib/http-scenario-evidence.ts"; + +export type { + AttemptStatus, + BodyDisposition, + HttpScenarioAssertionGroups, + RetryReason, +} from "../../../scripts/lib/http-scenario-evidence.ts"; + +export const HTTP_SCENARIO_RECEIPT_SCHEMA_VERSION = 1 as const; + export const HTTP_SCENARIO_IDS = Object.freeze([ "success", "empty", @@ -9,37 +25,411 @@ export const HTTP_SCENARIO_IDS = Object.freeze([ "malformed-json", "envelope-mismatch", "schema-mismatch", - "auth-recover-once", - "auth-persistent-401", + "response-too-large", + "unauthenticated-401", "forbidden-403", "not-found-404", "conflict-409", "validation-422", "rate-limited-429", "server-retry-success", - "server-terminal-500", + "server-terminal-503", ] as const); export type HttpScenarioId = (typeof HTTP_SCENARIO_IDS)[number]; -export const OPERATION_SCENARIO_CATALOG = Object.freeze({ +export const HTTP_SCENARIO_OPERATION_IDS = Object.freeze([ + "LIST_REFERENCE_RESOURCES", + "GET_REFERENCE_RESOURCE", + "CREATE_REFERENCE_RESOURCE", +] as const); + +export type HttpScenarioOperationId = + (typeof HTTP_SCENARIO_OPERATION_IDS)[number]; +export type HttpScenarioExecutionId = + `${HttpScenarioOperationId}::${HttpScenarioId}`; + +export type HttpScenarioExpectation = Readonly<{ + executionId: HttpScenarioExecutionId; + operationId: HttpScenarioOperationId; + scenarioId: HttpScenarioId; + expected: HttpScenarioAssertionGroups; + testDeadlineOverrideMs: number | null; +}>; + +const encoder = new TextEncoder(); +const jsonBytes = (value: unknown) => encoder.encode(JSON.stringify(value)).byteLength; + +const RESOURCE = Object.freeze({ + id: "reference-1", + name: "Reference", + createdAt: "2026-07-26T00:00:00.000Z", +}); +const CREATED = Object.freeze({ + id: "reference-created", + name: "Created", + createdAt: "2026-07-26T00:00:00.000Z", +}); + +function operationFacts(operationId: HttpScenarioOperationId) { + const command = operationId === "CREATE_REFERENCE_RESOURCE"; + return Object.freeze({ + command, + ceiling: + operationId === "LIST_REFERENCE_RESOURCES" ? 262_144 : 32_768, + payload: + operationId === "LIST_REFERENCE_RESOURCES" + ? [RESOURCE] + : operationId === "GET_REFERENCE_RESOURCE" + ? RESOURCE + : CREATED, + }); +} + +function problem(status: number, code: string) { + return { + type: `https://api.test/problems/${code.toLowerCase()}`, + title: code, + status, + code, + }; +} + +function attemptBody( + disposition: BodyDisposition, + pulledBytes: number, + ceiling: number, +) { + return Object.freeze({ disposition, pulledBytes, ceiling }); +} + +function expectationFor( + operationId: HttpScenarioOperationId, + scenarioId: HttpScenarioId, +): HttpScenarioExpectation { + const { command, ceiling, payload } = operationFacts(operationId); + let statuses: AttemptStatus[] = [200]; + let kind = "SUCCESS"; + let detail: string | null = null; + let outcomeEffect = command ? "APPLIED_CONFIRMED" : "NOT_APPLICABLE"; + let observer = outcomeEffect; + let retryReasons: RetryReason[] = []; + let media: (string | null)[] = ["application/json"]; + let bodies = [attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes(payload), ceiling)]; + let scopeEnd: "CURRENT" | "STALE" = "CURRENT"; + let scopeSignal: "ACTIVE" | "ABORTED" = "ACTIVE"; + let cancellationOwner: "NONE" | "CALLER" | "SCOPE_FENCE" | "DEADLINE" = + "NONE"; + let testDeadlineOverrideMs: number | null = null; + + switch (scenarioId) { + case "success": + case "slow": + break; + case "empty": + bodies = [attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes([]), ceiling)]; + break; + case "network-error": { + const attemptCount = command ? 1 : 3; + statuses = Array.from({ length: attemptCount }, () => "NETWORK_REJECTION"); + kind = "TRANSPORT_FAILURE"; + detail = "NETWORK_FAILURE"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_STARTED"; + observer = "NETWORK_FAILURE"; + retryReasons = command ? [] : ["NETWORK_FAILURE", "NETWORK_FAILURE"]; + media = Array.from({ length: attemptCount }, () => null); + bodies = Array.from({ length: attemptCount }, () => + attemptBody("NO_RESPONSE", 0, 0), + ); + break; + } + case "timeout": + statuses = ["PENDING_ABORT"]; + kind = "TRANSPORT_FAILURE"; + detail = "TIMEOUT"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_STARTED"; + observer = "TIMEOUT"; + media = [null]; + bodies = [attemptBody("NO_RESPONSE", 0, 0)]; + cancellationOwner = "DEADLINE"; + testDeadlineOverrideMs = 500; + break; + case "aborted": + statuses = ["PENDING_ABORT"]; + media = [null]; + bodies = [attemptBody("NO_RESPONSE", 0, 0)]; + if (operationId === "LIST_REFERENCE_RESOURCES") { + kind = "CANCELLED"; + outcomeEffect = "NOT_STARTED"; + observer = "CANCELLED"; + cancellationOwner = "CALLER"; + } else { + scopeSignal = "ABORTED"; + kind = "TRANSPORT_FAILURE"; + detail = "ABORTED_BY_SCOPE"; + outcomeEffect = "NOT_STARTED"; + observer = "SCOPE_FENCED"; + scopeEnd = "STALE"; + cancellationOwner = "SCOPE_FENCE"; + } + break; + case "content-type-mismatch": + kind = "CONTRACT_VIOLATION"; + detail = "CONTENT_TYPE_MISMATCH"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "CONTRACT_VIOLATION"; + media = ["text/html"]; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + encoder.encode("not json").byteLength, + ceiling, + ), + ]; + break; + case "malformed-json": + kind = "CONTRACT_VIOLATION"; + detail = "JSON_INVALID"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "CONTRACT_VIOLATION"; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + encoder.encode("{invalid").byteLength, + ceiling, + ), + ]; + break; + case "envelope-mismatch": + kind = "CONTRACT_VIOLATION"; + detail = "SUCCESS_SCHEMA_INVALID"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "CONTRACT_VIOLATION"; + bodies = [ + attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes({ data: payload }), ceiling), + ]; + break; + case "schema-mismatch": + kind = "CONTRACT_VIOLATION"; + detail = "SUCCESS_SCHEMA_INVALID"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "CONTRACT_VIOLATION"; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes({ unexpected: true }), + ceiling, + ), + ]; + break; + case "response-too-large": + kind = "CONTRACT_VIOLATION"; + detail = "RESPONSE_TOO_LARGE"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "RESPONSE_TOO_LARGE"; + bodies = [attemptBody("REJECTED_LIMIT", 0, ceiling)]; + break; + case "unauthenticated-401": + statuses = [401]; + kind = "UNAUTHENTICATED"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "UNAUTHENTICATED"; + bodies = [attemptBody("CANCELLED_WITHOUT_READ", 0, 0)]; + break; + case "forbidden-403": + statuses = [403]; + kind = "FORBIDDEN"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "FORBIDDEN"; + bodies = [attemptBody("CANCELLED_WITHOUT_READ", 0, 0)]; + break; + case "not-found-404": + statuses = [404]; + kind = "PROBLEM"; + detail = "404"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLIED"; + observer = command ? "MAYBE_APPLIED" : "NOT_STARTED"; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes(problem(404, "NOT_FOUND")), + 65_536, + ), + ]; + break; + case "conflict-409": + statuses = [409]; + kind = "PROBLEM"; + detail = "409"; + outcomeEffect = "NOT_APPLIED"; + observer = command ? "NOT_APPLIED" : "NOT_STARTED"; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes(problem(409, "CONFLICT")), + 65_536, + ), + ]; + break; + case "validation-422": + statuses = [422]; + kind = "PROBLEM"; + detail = "422"; + outcomeEffect = "NOT_APPLIED"; + observer = command ? "NOT_APPLIED" : "NOT_STARTED"; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes(problem(422, "VALIDATION_REJECTED")), + 65_536, + ), + ]; + break; + case "rate-limited-429": { + const attemptCount = command ? 1 : 3; + statuses = Array.from({ length: attemptCount }, () => 429); + kind = "RATE_LIMITED"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLICABLE"; + observer = "RATE_LIMITED"; + retryReasons = command ? [] : ["HTTP_429", "HTTP_429"]; + media = Array.from({ length: attemptCount }, () => "application/json"); + bodies = Array.from({ length: attemptCount }, () => + attemptBody("CANCELLED_WITHOUT_READ", 0, 0), + ); + break; + } + case "server-retry-success": + if (command) { + statuses = [503]; + kind = "PROBLEM"; + detail = "503"; + outcomeEffect = "MAYBE_APPLIED"; + observer = "MAYBE_APPLIED"; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes(problem(503, "SERVER_FAILURE")), + 65_536, + ), + ]; + } else { + statuses = [503, 200]; + retryReasons = ["HTTP_503"]; + media = ["application/json", "application/json"]; + bodies = [ + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes(problem(503, "SERVER_FAILURE")), + 65_536, + ), + attemptBody("FULLY_READ_WITHIN_BOUND", jsonBytes(payload), ceiling), + ]; + } + break; + case "server-terminal-503": { + const attemptCount = command ? 1 : 3; + statuses = Array.from({ length: attemptCount }, () => 503); + kind = "PROBLEM"; + detail = "503"; + outcomeEffect = command ? "MAYBE_APPLIED" : "NOT_APPLIED"; + observer = command ? "MAYBE_APPLIED" : "NOT_STARTED"; + retryReasons = command ? [] : ["HTTP_503", "HTTP_503"]; + media = Array.from({ length: attemptCount }, () => "application/json"); + bodies = Array.from({ length: attemptCount }, () => + attemptBody( + "FULLY_READ_WITHIN_BOUND", + jsonBytes(problem(503, "SERVER_FAILURE")), + 65_536, + ), + ); + break; + } + } + + const executionId = `${operationId}::${scenarioId}` as const; + return Object.freeze({ + executionId, + operationId, + scenarioId, + expected: Object.freeze({ + status: Object.freeze({ + attempts: Object.freeze(statuses), + final: statuses.at(-1)!, + }), + outcome: Object.freeze({ kind, detail }), + effect: Object.freeze({ outcome: outcomeEffect, observer }), + retry: Object.freeze({ + count: retryReasons.length, + reasons: Object.freeze(retryReasons), + }), + fetch: Object.freeze({ + count: statuses.length, + observerAttempts: statuses.length, + agrees: true, + }), + media: Object.freeze({ + attempts: Object.freeze(media), + final: media.at(-1) ?? null, + }), + body: Object.freeze({ attempts: Object.freeze(bodies) }), + scope: Object.freeze({ + start: "CURRENT" as const, + end: scopeEnd, + signal: scopeSignal, + cancellationOwner, + }), + }), + testDeadlineOverrideMs, + }); +} + +const OPERATION_SCENARIOS = Object.freeze({ LIST_REFERENCE_RESOURCES: HTTP_SCENARIO_IDS, + GET_REFERENCE_RESOURCE: Object.freeze( + HTTP_SCENARIO_IDS.filter((scenario) => scenario !== "empty"), + ), CREATE_REFERENCE_RESOURCE: Object.freeze( HTTP_SCENARIO_IDS.filter( - (scenario) => !["empty", "aborted"].includes(scenario), + (scenario) => scenario !== "empty" && scenario !== "aborted", ), ), - GET_REFERENCE_RESOURCE: HTTP_SCENARIO_IDS, -} satisfies Readonly>); +} satisfies Readonly< + Record +>); + +export const HTTP_SCENARIO_EXPECTATIONS = Object.freeze( + HTTP_SCENARIO_OPERATION_IDS.flatMap((operationId) => + OPERATION_SCENARIOS[operationId].map((scenarioId) => + expectationFor(operationId, scenarioId), + ), + ), +); + +export const HTTP_SCENARIO_EXECUTION_IDS = Object.freeze( + HTTP_SCENARIO_EXPECTATIONS.map((expectation) => expectation.executionId), +); + +export const OPERATION_SCENARIO_CATALOG = Object.freeze( + Object.fromEntries( + HTTP_SCENARIO_OPERATION_IDS.map((operationId) => [ + operationId, + Object.freeze( + HTTP_SCENARIO_EXPECTATIONS.filter( + (expectation) => expectation.operationId === operationId, + ).map((expectation) => expectation.scenarioId), + ), + ]), + ) as Readonly< + Record + >, +); export function assertOperationScenario( - operationId: keyof typeof OPERATION_SCENARIO_CATALOG, + operationId: HttpScenarioOperationId, scenario: HttpScenarioId, ) { if (!OPERATION_SCENARIO_CATALOG[operationId].includes(scenario)) { - throw new Error( - `Scenario ${scenario} is not declared for ${operationId}`, - ); + throw new Error(`Scenario ${scenario} is not declared for ${operationId}`); } return scenario; } diff --git a/tests/unit/http-execution-v3.test.ts b/tests/unit/http-execution-v3.test.ts index 5882adf..555f913 100644 --- a/tests/unit/http-execution-v3.test.ts +++ b/tests/unit/http-execution-v3.test.ts @@ -65,6 +65,30 @@ async function flushMicrotasks(): Promise { } describe("descriptor-driven HTTP execution lifetime", () => { + it("normalizes a read-side 429 to the non-applicable effect vocabulary", async () => { + const executor = createContractHttpExecutor({ + baseUrl: "https://api.example/", + maxRetryAttempts: 0, + attachCredentials: () => ({ + kind: "READY", + headers: {}, + credentials: "omit", + }), + fetcher: async () => + Response.json( + { type: "about:blank", title: "limited", status: 429 }, + { status: 429 }, + ), + }); + + await expect( + executor.execute(installed, { limit: 20 }, { scope }), + ).resolves.toMatchObject({ + kind: "RATE_LIMITED", + effect: "NOT_APPLICABLE", + }); + }); + it.each([ ["absent intent", () => undefined], ["empty key", () => mutationIntent({ idempotencyKey: "" })], @@ -471,6 +495,67 @@ describe("descriptor-driven HTTP execution lifetime", () => { vi.useRealTimers(); }); + it("never retries a deadline-owned abort while the monotonic clock still has a sub-tick budget", async () => { + vi.useFakeTimers(); + try { + const iterationCount = 100; + const sleep = vi.fn(async () => { + throw new Error("a deadline-owned abort must not enter retry sleep"); + }); + const fetcher = vi.fn( + (_input: RequestInfo | URL, init?: RequestInit) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(new DOMException("Aborted", "AbortError")), + { once: true }, + ); + }), + ); + const observe = vi.fn(); + const executor = createContractHttpExecutor({ + baseUrl: "https://api.example/", + maxRetryAttempts: 2, + attachCredentials: () => ({ + kind: "READY", + headers: {}, + credentials: "omit", + }), + fetcher, + monotonicNow: () => 0, + random: () => 0, + sleep, + observe, + }); + + for (let iteration = 0; iteration < iterationCount; iteration += 1) { + const result = executor.execute( + operation({ deadlineMs: 5 }), + { limit: 20 }, + { scope }, + ); + await vi.advanceTimersByTimeAsync(5); + await flushMicrotasks(); + + await expect(result).resolves.toMatchObject({ + kind: "TRANSPORT_FAILURE", + failure: { kind: "TIMEOUT" }, + }); + } + + expect(fetcher).toHaveBeenCalledTimes(iterationCount); + expect(sleep).not.toHaveBeenCalled(); + expect(observe).toHaveBeenCalledTimes(iterationCount); + for (const [observation] of observe.mock.calls) { + expect(observation).toEqual( + expect.objectContaining({ attempts: 1, certainty: "TIMEOUT" }), + ); + } + } finally { + vi.useRealTimers(); + } + }); + it("cancels a non-cooperative retry sleep when the caller aborts", async () => { const caller = new AbortController(); let sleepSignal: AbortSignal | undefined; diff --git a/tests/unit/http-scenario-evidence.test.ts b/tests/unit/http-scenario-evidence.test.ts new file mode 100644 index 0000000..360bfd3 --- /dev/null +++ b/tests/unit/http-scenario-evidence.test.ts @@ -0,0 +1,294 @@ +import { spawnSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + createBoundedPnpmScriptInvocation, + formatPnpmScriptFailure, +} from "../../scripts/lib/bounded-pnpm-script.ts"; +import { + computeHttpScenarioCatalogDigest, + httpScenarioExpectationSchema, + httpScenarioReceiptSchema, + type HttpScenarioAssertionGroups, + type HttpScenarioExpectation, +} from "../../scripts/lib/http-scenario-evidence.ts"; + +type MutableReceipt = { + schemaVersion: number; + catalogDigest: string; + catalogTotal: number; + executedIds: string[]; + rows: Array<{ + executionId: string; + expected: HttpScenarioAssertionGroups; + observed: HttpScenarioAssertionGroups; + testDeadlineOverrideMs: number | null; + }>; +}; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + rm(directory, { recursive: true, force: true }), + ), + ); +}); + +async function validFixture() { + const source = JSON.parse( + await readFile( + "tests/fixtures/test-evidence/scenarios/catalog.json", + "utf8", + ), + ) as Readonly>; + const expectations = httpScenarioExpectationSchema + .array() + .parse(source.HTTP_SCENARIO_EXPECTATIONS) as HttpScenarioExpectation[]; + const rows = expectations + .map((entry) => ({ + executionId: entry.executionId, + expected: structuredClone(entry.expected), + observed: structuredClone(entry.expected), + testDeadlineOverrideMs: entry.testDeadlineOverrideMs, + })) + .sort((left, right) => left.executionId.localeCompare(right.executionId)); + const receipt: MutableReceipt = { + schemaVersion: 1, + catalogDigest: computeHttpScenarioCatalogDigest(1, expectations), + catalogTotal: expectations.length, + executedIds: rows.map((row) => row.executionId), + rows, + }; + return { expectations, receipt }; +} + +async function runFixture( + expectations: readonly HttpScenarioExpectation[], + receipt: MutableReceipt, +) { + await mkdir(".tmp", { recursive: true }); + const directory = await mkdtemp(path.resolve(".tmp/http-scenario-evidence-")); + temporaryDirectories.push(directory); + const sourceRoot = path.join(directory, "source"); + const catalogPath = path.join(directory, "catalog.json"); + const receiptPath = path.join(directory, "receipt.json"); + const policyPath = path.join(directory, "policy.json"); + const artifactPath = path.join(directory, "report.json"); + await mkdir(sourceRoot); + await writeFile( + catalogPath, + `${JSON.stringify({ HTTP_SCENARIO_EXPECTATIONS: expectations }, null, 2)}\n`, + ); + await writeFile(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`); + await writeFile( + policyPath, + `${JSON.stringify( + { + schemaVersion: 2, + scenarioCatalogs: [ + { + owner: "mutation-fixture", + path: catalogPath, + expectationExport: "HTTP_SCENARIO_EXPECTATIONS", + receiptPath, + receiptSchemaVersion: 1, + }, + ], + sourceContracts: [], + }, + null, + 2, + )}\n`, + ); + return spawnSync( + process.execPath, + [ + "scripts/check-test-evidence.ts", + "--scenario-only", + "--source-root", + sourceRoot, + "--policy", + policyPath, + "--catalog", + catalogPath, + "--receipt", + receiptPath, + "--artifact", + artifactPath, + ], + { encoding: "utf8", cwd: process.cwd() }, + ); +} + +describe("HTTP scenario evidence receipt schema", () => { + it("bounds every orchestration child by time and captured output", () => { + const environment = { CI: "true" }; + + expect( + createBoundedPnpmScriptInvocation({ + nodePath: "/runtime/node", + pnpmCli: "/runtime/pnpm.cjs", + script: "test:http-scenario-catalog", + environment, + }), + ).toEqual({ + command: "/runtime/node", + arguments: [ + "/runtime/pnpm.cjs", + "run", + "test:http-scenario-catalog", + ], + options: { + encoding: "utf8", + env: environment, + killSignal: "SIGTERM", + maxBuffer: 16 * 1024 * 1024, + timeout: 60_000, + }, + }); + }); + + it("preserves timeout code and termination signal in child diagnostics", () => { + const error = Object.assign(new Error("spawn timed out"), { + code: "ETIMEDOUT", + }); + + expect( + formatPnpmScriptFailure("test:http-scenario-catalog", { + status: null, + signal: "SIGTERM", + error, + }), + ).toBe( + "test:http-scenario-catalog failed: exit=null, signal=SIGTERM, error=ETIMEDOUT: spawn timed out", + ); + }); + + it("rejects internally inconsistent assertion groups as schema drift", async () => { + const receipt = JSON.parse( + await readFile( + "tests/fixtures/test-evidence/scenarios/receipt-semantic-invalid.json", + "utf8", + ), + ); + + const result = httpScenarioReceiptSchema.safeParse(receipt); + + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: ["rows", 0, "expected", "fetch", "agrees"], + }), + expect.objectContaining({ + path: ["rows", 0, "observed", "fetch", "agrees"], + }), + ]), + ); + } + }); + + it.each([ + { + label: "zero declarations and executions", + diagnostic: "[TEST-EVIDENCE-CATALOG-ZERO]", + mutate(expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + expectations.splice(0); + receipt.catalogTotal = 0; + receipt.executedIds.splice(0); + receipt.rows.splice(0); + }, + }, + { + label: "a missing execution", + diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-MISSING]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + receipt.executedIds.pop(); + receipt.rows.pop(); + }, + }, + { + label: "an extra execution", + diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-EXTRA]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + const extra = structuredClone(receipt.rows.at(-1)!); + extra.executionId = "FIXTURE_OPERATION::zz-extra"; + receipt.executedIds.push(extra.executionId); + receipt.rows.push(extra); + }, + }, + { + label: "a duplicate execution", + diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + receipt.executedIds.splice(1, 0, receipt.executedIds[0]!); + receipt.rows.splice(1, 0, structuredClone(receipt.rows[0]!)); + }, + }, + { + label: "digest drift", + diagnostic: "[TEST-EVIDENCE-CATALOG-DIGEST]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + receipt.catalogDigest = `sha256:${"0".repeat(64)}`; + }, + preserveDigest: true, + }, + { + label: "unsorted execution rows", + diagnostic: "[TEST-EVIDENCE-EXECUTION-ID-UNSORTED]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + receipt.executedIds.reverse(); + receipt.rows.reverse(); + }, + }, + { + label: "expected value drift", + diagnostic: "[TEST-EVIDENCE-EXPECTED-DRIFT]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + const changed = { + ...receipt.rows[0]!.expected, + outcome: { kind: "FORBIDDEN", detail: null }, + }; + receipt.rows[0]!.expected = changed; + receipt.rows[0]!.observed = structuredClone(changed); + }, + }, + { + label: "observed value mismatch", + diagnostic: "[TEST-EVIDENCE-OBSERVATION-MISMATCH]", + mutate(_expectations: HttpScenarioExpectation[], receipt: MutableReceipt) { + receipt.rows[0]!.observed = { + ...receipt.rows[0]!.observed, + outcome: { kind: "FORBIDDEN", detail: null }, + }; + }, + }, + ])("rejects $label with its stable diagnostic", async (fixture) => { + const { expectations, receipt } = await validFixture(); + fixture.mutate(expectations, receipt); + if (!fixture.preserveDigest) { + receipt.catalogDigest = computeHttpScenarioCatalogDigest( + receipt.schemaVersion, + expectations, + ); + } + + const result = await runFixture(expectations, receipt); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(`${result.stdout}\n${result.stderr}`).toContain(fixture.diagnostic); + }); +}); diff --git a/tests/unit/validated-json-artifact.test.ts b/tests/unit/validated-json-artifact.test.ts index f17a2ad..47b94ef 100644 --- a/tests/unit/validated-json-artifact.test.ts +++ b/tests/unit/validated-json-artifact.test.ts @@ -7,6 +7,7 @@ import { rm, writeFile, } from "node:fs/promises"; +import { constants } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -41,6 +42,67 @@ afterEach(async () => { }); describe("validated JSON artifact writer", () => { + it("syncs an O_NOFOLLOW exclusive temp and its directory around rename", async () => { + const events: string[] = []; + let openFlags = 0; + let openMode = 0; + const writer = createValidatedJsonArtifactWriter({ + createNonce: () => "durable", + fileSystem: { + open: async (_target, flags, mode) => { + openFlags = flags; + openMode = mode; + events.push(`open:${flags}`); + return { + writeFile: async () => { + events.push("write"); + }, + sync: async () => { + events.push("file-sync"); + }, + close: async () => { + events.push("file-close"); + }, + }; + }, + openDirectory: async () => ({ + sync: async () => { + events.push("directory-sync"); + }, + close: async () => { + events.push("directory-close"); + }, + }), + rename: async () => { + events.push("rename"); + }, + rm: async () => {}, + }, + }); + + await writer({ + path: "/tmp/report.json", + schema: artifactSchema, + value: { schemaVersion: 1, name: "valid" }, + }); + + expect(events).toHaveLength(7); + expect(events[0]).toMatch(/^open:\d+$/); + expect(openFlags & constants.O_WRONLY).toBe(constants.O_WRONLY); + expect(openFlags & constants.O_CREAT).toBe(constants.O_CREAT); + expect(openFlags & constants.O_EXCL).toBe(constants.O_EXCL); + expect(openFlags & constants.O_NOFOLLOW).toBe(constants.O_NOFOLLOW); + expect(openMode).toBe(0o600); + expect(events.slice(1)).toEqual([ + "write", + "file-sync", + "file-close", + "rename", + "directory-sync", + "directory-close", + ]); + }); + it.each(["existing", "missing"] as const)( "rejects invalid %s artifacts before changing destination state", async (destinationState) => { @@ -123,12 +185,20 @@ describe("validated JSON artifact writer", () => { return { writeFile: async (data, encoding) => handle.writeFile(data, encoding), + sync: async () => handle.sync(), close: async () => { await handle.close(); throw closeError; }, }; }, + openDirectory: async (target) => { + const handle = await open(target, constants.O_RDONLY); + return { + sync: async () => handle.sync(), + close: async () => handle.close(), + }; + }, rename, rm, }, @@ -165,12 +235,20 @@ describe("validated JSON artifact writer", () => { await handle.writeFile(data, encoding); throw writeError; }, + sync: async () => handle.sync(), close: async () => { await handle.close(); throw closeError; }, }; }, + openDirectory: async (target) => { + const handle = await open(target, constants.O_RDONLY); + return { + sync: async () => handle.sync(), + close: async () => handle.close(), + }; + }, rename, rm, }, @@ -244,6 +322,14 @@ describe("validated JSON artifact writer", () => { throw new Error("injected write failure"); } }, + sync: async () => handle.sync(), + close: async () => handle.close(), + }; + }, + openDirectory: async (target) => { + const handle = await open(target, constants.O_RDONLY); + return { + sync: async () => handle.sync(), close: async () => handle.close(), }; },