test: execute the HTTP scenario catalog
This commit is contained in:
DongHyeonka
+333
-128
@@ -1,24 +1,62 @@
|
||||
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readFile, readdir, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
type ScenarioCatalogContribution = Readonly<{
|
||||
owner: string;
|
||||
path: string;
|
||||
arrayExport: string;
|
||||
minimumEntries: number;
|
||||
}>;
|
||||
import { z } from "zod";
|
||||
|
||||
type SourceContractContribution = Readonly<{
|
||||
owner: string;
|
||||
path: string;
|
||||
requiredTokens: readonly string[];
|
||||
}>;
|
||||
import {
|
||||
computeHttpScenarioCatalogDigest,
|
||||
httpScenarioExpectationSchema,
|
||||
httpScenarioReceiptSchema,
|
||||
sameScenarioJson,
|
||||
type HttpScenarioExpectation,
|
||||
type HttpScenarioReceipt,
|
||||
} from "./lib/http-scenario-evidence.ts";
|
||||
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
||||
|
||||
type TestEvidencePolicy = Readonly<{
|
||||
schemaVersion: number;
|
||||
scenarioCatalogs: readonly unknown[];
|
||||
sourceContracts: readonly unknown[];
|
||||
}>;
|
||||
const scenarioContributionSchema = z
|
||||
.object({
|
||||
owner: z.string().min(1),
|
||||
path: z.string().min(1),
|
||||
expectationExport: z.string().min(1),
|
||||
receiptPath: z.string().min(1),
|
||||
receiptSchemaVersion: z.number().int().positive(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const sourceContractSchema = z
|
||||
.object({
|
||||
owner: z.string().min(1),
|
||||
path: z.string().min(1),
|
||||
requiredTokens: z.array(z.string().min(1)),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const policySchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(2),
|
||||
scenarioCatalogs: z.array(scenarioContributionSchema),
|
||||
sourceContracts: z.array(sourceContractSchema),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const reportSchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(2),
|
||||
sourceRoot: z.string().min(1),
|
||||
status: z.enum(["PASS", "FAIL"]),
|
||||
facts: z
|
||||
.object({
|
||||
scannedFiles: z.number().int().nonnegative(),
|
||||
visualBaselines: z.number().int().nonnegative(),
|
||||
sharedScenarios: z.number().int().nonnegative(),
|
||||
declaredScenarioExecutions: z.number().int().nonnegative(),
|
||||
executedScenarioExecutions: z.number().int().nonnegative(),
|
||||
})
|
||||
.strict(),
|
||||
failures: z.array(z.string()),
|
||||
})
|
||||
.strict();
|
||||
|
||||
function argumentValue(name: string, fallback: string): string {
|
||||
const index = process.argv.indexOf(name);
|
||||
@@ -27,6 +65,11 @@ function argumentValue(name: string, fallback: string): string {
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function optionalArgumentValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(name);
|
||||
return index >= 0 ? process.argv[index + 1] : undefined;
|
||||
}
|
||||
|
||||
const sourceRoot = argumentValue("--source-root", "tests");
|
||||
const artifactPath = argumentValue(
|
||||
"--artifact",
|
||||
@@ -36,13 +79,21 @@ const policyPath = argumentValue(
|
||||
"--policy",
|
||||
"config/testing/test-evidence.json",
|
||||
);
|
||||
const explicitCatalogPath = optionalArgumentValue("--catalog");
|
||||
const explicitReceiptPath = optionalArgumentValue("--receipt");
|
||||
const fixtureMode = sourceRoot !== "tests";
|
||||
const sourceOnly = process.argv.includes("--source-only");
|
||||
const scenarioOnly = process.argv.includes("--scenario-only");
|
||||
const skipScenarioExecutions = process.argv.includes(
|
||||
"--skip-scenario-executions",
|
||||
);
|
||||
const failures: string[] = [];
|
||||
const facts = {
|
||||
scannedFiles: 0,
|
||||
visualBaselines: 0,
|
||||
sharedScenarios: 0,
|
||||
declaredScenarioExecutions: 0,
|
||||
executedScenarioExecutions: 0,
|
||||
};
|
||||
|
||||
async function filesBelow(target: string): Promise<string[]> {
|
||||
@@ -59,34 +110,268 @@ async function filesBelow(target: string): Promise<string[]> {
|
||||
}
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
function duplicates(values: readonly string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const duplicate = new Set<string>();
|
||||
for (const value of values) {
|
||||
if (seen.has(value)) duplicate.add(value);
|
||||
seen.add(value);
|
||||
}
|
||||
return [...duplicate].sort();
|
||||
}
|
||||
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
const skipPattern =
|
||||
/\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
function isSorted(values: readonly string[]): boolean {
|
||||
return values.every(
|
||||
(value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0,
|
||||
);
|
||||
}
|
||||
|
||||
async function readJson(target: string): Promise<unknown> {
|
||||
return JSON.parse(await readFile(target, "utf8"));
|
||||
}
|
||||
|
||||
async function loadCatalogExport(
|
||||
target: string,
|
||||
exportName: string,
|
||||
): Promise<unknown> {
|
||||
if (target.endsWith(".json")) {
|
||||
const document = await readJson(target);
|
||||
return (document as Readonly<Record<string, unknown>>)[exportName];
|
||||
}
|
||||
const module = (await import(
|
||||
`${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}`
|
||||
)) as Readonly<Record<string, unknown>>;
|
||||
return module[exportName];
|
||||
}
|
||||
|
||||
async function checkScenarioContribution(
|
||||
contribution: z.output<typeof scenarioContributionSchema>,
|
||||
): Promise<void> {
|
||||
const catalogPath = explicitCatalogPath ?? contribution.path;
|
||||
const receiptPath = explicitReceiptPath ?? contribution.receiptPath;
|
||||
let expectations: readonly HttpScenarioExpectation[];
|
||||
try {
|
||||
expectations = z
|
||||
.array(httpScenarioExpectationSchema)
|
||||
.parse(
|
||||
await loadCatalogExport(catalogPath, contribution.expectationExport),
|
||||
);
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
facts.declaredScenarioExecutions += expectations.length;
|
||||
facts.sharedScenarios += new Set(
|
||||
expectations.map((entry) => entry.scenarioId),
|
||||
).size;
|
||||
const declarationIds = expectations.map((entry) => entry.executionId);
|
||||
if (declarationIds.length === 0) {
|
||||
failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`);
|
||||
}
|
||||
const duplicateDeclarations = duplicates(declarationIds);
|
||||
if (duplicateDeclarations.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`,
|
||||
);
|
||||
}
|
||||
for (const entry of expectations) {
|
||||
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
if (wholeUiMask.test(source)) {
|
||||
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
||||
|
||||
let receipt: HttpScenarioReceipt;
|
||||
try {
|
||||
const parsedReceipt = httpScenarioReceiptSchema.safeParse(
|
||||
await readJson(receiptPath),
|
||||
);
|
||||
if (!parsedReceipt.success) {
|
||||
if (
|
||||
parsedReceipt.error.issues.some((issue) =>
|
||||
issue.message.includes("must be sorted by execution ID"),
|
||||
)
|
||||
) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
parsedReceipt.error.issues.some((issue) =>
|
||||
issue.message.includes("row IDs must exactly equal executed IDs"),
|
||||
)
|
||||
) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
||||
}
|
||||
throw parsedReceipt.error;
|
||||
}
|
||||
receipt = parsedReceipt.data;
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
facts.executedScenarioExecutions += receipt.rows.length;
|
||||
if (receipt.schemaVersion !== contribution.receiptSchemaVersion) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`,
|
||||
);
|
||||
}
|
||||
if (receipt.rows.length === 0 || receipt.executedIds.length === 0) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`);
|
||||
}
|
||||
if (receipt.catalogTotal !== declarationIds.length) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`,
|
||||
);
|
||||
}
|
||||
const duplicateExecutedIds = duplicates(receipt.executedIds);
|
||||
const rowIds = receipt.rows.map((row) => row.executionId);
|
||||
const duplicateRows = duplicates(rowIds);
|
||||
if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`);
|
||||
}
|
||||
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
||||
}
|
||||
|
||||
const declared = new Set(declarationIds);
|
||||
const executed = new Set(receipt.executedIds);
|
||||
const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort();
|
||||
const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort();
|
||||
if (missing.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (extra.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const expectedDigest = computeHttpScenarioCatalogDigest(
|
||||
receipt.schemaVersion,
|
||||
expectations,
|
||||
);
|
||||
if (receipt.catalogDigest !== expectedDigest) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`,
|
||||
);
|
||||
}
|
||||
|
||||
const declarationsById = new Map(
|
||||
expectations.map((entry) => [entry.executionId, entry] as const),
|
||||
);
|
||||
for (const row of receipt.rows) {
|
||||
const declaration = declarationsById.get(row.executionId);
|
||||
if (!declaration) continue;
|
||||
if (!sameScenarioJson(row.expected, declaration.expected)) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`,
|
||||
);
|
||||
}
|
||||
if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`,
|
||||
);
|
||||
}
|
||||
if (!sameScenarioJson(row.observed, row.expected)) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!fixtureMode) {
|
||||
if (!scenarioOnly) {
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
if (wholeUiMask.test(source)) {
|
||||
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let parsedPolicy: z.output<typeof policySchema> | undefined;
|
||||
if (!fixtureMode || scenarioOnly) {
|
||||
try {
|
||||
parsedPolicy = policySchema.parse(await readJson(policyPath));
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedPolicy) {
|
||||
for (const contract of parsedPolicy.sourceContracts) {
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(contract.path, "utf8");
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const token of contract.requiredTokens) {
|
||||
if (!source.includes(token)) {
|
||||
failures.push(
|
||||
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const shouldCheckScenarioExecutions =
|
||||
scenarioOnly ||
|
||||
(!sourceOnly && !skipScenarioExecutions && !fixtureMode);
|
||||
if (shouldCheckScenarioExecutions) {
|
||||
if (parsedPolicy.scenarioCatalogs.length === 0) {
|
||||
failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs");
|
||||
}
|
||||
if (
|
||||
(explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) &&
|
||||
parsedPolicy.scenarioCatalogs.length !== 1
|
||||
) {
|
||||
failures.push(
|
||||
"[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution",
|
||||
);
|
||||
} else {
|
||||
for (const contribution of parsedPolicy.scenarioCatalogs) {
|
||||
await checkScenarioContribution(contribution);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!fixtureMode && !scenarioOnly) {
|
||||
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
||||
for (const token of [
|
||||
"pnpm build",
|
||||
@@ -113,89 +398,6 @@ if (!fixtureMode) {
|
||||
}
|
||||
}
|
||||
|
||||
let evidencePolicy: unknown;
|
||||
try {
|
||||
evidencePolicy = JSON.parse(await readFile(policyPath, "utf8"));
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${policyPath}: cannot read test evidence policy (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
}
|
||||
const policy = evidencePolicy as Partial<TestEvidencePolicy> | undefined;
|
||||
if (
|
||||
policy?.schemaVersion !== 1 ||
|
||||
!Array.isArray(policy.scenarioCatalogs) ||
|
||||
!Array.isArray(policy.sourceContracts)
|
||||
) {
|
||||
failures.push(`${policyPath}: invalid test evidence policy`);
|
||||
} else {
|
||||
for (const candidate of policy.scenarioCatalogs) {
|
||||
const contribution = candidate as Partial<ScenarioCatalogContribution>;
|
||||
const minimumEntries = contribution.minimumEntries;
|
||||
if (
|
||||
typeof contribution?.owner !== "string" ||
|
||||
typeof contribution?.path !== "string" ||
|
||||
typeof contribution?.arrayExport !== "string" ||
|
||||
typeof minimumEntries !== "number" ||
|
||||
!Number.isInteger(minimumEntries) ||
|
||||
minimumEntries < 1
|
||||
) {
|
||||
failures.push(`${policyPath}: invalid scenario catalog contribution`);
|
||||
continue;
|
||||
}
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(contribution.path, "utf8");
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${contribution.path}: cannot read scenario catalog (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const arrayPattern = new RegExp(
|
||||
`${escapeRegExp(contribution.arrayExport)}\\s*=\\s*Object\\.freeze\\(\\[([\\s\\S]*?)\\]\\s*as const\\)`,
|
||||
);
|
||||
const entryCount = (
|
||||
arrayPattern.exec(source)?.[1]?.match(/"[^"]+"/g) ?? []
|
||||
).length;
|
||||
facts.sharedScenarios += entryCount;
|
||||
if (entryCount < minimumEntries) {
|
||||
failures.push(
|
||||
`${contribution.path}: ${contribution.owner} requires at least ${minimumEntries} shared scenarios`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const candidate of policy.sourceContracts) {
|
||||
const contract = candidate as Partial<SourceContractContribution>;
|
||||
const requiredTokens = contract.requiredTokens;
|
||||
if (
|
||||
typeof contract?.owner !== "string" ||
|
||||
typeof contract?.path !== "string" ||
|
||||
!Array.isArray(requiredTokens) ||
|
||||
requiredTokens.some((token: unknown) => typeof token !== "string")
|
||||
) {
|
||||
failures.push(`${policyPath}: invalid source contract contribution`);
|
||||
continue;
|
||||
}
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(contract.path, "utf8");
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const token of requiredTokens as readonly string[]) {
|
||||
if (!source.includes(token)) {
|
||||
failures.push(
|
||||
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
|
||||
(file) => file.endsWith(".png"),
|
||||
);
|
||||
@@ -223,14 +425,13 @@ if (!fixtureMode) {
|
||||
}
|
||||
}
|
||||
|
||||
const requiredBuiltFiles = [
|
||||
for (const required of [
|
||||
"dist/index.html",
|
||||
"dist/config.json",
|
||||
"dist/release-manifest.json",
|
||||
"dist/runtime-config.schema.json",
|
||||
"dist/.vite/manifest.json",
|
||||
];
|
||||
for (const required of requiredBuiltFiles) {
|
||||
]) {
|
||||
if ((await filesBelow(required)).length === 0) {
|
||||
failures.push(`built-dist contract missing ${required}`);
|
||||
}
|
||||
@@ -245,18 +446,22 @@ if (!fixtureMode) {
|
||||
}
|
||||
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
schemaVersion: 2 as const,
|
||||
sourceRoot,
|
||||
status: failures.length === 0 ? "PASS" : "FAIL",
|
||||
status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const),
|
||||
facts,
|
||||
failures,
|
||||
};
|
||||
await mkdir(path.dirname(artifactPath), { recursive: true });
|
||||
await writeFile(artifactPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
await writeValidatedJsonArtifact({
|
||||
path: artifactPath,
|
||||
schema: reportSchema,
|
||||
value: report,
|
||||
});
|
||||
if (failures.length > 0) {
|
||||
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(
|
||||
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.sharedScenarios} scenarios)\n`,
|
||||
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`,
|
||||
);
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process";
|
||||
|
||||
export const PNPM_SCRIPT_TIMEOUT_MS = 60_000;
|
||||
export const PNPM_SCRIPT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
type BoundedPnpmScriptInvocation = Readonly<{
|
||||
command: string;
|
||||
arguments: readonly string[];
|
||||
options: SpawnSyncOptionsWithStringEncoding;
|
||||
}>;
|
||||
|
||||
type PnpmScriptResult = Readonly<{
|
||||
status: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
error?: Error;
|
||||
}>;
|
||||
|
||||
export function createBoundedPnpmScriptInvocation(input: Readonly<{
|
||||
nodePath: string;
|
||||
pnpmCli: string;
|
||||
script: string;
|
||||
environment: NodeJS.ProcessEnv;
|
||||
}>): BoundedPnpmScriptInvocation {
|
||||
return Object.freeze({
|
||||
command: input.nodePath,
|
||||
arguments: Object.freeze([input.pnpmCli, "run", input.script]),
|
||||
options: Object.freeze({
|
||||
encoding: "utf8",
|
||||
env: input.environment,
|
||||
killSignal: "SIGTERM",
|
||||
maxBuffer: PNPM_SCRIPT_MAX_OUTPUT_BYTES,
|
||||
timeout: PNPM_SCRIPT_TIMEOUT_MS,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function formatPnpmScriptFailure(
|
||||
script: string,
|
||||
result: PnpmScriptResult,
|
||||
): string {
|
||||
const code =
|
||||
result.error && "code" in result.error &&
|
||||
typeof result.error.code === "string"
|
||||
? result.error.code
|
||||
: null;
|
||||
const message = result.error?.message.trim().replace(/\s+/g, " ") ?? null;
|
||||
const error =
|
||||
result.error === undefined
|
||||
? "none"
|
||||
: `${code ?? result.error.name}: ${message || "no message"}`;
|
||||
return `${script} failed: exit=${String(result.status)}, signal=${result.signal ?? "none"}, error=${error}`;
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
export type AttemptStatus = number | "NETWORK_REJECTION" | "PENDING_ABORT";
|
||||
export type RetryReason = "NETWORK_FAILURE" | "HTTP_429" | "HTTP_503";
|
||||
export type BodyDisposition =
|
||||
| "FULLY_READ_WITHIN_BOUND"
|
||||
| "CANCELLED_WITHOUT_READ"
|
||||
| "REJECTED_LIMIT"
|
||||
| "NO_RESPONSE";
|
||||
|
||||
const attemptStatusSchema = z.union([
|
||||
z.number().int().min(100).max(599),
|
||||
z.literal("NETWORK_REJECTION"),
|
||||
z.literal("PENDING_ABORT"),
|
||||
]);
|
||||
|
||||
export const httpScenarioAssertionGroupsSchema = z
|
||||
.object({
|
||||
status: z
|
||||
.object({
|
||||
attempts: z.array(attemptStatusSchema).min(1),
|
||||
final: attemptStatusSchema,
|
||||
})
|
||||
.strict(),
|
||||
outcome: z.object({ kind: z.string().min(1), detail: z.string().nullable() }).strict(),
|
||||
effect: z.object({ outcome: z.string().min(1), observer: z.string().min(1) }).strict(),
|
||||
retry: z
|
||||
.object({
|
||||
count: z.number().int().nonnegative(),
|
||||
reasons: z.array(z.enum(["NETWORK_FAILURE", "HTTP_429", "HTTP_503"])),
|
||||
})
|
||||
.strict(),
|
||||
fetch: z
|
||||
.object({
|
||||
count: z.number().int().positive(),
|
||||
observerAttempts: z.number().int().positive(),
|
||||
agrees: z.boolean(),
|
||||
})
|
||||
.strict(),
|
||||
media: z.object({ attempts: z.array(z.string().nullable()).min(1), final: z.string().nullable() }).strict(),
|
||||
body: z
|
||||
.object({
|
||||
attempts: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
disposition: z.enum([
|
||||
"FULLY_READ_WITHIN_BOUND",
|
||||
"CANCELLED_WITHOUT_READ",
|
||||
"REJECTED_LIMIT",
|
||||
"NO_RESPONSE",
|
||||
]),
|
||||
pulledBytes: z.number().int().nonnegative(),
|
||||
ceiling: z.number().int().nonnegative(),
|
||||
})
|
||||
.strict(),
|
||||
)
|
||||
.min(1),
|
||||
})
|
||||
.strict(),
|
||||
scope: z
|
||||
.object({
|
||||
start: z.literal("CURRENT"),
|
||||
end: z.enum(["CURRENT", "STALE"]),
|
||||
signal: z.enum(["ACTIVE", "ABORTED"]),
|
||||
cancellationOwner: z.enum(["NONE", "CALLER", "SCOPE_FENCE", "DEADLINE"]),
|
||||
})
|
||||
.strict(),
|
||||
})
|
||||
.strict()
|
||||
.superRefine((groups, context) => {
|
||||
const fail = (path: (string | number)[], message: string) => {
|
||||
context.addIssue({ code: "custom", path, message });
|
||||
};
|
||||
if (groups.status.final !== groups.status.attempts.at(-1)) {
|
||||
fail(["status", "final"], "must equal the final attempt status");
|
||||
}
|
||||
if (groups.retry.count !== groups.retry.reasons.length) {
|
||||
fail(["retry", "count"], "must equal retry reasons length");
|
||||
}
|
||||
if (groups.fetch.count !== groups.status.attempts.length) {
|
||||
fail(["fetch", "count"], "must equal status attempts length");
|
||||
}
|
||||
if (groups.fetch.count !== groups.media.attempts.length) {
|
||||
fail(["media", "attempts"], "must equal fetch count");
|
||||
}
|
||||
if (groups.fetch.count !== groups.body.attempts.length) {
|
||||
fail(["body", "attempts"], "must equal fetch count");
|
||||
}
|
||||
if (groups.fetch.observerAttempts !== groups.fetch.count) {
|
||||
fail(["fetch", "observerAttempts"], "must equal physical fetch count");
|
||||
}
|
||||
if (!groups.fetch.agrees) {
|
||||
fail(["fetch", "agrees"], "must prove physical/observer agreement");
|
||||
}
|
||||
if (groups.media.final !== groups.media.attempts.at(-1)) {
|
||||
fail(["media", "final"], "must equal the final attempt media essence");
|
||||
}
|
||||
});
|
||||
|
||||
export type HttpScenarioAssertionGroups = Readonly<{
|
||||
status: Readonly<{ attempts: readonly AttemptStatus[]; final: AttemptStatus }>;
|
||||
outcome: Readonly<{ kind: string; detail: string | null }>;
|
||||
effect: Readonly<{ outcome: string; observer: string }>;
|
||||
retry: Readonly<{ count: number; reasons: readonly RetryReason[] }>;
|
||||
fetch: Readonly<{
|
||||
count: number;
|
||||
observerAttempts: number;
|
||||
agrees: boolean;
|
||||
}>;
|
||||
media: Readonly<{
|
||||
attempts: readonly (string | null)[];
|
||||
final: string | null;
|
||||
}>;
|
||||
body: Readonly<{
|
||||
attempts: readonly Readonly<{
|
||||
disposition: BodyDisposition;
|
||||
pulledBytes: number;
|
||||
ceiling: number;
|
||||
}>[];
|
||||
}>;
|
||||
scope: Readonly<{
|
||||
start: "CURRENT";
|
||||
end: "CURRENT" | "STALE";
|
||||
signal: "ACTIVE" | "ABORTED";
|
||||
cancellationOwner: "NONE" | "CALLER" | "SCOPE_FENCE" | "DEADLINE";
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export const httpScenarioExpectationSchema = z
|
||||
.object({
|
||||
executionId: z.string().min(1),
|
||||
operationId: z.string().min(1),
|
||||
scenarioId: z.string().min(1),
|
||||
expected: httpScenarioAssertionGroupsSchema,
|
||||
testDeadlineOverrideMs: z.number().int().positive().nullable(),
|
||||
})
|
||||
.strict()
|
||||
.superRefine((entry, context) => {
|
||||
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["executionId"],
|
||||
message: "must equal operationId::scenarioId",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type HttpScenarioExpectation = Readonly<{
|
||||
executionId: string;
|
||||
operationId: string;
|
||||
scenarioId: string;
|
||||
expected: HttpScenarioAssertionGroups;
|
||||
testDeadlineOverrideMs: number | null;
|
||||
}>;
|
||||
|
||||
export const httpScenarioReceiptRowSchema = z
|
||||
.object({
|
||||
executionId: z.string().min(1),
|
||||
expected: httpScenarioAssertionGroupsSchema,
|
||||
observed: httpScenarioAssertionGroupsSchema,
|
||||
testDeadlineOverrideMs: z.number().int().positive().nullable(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const httpScenarioReceiptSchema = z
|
||||
.object({
|
||||
schemaVersion: z.number().int().positive(),
|
||||
catalogDigest: z.string().regex(/^sha256:[0-9a-f]{64}$/),
|
||||
catalogTotal: z.number().int().nonnegative(),
|
||||
executedIds: z.array(z.string().min(1)),
|
||||
rows: z.array(httpScenarioReceiptRowSchema),
|
||||
})
|
||||
.strict()
|
||||
.superRefine((receipt, context) => {
|
||||
const rowIds = receipt.rows.map((row) => row.executionId);
|
||||
const sortedIds = [...receipt.executedIds].sort((left, right) =>
|
||||
left.localeCompare(right),
|
||||
);
|
||||
const sortedRowIds = [...rowIds].sort((left, right) =>
|
||||
left.localeCompare(right),
|
||||
);
|
||||
if (!sameScenarioJson(receipt.executedIds, sortedIds)) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["executedIds"],
|
||||
message: "must be sorted by execution ID",
|
||||
});
|
||||
}
|
||||
if (!sameScenarioJson(rowIds, sortedRowIds)) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["rows"],
|
||||
message: "must be sorted by execution ID",
|
||||
});
|
||||
}
|
||||
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["rows"],
|
||||
message: "row IDs must exactly equal executed IDs",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type HttpScenarioReceipt = Readonly<{
|
||||
schemaVersion: number;
|
||||
catalogDigest: string;
|
||||
catalogTotal: number;
|
||||
executedIds: readonly string[];
|
||||
rows: readonly Readonly<{
|
||||
executionId: string;
|
||||
expected: HttpScenarioAssertionGroups;
|
||||
observed: HttpScenarioAssertionGroups;
|
||||
testDeadlineOverrideMs: number | null;
|
||||
}>[];
|
||||
}>;
|
||||
|
||||
export function stableScenarioJson(value: unknown): string {
|
||||
const normalize = (candidate: unknown): unknown => {
|
||||
if (Array.isArray(candidate)) return candidate.map(normalize);
|
||||
if (candidate && typeof candidate === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(candidate as Readonly<Record<string, unknown>>)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, nested]) => [key, normalize(nested)]),
|
||||
);
|
||||
}
|
||||
return candidate;
|
||||
};
|
||||
return JSON.stringify(normalize(value));
|
||||
}
|
||||
|
||||
export function sameScenarioJson(left: unknown, right: unknown): boolean {
|
||||
return stableScenarioJson(left) === stableScenarioJson(right);
|
||||
}
|
||||
|
||||
export function computeHttpScenarioCatalogDigest(
|
||||
schemaVersion: number,
|
||||
expectations: readonly HttpScenarioExpectation[],
|
||||
): `sha256:${string}` {
|
||||
const tuples = [...expectations]
|
||||
.sort((left, right) => left.executionId.localeCompare(right.executionId))
|
||||
.map((entry) => [
|
||||
entry.executionId,
|
||||
entry.expected.status,
|
||||
entry.expected.outcome,
|
||||
entry.expected.effect,
|
||||
entry.expected.retry,
|
||||
entry.expected.fetch,
|
||||
entry.expected.media,
|
||||
entry.expected.body,
|
||||
entry.expected.scope,
|
||||
entry.testDeadlineOverrideMs,
|
||||
]);
|
||||
return `sha256:${createHash("sha256")
|
||||
.update(stableScenarioJson([schemaVersion, ...tuples]))
|
||||
.digest("hex")}`;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import {
|
||||
open as openFile,
|
||||
rename as renameFile,
|
||||
@@ -15,8 +16,13 @@ export type ValidatedJsonArtifactInput = Readonly<{
|
||||
}>;
|
||||
|
||||
export type ValidatedJsonArtifactFileSystem = Readonly<{
|
||||
open: (path: string, flags: "wx") => Promise<{
|
||||
open: (path: string, flags: number, mode: number) => Promise<{
|
||||
writeFile(data: string, encoding: "utf8"): Promise<unknown>;
|
||||
sync(): Promise<unknown>;
|
||||
close(): Promise<unknown>;
|
||||
}>;
|
||||
openDirectory: (path: string) => Promise<{
|
||||
sync(): Promise<unknown>;
|
||||
close(): Promise<unknown>;
|
||||
}>;
|
||||
rename: (source: string, destination: string) => Promise<unknown>;
|
||||
@@ -29,11 +35,21 @@ type ValidatedJsonArtifactWriterDependencies = Readonly<{
|
||||
}>;
|
||||
|
||||
const defaultFileSystem: ValidatedJsonArtifactFileSystem = Object.freeze({
|
||||
open: async (target, flags) => openFile(target, flags),
|
||||
open: async (target, flags, mode) => openFile(target, flags, mode),
|
||||
openDirectory: async (target) => openFile(target, constants.O_RDONLY),
|
||||
rename: async (source, destination) => renameFile(source, destination),
|
||||
rm: async (target, options) => removeFile(target, options),
|
||||
});
|
||||
|
||||
function hasErrorCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
"code" in error &&
|
||||
error.code === code
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a writer whose only publish operation is an atomic sibling rename.
|
||||
* Dependency injection is limited to the file-system boundary so failure
|
||||
@@ -60,12 +76,20 @@ export function createValidatedJsonArtifactWriter(
|
||||
);
|
||||
let ownsTemporaryFile = false;
|
||||
try {
|
||||
const handle = await fileSystem.open(temporaryPath, "wx");
|
||||
const handle = await fileSystem.open(
|
||||
temporaryPath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_EXCL |
|
||||
constants.O_NOFOLLOW,
|
||||
0o600,
|
||||
);
|
||||
ownsTemporaryFile = true;
|
||||
let writeFailed = false;
|
||||
let writeFailure: unknown;
|
||||
try {
|
||||
await handle.writeFile(`${serialized}\n`, "utf8");
|
||||
await handle.sync();
|
||||
} catch (error) {
|
||||
writeFailed = true;
|
||||
writeFailure = error;
|
||||
@@ -81,6 +105,21 @@ export function createValidatedJsonArtifactWriter(
|
||||
if (writeFailed) throw writeFailure;
|
||||
if (closeFailed) throw closeFailure;
|
||||
await fileSystem.rename(temporaryPath, input.path);
|
||||
ownsTemporaryFile = false;
|
||||
const directoryHandle = await fileSystem.openDirectory(
|
||||
path.dirname(input.path),
|
||||
);
|
||||
try {
|
||||
try {
|
||||
await directoryHandle.sync();
|
||||
} catch (error) {
|
||||
if (!hasErrorCode(error, "EINVAL") && !hasErrorCode(error, "ENOTSUP")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await directoryHandle.close();
|
||||
}
|
||||
} catch (error) {
|
||||
if (ownsTemporaryFile) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { access, rm } from "node:fs/promises";
|
||||
|
||||
import {
|
||||
createBoundedPnpmScriptInvocation,
|
||||
formatPnpmScriptFailure,
|
||||
} from "./lib/bounded-pnpm-script.ts";
|
||||
|
||||
const receiptPath = "artifacts/tests/http-scenario-executions.json";
|
||||
const expectedNegativeDiagnostic =
|
||||
"[TEST-EVIDENCE-EXECUTION-ID-MISSING]";
|
||||
const pnpmCli = process.env.npm_execpath;
|
||||
if (!pnpmCli) throw new Error("npm_execpath is required");
|
||||
|
||||
await rm(receiptPath, { force: true });
|
||||
try {
|
||||
await access(receiptPath);
|
||||
throw new Error(`stale scenario receipt remains: ${receiptPath}`);
|
||||
} catch (error) {
|
||||
if (
|
||||
typeof error !== "object" ||
|
||||
error === null ||
|
||||
!("code" in error) ||
|
||||
error.code !== "ENOENT"
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
runExpectedPass("test:http-scenario-catalog");
|
||||
runExpectedPass("check:http-scenario-evidence");
|
||||
const negative = run("check:http-scenario-evidence:fixture");
|
||||
if (
|
||||
negative.status !== 1 ||
|
||||
!`${negative.stdout}\n${negative.stderr}`.includes(expectedNegativeDiagnostic)
|
||||
) {
|
||||
process.stderr.write(negative.stdout);
|
||||
process.stderr.write(negative.stderr);
|
||||
throw new Error(
|
||||
`scenario negative fixture identity mismatch: ${formatPnpmScriptFailure(
|
||||
"check:http-scenario-evidence:fixture",
|
||||
negative,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
"HTTP scenario evidence: PASS (stale receipt removed, 54 executed, missing-ID fixture rejected)\n",
|
||||
);
|
||||
|
||||
function run(script: string) {
|
||||
const invocation = createBoundedPnpmScriptInvocation({
|
||||
nodePath: process.execPath,
|
||||
pnpmCli: pnpmCli!,
|
||||
script,
|
||||
environment: process.env,
|
||||
});
|
||||
return spawnSync(
|
||||
invocation.command,
|
||||
[...invocation.arguments],
|
||||
invocation.options,
|
||||
);
|
||||
}
|
||||
|
||||
function runExpectedPass(script: string): void {
|
||||
const result = run(script);
|
||||
process.stdout.write(result.stdout);
|
||||
process.stderr.write(result.stderr);
|
||||
if (result.status !== 0 || result.error !== undefined) {
|
||||
throw new Error(formatPnpmScriptFailure(script, result));
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ const commonTestPaths = [
|
||||
const featureOwnedPaths = [
|
||||
featureSource,
|
||||
featureTests,
|
||||
"tests/integration/http-scenario-catalog.test.ts",
|
||||
"tests/e2e/reference-form.spec.ts",
|
||||
"tests/e2e/reference-route.spec.ts",
|
||||
"tests/mocks",
|
||||
|
||||
Reference in New Issue
Block a user