import { mkdir, readFile, readdir, stat } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { z } from "zod"; import { computeHttpScenarioCatalogDigest, httpScenarioExpectationSchema, httpScenarioReceiptSchema, sameScenarioJson, type HttpScenarioExpectation, type HttpScenarioReceipt, } from "./lib/http-scenario-evidence.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts"; const scenarioContributionSchema = z .object({ owner: z.string().min(1), path: z.string().min(1), expectationExport: z.string().min(1), receiptPath: z.string().min(1), receiptSchemaVersion: z.number().int().positive(), }) .strict(); const sourceContractSchema = z .object({ owner: z.string().min(1), path: z.string().min(1), requiredTokens: z.array(z.string().min(1)), }) .strict(); const policySchema = z .object({ schemaVersion: z.literal(2), scenarioCatalogs: z.array(scenarioContributionSchema), sourceContracts: z.array(sourceContractSchema), }) .strict(); function argumentValue(name: string, fallback: string): string { const index = process.argv.indexOf(name); return index >= 0 && process.argv[index + 1] ? process.argv[index + 1] : fallback; } function optionalArgumentValue(name: string): string | undefined { const index = process.argv.indexOf(name); return index >= 0 ? process.argv[index + 1] : undefined; } const sourceRoot = argumentValue("--source-root", "tests"); const artifactPath = argumentValue( "--artifact", "artifacts/quality/test-evidence.json", ); const policyPath = argumentValue( "--policy", "config/testing/test-evidence.json", ); const explicitCatalogPath = optionalArgumentValue("--catalog"); const explicitReceiptPath = optionalArgumentValue("--receipt"); const fixtureMode = sourceRoot !== "tests"; const sourceOnly = process.argv.includes("--source-only"); const scenarioOnly = process.argv.includes("--scenario-only"); const skipScenarioExecutions = process.argv.includes( "--skip-scenario-executions", ); const failures: string[] = []; const facts = { scannedFiles: 0, visualBaselines: 0, sharedScenarios: 0, declaredScenarioExecutions: 0, executedScenarioExecutions: 0, }; async function filesBelow(target: string): Promise { try { const metadata = await stat(target); if (metadata.isFile()) return [target]; const entries = await readdir(target, { withFileTypes: true }); const groups = await Promise.all( entries.map((entry) => filesBelow(path.join(target, entry.name))), ); return groups.flat(); } catch { return []; } } function duplicates(values: readonly string[]): string[] { const seen = new Set(); const duplicate = new Set(); for (const value of values) { if (seen.has(value)) duplicate.add(value); seen.add(value); } return [...duplicate].sort(); } function isSorted(values: readonly string[]): boolean { return values.every( (value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0, ); } async function readJson(target: string): Promise { return JSON.parse(await readFile(target, "utf8")); } async function loadCatalogExport( target: string, exportName: string, ): Promise { if (target.endsWith(".json")) { const document = await readJson(target); return (document as Readonly>)[exportName]; } const module = (await import( `${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}` )) as Readonly>; return module[exportName]; } async function checkScenarioContribution( contribution: z.output, ): Promise { const catalogPath = explicitCatalogPath ?? contribution.path; const receiptPath = explicitReceiptPath ?? contribution.receiptPath; let expectations: readonly HttpScenarioExpectation[]; try { expectations = z .array(httpScenarioExpectationSchema) .parse( await loadCatalogExport(catalogPath, contribution.expectationExport), ); } catch (error) { failures.push( `[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`, ); return; } facts.declaredScenarioExecutions += expectations.length; facts.sharedScenarios += new Set( expectations.map((entry) => entry.scenarioId), ).size; const declarationIds = expectations.map((entry) => entry.executionId); if (declarationIds.length === 0) { failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`); } const duplicateDeclarations = duplicates(declarationIds); if (duplicateDeclarations.length > 0) { failures.push( `[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`, ); } for (const entry of expectations) { if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) { failures.push( `[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`, ); } } let receipt: HttpScenarioReceipt; try { const parsedReceipt = httpScenarioReceiptSchema.safeParse( await readJson(receiptPath), ); if (!parsedReceipt.success) { if ( parsedReceipt.error.issues.some((issue) => issue.message.includes("must be sorted by execution ID"), ) ) { failures.push( `[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`, ); } if ( parsedReceipt.error.issues.some((issue) => issue.message.includes("row IDs must exactly equal executed IDs"), ) ) { failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`); } throw parsedReceipt.error; } receipt = parsedReceipt.data; } catch (error) { failures.push( `[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`, ); return; } facts.executedScenarioExecutions += receipt.rows.length; if (receipt.schemaVersion !== contribution.receiptSchemaVersion) { failures.push( `[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`, ); } if (receipt.rows.length === 0 || receipt.executedIds.length === 0) { failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`); } if (receipt.catalogTotal !== declarationIds.length) { failures.push( `[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`, ); } const duplicateExecutedIds = duplicates(receipt.executedIds); const rowIds = receipt.rows.map((row) => row.executionId); const duplicateRows = duplicates(rowIds); if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) { failures.push( `[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`, ); } if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) { failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`); } if (!sameScenarioJson(receipt.executedIds, rowIds)) { failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`); } const declared = new Set(declarationIds); const executed = new Set(receipt.executedIds); const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort(); const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort(); if (missing.length > 0) { failures.push( `[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`, ); } if (extra.length > 0) { failures.push( `[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`, ); } const expectedDigest = computeHttpScenarioCatalogDigest( receipt.schemaVersion, expectations, ); if (receipt.catalogDigest !== expectedDigest) { failures.push( `[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`, ); } const declarationsById = new Map( expectations.map((entry) => [entry.executionId, entry] as const), ); for (const row of receipt.rows) { const declaration = declarationsById.get(row.executionId); if (!declaration) continue; if (!sameScenarioJson(row.expected, declaration.expected)) { failures.push( `[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`, ); } if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) { failures.push( `[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`, ); } if (!sameScenarioJson(row.observed, row.expected)) { failures.push( `[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`, ); } } } if (!scenarioOnly) { const sourceFiles = (await filesBelow(sourceRoot)).filter( (file) => fixtureMode || !file.split(path.sep).includes("fixtures"), ); for (const file of sourceFiles) { if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue; const source = await readFile(file, "utf8"); facts.scannedFiles += 1; const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g; if (skipPattern.test(source)) { const quarantine = /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/; if (!quarantine.test(source)) { failures.push(`${file}: skip/fixme lacks owned expiring quarantine`); } } const wholeUiMask = /\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i; if (wholeUiMask.test(source)) { failures.push(`${file}: screenshot mask may not cover the whole UI`); } } } let parsedPolicy: z.output | undefined; if (!fixtureMode || scenarioOnly) { try { parsedPolicy = policySchema.parse(await readJson(policyPath)); } catch (error) { failures.push( `[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`, ); } } if (parsedPolicy) { for (const contract of parsedPolicy.sourceContracts) { let source: string; try { source = await readFile(contract.path, "utf8"); } catch (error) { failures.push( `${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`, ); continue; } for (const token of contract.requiredTokens) { if (!source.includes(token)) { failures.push( `${contract.path}: ${contract.owner} evidence contract is missing ${token}`, ); } } } const shouldCheckScenarioExecutions = scenarioOnly || (!sourceOnly && !skipScenarioExecutions && !fixtureMode); if (shouldCheckScenarioExecutions) { if (parsedPolicy.scenarioCatalogs.length === 0) { failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs"); } if ( (explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) && parsedPolicy.scenarioCatalogs.length !== 1 ) { failures.push( "[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution", ); } else { for (const contribution of parsedPolicy.scenarioCatalogs) { await checkScenarioContribution(contribution); } } } } if (!fixtureMode && !scenarioOnly) { const e2eConfig = await readFile("playwright.config.ts", "utf8"); for (const token of [ "pnpm build", "pnpm preview", "reuseExistingServer: false", '"junit"', 'trace: "retain-on-failure"', '"chromium-compact"', '"firefox"', '"webkit"', ]) { if (!e2eConfig.includes(token)) { failures.push(`playwright.config.ts missing release evidence token ${token}`); } } const e2eFiles = (await filesBelow("tests/e2e")).filter((file) => /\.spec\.ts$/.test(file), ); for (const file of e2eFiles) { const source = await readFile(file, "utf8"); if (!source.includes("support/browser/strict-browser-test")) { failures.push(`${file}: bypasses strict browser fixture`); } } const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter( (file) => file.endsWith(".png"), ); facts.visualBaselines = baselineFiles.length; if (facts.visualBaselines < 4) { failures.push("visual baseline requires at least four risk surfaces"); } for (const required of [ "playwright.storybook.config.ts", "playwright.visual.config.ts", "tests/storybook/workshop.spec.ts", ]) { if ((await filesBelow(required)).length === 0) { failures.push(`test evidence missing ${required}`); } } if (!sourceOnly) { for (const required of [ "artifacts/tests/storybook/results.xml", "artifacts/tests/visual/results.xml", ]) { if ((await filesBelow(required)).length === 0) { failures.push(`test evidence missing ${required}`); } } for (const required of [ "dist/index.html", "dist/config.json", "dist/release-manifest.json", "dist/runtime-config.schema.json", "dist/.vite/manifest.json", ]) { if ((await filesBelow(required)).length === 0) { failures.push(`built-dist contract missing ${required}`); } } const sourceMaps = (await filesBelow("dist")).filter((file) => file.endsWith(".map"), ); if (sourceMaps.length > 0) { failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`); } } } const report = { schemaVersion: 2 as const, sourceRoot, status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const), facts, failures, }; await mkdir(path.dirname(artifactPath), { recursive: true }); await writeValidatedJsonArtifact({ path: artifactPath, schema: testEvidenceReportSchema, value: report, }); if (failures.length > 0) { process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`); process.exit(1); } process.stdout.write( `Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`, );