import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; type ScenarioCatalogContribution = Readonly<{ owner: string; path: string; arrayExport: string; minimumEntries: number; }>; type SourceContractContribution = Readonly<{ owner: string; path: string; requiredTokens: readonly string[]; }>; type TestEvidencePolicy = Readonly<{ schemaVersion: number; scenarioCatalogs: readonly unknown[]; sourceContracts: readonly unknown[]; }>; function argumentValue(name: string, fallback: string): string { const index = process.argv.indexOf(name); return index >= 0 && process.argv[index + 1] ? process.argv[index + 1] : fallback; } const sourceRoot = argumentValue("--source-root", "tests"); const artifactPath = argumentValue( "--artifact", "artifacts/quality/test-evidence.json", ); const policyPath = argumentValue( "--policy", "config/testing/test-evidence.json", ); const fixtureMode = sourceRoot !== "tests"; const sourceOnly = process.argv.includes("--source-only"); const failures: string[] = []; const facts = { scannedFiles: 0, visualBaselines: 0, sharedScenarios: 0, }; async function filesBelow(target: string): Promise { try { const metadata = await stat(target); if (metadata.isFile()) return [target]; const entries = await readdir(target, { withFileTypes: true }); const groups = await Promise.all( entries.map((entry) => filesBelow(path.join(target, entry.name))), ); return groups.flat(); } catch { return []; } } function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } const sourceFiles = (await filesBelow(sourceRoot)).filter( (file) => fixtureMode || !file.split(path.sep).includes("fixtures"), ); for (const file of sourceFiles) { if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue; const source = await readFile(file, "utf8"); facts.scannedFiles += 1; const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g; if (skipPattern.test(source)) { const quarantine = /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/; if (!quarantine.test(source)) { failures.push(`${file}: skip/fixme lacks owned expiring quarantine`); } } const wholeUiMask = /\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i; if (wholeUiMask.test(source)) { failures.push(`${file}: screenshot mask may not cover the whole UI`); } } if (!fixtureMode) { const e2eConfig = await readFile("playwright.config.ts", "utf8"); for (const token of [ "pnpm build", "pnpm preview", "reuseExistingServer: false", '"junit"', 'trace: "retain-on-failure"', '"chromium-compact"', '"firefox"', '"webkit"', ]) { if (!e2eConfig.includes(token)) { failures.push(`playwright.config.ts missing release evidence token ${token}`); } } const e2eFiles = (await filesBelow("tests/e2e")).filter((file) => /\.spec\.ts$/.test(file), ); for (const file of e2eFiles) { const source = await readFile(file, "utf8"); if (!source.includes("support/browser/strict-browser-test")) { failures.push(`${file}: bypasses strict browser fixture`); } } let evidencePolicy: unknown; try { evidencePolicy = JSON.parse(await readFile(policyPath, "utf8")); } catch (error) { failures.push( `${policyPath}: cannot read test evidence policy (${error instanceof Error ? error.message : String(error)})`, ); } const policy = evidencePolicy as Partial | undefined; if ( policy?.schemaVersion !== 1 || !Array.isArray(policy.scenarioCatalogs) || !Array.isArray(policy.sourceContracts) ) { failures.push(`${policyPath}: invalid test evidence policy`); } else { for (const candidate of policy.scenarioCatalogs) { const contribution = candidate as Partial; const minimumEntries = contribution.minimumEntries; if ( typeof contribution?.owner !== "string" || typeof contribution?.path !== "string" || typeof contribution?.arrayExport !== "string" || typeof minimumEntries !== "number" || !Number.isInteger(minimumEntries) || minimumEntries < 1 ) { failures.push(`${policyPath}: invalid scenario catalog contribution`); continue; } let source: string; try { source = await readFile(contribution.path, "utf8"); } catch (error) { failures.push( `${contribution.path}: cannot read scenario catalog (${error instanceof Error ? error.message : String(error)})`, ); continue; } const arrayPattern = new RegExp( `${escapeRegExp(contribution.arrayExport)}\\s*=\\s*Object\\.freeze\\(\\[([\\s\\S]*?)\\]\\s*as const\\)`, ); const entryCount = ( arrayPattern.exec(source)?.[1]?.match(/"[^"]+"/g) ?? [] ).length; facts.sharedScenarios += entryCount; if (entryCount < minimumEntries) { failures.push( `${contribution.path}: ${contribution.owner} requires at least ${minimumEntries} shared scenarios`, ); } } for (const candidate of policy.sourceContracts) { const contract = candidate as Partial; const requiredTokens = contract.requiredTokens; if ( typeof contract?.owner !== "string" || typeof contract?.path !== "string" || !Array.isArray(requiredTokens) || requiredTokens.some((token: unknown) => typeof token !== "string") ) { failures.push(`${policyPath}: invalid source contract contribution`); continue; } let source: string; try { source = await readFile(contract.path, "utf8"); } catch (error) { failures.push( `${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`, ); continue; } for (const token of requiredTokens as readonly string[]) { if (!source.includes(token)) { failures.push( `${contract.path}: ${contract.owner} evidence contract is missing ${token}`, ); } } } } const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter( (file) => file.endsWith(".png"), ); facts.visualBaselines = baselineFiles.length; if (facts.visualBaselines < 4) { failures.push("visual baseline requires at least four risk surfaces"); } for (const required of [ "playwright.storybook.config.ts", "playwright.visual.config.ts", "tests/storybook/workshop.spec.ts", ]) { if ((await filesBelow(required)).length === 0) { failures.push(`test evidence missing ${required}`); } } if (!sourceOnly) { for (const required of [ "artifacts/tests/storybook/results.xml", "artifacts/tests/visual/results.xml", ]) { if ((await filesBelow(required)).length === 0) { failures.push(`test evidence missing ${required}`); } } const requiredBuiltFiles = [ "dist/index.html", "dist/config.json", "dist/release-manifest.json", "dist/runtime-config.schema.json", "dist/.vite/manifest.json", ]; for (const required of requiredBuiltFiles) { if ((await filesBelow(required)).length === 0) { failures.push(`built-dist contract missing ${required}`); } } const sourceMaps = (await filesBelow("dist")).filter((file) => file.endsWith(".map"), ); if (sourceMaps.length > 0) { failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`); } } } const report = { schemaVersion: 1, sourceRoot, status: failures.length === 0 ? "PASS" : "FAIL", facts, failures, }; await mkdir(path.dirname(artifactPath), { recursive: true }); await writeFile(artifactPath, `${JSON.stringify(report, null, 2)}\n`); if (failures.length > 0) { process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`); process.exit(1); } process.stdout.write( `Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.sharedScenarios} scenarios)\n`, );