451 lines
14 KiB
TypeScript
451 lines
14 KiB
TypeScript
import { mkdir, readFile, readdir, stat } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
import { z } from "zod";
|
|
|
|
import {
|
|
computeHttpScenarioCatalogDigest,
|
|
httpScenarioExpectationSchema,
|
|
httpScenarioReceiptSchema,
|
|
sameScenarioJson,
|
|
type HttpScenarioExpectation,
|
|
type HttpScenarioReceipt,
|
|
} from "./lib/http-scenario-evidence.ts";
|
|
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
|
import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts";
|
|
|
|
const scenarioContributionSchema = z
|
|
.object({
|
|
owner: z.string().min(1),
|
|
path: z.string().min(1),
|
|
expectationExport: z.string().min(1),
|
|
receiptPath: z.string().min(1),
|
|
receiptSchemaVersion: z.number().int().positive(),
|
|
})
|
|
.strict();
|
|
|
|
const sourceContractSchema = z
|
|
.object({
|
|
owner: z.string().min(1),
|
|
path: z.string().min(1),
|
|
requiredTokens: z.array(z.string().min(1)),
|
|
})
|
|
.strict();
|
|
|
|
const policySchema = z
|
|
.object({
|
|
schemaVersion: z.literal(2),
|
|
scenarioCatalogs: z.array(scenarioContributionSchema),
|
|
sourceContracts: z.array(sourceContractSchema),
|
|
})
|
|
.strict();
|
|
|
|
function argumentValue(name: string, fallback: string): string {
|
|
const index = process.argv.indexOf(name);
|
|
return index >= 0 && process.argv[index + 1]
|
|
? process.argv[index + 1]
|
|
: fallback;
|
|
}
|
|
|
|
function optionalArgumentValue(name: string): string | undefined {
|
|
const index = process.argv.indexOf(name);
|
|
return index >= 0 ? process.argv[index + 1] : undefined;
|
|
}
|
|
|
|
const sourceRoot = argumentValue("--source-root", "tests");
|
|
const artifactPath = argumentValue(
|
|
"--artifact",
|
|
"artifacts/quality/test-evidence.json",
|
|
);
|
|
const policyPath = argumentValue(
|
|
"--policy",
|
|
"config/testing/test-evidence.json",
|
|
);
|
|
const explicitCatalogPath = optionalArgumentValue("--catalog");
|
|
const explicitReceiptPath = optionalArgumentValue("--receipt");
|
|
const fixtureMode = sourceRoot !== "tests";
|
|
const sourceOnly = process.argv.includes("--source-only");
|
|
const scenarioOnly = process.argv.includes("--scenario-only");
|
|
const skipScenarioExecutions = process.argv.includes(
|
|
"--skip-scenario-executions",
|
|
);
|
|
const failures: string[] = [];
|
|
const facts = {
|
|
scannedFiles: 0,
|
|
visualBaselines: 0,
|
|
sharedScenarios: 0,
|
|
declaredScenarioExecutions: 0,
|
|
executedScenarioExecutions: 0,
|
|
};
|
|
|
|
async function filesBelow(target: string): Promise<string[]> {
|
|
try {
|
|
const metadata = await stat(target);
|
|
if (metadata.isFile()) return [target];
|
|
const entries = await readdir(target, { withFileTypes: true });
|
|
const groups = await Promise.all(
|
|
entries.map((entry) => filesBelow(path.join(target, entry.name))),
|
|
);
|
|
return groups.flat();
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function duplicates(values: readonly string[]): string[] {
|
|
const seen = new Set<string>();
|
|
const duplicate = new Set<string>();
|
|
for (const value of values) {
|
|
if (seen.has(value)) duplicate.add(value);
|
|
seen.add(value);
|
|
}
|
|
return [...duplicate].sort();
|
|
}
|
|
|
|
function isSorted(values: readonly string[]): boolean {
|
|
return values.every(
|
|
(value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0,
|
|
);
|
|
}
|
|
|
|
async function readJson(target: string): Promise<unknown> {
|
|
return JSON.parse(await readFile(target, "utf8"));
|
|
}
|
|
|
|
async function loadCatalogExport(
|
|
target: string,
|
|
exportName: string,
|
|
): Promise<unknown> {
|
|
if (target.endsWith(".json")) {
|
|
const document = await readJson(target);
|
|
return (document as Readonly<Record<string, unknown>>)[exportName];
|
|
}
|
|
const module = (await import(
|
|
`${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}`
|
|
)) as Readonly<Record<string, unknown>>;
|
|
return module[exportName];
|
|
}
|
|
|
|
async function checkScenarioContribution(
|
|
contribution: z.output<typeof scenarioContributionSchema>,
|
|
): Promise<void> {
|
|
const catalogPath = explicitCatalogPath ?? contribution.path;
|
|
const receiptPath = explicitReceiptPath ?? contribution.receiptPath;
|
|
let expectations: readonly HttpScenarioExpectation[];
|
|
try {
|
|
expectations = z
|
|
.array(httpScenarioExpectationSchema)
|
|
.parse(
|
|
await loadCatalogExport(catalogPath, contribution.expectationExport),
|
|
);
|
|
} catch (error) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
facts.declaredScenarioExecutions += expectations.length;
|
|
facts.sharedScenarios += new Set(
|
|
expectations.map((entry) => entry.scenarioId),
|
|
).size;
|
|
const declarationIds = expectations.map((entry) => entry.executionId);
|
|
if (declarationIds.length === 0) {
|
|
failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`);
|
|
}
|
|
const duplicateDeclarations = duplicates(declarationIds);
|
|
if (duplicateDeclarations.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`,
|
|
);
|
|
}
|
|
for (const entry of expectations) {
|
|
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
let receipt: HttpScenarioReceipt;
|
|
try {
|
|
const parsedReceipt = httpScenarioReceiptSchema.safeParse(
|
|
await readJson(receiptPath),
|
|
);
|
|
if (!parsedReceipt.success) {
|
|
if (
|
|
parsedReceipt.error.issues.some((issue) =>
|
|
issue.message.includes("must be sorted by execution ID"),
|
|
)
|
|
) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`,
|
|
);
|
|
}
|
|
if (
|
|
parsedReceipt.error.issues.some((issue) =>
|
|
issue.message.includes("row IDs must exactly equal executed IDs"),
|
|
)
|
|
) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
|
}
|
|
throw parsedReceipt.error;
|
|
}
|
|
receipt = parsedReceipt.data;
|
|
} catch (error) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
return;
|
|
}
|
|
facts.executedScenarioExecutions += receipt.rows.length;
|
|
if (receipt.schemaVersion !== contribution.receiptSchemaVersion) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`,
|
|
);
|
|
}
|
|
if (receipt.rows.length === 0 || receipt.executedIds.length === 0) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`);
|
|
}
|
|
if (receipt.catalogTotal !== declarationIds.length) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`,
|
|
);
|
|
}
|
|
const duplicateExecutedIds = duplicates(receipt.executedIds);
|
|
const rowIds = receipt.rows.map((row) => row.executionId);
|
|
const duplicateRows = duplicates(rowIds);
|
|
if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`,
|
|
);
|
|
}
|
|
if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`);
|
|
}
|
|
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
|
}
|
|
|
|
const declared = new Set(declarationIds);
|
|
const executed = new Set(receipt.executedIds);
|
|
const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort();
|
|
const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort();
|
|
if (missing.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`,
|
|
);
|
|
}
|
|
if (extra.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`,
|
|
);
|
|
}
|
|
|
|
const expectedDigest = computeHttpScenarioCatalogDigest(
|
|
receipt.schemaVersion,
|
|
expectations,
|
|
);
|
|
if (receipt.catalogDigest !== expectedDigest) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`,
|
|
);
|
|
}
|
|
|
|
const declarationsById = new Map(
|
|
expectations.map((entry) => [entry.executionId, entry] as const),
|
|
);
|
|
for (const row of receipt.rows) {
|
|
const declaration = declarationsById.get(row.executionId);
|
|
if (!declaration) continue;
|
|
if (!sameScenarioJson(row.expected, declaration.expected)) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`,
|
|
);
|
|
}
|
|
if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`,
|
|
);
|
|
}
|
|
if (!sameScenarioJson(row.observed, row.expected)) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!scenarioOnly) {
|
|
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
|
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
|
);
|
|
for (const file of sourceFiles) {
|
|
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
|
const source = await readFile(file, "utf8");
|
|
facts.scannedFiles += 1;
|
|
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
|
if (skipPattern.test(source)) {
|
|
const quarantine =
|
|
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
|
if (!quarantine.test(source)) {
|
|
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
|
}
|
|
}
|
|
const wholeUiMask =
|
|
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
|
if (wholeUiMask.test(source)) {
|
|
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
|
}
|
|
}
|
|
}
|
|
|
|
let parsedPolicy: z.output<typeof policySchema> | undefined;
|
|
if (!fixtureMode || scenarioOnly) {
|
|
try {
|
|
parsedPolicy = policySchema.parse(await readJson(policyPath));
|
|
} catch (error) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
if (parsedPolicy) {
|
|
for (const contract of parsedPolicy.sourceContracts) {
|
|
let source: string;
|
|
try {
|
|
source = await readFile(contract.path, "utf8");
|
|
} catch (error) {
|
|
failures.push(
|
|
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
|
);
|
|
continue;
|
|
}
|
|
for (const token of contract.requiredTokens) {
|
|
if (!source.includes(token)) {
|
|
failures.push(
|
|
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
const shouldCheckScenarioExecutions =
|
|
scenarioOnly ||
|
|
(!sourceOnly && !skipScenarioExecutions && !fixtureMode);
|
|
if (shouldCheckScenarioExecutions) {
|
|
if (parsedPolicy.scenarioCatalogs.length === 0) {
|
|
failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs");
|
|
}
|
|
if (
|
|
(explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) &&
|
|
parsedPolicy.scenarioCatalogs.length !== 1
|
|
) {
|
|
failures.push(
|
|
"[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution",
|
|
);
|
|
} else {
|
|
for (const contribution of parsedPolicy.scenarioCatalogs) {
|
|
await checkScenarioContribution(contribution);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!fixtureMode && !scenarioOnly) {
|
|
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
|
for (const token of [
|
|
"pnpm build",
|
|
"pnpm preview",
|
|
"reuseExistingServer: false",
|
|
'"junit"',
|
|
'trace: "retain-on-failure"',
|
|
'"chromium-compact"',
|
|
'"firefox"',
|
|
'"webkit"',
|
|
]) {
|
|
if (!e2eConfig.includes(token)) {
|
|
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
|
}
|
|
}
|
|
|
|
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
|
/\.spec\.ts$/.test(file),
|
|
);
|
|
for (const file of e2eFiles) {
|
|
const source = await readFile(file, "utf8");
|
|
if (!source.includes("support/browser/strict-browser-test")) {
|
|
failures.push(`${file}: bypasses strict browser fixture`);
|
|
}
|
|
}
|
|
|
|
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
|
|
(file) => file.endsWith(".png"),
|
|
);
|
|
facts.visualBaselines = baselineFiles.length;
|
|
if (facts.visualBaselines < 4) {
|
|
failures.push("visual baseline requires at least four risk surfaces");
|
|
}
|
|
for (const required of [
|
|
"playwright.storybook.config.ts",
|
|
"playwright.visual.config.ts",
|
|
"tests/storybook/workshop.spec.ts",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`test evidence missing ${required}`);
|
|
}
|
|
}
|
|
|
|
if (!sourceOnly) {
|
|
for (const required of [
|
|
"artifacts/tests/storybook/results.xml",
|
|
"artifacts/tests/visual/results.xml",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`test evidence missing ${required}`);
|
|
}
|
|
}
|
|
|
|
for (const required of [
|
|
"dist/index.html",
|
|
"dist/config.json",
|
|
"dist/release-manifest.json",
|
|
"dist/runtime-config.schema.json",
|
|
"dist/.vite/manifest.json",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`built-dist contract missing ${required}`);
|
|
}
|
|
}
|
|
const sourceMaps = (await filesBelow("dist")).filter((file) =>
|
|
file.endsWith(".map"),
|
|
);
|
|
if (sourceMaps.length > 0) {
|
|
failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const report = {
|
|
schemaVersion: 2 as const,
|
|
sourceRoot,
|
|
status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const),
|
|
facts,
|
|
failures,
|
|
};
|
|
await mkdir(path.dirname(artifactPath), { recursive: true });
|
|
await writeValidatedJsonArtifact({
|
|
path: artifactPath,
|
|
schema: testEvidenceReportSchema,
|
|
value: report,
|
|
});
|
|
if (failures.length > 0) {
|
|
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
|
|
process.exit(1);
|
|
}
|
|
process.stdout.write(
|
|
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`,
|
|
);
|