263 lines
8.2 KiB
TypeScript
263 lines
8.2 KiB
TypeScript
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
type ScenarioCatalogContribution = Readonly<{
|
|
owner: string;
|
|
path: string;
|
|
arrayExport: string;
|
|
minimumEntries: number;
|
|
}>;
|
|
|
|
type SourceContractContribution = Readonly<{
|
|
owner: string;
|
|
path: string;
|
|
requiredTokens: readonly string[];
|
|
}>;
|
|
|
|
type TestEvidencePolicy = Readonly<{
|
|
schemaVersion: number;
|
|
scenarioCatalogs: readonly unknown[];
|
|
sourceContracts: readonly unknown[];
|
|
}>;
|
|
|
|
function argumentValue(name: string, fallback: string): string {
|
|
const index = process.argv.indexOf(name);
|
|
return index >= 0 && process.argv[index + 1]
|
|
? process.argv[index + 1]
|
|
: fallback;
|
|
}
|
|
|
|
const sourceRoot = argumentValue("--source-root", "tests");
|
|
const artifactPath = argumentValue(
|
|
"--artifact",
|
|
"artifacts/quality/test-evidence.json",
|
|
);
|
|
const policyPath = argumentValue(
|
|
"--policy",
|
|
"config/testing/test-evidence.json",
|
|
);
|
|
const fixtureMode = sourceRoot !== "tests";
|
|
const sourceOnly = process.argv.includes("--source-only");
|
|
const failures: string[] = [];
|
|
const facts = {
|
|
scannedFiles: 0,
|
|
visualBaselines: 0,
|
|
sharedScenarios: 0,
|
|
};
|
|
|
|
async function filesBelow(target: string): Promise<string[]> {
|
|
try {
|
|
const metadata = await stat(target);
|
|
if (metadata.isFile()) return [target];
|
|
const entries = await readdir(target, { withFileTypes: true });
|
|
const groups = await Promise.all(
|
|
entries.map((entry) => filesBelow(path.join(target, entry.name))),
|
|
);
|
|
return groups.flat();
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function escapeRegExp(value: string): string {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
|
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
|
);
|
|
for (const file of sourceFiles) {
|
|
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
|
const source = await readFile(file, "utf8");
|
|
facts.scannedFiles += 1;
|
|
const skipPattern =
|
|
/\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
|
if (skipPattern.test(source)) {
|
|
const quarantine =
|
|
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
|
if (!quarantine.test(source)) {
|
|
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
|
}
|
|
}
|
|
const wholeUiMask =
|
|
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
|
if (wholeUiMask.test(source)) {
|
|
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
|
}
|
|
}
|
|
|
|
if (!fixtureMode) {
|
|
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
|
for (const token of [
|
|
"pnpm build",
|
|
"pnpm preview",
|
|
"reuseExistingServer: false",
|
|
'"junit"',
|
|
'trace: "retain-on-failure"',
|
|
'"chromium-compact"',
|
|
'"firefox"',
|
|
'"webkit"',
|
|
]) {
|
|
if (!e2eConfig.includes(token)) {
|
|
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
|
}
|
|
}
|
|
|
|
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
|
/\.spec\.ts$/.test(file),
|
|
);
|
|
for (const file of e2eFiles) {
|
|
const source = await readFile(file, "utf8");
|
|
if (!source.includes("support/browser/strict-browser-test")) {
|
|
failures.push(`${file}: bypasses strict browser fixture`);
|
|
}
|
|
}
|
|
|
|
let evidencePolicy: unknown;
|
|
try {
|
|
evidencePolicy = JSON.parse(await readFile(policyPath, "utf8"));
|
|
} catch (error) {
|
|
failures.push(
|
|
`${policyPath}: cannot read test evidence policy (${error instanceof Error ? error.message : String(error)})`,
|
|
);
|
|
}
|
|
const policy = evidencePolicy as Partial<TestEvidencePolicy> | undefined;
|
|
if (
|
|
policy?.schemaVersion !== 1 ||
|
|
!Array.isArray(policy.scenarioCatalogs) ||
|
|
!Array.isArray(policy.sourceContracts)
|
|
) {
|
|
failures.push(`${policyPath}: invalid test evidence policy`);
|
|
} else {
|
|
for (const candidate of policy.scenarioCatalogs) {
|
|
const contribution = candidate as Partial<ScenarioCatalogContribution>;
|
|
const minimumEntries = contribution.minimumEntries;
|
|
if (
|
|
typeof contribution?.owner !== "string" ||
|
|
typeof contribution?.path !== "string" ||
|
|
typeof contribution?.arrayExport !== "string" ||
|
|
typeof minimumEntries !== "number" ||
|
|
!Number.isInteger(minimumEntries) ||
|
|
minimumEntries < 1
|
|
) {
|
|
failures.push(`${policyPath}: invalid scenario catalog contribution`);
|
|
continue;
|
|
}
|
|
let source: string;
|
|
try {
|
|
source = await readFile(contribution.path, "utf8");
|
|
} catch (error) {
|
|
failures.push(
|
|
`${contribution.path}: cannot read scenario catalog (${error instanceof Error ? error.message : String(error)})`,
|
|
);
|
|
continue;
|
|
}
|
|
const arrayPattern = new RegExp(
|
|
`${escapeRegExp(contribution.arrayExport)}\\s*=\\s*Object\\.freeze\\(\\[([\\s\\S]*?)\\]\\s*as const\\)`,
|
|
);
|
|
const entryCount = (
|
|
arrayPattern.exec(source)?.[1]?.match(/"[^"]+"/g) ?? []
|
|
).length;
|
|
facts.sharedScenarios += entryCount;
|
|
if (entryCount < minimumEntries) {
|
|
failures.push(
|
|
`${contribution.path}: ${contribution.owner} requires at least ${minimumEntries} shared scenarios`,
|
|
);
|
|
}
|
|
}
|
|
for (const candidate of policy.sourceContracts) {
|
|
const contract = candidate as Partial<SourceContractContribution>;
|
|
const requiredTokens = contract.requiredTokens;
|
|
if (
|
|
typeof contract?.owner !== "string" ||
|
|
typeof contract?.path !== "string" ||
|
|
!Array.isArray(requiredTokens) ||
|
|
requiredTokens.some((token: unknown) => typeof token !== "string")
|
|
) {
|
|
failures.push(`${policyPath}: invalid source contract contribution`);
|
|
continue;
|
|
}
|
|
let source: string;
|
|
try {
|
|
source = await readFile(contract.path, "utf8");
|
|
} catch (error) {
|
|
failures.push(
|
|
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
|
);
|
|
continue;
|
|
}
|
|
for (const token of requiredTokens as readonly string[]) {
|
|
if (!source.includes(token)) {
|
|
failures.push(
|
|
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
|
|
(file) => file.endsWith(".png"),
|
|
);
|
|
facts.visualBaselines = baselineFiles.length;
|
|
if (facts.visualBaselines < 4) {
|
|
failures.push("visual baseline requires at least four risk surfaces");
|
|
}
|
|
for (const required of [
|
|
"playwright.storybook.config.ts",
|
|
"playwright.visual.config.ts",
|
|
"tests/storybook/workshop.spec.ts",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`test evidence missing ${required}`);
|
|
}
|
|
}
|
|
|
|
if (!sourceOnly) {
|
|
for (const required of [
|
|
"artifacts/tests/storybook/results.xml",
|
|
"artifacts/tests/visual/results.xml",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`test evidence missing ${required}`);
|
|
}
|
|
}
|
|
|
|
const requiredBuiltFiles = [
|
|
"dist/index.html",
|
|
"dist/config.json",
|
|
"dist/release-manifest.json",
|
|
"dist/runtime-config.schema.json",
|
|
"dist/.vite/manifest.json",
|
|
];
|
|
for (const required of requiredBuiltFiles) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`built-dist contract missing ${required}`);
|
|
}
|
|
}
|
|
const sourceMaps = (await filesBelow("dist")).filter((file) =>
|
|
file.endsWith(".map"),
|
|
);
|
|
if (sourceMaps.length > 0) {
|
|
failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const report = {
|
|
schemaVersion: 1,
|
|
sourceRoot,
|
|
status: failures.length === 0 ? "PASS" : "FAIL",
|
|
facts,
|
|
failures,
|
|
};
|
|
await mkdir(path.dirname(artifactPath), { recursive: true });
|
|
await writeFile(artifactPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
if (failures.length > 0) {
|
|
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
|
|
process.exit(1);
|
|
}
|
|
process.stdout.write(
|
|
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.sharedScenarios} scenarios)\n`,
|
|
);
|