test: execute the HTTP scenario catalog
This commit is contained in:
DongHyeonka
+333
-128
@@ -1,24 +1,62 @@
|
||||
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readFile, readdir, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
type ScenarioCatalogContribution = Readonly<{
|
||||
owner: string;
|
||||
path: string;
|
||||
arrayExport: string;
|
||||
minimumEntries: number;
|
||||
}>;
|
||||
import { z } from "zod";
|
||||
|
||||
type SourceContractContribution = Readonly<{
|
||||
owner: string;
|
||||
path: string;
|
||||
requiredTokens: readonly string[];
|
||||
}>;
|
||||
import {
|
||||
computeHttpScenarioCatalogDigest,
|
||||
httpScenarioExpectationSchema,
|
||||
httpScenarioReceiptSchema,
|
||||
sameScenarioJson,
|
||||
type HttpScenarioExpectation,
|
||||
type HttpScenarioReceipt,
|
||||
} from "./lib/http-scenario-evidence.ts";
|
||||
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
||||
|
||||
type TestEvidencePolicy = Readonly<{
|
||||
schemaVersion: number;
|
||||
scenarioCatalogs: readonly unknown[];
|
||||
sourceContracts: readonly unknown[];
|
||||
}>;
|
||||
const scenarioContributionSchema = z
|
||||
.object({
|
||||
owner: z.string().min(1),
|
||||
path: z.string().min(1),
|
||||
expectationExport: z.string().min(1),
|
||||
receiptPath: z.string().min(1),
|
||||
receiptSchemaVersion: z.number().int().positive(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const sourceContractSchema = z
|
||||
.object({
|
||||
owner: z.string().min(1),
|
||||
path: z.string().min(1),
|
||||
requiredTokens: z.array(z.string().min(1)),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const policySchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(2),
|
||||
scenarioCatalogs: z.array(scenarioContributionSchema),
|
||||
sourceContracts: z.array(sourceContractSchema),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const reportSchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(2),
|
||||
sourceRoot: z.string().min(1),
|
||||
status: z.enum(["PASS", "FAIL"]),
|
||||
facts: z
|
||||
.object({
|
||||
scannedFiles: z.number().int().nonnegative(),
|
||||
visualBaselines: z.number().int().nonnegative(),
|
||||
sharedScenarios: z.number().int().nonnegative(),
|
||||
declaredScenarioExecutions: z.number().int().nonnegative(),
|
||||
executedScenarioExecutions: z.number().int().nonnegative(),
|
||||
})
|
||||
.strict(),
|
||||
failures: z.array(z.string()),
|
||||
})
|
||||
.strict();
|
||||
|
||||
function argumentValue(name: string, fallback: string): string {
|
||||
const index = process.argv.indexOf(name);
|
||||
@@ -27,6 +65,11 @@ function argumentValue(name: string, fallback: string): string {
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function optionalArgumentValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(name);
|
||||
return index >= 0 ? process.argv[index + 1] : undefined;
|
||||
}
|
||||
|
||||
const sourceRoot = argumentValue("--source-root", "tests");
|
||||
const artifactPath = argumentValue(
|
||||
"--artifact",
|
||||
@@ -36,13 +79,21 @@ const policyPath = argumentValue(
|
||||
"--policy",
|
||||
"config/testing/test-evidence.json",
|
||||
);
|
||||
const explicitCatalogPath = optionalArgumentValue("--catalog");
|
||||
const explicitReceiptPath = optionalArgumentValue("--receipt");
|
||||
const fixtureMode = sourceRoot !== "tests";
|
||||
const sourceOnly = process.argv.includes("--source-only");
|
||||
const scenarioOnly = process.argv.includes("--scenario-only");
|
||||
const skipScenarioExecutions = process.argv.includes(
|
||||
"--skip-scenario-executions",
|
||||
);
|
||||
const failures: string[] = [];
|
||||
const facts = {
|
||||
scannedFiles: 0,
|
||||
visualBaselines: 0,
|
||||
sharedScenarios: 0,
|
||||
declaredScenarioExecutions: 0,
|
||||
executedScenarioExecutions: 0,
|
||||
};
|
||||
|
||||
async function filesBelow(target: string): Promise<string[]> {
|
||||
@@ -59,34 +110,268 @@ async function filesBelow(target: string): Promise<string[]> {
|
||||
}
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
function duplicates(values: readonly string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const duplicate = new Set<string>();
|
||||
for (const value of values) {
|
||||
if (seen.has(value)) duplicate.add(value);
|
||||
seen.add(value);
|
||||
}
|
||||
return [...duplicate].sort();
|
||||
}
|
||||
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
const skipPattern =
|
||||
/\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
function isSorted(values: readonly string[]): boolean {
|
||||
return values.every(
|
||||
(value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0,
|
||||
);
|
||||
}
|
||||
|
||||
async function readJson(target: string): Promise<unknown> {
|
||||
return JSON.parse(await readFile(target, "utf8"));
|
||||
}
|
||||
|
||||
async function loadCatalogExport(
|
||||
target: string,
|
||||
exportName: string,
|
||||
): Promise<unknown> {
|
||||
if (target.endsWith(".json")) {
|
||||
const document = await readJson(target);
|
||||
return (document as Readonly<Record<string, unknown>>)[exportName];
|
||||
}
|
||||
const module = (await import(
|
||||
`${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}`
|
||||
)) as Readonly<Record<string, unknown>>;
|
||||
return module[exportName];
|
||||
}
|
||||
|
||||
async function checkScenarioContribution(
|
||||
contribution: z.output<typeof scenarioContributionSchema>,
|
||||
): Promise<void> {
|
||||
const catalogPath = explicitCatalogPath ?? contribution.path;
|
||||
const receiptPath = explicitReceiptPath ?? contribution.receiptPath;
|
||||
let expectations: readonly HttpScenarioExpectation[];
|
||||
try {
|
||||
expectations = z
|
||||
.array(httpScenarioExpectationSchema)
|
||||
.parse(
|
||||
await loadCatalogExport(catalogPath, contribution.expectationExport),
|
||||
);
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
facts.declaredScenarioExecutions += expectations.length;
|
||||
facts.sharedScenarios += new Set(
|
||||
expectations.map((entry) => entry.scenarioId),
|
||||
).size;
|
||||
const declarationIds = expectations.map((entry) => entry.executionId);
|
||||
if (declarationIds.length === 0) {
|
||||
failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`);
|
||||
}
|
||||
const duplicateDeclarations = duplicates(declarationIds);
|
||||
if (duplicateDeclarations.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`,
|
||||
);
|
||||
}
|
||||
for (const entry of expectations) {
|
||||
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
if (wholeUiMask.test(source)) {
|
||||
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
||||
|
||||
let receipt: HttpScenarioReceipt;
|
||||
try {
|
||||
const parsedReceipt = httpScenarioReceiptSchema.safeParse(
|
||||
await readJson(receiptPath),
|
||||
);
|
||||
if (!parsedReceipt.success) {
|
||||
if (
|
||||
parsedReceipt.error.issues.some((issue) =>
|
||||
issue.message.includes("must be sorted by execution ID"),
|
||||
)
|
||||
) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
parsedReceipt.error.issues.some((issue) =>
|
||||
issue.message.includes("row IDs must exactly equal executed IDs"),
|
||||
)
|
||||
) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
||||
}
|
||||
throw parsedReceipt.error;
|
||||
}
|
||||
receipt = parsedReceipt.data;
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
facts.executedScenarioExecutions += receipt.rows.length;
|
||||
if (receipt.schemaVersion !== contribution.receiptSchemaVersion) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`,
|
||||
);
|
||||
}
|
||||
if (receipt.rows.length === 0 || receipt.executedIds.length === 0) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`);
|
||||
}
|
||||
if (receipt.catalogTotal !== declarationIds.length) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`,
|
||||
);
|
||||
}
|
||||
const duplicateExecutedIds = duplicates(receipt.executedIds);
|
||||
const rowIds = receipt.rows.map((row) => row.executionId);
|
||||
const duplicateRows = duplicates(rowIds);
|
||||
if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`);
|
||||
}
|
||||
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
|
||||
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
||||
}
|
||||
|
||||
const declared = new Set(declarationIds);
|
||||
const executed = new Set(receipt.executedIds);
|
||||
const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort();
|
||||
const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort();
|
||||
if (missing.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (extra.length > 0) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const expectedDigest = computeHttpScenarioCatalogDigest(
|
||||
receipt.schemaVersion,
|
||||
expectations,
|
||||
);
|
||||
if (receipt.catalogDigest !== expectedDigest) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`,
|
||||
);
|
||||
}
|
||||
|
||||
const declarationsById = new Map(
|
||||
expectations.map((entry) => [entry.executionId, entry] as const),
|
||||
);
|
||||
for (const row of receipt.rows) {
|
||||
const declaration = declarationsById.get(row.executionId);
|
||||
if (!declaration) continue;
|
||||
if (!sameScenarioJson(row.expected, declaration.expected)) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`,
|
||||
);
|
||||
}
|
||||
if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`,
|
||||
);
|
||||
}
|
||||
if (!sameScenarioJson(row.observed, row.expected)) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!fixtureMode) {
|
||||
if (!scenarioOnly) {
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
if (wholeUiMask.test(source)) {
|
||||
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let parsedPolicy: z.output<typeof policySchema> | undefined;
|
||||
if (!fixtureMode || scenarioOnly) {
|
||||
try {
|
||||
parsedPolicy = policySchema.parse(await readJson(policyPath));
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedPolicy) {
|
||||
for (const contract of parsedPolicy.sourceContracts) {
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(contract.path, "utf8");
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const token of contract.requiredTokens) {
|
||||
if (!source.includes(token)) {
|
||||
failures.push(
|
||||
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const shouldCheckScenarioExecutions =
|
||||
scenarioOnly ||
|
||||
(!sourceOnly && !skipScenarioExecutions && !fixtureMode);
|
||||
if (shouldCheckScenarioExecutions) {
|
||||
if (parsedPolicy.scenarioCatalogs.length === 0) {
|
||||
failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs");
|
||||
}
|
||||
if (
|
||||
(explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) &&
|
||||
parsedPolicy.scenarioCatalogs.length !== 1
|
||||
) {
|
||||
failures.push(
|
||||
"[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution",
|
||||
);
|
||||
} else {
|
||||
for (const contribution of parsedPolicy.scenarioCatalogs) {
|
||||
await checkScenarioContribution(contribution);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!fixtureMode && !scenarioOnly) {
|
||||
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
||||
for (const token of [
|
||||
"pnpm build",
|
||||
@@ -113,89 +398,6 @@ if (!fixtureMode) {
|
||||
}
|
||||
}
|
||||
|
||||
let evidencePolicy: unknown;
|
||||
try {
|
||||
evidencePolicy = JSON.parse(await readFile(policyPath, "utf8"));
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${policyPath}: cannot read test evidence policy (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
}
|
||||
const policy = evidencePolicy as Partial<TestEvidencePolicy> | undefined;
|
||||
if (
|
||||
policy?.schemaVersion !== 1 ||
|
||||
!Array.isArray(policy.scenarioCatalogs) ||
|
||||
!Array.isArray(policy.sourceContracts)
|
||||
) {
|
||||
failures.push(`${policyPath}: invalid test evidence policy`);
|
||||
} else {
|
||||
for (const candidate of policy.scenarioCatalogs) {
|
||||
const contribution = candidate as Partial<ScenarioCatalogContribution>;
|
||||
const minimumEntries = contribution.minimumEntries;
|
||||
if (
|
||||
typeof contribution?.owner !== "string" ||
|
||||
typeof contribution?.path !== "string" ||
|
||||
typeof contribution?.arrayExport !== "string" ||
|
||||
typeof minimumEntries !== "number" ||
|
||||
!Number.isInteger(minimumEntries) ||
|
||||
minimumEntries < 1
|
||||
) {
|
||||
failures.push(`${policyPath}: invalid scenario catalog contribution`);
|
||||
continue;
|
||||
}
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(contribution.path, "utf8");
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${contribution.path}: cannot read scenario catalog (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const arrayPattern = new RegExp(
|
||||
`${escapeRegExp(contribution.arrayExport)}\\s*=\\s*Object\\.freeze\\(\\[([\\s\\S]*?)\\]\\s*as const\\)`,
|
||||
);
|
||||
const entryCount = (
|
||||
arrayPattern.exec(source)?.[1]?.match(/"[^"]+"/g) ?? []
|
||||
).length;
|
||||
facts.sharedScenarios += entryCount;
|
||||
if (entryCount < minimumEntries) {
|
||||
failures.push(
|
||||
`${contribution.path}: ${contribution.owner} requires at least ${minimumEntries} shared scenarios`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const candidate of policy.sourceContracts) {
|
||||
const contract = candidate as Partial<SourceContractContribution>;
|
||||
const requiredTokens = contract.requiredTokens;
|
||||
if (
|
||||
typeof contract?.owner !== "string" ||
|
||||
typeof contract?.path !== "string" ||
|
||||
!Array.isArray(requiredTokens) ||
|
||||
requiredTokens.some((token: unknown) => typeof token !== "string")
|
||||
) {
|
||||
failures.push(`${policyPath}: invalid source contract contribution`);
|
||||
continue;
|
||||
}
|
||||
let source: string;
|
||||
try {
|
||||
source = await readFile(contract.path, "utf8");
|
||||
} catch (error) {
|
||||
failures.push(
|
||||
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const token of requiredTokens as readonly string[]) {
|
||||
if (!source.includes(token)) {
|
||||
failures.push(
|
||||
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
|
||||
(file) => file.endsWith(".png"),
|
||||
);
|
||||
@@ -223,14 +425,13 @@ if (!fixtureMode) {
|
||||
}
|
||||
}
|
||||
|
||||
const requiredBuiltFiles = [
|
||||
for (const required of [
|
||||
"dist/index.html",
|
||||
"dist/config.json",
|
||||
"dist/release-manifest.json",
|
||||
"dist/runtime-config.schema.json",
|
||||
"dist/.vite/manifest.json",
|
||||
];
|
||||
for (const required of requiredBuiltFiles) {
|
||||
]) {
|
||||
if ((await filesBelow(required)).length === 0) {
|
||||
failures.push(`built-dist contract missing ${required}`);
|
||||
}
|
||||
@@ -245,18 +446,22 @@ if (!fixtureMode) {
|
||||
}
|
||||
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
schemaVersion: 2 as const,
|
||||
sourceRoot,
|
||||
status: failures.length === 0 ? "PASS" : "FAIL",
|
||||
status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const),
|
||||
facts,
|
||||
failures,
|
||||
};
|
||||
await mkdir(path.dirname(artifactPath), { recursive: true });
|
||||
await writeFile(artifactPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
await writeValidatedJsonArtifact({
|
||||
path: artifactPath,
|
||||
schema: reportSchema,
|
||||
value: report,
|
||||
});
|
||||
if (failures.length > 0) {
|
||||
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(
|
||||
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.sharedScenarios} scenarios)\n`,
|
||||
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user