Files
tech-log-frontend/scripts/check-test-evidence.ts
T
DongHyeonka c80320dfc7 chore: sync the frontend template from ebea300 to af3d8e6
Carries 359 template commits, tag v0.2.0 plus one documentation commit: the
generator-built feature index with its app, bounded-context and feature
declarations, the src/platform kernel and scale tooling, the 2026-09-24 src
review fixes, and the 2026-09-25 maturity review round with its CHANGELOG
[보안]/[계약] rows. 691 of the 824 template paths are applied, 85 of them
merged with product changes; 127 are not carried.

Product identity is unchanged: name, version 0.1.0, routes, design and the
TechLog domain.

Three decisions worth naming. TechLog stays composed directly, as the last
two syncs recorded; its directory carries a NOT_INSTALLED marker that says so,
and every installed-* composer takes the template's projection with TechLog
added on top. The reference sample is no longer installed: the template now
installs a feature only through feature-module.ts and context-routes.ts,
which load the screens this product deleted, so its three operations, six
schemas and one topic leave the registries, recorded as registry change
evidence. And AppRouter mounts the platform and TechLog routes only; the
template's bounded-context route gateway waits for a generator-installed
feature.

Required fixes: readSubject on the BFF session owner (without it every HTTP
Studio session would close as integration-failed), zod through the facade in
nine TechLog files, noUnusedLocals, and public/config.json written out to
match the local profile. Derived baselines were recomputed after reproducing
both sides' values: CI contract 101 commands / 106 references / 141
artifacts, gate shape c9adcf85; registry approval 2dc6c3a3; the runtime
config schema baseline equals the schema at 8210d48.

Verified here: types, lint, architecture, adapter inventory, CI contract,
i18n, design system, diagnostics, browser security, dev release manifest,
TechLog contract and build pass; 220 component, 155 integration, 15
contract, 418 TechLog, 43 reference-feature, 18 recipe and 46
release-coherence tests pass, and 2381 of 2383 unit tests. Reproduced at
8210d48 first and unchanged: check:registries (byte-identical),
test:runtime-schema, verify:documentation. Brought by the template and
escalated rather than suppressed: check:request-fanout on TechLog's cursor
pagination, check:remediation-ledger (fails upstream too), the shared-chunk
hash test, and the registry-dependent sample removal and second-feature
tests. Playwright suites were not run.

Full record, the five open questions and the follow-ups in
docs/operations/template-sync-2026-09-29.md.

(cherry picked from commit b2170cf265dea45dc9cc05d3c61e627e97f31cdb)
2026-11-12 08:28:54 +09:00

472 lines
15 KiB
TypeScript

import { mkdir, readFile, readdir, stat } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { z } from "zod";
import {
computeHttpScenarioCatalogDigest,
httpScenarioExpectationSchema,
httpScenarioReceiptSchema,
sameScenarioJson,
type HttpScenarioExpectation,
type HttpScenarioReceipt,
} from "./lib/http-scenario-evidence.ts";
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts";
const scenarioContributionSchema = z
.object({
owner: z.string().min(1),
path: z.string().min(1),
expectationExport: z.string().min(1),
receiptPath: z.string().min(1),
receiptSchemaVersion: z.number().int().positive(),
})
.strict();
const sourceContractSchema = z
.object({
owner: z.string().min(1),
path: z.string().min(1),
requiredTokens: z.array(z.string().min(1)),
})
.strict();
const policySchema = z
.object({
schemaVersion: z.literal(2),
scenarioCatalogs: z.array(scenarioContributionSchema),
sourceContracts: z.array(sourceContractSchema),
})
.strict();
function argumentValue(name: string, fallback: string): string {
const index = process.argv.indexOf(name);
return index >= 0 && process.argv[index + 1]
? process.argv[index + 1]
: fallback;
}
function optionalArgumentValue(name: string): string | undefined {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
const sourceRoot = argumentValue("--source-root", "tests");
const artifactPath = argumentValue(
"--artifact",
"artifacts/quality/test-evidence.json",
);
const policyPath = argumentValue(
"--policy",
"config/testing/test-evidence.json",
);
const explicitCatalogPath = optionalArgumentValue("--catalog");
const explicitReceiptPath = optionalArgumentValue("--receipt");
const fixtureMode = sourceRoot !== "tests";
const sourceOnly = process.argv.includes("--source-only");
const scenarioOnly = process.argv.includes("--scenario-only");
const skipScenarioExecutions = process.argv.includes(
"--skip-scenario-executions",
);
const failures: string[] = [];
const facts = {
scannedFiles: 0,
visualBaselines: 0,
sharedScenarios: 0,
declaredScenarioExecutions: 0,
executedScenarioExecutions: 0,
};
async function filesBelow(target: string): Promise<string[]> {
try {
const metadata = await stat(target);
if (metadata.isFile()) return [target];
const entries = await readdir(target, { withFileTypes: true });
const groups = await Promise.all(
entries.map((entry) => filesBelow(path.join(target, entry.name))),
);
return groups.flat();
} catch {
return [];
}
}
function duplicates(values: readonly string[]): string[] {
const seen = new Set<string>();
const duplicate = new Set<string>();
for (const value of values) {
if (seen.has(value)) duplicate.add(value);
seen.add(value);
}
return [...duplicate].sort();
}
function isSorted(values: readonly string[]): boolean {
return values.every(
(value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0,
);
}
async function readJson(target: string): Promise<unknown> {
return JSON.parse(await readFile(target, "utf8"));
}
async function loadCatalogExport(
target: string,
exportName: string,
): Promise<unknown> {
if (target.endsWith(".json")) {
const document = await readJson(target);
return (document as Readonly<Record<string, unknown>>)[exportName];
}
const module = (await import(
`${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}`
)) as Readonly<Record<string, unknown>>;
return module[exportName];
}
async function checkScenarioContribution(
contribution: z.output<typeof scenarioContributionSchema>,
): Promise<void> {
const catalogPath = explicitCatalogPath ?? contribution.path;
const receiptPath = explicitReceiptPath ?? contribution.receiptPath;
let expectations: readonly HttpScenarioExpectation[];
try {
expectations = z
.array(httpScenarioExpectationSchema)
.parse(
await loadCatalogExport(catalogPath, contribution.expectationExport),
);
} catch (error) {
failures.push(
`[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`,
);
return;
}
facts.declaredScenarioExecutions += expectations.length;
facts.sharedScenarios += new Set(
expectations.map((entry) => entry.scenarioId),
).size;
const declarationIds = expectations.map((entry) => entry.executionId);
if (declarationIds.length === 0) {
failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`);
}
const duplicateDeclarations = duplicates(declarationIds);
if (duplicateDeclarations.length > 0) {
failures.push(
`[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`,
);
}
for (const entry of expectations) {
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`,
);
}
}
let receipt: HttpScenarioReceipt;
try {
const parsedReceipt = httpScenarioReceiptSchema.safeParse(
await readJson(receiptPath),
);
if (!parsedReceipt.success) {
if (
parsedReceipt.error.issues.some((issue) =>
issue.message.includes("must be sorted by execution ID"),
)
) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`,
);
}
if (
parsedReceipt.error.issues.some((issue) =>
issue.message.includes("row IDs must exactly equal executed IDs"),
)
) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
}
throw parsedReceipt.error;
}
receipt = parsedReceipt.data;
} catch (error) {
failures.push(
`[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`,
);
return;
}
facts.executedScenarioExecutions += receipt.rows.length;
if (receipt.schemaVersion !== contribution.receiptSchemaVersion) {
failures.push(
`[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`,
);
}
if (receipt.rows.length === 0 || receipt.executedIds.length === 0) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`);
}
if (receipt.catalogTotal !== declarationIds.length) {
failures.push(
`[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`,
);
}
const duplicateExecutedIds = duplicates(receipt.executedIds);
const rowIds = receipt.rows.map((row) => row.executionId);
const duplicateRows = duplicates(rowIds);
if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`,
);
}
if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`);
}
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
}
const declared = new Set(declarationIds);
const executed = new Set(receipt.executedIds);
const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort();
const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort();
if (missing.length > 0) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`,
);
}
if (extra.length > 0) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`,
);
}
const expectedDigest = computeHttpScenarioCatalogDigest(
receipt.schemaVersion,
expectations,
);
if (receipt.catalogDigest !== expectedDigest) {
failures.push(
`[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`,
);
}
const declarationsById = new Map(
expectations.map((entry) => [entry.executionId, entry] as const),
);
for (const row of receipt.rows) {
const declaration = declarationsById.get(row.executionId);
if (!declaration) continue;
if (!sameScenarioJson(row.expected, declaration.expected)) {
failures.push(
`[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`,
);
}
if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) {
failures.push(
`[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`,
);
}
if (!sameScenarioJson(row.observed, row.expected)) {
failures.push(
`[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`,
);
}
}
}
if (!scenarioOnly) {
const sourceFiles = (await filesBelow(sourceRoot)).filter(
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
);
const today = new Date().toISOString().slice(0, 10);
for (const file of sourceFiles) {
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
const source = await readFile(file, "utf8");
facts.scannedFiles += 1;
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
}
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
if (skipPattern.test(source)) {
const quarantines = [
...source.matchAll(
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
),
];
if (quarantines.length === 0) {
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
}
for (const [, expires = ""] of quarantines) {
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
}
}
const wholeUiMask =
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
if (wholeUiMask.test(source)) {
failures.push(`${file}: screenshot mask may not cover the whole UI`);
}
}
}
let parsedPolicy: z.output<typeof policySchema> | undefined;
if (!fixtureMode || scenarioOnly) {
try {
parsedPolicy = policySchema.parse(await readJson(policyPath));
} catch (error) {
failures.push(
`[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
if (parsedPolicy) {
for (const contract of parsedPolicy.sourceContracts) {
let source: string;
try {
source = await readFile(contract.path, "utf8");
} catch (error) {
failures.push(
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
);
continue;
}
for (const token of contract.requiredTokens) {
if (!source.includes(token)) {
failures.push(
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
);
}
}
}
const shouldCheckScenarioExecutions =
scenarioOnly ||
(!sourceOnly && !skipScenarioExecutions && !fixtureMode);
if (shouldCheckScenarioExecutions) {
if (parsedPolicy.scenarioCatalogs.length === 0) {
failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs");
}
if (
(explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) &&
parsedPolicy.scenarioCatalogs.length !== 1
) {
failures.push(
"[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution",
);
} else {
for (const contribution of parsedPolicy.scenarioCatalogs) {
await checkScenarioContribution(contribution);
}
}
}
}
if (!fixtureMode && !scenarioOnly) {
const e2eConfig = await readFile("playwright.config.ts", "utf8");
for (const token of [
"pnpm build",
"pnpm preview",
"reuseExistingServer: false",
'"junit"',
'trace: "retain-on-failure"',
'"chromium-compact"',
'"firefox"',
'"webkit"',
]) {
if (!e2eConfig.includes(token)) {
failures.push(`playwright.config.ts missing release evidence token ${token}`);
}
}
// playwright.dev.config.ts spreads playwright.config.ts.
for (const config of [
"playwright.config.ts",
"playwright.visual.config.ts",
"playwright.storybook.config.ts",
"playwright.capabilities.config.ts",
]) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
/\.spec\.ts$/.test(file),
);
for (const file of e2eFiles) {
const source = await readFile(file, "utf8");
if (!source.includes("support/browser/strict-browser-test")) {
failures.push(`${file}: bypasses strict browser fixture`);
}
}
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
(file) => file.endsWith(".png"),
);
facts.visualBaselines = baselineFiles.length;
if (facts.visualBaselines < 4) {
failures.push("visual baseline requires at least four risk surfaces");
}
for (const required of [
"playwright.storybook.config.ts",
"playwright.visual.config.ts",
"tests/storybook/workshop.spec.ts",
]) {
if ((await filesBelow(required)).length === 0) {
failures.push(`test evidence missing ${required}`);
}
}
if (!sourceOnly) {
for (const required of [
"artifacts/tests/storybook/results.xml",
"artifacts/tests/visual/results.xml",
]) {
if ((await filesBelow(required)).length === 0) {
failures.push(`test evidence missing ${required}`);
}
}
for (const required of [
"dist/index.html",
"dist/config.json",
"dist/release-manifest.json",
"dist/runtime-config.schema.json",
"dist/.vite/manifest.json",
]) {
if ((await filesBelow(required)).length === 0) {
failures.push(`built-dist contract missing ${required}`);
}
}
const sourceMaps = (await filesBelow("dist")).filter((file) =>
file.endsWith(".map"),
);
if (sourceMaps.length > 0) {
failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`);
}
}
}
const report = {
schemaVersion: 2 as const,
sourceRoot,
status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const),
facts,
failures,
};
await mkdir(path.dirname(artifactPath), { recursive: true });
await writeValidatedJsonArtifact({
path: artifactPath,
schema: testEvidenceReportSchema,
value: report,
});
if (failures.length > 0) {
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
process.exit(1);
}
process.stdout.write(
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`,
);