Carries 359 template commits, tag v0.2.0 plus one documentation commit: the generator-built feature index with its app, bounded-context and feature declarations, the src/platform kernel and scale tooling, the 2026-09-24 src review fixes, and the 2026-09-25 maturity review round with its CHANGELOG [보안]/[계약] rows. 691 of the 824 template paths are applied, 85 of them merged with product changes; 127 are not carried. Product identity is unchanged: name, version 0.1.0, routes, design and the TechLog domain. Three decisions worth naming. TechLog stays composed directly, as the last two syncs recorded; its directory carries a NOT_INSTALLED marker that says so, and every installed-* composer takes the template's projection with TechLog added on top. The reference sample is no longer installed: the template now installs a feature only through feature-module.ts and context-routes.ts, which load the screens this product deleted, so its three operations, six schemas and one topic leave the registries, recorded as registry change evidence. And AppRouter mounts the platform and TechLog routes only; the template's bounded-context route gateway waits for a generator-installed feature. Required fixes: readSubject on the BFF session owner (without it every HTTP Studio session would close as integration-failed), zod through the facade in nine TechLog files, noUnusedLocals, and public/config.json written out to match the local profile. Derived baselines were recomputed after reproducing both sides' values: CI contract 101 commands / 106 references / 141 artifacts, gate shape c9adcf85; registry approval 2dc6c3a3; the runtime config schema baseline equals the schema at8210d48. Verified here: types, lint, architecture, adapter inventory, CI contract, i18n, design system, diagnostics, browser security, dev release manifest, TechLog contract and build pass; 220 component, 155 integration, 15 contract, 418 TechLog, 43 reference-feature, 18 recipe and 46 release-coherence tests pass, and 2381 of 2383 unit tests. Reproduced at8210d48first and unchanged: check:registries (byte-identical), test:runtime-schema, verify:documentation. Brought by the template and escalated rather than suppressed: check:request-fanout on TechLog's cursor pagination, check:remediation-ledger (fails upstream too), the shared-chunk hash test, and the registry-dependent sample removal and second-feature tests. Playwright suites were not run. Full record, the five open questions and the follow-ups in docs/operations/template-sync-2026-09-29.md. (cherry picked from commit b2170cf265dea45dc9cc05d3c61e627e97f31cdb)
472 lines
15 KiB
TypeScript
472 lines
15 KiB
TypeScript
import { mkdir, readFile, readdir, stat } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
import { z } from "zod";
|
|
|
|
import {
|
|
computeHttpScenarioCatalogDigest,
|
|
httpScenarioExpectationSchema,
|
|
httpScenarioReceiptSchema,
|
|
sameScenarioJson,
|
|
type HttpScenarioExpectation,
|
|
type HttpScenarioReceipt,
|
|
} from "./lib/http-scenario-evidence.ts";
|
|
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
|
import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts";
|
|
|
|
const scenarioContributionSchema = z
|
|
.object({
|
|
owner: z.string().min(1),
|
|
path: z.string().min(1),
|
|
expectationExport: z.string().min(1),
|
|
receiptPath: z.string().min(1),
|
|
receiptSchemaVersion: z.number().int().positive(),
|
|
})
|
|
.strict();
|
|
|
|
const sourceContractSchema = z
|
|
.object({
|
|
owner: z.string().min(1),
|
|
path: z.string().min(1),
|
|
requiredTokens: z.array(z.string().min(1)),
|
|
})
|
|
.strict();
|
|
|
|
const policySchema = z
|
|
.object({
|
|
schemaVersion: z.literal(2),
|
|
scenarioCatalogs: z.array(scenarioContributionSchema),
|
|
sourceContracts: z.array(sourceContractSchema),
|
|
})
|
|
.strict();
|
|
|
|
function argumentValue(name: string, fallback: string): string {
|
|
const index = process.argv.indexOf(name);
|
|
return index >= 0 && process.argv[index + 1]
|
|
? process.argv[index + 1]
|
|
: fallback;
|
|
}
|
|
|
|
function optionalArgumentValue(name: string): string | undefined {
|
|
const index = process.argv.indexOf(name);
|
|
return index >= 0 ? process.argv[index + 1] : undefined;
|
|
}
|
|
|
|
const sourceRoot = argumentValue("--source-root", "tests");
|
|
const artifactPath = argumentValue(
|
|
"--artifact",
|
|
"artifacts/quality/test-evidence.json",
|
|
);
|
|
const policyPath = argumentValue(
|
|
"--policy",
|
|
"config/testing/test-evidence.json",
|
|
);
|
|
const explicitCatalogPath = optionalArgumentValue("--catalog");
|
|
const explicitReceiptPath = optionalArgumentValue("--receipt");
|
|
const fixtureMode = sourceRoot !== "tests";
|
|
const sourceOnly = process.argv.includes("--source-only");
|
|
const scenarioOnly = process.argv.includes("--scenario-only");
|
|
const skipScenarioExecutions = process.argv.includes(
|
|
"--skip-scenario-executions",
|
|
);
|
|
const failures: string[] = [];
|
|
const facts = {
|
|
scannedFiles: 0,
|
|
visualBaselines: 0,
|
|
sharedScenarios: 0,
|
|
declaredScenarioExecutions: 0,
|
|
executedScenarioExecutions: 0,
|
|
};
|
|
|
|
async function filesBelow(target: string): Promise<string[]> {
|
|
try {
|
|
const metadata = await stat(target);
|
|
if (metadata.isFile()) return [target];
|
|
const entries = await readdir(target, { withFileTypes: true });
|
|
const groups = await Promise.all(
|
|
entries.map((entry) => filesBelow(path.join(target, entry.name))),
|
|
);
|
|
return groups.flat();
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function duplicates(values: readonly string[]): string[] {
|
|
const seen = new Set<string>();
|
|
const duplicate = new Set<string>();
|
|
for (const value of values) {
|
|
if (seen.has(value)) duplicate.add(value);
|
|
seen.add(value);
|
|
}
|
|
return [...duplicate].sort();
|
|
}
|
|
|
|
function isSorted(values: readonly string[]): boolean {
|
|
return values.every(
|
|
(value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0,
|
|
);
|
|
}
|
|
|
|
async function readJson(target: string): Promise<unknown> {
|
|
return JSON.parse(await readFile(target, "utf8"));
|
|
}
|
|
|
|
async function loadCatalogExport(
|
|
target: string,
|
|
exportName: string,
|
|
): Promise<unknown> {
|
|
if (target.endsWith(".json")) {
|
|
const document = await readJson(target);
|
|
return (document as Readonly<Record<string, unknown>>)[exportName];
|
|
}
|
|
const module = (await import(
|
|
`${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}`
|
|
)) as Readonly<Record<string, unknown>>;
|
|
return module[exportName];
|
|
}
|
|
|
|
async function checkScenarioContribution(
|
|
contribution: z.output<typeof scenarioContributionSchema>,
|
|
): Promise<void> {
|
|
const catalogPath = explicitCatalogPath ?? contribution.path;
|
|
const receiptPath = explicitReceiptPath ?? contribution.receiptPath;
|
|
let expectations: readonly HttpScenarioExpectation[];
|
|
try {
|
|
expectations = z
|
|
.array(httpScenarioExpectationSchema)
|
|
.parse(
|
|
await loadCatalogExport(catalogPath, contribution.expectationExport),
|
|
);
|
|
} catch (error) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
facts.declaredScenarioExecutions += expectations.length;
|
|
facts.sharedScenarios += new Set(
|
|
expectations.map((entry) => entry.scenarioId),
|
|
).size;
|
|
const declarationIds = expectations.map((entry) => entry.executionId);
|
|
if (declarationIds.length === 0) {
|
|
failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`);
|
|
}
|
|
const duplicateDeclarations = duplicates(declarationIds);
|
|
if (duplicateDeclarations.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`,
|
|
);
|
|
}
|
|
for (const entry of expectations) {
|
|
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
let receipt: HttpScenarioReceipt;
|
|
try {
|
|
const parsedReceipt = httpScenarioReceiptSchema.safeParse(
|
|
await readJson(receiptPath),
|
|
);
|
|
if (!parsedReceipt.success) {
|
|
if (
|
|
parsedReceipt.error.issues.some((issue) =>
|
|
issue.message.includes("must be sorted by execution ID"),
|
|
)
|
|
) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`,
|
|
);
|
|
}
|
|
if (
|
|
parsedReceipt.error.issues.some((issue) =>
|
|
issue.message.includes("row IDs must exactly equal executed IDs"),
|
|
)
|
|
) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
|
}
|
|
throw parsedReceipt.error;
|
|
}
|
|
receipt = parsedReceipt.data;
|
|
} catch (error) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
return;
|
|
}
|
|
facts.executedScenarioExecutions += receipt.rows.length;
|
|
if (receipt.schemaVersion !== contribution.receiptSchemaVersion) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`,
|
|
);
|
|
}
|
|
if (receipt.rows.length === 0 || receipt.executedIds.length === 0) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`);
|
|
}
|
|
if (receipt.catalogTotal !== declarationIds.length) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`,
|
|
);
|
|
}
|
|
const duplicateExecutedIds = duplicates(receipt.executedIds);
|
|
const rowIds = receipt.rows.map((row) => row.executionId);
|
|
const duplicateRows = duplicates(rowIds);
|
|
if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`,
|
|
);
|
|
}
|
|
if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`);
|
|
}
|
|
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
|
|
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
|
|
}
|
|
|
|
const declared = new Set(declarationIds);
|
|
const executed = new Set(receipt.executedIds);
|
|
const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort();
|
|
const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort();
|
|
if (missing.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`,
|
|
);
|
|
}
|
|
if (extra.length > 0) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`,
|
|
);
|
|
}
|
|
|
|
const expectedDigest = computeHttpScenarioCatalogDigest(
|
|
receipt.schemaVersion,
|
|
expectations,
|
|
);
|
|
if (receipt.catalogDigest !== expectedDigest) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`,
|
|
);
|
|
}
|
|
|
|
const declarationsById = new Map(
|
|
expectations.map((entry) => [entry.executionId, entry] as const),
|
|
);
|
|
for (const row of receipt.rows) {
|
|
const declaration = declarationsById.get(row.executionId);
|
|
if (!declaration) continue;
|
|
if (!sameScenarioJson(row.expected, declaration.expected)) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`,
|
|
);
|
|
}
|
|
if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`,
|
|
);
|
|
}
|
|
if (!sameScenarioJson(row.observed, row.expected)) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!scenarioOnly) {
|
|
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
|
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
|
);
|
|
const today = new Date().toISOString().slice(0, 10);
|
|
for (const file of sourceFiles) {
|
|
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
|
const source = await readFile(file, "utf8");
|
|
facts.scannedFiles += 1;
|
|
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
|
|
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
|
|
}
|
|
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
|
if (skipPattern.test(source)) {
|
|
const quarantines = [
|
|
...source.matchAll(
|
|
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
|
|
),
|
|
];
|
|
if (quarantines.length === 0) {
|
|
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
|
}
|
|
for (const [, expires = ""] of quarantines) {
|
|
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
|
|
}
|
|
}
|
|
const wholeUiMask =
|
|
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
|
if (wholeUiMask.test(source)) {
|
|
failures.push(`${file}: screenshot mask may not cover the whole UI`);
|
|
}
|
|
}
|
|
}
|
|
|
|
let parsedPolicy: z.output<typeof policySchema> | undefined;
|
|
if (!fixtureMode || scenarioOnly) {
|
|
try {
|
|
parsedPolicy = policySchema.parse(await readJson(policyPath));
|
|
} catch (error) {
|
|
failures.push(
|
|
`[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
if (parsedPolicy) {
|
|
for (const contract of parsedPolicy.sourceContracts) {
|
|
let source: string;
|
|
try {
|
|
source = await readFile(contract.path, "utf8");
|
|
} catch (error) {
|
|
failures.push(
|
|
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
|
|
);
|
|
continue;
|
|
}
|
|
for (const token of contract.requiredTokens) {
|
|
if (!source.includes(token)) {
|
|
failures.push(
|
|
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
const shouldCheckScenarioExecutions =
|
|
scenarioOnly ||
|
|
(!sourceOnly && !skipScenarioExecutions && !fixtureMode);
|
|
if (shouldCheckScenarioExecutions) {
|
|
if (parsedPolicy.scenarioCatalogs.length === 0) {
|
|
failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs");
|
|
}
|
|
if (
|
|
(explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) &&
|
|
parsedPolicy.scenarioCatalogs.length !== 1
|
|
) {
|
|
failures.push(
|
|
"[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution",
|
|
);
|
|
} else {
|
|
for (const contribution of parsedPolicy.scenarioCatalogs) {
|
|
await checkScenarioContribution(contribution);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!fixtureMode && !scenarioOnly) {
|
|
const e2eConfig = await readFile("playwright.config.ts", "utf8");
|
|
for (const token of [
|
|
"pnpm build",
|
|
"pnpm preview",
|
|
"reuseExistingServer: false",
|
|
'"junit"',
|
|
'trace: "retain-on-failure"',
|
|
'"chromium-compact"',
|
|
'"firefox"',
|
|
'"webkit"',
|
|
]) {
|
|
if (!e2eConfig.includes(token)) {
|
|
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
|
}
|
|
}
|
|
// playwright.dev.config.ts spreads playwright.config.ts.
|
|
for (const config of [
|
|
"playwright.config.ts",
|
|
"playwright.visual.config.ts",
|
|
"playwright.storybook.config.ts",
|
|
"playwright.capabilities.config.ts",
|
|
]) {
|
|
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
|
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
|
}
|
|
}
|
|
|
|
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
|
/\.spec\.ts$/.test(file),
|
|
);
|
|
for (const file of e2eFiles) {
|
|
const source = await readFile(file, "utf8");
|
|
if (!source.includes("support/browser/strict-browser-test")) {
|
|
failures.push(`${file}: bypasses strict browser fixture`);
|
|
}
|
|
}
|
|
|
|
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
|
|
(file) => file.endsWith(".png"),
|
|
);
|
|
facts.visualBaselines = baselineFiles.length;
|
|
if (facts.visualBaselines < 4) {
|
|
failures.push("visual baseline requires at least four risk surfaces");
|
|
}
|
|
for (const required of [
|
|
"playwright.storybook.config.ts",
|
|
"playwright.visual.config.ts",
|
|
"tests/storybook/workshop.spec.ts",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`test evidence missing ${required}`);
|
|
}
|
|
}
|
|
|
|
if (!sourceOnly) {
|
|
for (const required of [
|
|
"artifacts/tests/storybook/results.xml",
|
|
"artifacts/tests/visual/results.xml",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`test evidence missing ${required}`);
|
|
}
|
|
}
|
|
|
|
for (const required of [
|
|
"dist/index.html",
|
|
"dist/config.json",
|
|
"dist/release-manifest.json",
|
|
"dist/runtime-config.schema.json",
|
|
"dist/.vite/manifest.json",
|
|
]) {
|
|
if ((await filesBelow(required)).length === 0) {
|
|
failures.push(`built-dist contract missing ${required}`);
|
|
}
|
|
}
|
|
const sourceMaps = (await filesBelow("dist")).filter((file) =>
|
|
file.endsWith(".map"),
|
|
);
|
|
if (sourceMaps.length > 0) {
|
|
failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const report = {
|
|
schemaVersion: 2 as const,
|
|
sourceRoot,
|
|
status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const),
|
|
facts,
|
|
failures,
|
|
};
|
|
await mkdir(path.dirname(artifactPath), { recursive: true });
|
|
await writeValidatedJsonArtifact({
|
|
path: artifactPath,
|
|
schema: testEvidenceReportSchema,
|
|
value: report,
|
|
});
|
|
if (failures.length > 0) {
|
|
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
|
|
process.exit(1);
|
|
}
|
|
process.stdout.write(
|
|
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`,
|
|
);
|