Files
clean-architecture-frontend…/scripts/check-test-evidence.ts
DongHyeonka 3f64c68991 refactor(layout): 남은 루트 경로 목록도 등록부 집합에서 고른다
정리 4단계의 R2 검사는 등록부를 쓰는 7곳만 봤다. 그 밖에 루트 이름을
담은 목록이 코드 15파일에 19개 따로 있어, 루트 항목 하나를 옮기면
여전히 여러 곳을 같이 고쳐야 했다.

- 등록부에 집합 12개를 더했다. 사본 복사에서 건너뛰는 항목 셋
  (checkout·provider·공급망 fixture), fixture 코드 범위, pnpm 매니페스트,
  Playwright 설정, visual 증거, 문서 입력, 에이전트 하네스, tsconfig
  묶음 셋. 각 집합 설명에 읽는 파일을 적었다.
- 사본이 건너뛰는 node_modules·dist·.tmp 같은 추적 안 하는 항목은
  종류 untracked(optional)로 적는다. 추적 파일이 있으면 시험이 실패한다.
- 19개 목록이 모두 집합을 고른다. 집합의 원소는 전과 같아 동작은
  바뀌지 않는다.
- R2 시험이 추적되는 코드 파일 전체를 본다. 루트 항목(이나 그 안
  경로)만 담은 문자열 배열에 추적되는 루트 이름이 둘 이상이면 실패.
  인자 목록은 `--` 뒤만 본다. main의 옛 목록 18개를 이 검사가 잡는
  것을 확인했다.
- 집합마다 selectRepositoryPaths·selectRepositoryEntries로 고르는 코드가
  있는지 본다. 문서를 복사하지 않는 제거 drill(FE-GATE-020)은 문서
  시험(REMOVAL_FIXTURE_DOCUMENT_TESTS)을 지우는데, 집합 셋
  (agent-harness·documentation-inputs·feature-typecheck)은 그 시험만
  읽는다. 시험이 이 셋을 적어 두고, 문서 시험이 지워진 사본에서만 셋이
  비는 것을 허용한다. 셋이 바뀌면 전체 checkout에서 실패한다.
- 파서가 kinds·sets 설명과 코드의 목록이 같은지 본다.
- .gitignore 주석이 a11y 기록을 아직 artifacts/ 아래라고 적던 것을
  고쳤다(정리 3단계에서 docs/로 옮김).

검증(desktop): test:unit, check:types, check:ci, lint, verify:documentation.
repository-layout 시험을 문서 없는 제거 사본과 feature-scaffold 사본에서도
돌렸고, 집합 하나의 읽는 곳을 지우면 두 곳 모두 실패한다.
2026-11-12 01:17:55 +09:00

464 lines
15 KiB
TypeScript

import { mkdir, readFile, readdir, stat } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { z } from "zod";
import {
computeHttpScenarioCatalogDigest,
httpScenarioExpectationSchema,
httpScenarioReceiptSchema,
sameScenarioJson,
type HttpScenarioExpectation,
type HttpScenarioReceipt,
} from "./lib/http-scenario-evidence.ts";
import { selectRepositoryPaths } from "./lib/repository-layout.ts";
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
import { testEvidenceReportSchema } from "./lib/test-evidence-artifact.ts";
const scenarioContributionSchema = z
.object({
owner: z.string().min(1),
path: z.string().min(1),
expectationExport: z.string().min(1),
receiptPath: z.string().min(1),
receiptSchemaVersion: z.number().int().positive(),
})
.strict();
const sourceContractSchema = z
.object({
owner: z.string().min(1),
path: z.string().min(1),
requiredTokens: z.array(z.string().min(1)),
})
.strict();
const policySchema = z
.object({
schemaVersion: z.literal(2),
scenarioCatalogs: z.array(scenarioContributionSchema),
sourceContracts: z.array(sourceContractSchema),
})
.strict();
function argumentValue(name: string, fallback: string): string {
const index = process.argv.indexOf(name);
return index >= 0 && process.argv[index + 1]
? process.argv[index + 1]
: fallback;
}
function optionalArgumentValue(name: string): string | undefined {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
const sourceRoot = argumentValue("--source-root", "tests");
const artifactPath = argumentValue(
"--artifact",
"artifacts/quality/test-evidence.json",
);
const policyPath = argumentValue(
"--policy",
"config/testing/test-evidence.json",
);
const explicitCatalogPath = optionalArgumentValue("--catalog");
const explicitReceiptPath = optionalArgumentValue("--receipt");
const fixtureMode = sourceRoot !== "tests";
const sourceOnly = process.argv.includes("--source-only");
const scenarioOnly = process.argv.includes("--scenario-only");
const skipScenarioExecutions = process.argv.includes(
"--skip-scenario-executions",
);
const failures: string[] = [];
const facts = {
scannedFiles: 0,
visualBaselines: 0,
sharedScenarios: 0,
declaredScenarioExecutions: 0,
executedScenarioExecutions: 0,
};
async function filesBelow(target: string): Promise<string[]> {
try {
const metadata = await stat(target);
if (metadata.isFile()) return [target];
const entries = await readdir(target, { withFileTypes: true });
const groups = await Promise.all(
entries.map((entry) => filesBelow(path.join(target, entry.name))),
);
return groups.flat();
} catch {
return [];
}
}
function duplicates(values: readonly string[]): string[] {
const seen = new Set<string>();
const duplicate = new Set<string>();
for (const value of values) {
if (seen.has(value)) duplicate.add(value);
seen.add(value);
}
return [...duplicate].sort();
}
function isSorted(values: readonly string[]): boolean {
return values.every(
(value, index) => index === 0 || values[index - 1]!.localeCompare(value) <= 0,
);
}
async function readJson(target: string): Promise<unknown> {
return JSON.parse(await readFile(target, "utf8"));
}
async function loadCatalogExport(
target: string,
exportName: string,
): Promise<unknown> {
if (target.endsWith(".json")) {
const document = await readJson(target);
return (document as Readonly<Record<string, unknown>>)[exportName];
}
const module = (await import(
`${pathToFileURL(path.resolve(target)).href}?test-evidence=${Date.now()}`
)) as Readonly<Record<string, unknown>>;
return module[exportName];
}
async function checkScenarioContribution(
contribution: z.output<typeof scenarioContributionSchema>,
): Promise<void> {
const catalogPath = explicitCatalogPath ?? contribution.path;
const receiptPath = explicitReceiptPath ?? contribution.receiptPath;
let expectations: readonly HttpScenarioExpectation[];
try {
expectations = z
.array(httpScenarioExpectationSchema)
.parse(
await loadCatalogExport(catalogPath, contribution.expectationExport),
);
} catch (error) {
failures.push(
`[TEST-EVIDENCE-CATALOG-SCHEMA] ${catalogPath}: ${error instanceof Error ? error.message : String(error)}`,
);
return;
}
facts.declaredScenarioExecutions += expectations.length;
facts.sharedScenarios += new Set(
expectations.map((entry) => entry.scenarioId),
).size;
const declarationIds = expectations.map((entry) => entry.executionId);
if (declarationIds.length === 0) {
failures.push(`[TEST-EVIDENCE-CATALOG-ZERO] ${catalogPath}: no declarations`);
}
const duplicateDeclarations = duplicates(declarationIds);
if (duplicateDeclarations.length > 0) {
failures.push(
`[TEST-EVIDENCE-DECLARATION-DUPLICATE] ${catalogPath}: ${duplicateDeclarations.join(", ")}`,
);
}
for (const entry of expectations) {
if (entry.executionId !== `${entry.operationId}::${entry.scenarioId}`) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-SCHEMA] ${catalogPath}: ${entry.executionId}`,
);
}
}
let receipt: HttpScenarioReceipt;
try {
const parsedReceipt = httpScenarioReceiptSchema.safeParse(
await readJson(receiptPath),
);
if (!parsedReceipt.success) {
if (
parsedReceipt.error.issues.some((issue) =>
issue.message.includes("must be sorted by execution ID"),
)
) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`,
);
}
if (
parsedReceipt.error.issues.some((issue) =>
issue.message.includes("row IDs must exactly equal executed IDs"),
)
) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
}
throw parsedReceipt.error;
}
receipt = parsedReceipt.data;
} catch (error) {
failures.push(
`[TEST-EVIDENCE-RECEIPT-SCHEMA] ${receiptPath}: ${error instanceof Error ? error.message : String(error)}`,
);
return;
}
facts.executedScenarioExecutions += receipt.rows.length;
if (receipt.schemaVersion !== contribution.receiptSchemaVersion) {
failures.push(
`[TEST-EVIDENCE-RECEIPT-VERSION] ${receiptPath}: expected ${contribution.receiptSchemaVersion}, observed ${receipt.schemaVersion}`,
);
}
if (receipt.rows.length === 0 || receipt.executedIds.length === 0) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ZERO] ${receiptPath}: no executions`);
}
if (receipt.catalogTotal !== declarationIds.length) {
failures.push(
`[TEST-EVIDENCE-CATALOG-TOTAL] ${receiptPath}: expected ${declarationIds.length}, observed ${receipt.catalogTotal}`,
);
}
const duplicateExecutedIds = duplicates(receipt.executedIds);
const rowIds = receipt.rows.map((row) => row.executionId);
const duplicateRows = duplicates(rowIds);
if (duplicateExecutedIds.length > 0 || duplicateRows.length > 0) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-DUPLICATE] ${receiptPath}: ${[...new Set([...duplicateExecutedIds, ...duplicateRows])].sort().join(", ")}`,
);
}
if (!isSorted(receipt.executedIds) || !isSorted(rowIds)) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ID-UNSORTED] ${receiptPath}`);
}
if (!sameScenarioJson(receipt.executedIds, rowIds)) {
failures.push(`[TEST-EVIDENCE-EXECUTION-ROW-DRIFT] ${receiptPath}`);
}
const declared = new Set(declarationIds);
const executed = new Set(receipt.executedIds);
const missing = [...declared].filter((executionId) => !executed.has(executionId)).sort();
const extra = [...executed].filter((executionId) => !declared.has(executionId)).sort();
if (missing.length > 0) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-MISSING] ${receiptPath}: ${missing.join(", ")}`,
);
}
if (extra.length > 0) {
failures.push(
`[TEST-EVIDENCE-EXECUTION-ID-EXTRA] ${receiptPath}: ${extra.join(", ")}`,
);
}
const expectedDigest = computeHttpScenarioCatalogDigest(
receipt.schemaVersion,
expectations,
);
if (receipt.catalogDigest !== expectedDigest) {
failures.push(
`[TEST-EVIDENCE-CATALOG-DIGEST] ${receiptPath}: expected ${expectedDigest}, observed ${receipt.catalogDigest}`,
);
}
const declarationsById = new Map(
expectations.map((entry) => [entry.executionId, entry] as const),
);
for (const row of receipt.rows) {
const declaration = declarationsById.get(row.executionId);
if (!declaration) continue;
if (!sameScenarioJson(row.expected, declaration.expected)) {
failures.push(
`[TEST-EVIDENCE-EXPECTED-DRIFT] ${receiptPath}: ${row.executionId}`,
);
}
if (row.testDeadlineOverrideMs !== declaration.testDeadlineOverrideMs) {
failures.push(
`[TEST-EVIDENCE-DEADLINE-DRIFT] ${receiptPath}: ${row.executionId}`,
);
}
if (!sameScenarioJson(row.observed, row.expected)) {
failures.push(
`[TEST-EVIDENCE-OBSERVATION-MISMATCH] ${receiptPath}: ${row.executionId}`,
);
}
}
}
if (!scenarioOnly) {
const sourceFiles = (await filesBelow(sourceRoot)).filter(
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
);
const today = new Date().toISOString().slice(0, 10);
for (const file of sourceFiles) {
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
const source = await readFile(file, "utf8");
facts.scannedFiles += 1;
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
}
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
if (skipPattern.test(source)) {
const quarantines = [
...source.matchAll(
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
),
];
if (quarantines.length === 0) {
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
}
for (const [, expires = ""] of quarantines) {
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
}
}
const wholeUiMask =
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
if (wholeUiMask.test(source)) {
failures.push(`${file}: screenshot mask may not cover the whole UI`);
}
}
}
let parsedPolicy: z.output<typeof policySchema> | undefined;
if (!fixtureMode || scenarioOnly) {
try {
parsedPolicy = policySchema.parse(await readJson(policyPath));
} catch (error) {
failures.push(
`[TEST-EVIDENCE-POLICY-SCHEMA] ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
if (parsedPolicy) {
for (const contract of parsedPolicy.sourceContracts) {
let source: string;
try {
source = await readFile(contract.path, "utf8");
} catch (error) {
failures.push(
`${contract.path}: cannot read source contract (${error instanceof Error ? error.message : String(error)})`,
);
continue;
}
for (const token of contract.requiredTokens) {
if (!source.includes(token)) {
failures.push(
`${contract.path}: ${contract.owner} evidence contract is missing ${token}`,
);
}
}
}
const shouldCheckScenarioExecutions =
scenarioOnly ||
(!sourceOnly && !skipScenarioExecutions && !fixtureMode);
if (shouldCheckScenarioExecutions) {
if (parsedPolicy.scenarioCatalogs.length === 0) {
failures.push("[TEST-EVIDENCE-CATALOG-ZERO] policy has no scenario catalogs");
}
if (
(explicitCatalogPath !== undefined || explicitReceiptPath !== undefined) &&
parsedPolicy.scenarioCatalogs.length !== 1
) {
failures.push(
"[TEST-EVIDENCE-FIXTURE-SCOPE] explicit catalog/receipt requires exactly one contribution",
);
} else {
for (const contribution of parsedPolicy.scenarioCatalogs) {
await checkScenarioContribution(contribution);
}
}
}
}
if (!fixtureMode && !scenarioOnly) {
const e2eConfig = await readFile("playwright.config.ts", "utf8");
for (const token of [
"pnpm build",
"pnpm preview",
"reuseExistingServer: false",
'"junit"',
'trace: "retain-on-failure"',
'"chromium-compact"',
'"firefox"',
'"webkit"',
]) {
if (!e2eConfig.includes(token)) {
failures.push(`playwright.config.ts missing release evidence token ${token}`);
}
}
// playwright.dev.config.ts spreads playwright.config.ts.
for (const config of selectRepositoryPaths("playwright-configs")) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
/\.spec\.ts$/.test(file),
);
for (const file of e2eFiles) {
const source = await readFile(file, "utf8");
if (!source.includes("support/browser/strict-browser-test")) {
failures.push(`${file}: bypasses strict browser fixture`);
}
}
const baselineFiles = (await filesBelow("tests/visual/__snapshots__")).filter(
(file) => file.endsWith(".png"),
);
facts.visualBaselines = baselineFiles.length;
if (facts.visualBaselines < 4) {
failures.push("visual baseline requires at least four risk surfaces");
}
for (const required of selectRepositoryPaths("visual-evidence")) {
if ((await filesBelow(required)).length === 0) {
failures.push(`test evidence missing ${required}`);
}
}
if (!sourceOnly) {
for (const required of [
"artifacts/tests/storybook/results.xml",
"artifacts/tests/visual/results.xml",
]) {
if ((await filesBelow(required)).length === 0) {
failures.push(`test evidence missing ${required}`);
}
}
for (const required of [
"dist/index.html",
"dist/config.json",
"dist/release-manifest.json",
"dist/runtime-config.schema.json",
"dist/.vite/manifest.json",
]) {
if ((await filesBelow(required)).length === 0) {
failures.push(`built-dist contract missing ${required}`);
}
}
const sourceMaps = (await filesBelow("dist")).filter((file) =>
file.endsWith(".map"),
);
if (sourceMaps.length > 0) {
failures.push(`production dist contains source maps: ${sourceMaps.join(", ")}`);
}
}
}
const report = {
schemaVersion: 2 as const,
sourceRoot,
status: failures.length === 0 ? ("PASS" as const) : ("FAIL" as const),
facts,
failures,
};
await mkdir(path.dirname(artifactPath), { recursive: true });
await writeValidatedJsonArtifact({
path: artifactPath,
schema: testEvidenceReportSchema,
value: report,
});
if (failures.length > 0) {
process.stderr.write(`Test evidence failed:\n- ${failures.join("\n- ")}\n`);
process.exit(1);
}
process.stdout.write(
`Test evidence: PASS (${facts.scannedFiles} files, ${facts.visualBaselines} baselines, ${facts.declaredScenarioExecutions} declared/${facts.executedScenarioExecutions} executed scenarios)\n`,
);