문제: `verify:documentation`은 명령 머리가 corepack·pnpm·node 같은 낱말일 때만 인자 경로를 보았다. 이 repo 문서의 명령 대부분은 `scripts/verify.sh exec -- node <경로>`, `scripts/verify.sh script <이름>`, `python3 scripts/…` 꼴이라, 그 안의 낡은 경로·없는 script 이름이 검사를 통과했다. 고친 것: `python3`를 명령 머리에 더하고, `scripts/verify.sh`는 그 자신을 경로로 보며 `exec --` 뒤를 다시 명령으로 읽고 `script` 뒤 낱말을 package script 이름으로 대조한다. 자리표시자(`<이름>`)와 `--` 뒤 인자는 보지 않는다. 위험·한계: 살아 있는 문서 전부가 지금 이 검사를 통과하는지는 desktop `verify:documentation`으로 확인한다. 검증: desktop `node scripts/run-vitest.ts run tests/unit/documentation-readiness.test.ts` 24건 통과(새 시험 포함). 검사기를 기준 커밋으로 되돌리면 새 시험이 실패한다. Refs: 1112/08-06
413 lines
20 KiB
TypeScript
413 lines
20 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { devNull, tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
import { selectRepositoryPaths } from "../../scripts/lib/repository-layout.ts";
|
|
import { childProcessTimeout } from "../helpers/child-process-timeout.ts";
|
|
|
|
// The pin says which commit of the paired backend the documents describe.
|
|
const INPUTS = selectRepositoryPaths("documentation-inputs");
|
|
const PAIRED_BACKEND = "clean-architecture-backend-template";
|
|
|
|
type Changes = Readonly<Record<string, (text: string) => string>>;
|
|
|
|
/**
|
|
* Runs verify:documentation on a copy of its inputs, each `changes` entry
|
|
* applied to its document and every `removed` file deleted. Code facts (src,
|
|
* package.json, gates.json) are read from this repository, not the copy. The
|
|
* copy is a `frontend` folder; `paired` builds what sits beside it, where a
|
|
* checkout of the paired repositories would be, and returns more changes (a
|
|
* pin to its commit).
|
|
*/
|
|
async function verify(
|
|
changes: Changes,
|
|
removed: readonly string[] = [],
|
|
paired: (parent: string) => Changes = () => ({}),
|
|
) {
|
|
const parent = await mkdtemp(path.join(tmpdir(), "documentation-readiness-"));
|
|
const root = path.join(parent, "frontend");
|
|
try {
|
|
const pairedChanges = paired(parent);
|
|
for (const input of INPUTS) await cp(input, path.join(root, input), { recursive: true });
|
|
for (const [document, mutate] of Object.entries({ ...changes, ...pairedChanges })) {
|
|
const target = path.join(root, document);
|
|
await writeFile(target, mutate(await readFile(target, "utf8")));
|
|
}
|
|
for (const file of removed) await rm(path.join(root, file));
|
|
const result = spawnSync(process.execPath, [path.resolve("scripts/verify-documentation-readiness.ts")], {
|
|
cwd: root,
|
|
encoding: "utf8",
|
|
});
|
|
const report = await readFile(path.join(root, "artifacts/quality/documentation-review.json"), "utf8")
|
|
.then((json) => JSON.parse(json) as {
|
|
documentReferences: { path: string; missingPaths?: string[]; unverifiedLinks?: string[] }[];
|
|
})
|
|
.catch(() => null);
|
|
return { ...result, report };
|
|
} finally {
|
|
await rm(parent, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Each case spawns verify:documentation, which reads every document. On a
|
|
* loaded host (the FE-GATE-020 drills run this file inside their copies) one
|
|
* run took more than the 10 s default. The longest case took 15.8 s in the
|
|
* desktop coverage bundle (2026-11-13).
|
|
*/
|
|
const VERIFY_TIMEOUT = childProcessTimeout(15_807);
|
|
|
|
function verifyWithReadme(mutate: (readme: string) => string, document = "README.md") {
|
|
return verify({ [document]: mutate });
|
|
}
|
|
|
|
/**
|
|
* docs-system.md 7: the document checks fail a pull request. The FE-GATE-017
|
|
* job runs only when a workflow_dispatch picks the documentation stage, so the
|
|
* merge stage (FE-GATE-005, this file) runs verify:documentation once on this
|
|
* repository as it is: every document, not the copy the cases below make.
|
|
*/
|
|
describe("verify:documentation on this repository", () => {
|
|
it("passes", () => {
|
|
const result = spawnSync(process.execPath, [path.resolve("scripts/verify-documentation-readiness.ts")], {
|
|
encoding: "utf8",
|
|
});
|
|
expect(result.status, `${result.stderr}${result.stdout}`).toBe(0);
|
|
}, VERIFY_TIMEOUT);
|
|
});
|
|
|
|
const gitEnvironment = {
|
|
...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("GIT_"))),
|
|
GIT_CONFIG_GLOBAL: devNull,
|
|
GIT_CONFIG_NOSYSTEM: "1",
|
|
};
|
|
function git(cwd: string, ...args: string[]): string {
|
|
const result = spawnSync(
|
|
"git",
|
|
["-c", "user.name=fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", ...args],
|
|
{ cwd, encoding: "utf8", env: gitEnvironment },
|
|
);
|
|
if (result.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr}`);
|
|
return result.stdout.trim();
|
|
}
|
|
|
|
/**
|
|
* A git checkout of the paired backend beside the copy: one commit per
|
|
* `history` entry (a file set to null is deleted), then `uncommitted` written
|
|
* to its working tree. Returns the commit ids.
|
|
*/
|
|
function pairedCheckout(
|
|
parent: string,
|
|
history: readonly Readonly<Record<string, string | null>>[],
|
|
uncommitted: Readonly<Record<string, string>> = {},
|
|
): string[] {
|
|
const root = path.join(parent, PAIRED_BACKEND);
|
|
const write = (files: Readonly<Record<string, string | null>>) => {
|
|
for (const [file, content] of Object.entries(files)) {
|
|
const target = path.join(root, file);
|
|
if (content === null) {
|
|
rmSync(target);
|
|
} else {
|
|
mkdirSync(path.dirname(target), { recursive: true });
|
|
writeFileSync(target, content);
|
|
}
|
|
}
|
|
};
|
|
mkdirSync(root, { recursive: true });
|
|
git(root, "init", "-q");
|
|
const commits: string[] = [];
|
|
for (const files of history) {
|
|
write(files);
|
|
git(root, "add", "-A");
|
|
git(root, "commit", "-q", "--allow-empty", "-m", `fixture ${commits.length}`);
|
|
commits.push(git(root, "rev-parse", "HEAD"));
|
|
}
|
|
write(uncommitted);
|
|
return commits;
|
|
}
|
|
|
|
function pinnedTo(commit: string): Changes {
|
|
return {
|
|
"config/contracts/backend-contract-pin.json": (pin) =>
|
|
pin.replace(/"commit": "[0-9a-f]+"/u, `"commit": "${commit}"`),
|
|
};
|
|
}
|
|
|
|
// 1101/14-06. A route missing from a scope document used to fail with only
|
|
// "canonical scoped-review evidence is incomplete", which points at the
|
|
// diagram review instead of the file to fix.
|
|
describe("verify:documentation failure output (1101/14-06)", () => {
|
|
it("passes the documents as they are", async () => {
|
|
const result = await verifyWithReadme((readme) => readme);
|
|
expect(result.status, result.stderr).toBe(0);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("names the route ID that has no manual review record", async () => {
|
|
const result = await verify({}, ["docs/accessibility/route-reviews/EXAMPLES_PLATFORM.md"]);
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("docs/accessibility/route-reviews/ has no record for route IDs: EXAMPLES_PLATFORM");
|
|
expect(result.stderr).toMatch(/FAIL_UNVERIFIED \(failed: route scope\)/u);
|
|
expect(result.stderr).not.toContain("scoped-review evidence");
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("names every failed check", async () => {
|
|
const result = await verify(
|
|
{ "README.md": (readme) => readme.replaceAll("`RELEASE_TARGET`", "") },
|
|
["docs/accessibility/route-reviews/EXAMPLES_PLATFORM.md"],
|
|
);
|
|
expect(result.stderr).toContain("README.md does not name: RELEASE_TARGET");
|
|
expect(result.stderr).toMatch(/\(failed: route scope, runtime variables\)/u);
|
|
}, VERIFY_TIMEOUT);
|
|
});
|
|
|
|
// 1101/14-05. README and the manual checklist listed every route ID, so each
|
|
// new feature edited two template-owned documents. The scope is the record
|
|
// files now, and the current contract documents (docs/README.md §1) may only
|
|
// name paths, package scripts and gate IDs that exist.
|
|
describe("verify:documentation current contract documents (1101/14-05)", () => {
|
|
// Written without the sample's route prefix: the sample removal drill runs
|
|
// this file and refuses any file that still names the removed sample.
|
|
const ROUTE_IDS = /`(?:APP_HOME|EXAMPLES_[A-Z]+|NOT_FOUND|[A-Z]+_RESOURCE_[A-Z]+)`/gu;
|
|
|
|
it("does not need README or the checklist to list route IDs", async () => {
|
|
const result = await verify({
|
|
"README.md": (readme) => readme.replaceAll(ROUTE_IDS, ""),
|
|
"docs/accessibility/manual-checklist.md": (checklist) => checklist.replaceAll(ROUTE_IDS, ""),
|
|
});
|
|
expect(result.status, result.stderr).toBe(0);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it.each([
|
|
["README.md", "scripts/verify-documentation-readiness.ts", "scripts/verify-documentation-readyness.ts"],
|
|
["docs/contracts/compatibility.md", "scripts/check-compatibility.ts", "scripts/check-compatibilty.ts"],
|
|
// reached through the `operations/` directory link
|
|
["docs/operations/release-cache-rollback.md", "scripts/lib/hosting-probe.ts", "scripts/lib/hosting-prob.ts"],
|
|
])("fails when %s names a path that does not exist", async (document, before, after) => {
|
|
const result = await verifyWithReadme((text) => text.replace(`\`${before}\``, `\`${after}\``), document);
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain(`${document} names paths that do not exist: ${after}`);
|
|
expect(result.stderr).toMatch(/\(failed: document references\)/u);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("fails when docs/README.md links a document that does not exist", async () => {
|
|
const result = await verifyWithReadme(
|
|
(index) => index.replace("(architecture/layers.md)", "(architecture/layer.md)"),
|
|
"docs/README.md",
|
|
);
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("docs/README.md names paths that do not exist: docs/architecture/layer.md");
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("fails when a document names a package script or a gate ID that does not exist", async () => {
|
|
const result = await verify({
|
|
"README.md": (readme) => readme.replace("`verify:documentation`", "`verify:documentaton`"),
|
|
"docs/README.md": (index) => index.replace("(FE-GATE-017)", "(FE-GATE-099)"),
|
|
});
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("README.md names package scripts that do not exist: verify:documentaton");
|
|
expect(result.stderr).toContain("docs/README.md names gate IDs that config/ci/gates.json does not define: FE-GATE-099");
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("does not ask generated output to exist", async () => {
|
|
const result = await verifyWithReadme((readme) => `${readme}\nSee \`artifacts/quality/not-generated-yet.json\`.\n`);
|
|
expect(result.status, result.stderr).toBe(0);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("accepts a path under history a copy deleted, and only that", async () => {
|
|
const naming = (readme: string) => `${readme}\nSee \`docs/retired-notes/plan.md\`.\n`;
|
|
const deleted = await verifyWithReadme((readme) =>
|
|
naming(readme).replace("| 경로 | 내용 |\n| --- | --- |\n", "$&| `docs/retired-notes/` | 지웠다 |\n"),
|
|
);
|
|
expect(deleted.status, deleted.stderr).toBe(0);
|
|
|
|
const missing = await verifyWithReadme(naming);
|
|
expect(missing.status).toBe(1);
|
|
expect(missing.stderr).toContain("README.md names paths that do not exist: docs/retired-notes/plan.md");
|
|
}, VERIFY_TIMEOUT);
|
|
});
|
|
|
|
// 1103/12-05. Paths inside a command and links into the paired repositories
|
|
// were never looked up: a deleted V2 test stayed in a `vitest run` command,
|
|
// and a broken link to the backend template's contract passed.
|
|
describe("verify:documentation commands and sibling links (1103/12-05)", () => {
|
|
const CI_DOC = "docs/operations/ci-quality-gates.md";
|
|
const BACKEND = "clean-architecture-backend-template/docs/web/http-contract.md";
|
|
|
|
it.each([
|
|
["an inline command", "`corepack pnpm exec vitest run tests/integration/no-such-client.test.ts`"],
|
|
["a shell block", "```bash\nnode tests/integration/no-such-client.test.ts --flag\n```"],
|
|
])("fails when %s names a path that does not exist", async (_shape, line) => {
|
|
const result = await verify({ [CI_DOC]: (text) => `${text}\n${line}\n` });
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain(`${CI_DOC} names paths that do not exist: tests/integration/no-such-client.test.ts`);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("looks up a command's existing arguments and leaves placeholders alone", async () => {
|
|
const result = await verify({
|
|
[CI_DOC]: (text) => `${text}\n\`corepack pnpm exec vitest run tests/unit/documentation-readiness.test.ts -t "x"\`\n` +
|
|
"```bash\ncorepack pnpm drill:runbook -- FE-RB-005 --release-root <release dir>\n```\n" +
|
|
"Any script is run as `node scripts/…`.\n" +
|
|
"`scripts/verify.sh exec -- node scripts/run-vitest.ts run tests/unit/documentation-readiness.test.ts`, " +
|
|
"`scripts/verify.sh script verify:documentation lint -- --x`, `scripts/verify.sh script <이름>…`, " +
|
|
"`python3 scripts/check-work-docs.py --history <경로>`\n",
|
|
});
|
|
expect(result.status, result.stderr).toBe(0);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
// 1112/08-06. Commands through this repository's entry point, and python3
|
|
// commands, were not read, so a stale path or script name in them passed.
|
|
it("reads commands run through scripts/verify.sh and python3", async () => {
|
|
const result = await verify({
|
|
[CI_DOC]: (text) => `${text}\n\`scripts/verify.sh exec -- node scripts/no-such.ts\`\n` +
|
|
"```bash\nVERIFY_SNAPSHOT=1 scripts/verify.sh exec -- node tests/unit/no-such-shell.test.ts\n```\n" +
|
|
"`python3 scripts/no-such.py --history docs`\n`scripts/verify.sh script no:such-script -- --flag`\n",
|
|
});
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain(
|
|
`${CI_DOC} names paths that do not exist: scripts/no-such.py, scripts/no-such.ts, tests/unit/no-such-shell.test.ts`,
|
|
);
|
|
expect(result.stderr).toContain(`${CI_DOC} names package scripts that do not exist: no:such-script`);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("checks a link or a path into a paired repository checked out beside this one", async () => {
|
|
const naming = (target: string) => (text: string) =>
|
|
`${text}\n[contract](../../../${target}) and \`${target}\`\n`;
|
|
// 1105/12-02. The checkout is looked up at the pinned commit.
|
|
const pinnedBackend = (parent: string) =>
|
|
pinnedTo(pairedCheckout(parent, [{ "docs/web/http-contract.md": "# contract\n" }])[0]!);
|
|
const present = await verify({ [CI_DOC]: naming(BACKEND) }, [], pinnedBackend);
|
|
// Only this document is under test: other documents name paired-backend
|
|
// paths the stub checkout does not have.
|
|
expect(present.stderr).not.toContain(`${CI_DOC} names paths that do not exist`);
|
|
expect(
|
|
present.report?.documentReferences.find((entry) => entry.path === CI_DOC)?.missingPaths,
|
|
).toEqual([]);
|
|
|
|
const broken = await verify(
|
|
{ [CI_DOC]: naming("clean-architecture-backend-template/docs/web/no-such.md") },
|
|
[],
|
|
pinnedBackend,
|
|
);
|
|
expect(broken.status).toBe(1);
|
|
expect(broken.stderr).toContain(
|
|
`${CI_DOC} names paths that do not exist: ../clean-architecture-backend-template/docs/web/no-such.md`,
|
|
);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("reports a link into a paired repository that is not checked out as unverified", async () => {
|
|
const result = await verify({
|
|
[CI_DOC]: (text) => `${text}\n[contract](../../../clean-architecture-backend-template/docs/web/no-such.md)\n`,
|
|
});
|
|
expect(result.status, result.stderr).toBe(0);
|
|
expect(result.stdout).toContain("not verified");
|
|
expect(result.report?.documentReferences.find((entry) => entry.path === CI_DOC)?.unverifiedLinks)
|
|
.toEqual(["../clean-architecture-backend-template/docs/web/no-such.md"]);
|
|
}, VERIFY_TIMEOUT);
|
|
});
|
|
|
|
// 1105/12-02. Paired references were looked up in the working tree of the
|
|
// checkout beside this one, so the same commit failed on the desktop (a
|
|
// sibling months behind with uncommitted changes), passed with 17 references
|
|
// unverified where no sibling was, and a misspelled repository passed as
|
|
// unverified. They are looked up at the commit the pin names.
|
|
describe("verify:documentation paired references at the pinned commit (1105/12-02)", () => {
|
|
const CI_DOC = "docs/operations/ci-quality-gates.md";
|
|
const naming = (...targets: string[]) => (text: string) =>
|
|
`${text}\n${targets.map((target) => `[link](../../../${target}) and \`${target}\``).join("\n")}\n`;
|
|
const entryOf = (result: Awaited<ReturnType<typeof verify>>) =>
|
|
result.report?.documentReferences.find((entry) => entry.path === CI_DOC);
|
|
|
|
it("looks a reference up in the pinned commit, not in a later HEAD or the working tree", async () => {
|
|
const result = await verify(
|
|
{
|
|
[CI_DOC]: naming(
|
|
`${PAIRED_BACKEND}/docs/web/http-contract.md`,
|
|
`${PAIRED_BACKEND}/docs/web/after-pin.md`,
|
|
`${PAIRED_BACKEND}/docs/web/uncommitted.md`,
|
|
),
|
|
},
|
|
[],
|
|
(parent) => {
|
|
const [pinned] = pairedCheckout(
|
|
parent,
|
|
[
|
|
{ "docs/web/http-contract.md": "# contract\n" },
|
|
{ "docs/web/http-contract.md": null, "docs/web/after-pin.md": "# later\n" },
|
|
],
|
|
{ "docs/web/uncommitted.md": "# draft\n" },
|
|
);
|
|
return pinnedTo(pinned!);
|
|
},
|
|
);
|
|
// Other documents name paired paths the stub does not have, so only this
|
|
// document's entry is compared.
|
|
expect(entryOf(result)?.missingPaths).toEqual([
|
|
`../${PAIRED_BACKEND}/docs/web/after-pin.md`,
|
|
`../${PAIRED_BACKEND}/docs/web/uncommitted.md`,
|
|
]);
|
|
expect(entryOf(result)?.unverifiedLinks).toEqual([]);
|
|
expect(result.stderr).toContain("at the pinned commit (looked up in clean-architecture-backend-template ");
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("reports the references as not verified when the checkout does not have the pinned commit", async () => {
|
|
// The desktop case: a sibling whose history stops before the pin, with
|
|
// none of the files the documents name.
|
|
const result = await verify({ [CI_DOC]: naming(`${PAIRED_BACKEND}/docs/web/no-such.md`) }, [], (parent) => {
|
|
pairedCheckout(parent, [{ "README.md": "# an older backend\n" }]);
|
|
return {};
|
|
});
|
|
expect(result.status, result.stderr).toBe(0);
|
|
expect(entryOf(result)?.missingPaths).toEqual([]);
|
|
expect(entryOf(result)?.unverifiedLinks).toEqual([`../${PAIRED_BACKEND}/docs/web/no-such.md`]);
|
|
expect(result.stdout).toContain("not verified");
|
|
expect(result.stdout).toContain("does not have the pinned commit");
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("fails a link out of the repository into a repository that is not paired", async () => {
|
|
const misspelled = "clean-architecture-backend-templat/docs/web/http-contract.md";
|
|
const result = await verify({ [CI_DOC]: (text) => `${text}\n[contract](../../../${misspelled})\n` });
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain(`${CI_DOC} names paths that do not exist: ../${misspelled}`);
|
|
}, VERIFY_TIMEOUT);
|
|
});
|
|
|
|
// 1101/14-03. The entry documents drew a top-level domain layer that does not
|
|
// exist, left out three of the nine src folders, and layers.md named a
|
|
// minority of the architecture rules while claiming it could not drift.
|
|
describe("verify:documentation source structure (1101/14-03)", () => {
|
|
it("fails when layers.md loses a rule name, as renaming a rule does", async () => {
|
|
const result = await verifyWithReadme(
|
|
(layers) => layers.replaceAll("`platform-kernel-is-product-neutral`", "`platform-kernel`"),
|
|
"docs/architecture/layers.md",
|
|
);
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain(
|
|
"docs/architecture/layers.md does not name .dependency-cruiser.json rules: platform-kernel-is-product-neutral",
|
|
);
|
|
expect(result.stderr).toMatch(/\(failed: source structure\)/u);
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("fails when a document names a top-level src folder that does not exist", async () => {
|
|
const result = await verifyWithReadme((readme) =>
|
|
readme.replace("presentation -> application -> contracts", "presentation -> application -> `src/domain`"),
|
|
);
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("README.md names src folders that do not exist: domain");
|
|
}, VERIFY_TIMEOUT);
|
|
|
|
it("fails when a document describes a folder differently from layers.md", async () => {
|
|
const result = await verifyWithReadme(
|
|
(overview) => overview.replace("- `src/platform` — 어느 폴더든", "- `src/platform` — 공용"),
|
|
"docs/architecture/overview.md",
|
|
);
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain(
|
|
"docs/architecture/overview.md does not describe src folders as the layers.md table does: platform",
|
|
);
|
|
}, VERIFY_TIMEOUT);
|
|
});
|