import { spawnSync } from "node:child_process"; import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; const INPUTS = ["README.md", "docs", "artifacts/tests/a11y-manual"]; /** * Runs verify:documentation on a copy of its inputs, each `changes` entry * applied to its document and every `removed` file deleted. Code facts (src, * package.json, gates.json) are read from this repository, not the copy. */ async function verify( changes: Readonly string>>, removed: readonly string[] = [], ) { const root = await mkdtemp(path.join(tmpdir(), "documentation-readiness-")); try { for (const input of INPUTS) await cp(input, path.join(root, input), { recursive: true }); for (const [document, mutate] of Object.entries(changes)) { const target = path.join(root, document); await writeFile(target, mutate(await readFile(target, "utf8"))); } for (const file of removed) await rm(path.join(root, file)); return spawnSync(process.execPath, [path.resolve("scripts/verify-documentation-readiness.ts")], { cwd: root, encoding: "utf8", }); } finally { await rm(root, { recursive: true, force: true }); } } function verifyWithReadme(mutate: (readme: string) => string, document = "README.md") { return verify({ [document]: mutate }); } // 1101/14-06. A route missing from a scope document used to fail with only // "canonical scoped-review evidence is incomplete", which points at the // diagram review instead of the file to fix. describe("verify:documentation failure output (1101/14-06)", () => { it("passes the documents as they are", async () => { const result = await verifyWithReadme((readme) => readme); expect(result.status, result.stderr).toBe(0); }); it("names the route ID that has no manual review record", async () => { const result = await verify({}, ["artifacts/tests/a11y-manual/EXAMPLES_PLATFORM.md"]); expect(result.status).toBe(1); expect(result.stderr).toContain("artifacts/tests/a11y-manual/ has no record for route IDs: EXAMPLES_PLATFORM"); expect(result.stderr).toMatch(/FAIL_UNVERIFIED \(failed: route scope\)/u); expect(result.stderr).not.toContain("scoped-review evidence"); }); it("names every failed check", async () => { const result = await verify( { "README.md": (readme) => readme.replaceAll("`RELEASE_TARGET`", "") }, ["artifacts/tests/a11y-manual/EXAMPLES_PLATFORM.md"], ); expect(result.stderr).toContain("README.md does not name: RELEASE_TARGET"); expect(result.stderr).toMatch(/\(failed: route scope, runtime variables\)/u); }); }); // 1101/14-05. README and the manual checklist listed every route ID, so each // new feature edited two template-owned documents. The scope is the record // files now, and the current contract documents (docs/README.md §1) may only // name paths, package scripts and gate IDs that exist. describe("verify:documentation current contract documents (1101/14-05)", () => { const ROUTE_IDS = /`(?:APP_HOME|EXAMPLES_[A-Z]+|NOT_FOUND|REFERENCE_RESOURCE_[A-Z]+)`/gu; it("does not need README or the checklist to list route IDs", async () => { const result = await verify({ "README.md": (readme) => readme.replaceAll(ROUTE_IDS, ""), "docs/accessibility/manual-checklist.md": (checklist) => checklist.replaceAll(ROUTE_IDS, ""), }); expect(result.status, result.stderr).toBe(0); }); it.each([ ["README.md", "scripts/verify-documentation-readiness.ts", "scripts/verify-documentation-readyness.ts"], ["docs/contracts/compatibility.md", "scripts/check-compatibility.ts", "scripts/check-compatibilty.ts"], // reached through the `operations/` directory link ["docs/operations/release-cache-rollback.md", "scripts/lib/hosting-probe.ts", "scripts/lib/hosting-prob.ts"], ])("fails when %s names a path that does not exist", async (document, before, after) => { const result = await verifyWithReadme((text) => text.replace(`\`${before}\``, `\`${after}\``), document); expect(result.status).toBe(1); expect(result.stderr).toContain(`${document} names paths that do not exist: ${after}`); expect(result.stderr).toMatch(/\(failed: document references\)/u); }); it("fails when docs/README.md links a document that does not exist", async () => { const result = await verifyWithReadme( (index) => index.replace("(architecture/layers.md)", "(architecture/layer.md)"), "docs/README.md", ); expect(result.status).toBe(1); expect(result.stderr).toContain("docs/README.md names paths that do not exist: docs/architecture/layer.md"); }); it("fails when a document names a package script or a gate ID that does not exist", async () => { const result = await verify({ "README.md": (readme) => readme.replace("`verify:documentation` fails", "`verify:documentaton` fails"), "docs/README.md": (index) => index.replace("(FE-GATE-017)", "(FE-GATE-099)"), }); expect(result.status).toBe(1); expect(result.stderr).toContain("README.md names package scripts that do not exist: verify:documentaton"); expect(result.stderr).toContain("docs/README.md names gate IDs that config/ci/gates.json does not define: FE-GATE-099"); }); it("does not ask generated output to exist", async () => { const result = await verifyWithReadme((readme) => `${readme}\nSee \`artifacts/quality/not-generated-yet.json\`.\n`); expect(result.status, result.stderr).toBe(0); }); it("accepts a path under history a copy deleted, and only that", async () => { const naming = (readme: string) => `${readme}\nSee \`docs/retired-notes/plan.md\`.\n`; const deleted = await verifyWithReadme((readme) => naming(readme).replace("| Path | Contents |\n| --- | --- |\n", "$&| `docs/retired-notes/` | deleted |\n"), ); expect(deleted.status, deleted.stderr).toBe(0); const missing = await verifyWithReadme(naming); expect(missing.status).toBe(1); expect(missing.stderr).toContain("README.md names paths that do not exist: docs/retired-notes/plan.md"); }); }); // 1101/14-03. The entry documents drew a top-level domain layer that does not // exist, left out three of the nine src folders, and layers.md named a // minority of the architecture rules while claiming it could not drift. describe("verify:documentation source structure (1101/14-03)", () => { it("fails when layers.md loses a rule name, as renaming a rule does", async () => { const result = await verifyWithReadme( (layers) => layers.replaceAll("`platform-kernel-is-product-neutral`", "`platform-kernel`"), "docs/architecture/layers.md", ); expect(result.status).toBe(1); expect(result.stderr).toContain( "docs/architecture/layers.md does not name .dependency-cruiser.json rules: platform-kernel-is-product-neutral", ); expect(result.stderr).toMatch(/\(failed: source structure\)/u); }); it("fails when a document names a top-level src folder that does not exist", async () => { const result = await verifyWithReadme((readme) => readme.replace("presentation -> application -> contracts", "presentation -> application -> `src/domain`"), ); expect(result.status).toBe(1); expect(result.stderr).toContain("README.md names src folders that do not exist: domain"); }); it("fails when a document describes a folder differently from layers.md", async () => { const result = await verifyWithReadme( (overview) => overview.replace("- `src/platform` — product-neutral", "- `src/platform` — shared"), "docs/architecture/overview.md", ); expect(result.status).toBe(1); expect(result.stderr).toContain( "docs/architecture/overview.md does not describe src folders as the layers.md table does: platform", ); }); });