diff --git a/scripts/verify-documentation-readiness.ts b/scripts/verify-documentation-readiness.ts index cf03bd7e..4cebe964 100644 --- a/scripts/verify-documentation-readiness.ts +++ b/scripts/verify-documentation-readiness.ts @@ -112,12 +112,17 @@ const missingRuntimeVariables = RUNTIME_VARIABLES.filter( (variable) => !readme.includes(`\`${variable}\``), ); -const passed = - reportDigestValid && - routeScopeDocumented && - missingRuntimeVariables.length === 0 && - results.length === 2 && - results.every((result) => result.passed); +// 1101/14-06. Each failed check is named, so the stderr points at the file to +// fix rather than at the diagram review whatever failed. +const failedChecks = [ + ...(routeScopeDocumented ? [] : ["route scope"]), + ...(missingRuntimeVariables.length === 0 ? [] : ["runtime variables"]), + ...(reportDigestValid ? [] : ["review report digest"]), + ...(results.length === 2 && results.every((result) => result.passed) + ? [] + : ["diagram reviews"]), +]; +const passed = failedChecks.length === 0; await mkdir("artifacts/quality", { recursive: true }); await writeValidatedJsonArtifact({ path: "artifacts/quality/documentation-review.json", @@ -138,11 +143,30 @@ await writeValidatedJsonArtifact({ }, }); if (!passed) { + for (const entry of routeScope) { + if (entry.documented) continue; + process.stderr.write(`${entry.path} does not name route IDs: ${entry.missingRouteIds.join(", ")}\n`); + } if (missingRuntimeVariables.length > 0) { process.stderr.write(`README.md does not name: ${missingRuntimeVariables.join(", ")}\n`); } + if (!reportDigestValid) { + process.stderr.write( + `${ledger.evidenceReport.repoPath} does not contain the ledger's canonicalSha256\n`, + ); + } + for (const result of results) { + if (result.passed) continue; + process.stderr.write( + `diagram review ${result.diagram} (${result.sourcePath}) is not a recorded PASS in ` + + `${ledger.evidenceReport.repoPath} (docs/architecture/review-ledger.json)\n`, + ); + } + if (results.length !== 2) { + process.stderr.write(`review ledger has ${results.length} diagram reviews; 2 are required\n`); + } process.stderr.write( - "Documentation readiness: FAIL_UNVERIFIED (canonical scoped-review evidence is incomplete)\n", + `Documentation readiness: FAIL_UNVERIFIED (failed: ${failedChecks.join(", ")})\n`, ); process.exit(1); } diff --git a/tests/unit/documentation-readiness.test.ts b/tests/unit/documentation-readiness.test.ts new file mode 100644 index 00000000..55f677dc --- /dev/null +++ b/tests/unit/documentation-readiness.test.ts @@ -0,0 +1,55 @@ +import { spawnSync } from "node:child_process"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const INPUTS = [ + "README.md", + "docs/accessibility/manual-checklist.md", + "docs/architecture/review-ledger.json", + "docs/architecture/review-evidence.md", +]; + +/** Runs verify:documentation on a copy of its inputs, `mutate` applied to README.md. */ +async function verifyWithReadme(mutate: (readme: string) => string) { + const root = await mkdtemp(path.join(tmpdir(), "documentation-readiness-")); + try { + for (const input of INPUTS) await cp(input, path.join(root, input), { recursive: true }); + const readme = path.join(root, "README.md"); + await writeFile(readme, mutate(await readFile(readme, "utf8"))); + return spawnSync(process.execPath, [path.resolve("scripts/verify-documentation-readiness.ts")], { + cwd: root, + encoding: "utf8", + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +// 1101/14-06. A route missing from a scope document used to fail with only +// "canonical scoped-review evidence is incomplete", which points at the +// diagram review instead of the document to fix. +describe("verify:documentation failure output (1101/14-06)", () => { + it("passes the documents as they are", async () => { + const result = await verifyWithReadme((readme) => readme); + expect(result.status, result.stderr).toBe(0); + }); + + it("names the document and the route ID it is missing", async () => { + const result = await verifyWithReadme((readme) => readme.replaceAll("`EXAMPLES_PLATFORM`", "")); + expect(result.status).toBe(1); + expect(result.stderr).toContain("README.md does not name route IDs: EXAMPLES_PLATFORM"); + expect(result.stderr).toMatch(/FAIL_UNVERIFIED \(failed: route scope\)/u); + expect(result.stderr).not.toContain("scoped-review evidence"); + }); + + it("names every failed check", async () => { + const result = await verifyWithReadme((readme) => + readme.replaceAll("`EXAMPLES_PLATFORM`", "").replaceAll("`RELEASE_TARGET`", ""), + ); + expect(result.stderr).toContain("README.md does not name: RELEASE_TARGET"); + expect(result.stderr).toMatch(/\(failed: route scope, runtime variables\)/u); + }); +});