Adversarial re-verification of the gate itself found three holes. The disposition check matched the finding's verdict anywhere in the table row, and every row also carries the prior verdict — so `| NS-01 | PARTIAL | FIXED |` satisfied a receipt that said either. It now reads the disposition from its own column, which is the check the gate was supposed to be performing all along. An evidence path only had to exist. A row could point at an unrelated suite and look substantiated, so each evidence file must now name the finding it is evidence for; rows the review labelled differently declare their own markers rather than the check being loosened. The file-transfer bundle budget was a number typed into prose next to a number in config, which is exactly the evidence drift the cross-audit raised. The gate now compares them. Each hole was confirmed by breaking the input and watching the gate fail: a disposition disagreement, a budget changed to 60,000, and NS-07 pointed at the public cache suite. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
166 lines
6.3 KiB
TypeScript
166 lines
6.3 KiB
TypeScript
import { access, readFile } from "node:fs/promises";
|
|
|
|
/**
|
|
* GOV-03. Joins the machine-readable remediation dispositions against the prose
|
|
* ledger.
|
|
*
|
|
* The previous ledger declared "All 38 are now FIXED" while six of those rows
|
|
* were reproducibly partial. A sentence is cheap and a reviewer reads it as
|
|
* evidence, so this gate makes the claim derivable rather than authored: every
|
|
* disposition must name a test path that exists, the prose must carry the same
|
|
* verdict for the same id, and a blanket closure sentence is only allowed when
|
|
* nothing is still open.
|
|
*/
|
|
|
|
type Disposition = Readonly<{
|
|
id: string;
|
|
previous: string;
|
|
disposition: string;
|
|
summary: string;
|
|
evidence: readonly string[];
|
|
/**
|
|
* Labels the evidence files actually carry. Defaults to the finding id; a row
|
|
* declares its own when the review labelled the work differently, as the
|
|
* cross-audit ids did.
|
|
*/
|
|
markers?: readonly string[];
|
|
}>;
|
|
|
|
const DISPOSITIONS_PATH = "docs/operations/adapter-remediation-dispositions.json";
|
|
const LEDGER_PATH = "docs/operations/adapter-remediation-ledger.md";
|
|
/**
|
|
* Finding ids are only unique within a review pass — the second pass also used
|
|
* `SW-01` — so rows are matched inside this pass's section rather than anywhere
|
|
* in the document.
|
|
*/
|
|
const SECTION_HEADING = "## Third re-review (2026-08-14)";
|
|
const CLOSED_DISPOSITIONS: ReadonlySet<string> = new Set(["FIXED"]);
|
|
const RECIPES_PATH = "config/recipes/frontend-capability-recipes.json";
|
|
/** The table is `| id | prior verdict | disposition | evidence |`. */
|
|
const DISPOSITION_OFFSET_FROM_ID = 2;
|
|
/** Sentences that assert everything is done, and therefore need proof. */
|
|
const BLANKET_CLOSURE = /All\s+(?:\d+|findings|rows)[^.\n]*\b(?:FIXED|closed)\b/giu;
|
|
|
|
async function main(): Promise<void> {
|
|
const problems: string[] = [];
|
|
const raw: unknown = JSON.parse(await readFile(DISPOSITIONS_PATH, "utf8"));
|
|
if (
|
|
raw === null ||
|
|
typeof raw !== "object" ||
|
|
!Array.isArray((raw as { dispositions?: unknown }).dispositions)
|
|
) {
|
|
console.error(`${DISPOSITIONS_PATH}: dispositions array is required`);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
const dispositions = (raw as { dispositions: Disposition[] }).dispositions;
|
|
const document = await readFile(LEDGER_PATH, "utf8");
|
|
const sectionStart = document.indexOf(SECTION_HEADING);
|
|
if (sectionStart < 0) {
|
|
console.error(`${LEDGER_PATH}: missing section "${SECTION_HEADING}"`);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
const ledger = document.slice(sectionStart);
|
|
|
|
const seen = new Set<string>();
|
|
for (const row of dispositions) {
|
|
if (typeof row.id !== "string" || row.id.length === 0) {
|
|
problems.push("a disposition row has no id");
|
|
continue;
|
|
}
|
|
if (seen.has(row.id)) problems.push(`duplicate disposition id: ${row.id}`);
|
|
seen.add(row.id);
|
|
if (typeof row.disposition !== "string" || row.disposition.length === 0) {
|
|
problems.push(`${row.id}: disposition is required`);
|
|
}
|
|
if (!Array.isArray(row.evidence) || row.evidence.length === 0) {
|
|
problems.push(`${row.id}: at least one evidence path is required`);
|
|
continue;
|
|
}
|
|
const markers = row.markers ?? [row.id];
|
|
for (const path of row.evidence) {
|
|
let contents: string;
|
|
try {
|
|
contents = await readFile(path, "utf8");
|
|
} catch {
|
|
problems.push(`${row.id}: evidence path does not exist: ${path}`);
|
|
continue;
|
|
}
|
|
// A path that exists proves nothing on its own. The file has to name the
|
|
// finding it is evidence for, so a row cannot point at an unrelated suite
|
|
// and look substantiated.
|
|
if (markers.length > 0 && !markers.some((mark) => contents.includes(mark))) {
|
|
problems.push(
|
|
`${row.id}: ${path} does not mention ${markers.join(" or ")}`,
|
|
);
|
|
}
|
|
}
|
|
// The prose must carry the same verdict for the same id, so a reader of the
|
|
// document and a reader of the receipt cannot reach different conclusions.
|
|
const line = ledger
|
|
.split("\n")
|
|
.find((candidate) => candidate.includes(`| ${row.id} |`));
|
|
if (!line) {
|
|
problems.push(`${row.id}: no row in ${LEDGER_PATH}`);
|
|
continue;
|
|
}
|
|
// The row carries the prior verdict as well, so the disposition is read
|
|
// from its own column. Matching anywhere in the line let the "previous"
|
|
// cell satisfy the check and hid a disagreement between the two records.
|
|
const cells = line.split("|").map((cell) => cell.trim());
|
|
const recorded = cells[cells.indexOf(row.id) + DISPOSITION_OFFSET_FROM_ID];
|
|
if (recorded !== `\`${row.disposition}\``) {
|
|
problems.push(
|
|
`${row.id}: ${LEDGER_PATH} records ${
|
|
recorded ?? "nothing"
|
|
} where the receipt says \`${row.disposition}\``,
|
|
);
|
|
}
|
|
}
|
|
|
|
const open = dispositions.filter(
|
|
(row) => !CLOSED_DISPOSITIONS.has(row.disposition),
|
|
);
|
|
const blanketClaims = [...document.matchAll(BLANKET_CLOSURE)];
|
|
if (open.length > 0 && blanketClaims.length > 0) {
|
|
problems.push(
|
|
`${LEDGER_PATH} declares a blanket closure (${blanketClaims
|
|
.map((match) => `"${match[0]}"`)
|
|
.join(", ")}) while ${open.length} finding(s) are still open: ${open
|
|
.map((row) => `${row.id}=${row.disposition}`)
|
|
.join(", ")}`,
|
|
);
|
|
}
|
|
|
|
// GOV-05 / X-AUDIT-04. The most drift-prone evidence in this document is a
|
|
// number someone typed. The one number that gates a release is checked
|
|
// against its source of truth rather than trusted.
|
|
const recipes: unknown = JSON.parse(await readFile(RECIPES_PATH, "utf8"));
|
|
const fileTransfer = (
|
|
(recipes as { recipes?: readonly Record<string, unknown>[] }).recipes ?? []
|
|
).find((recipe) => recipe["id"] === "file-transfer");
|
|
const budget = fileTransfer?.["bundleBudgetGzipBytes"];
|
|
if (typeof budget !== "number") {
|
|
problems.push(`${RECIPES_PATH}: file-transfer has no bundle budget`);
|
|
} else if (!ledger.includes(budget.toLocaleString("en-US"))) {
|
|
problems.push(
|
|
`${LEDGER_PATH} does not state the configured file-transfer budget of ${budget.toLocaleString(
|
|
"en-US",
|
|
)} gzip bytes`,
|
|
);
|
|
}
|
|
|
|
if (problems.length > 0) {
|
|
for (const problem of problems) console.error(problem);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
console.log(
|
|
`Remediation ledger: ${dispositions.length} dispositions joined to ` +
|
|
`${LEDGER_PATH}; ${open.length} open; evidence paths verified`,
|
|
);
|
|
}
|
|
|
|
await main();
|