diff --git a/artifacts/tests/a11y-manual/APP_HOME.md b/artifacts/tests/a11y-manual/APP_HOME.md index cfd99fd..12a847a 100644 --- a/artifacts/tests/a11y-manual/APP_HOME.md +++ b/artifacts/tests/a11y-manual/APP_HOME.md @@ -1,15 +1,18 @@ # APP_HOME accessibility review Status: pending-manual-review - +Route ID: APP_HOME +Release ID: Reviewer: - -Keyboard: automated tab-order fixture passed; human review pending. - -Focus: automated visible-focus fixture passed; route-change review pending. - -Screen reader: pending. - -Reduced motion: automated media-query fixture passed; human review pending. - -Color signal: pending. +Reviewed at: +Signature: +Attestation: pending +M1 Keyboard: pending +M2 Visible focus: pending +M3 Route focus: pending +M4 Modal focus: not-applicable (no modal on this route) +M5 Error association: not-applicable (no form error on this route) +M6 Color signal: pending +M7 Reduced motion: pending +Screen reader: pending +Notes: Automated axe, keyboard-focus, and reduced-motion evidence is available; human review pending. diff --git a/artifacts/tests/a11y-manual/NOT_FOUND.md b/artifacts/tests/a11y-manual/NOT_FOUND.md new file mode 100644 index 0000000..f69a783 --- /dev/null +++ b/artifacts/tests/a11y-manual/NOT_FOUND.md @@ -0,0 +1,18 @@ +# NOT_FOUND accessibility review + +Status: pending-manual-review +Route ID: NOT_FOUND +Release ID: +Reviewer: +Reviewed at: +Signature: +Attestation: pending +M1 Keyboard: pending +M2 Visible focus: pending +M3 Route focus: pending +M4 Modal focus: not-applicable (no modal on this route) +M5 Error association: not-applicable (no form error on this route) +M6 Color signal: pending +M7 Reduced motion: pending +Screen reader: pending +Notes: Human review pending. diff --git a/artifacts/tests/a11y-manual/SAMPLE_RESOURCE_LIST.md b/artifacts/tests/a11y-manual/SAMPLE_RESOURCE_LIST.md new file mode 100644 index 0000000..5e160e8 --- /dev/null +++ b/artifacts/tests/a11y-manual/SAMPLE_RESOURCE_LIST.md @@ -0,0 +1,18 @@ +# SAMPLE_RESOURCE_LIST accessibility review + +Status: pending-manual-review +Route ID: SAMPLE_RESOURCE_LIST +Release ID: +Reviewer: +Reviewed at: +Signature: +Attestation: pending +M1 Keyboard: pending +M2 Visible focus: pending +M3 Route focus: pending +M4 Modal focus: not-applicable (no modal on this route) +M5 Error association: not-applicable (no form error on this route) +M6 Color signal: pending +M7 Reduced motion: pending +Screen reader: pending +Notes: Human review pending. diff --git a/docs/accessibility/manual-checklist.md b/docs/accessibility/manual-checklist.md index fc4bb55..9ef5f1d 100644 --- a/docs/accessibility/manual-checklist.md +++ b/docs/accessibility/manual-checklist.md @@ -1,17 +1,47 @@ # Manual accessibility review checklist Automated axe checks do not establish WCAG conformance. A human reviewer must -copy this checklist to `artifacts/tests/a11y-manual/.md`, execute it -on the release candidate, and sign it. +review all three route records in `artifacts/tests/a11y-manual/` against one +release candidate and sign them. Copy the template fields exactly; the gate +rejects blank identity/timestamp/signature fields, pending verdicts, mismatched +release IDs, or missing routes. -- Status: `pending` or `reviewed` -- Reviewer and reviewed-at timestamp -- Keyboard: all actions reachable in logical order -- Focus: visible, route changes deterministic, modal restore verified -- Screen reader: headings, live regions, errors, and actions announced once -- Reduced motion: non-essential animation suppressed -- Color signal: every state has text/icon/structure in addition to color -- Notes and linked defect IDs +Allowed item verdicts: -Passing the automated threshold means only that the tested pages had zero -critical/serious axe findings under the recorded browser run. +- `pass` +- `not-applicable ()` + +Required record: + +```text +Status: reviewed +Route ID: APP_HOME +Release ID: +Reviewer: +Reviewed at: +Signature: +Attestation: accepted +M1 Keyboard: pass +M2 Visible focus: pass +M3 Route focus: pass +M4 Modal focus: not-applicable (no modal on this route) +M5 Error association: not-applicable (no form error on this route) +M6 Color signal: pass +M7 Reduced motion: pass +Screen reader: pass +Notes: +``` + +The reviewer must verify: + +- M1: every action works without a pointing device +- M2: every focused element has a visible indicator +- M3: route transitions move focus to a deterministic target +- M4: modal focus is trapped and restored, when a modal exists +- M5: errors are programmatically associated with their controls, when present +- M6: state never relies on color alone +- M7: non-essential motion is suppressed with reduced-motion preference +- Screen reader: headings, live regions, errors, and actions are announced once + +Passing automated evidence means only that tested pages had no critical or +serious axe findings under the recorded browser run. diff --git a/scripts/lib/manual-a11y-evidence.mjs b/scripts/lib/manual-a11y-evidence.mjs new file mode 100644 index 0000000..a56dd61 --- /dev/null +++ b/scripts/lib/manual-a11y-evidence.mjs @@ -0,0 +1,58 @@ +export const MANUAL_A11Y_ROUTE_IDS = Object.freeze([ + "APP_HOME", + "SAMPLE_RESOURCE_LIST", + "NOT_FOUND", +]); + +const REVIEW_FIELDS = Object.freeze([ + "M1 Keyboard", + "M2 Visible focus", + "M3 Route focus", + "M4 Modal focus", + "M5 Error association", + "M6 Color signal", + "M7 Reduced motion", + "Screen reader", +]); + +/** @param {string} content */ +export function validateManualA11yEvidence(content) { + const fields = Object.fromEntries( + content + .split(/\r?\n/) + .map((line) => /^([^:]+):\s*(.*)$/.exec(line)) + .filter(Boolean) + .map((match) => [ + /** @type {RegExpExecArray} */ (match)[1].trim(), + /** @type {RegExpExecArray} */ (match)[2].trim(), + ]), + ); + const failures = []; + if (fields.Status !== "reviewed") failures.push("Status"); + if (!fields["Route ID"]) failures.push("Route ID"); + if (!fields["Release ID"]) failures.push("Release ID"); + if (!fields.Reviewer) failures.push("Reviewer"); + if (!fields.Signature) failures.push("Signature"); + if (fields.Attestation !== "accepted") failures.push("Attestation"); + if ( + !fields["Reviewed at"] || + !Number.isFinite(Date.parse(fields["Reviewed at"])) + ) { + failures.push("Reviewed at"); + } + + for (const field of REVIEW_FIELDS) { + const result = fields[field]; + if ( + result !== "pass" && + !/^not-applicable \(.+\)$/.test(result ?? "") + ) { + failures.push(field); + } + } + return Object.freeze({ + fields: Object.freeze(fields), + failures: Object.freeze(failures), + passed: failures.length === 0, + }); +} diff --git a/scripts/verify-a11y-manual.mjs b/scripts/verify-a11y-manual.mjs index 329f573..d594eb2 100644 --- a/scripts/verify-a11y-manual.mjs +++ b/scripts/verify-a11y-manual.mjs @@ -1,25 +1,71 @@ -import { readFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; -const evidence = await readFile( - "artifacts/tests/a11y-manual/APP_HOME.md", - "utf8", +import { + MANUAL_A11Y_ROUTE_IDS, + validateManualA11yEvidence, +} from "./lib/manual-a11y-evidence.mjs"; + +/** @type {Array<{ + * routeId: string; + * path: string; + * reviewer: string | null; + * reviewedAt: string | null; + * releaseId: string | null; + * failures: readonly string[]; + * passed: boolean; + * }>} */ +const results = []; +for (const routeId of MANUAL_A11Y_ROUTE_IDS) { + const path = `artifacts/tests/a11y-manual/${routeId}.md`; + const evidence = await readFile(path, "utf8"); + const validation = validateManualA11yEvidence(evidence); + const failures = + validation.fields["Route ID"] === routeId + ? validation.failures + : Object.freeze([...validation.failures, "Route ID mismatch"]); + results.push({ + routeId, + path, + reviewer: validation.fields.Reviewer ?? null, + reviewedAt: validation.fields["Reviewed at"] ?? null, + releaseId: validation.fields["Release ID"] ?? null, + failures, + passed: validation.passed && failures.length === 0, + }); +} +const releaseIds = new Set(results.map((result) => result.releaseId)); +const passed = + results.every((result) => result.passed) && + releaseIds.size === 1 && + results.every((result) => Boolean(result.releaseId)); + +await mkdir("artifacts/tests/a11y-manual", { recursive: true }); +await writeFile( + "artifacts/tests/a11y-manual/report.json", + `${JSON.stringify( + { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + scope: MANUAL_A11Y_ROUTE_IDS, + results, + coherentRelease: releaseIds.size === 1, + passed, + }, + null, + 2, + )}\n`, ); -const required = [ - "Status: reviewed", - "Reviewer:", - "Keyboard:", - "Focus:", - "Screen reader:", - "Reduced motion:", - "Color signal:", -]; - -const missing = required.filter((marker) => !evidence.includes(marker)); -if (missing.length > 0) { +if (!passed) { + const failures = results + .filter((result) => !result.passed) + .map((result) => `${result.routeId}: ${result.failures.join(", ")}`); + if (releaseIds.size !== 1) failures.push("release IDs do not match"); process.stderr.write( - `Manual accessibility evidence is incomplete: ${missing.join(", ")}\n`, + `Manual accessibility evidence is incomplete:\n${failures.join("\n")}\n`, ); process.exit(1); } -process.stdout.write("Manual accessibility evidence: PASS\n"); +process.stdout.write( + `Manual accessibility evidence: PASS (${results.length} routes)\n`, +); diff --git a/scripts/write-a11y-report.mjs b/scripts/write-a11y-report.mjs index f7304f9..73faf22 100644 --- a/scripts/write-a11y-report.mjs +++ b/scripts/write-a11y-report.mjs @@ -10,7 +10,7 @@ await writeFile( scope: ["APP_HOME", "SAMPLE_RESOURCE_LIST", "NOT_FOUND"], threshold: { critical: 0, serious: 0 }, automatedStatus: "passed", - manualReview: "see artifacts/tests/a11y-manual/APP_HOME.md", + manualReview: "see artifacts/tests/a11y-manual/report.json", }, null, 2, diff --git a/tests/unit/manual-a11y-evidence.test.js b/tests/unit/manual-a11y-evidence.test.js new file mode 100644 index 0000000..09d6f4b --- /dev/null +++ b/tests/unit/manual-a11y-evidence.test.js @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; + +import { validateManualA11yEvidence } from "../../scripts/lib/manual-a11y-evidence.mjs"; + +const reviewed = `Status: reviewed +Route ID: APP_HOME +Release ID: release-1 +Reviewer: reviewer@example.test +Reviewed at: 2026-07-25T12:00:00Z +Signature: review-record-1 +Attestation: accepted +M1 Keyboard: pass +M2 Visible focus: pass +M3 Route focus: pass +M4 Modal focus: not-applicable (no modal) +M5 Error association: not-applicable (no form error) +M6 Color signal: pass +M7 Reduced motion: pass +Screen reader: pass +Notes: no defects`; + +describe("manual accessibility evidence", () => { + it("accepts a complete signed human review record", () => { + expect(validateManualA11yEvidence(reviewed)).toMatchObject({ + failures: [], + passed: true, + }); + }); + + it("rejects pending, unsigned, or incomplete evidence", () => { + expect( + validateManualA11yEvidence( + reviewed + .replace("Status: reviewed", "Status: pending-manual-review") + .replace("Signature: review-record-1", "Signature:") + .replace("Screen reader: pass", "Screen reader: pending"), + ), + ).toMatchObject({ + failures: ["Status", "Signature", "Screen reader"], + passed: false, + }); + }); + + it("does not treat an unexplained not-applicable verdict as evidence", () => { + expect( + validateManualA11yEvidence( + reviewed.replace( + "M4 Modal focus: not-applicable (no modal)", + "M4 Modal focus: not-applicable", + ), + ), + ).toMatchObject({ + failures: ["M4 Modal focus"], + passed: false, + }); + }); +});