Compare commits

...
Author SHA1 Message Date
donghyeon-ka 2725c35c28 fix: require signed accessibility evidence per route 2026-07-25 22:21:23 +09:00
8 changed files with 272 additions and 42 deletions
+14 -11
View File
@@ -1,15 +1,18 @@
# APP_HOME accessibility review # APP_HOME accessibility review
Status: pending-manual-review Status: pending-manual-review
Route ID: APP_HOME
Release ID:
Reviewer: Reviewer:
Reviewed at:
Keyboard: automated tab-order fixture passed; human review pending. Signature:
Attestation: pending
Focus: automated visible-focus fixture passed; route-change review pending. M1 Keyboard: pending
M2 Visible focus: pending
Screen reader: pending. M3 Route focus: pending
M4 Modal focus: not-applicable (no modal on this route)
Reduced motion: automated media-query fixture passed; human review pending. M5 Error association: not-applicable (no form error on this route)
M6 Color signal: pending
Color signal: pending. M7 Reduced motion: pending
Screen reader: pending
Notes: Automated axe, keyboard-focus, and reduced-motion evidence is available; human review pending.
+18
View File
@@ -0,0 +1,18 @@
# NOT_FOUND accessibility review
Status: pending-manual-review
Route ID: NOT_FOUND
Release ID:
Reviewer:
Reviewed at:
Signature:
Attestation: pending
M1 Keyboard: pending
M2 Visible focus: pending
M3 Route focus: pending
M4 Modal focus: not-applicable (no modal on this route)
M5 Error association: not-applicable (no form error on this route)
M6 Color signal: pending
M7 Reduced motion: pending
Screen reader: pending
Notes: Human review pending.
@@ -0,0 +1,18 @@
# SAMPLE_RESOURCE_LIST accessibility review
Status: pending-manual-review
Route ID: SAMPLE_RESOURCE_LIST
Release ID:
Reviewer:
Reviewed at:
Signature:
Attestation: pending
M1 Keyboard: pending
M2 Visible focus: pending
M3 Route focus: pending
M4 Modal focus: not-applicable (no modal on this route)
M5 Error association: not-applicable (no form error on this route)
M6 Color signal: pending
M7 Reduced motion: pending
Screen reader: pending
Notes: Human review pending.
+42 -12
View File
@@ -1,17 +1,47 @@
# Manual accessibility review checklist # Manual accessibility review checklist
Automated axe checks do not establish WCAG conformance. A human reviewer must Automated axe checks do not establish WCAG conformance. A human reviewer must
copy this checklist to `artifacts/tests/a11y-manual/<route-id>.md`, execute it review all three route records in `artifacts/tests/a11y-manual/` against one
on the release candidate, and sign it. release candidate and sign them. Copy the template fields exactly; the gate
rejects blank identity/timestamp/signature fields, pending verdicts, mismatched
release IDs, or missing routes.
- Status: `pending` or `reviewed` Allowed item verdicts:
- Reviewer and reviewed-at timestamp
- Keyboard: all actions reachable in logical order
- Focus: visible, route changes deterministic, modal restore verified
- Screen reader: headings, live regions, errors, and actions announced once
- Reduced motion: non-essential animation suppressed
- Color signal: every state has text/icon/structure in addition to color
- Notes and linked defect IDs
Passing the automated threshold means only that the tested pages had zero - `pass`
critical/serious axe findings under the recorded browser run. - `not-applicable (<specific reason>)`
Required record:
```text
Status: reviewed
Route ID: APP_HOME
Release ID: <immutable release ID>
Reviewer: <human reviewer identity>
Reviewed at: <RFC 3339 timestamp>
Signature: <reviewer identity or approved signature reference>
Attestation: accepted
M1 Keyboard: pass
M2 Visible focus: pass
M3 Route focus: pass
M4 Modal focus: not-applicable (no modal on this route)
M5 Error association: not-applicable (no form error on this route)
M6 Color signal: pass
M7 Reduced motion: pass
Screen reader: pass
Notes: <observations and linked defect IDs>
```
The reviewer must verify:
- M1: every action works without a pointing device
- M2: every focused element has a visible indicator
- M3: route transitions move focus to a deterministic target
- M4: modal focus is trapped and restored, when a modal exists
- M5: errors are programmatically associated with their controls, when present
- M6: state never relies on color alone
- M7: non-essential motion is suppressed with reduced-motion preference
- Screen reader: headings, live regions, errors, and actions are announced once
Passing automated evidence means only that tested pages had no critical or
serious axe findings under the recorded browser run.
+58
View File
@@ -0,0 +1,58 @@
export const MANUAL_A11Y_ROUTE_IDS = Object.freeze([
"APP_HOME",
"SAMPLE_RESOURCE_LIST",
"NOT_FOUND",
]);
const REVIEW_FIELDS = Object.freeze([
"M1 Keyboard",
"M2 Visible focus",
"M3 Route focus",
"M4 Modal focus",
"M5 Error association",
"M6 Color signal",
"M7 Reduced motion",
"Screen reader",
]);
/** @param {string} content */
export function validateManualA11yEvidence(content) {
const fields = Object.fromEntries(
content
.split(/\r?\n/)
.map((line) => /^([^:]+):\s*(.*)$/.exec(line))
.filter(Boolean)
.map((match) => [
/** @type {RegExpExecArray} */ (match)[1].trim(),
/** @type {RegExpExecArray} */ (match)[2].trim(),
]),
);
const failures = [];
if (fields.Status !== "reviewed") failures.push("Status");
if (!fields["Route ID"]) failures.push("Route ID");
if (!fields["Release ID"]) failures.push("Release ID");
if (!fields.Reviewer) failures.push("Reviewer");
if (!fields.Signature) failures.push("Signature");
if (fields.Attestation !== "accepted") failures.push("Attestation");
if (
!fields["Reviewed at"] ||
!Number.isFinite(Date.parse(fields["Reviewed at"]))
) {
failures.push("Reviewed at");
}
for (const field of REVIEW_FIELDS) {
const result = fields[field];
if (
result !== "pass" &&
!/^not-applicable \(.+\)$/.test(result ?? "")
) {
failures.push(field);
}
}
return Object.freeze({
fields: Object.freeze(fields),
failures: Object.freeze(failures),
passed: failures.length === 0,
});
}
+64 -18
View File
@@ -1,25 +1,71 @@
import { readFile } from "node:fs/promises"; import { mkdir, readFile, writeFile } from "node:fs/promises";
const evidence = await readFile( import {
"artifacts/tests/a11y-manual/APP_HOME.md", MANUAL_A11Y_ROUTE_IDS,
"utf8", validateManualA11yEvidence,
} from "./lib/manual-a11y-evidence.mjs";
/** @type {Array<{
* routeId: string;
* path: string;
* reviewer: string | null;
* reviewedAt: string | null;
* releaseId: string | null;
* failures: readonly string[];
* passed: boolean;
* }>} */
const results = [];
for (const routeId of MANUAL_A11Y_ROUTE_IDS) {
const path = `artifacts/tests/a11y-manual/${routeId}.md`;
const evidence = await readFile(path, "utf8");
const validation = validateManualA11yEvidence(evidence);
const failures =
validation.fields["Route ID"] === routeId
? validation.failures
: Object.freeze([...validation.failures, "Route ID mismatch"]);
results.push({
routeId,
path,
reviewer: validation.fields.Reviewer ?? null,
reviewedAt: validation.fields["Reviewed at"] ?? null,
releaseId: validation.fields["Release ID"] ?? null,
failures,
passed: validation.passed && failures.length === 0,
});
}
const releaseIds = new Set(results.map((result) => result.releaseId));
const passed =
results.every((result) => result.passed) &&
releaseIds.size === 1 &&
results.every((result) => Boolean(result.releaseId));
await mkdir("artifacts/tests/a11y-manual", { recursive: true });
await writeFile(
"artifacts/tests/a11y-manual/report.json",
`${JSON.stringify(
{
schemaVersion: 1,
generatedAt: new Date().toISOString(),
scope: MANUAL_A11Y_ROUTE_IDS,
results,
coherentRelease: releaseIds.size === 1,
passed,
},
null,
2,
)}\n`,
); );
const required = [ if (!passed) {
"Status: reviewed", const failures = results
"Reviewer:", .filter((result) => !result.passed)
"Keyboard:", .map((result) => `${result.routeId}: ${result.failures.join(", ")}`);
"Focus:", if (releaseIds.size !== 1) failures.push("release IDs do not match");
"Screen reader:",
"Reduced motion:",
"Color signal:",
];
const missing = required.filter((marker) => !evidence.includes(marker));
if (missing.length > 0) {
process.stderr.write( process.stderr.write(
`Manual accessibility evidence is incomplete: ${missing.join(", ")}\n`, `Manual accessibility evidence is incomplete:\n${failures.join("\n")}\n`,
); );
process.exit(1); process.exit(1);
} }
process.stdout.write("Manual accessibility evidence: PASS\n"); process.stdout.write(
`Manual accessibility evidence: PASS (${results.length} routes)\n`,
);
+1 -1
View File
@@ -10,7 +10,7 @@ await writeFile(
scope: ["APP_HOME", "SAMPLE_RESOURCE_LIST", "NOT_FOUND"], scope: ["APP_HOME", "SAMPLE_RESOURCE_LIST", "NOT_FOUND"],
threshold: { critical: 0, serious: 0 }, threshold: { critical: 0, serious: 0 },
automatedStatus: "passed", automatedStatus: "passed",
manualReview: "see artifacts/tests/a11y-manual/APP_HOME.md", manualReview: "see artifacts/tests/a11y-manual/report.json",
}, },
null, null,
2, 2,
+57
View File
@@ -0,0 +1,57 @@
import { describe, expect, it } from "vitest";
import { validateManualA11yEvidence } from "../../scripts/lib/manual-a11y-evidence.mjs";
const reviewed = `Status: reviewed
Route ID: APP_HOME
Release ID: release-1
Reviewer: reviewer@example.test
Reviewed at: 2026-07-25T12:00:00Z
Signature: review-record-1
Attestation: accepted
M1 Keyboard: pass
M2 Visible focus: pass
M3 Route focus: pass
M4 Modal focus: not-applicable (no modal)
M5 Error association: not-applicable (no form error)
M6 Color signal: pass
M7 Reduced motion: pass
Screen reader: pass
Notes: no defects`;
describe("manual accessibility evidence", () => {
it("accepts a complete signed human review record", () => {
expect(validateManualA11yEvidence(reviewed)).toMatchObject({
failures: [],
passed: true,
});
});
it("rejects pending, unsigned, or incomplete evidence", () => {
expect(
validateManualA11yEvidence(
reviewed
.replace("Status: reviewed", "Status: pending-manual-review")
.replace("Signature: review-record-1", "Signature:")
.replace("Screen reader: pass", "Screen reader: pending"),
),
).toMatchObject({
failures: ["Status", "Signature", "Screen reader"],
passed: false,
});
});
it("does not treat an unexplained not-applicable verdict as evidence", () => {
expect(
validateManualA11yEvidence(
reviewed.replace(
"M4 Modal focus: not-applicable (no modal)",
"M4 Modal focus: not-applicable",
),
),
).toMatchObject({
failures: ["M4 Modal focus"],
passed: false,
});
});
});