From 757d1c736b1349b71f755cfd04f4a20e0cdf632f Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 5 Nov 2026 22:21:37 +0900 Subject: [PATCH] =?UTF-8?q?fix(a11y)!:=20=EC=8B=A4=ED=8C=A8=20=ED=99=94?= =?UTF-8?q?=EB=A9=B4=EC=9C=BC=EB=A1=9C=EB=A7=8C=20=EA=B2=80=EC=82=AC?= =?UTF-8?q?=ED=95=9C=20=EB=9D=BC=EC=9A=B0=ED=8A=B8=EB=A5=BC=20a11y.json?= =?UTF-8?q?=EC=97=90=EC=84=9C=20=ED=86=B5=EA=B3=BC=EB=A1=9C=20=EC=84=B8?= =?UTF-8?q?=EC=A7=80=20=EC=95=8A=EB=8A=94=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 문제 접근성 e2e는 보호 라우트의 목록 조회에 빈 목록을 답한다. 자원 하나를 읽는 상세 라우트(참조 기능의 상세, 생성기 --with-create의 상세)는 일반 답이 없어 실패 화면에서 axe를 돌린다. 그런데 scripts/write-a11y-report.ts는 설치된 라우트 전체를 scope에 넣고 automatedStatus "passed"를 쓴다. 화면 내용은 한 번도 검사하지 않은 라우트가 통과로 보고된다. annotation은 test 결과에만 남는다. 고른 방법과 버린 대안 - scripts/lib/a11y-data-state.ts(새 파일): 라우트 test가 끝날 때 자기 상태를 artifacts/tests/a11y-data-state/.json에 남긴다. 실패 화면으로만 봤으면 이유를 쓰고, 아니면 지운다. 그래서 라우트마다 마지막 실행이 정한다. 세 엔진이 같은 파일을 쓰므로 임시 파일에 쓰고 rename한다. - tests/e2e/accessibility.spec.ts: 위 기록을 남긴다(공개 라우트와 데이터 상태로 본 라우트는 지운다). - a11y.json schemaVersion 2: scope에는 자기 상태로 검사한 라우트만, notChecked에는 실패 화면으로만 본 라우트와 이유를 넣는다. 둘을 합치면 설치된 라우트 레지스트리와 같아야 한다(schema가 본다). - 상세 라우트를 실제 데이터 상태로 검사하는 안은 버렸다. 출력 validator에서 유효한 값을 일반적으로 만들 수 없고, 기능마다 견본 응답을 받는 자리를 새로 열어야 한다. - annotation을 Playwright JSON reporter로 읽는 안은 버렸다. test:a11y 스크립트의 reporter와 산출물 경로를 바꿔야 하고, 경로는 다른 작업 (12-04)이 바꾸는 중이다. - 매뉴얼(docs/manual/reference/create-feature.md)에 상세 라우트가 notChecked로 적히고 수동 검토가 맡는다고 적는다. 위험·한계 - a11y.json 모양이 바뀐다(schemaVersion 1 → 2, scope 의미, 새 필드). 템플릿 안의 소비자는 ci-artifact-validator 하나이고 같이 바꿨다. - 템플릿 상태에서도 참조 기능의 상세 라우트가 notChecked로 나온다. 지금까지 실제로 검사하지 않던 것이 드러나는 것이다. - 기록 디렉터리는 artifacts/tests 아래 고정 경로다. 12-04가 산출물 경로를 실행마다 나누면 이 경로도 따라가야 한다. - 라우트 test가 돌지 않은 실행에서는 그 라우트의 지난 기록이 남는다. 지난 기록이 "검사 안 함"이면 덜 주장하는 쪽이라 위험하지 않다. 검증 - desktop batch에서 확인: node scripts/run-vitest.ts run tests/unit/a11y-report.test.ts corepack pnpm test:a11y (세 엔진) 뒤 artifacts/tests/a11y.json의 notChecked에 REFERENCE_RESOURCE_DETAIL이 있고 scope에 없어야 한다. - RED: 부모 커밋에는 scripts/lib/a11y-data-state.ts가 없어 새 unit test 파일 전체가 실패한다. Refs: 1103/11-01 BREAKING CHANGE: artifacts/tests/a11y.json이 schemaVersion 2가 된다. scope는 실패 화면으로만 본 라우트를 빼고, 그 라우트는 새 notChecked 배열에 { routeId, reason }으로 들어간다. scope를 설치된 라우트 전체로 읽던 도구는 scope와 notChecked를 합쳐 읽는다. --- docs/manual/reference/create-feature.md | 3 +- scripts/contracts/release-artifacts.ts | 27 ++++++- scripts/lib/a11y-data-state.ts | 98 +++++++++++++++++++++++ scripts/write-a11y-report.ts | 21 +++-- tests/e2e/accessibility.spec.ts | 13 +-- tests/unit/a11y-report.test.ts | 101 ++++++++++++++++++++++++ 6 files changed, 247 insertions(+), 16 deletions(-) create mode 100644 scripts/lib/a11y-data-state.ts create mode 100644 tests/unit/a11y-report.test.ts diff --git a/docs/manual/reference/create-feature.md b/docs/manual/reference/create-feature.md index 93dc002f..e052d8c7 100644 --- a/docs/manual/reference/create-feature.md +++ b/docs/manual/reference/create-feature.md @@ -94,7 +94,8 @@ corepack pnpm create:feature [--context ] [--with-create [--key - 모든 operation이 `responseEnvelope: "ENVELOPE_V1"`이다. - 쿼리 1개: `LIST_STANDARD` profile. - 무효화 topic 1개, namespace 1개, edge 1개. -- 빈 배열 응답은 공유 빈 상태 화면으로 그린다. `tests/e2e/accessibility.spec.ts`는 이 목록 조회에 빈 목록을 Envelope 모양으로 답하고, 경고 상자가 없는지 본 뒤 axe를 돌린다. 그래서 생성한 라우트는 빈 상태 화면으로 검사된다. +- 빈 배열 응답은 공유 빈 상태 화면으로 그린다. `tests/e2e/accessibility.spec.ts`는 이 목록 조회에 빈 목록을 Envelope 모양으로 답하고, 경고 상자가 없는지 본 뒤 axe를 돌린다. 그래서 생성한 목록 라우트는 빈 상태 화면으로 검사된다. +- `--with-create`의 상세 라우트는 자원 하나를 읽는다. 빈 목록으로 답할 수 없어 실패 화면에서만 axe를 돌린다. 그래서 `artifacts/tests/a11y.json`은 이 라우트를 통과(`scope`)로 세지 않고 `notChecked`에 이유와 함께 적는다. 화면 내용의 접근성은 수동 검토 기록(`artifacts/tests/a11y-manual/.md`)이 맡는다. 한도·마감 같은 숫자는 `scripts/create-feature.ts`의 템플릿이 정본이다. diff --git a/scripts/contracts/release-artifacts.ts b/scripts/contracts/release-artifacts.ts index a40de465..0c22dc85 100644 --- a/scripts/contracts/release-artifacts.ts +++ b/scripts/contracts/release-artifacts.ts @@ -830,20 +830,41 @@ function addUniqueStringIssues( } } +/** + * 1103/11-01. `automatedStatus` speaks for `scope` only. A route axe saw only + * on its failure state (a detail page whose one-resource read has no generic + * answer, scripts/lib/a11y-data-state.ts) is in `notChecked` with its reason + * instead; its content is the manual review's. Together they are the + * installed route registry. Version 1 put every route in `scope`. + */ export const automatedA11yArtifactSchema = z .object({ - schemaVersion: z.literal(1), + schemaVersion: z.literal(2), generatedAt: timestamp, scope: z.array(nonEmptyString).min(1).max(128), + notChecked: z + .array(z.object({ routeId: nonEmptyString, reason: nonEmptyString }).strict()) + .max(128), threshold: z.object({ critical: z.literal(0), serious: z.literal(0) }).strict(), automatedStatus: z.literal("passed"), manualReview: z.literal("see artifacts/tests/a11y-manual/report.json"), }) .strict() .superRefine((artifact, context) => { + const installed: readonly string[] = MANUAL_A11Y_ROUTE_IDS; + const notCheckedIds = artifact.notChecked.map(({ routeId }) => routeId); addUniqueStringIssues(artifact.scope, ["scope"], context); - if (JSON.stringify(artifact.scope) !== JSON.stringify(MANUAL_A11Y_ROUTE_IDS)) { - context.addIssue({ code: "custom", path: ["scope"], message: "must match the installed route registry" }); + addUniqueStringIssues(notCheckedIds, ["notChecked"], context); + if (notCheckedIds.some((routeId) => !installed.includes(routeId))) { + context.addIssue({ code: "custom", path: ["notChecked"], message: "must name installed routes" }); + } + const checked = installed.filter((routeId) => !notCheckedIds.includes(routeId)); + if (JSON.stringify(artifact.scope) !== JSON.stringify(checked)) { + context.addIssue({ + code: "custom", + path: ["scope"], + message: "must be the installed route registry less the notChecked routes", + }); } }); diff --git a/scripts/lib/a11y-data-state.ts b/scripts/lib/a11y-data-state.ts new file mode 100644 index 00000000..81dfd740 --- /dev/null +++ b/scripts/lib/a11y-data-state.ts @@ -0,0 +1,98 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** + * 1103/11-01. Which routes `tests/e2e/accessibility.spec.ts` could scan only + * on their failure state. The spec answers a protected page's list reads with + * an empty list; a read of one resource (a detail page) has no generic answer, + * so that page is scanned on its error screen. Passing axe there says nothing + * about the page's own content, and `artifacts/tests/a11y.json` used to list + * it as passed anyway. + * + * Each route's test leaves its answer in this directory: a record when it saw + * only the failure state, no record otherwise. A route's latest run therefore + * decides. `scripts/write-a11y-report.ts` reads the records and reports those + * routes under `notChecked` instead of in the passed `scope`. + */ +export const A11Y_DATA_STATE_DIRECTORY = "artifacts/tests/a11y-data-state"; + +export type A11yNotCheckedRoute = Readonly<{ routeId: string; reason: string }>; + +const ROUTE_ID = /^[A-Z][A-Z0-9_]*$/u; + +function recordPath(directory: string, routeId: string): string { + if (!ROUTE_ID.test(routeId)) { + throw new TypeError(`not a route id: ${routeId}`); + } + return path.join(directory, `${routeId}.json`); +} + +/** `reason` is null when the route was scanned in its own state. */ +export async function recordA11yDataState( + routeId: string, + reason: string | null, + directory: string = A11Y_DATA_STATE_DIRECTORY, +): Promise { + const file = recordPath(directory, routeId); + if (reason === null) { + await rm(file, { force: true }); + return; + } + if (reason.length === 0) throw new TypeError("a not-checked route needs a reason"); + await mkdir(directory, { recursive: true }); + // Every engine records the same route at once: write whole, then rename. + const pending = `${file}.${randomUUID()}.tmp`; + await writeFile(pending, `${JSON.stringify({ routeId, reason })}\n`, "utf8"); + await rename(pending, file); +} + +/** The records for `routeIds`, in their order. A route with none was checked. */ +export async function readA11yNotCheckedRoutes( + routeIds: readonly string[], + directory: string = A11Y_DATA_STATE_DIRECTORY, +): Promise { + const notChecked: A11yNotCheckedRoute[] = []; + for (const routeId of routeIds) { + let text: string; + try { + text = await readFile(recordPath(directory, routeId), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + const record = JSON.parse(text) as Partial>; + if ( + record.routeId !== routeId || + typeof record.reason !== "string" || + record.reason.length === 0 + ) { + throw new TypeError(`malformed a11y data-state record for ${routeId}`); + } + notChecked.push(Object.freeze({ routeId, reason: record.reason })); + } + return Object.freeze(notChecked); +} + +/** + * The `artifacts/tests/a11y.json` value. `automatedStatus: "passed"` speaks + * for `scope` only: the installed routes axe scanned in their own state. + */ +export function automatedA11yReport( + input: Readonly<{ + routeIds: readonly string[]; + notChecked: readonly A11yNotCheckedRoute[]; + generatedAt: string; + }>, +) { + const notCheckedIds = new Set(input.notChecked.map(({ routeId }) => routeId)); + return { + schemaVersion: 2 as const, + generatedAt: input.generatedAt, + scope: input.routeIds.filter((routeId) => !notCheckedIds.has(routeId)), + notChecked: input.notChecked.map(({ routeId, reason }) => ({ routeId, reason })), + threshold: { critical: 0 as const, serious: 0 as const }, + automatedStatus: "passed" as const, + manualReview: "see artifacts/tests/a11y-manual/report.json" as const, + }; +} diff --git a/scripts/write-a11y-report.ts b/scripts/write-a11y-report.ts index f4d80e44..da9b06e2 100644 --- a/scripts/write-a11y-report.ts +++ b/scripts/write-a11y-report.ts @@ -1,19 +1,26 @@ import { mkdir } from "node:fs/promises"; import { automatedA11yArtifactSchema } from "./contracts/release-artifacts.ts"; +import { + automatedA11yReport, + readA11yNotCheckedRoutes, +} from "./lib/a11y-data-state.ts"; import { MANUAL_A11Y_ROUTE_IDS } from "./lib/manual-a11y-evidence.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; await mkdir("artifacts/tests", { recursive: true }); +// 1103/11-01. Routes the spec scanned only on their failure state are not +// reported as passed. +const notChecked = await readA11yNotCheckedRoutes(MANUAL_A11Y_ROUTE_IDS); await writeValidatedJsonArtifact({ path: "artifacts/tests/a11y.json", schema: automatedA11yArtifactSchema, - value: { - schemaVersion: 1, + value: automatedA11yReport({ + routeIds: MANUAL_A11Y_ROUTE_IDS, + notChecked, generatedAt: new Date().toISOString(), - scope: MANUAL_A11Y_ROUTE_IDS, - threshold: { critical: 0, serious: 0 }, - automatedStatus: "passed", - manualReview: "see artifacts/tests/a11y-manual/report.json", - }, + }), }); +for (const { routeId, reason } of notChecked) { + process.stdout.write(`a11y: ${routeId} not checked in its own state (${reason})\n`); +} diff --git a/tests/e2e/accessibility.spec.ts b/tests/e2e/accessibility.spec.ts index 189069fd..e8264ff4 100644 --- a/tests/e2e/accessibility.spec.ts +++ b/tests/e2e/accessibility.spec.ts @@ -1,4 +1,5 @@ import AxeBuilder from "@axe-core/playwright"; +import { recordA11yDataState } from "../../scripts/lib/a11y-data-state.ts"; import { expect, test, type Page } from "../support/browser/strict-browser-test.ts"; import { ROUTE_REGISTRY } from "../../src/features/installed-feature-contracts.ts"; import { COMPOSED_CONTRACT_CONTRIBUTIONS } from "../../src/features/loaded-feature-runtimes.ts"; @@ -72,18 +73,20 @@ for (const definition of ROUTES) { if (singleResourceReads.length === 0) { // A shape the page does not read fails here, before axe passes it. await expect(page.getByRole("alert")).toHaveCount(0); + await recordA11yDataState(definition.routeId, null); } else { // No generic body answers a read of one resource: a detail page is - // scanned in its failure state, and the report says which. - test.info().annotations.push({ - type: "a11y-data-state", - description: `failure state: ${singleResourceReads.join(", ")} reads one resource`, - }); + // scanned in its failure state. 1103/11-01. a11y.json then lists it + // under `notChecked`, not as passed. + const reason = `scanned on its failure state only: ${singleResourceReads.join(", ")} reads one resource`; + test.info().annotations.push({ type: "a11y-data-state", description: reason }); + await recordA11yDataState(definition.routeId, reason); } } else { await page.goto(route); const title = CATALOG[`route.${definition.routeId}.title`]; await expect(heading).toHaveText(title ?? definition.title); + await recordA11yDataState(definition.routeId, null); } expect(await blockingViolations(page)).toEqual([]); }); diff --git a/tests/unit/a11y-report.test.ts b/tests/unit/a11y-report.test.ts new file mode 100644 index 00000000..330b46a9 --- /dev/null +++ b/tests/unit/a11y-report.test.ts @@ -0,0 +1,101 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { automatedA11yArtifactSchema } from "../../scripts/contracts/release-artifacts.ts"; +import { + automatedA11yReport, + readA11yNotCheckedRoutes, + recordA11yDataState, +} from "../../scripts/lib/a11y-data-state.ts"; +import { MANUAL_A11Y_ROUTE_IDS } from "../../scripts/lib/manual-a11y-evidence.ts"; + +/** + * 1103/11-01. The accessibility spec scans a detail page on its failure state + * (no generic answer exists for a read of one resource). `a11y.json` must not + * count that route as passed: it goes to `notChecked` with its reason. + */ +const ROUTES: readonly string[] = MANUAL_A11Y_ROUTE_IDS; +const [CHECKED_ROUTE = "", FAILURE_STATE_ROUTE = ""] = ROUTES; +const REASON = "scanned on its failure state only: TEST_GET_ENTITY reads one resource"; +const GENERATED_AT = "2026-11-05T00:00:00.000Z"; + +const directories: string[] = []; + +async function recordDirectory(): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "a11y-data-state-")); + directories.push(directory); + return directory; +} + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((directory) => + rm(directory, { recursive: true, force: true }), + ), + ); +}); + +describe("automated accessibility report", () => { + it("needs two installed routes to tell a checked route from a failure-state one", () => { + expect(ROUTES.length).toBeGreaterThanOrEqual(2); + }); + + it("reports a route scanned only on its failure state as not checked", async () => { + const directory = await recordDirectory(); + await recordA11yDataState(CHECKED_ROUTE, null, directory); + await recordA11yDataState(FAILURE_STATE_ROUTE, REASON, directory); + + const report = automatedA11yReport({ + routeIds: ROUTES, + notChecked: await readA11yNotCheckedRoutes(ROUTES, directory), + generatedAt: GENERATED_AT, + }); + + expect(report.scope).toContain(CHECKED_ROUTE); + expect(report.scope).not.toContain(FAILURE_STATE_ROUTE); + expect(report.notChecked).toEqual([ + { routeId: FAILURE_STATE_ROUTE, reason: REASON }, + ]); + expect(automatedA11yArtifactSchema.safeParse(report).success).toBe(true); + }); + + it("lets a route's latest run decide", async () => { + const directory = await recordDirectory(); + await recordA11yDataState(FAILURE_STATE_ROUTE, REASON, directory); + await recordA11yDataState(FAILURE_STATE_ROUTE, null, directory); + + await expect(readA11yNotCheckedRoutes(ROUTES, directory)).resolves.toEqual([]); + }); + + it("refuses a report that counts a failure-state route as passed", () => { + const report = automatedA11yReport({ + routeIds: ROUTES, + notChecked: [{ routeId: FAILURE_STATE_ROUTE, reason: REASON }], + generatedAt: GENERATED_AT, + }); + + expect( + automatedA11yArtifactSchema.safeParse({ ...report, scope: [...ROUTES] }) + .success, + ).toBe(false); + }); + + it("refuses a report that drops an installed route from both lists", () => { + const report = automatedA11yReport({ + routeIds: ROUTES, + notChecked: [], + generatedAt: GENERATED_AT, + }); + + expect(automatedA11yArtifactSchema.safeParse(report).success).toBe(true); + expect( + automatedA11yArtifactSchema.safeParse({ + ...report, + scope: ROUTES.filter((routeId) => routeId !== FAILURE_STATE_ROUTE), + }).success, + ).toBe(false); + }); +});