From e8b116306c192d2f558a74afb164073fed0ebdfd Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 00:10:22 +0900 Subject: [PATCH 01/15] test(reference-feature): give route-entry waits room under host load (10-02) Each installed route resolves the context route table and a lazy page, and React holds each Suspense reveal for up to 300ms, so the 1s findBy default failed on a busy host with a misleading missing-heading message. Co-Authored-By: Claude Opus 5.5 --- .../reference-feature/reference-page.test.tsx | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/features/reference-feature/reference-page.test.tsx b/tests/features/reference-feature/reference-page.test.tsx index 43fe742c..861b5011 100644 --- a/tests/features/reference-feature/reference-page.test.tsx +++ b/tests/features/reference-feature/reference-page.test.tsx @@ -4,9 +4,15 @@ import { QueryClient, QueryClientProvider, } from "@tanstack/react-query"; -import { render, screen, waitFor } from "@testing-library/react"; +import { + configure, + getConfig, + render, + screen, + waitFor, +} from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { describe, expect, it, vi } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDemoSessionAdapter } from "../../../src/adapters/auth/external-session-adapter.ts"; import type { AuthSessionPort } from "../../../src/application/ports/auth-session-port.ts"; @@ -122,6 +128,16 @@ function inputWith( } describe("reference feature page states", () => { + // Entering an installed route resolves the context route table and then a + // lazy page, and React holds each Suspense reveal for up to 300ms. That + // floor alone uses half of Testing Library's 1s findBy default, so a busy + // host misses it and reports a missing heading instead of a slow route. + // Keep the wider wait scoped to this file; the test timeout still catches + // a real hang. + const defaultAsyncUtilTimeout = getConfig().asyncUtilTimeout; + beforeAll(() => configure({ asyncUtilTimeout: 5_000 })); + afterAll(() => configure({ asyncUtilTimeout: defaultAsyncUtilTimeout })); + it("mounts list and detail keys under the installed governed namespace", async () => { expect(REFERENCE_RESOURCE_QUERY_NAMESPACE).toEqual({ namespaceId: "reference-resource", From 72e7ae1139ee8ff82580cb9f3f29a92c5f69cad8 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 00:22:16 +0900 Subject: [PATCH 02/15] test(vitest): collect only Vitest files and document running one file (10-05) A bare vitest run collected 26 Playwright specs and the host-bound system suite. The documented one-file command goes through run-vitest.ts, which renders the gitignored product index a fresh clone lacks. Co-Authored-By: Claude Opus 5.5 --- docs/testing/taxonomy.md | 18 ++++++++++++ tests/integration/vitest-collection.test.ts | 31 +++++++++++++++++++++ vitest.config.ts | 10 +++++++ 3 files changed, 59 insertions(+) create mode 100644 tests/integration/vitest-collection.test.ts diff --git a/docs/testing/taxonomy.md b/docs/testing/taxonomy.md index 421a3d6d..d1ce9feb 100644 --- a/docs/testing/taxonomy.md +++ b/docs/testing/taxonomy.md @@ -37,6 +37,24 @@ That runner: `vitest.config.ts` also fixes `NODE_ENV=test` so a direct Vitest invocation cannot accidentally select React's production behavior. +To run one test file, use the same runner: + +```sh +node scripts/run-vitest.ts run tests/component/router.test.tsx +``` + +The runner renders the gitignored `src/.generated/` product index first +(`scripts/run-vitest.ts`), so this works in a fresh clone. A direct +`pnpm exec vitest run ` skips that step and fails there with +`Failed to resolve import "../.generated/product-feature-index.ts"`. +`pnpm test:unit -- ` does not narrow the run, because the script already +passes the whole `tests/unit` directory. + +`vitest.config.ts` collects only `tests/**/*.test.{ts,tsx}`; Playwright +`*.spec.ts` files are not Vitest tests. `tests/system/` is collected only when +the command names it, as `test:system` does, so a bare `vitest` run does not +start the CI-runner suite. + The system suite additionally runs `scripts/check-system-test-prerequisites.ts` and fails with one prerequisite report when the CI-runner host does not provide its required Linux facilities. diff --git a/tests/integration/vitest-collection.test.ts b/tests/integration/vitest-collection.test.ts new file mode 100644 index 00000000..7ca0cfe4 --- /dev/null +++ b/tests/integration/vitest-collection.test.ts @@ -0,0 +1,31 @@ +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +function listTestFiles(...filters: string[]): string[] { + const result = spawnSync( + process.execPath, + ["node_modules/vitest/vitest.mjs", "list", "--filesOnly", ...filters], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + return result.stdout.split("\n").filter((line) => line.length > 0); +} + +describe("Vitest file collection", () => { + it("leaves Playwright specs and host system tests out of a bare run", () => { + const files = listTestFiles(); + + expect(files).toContain("tests/integration/vitest-collection.test.ts"); + expect(files.filter((file) => file.endsWith(".spec.ts"))).toEqual([]); + expect(files.filter((file) => file.startsWith("tests/system/"))).toEqual( + [], + ); + }, 30_000); + + it("still collects system tests when test:system names them", () => { + expect(listTestFiles("tests/system")).toContain( + "tests/system/ci-runner/ci-evidence-freshness.test.ts", + ); + }, 30_000); +}); diff --git a/vitest.config.ts b/vitest.config.ts index b46a2507..9235ef63 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,6 +6,13 @@ import { configDefaults, defineConfig } from "vitest/config"; // protects direct `vitest` invocations as well. process.env.NODE_ENV = "test"; +// tests/system needs CI-runner host facilities (see docs/testing/taxonomy.md). +// It runs only when a caller names it, as `test:system` does; a bare `vitest` +// run leaves it out. +const systemTestsRequested = process.argv.some((argument) => + argument.replaceAll("\\", "/").replace(/^\.\//u, "").startsWith("tests/system"), +); + export default defineConfig({ test: { globals: false, @@ -14,6 +21,8 @@ export default defineConfig({ clearMocks: true, mockReset: true, testTimeout: 10_000, + // Playwright specs (*.spec.ts) share tests/ but belong to Playwright. + include: ["tests/**/*.test.{ts,tsx}"], exclude: [ ...configDefaults.exclude, ".tmp/**", @@ -22,6 +31,7 @@ export default defineConfig({ // reports failures that belong to neither. ".worktrees/**", "tests/fixtures/v8-coverage-counter-semantics/**", + ...(systemTestsRequested ? [] : ["tests/system/**"]), ], coverage: { provider: "v8", From d090054ef2a0d4e4ec272d3774b421310ec7a526 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 00:22:16 +0900 Subject: [PATCH 03/15] test(setup): restore real timers after every test (10-06) Five tests restore fake timers only at the end of their body, so a failed assertion left fake timers to the next test in the same file. Co-Authored-By: Claude Opus 5.5 --- tests/setup.ts | 5 ++++- tests/unit/test-setup.test.ts | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test-setup.test.ts diff --git a/tests/setup.ts b/tests/setup.ts index e2621932..d9a6d8e8 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -1,7 +1,10 @@ import "@testing-library/jest-dom/vitest"; import { cleanup } from "@testing-library/react"; -import { afterEach } from "vitest"; +import { afterEach, vi } from "vitest"; afterEach(() => { cleanup(); + // A test that fails before its own vi.useRealTimers() must not leave fake + // timers to the next test in the file. + vi.useRealTimers(); }); diff --git a/tests/unit/test-setup.test.ts b/tests/unit/test-setup.test.ts new file mode 100644 index 00000000..0e921b3b --- /dev/null +++ b/tests/unit/test-setup.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it, vi } from "vitest"; + +// tests/setup.ts restores real timers after every test, so a test that fails +// before its own vi.useRealTimers() does not hand fake timers to the next one. +describe("global test setup", () => { + it("may end with fake timers still installed", () => { + vi.useFakeTimers(); + expect(vi.isFakeTimers()).toBe(true); + }); + + it("starts the next test on real timers", () => { + expect(vi.isFakeTimers()).toBe(false); + }); +}); From d68e25a2ad5e8482a97b07dbc9f9e33ce0af3016 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 00:28:02 +0900 Subject: [PATCH 04/15] test(coverage): ratchet live retry, contract bridge and IndexedDB modules (10-04) The policy guarded legacy retry-policy.ts but not the live V3 retry and contract path rules, and the strategy doc claimed 19 modules at 90% targets. The doc now states the enforced ratchet. The below-threshold fixture used the pre-counter summary shape and failed with a TypeError instead of the 'Risk coverage failed:' diagnostic the negative gate expects. Co-Authored-By: Claude Opus 5.5 --- config/testing/risk-coverage.json | 35 ++++++++++++++++++- .../frontend-platform-testing-strategy.md | 31 +++++++++++++--- tests/fixtures/coverage/below-threshold.json | 16 ++++----- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/config/testing/risk-coverage.json b/config/testing/risk-coverage.json index f3b5eb7b..3095ac12 100644 --- a/config/testing/risk-coverage.json +++ b/config/testing/risk-coverage.json @@ -178,6 +178,36 @@ "functions": 90, "branches": 80 } + }, + { + "path": "src/adapters/http/http-contract-bridge.ts", + "owner": "http-runtime", + "minimum": { + "lines": 85, + "statements": 79, + "functions": 100, + "branches": 73 + } + }, + { + "path": "src/adapters/http/http-retry-lifecycle.ts", + "owner": "http-runtime", + "minimum": { + "lines": 100, + "statements": 100, + "functions": 100, + "branches": 100 + } + }, + { + "path": "src/adapters/storage/indexeddb/indexeddb-runtime.ts", + "owner": "storage-runtime", + "minimum": { + "lines": 79, + "statements": 76, + "functions": 73, + "branches": 72 + } } ], "highRiskPaths": [ @@ -197,7 +227,10 @@ "src/bootstrap/read-bounded-boot-json.ts", "src/contracts/diagnostics.ts", "src/features/reference-feature/adapters/reference-http-gateway.ts", - "src/presentation/adapters/query/application-query.ts" + "src/presentation/adapters/query/application-query.ts", + "src/adapters/http/http-contract-bridge.ts", + "src/adapters/http/http-retry-lifecycle.ts", + "src/adapters/storage/indexeddb/indexeddb-runtime.ts" ], "waivers": [] } diff --git a/docs/testing/frontend-platform-testing-strategy.md b/docs/testing/frontend-platform-testing-strategy.md index 3e6f9fc5..4ddbfd4a 100644 --- a/docs/testing/frontend-platform-testing-strategy.md +++ b/docs/testing/frontend-platform-testing-strategy.md @@ -153,11 +153,25 @@ Chromium, Firefox, WebKit과 compact project로 실행한다. 빠른 Vite 개발 #### 위험 기반 coverage -V8 text/JSON/LCOV를 생성하고 전체 기준과 retry/storage/telemetry/application -composition/compatibility/performance/promotion/chunk/diagnostics/reference HTTP -operation/query-mutation/registry compatibility 19개 high-risk module에 80개 -scoped threshold를 적용한다. critical module 누락 또는 threshold 미달 fixture는 -merge gate를 실패시킨다. +V8 text/JSON/LCOV를 생성하고 `config/testing/risk-coverage.json`의 하한을 +적용한다. 하한은 전체 평균 4개(`summary`: lines 75, statements 73, functions 80, +branches 68)와 `criticalModules` 20개의 모듈별 하한 80개(모듈마다 4개)다. +`check-risk-coverage.ts`는 두 종류를 합쳐 "84 thresholds"로 보고한다. 모듈은 +HTTP 실행(`http-execution-v3.ts`, live 재시도 규칙 `http-retry-lifecycle.ts`, +계약 경로 규칙 `http-contract-bridge.ts`, legacy V2 `retry-policy.ts`, 요청 조립, +bounded body/JSON), server-state scope, Service Worker lifecycle, 저장소 +(`browser-storage-adapter.ts`, `indexeddb-runtime.ts`), telemetry, application +조립, compatibility, chunk recovery, release manifest, boot JSON, diagnostics, +reference HTTP gateway, application query다. 모듈별 하한은 목표치가 아니라 +등록할 때 측정값 근처로 둔 ratchet이다(되돌아가지 못하게 막는 하한). 새로 +넣은 세 모듈은 2026-09-26 측정값을 내림한 값이다. +critical module 누락 또는 threshold 미달 fixture는 merge gate를 실패시킨다. + +realtime(`src/adapters/realtime/`)과 Web Push(`src/adapters/web-push/`)는 +`test:realtime-removal`이 통째로 지우는 선택 runtime이라 +(`scripts/test-realtime-runtime-removal.ts`의 `runtimePaths`) 모듈별 하한에 넣지 +않는다. 넣으면 그 runtime을 지운 다운스트림의 coverage gate가 없는 파일 때문에 +실패한다. 두 영역은 전체 평균 하한만 받는다. Repository inventory의 정적 provenance는 runtime 실행 가능 여부가 아니라 현재 Vitest/V8 producer가 file counter를 생성하는 문장의 존재 여부를 나타낸다. 따라서 @@ -951,6 +965,13 @@ fixture로 확인한다. ### 14.2 위험별 목표 +아래 표는 올려 갈 **목표**다. 지금 gate가 막는 값은 이 표가 아니라 +`config/testing/risk-coverage.json`의 현재값 ratchet이다(2.3절 "위험 기반 +coverage"). 예를 들어 `service-worker-lifecycle.ts`의 branch 하한은 43, +`http-execution-v3.ts`는 52로 표의 목표보다 낮다. 테스트를 보강해 측정값이 +오르면 그 모듈의 하한을 새 측정값으로 올린다. 하한을 내리는 변경은 이유를 +커밋 메시지에 적는다. + | 영역 | Line/statement 참고 | Branch 우선 목표 | | --- | --- | --- | | registry/codec/schema | 95~100% | 100%에 가깝게 | diff --git a/tests/fixtures/coverage/below-threshold.json b/tests/fixtures/coverage/below-threshold.json index 1df4f7e2..0e973e44 100644 --- a/tests/fixtures/coverage/below-threshold.json +++ b/tests/fixtures/coverage/below-threshold.json @@ -1,14 +1,14 @@ { "total": { - "lines": { "pct": 0 }, - "statements": { "pct": 0 }, - "functions": { "pct": 0 }, - "branches": { "pct": 0 } + "lines": { "total": 10, "covered": 0, "skipped": 0, "pct": 0 }, + "statements": { "total": 10, "covered": 0, "skipped": 0, "pct": 0 }, + "functions": { "total": 2, "covered": 0, "skipped": 0, "pct": 0 }, + "branches": { "total": 4, "covered": 0, "skipped": 0, "pct": 0 } }, "src/adapters/http/http-execution-v3.ts": { - "lines": { "pct": 0 }, - "statements": { "pct": 0 }, - "functions": { "pct": 0 }, - "branches": { "pct": 0 } + "lines": { "total": 10, "covered": 0, "skipped": 0, "pct": 0 }, + "statements": { "total": 10, "covered": 0, "skipped": 0, "pct": 0 }, + "functions": { "total": 2, "covered": 0, "skipped": 0, "pct": 0 }, + "branches": { "total": 4, "covered": 0, "skipped": 0, "pct": 0 } } } From 84f2ab929f9b5cf9aed087e4a6cf8fbc187bc2dd Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 00:30:28 +0900 Subject: [PATCH 05/15] docs(testing): match determinism, scenario and taxonomy text to the code (10-07) The strategy doc promised clock/random/scheduler helpers that no test used (the helper's clearTimeout did nothing), listed three scenario IDs the catalog does not have, and the taxonomy described an older test:all and a child-process rule tests/unit does not follow. Removes the unused helper. Co-Authored-By: Claude Opus 5.5 --- .../frontend-platform-testing-strategy.md | 18 +++-- docs/testing/taxonomy.md | 26 ++++--- tests/support/runtime/determinism.ts | 69 ------------------- 3 files changed, 31 insertions(+), 82 deletions(-) delete mode 100644 tests/support/runtime/determinism.ts diff --git a/docs/testing/frontend-platform-testing-strategy.md b/docs/testing/frontend-platform-testing-strategy.md index 4ddbfd4a..866bdec4 100644 --- a/docs/testing/frontend-platform-testing-strategy.md +++ b/docs/testing/frontend-platform-testing-strategy.md @@ -142,8 +142,12 @@ rasterization과 실제 device farm은 프로젝트가 요구할 때 연결한 operation별 success/empty/slow/network/timeout/content/envelope/schema/auth/ 403/404/409/422/429/retry/terminal을 포함한 19개 scenario catalog와 strict -unhandled-request server를 공유한다. clock/random/scheduler/storage helper와 -browser console/page/network failure 정책으로 비결정적 우회를 차단한다. +unhandled-request server(`tests/mocks/server.ts`의 +`onUnhandledRequest: "error"`)를 공유한다. browser test는 공용 fixture +(`tests/support/browser/strict-browser-test.ts`)가 console error/warning, +page error, 실패한 request를 test 실패로 만든다. 시간은 각 test가 +`vi.useFakeTimers()`로 통제하고, 전역 `afterEach`가 실제 timer로 되돌린다 +(`tests/setup.ts`). 공용 clock/random/scheduler helper는 두지 않는다. #### Built-dist와 compact E2E @@ -738,15 +742,19 @@ tests/factories/ | `malformed-json` | 파싱 불가능한 JSON | | `envelope-mismatch` | top-level contract 위반 | | `schema-mismatch` | data payload contract 위반 | -| `auth-recover-once` | 첫 401 후 복구 성공 | -| `auth-persistent-401` | 복구 뒤에도 401 | +| `response-too-large` | 응답 본문이 크기 상한을 넘어 읽기를 멈춤 | +| `unauthenticated-401` | 401. 명령은 적용되지 않음(`NOT_APPLIED`) | | `forbidden-403` | 권한 없음 | | `not-found-404` | 리소스 없음 | | `conflict-409` | mutation conflict | | `validation-422` | 승인된 field/global error | | `rate-limited-429` | Retry-After 포함 | | `server-retry-success` | bounded retry 뒤 성공 | -| `server-terminal-500` | retry 소진 | +| `server-terminal-503` | 503. 조회는 retry 2회 뒤 소진, 명령은 retry 없이 결과 불명(`MAYBE_APPLIED`) | + +이 표는 `tests/mocks/scenarios/catalog.ts`의 `HTTP_SCENARIO_IDS`와 같은 순서다. +401 뒤 인증 복구 흐름은 catalog가 아니라 `tests/integration/auth-recovery.test.ts`가 +다룬다. ### 11.3 Catalog 규칙 diff --git a/docs/testing/taxonomy.md b/docs/testing/taxonomy.md index d1ce9feb..46e5d172 100644 --- a/docs/testing/taxonomy.md +++ b/docs/testing/taxonomy.md @@ -16,9 +16,11 @@ Each gate is blocking in its declared scope. Failures are not downgraded with | end-to-end | `pnpm test:e2e` | pinned browser engines | `artifacts/tests/e2e/` | | accessibility | `pnpm test:a11y` | pinned browser engines | `artifacts/tests/a11y.json` | -`test:all` is the normal product-development loop. It intentionally includes -runtime-schema, unit, capability-contract, component, integration, reference -feature and recipe suites, but does not include `test:system`. CI-runner and +`test:all` is the normal product-development loop. It runs, in order, +`test:runtime-schema`, `test:unit`, `test:contract`, `test:component`, +`test:integration`, `test:reference-feature`, `test:platform` (`tests/platform` +and `tests/architecture`), `check:release-coherence` and `test:recipes` +(`package.json` `test:all`). It does not include `test:system`. CI-runner and supply-chain assurance has different host prerequisites and is invoked explicitly in the assurance path. @@ -82,11 +84,19 @@ CI / release assurance: A host-level process/sandbox test must not be placed in `tests/unit` merely because it uses Vitest. The classification follows the system boundary and -prerequisites, not the test framework. Process-heavy tests that spawn child -processes but do not require privileged host facilities belong in -`tests/integration`; for example the provider guardian transaction protocol -lives at `tests/integration/provider-guardian-transaction.test.ts`. Its -handshake timeout is a test watchdog, not a production protocol deadline. +prerequisites, not the test framework: + +- Tests that need systemd, bubblewrap, cgroup v2 or CI-provider process + controls belong in `tests/system`. +- Tests of a long-lived child-process protocol belong in `tests/integration`; + for example the provider guardian transaction protocol lives at + `tests/integration/provider-guardian-transaction.test.ts`. Its handshake + timeout is a test watchdog, not a production protocol deadline. +- A script's own tests may start that script, Node or Vitest as a short-lived + child process to check its command-line contract, and stay in `tests/unit` + next to the script's other tests. Such a suite declares its own longer + timeout instead of raising the global one; for example + `tests/unit/risk-coverage.test.ts` uses `PROCESS_HEAVY_TIMEOUT_MS`. End-to-end and automated accessibility scenarios run on the pinned Chromium, Firefox, and WebKit engines. The responsive contract explicitly exercises diff --git a/tests/support/runtime/determinism.ts b/tests/support/runtime/determinism.ts deleted file mode 100644 index 18413e18..00000000 --- a/tests/support/runtime/determinism.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { vi } from "vitest"; - -export function createDeterministicClock(start = 0) { - let current = start; - return Object.freeze({ - now: () => current, - sleep: async (milliseconds: number, signal?: AbortSignal) => { - if (signal?.aborted) throw new DOMException("aborted", "AbortError"); - current += Math.max(0, milliseconds); - }, - advance(milliseconds: number) { - current += Math.max(0, milliseconds); - }, - }); -} - -export function createSeededRandom(seed = 1) { - let state = seed >>> 0; - return () => { - state = (state * 1_664_525 + 1_013_904_223) >>> 0; - return state / 0x1_0000_0000; - }; -} - -export function createControlledScheduler() { - const callbacks: Array<() => void> = []; - return Object.freeze({ - callbacks, - setTimeout: vi.fn((callback: () => void) => { - callbacks.push(callback); - return callbacks.length - 1; - }), - clearTimeout: vi.fn(), - runNext() { - callbacks.shift()?.(); - }, - }); -} - -export function createRecordingStorage() { - const values = new Map(); - const operations: Array< - Readonly<{ operation: "get" | "set" | "remove"; key: string }> - > = []; - const storage: Storage = { - getItem(key) { - operations.push({ operation: "get", key }); - return values.get(key) ?? null; - }, - setItem(key, value) { - operations.push({ operation: "set", key }); - values.set(key, value); - }, - removeItem(key) { - operations.push({ operation: "remove", key }); - values.delete(key); - }, - clear() { - values.clear(); - }, - key(index) { - return [...values.keys()][index] ?? null; - }, - get length() { - return values.size; - }, - }; - return Object.freeze({ storage, operations, values }); -} From 7870c48b5fcf22103e912888fcf1cd5a0d2338aa Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 01:01:49 +0900 Subject: [PATCH 06/15] test(evidence): forbid focused tests and expired quarantines (11-08) Only the capabilities config set forbidOnly, so a committed test.only ran one e2e, visual or storybook test and passed. The skip check accepted any quarantine marker without comparing its expiry date to today. Co-Authored-By: Claude Opus 5.5 --- playwright.config.ts | 1 + playwright.storybook.config.ts | 1 + playwright.visual.config.ts | 1 + scripts/check-test-evidence.ts | 27 ++++++- .../forbidden/expired-quarantine.fixture | 2 + .../forbidden/focused-test.fixture | 1 + tests/unit/test-evidence-source-rules.test.ts | 70 +++++++++++++++++++ 7 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture create mode 100644 tests/fixtures/test-evidence/forbidden/focused-test.fixture create mode 100644 tests/unit/test-evidence-source-rules.test.ts diff --git a/playwright.config.ts b/playwright.config.ts index 83fe0819..b6523844 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./tests/e2e", + forbidOnly: true, outputDir: "./artifacts/tests/e2e/results", reporter: [ ["list"], diff --git a/playwright.storybook.config.ts b/playwright.storybook.config.ts index 4c2fac04..75abbe7c 100644 --- a/playwright.storybook.config.ts +++ b/playwright.storybook.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./tests/storybook", + forbidOnly: true, outputDir: "./artifacts/tests/storybook/results", reporter: [ ["list"], diff --git a/playwright.visual.config.ts b/playwright.visual.config.ts index f66f9967..36528303 100644 --- a/playwright.visual.config.ts +++ b/playwright.visual.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./tests/visual", + forbidOnly: true, snapshotDir: "./tests/visual/__snapshots__", outputDir: "./artifacts/tests/visual/results", reporter: [ diff --git a/scripts/check-test-evidence.ts b/scripts/check-test-evidence.ts index 9158bde6..ab3e0dc6 100644 --- a/scripts/check-test-evidence.ts +++ b/scripts/check-test-evidence.ts @@ -281,17 +281,27 @@ if (!scenarioOnly) { const sourceFiles = (await filesBelow(sourceRoot)).filter( (file) => fixtureMode || !file.split(path.sep).includes("fixtures"), ); + const today = new Date().toISOString().slice(0, 10); for (const file of sourceFiles) { if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue; const source = await readFile(file, "utf8"); facts.scannedFiles += 1; + if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) { + failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`); + } const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g; if (skipPattern.test(source)) { - const quarantine = - /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/; - if (!quarantine.test(source)) { + const quarantines = [ + ...source.matchAll( + /quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g, + ), + ]; + if (quarantines.length === 0) { failures.push(`${file}: skip/fixme lacks owned expiring quarantine`); } + for (const [, expires = ""] of quarantines) { + if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`); + } } const wholeUiMask = /\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i; @@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) { failures.push(`playwright.config.ts missing release evidence token ${token}`); } } + // playwright.dev.config.ts spreads playwright.config.ts. + for (const config of [ + "playwright.config.ts", + "playwright.visual.config.ts", + "playwright.storybook.config.ts", + "playwright.capabilities.config.ts", + ]) { + if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) { + failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`); + } + } const e2eFiles = (await filesBelow("tests/e2e")).filter((file) => /\.spec\.ts$/.test(file), diff --git a/tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture b/tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture new file mode 100644 index 00000000..5fb8dab3 --- /dev/null +++ b/tests/fixtures/test-evidence/forbidden/expired-quarantine.fixture @@ -0,0 +1,2 @@ +// quarantine(owner=frontend-platform, defect=expired-fixture, expires=2020-01-01) +test.skip("quarantine whose expiry date has passed", async () => {}); diff --git a/tests/fixtures/test-evidence/forbidden/focused-test.fixture b/tests/fixtures/test-evidence/forbidden/focused-test.fixture new file mode 100644 index 00000000..f3481dc1 --- /dev/null +++ b/tests/fixtures/test-evidence/forbidden/focused-test.fixture @@ -0,0 +1 @@ +test.only("focused test that would pass alone", async () => {}); diff --git a/tests/unit/test-evidence-source-rules.test.ts b/tests/unit/test-evidence-source-rules.test.ts new file mode 100644 index 00000000..abd36c8f --- /dev/null +++ b/tests/unit/test-evidence-source-rules.test.ts @@ -0,0 +1,70 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterAll, describe, expect, it } from "vitest"; + +const FORBIDDEN = "tests/fixtures/test-evidence/forbidden"; +const roots: string[] = []; + +afterAll(async () => { + await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))); +}); + +function checkSource(sourceRoot: string) { + const result = spawnSync( + process.execPath, + [ + "scripts/check-test-evidence.ts", + "--source-only", + "--source-root", + sourceRoot, + "--artifact", + "artifacts/quality/test-evidence-source-rules.json", + ], + { encoding: "utf8" }, + ); + return { status: result.status, stderr: result.stderr }; +} + +// 11-08. Each forbidden fixture is checked alone, so a rule that stops firing +// is not hidden by another fixture in the same directory still failing. +describe("test evidence source rules", () => { + it("rejects a focused test", () => { + const result = checkSource(`${FORBIDDEN}/focused-test.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("focused test"); + }, 30_000); + + it("rejects a quarantine whose expiry date has passed", () => { + const result = checkSource(`${FORBIDDEN}/expired-quarantine.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("quarantine expired on 2020-01-01"); + }, 30_000); + + it("rejects a skip without a quarantine", () => { + const result = checkSource(`${FORBIDDEN}/unowned-skip.fixture`); + expect(result.status).toBe(1); + expect(result.stderr).toContain("lacks owned expiring quarantine"); + }, 30_000); + + it("accepts a skip under an unexpired quarantine", async () => { + const root = await mkdtemp(path.join(tmpdir(), "test-evidence-quarantine-")); + roots.push(root); + const file = path.join(root, "quarantined.fixture"); + // Built from parts so this file is not itself a skip the checker scans. + const skip = ["test", "skip"].join("."); + await writeFile( + file, + [ + `// ${"quarantine"}(owner=frontend-platform, defect=open-fixture, expires=2999-12-31)`, + `${skip}("quarantined", async () => {});`, + "", + ].join("\n"), + ); + const result = checkSource(file); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + }, 30_000); +}); From 48b84951b805d84b3e6db8f773ccf77ca3453c0f Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:00:35 +0900 Subject: [PATCH 07/15] test(e2e): assert reference data and cover save success and retry (11-06) The route spec only checked that the protected screen opened, so a schema violating 200 or a 503 still passed it. It now asserts the mocked row, and a new spec saves once and retries after a 429 in a real browser. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/reference-route.spec.ts | 2 + tests/e2e/reference-save.spec.ts | 100 +++++++++++++++++++ tests/support/browser/strict-browser-test.ts | 21 +++- 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/reference-save.spec.ts diff --git a/tests/e2e/reference-route.spec.ts b/tests/e2e/reference-route.spec.ts index dd410823..677581c8 100644 --- a/tests/e2e/reference-route.spec.ts +++ b/tests/e2e/reference-route.spec.ts @@ -29,4 +29,6 @@ test("opens the protected integration route through the local demo seam", async page.getByRole("heading", { name: protectedRoute.title }), ).toBeVisible(); await expect(page.getByText("인증됨")).toBeVisible(); + // The mocked row went through the production codec, mapper and query cache. + await expect(page.getByText("Browser reference")).toBeVisible(); }); diff --git a/tests/e2e/reference-save.spec.ts b/tests/e2e/reference-save.spec.ts new file mode 100644 index 00000000..5396e159 --- /dev/null +++ b/tests/e2e/reference-save.spec.ts @@ -0,0 +1,100 @@ +import type { Page, Route } from "@playwright/test"; +import { expect, test } from "../support/browser/strict-browser-test.ts"; + +const CREATE_URL = "http://localhost:8080/api/reference-resources"; + +type CreateCall = Readonly<{ name: unknown; idempotencyKey: string | null }>; + +async function openForm(page: Page) { + await page.route(`${CREATE_URL}?*`, (route) => route.fulfill({ json: [] })); + await page.goto("/examples/reference-resources/new"); + await page.getByRole("button", { name: "로그인 시작" }).click(); + await expect( + page.getByRole("heading", { name: "Reference resource 만들기" }), + ).toBeVisible(); +} + +/** Answers each POST with the next responder and records what was sent. */ +async function routeCreate( + page: Page, + responders: readonly ((route: Route) => Promise)[], +): Promise { + const calls: CreateCall[] = []; + await page.route(CREATE_URL, async (route) => { + const request = route.request(); + if (request.method() !== "POST") return route.fallback(); + calls.push({ + name: (request.postDataJSON() as { name?: unknown }).name, + idempotencyKey: await request.headerValue("idempotency-key"), + }); + const respond = responders[calls.length - 1]; + if (!respond) throw new Error(`unexpected create call ${calls.length}`); + await respond(route); + }); + return calls; +} + +const created = (route: Route) => + route.fulfill({ + status: 201, + json: { + id: "browser-created", + name: "Saved in the browser", + createdAt: "2026-07-26T00:00:00.000Z", + }, + }); + +test("saves a new reference resource through the production command path", async ({ + page, +}) => { + await openForm(page); + const calls = await routeCreate(page, [created]); + + await page + .getByRole("textbox", { name: /새 항목 이름/ }) + .fill("Saved in the browser"); + await page.getByRole("button", { name: "저장" }).click(); + + await expect(page.getByRole("status")).toHaveText("저장했습니다."); + expect(calls).toHaveLength(1); + expect(calls[0]?.name).toBe("Saved in the browser"); + expect(calls[0]?.idempotencyKey).toBeTruthy(); +}); + +test.describe("after a rate-limited save", () => { + // Chromium and WebKit log a 4xx fetch response to the console, and the + // client cancels the 429 body unread, which Chromium reports as aborted. + test.use({ + allowedBrowserFailure: + /^console:error:.*status of 429|^requestfailed:POST:\/api\/reference-resources:net::ERR_ABORTED$/u, + }); + + test("keeps the input and saves on retry", async ({ page }) => { + await openForm(page); + const calls = await routeCreate(page, [ + (route) => + route.fulfill({ + status: 429, + headers: { "retry-after": "1" }, + json: { title: "Too many requests" }, + }), + created, + ]); + + const name = page.getByRole("textbox", { name: /새 항목 이름/ }); + await name.fill("Saved in the browser"); + await page.getByRole("button", { name: "저장" }).click(); + + await expect( + page.getByText("요청이 많습니다. 잠시 후 다시 시도해 주세요."), + ).toBeVisible(); + await expect(name).toHaveValue("Saved in the browser"); + await page.getByRole("button", { name: "다시 시도" }).click(); + + await expect(page.getByRole("status")).toHaveText("저장했습니다."); + expect(calls.map((call) => call.name)).toEqual([ + "Saved in the browser", + "Saved in the browser", + ]); + }); +}); diff --git a/tests/support/browser/strict-browser-test.ts b/tests/support/browser/strict-browser-test.ts index 93e22a8c..23aaad7b 100644 --- a/tests/support/browser/strict-browser-test.ts +++ b/tests/support/browser/strict-browser-test.ts @@ -12,8 +12,18 @@ function ignoredConsole(message: ConsoleMessage) { ); } -export const test = base.extend({ - page: async ({ page }, use) => { +export const test = base.extend<{ + /** + * Browser failures a test provokes on purpose, matched against the recorded + * `console:…`, `pageerror:…` or `requestfailed:…` line (for example the + * browser's log line for a mocked 4xx). Everything else still fails. One + * pattern, not an array: `test.use` reads an array as a `[value, options]` + * fixture tuple. + */ + allowedBrowserFailure: RegExp | undefined; +}>({ + allowedBrowserFailure: [undefined, { option: true }], + page: async ({ page, allowedBrowserFailure }, use) => { const failures: string[] = []; const onConsole = (message: ConsoleMessage) => { if ( @@ -38,7 +48,12 @@ export const test = base.extend({ page.off("console", onConsole); page.off("pageerror", onPageError); page.off("requestfailed", onRequestFailed); - expect(failures, "unexpected browser console/page errors").toEqual([]); + expect( + failures.filter( + (failure) => !allowedBrowserFailure?.test(failure), + ), + "unexpected browser console/page errors", + ).toEqual([]); }, }); From a96fd241db6c9f3814baefa71b2258fbb90e2494 Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:06:23 +0900 Subject: [PATCH 08/15] test(scale): measure one app growing with real-shape features (11-04) The scale evidence came from stub features, so the cost of one app growing towards its declared ceiling was never measured. Copies of the reference feature at 25 and 100 are now the baseline, and the acceptance matrix says what the ceilings rest on and which axes stay unverified. Co-Authored-By: Claude Opus 5.5 --- config/scale/feature-shape-growth.json | 210 ++++++++++++++ .../2026-09-23-extreme-scale-acceptance.md | 26 +- scripts/scale/measure-feature-shape-growth.ts | 257 ++++++++++++++++++ tests/unit/feature-shape-growth.test.ts | 103 +++++++ 4 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 config/scale/feature-shape-growth.json create mode 100644 scripts/scale/measure-feature-shape-growth.ts create mode 100644 tests/unit/feature-shape-growth.test.ts diff --git a/config/scale/feature-shape-growth.json b/config/scale/feature-shape-growth.json new file mode 100644 index 00000000..2a28b36e --- /dev/null +++ b/config/scale/feature-shape-growth.json @@ -0,0 +1,210 @@ +{ + "schemaVersion": 1, + "measuredAt": "2026-09-27T04:03:00.821Z", + "command": "node scripts/scale/measure-feature-shape-growth.ts --sizes=0,25,100 --repeats=2", + "environment": { + "node": "24.14.0", + "platform": "linux", + "arch": "x64", + "cpuModel": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", + "logicalCpus": 12, + "availableParallelism": 12, + "typescript": "Version 7.0.2", + "vite": "vite/8.1.5 linux-x64 node-v24.14.0" + }, + "sourceFeature": "src/features/reference-feature", + "runs": [ + { + "repeat": 1, + "addedFeatures": 0, + "routes": 10, + "tsc": { + "passed": true, + "firstError": null, + "files": 840, + "types": 284678, + "symbols": 802946, + "instantiations": 712089, + "memoryKb": 520454, + "wallMs": 3408 + }, + "build": { + "wallMs": 1208, + "initialJavascriptFiles": 13, + "initialJavascriptGzipBytes": 200001, + "lazyJavascriptFiles": 14, + "lazyJavascriptGzipBytes": 24938 + }, + "loadBefore": [ + 25.05, + 16.88, + 7.75 + ], + "loadAfter": [ + 26.4, + 17.3, + 7.93 + ] + }, + { + "repeat": 2, + "addedFeatures": 0, + "routes": 10, + "tsc": { + "passed": true, + "firstError": null, + "files": 840, + "types": 284678, + "symbols": 802946, + "instantiations": 712089, + "memoryKb": 520395, + "wallMs": 3930 + }, + "build": { + "wallMs": 1526, + "initialJavascriptFiles": 13, + "initialJavascriptGzipBytes": 200001, + "lazyJavascriptFiles": 14, + "lazyJavascriptGzipBytes": 24938 + }, + "loadBefore": [ + 26.4, + 17.3, + 7.93 + ], + "loadAfter": [ + 27.65, + 17.71, + 8.12 + ] + }, + { + "repeat": 1, + "addedFeatures": 25, + "routes": 110, + "tsc": { + "passed": true, + "firstError": null, + "files": 1366, + "types": 1055723, + "symbols": 2831441, + "instantiations": 3579093, + "memoryKb": 1422337, + "wallMs": 8121 + }, + "build": { + "wallMs": 1676, + "initialJavascriptFiles": 39, + "initialJavascriptGzipBytes": 274493, + "lazyJavascriptFiles": 165, + "lazyJavascriptGzipBytes": 171638 + }, + "loadBefore": [ + 27.65, + 17.71, + 8.12 + ], + "loadAfter": [ + 27.17, + 17.93, + 8.29 + ] + }, + { + "repeat": 2, + "addedFeatures": 25, + "routes": 110, + "tsc": { + "passed": true, + "firstError": null, + "files": 1366, + "types": 1055723, + "symbols": 2831441, + "instantiations": 3579093, + "memoryKb": 1422431, + "wallMs": 8714 + }, + "build": { + "wallMs": 2140, + "initialJavascriptFiles": 39, + "initialJavascriptGzipBytes": 274493, + "lazyJavascriptFiles": 165, + "lazyJavascriptGzipBytes": 171638 + }, + "loadBefore": [ + 27.17, + 17.93, + 8.29 + ], + "loadAfter": [ + 27.55, + 18.46, + 8.62 + ] + }, + { + "repeat": 1, + "addedFeatures": 100, + "routes": 410, + "tsc": { + "passed": true, + "firstError": null, + "files": 2941, + "types": 3349868, + "symbols": 8940533, + "instantiations": 12157304, + "memoryKb": 4195900, + "wallMs": 31315 + }, + "build": { + "wallMs": 6010, + "initialJavascriptFiles": 114, + "initialJavascriptGzipBytes": 490658, + "lazyJavascriptFiles": 615, + "lazyJavascriptGzipBytes": 704938 + }, + "loadBefore": [ + 27.43, + 18.59, + 8.71 + ], + "loadAfter": [ + 29.72, + 20.12, + 9.64 + ] + }, + { + "repeat": 2, + "addedFeatures": 100, + "routes": 410, + "tsc": { + "passed": true, + "firstError": null, + "files": 2941, + "types": 3349868, + "symbols": 8940533, + "instantiations": 12157304, + "memoryKb": 4195704, + "wallMs": 43234 + }, + "build": { + "wallMs": 3702, + "initialJavascriptFiles": 114, + "initialJavascriptGzipBytes": 490658, + "lazyJavascriptFiles": 615, + "lazyJavascriptGzipBytes": 704938 + }, + "loadBefore": [ + 29.72, + 20.12, + 9.64 + ], + "loadAfter": [ + 35.75, + 22.97, + 11.09 + ] + } + ] +} diff --git a/docs/reviews/2026-09-23-extreme-scale-acceptance.md b/docs/reviews/2026-09-23-extreme-scale-acceptance.md index f62aba3c..b12c2e43 100644 --- a/docs/reviews/2026-09-23-extreme-scale-acceptance.md +++ b/docs/reviews/2026-09-23-extreme-scale-acceptance.md @@ -8,7 +8,7 @@ Plan bundle: `docs/reviews/2026-09-23-extreme-scale-frontend-plan-bundle/`. One | 2 | Composition does not contain known quadratic accumulation patterns. | 완료 | `src/platform/cardinality/compose-unique-record.ts`; `tests/platform/cardinality/compose-unique-record.test.ts`; `tests/platform/cardinality/extreme-scale-registry.test.ts`; growth gate `scripts/scale/run-composition-scale.ts` + `src/platform/cardinality/growth-regression.ts` (`tests/scale/composition-scale.test.ts`, `tests/scale/growth-regression.test.ts`, artifact `artifacts/scale/main/composition-s0-s2.json`); stored-baseline gate `pnpm check:scale:baseline` (`config/scale/benchmark-baseline.json`) | | 3 | Membership checks on scale-sensitive paths use appropriate indexed structures. | 완료 | `src/contracts/product-features.ts` (`Set` membership); `tests/unit/product-features.test.ts`; `tests/features/product-features-scale.test.ts`; 100k selection/resolution in `tests/platform/cardinality/extreme-scale-registry.test.ts`; workloads `product-feature-selection`/`product-feature-resolution` in `scripts/scale/composition-workloads.ts` | | 4 | Concrete imports across unrelated features fail architecture checks. | 완료 | `.dependency-cruiser.json` rules `features-do-not-import-other-features`, `contexts-do-not-import-other-contexts`; rejected fixtures `tests/fixtures/architecture/dependency-graph/cross-boundary/`; assertions in `scripts/check-architecture.ts`; `pnpm check:architecture` | -| 5 | Context/app cardinality ceilings are explicit and testable. | 완료 | `src/platform/cardinality/cardinality-budget.ts`; `src/contexts/bounded-context-module.ts`; `src/apps/deployable-app-module.ts`; `tests/architecture/topology-cardinality.test.ts`; `tests/architecture/module-hierarchy.test.ts`; `tests/platform/cardinality/cardinality-budget.test.ts` | +| 5 | Context/app cardinality ceilings are explicit and testable. | 완료 | `src/platform/cardinality/cardinality-budget.ts`; `src/contexts/bounded-context-module.ts`; `src/apps/deployable-app-module.ts`; `tests/architecture/topology-cardinality.test.ts`; `tests/architecture/module-hierarchy.test.ts`; `tests/platform/cardinality/cardinality-budget.test.ts`. The ceilings are declared limits, not a measured capacity: see "Growth inside one app" below (11-04) | | 6 | An unchanged local app does not grow materially when global logical topology grows toward S3/S4 fixtures. | 완료 | (1) Ownership source: `src/apps/template-app/app-module.ts`, `src/contexts/reference-context/context-module.ts`, `FEATURE_CONTEXT_ID`. (2) Production projection: `scripts/generate-product-feature-index.ts` selects one app (`--app=`/`PRODUCT_APP_ID`) → `src/.generated/{product-feature-index,product-context-index,product-app}.ts` → `src/apps/template-app.ts`. (3) S0→S4 invariant: `tests/scale/global-local-independence.test.ts` — S0/S1/S2 real on-disk repos through the generator command (identical generated files, sha256 and reachable source graph); S0→S4 through the same `selectAppIndexes` (identical feature/context ids, route roots, generated files, hashes and compile-input paths; only App A's declarations read at 1M), pinned to the filesystem source by an equivalence test. (4) Build-input gate: `scripts/check-app-build-input.ts` (`pnpm check:app-build-input`, and `check:app-build-input:bundle` on the fresh production bundle), negative fixtures in `tests/architecture/app-build-input.test.ts` | | 7 | Initial routing is hierarchical/lazy rather than a product-wide flat feature table. | 완료 | `src/platform/routing/context-route-module.ts`; `src/presentation/routes/app-router.tsx` (`ContextRouteGateway`); `tests/platform/routing/context-route-module.test.ts`; `tests/architecture/context-lazy-loading.test.ts`; `tests/features/reference-feature/installed-context-routing.test.ts` | | 8 | User interactions carry bounded request, retry, concurrency, cancellation, and deadline semantics. | 완료 | `src/platform/execution/execution-context.ts`; `src/platform/traffic/{request-budget,concurrency-budget,traffic-budget}.ts`; `src/platform/resilience/{retry-budget,deadline}.ts`; enforcement in `src/adapters/http/http-execution-v3.ts`; operation policy `HttpExecutionPolicy.traffic` (`src/contracts/external-contract-runtime.ts`); `tests/platform/traffic/http-budget-integration.test.ts`; `tests/platform/traffic/traffic-amplification.test.ts`; `pnpm check:traffic` | @@ -24,6 +24,30 @@ Plan bundle: `docs/reviews/2026-09-23-extreme-scale-frontend-plan-bundle/`. One Overall: **완료** — every spec §54 criterion is `완료`. +## Growth inside one app (11-04) + +The rows above show that one app does not change when the *global* topology grows. What one app costs as its *own* feature count grows is measured separately, with copies of the real reference feature: `scripts/scale/measure-feature-shape-growth.ts` copies `src/features/reference-feature` N times with every feature-owned identifier renamed (feature id, routes, operations, schemas, query namespace, invalidation topic, messages), into contexts of 200 features each, and proves that the copies compose (the registries throw at module evaluation on any duplicate). The baseline is `config/scale/feature-shape-growth.json`, re-created with `node scripts/scale/measure-feature-shape-growth.ts --sizes=0,25,100 --repeats=2 --write` and checked by `tests/unit/feature-shape-growth.test.ts`. + +Measured 2026-09-27, two runs per size, Intel i5-10400F (12 logical CPUs), Node 24.14.0, TypeScript 7.0.2, Vite 8.1.5. Load average 25–30 during the runs (other agents were building in parallel), so wall times are upper bounds; counts and bytes are identical in both runs. + +| Added features | Routes | tsc instantiations | tsc memory | tsc wall (run 1 / 2) | vite build wall | Initial JS files / gzip | Lazy JS files / gzip | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 10 | 712,089 | 0.5 GB | 3.4 s / 3.9 s | 1.2 s / 1.5 s | 13 / 200,001 B | 14 / 24,938 B | +| 25 | 110 | 3,579,093 | 1.4 GB | 8.1 s / 8.7 s | 1.7 s / 2.1 s | 39 / 274,493 B | 165 / 171,638 B | +| 100 | 410 | 12,157,304 | 4.2 GB | 31.3 s / 43.2 s | 6.0 s / 3.7 s | 114 / 490,658 B | 615 / 704,938 B | + +What this means for the declared ceilings (`CONTEXT_MAX_FEATURES = 200`, `src/contexts/reference-context/context-module.ts:10`; `APP_MAX_CONTEXTS = 20`, `src/apps/template-app/app-module.ts:7`): + +| Growth axis | Verified | Condition | Not verified | +| --- | --- | --- | --- | +| Typecheck of one app | Linear, about 115k instantiations and 37 MB of tsc memory per feature, up to 100 features | Reference-shaped features, `tsconfig.app.json`, the host above | 200 features (one full context) and beyond; 4,000 (20 × 200) would need on the order of 150 GB by the same slope | +| Production build | Completes at 100 features in seconds | Same | Beyond 100 | +| Initial JavaScript | Grows by about 2.9 KB gzip and one file per feature: every installed feature's contracts and module load at boot | Same | — the initial budget (204,800 B, `config/performance/budgets.json`) (about 196.5 KB used by `check:bundle`) is exceeded after about three more reference-shaped features, far below the context ceiling | +| Lazy JavaScript | About 6.8 KB gzip and six files per feature, every file far below the 120 KiB lazy-chunk budget | Same | Download cost of many lazy files | +| Routes | 4 per feature, composed without conflict | Same | Router matching cost at runtime | + +So the ceilings bound the *count*; they are not a verified capacity. The practical limit of one app today is the initial JavaScript budget, then typecheck memory. 400 features were not measured (D-18 makes it optional; at the observed slope a run needs about 16 GB of tsc memory). + ## Executable evidence (supported runtime) Re-run on Node **v24.11.0** / pnpm 11.17.0 inside the `workmachine` container (whose default Node v22.23.2 is below `engines.node`): diff --git a/scripts/scale/measure-feature-shape-growth.ts b/scripts/scale/measure-feature-shape-growth.ts new file mode 100644 index 00000000..dba1ec0e --- /dev/null +++ b/scripts/scale/measure-feature-shape-growth.ts @@ -0,0 +1,257 @@ +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { availableParallelism, cpus, loadavg, tmpdir } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { gzipSync } from "node:zlib"; + +import { classifyViteJavascript } from "../lib/classify-vite-bundle.ts"; +import { linkFixtureNodeModules } from "../lib/fixture-node-modules.ts"; +import { parseTscExtendedDiagnostics, type TscDiagnostics } from "./collect-tsc-diagnostics.ts"; + +/** + * 11-04 (D-18). Cost of one app as its feature count grows, measured with + * features shaped like the real one: `src/features/reference-feature` copied N + * times with every feature-owned identifier renamed (feature id, route ids and + * paths, API operations, schemas, query namespace, invalidation topic, + * messages). The copies live in contexts of their own (200 features each, the + * context ceiling) so the reference context is unchanged. + * + * For each N the script runs the product generator, `tsc --extendedDiagnostics` + * over tsconfig.app.json and `vite build`, and records compiler counts, wall + * times and initial/lazy JavaScript gzip bytes. Deterministic counts repeat + * exactly; times depend on the host, so the CPU and load average are recorded + * next to them. A typecheck that fails (for example TS2590, a union too complex + * to represent) is recorded as a result with its first error. + * + * node scripts/scale/measure-feature-shape-growth.ts [--sizes=0,25,100] [--repeats=2] [--write] + * + * `--write` stores the result as config/scale/feature-shape-growth.json. + */ +export const FEATURE_SHAPE_BASELINE_PATH = "config/scale/feature-shape-growth.json"; + +const SOURCE_FEATURE = "src/features/reference-feature"; +const FEATURES_PER_CONTEXT = 200; +const BUILD_INPUTS = [ + "src", + "public", + "index.html", + "package.json", + "vite.config.ts", + "tsconfig.base.json", + "tsconfig.app.json", + "scripts/lib/vite-module-inventory.ts", +] as const; +// Platform-owned names that only happen to contain "reference". +const PLATFORM_TOKENS = [ + "REFERENCE_EXTERNAL_BEARER", + "REFERENCE_FEATURE_V1", + "supportReference", +] as const; + +/** Renames every feature-owned identifier of the reference feature for copy `id`. */ +export function renameReferenceFeature( + text: string, + id: string, + context: Readonly<{ id: string; routeRoot: string }>, +): string { + const upper = id.toUpperCase(); + let renamed = text; + PLATFORM_TOKENS.forEach((token, index) => { + renamed = renamed.replaceAll(token, `\u0000${index}\u0000`); + }); + renamed = renamed + .replaceAll("/examples/reference-resources", `/${context.routeRoot}/${id}-resources`) + .replaceAll("qinv.01k10f7m3w9p6r2c8v5n4x", `qinv.shape.${id}`) + .replaceAll("NoRequest", `${upper}NoRequest`) + .replaceAll("reference-context", context.id) + .replaceAll("REFERENCE", upper) + .replaceAll("Reference", upper) + .replaceAll("reference", id); + PLATFORM_TOKENS.forEach((token, index) => { + renamed = renamed.replaceAll(`\u0000${index}\u0000`, token); + }); + return renamed; +} + +async function copyRenamed(from: string, to: string, rename: (text: string) => string) { + await mkdir(to, { recursive: true }); + for (const entry of await readdir(from, { withFileTypes: true })) { + const source = path.join(from, entry.name); + const target = path.join(to, rename(entry.name)); + if (entry.isDirectory()) await copyRenamed(source, target, rename); + else await writeFile(target, rename(await readFile(source, "utf8"))); + } +} + +async function createFixture(features: number): Promise { + const root = await mkdtemp(path.join(tmpdir(), `feature-shape-${features}-`)); + for (const input of BUILD_INPUTS) { + await mkdir(path.dirname(path.join(root, input)), { recursive: true }); + await cp(path.resolve(input), path.join(root, input), { recursive: true }); + } + await rm(path.join(root, "src/.generated"), { recursive: true, force: true }); + await linkFixtureNodeModules(root); + for (let index = 0; index < features; index += 1) { + const contextNumber = Math.floor(index / FEATURES_PER_CONTEXT); + const context = { id: `shape-context-${contextNumber}`, routeRoot: `shape-${contextNumber}` }; + if (index % FEATURES_PER_CONTEXT === 0) { + const contextFile = path.join(root, "src/contexts", context.id, "context-module.ts"); + await mkdir(path.dirname(contextFile), { recursive: true }); + await writeFile( + contextFile, + `export const CONTEXT_MODULE_ID = "${context.id}";\n` + + `export const CONTEXT_APP_ID = "template-app";\n` + + `export const CONTEXT_ROUTE_PATH = "${context.routeRoot}/*";\n` + + `export const CONTEXT_MAX_FEATURES = ${FEATURES_PER_CONTEXT};\n`, + ); + } + const id = `s${String(index).padStart(3, "0")}`; + await copyRenamed( + path.resolve(SOURCE_FEATURE), + path.join(root, "src/features", `${id}-feature`), + (text) => renameReferenceFeature(text, id, context), + ); + } + return root; +} + +function run( + command: string, + args: readonly string[], + cwd: string, + options: Readonly<{ allowFailure?: boolean }> = {}, +) { + const started = performance.now(); + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + const ms = Math.round(performance.now() - started); + if (result.status !== 0 && !(options.allowFailure && result.stdout.includes("Instantiations:"))) { + throw new Error( + `${path.basename(command)} ${args.join(" ")} failed in ${cwd}:\n${(result.stdout + result.stderr).slice(0, 4_000)}`, + ); + } + return { ms, passed: result.status === 0, stdout: result.stdout }; +} + +const load = () => loadavg().map((value) => Math.round(value * 100) / 100); + +async function javascriptGzipBytes(root: string, files: readonly string[]) { + let bytes = 0; + for (const file of files) bytes += gzipSync(await readFile(path.join(root, "dist", file))).byteLength; + return bytes; +} + +async function measureOnce(root: string, features: number) { + const loadBefore = load(); + run(process.execPath, [path.resolve("scripts/generate-product-feature-index.ts"), `--root=${root}`], root); + // The composed registries throw at module evaluation on a duplicate route, + // operation, schema, namespace or message key: prove the copies compose. + const contracts = (await import( + `${pathToFileURL(path.join(root, "src/features/installed-feature-contracts.ts")).href}?features=${features}` + )) as typeof import("../../src/features/installed-feature-contracts.ts"); + const routes = Object.keys(contracts.ROUTE_REGISTRY).length; + const tsc = run( + path.resolve("node_modules/.bin/tsc"), + ["--project", path.join(root, "tsconfig.app.json"), "--noEmit", "--extendedDiagnostics"], + root, + // A type error is a result here (the typecheck stops scaling), not a crash. + { allowFailure: true }, + ); + await rm(path.join(root, "dist"), { recursive: true, force: true }); + const build = run(path.resolve("node_modules/.bin/vite"), ["build", "--logLevel", "error"], root); + const manifest = JSON.parse(await readFile(path.join(root, "dist/.vite/manifest.json"), "utf8")); + const javascript = classifyViteJavascript(manifest); + const diagnostics: TscDiagnostics = parseTscExtendedDiagnostics(tsc.stdout); + return { + addedFeatures: features, + routes, + tsc: { + passed: tsc.passed, + firstError: tsc.stdout.split("\n").find((line) => line.includes("error TS")) ?? null, + files: diagnostics.files, + types: diagnostics.types, + symbols: diagnostics.symbols, + instantiations: diagnostics.instantiations, + memoryKb: diagnostics.memoryKb ?? null, + wallMs: tsc.ms, + }, + build: { + wallMs: build.ms, + initialJavascriptFiles: javascript.initialFiles.length, + initialJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.initialFiles), + lazyJavascriptFiles: javascript.lazyFiles.length, + lazyJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.lazyFiles), + }, + loadBefore, + loadAfter: load(), + }; +} + +function version(bin: string): string { + const result = spawnSync(path.resolve("node_modules/.bin", bin), ["--version"], { encoding: "utf8" }); + return (result.stdout ?? "").trim(); +} + +async function main(): Promise { + const argument = (name: string) => + process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3); + const sizes = (argument("sizes") ?? "0,25,100").split(",").map(Number); + const repeats = Number(argument("repeats") ?? "2"); + if (!sizes.every((size) => Number.isSafeInteger(size) && size >= 0) || !(repeats >= 1)) { + throw new TypeError("--sizes must be nonnegative integers and --repeats at least 1"); + } + const runs = []; + for (const features of sizes) { + const root = await createFixture(features); + try { + for (let repeat = 1; repeat <= repeats; repeat += 1) { + const measured = await measureOnce(root, features); + runs.push({ repeat, ...measured }); + process.stdout.write( + `features +${features} (repeat ${repeat}): routes=${measured.routes} ` + + `instantiations=${measured.tsc.instantiations} tsc=${measured.tsc.wallMs}ms ` + + `build=${measured.build.wallMs}ms initial=${measured.build.initialJavascriptGzipBytes}B ` + + `lazy=${measured.build.lazyJavascriptGzipBytes}B load=${measured.loadBefore.join("/")}` + + `${measured.tsc.passed ? "" : ` typecheck FAILED: ${measured.tsc.firstError}`}\n`, + ); + // Compiler results repeat exactly; a failed typecheck costs minutes and + // gigabytes again for no new information. + if (!measured.tsc.passed) break; + } + } finally { + await rm(root, { recursive: true, force: true }); + } + } + const cpu = cpus(); + const result = { + schemaVersion: 1, + measuredAt: new Date().toISOString(), + command: `node scripts/scale/measure-feature-shape-growth.ts --sizes=${sizes.join(",")} --repeats=${repeats}`, + environment: { + node: process.versions.node, + platform: process.platform, + arch: process.arch, + cpuModel: cpu[0]?.model.trim() ?? "unknown", + logicalCpus: cpu.length, + availableParallelism: availableParallelism(), + typescript: version("tsc"), + vite: version("vite"), + }, + sourceFeature: SOURCE_FEATURE, + runs, + }; + const output = process.argv.includes("--write") + ? FEATURE_SHAPE_BASELINE_PATH + : "artifacts/scale/feature-shape-growth.json"; + await mkdir(path.dirname(output), { recursive: true }); + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); + process.stdout.write(`feature-shape growth: ${output}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await main(); +} diff --git a/tests/unit/feature-shape-growth.test.ts b/tests/unit/feature-shape-growth.test.ts new file mode 100644 index 00000000..fffc0b52 --- /dev/null +++ b/tests/unit/feature-shape-growth.test.ts @@ -0,0 +1,103 @@ +import { readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; + +import { + FEATURE_SHAPE_BASELINE_PATH, + renameReferenceFeature, +} from "../../scripts/scale/measure-feature-shape-growth.ts"; + +/** + * 11-04 (D-18). The scale evidence used to come from stub features that only + * differ by name, so the cost of one app growing towards its declared ceiling + * (20 contexts x 200 features) was never measured. The committed baseline is a + * measurement with copies of the real reference feature. + */ +const context = { id: "shape-context-0", routeRoot: "shape-0" } as const; + +type Run = Readonly<{ + addedFeatures: number; + routes: number; + tsc: Readonly<{ passed: boolean; instantiations: number; wallMs: number }>; + build: Readonly<{ + wallMs: number; + initialJavascriptGzipBytes: number; + lazyJavascriptFiles: number; + }>; + loadBefore: readonly number[]; +}>; + +describe("feature-shape growth measurement (11-04)", () => { + it("renames every feature-owned identifier of a reference copy", () => { + const source = [ + 'featureId: "reference-feature"', + 'path: "/examples/reference-resources/:id"', + "REFERENCE_RESOURCE_LIST", + "ReferenceResourcePage", + 'contextId: "reference-context"', + 'topic: "qinv.01k10f7m3w9p6r2c8v5n4x"', + "NoRequest", + ].join("\n"); + + const renamed = renameReferenceFeature(source, "s007", context); + + expect(renamed).not.toMatch(/reference/iu); + expect(renamed).toContain('featureId: "s007-feature"'); + expect(renamed).toContain('path: "/shape-0/s007-resources/:id"'); + expect(renamed).toContain("S007_RESOURCE_LIST"); + expect(renamed).toContain("S007ResourcePage"); + expect(renamed).toContain('contextId: "shape-context-0"'); + expect(renamed).toContain('topic: "qinv.shape.s007"'); + expect(renamed).toContain("S007NoRequest"); + }); + + it("keeps platform-owned names that only contain the word", () => { + const source = + "REFERENCE_EXTERNAL_BEARER REFERENCE_FEATURE_V1 supportReference"; + + expect(renameReferenceFeature(source, "s001", context)).toBe(source); + }); + + it("commits a baseline measured at 25 and 100 real-shape features, twice each", async () => { + const baseline = JSON.parse( + await readFile(FEATURE_SHAPE_BASELINE_PATH, "utf8"), + ) as Readonly<{ + environment: Readonly>; + runs: readonly Run[]; + }>; + expect(baseline.environment).toMatchObject({ + cpuModel: expect.any(String), + logicalCpus: expect.any(Number), + }); + + const bySize = (size: number) => + baseline.runs.filter((run) => run.addedFeatures === size); + const [origin] = bySize(0); + expect(origin?.tsc.passed).toBe(true); + for (const size of [25, 100]) { + const runs = bySize(size); + // A failed typecheck is the result and is not repeated. + expect(runs.length).toBeGreaterThanOrEqual(runs[0]?.tsc.passed ? 2 : 1); + for (const run of runs) { + // Every copy contributes the reference feature's own routes. + expect((run.routes - origin!.routes) % size).toBe(0); + expect(run.routes).toBeGreaterThan(origin!.routes); + expect(run.build.lazyJavascriptFiles).toBeGreaterThan( + origin!.build.lazyJavascriptFiles, + ); + expect(run.loadBefore).toHaveLength(3); + } + // Compiler counts are deterministic; only times vary with the host. + expect(new Set(runs.map((run) => run.tsc.instantiations)).size).toBe(1); + } + }); + + it("states in the acceptance matrix what the ceilings rest on", async () => { + const acceptance = await readFile( + "docs/reviews/2026-09-23-extreme-scale-acceptance.md", + "utf8", + ); + expect(acceptance).toContain(FEATURE_SHAPE_BASELINE_PATH); + expect(acceptance).toMatch(/\| Growth axis \| Verified \| Condition \| Not verified \|/u); + }); +}); From 34c97885ea0aebe124384c43ccbe5f46a6fee48d Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:10:13 +0900 Subject: [PATCH 09/15] test(e2e): recover an old tab's missing chunk without a service worker (11-05) Old-tab chunk recovery in the default configuration only had jsdom tests, and the old-asset test checked its own hosting fixture. A spec now serves two real builds: the old tab reloads once into the new one and not a second time. The fixture test and the acceptance row now say what they cover. Co-Authored-By: Claude Opus 5.5 --- .../2026-09-23-extreme-scale-acceptance.md | 2 +- tests/e2e/old-tab-chunk-recovery.spec.ts | 125 ++++++++++++++++++ tests/e2e/support/fixture-builds.ts | 97 ++++++++++++++ tests/release/old-asset-reachability.test.ts | 10 +- 4 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/old-tab-chunk-recovery.spec.ts create mode 100644 tests/e2e/support/fixture-builds.ts diff --git a/docs/reviews/2026-09-23-extreme-scale-acceptance.md b/docs/reviews/2026-09-23-extreme-scale-acceptance.md index b12c2e43..264c9935 100644 --- a/docs/reviews/2026-09-23-extreme-scale-acceptance.md +++ b/docs/reviews/2026-09-23-extreme-scale-acceptance.md @@ -20,7 +20,7 @@ Plan bundle: `docs/reviews/2026-09-23-extreme-scale-frontend-plan-bundle/`. One | 14 | Telemetry failure does not fail business behavior. | 완료 | `tests/platform/telemetry/telemetry-pipeline.test.ts` (transport throw/reject); `tests/platform/telemetry/telemetry-port.test.ts` (`failOpenTelemetryPort`); `tests/features/reference-feature/reference-telemetry.test.ts`; collector outage in `tests/scale/telemetry/telemetry-storm.test.ts` | | 15 | Failure boundaries prevent one feature/context from unnecessarily collapsing unrelated UI. | 완료 | `src/platform/errors/{scoped-error-boundary,feature-error-boundary,context-error-boundary}.tsx`; wiring in `src/presentation/routes/app-router.tsx`; `tests/platform/errors/failure-boundary.test.tsx` | | 16 | Spike and sustained-load tests detect request, retry, telemetry, and memory amplification. | 완료 | `tests/scale/traffic/*`, `tests/scale/telemetry/telemetry-storm.test.ts`, `tests/scale/session/*`; real Chromium soak `scripts/scale/browser-soak.ts` against the fresh build with telemetry on and a 503 API: 2,012 cycles, heap/listeners/timers/storage stable, 0 in-flight at end, 1.43 API requests per reference visit (≤ 3), telemetry ≤ 14 posts/min (≤ 40), checks in `artifacts/scale/nightly/browser-soak.json`; evaluator tests `tests/scale/browser-soak.contract.test.ts`; skipped ⇒ tier `incomplete` (non-zero). Platform telemetry adapter now send-rate limited (`src/adapters/telemetry/best-effort-telemetry.ts`, tests in `tests/unit/telemetry.test.ts`) | -| 17 | Multi-replica release tests verify fail-closed coherence and old-asset reachability. | 완료 | `src/platform/release/release-coherence.ts` (delegated to by `src/bootstrap/load-runtime-config.ts`, `src/bootstrap/load-release-manifest.ts`); `src/platform/release/release-mismatch-recovery.ts` + `src/bootstrap/boot-release-recovery.ts`; `tests/platform/release/*.test.ts`; `tests/release/{old-asset-reachability,mixed-replica,release-matrix}.test.ts`; `pnpm check:release-coherence` | +| 17 | Multi-replica release tests verify fail-closed coherence and old-asset reachability. | 완료 | `src/platform/release/release-coherence.ts` (delegated to by `src/bootstrap/load-runtime-config.ts`, `src/bootstrap/load-release-manifest.ts`); `src/platform/release/release-mismatch-recovery.ts` + `src/bootstrap/boot-release-recovery.ts`; `tests/platform/release/*.test.ts`; `tests/release/{mixed-replica,release-matrix}.test.ts`; `pnpm check:release-coherence`. Old-asset reachability is a hosting provider duty (`docs/operations/release-cache-rollback.md`); `tests/release/old-asset-reachability.test.ts` checks only the in-memory hosting fixture against `config/hosting/cache-policy.json`. An old tab of the default build (no Service Worker) whose chunk is gone after a deploy is checked in real browsers by `tests/e2e/old-tab-chunk-recovery.spec.ts`: one reload into the new build, no second reload (11-05) | Overall: **완료** — every spec §54 criterion is `완료`. diff --git a/tests/e2e/old-tab-chunk-recovery.spec.ts b/tests/e2e/old-tab-chunk-recovery.spec.ts new file mode 100644 index 00000000..73001f62 --- /dev/null +++ b/tests/e2e/old-tab-chunk-recovery.spec.ts @@ -0,0 +1,125 @@ +import { appendFile, readFile } from "node:fs/promises"; +import path from "node:path"; + +import { ROUTE_REGISTRY } from "../../src/features/installed-feature-contracts.ts"; +import { expect, test } from "../support/browser/strict-browser-test.ts"; +import { + prepareFixtureBuilds, + removeFixtureBuilds, + type FixtureBuilds, +} from "./support/fixture-builds.ts"; +import { startBuildServer } from "./support/service-worker-builds.ts"; + +/** + * 11-05. A tab opened on build A, in the default configuration (no Service + * Worker), after build B is deployed. B changes two lazy route pages, so their + * A chunks no longer exist on the server. The first missing chunk reloads the + * tab once, into B. A second missing chunk (the server alternates back to A, + * as mismatched replicas do) must not reload again: the tab shows the + * recovery failure instead. + */ + +const PLATFORM = ROUTE_REGISTRY.EXAMPLES_PLATFORM; +const UI = ROUTE_REGISTRY.EXAMPLES_UI; +const PAGES = [ + "src/presentation/examples/platform-overview-page.tsx", + "src/presentation/examples/ui-gallery-page.tsx", +] as const; + +let builds: FixtureBuilds<"a" | "b"> | undefined; + +test.beforeAll(async () => { + test.setTimeout(300_000); + builds = await prepareFixtureBuilds("chunk-recovery", [ + { name: "a" }, + { + name: "b", + edit: async (root) => { + for (const page of PAGES) { + await appendFile( + path.join(root, page), + '\n(globalThis as Record).__caFixtureBuild = "b";\n', + ); + } + }, + }, + ]); +}); + +test.afterAll(async () => { + await removeFixtureBuilds(builds); +}); + +async function routeChunk(dist: string, chunkId: string): Promise { + const manifest = JSON.parse( + await readFile(path.join(dist, "release-manifest.json"), "utf8"), + ) as { routeChunks: Record }; + return `/${manifest.routeChunks[chunkId]}`; +} + +// Each missing page chunk fails its request and the dynamic import. Chromium +// may also log the 404 itself (without its URL), and Firefox logs the error +// object React caught. Nothing else is allowed. +const MISSING_PAGE = String.raw`/assets/(?:platform-overview|ui-gallery)-page-[\w-]+\.js`; +test.use({ + allowedBrowserFailure: new RegExp( + [ + String.raw`^requestfailed:GET:${MISSING_PAGE}:`, + String.raw`^console:error:TypeError: Failed to fetch dynamically imported module: http://127\.0\.0\.1:\d+${MISSING_PAGE}$`, + String.raw`^console:error:\[JavaScript Error: "Loading module from “http://127\.0\.0\.1:\d+${MISSING_PAGE}” was blocked`, + String.raw`^console:error:Failed to load resource: the server responded with a status of 404 \(Not Found\)$`, + String.raw`^console:error:JSHandle@object$`, + ].join("|"), + "u", + ), +}); + +test("an old tab reloads once into the new build, then stops reloading", async ({ + page, +}) => { + if (!builds) throw new Error("Fixture builds were not prepared."); + const aPlatform = await routeChunk(builds.dist.a, PLATFORM.chunkId); + const bPlatform = await routeChunk(builds.dist.b, PLATFORM.chunkId); + const bUi = await routeChunk(builds.dist.b, UI.chunkId); + expect(bPlatform).not.toBe(aPlatform); + expect(bUi).not.toBe(await routeChunk(builds.dist.a, UI.chunkId)); + + const server = await startBuildServer(builds.dist); + try { + await page.goto(`${server.origin}/`); + await expect( + page.getByRole("link", { name: PLATFORM.navigationLabel!, exact: true }), + ).toBeVisible(); + let loads = 0; + page.on("load", () => { + loads += 1; + }); + + // Build B is deployed while the A tab stays open. + server.serve("b"); + await page + .getByRole("link", { name: PLATFORM.navigationLabel!, exact: true }) + .click(); + + await expect(page.getByRole("heading", { name: PLATFORM.title })).toBeVisible(); + expect(loads).toBe(1); + const served = server.served.map((entry) => entry.path); + expect(served).not.toContain(aPlatform); + expect(served).toContain(bPlatform); + await expect + .poll(() => page.evaluate(() => (globalThis as Record).__caFixtureBuild)) + .toBe("b"); + + // The replicas alternate: the B tab now meets A, which lacks B's chunk. + server.serve("a"); + await page.getByRole("link", { name: UI.navigationLabel!, exact: true }).click(); + + await expect(page.getByRole("alert")).toContainText( + "화면 자산을 복구하지 못했습니다.", + ); + expect(loads).toBe(1); + expect(page.url()).toBe(`${server.origin}${UI.path}`); + } finally { + await server.close(); + } +}); diff --git a/tests/e2e/support/fixture-builds.ts b/tests/e2e/support/fixture-builds.ts new file mode 100644 index 00000000..a29238a6 --- /dev/null +++ b/tests/e2e/support/fixture-builds.ts @@ -0,0 +1,97 @@ +import { spawnSync } from "node:child_process"; +import { cp, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { linkFixtureNodeModules } from "../../../scripts/lib/fixture-node-modules.ts"; + +/** + * Real production builds of an edited temp copy of the repository, in the + * template's default configuration (`serviceWorker: null`). `edit` changes the + * copy right before that build runs, and edits persist into later builds. The + * working tree and its `dist` (served by the e2e preview server) are never + * touched. Serve the result with `startBuildServer` from + * `service-worker-builds.ts`. + */ + +const repoRoot = path.resolve(import.meta.dirname, "../../.."); +const skipped = new Set([ + "node_modules", + ".git", + ".claude", + ".generated", + ".tmp", + "dist", + "artifacts", + "graphify-out", +]); + +export type FixtureBuild = Readonly<{ + name: Name; + edit?: (root: string) => Promise; +}>; + +export type FixtureBuilds = Readonly<{ + root: string; + dist: Readonly>; +}>; + +/** Replaces the one occurrence of `search` in `file` of the copy. */ +export async function editOnce( + file: string, + search: string, + replacement: string, +): Promise { + const source = await readFile(file, "utf8"); + if (source.split(search).length !== 2) { + throw new Error(`Expected exactly one "${search}" in ${file}.`); + } + await writeFile(file, source.replace(search, replacement)); +} + +function run(cwd: string, script: string, buildId: string): void { + const result = spawnSync("node", [script], { + cwd, + encoding: "utf8", + env: { ...process.env, VITE_BUILD_ID: buildId, RELEASE_ID: buildId }, + }); + if (result.status !== 0) { + throw new Error( + `${script} failed for ${buildId}:\n${result.stdout}\n${result.stderr}`, + ); + } +} + +export async function prepareFixtureBuilds( + prefix: string, + builds: readonly FixtureBuild[], +): Promise> { + const root = await mkdtemp(path.join(tmpdir(), `${prefix}-`)); + try { + await cp(repoRoot, root, { + recursive: true, + filter: (source) => + !skipped.has(path.relative(repoRoot, source).split(path.sep)[0]!), + }); + await linkFixtureNodeModules(root, repoRoot); + const dist = {} as Record; + for (const build of builds) { + await build.edit?.(root); + const buildId = `${prefix}-${build.name}`; + run(root, "scripts/generate-product-feature-index.ts", buildId); + run(root, "scripts/build-frontend.ts", buildId); + dist[build.name] = path.join(root, `dist-${build.name}`); + await rename(path.join(root, "dist"), dist[build.name]); + } + return { root, dist }; + } catch (error) { + await rm(root, { recursive: true, force: true }); + throw error; + } +} + +export async function removeFixtureBuilds( + builds: Readonly<{ root: string }> | undefined, +): Promise { + if (builds) await rm(builds.root, { recursive: true, force: true }); +} diff --git a/tests/release/old-asset-reachability.test.ts b/tests/release/old-asset-reachability.test.ts index 2f6dcc62..a8852549 100644 --- a/tests/release/old-asset-reachability.test.ts +++ b/tests/release/old-asset-reachability.test.ts @@ -2,7 +2,15 @@ import { describe, expect, it } from "vitest"; import { CACHE_POLICY, createReleaseFixture } from "./release-fixture.ts"; -describe("old hashed-asset reachability", () => { +/** + * 11-05. A contract of the in-memory hosting model in `release-fixture.ts` + * against `config/hosting/cache-policy.json`: no product code, deploy script or + * browser runs here. Keeping old assets reachable is the hosting provider's + * job (`docs/operations/release-cache-rollback.md`). What an old tab does when + * an asset is gone anyway is checked in a real browser by + * `tests/e2e/old-tab-chunk-recovery.spec.ts`. + */ +describe("old hashed-asset reachability (hosting fixture contract)", () => { it("keeps R10 hashed assets reachable after R11 becomes active", async () => { const deployment = createReleaseFixture(["R10", "R11"]); deployment.activate("R11"); From 8e5c51e81354880a3f7e5b701a7ef74c1fb362c8 Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:14:03 +0900 Subject: [PATCH 10/15] perf(lab): repeat lab runs, judge the median and pin the measured server (11-07) One run decided the release gate, the interaction included Playwright round trips, and a server already on 4173 was measured instead of the build. Runs now repeat with every sample kept, time the interaction in the page, use --strictPort and check the served build, and stop the whole preview group. Co-Authored-By: Claude Opus 5.5 --- docs/operations/performance-evidence.md | 26 ++- scripts/lib/lab-performance.ts | 89 ++++++++++ scripts/test-performance.ts | 206 +++++++++++++++++++----- tests/unit/lab-performance.test.ts | 106 ++++++++++++ 4 files changed, 380 insertions(+), 47 deletions(-) create mode 100644 scripts/lib/lab-performance.ts create mode 100644 tests/unit/lab-performance.test.ts diff --git a/docs/operations/performance-evidence.md b/docs/operations/performance-evidence.md index 176891f6..9faa1659 100644 --- a/docs/operations/performance-evidence.md +++ b/docs/operations/performance-evidence.md @@ -41,10 +41,28 @@ Performance evidence is deliberately split by measurement context: 예산을 올릴 때는 이 표의 기준선과 새 측정값, 이유를 같은 변경에 적는다. lab 측정(`scripts/test-performance.ts`, FE-GATE-026)은 local 프로필 build를 -`vite preview`로 띄워 한 번만 잰다. named interaction 시간은 Node의 -`performance.now()`로 Playwright 클릭부터 heading 대기까지 재므로 Playwright -왕복 시간이 들어간다. 한 번 측정한 값은 흔들린다. 경계 근처 값으로 판단할 -때는 여러 번 돌려 중앙값을 본다. +`vite preview --strictPort`로 띄워 잰다(리뷰 11-07). + +- 포트는 `LAB_PERFORMANCE_PORT`(기본 4173)다. 서버를 띄우기 전에 그 포트가 + 이미 응답하면 실패하고, 응답하는 `/release-manifest.json`의 build·release가 + 로컬 `dist/release-manifest.json`과 다르면 실패한다 + (`scripts/lib/lab-performance.ts`, `tests/unit/lab-performance.test.ts`). +- 새 브라우저 context에서 `--runs=N`(기본 5)번 재고, 예산은 중앙값으로 + 판정한다. 모든 측정값과 최소·최대는 `lab.json`의 `context.sampling`에 남는다. +- named interaction은 페이지 안의 시계로 잰다. 클릭 이벤트의 `timeStamp`부터 + 대상 heading이 DOM에 들어온 시각까지다. Playwright 왕복 시간은 들어가지 않고, + lazy chunk 다운로드(200 KB/s 제한)는 들어간다. 클릭의 Event Timing entry + 길이(INP의 입력값)는 `context.sampling.clickEventDurationMs`에 따로 남는다. +- CPU 모델, 논리 CPU 수, 측정 전후 loadavg를 `context.runner`에 남긴다. +- preview 서버는 자기 프로세스 그룹으로 띄우고 끝날 때 그룹 전체를 멈춘다. + `corepack`에만 신호를 보내던 때는 vite가 포트를 계속 잡고 있었다. + +2026-09-27 한 빌드를 10회 잰 결과(i5-10400F 12 CPU, loadavg 11~17): 중앙값 +LCP 2,386ms, interaction 184.5ms로 통과했다. 한 번씩 판정했다면 interaction +10회 중 4회(최대 274ms), LCP 10회 중 2회(최대 2,644ms)가 실패였다. 같은 +빌드를 loadavg 7~9에서 다시 10회 재면 interaction 120~148ms, LCP +1,968~2,072ms로 모두 통과했다. 한 번 측정한 판정은 호스트 부하에 따라 +뒤집히고, 중앙값 판정은 두 번 모두 통과였다. The field minimum eligible-sample threshold is intentionally unresolved until a privacy-approved telemetry baseline exists. Therefore the field command diff --git a/scripts/lib/lab-performance.ts b/scripts/lib/lab-performance.ts new file mode 100644 index 00000000..0c01699d --- /dev/null +++ b/scripts/lib/lab-performance.ts @@ -0,0 +1,89 @@ +/** + * 11-07. The pieces of the lab performance gate (`scripts/test-performance.ts`) + * that decide what is measured and whether the numbers can be trusted. + */ + +export const DEFAULT_LAB_PORT = 4173; +export const DEFAULT_LAB_RUNS = 5; + +/** + * `vite preview` arguments. Without `--strictPort` vite moved to the next free + * port when the requested one was taken, and the gate measured whatever server + * already listened there. + */ +export function previewArguments(port: number): readonly string[] { + return [ + "pnpm", + "preview", + "--host", + "127.0.0.1", + "--port", + String(port), + "--strictPort", + ]; +} + +export type SampleSummary = Readonly<{ + median: number; + min: number; + max: number; + samples: readonly number[]; +}>; + +export function summarizeSamples(samples: readonly number[]): SampleSummary { + if (samples.length === 0) throw new RangeError("no samples to summarize"); + const sorted = [...samples].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + const median = + sorted.length % 2 === 1 + ? sorted[middle]! + : (sorted[middle - 1]! + sorted[middle]!) / 2; + return Object.freeze({ + median, + min: sorted[0]!, + max: sorted.at(-1)!, + samples: Object.freeze([...samples]), + }); +} + +/** Fails when anything already answers where the preview server will listen. */ +export async function assertPortUnused( + baseUrl: string, + fetchImpl: typeof fetch = fetch, +): Promise { + let answered = false; + try { + await fetchImpl(baseUrl, { signal: AbortSignal.timeout(1_000) }); + answered = true; + } catch { + // Nothing is listening: the expected state. + } + if (answered) { + throw new Error( + `${baseUrl} already answers before the preview server starts; another server holds the port.`, + ); + } +} + +/** + * Compares the release the server actually serves with the local build that + * is being measured, so the numbers belong to that build. + */ +export async function assertServedRelease( + baseUrl: string, + expected: Readonly<{ buildId: string; releaseId: string }>, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await fetchImpl(new URL("/release-manifest.json", baseUrl), { + cache: "no-store", + }); + const served = response.ok + ? ((await response.json()) as Partial>) + : {}; + if (served.buildId !== expected.buildId || served.releaseId !== expected.releaseId) { + throw new Error( + `${baseUrl} serves build ${String(served.buildId)}/${String(served.releaseId)}, ` + + `not the measured ${expected.buildId}/${expected.releaseId}.`, + ); + } +} diff --git a/scripts/test-performance.ts b/scripts/test-performance.ts index 00547c8a..4c82ebdb 100644 --- a/scripts/test-performance.ts +++ b/scripts/test-performance.ts @@ -1,10 +1,18 @@ import { spawn } from "node:child_process"; import { mkdir, readFile } from "node:fs/promises"; -import { performance } from "node:perf_hooks"; +import { cpus, loadavg } from "node:os"; import process from "node:process"; -import { chromium } from "@playwright/test"; +import { chromium, type Browser } from "@playwright/test"; +import { + assertPortUnused, + assertServedRelease, + DEFAULT_LAB_PORT, + DEFAULT_LAB_RUNS, + previewArguments, + summarizeSamples, +} from "./lib/lab-performance.ts"; import { evaluateLabBudget } from "./lib/performance-budgets.ts"; import { ROUTE_REGISTRY } from "../src/features/installed-feature-contracts.ts"; import { labPerformanceArtifactSchema } from "./contracts/release-artifacts.ts"; @@ -13,6 +21,9 @@ import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; type ContractPerformanceEvidence = { lcpMs: number; cls: number; + clickAt: number | null; + headingAt: number | null; + eventDurationMs: number | null; }; type ContractPerformanceWindow = Window & { @@ -24,15 +35,37 @@ type LayoutShiftEntry = PerformanceEntry & { value: number; }; -const server = spawn( - "corepack", - ["pnpm", "preview", "--host", "127.0.0.1", "--port", "4173"], - { stdio: "ignore" }, -); -const baseUrl = "http://127.0.0.1:4173"; +const argument = (name: string) => + process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3); +const port = Number(process.env.LAB_PERFORMANCE_PORT ?? DEFAULT_LAB_PORT); +const runs = Number(argument("runs") ?? DEFAULT_LAB_RUNS); +if (!Number.isSafeInteger(port) || port <= 0 || !Number.isSafeInteger(runs) || runs < 1) { + throw new TypeError("LAB_PERFORMANCE_PORT must be a port and --runs a positive integer."); +} +const baseUrl = `http://127.0.0.1:${port}`; +const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM; +const targetLabel = target.navigationLabel; +if (!targetLabel) { + throw new Error("Performance route must be present in navigation."); +} + +// 11-07. A server already on the port used to be measured in place of this +// build: vite moved to another port and the gate never noticed. +await assertPortUnused(baseUrl); +// Its own process group: `corepack` starts pnpm, which starts vite, and a +// signal to `corepack` alone left vite serving the port after the gate ended. +const server = spawn("corepack", [...previewArguments(port)], { + stdio: "ignore", + detached: true, +}); +let serverExit: string | null = null; +server.on("exit", (code, signal) => { + serverExit = `preview server exited (${signal ?? code})`; +}); async function waitForServer() { - for (let attempt = 0; attempt < 50; attempt += 1) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (serverExit) throw new Error(serverExit); try { const response = await fetch(baseUrl); if (response.ok) return; @@ -44,19 +77,14 @@ async function waitForServer() { throw new Error("Preview server did not become ready."); } -try { - await waitForServer(); - const release = JSON.parse( - await readFile("dist/release-manifest.json", "utf8"), - ); - const thresholds = JSON.parse( - await readFile("config/performance/budgets.json", "utf8"), - ).lab; - const browser = await chromium.launch(); +type RunMetrics = { lcpMs: number; cls: number; namedInteractionMs: number; eventDurationMs: number | null }; + +async function measureOnce(browser: Browser): Promise { + // Every run is a cold, isolated context. + const context = await browser.newContext({ + viewport: { width: 1280, height: 720 }, + }); try { - const context = await browser.newContext({ - viewport: { width: 1280, height: 720 }, - }); const page = await context.newPage(); const cdp = await context.newCDPSession(page); await cdp.send("Network.enable"); @@ -68,8 +96,14 @@ try { connectionType: "cellular4g", }); await cdp.send("Emulation.setCPUThrottlingRate", { rate: 4 }); - await page.addInitScript(() => { - const evidence = { lcpMs: 0, cls: 0 }; + await page.addInitScript((title: string) => { + const evidence: ContractPerformanceEvidence = { + lcpMs: 0, + cls: 0, + clickAt: null, + headingAt: null, + eventDurationMs: null, + }; (window as ContractPerformanceWindow).__contractPerformance = evidence; new PerformanceObserver((list) => { for (const entry of list.getEntries()) evidence.lcpMs = entry.startTime; @@ -82,34 +116,97 @@ try { } } }).observe({ type: "layout-shift", buffered: true }); - }); + // 11-07. The interaction is timed by the page's own clock, from the + // click event to the target heading entering the DOM, so Playwright's + // round trips are not in it. The Event Timing entry of the click (the + // INP input) is recorded next to it. + new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + if (entry.name === "click") { + evidence.eventDurationMs = Math.max(evidence.eventDurationMs ?? 0, entry.duration); + } + } + }).observe({ type: "event", buffered: true, durationThreshold: 16 } as PerformanceObserverInit); + document.addEventListener( + "click", + (event) => { + evidence.clickAt ??= event.timeStamp; + }, + { capture: true }, + ); + new MutationObserver((_records, observer) => { + if (evidence.clickAt === null) return; + const found = [...document.querySelectorAll("h1, h2")].some( + (heading) => heading.textContent?.trim() === title, + ); + if (found) { + evidence.headingAt = performance.now(); + observer.disconnect(); + } + }).observe(document, { childList: true, subtree: true, characterData: true }); + }, target.title); await page.goto(baseUrl, { waitUntil: "networkidle" }); - const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM; - const targetLabel = target.navigationLabel; - if (!targetLabel) { - throw new Error("Performance route must be present in navigation."); - } - const interactionStarted = performance.now(); // Playwright matches accessible names by substring, so the navigation entry // "플랫폼 구성" also matched the home page's "플랫폼 구성 보기" call to - // action and the locator resolved to two links. That is a strict-mode - // violation before the first measurement is taken, so no lab performance - // evidence could be produced at all — the run failed for an ambiguous - // selector rather than for anything about performance. + // action; `exact` keeps the locator on the navigation link. await page.getByRole("link", { name: targetLabel, exact: true }).click(); await page.getByRole("heading", { name: target.title }).waitFor(); - const namedInteractionMs = performance.now() - interactionStarted; + await page.waitForFunction( + () => (window as ContractPerformanceWindow).__contractPerformance?.headingAt !== null, + ); const paint = await page.evaluate( () => (window as ContractPerformanceWindow).__contractPerformance, ); - if (!paint) { + if (!paint || paint.clickAt === null || paint.headingAt === null) { throw new Error("Browser performance evidence was not initialized."); } + return { + lcpMs: Math.round(paint.lcpMs), + cls: Number(paint.cls.toFixed(4)), + namedInteractionMs: Math.round(paint.headingAt - paint.clickAt), + eventDurationMs: + paint.eventDurationMs === null ? null : Math.round(paint.eventDurationMs), + }; + } finally { + await context.close(); + } +} + +const load = () => loadavg().map((value) => Math.round(value * 100) / 100); + +try { + await waitForServer(); + const release = JSON.parse( + await readFile("dist/release-manifest.json", "utf8"), + ); + await assertServedRelease(baseUrl, release); + const thresholds = JSON.parse( + await readFile("config/performance/budgets.json", "utf8"), + ).lab; + const browser = await chromium.launch(); + try { + const loadBefore = load(); + const samples: RunMetrics[] = []; + for (let run = 0; run < runs; run += 1) samples.push(await measureOnce(browser)); + const loadAfter = load(); + // The served build must not have changed under the runs either. + await assertServedRelease(baseUrl, release); + const lcp = summarizeSamples(samples.map((sample) => sample.lcpMs)); + const cls = summarizeSamples(samples.map((sample) => sample.cls)); + const interaction = summarizeSamples( + samples.map((sample) => sample.namedInteractionMs), + ); + const eventDurations = samples.flatMap((sample) => + sample.eventDurationMs === null ? [] : [sample.eventDurationMs], + ); const contextMetadata = { runner: { platform: process.platform, architecture: process.arch, nodeVersion: process.version, + cpuModel: cpus()[0]?.model.trim() ?? "unknown", + logicalCpus: cpus().length, + loadAverage: { before: loadBefore, after: loadAfter }, }, browser: { name: "chromium", version: await browser.version() }, viewport: { width: 1280, height: 720 }, @@ -121,12 +218,29 @@ try { }, cpu: { throttlingRate: 4 }, cache: { state: "cold", isolation: "new-browser-context" }, - build: { buildId: release.buildId, releaseId: release.releaseId }, + build: { + buildId: release.buildId, + releaseId: release.releaseId, + served: `${baseUrl}/release-manifest.json`, + }, + // 11-07. The budget is judged on the median of cold runs; every run and + // the spread stay in the evidence. + sampling: { + runs, + statistic: "median", + interactionClock: "page: click event to target heading in DOM", + lcpMs: lcp, + cls, + namedInteractionMs: interaction, + clickEventDurationMs: eventDurations.length + ? summarizeSamples(eventDurations) + : null, + }, }; const metrics = { - lcpMs: Math.round(paint.lcpMs), - cls: Number(paint.cls.toFixed(4)), - namedInteractionMs: Math.round(namedInteractionMs), + lcpMs: lcp.median, + cls: cls.median, + namedInteractionMs: interaction.median, }; const result = evaluateLabBudget( { context: contextMetadata, metrics }, @@ -164,14 +278,20 @@ try { }, }); if (!passed) { - throw new Error(`Lab performance failed: ${JSON.stringify(metrics)}`); + throw new Error( + `Lab performance failed (median of ${runs}): ${JSON.stringify(metrics)}; ` + + `interaction samples ${interaction.samples.join(",")}`, + ); } process.stdout.write( - `Lab performance: PASS (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms)\n`, + `Lab performance: PASS, median of ${runs} (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms ` + + `[${interaction.min}-${interaction.max}])\n`, ); } finally { await browser.close(); } } finally { - server.kill("SIGTERM"); + if (server.pid !== undefined && serverExit === null) { + process.kill(-server.pid, "SIGTERM"); + } } diff --git a/tests/unit/lab-performance.test.ts b/tests/unit/lab-performance.test.ts new file mode 100644 index 00000000..c3ea9d82 --- /dev/null +++ b/tests/unit/lab-performance.test.ts @@ -0,0 +1,106 @@ +import { spawn } from "node:child_process"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + assertPortUnused, + assertServedRelease, + previewArguments, + summarizeSamples, +} from "../../scripts/lib/lab-performance.ts"; + +/** + * 11-07. The lab performance gate measured once, and measured whatever server + * held its port: `vite preview` without `--strictPort` moved elsewhere and the + * script never checked which build answered. + */ +const servers: Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => new Promise((resolve) => server.close(() => resolve())), + ), + ); +}); + +async function foreignServer(manifest: unknown): Promise { + const server = createServer((_request, response) => { + response + .writeHead(200, { "content-type": "application/json" }) + .end(JSON.stringify(manifest)); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +describe("lab performance gate (11-07)", () => { + it("starts the preview server on exactly the requested port", () => { + expect(previewArguments(4173)).toEqual([ + "pnpm", + "preview", + "--host", + "127.0.0.1", + "--port", + "4173", + "--strictPort", + ]); + }); + + it("summarizes repeated runs by their median and keeps every sample", () => { + expect(summarizeSamples([326, 1103, 410, 390, 505])).toEqual({ + median: 410, + min: 326, + max: 1103, + samples: [326, 1103, 410, 390, 505], + }); + expect(summarizeSamples([1, 4, 2, 3]).median).toBe(2.5); + expect(() => summarizeSamples([])).toThrow(RangeError); + }); + + it("refuses a port that another server already answers on", async () => { + const baseUrl = await foreignServer({}); + await expect(assertPortUnused(baseUrl)).rejects.toThrow(/another server/u); + await expect(assertPortUnused("http://127.0.0.1:9")).resolves.toBeUndefined(); + }); + + it("refuses a server that serves a different build", async () => { + const baseUrl = await foreignServer({ buildId: "other", releaseId: "r1" }); + await expect( + assertServedRelease(baseUrl, { buildId: "measured", releaseId: "r1" }), + ).rejects.toThrow(/serves build other\/r1, not the measured measured\/r1/u); + await expect( + assertServedRelease(baseUrl, { buildId: "other", releaseId: "r1" }), + ).resolves.toBeUndefined(); + }); + + it("fails the gate before measuring when its port is taken", async () => { + const baseUrl = await foreignServer({ buildId: "other", releaseId: "r1" }); + const child = spawn( + process.execPath, + [path.resolve("scripts/test-performance.ts")], + { + env: { ...process.env, LAB_PERFORMANCE_PORT: new URL(baseUrl).port }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + const code = await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 20_000); + child.on("exit", (exitCode) => { + clearTimeout(timer); + resolve(exitCode); + }); + }); + + expect(code).not.toBe(0); + expect(stderr).toMatch(/another server holds the port/u); + }, 30_000); +}); From eb5264ce50b34dca63687ff01dab8ec4dc9e5196 Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:16:31 +0900 Subject: [PATCH 11/15] test(e2e): open a build that throws at module evaluation (F-2) 09-03 moved the application graph behind a dynamic import, but only jsdom checked it. A real production build whose installed messages collide now opens in a browser and must show the boot error shell, not the loading shell. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/boot-module-failure.spec.ts | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/e2e/boot-module-failure.spec.ts diff --git a/tests/e2e/boot-module-failure.spec.ts b/tests/e2e/boot-module-failure.spec.ts new file mode 100644 index 00000000..5e1f5718 --- /dev/null +++ b/tests/e2e/boot-module-failure.spec.ts @@ -0,0 +1,59 @@ +import path from "node:path"; + +import { expect, test } from "../support/browser/strict-browser-test.ts"; +import { + editOnce, + prepareFixtureBuilds, + removeFixtureBuilds, + type FixtureBuilds, +} from "./support/fixture-builds.ts"; +import { startBuildServer } from "./support/service-worker-builds.ts"; + +/** + * 09-03 (F-2). A production build whose application graph throws while its + * modules evaluate: the installed features are composed twice, so + * `installed-feature-messages.ts` meets every message key a second time and + * throws at module top level. The static "loading" shell used to stay up + * forever; the boot error shell has to replace it. + */ + +let builds: FixtureBuilds<"broken"> | undefined; + +test.beforeAll(async () => { + test.setTimeout(300_000); + builds = await prepareFixtureBuilds("boot-module-failure", [ + { + name: "broken", + edit: (root) => + editOnce( + path.join(root, "src/features/installed-feature-messages.ts"), + "composeInstalledMessageCatalogs(\n INSTALLED_PRODUCT_FEATURE_MODULES,\n)", + "composeInstalledMessageCatalogs([\n ...INSTALLED_PRODUCT_FEATURE_MODULES,\n ...INSTALLED_PRODUCT_FEATURE_MODULES,\n])", + ), + }, + ]); +}); + +test.afterAll(async () => { + await removeFixtureBuilds(builds); +}); + +test("shows the boot error shell when a module throws while it evaluates", async ({ + page, +}) => { + if (!builds) throw new Error("Fixture build was not prepared."); + const broken = builds.dist.broken; + const server = await startBuildServer({ a: broken, b: broken }); + try { + await page.goto(`${server.origin}/`); + + const shell = page.getByRole("alert"); + await expect(shell).toContainText("애플리케이션을 시작할 수 없습니다."); + await expect(shell).toContainText("BOOT_UNEXPECTED_FAILURE"); + await expect(shell).toContainText("TypeError"); + await expect(shell).toContainText("boot-module-failure-broken"); + await expect(page.locator("[data-boot-shell]")).toHaveCount(0); + } finally { + await server.close(); + } +}); From 66aca381797f97467fe145a0a379553df4650a45 Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:16:31 +0900 Subject: [PATCH 12/15] test(unit): remove the temp roots three suites left in /tmp (00-03) Each run left 35 directories behind (repository-inventory, inventory-outside, sw-assets, generator), and thousands piled up. They are now removed after every test. Co-Authored-By: Claude Opus 5.5 --- .../generate-product-feature-index.test.ts | 13 ++++++++++++- tests/unit/repository-file-inventory.test.ts | 16 ++++++++++++++-- .../service-worker-web-push-remediation.test.ts | 13 +++++++++++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/tests/scripts/generate-product-feature-index.test.ts b/tests/scripts/generate-product-feature-index.test.ts index 1f1bcfd4..df5387ce 100644 --- a/tests/scripts/generate-product-feature-index.test.ts +++ b/tests/scripts/generate-product-feature-index.test.ts @@ -1,4 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { rm } from "node:fs/promises"; + +import { afterEach, describe, expect, it } from "vitest"; import { generateAppIndexes, @@ -80,11 +82,20 @@ describe("feature index generator", () => { }); describe("app-scoped selection from ownership declarations", () => { + // 00-03. The fake repositories used to stay in /tmp. + const temporaryRoots: string[] = []; + afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); + }); + async function repo(files: Record) { const { mkdtemp, mkdir, writeFile } = await import("node:fs/promises"); const { tmpdir } = await import("node:os"); const path = await import("node:path"); const root = await mkdtemp(path.join(tmpdir(), "generator-")); + temporaryRoots.push(root); for (const [file, content] of Object.entries(files)) { await mkdir(path.dirname(path.join(root, file)), { recursive: true }); await writeFile(path.join(root, file), content); diff --git a/tests/unit/repository-file-inventory.test.ts b/tests/unit/repository-file-inventory.test.ts index 8293e62a..73e6b31c 100644 --- a/tests/unit/repository-file-inventory.test.ts +++ b/tests/unit/repository-file-inventory.test.ts @@ -1,8 +1,8 @@ -import { mkdtemp, mkdir, symlink, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { buildRepositoryFileInventory, @@ -23,8 +23,19 @@ function gitResult( }; } +// 00-03. Every temp root is removed after its test; they used to pile up in +// /tmp by the thousand. +const temporaryRoots: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + async function repositoryFixture() { const root = await mkdtemp(path.join(tmpdir(), "repository-inventory-")); + temporaryRoots.push(root); await mkdir(path.join(root, "src")); await mkdir(path.join(root, "docs")); await writeFile(path.join(root, "src", "tracked.ts"), "tracked\n"); @@ -203,6 +214,7 @@ describe("repository file inventory", () => { it("rejects a tracked symlink whose real path escapes the repository", async () => { const repositoryRoot = await repositoryFixture(); const outside = await mkdtemp(path.join(tmpdir(), "inventory-outside-")); + temporaryRoots.push(outside); await writeFile(path.join(outside, "secret.ts"), "secret\n"); await symlink( path.join(outside, "secret.ts"), diff --git a/tests/unit/service-worker-web-push-remediation.test.ts b/tests/unit/service-worker-web-push-remediation.test.ts index 3b7994c6..68cfa7cc 100644 --- a/tests/unit/service-worker-web-push-remediation.test.ts +++ b/tests/unit/service-worker-web-push-remediation.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -80,10 +80,19 @@ describe("SW-RR-03 one authoritative cacheable asset table", () => { * had just produced, failing the release build at install time. */ describe("SW-02 the generator and the decoder share one path grammar", () => { + // 00-03. The dist fixtures used to stay in /tmp. + const temporaryRoots: string[] = []; + afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); + }); + async function distWith( files: Readonly>, ): Promise { const root = await mkdtemp(path.join(tmpdir(), "sw-assets-")); + temporaryRoots.push(root); for (const [relative, content] of Object.entries(files)) { const absolute = path.join(root, relative); await mkdir(path.dirname(absolute), { recursive: true }); From a8520f6dfb477b00d7603b6555eb7efbdb20adfa Mon Sep 17 00:00:00 2001 From: donghyeon-ka Date: Sun, 27 Sep 2026 13:20:12 +0900 Subject: [PATCH 13/15] fix(fixtures): keep pnpm in a linked fixture from installing (F-4) pnpm verifies the install before a script, and a fixture running it with CI=true against an install made without CI saw other settings and installed through the linked .pnpm and .bin, rewriting the shared lock.yaml and shims. Every fixture pnpm run now turns that check off; a test pins it for all. Co-Authored-By: Claude Opus 5.5 --- .../check-supply-chain-provider-fixtures.ts | 6 +- scripts/lib/fixture-node-modules.ts | 14 ++ scripts/lib/removal-fixture.ts | 9 +- .../security-followup-archive.test.ts | 6 +- .../ci-runner/ci-artifact-contract.test.ts | 6 +- tests/unit/fixture-node-modules.test.ts | 145 +++++++++++++++++- 6 files changed, 177 insertions(+), 9 deletions(-) diff --git a/scripts/check-supply-chain-provider-fixtures.ts b/scripts/check-supply-chain-provider-fixtures.ts index 21646494..80f702d9 100644 --- a/scripts/check-supply-chain-provider-fixtures.ts +++ b/scripts/check-supply-chain-provider-fixtures.ts @@ -41,7 +41,10 @@ import { releaseCandidateManifestSchema, } from "./lib/release-candidate.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; -import { linkFixtureNodeModules } from "./lib/fixture-node-modules.ts"; +import { + FIXTURE_PNPM_ENVIRONMENT, + linkFixtureNodeModules, +} from "./lib/fixture-node-modules.ts"; const repositoryRoot = process.cwd(); const fixtureRoot = await mkdtemp(path.join(tmpdir(), "provider-exact-five-fixture-")); @@ -74,6 +77,7 @@ try { maxBuffer: 32 * 1024 * 1024, env: { ...process.env, + ...FIXTURE_PNPM_ENVIRONMENT, CI: "true", VITE_BUILD_ID: "provider-exact-five-fixture", VITE_COMMIT_SHA: revision, diff --git a/scripts/lib/fixture-node-modules.ts b/scripts/lib/fixture-node-modules.ts index 85bb7ce2..26e3e7b5 100644 --- a/scripts/lib/fixture-node-modules.ts +++ b/scripts/lib/fixture-node-modules.ts @@ -27,6 +27,20 @@ import path from "node:path"; * the linked directories. It is rewritten to name the fixture root instead: * the packages are the repository's own, already validated. */ +/** + * F-4. The environment for every `pnpm` a linked fixture runs. pnpm verifies + * the install before a script and, when the fixture's copied state disagrees + * with its own settings (for example `CI=true` against an install made + * without it), installs. `.pnpm` and `.bin` are links into the repository's + * node_modules, so that install rewrote the shared `.pnpm/lock.yaml` and bin + * shims. A fixture started through `pnpm