diff --git a/docs/operations/performance-evidence.md b/docs/operations/performance-evidence.md index 176891f6..9faa1659 100644 --- a/docs/operations/performance-evidence.md +++ b/docs/operations/performance-evidence.md @@ -41,10 +41,28 @@ Performance evidence is deliberately split by measurement context: 예산을 올릴 때는 이 표의 기준선과 새 측정값, 이유를 같은 변경에 적는다. lab 측정(`scripts/test-performance.ts`, FE-GATE-026)은 local 프로필 build를 -`vite preview`로 띄워 한 번만 잰다. named interaction 시간은 Node의 -`performance.now()`로 Playwright 클릭부터 heading 대기까지 재므로 Playwright -왕복 시간이 들어간다. 한 번 측정한 값은 흔들린다. 경계 근처 값으로 판단할 -때는 여러 번 돌려 중앙값을 본다. +`vite preview --strictPort`로 띄워 잰다(리뷰 11-07). + +- 포트는 `LAB_PERFORMANCE_PORT`(기본 4173)다. 서버를 띄우기 전에 그 포트가 + 이미 응답하면 실패하고, 응답하는 `/release-manifest.json`의 build·release가 + 로컬 `dist/release-manifest.json`과 다르면 실패한다 + (`scripts/lib/lab-performance.ts`, `tests/unit/lab-performance.test.ts`). +- 새 브라우저 context에서 `--runs=N`(기본 5)번 재고, 예산은 중앙값으로 + 판정한다. 모든 측정값과 최소·최대는 `lab.json`의 `context.sampling`에 남는다. +- named interaction은 페이지 안의 시계로 잰다. 클릭 이벤트의 `timeStamp`부터 + 대상 heading이 DOM에 들어온 시각까지다. Playwright 왕복 시간은 들어가지 않고, + lazy chunk 다운로드(200 KB/s 제한)는 들어간다. 클릭의 Event Timing entry + 길이(INP의 입력값)는 `context.sampling.clickEventDurationMs`에 따로 남는다. +- CPU 모델, 논리 CPU 수, 측정 전후 loadavg를 `context.runner`에 남긴다. +- preview 서버는 자기 프로세스 그룹으로 띄우고 끝날 때 그룹 전체를 멈춘다. + `corepack`에만 신호를 보내던 때는 vite가 포트를 계속 잡고 있었다. + +2026-09-27 한 빌드를 10회 잰 결과(i5-10400F 12 CPU, loadavg 11~17): 중앙값 +LCP 2,386ms, interaction 184.5ms로 통과했다. 한 번씩 판정했다면 interaction +10회 중 4회(최대 274ms), LCP 10회 중 2회(최대 2,644ms)가 실패였다. 같은 +빌드를 loadavg 7~9에서 다시 10회 재면 interaction 120~148ms, LCP +1,968~2,072ms로 모두 통과했다. 한 번 측정한 판정은 호스트 부하에 따라 +뒤집히고, 중앙값 판정은 두 번 모두 통과였다. The field minimum eligible-sample threshold is intentionally unresolved until a privacy-approved telemetry baseline exists. Therefore the field command diff --git a/scripts/lib/lab-performance.ts b/scripts/lib/lab-performance.ts new file mode 100644 index 00000000..0c01699d --- /dev/null +++ b/scripts/lib/lab-performance.ts @@ -0,0 +1,89 @@ +/** + * 11-07. The pieces of the lab performance gate (`scripts/test-performance.ts`) + * that decide what is measured and whether the numbers can be trusted. + */ + +export const DEFAULT_LAB_PORT = 4173; +export const DEFAULT_LAB_RUNS = 5; + +/** + * `vite preview` arguments. Without `--strictPort` vite moved to the next free + * port when the requested one was taken, and the gate measured whatever server + * already listened there. + */ +export function previewArguments(port: number): readonly string[] { + return [ + "pnpm", + "preview", + "--host", + "127.0.0.1", + "--port", + String(port), + "--strictPort", + ]; +} + +export type SampleSummary = Readonly<{ + median: number; + min: number; + max: number; + samples: readonly number[]; +}>; + +export function summarizeSamples(samples: readonly number[]): SampleSummary { + if (samples.length === 0) throw new RangeError("no samples to summarize"); + const sorted = [...samples].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + const median = + sorted.length % 2 === 1 + ? sorted[middle]! + : (sorted[middle - 1]! + sorted[middle]!) / 2; + return Object.freeze({ + median, + min: sorted[0]!, + max: sorted.at(-1)!, + samples: Object.freeze([...samples]), + }); +} + +/** Fails when anything already answers where the preview server will listen. */ +export async function assertPortUnused( + baseUrl: string, + fetchImpl: typeof fetch = fetch, +): Promise { + let answered = false; + try { + await fetchImpl(baseUrl, { signal: AbortSignal.timeout(1_000) }); + answered = true; + } catch { + // Nothing is listening: the expected state. + } + if (answered) { + throw new Error( + `${baseUrl} already answers before the preview server starts; another server holds the port.`, + ); + } +} + +/** + * Compares the release the server actually serves with the local build that + * is being measured, so the numbers belong to that build. + */ +export async function assertServedRelease( + baseUrl: string, + expected: Readonly<{ buildId: string; releaseId: string }>, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await fetchImpl(new URL("/release-manifest.json", baseUrl), { + cache: "no-store", + }); + const served = response.ok + ? ((await response.json()) as Partial>) + : {}; + if (served.buildId !== expected.buildId || served.releaseId !== expected.releaseId) { + throw new Error( + `${baseUrl} serves build ${String(served.buildId)}/${String(served.releaseId)}, ` + + `not the measured ${expected.buildId}/${expected.releaseId}.`, + ); + } +} diff --git a/scripts/test-performance.ts b/scripts/test-performance.ts index 00547c8a..4c82ebdb 100644 --- a/scripts/test-performance.ts +++ b/scripts/test-performance.ts @@ -1,10 +1,18 @@ import { spawn } from "node:child_process"; import { mkdir, readFile } from "node:fs/promises"; -import { performance } from "node:perf_hooks"; +import { cpus, loadavg } from "node:os"; import process from "node:process"; -import { chromium } from "@playwright/test"; +import { chromium, type Browser } from "@playwright/test"; +import { + assertPortUnused, + assertServedRelease, + DEFAULT_LAB_PORT, + DEFAULT_LAB_RUNS, + previewArguments, + summarizeSamples, +} from "./lib/lab-performance.ts"; import { evaluateLabBudget } from "./lib/performance-budgets.ts"; import { ROUTE_REGISTRY } from "../src/features/installed-feature-contracts.ts"; import { labPerformanceArtifactSchema } from "./contracts/release-artifacts.ts"; @@ -13,6 +21,9 @@ import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; type ContractPerformanceEvidence = { lcpMs: number; cls: number; + clickAt: number | null; + headingAt: number | null; + eventDurationMs: number | null; }; type ContractPerformanceWindow = Window & { @@ -24,15 +35,37 @@ type LayoutShiftEntry = PerformanceEntry & { value: number; }; -const server = spawn( - "corepack", - ["pnpm", "preview", "--host", "127.0.0.1", "--port", "4173"], - { stdio: "ignore" }, -); -const baseUrl = "http://127.0.0.1:4173"; +const argument = (name: string) => + process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3); +const port = Number(process.env.LAB_PERFORMANCE_PORT ?? DEFAULT_LAB_PORT); +const runs = Number(argument("runs") ?? DEFAULT_LAB_RUNS); +if (!Number.isSafeInteger(port) || port <= 0 || !Number.isSafeInteger(runs) || runs < 1) { + throw new TypeError("LAB_PERFORMANCE_PORT must be a port and --runs a positive integer."); +} +const baseUrl = `http://127.0.0.1:${port}`; +const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM; +const targetLabel = target.navigationLabel; +if (!targetLabel) { + throw new Error("Performance route must be present in navigation."); +} + +// 11-07. A server already on the port used to be measured in place of this +// build: vite moved to another port and the gate never noticed. +await assertPortUnused(baseUrl); +// Its own process group: `corepack` starts pnpm, which starts vite, and a +// signal to `corepack` alone left vite serving the port after the gate ended. +const server = spawn("corepack", [...previewArguments(port)], { + stdio: "ignore", + detached: true, +}); +let serverExit: string | null = null; +server.on("exit", (code, signal) => { + serverExit = `preview server exited (${signal ?? code})`; +}); async function waitForServer() { - for (let attempt = 0; attempt < 50; attempt += 1) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (serverExit) throw new Error(serverExit); try { const response = await fetch(baseUrl); if (response.ok) return; @@ -44,19 +77,14 @@ async function waitForServer() { throw new Error("Preview server did not become ready."); } -try { - await waitForServer(); - const release = JSON.parse( - await readFile("dist/release-manifest.json", "utf8"), - ); - const thresholds = JSON.parse( - await readFile("config/performance/budgets.json", "utf8"), - ).lab; - const browser = await chromium.launch(); +type RunMetrics = { lcpMs: number; cls: number; namedInteractionMs: number; eventDurationMs: number | null }; + +async function measureOnce(browser: Browser): Promise { + // Every run is a cold, isolated context. + const context = await browser.newContext({ + viewport: { width: 1280, height: 720 }, + }); try { - const context = await browser.newContext({ - viewport: { width: 1280, height: 720 }, - }); const page = await context.newPage(); const cdp = await context.newCDPSession(page); await cdp.send("Network.enable"); @@ -68,8 +96,14 @@ try { connectionType: "cellular4g", }); await cdp.send("Emulation.setCPUThrottlingRate", { rate: 4 }); - await page.addInitScript(() => { - const evidence = { lcpMs: 0, cls: 0 }; + await page.addInitScript((title: string) => { + const evidence: ContractPerformanceEvidence = { + lcpMs: 0, + cls: 0, + clickAt: null, + headingAt: null, + eventDurationMs: null, + }; (window as ContractPerformanceWindow).__contractPerformance = evidence; new PerformanceObserver((list) => { for (const entry of list.getEntries()) evidence.lcpMs = entry.startTime; @@ -82,34 +116,97 @@ try { } } }).observe({ type: "layout-shift", buffered: true }); - }); + // 11-07. The interaction is timed by the page's own clock, from the + // click event to the target heading entering the DOM, so Playwright's + // round trips are not in it. The Event Timing entry of the click (the + // INP input) is recorded next to it. + new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + if (entry.name === "click") { + evidence.eventDurationMs = Math.max(evidence.eventDurationMs ?? 0, entry.duration); + } + } + }).observe({ type: "event", buffered: true, durationThreshold: 16 } as PerformanceObserverInit); + document.addEventListener( + "click", + (event) => { + evidence.clickAt ??= event.timeStamp; + }, + { capture: true }, + ); + new MutationObserver((_records, observer) => { + if (evidence.clickAt === null) return; + const found = [...document.querySelectorAll("h1, h2")].some( + (heading) => heading.textContent?.trim() === title, + ); + if (found) { + evidence.headingAt = performance.now(); + observer.disconnect(); + } + }).observe(document, { childList: true, subtree: true, characterData: true }); + }, target.title); await page.goto(baseUrl, { waitUntil: "networkidle" }); - const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM; - const targetLabel = target.navigationLabel; - if (!targetLabel) { - throw new Error("Performance route must be present in navigation."); - } - const interactionStarted = performance.now(); // Playwright matches accessible names by substring, so the navigation entry // "플랫폼 구성" also matched the home page's "플랫폼 구성 보기" call to - // action and the locator resolved to two links. That is a strict-mode - // violation before the first measurement is taken, so no lab performance - // evidence could be produced at all — the run failed for an ambiguous - // selector rather than for anything about performance. + // action; `exact` keeps the locator on the navigation link. await page.getByRole("link", { name: targetLabel, exact: true }).click(); await page.getByRole("heading", { name: target.title }).waitFor(); - const namedInteractionMs = performance.now() - interactionStarted; + await page.waitForFunction( + () => (window as ContractPerformanceWindow).__contractPerformance?.headingAt !== null, + ); const paint = await page.evaluate( () => (window as ContractPerformanceWindow).__contractPerformance, ); - if (!paint) { + if (!paint || paint.clickAt === null || paint.headingAt === null) { throw new Error("Browser performance evidence was not initialized."); } + return { + lcpMs: Math.round(paint.lcpMs), + cls: Number(paint.cls.toFixed(4)), + namedInteractionMs: Math.round(paint.headingAt - paint.clickAt), + eventDurationMs: + paint.eventDurationMs === null ? null : Math.round(paint.eventDurationMs), + }; + } finally { + await context.close(); + } +} + +const load = () => loadavg().map((value) => Math.round(value * 100) / 100); + +try { + await waitForServer(); + const release = JSON.parse( + await readFile("dist/release-manifest.json", "utf8"), + ); + await assertServedRelease(baseUrl, release); + const thresholds = JSON.parse( + await readFile("config/performance/budgets.json", "utf8"), + ).lab; + const browser = await chromium.launch(); + try { + const loadBefore = load(); + const samples: RunMetrics[] = []; + for (let run = 0; run < runs; run += 1) samples.push(await measureOnce(browser)); + const loadAfter = load(); + // The served build must not have changed under the runs either. + await assertServedRelease(baseUrl, release); + const lcp = summarizeSamples(samples.map((sample) => sample.lcpMs)); + const cls = summarizeSamples(samples.map((sample) => sample.cls)); + const interaction = summarizeSamples( + samples.map((sample) => sample.namedInteractionMs), + ); + const eventDurations = samples.flatMap((sample) => + sample.eventDurationMs === null ? [] : [sample.eventDurationMs], + ); const contextMetadata = { runner: { platform: process.platform, architecture: process.arch, nodeVersion: process.version, + cpuModel: cpus()[0]?.model.trim() ?? "unknown", + logicalCpus: cpus().length, + loadAverage: { before: loadBefore, after: loadAfter }, }, browser: { name: "chromium", version: await browser.version() }, viewport: { width: 1280, height: 720 }, @@ -121,12 +218,29 @@ try { }, cpu: { throttlingRate: 4 }, cache: { state: "cold", isolation: "new-browser-context" }, - build: { buildId: release.buildId, releaseId: release.releaseId }, + build: { + buildId: release.buildId, + releaseId: release.releaseId, + served: `${baseUrl}/release-manifest.json`, + }, + // 11-07. The budget is judged on the median of cold runs; every run and + // the spread stay in the evidence. + sampling: { + runs, + statistic: "median", + interactionClock: "page: click event to target heading in DOM", + lcpMs: lcp, + cls, + namedInteractionMs: interaction, + clickEventDurationMs: eventDurations.length + ? summarizeSamples(eventDurations) + : null, + }, }; const metrics = { - lcpMs: Math.round(paint.lcpMs), - cls: Number(paint.cls.toFixed(4)), - namedInteractionMs: Math.round(namedInteractionMs), + lcpMs: lcp.median, + cls: cls.median, + namedInteractionMs: interaction.median, }; const result = evaluateLabBudget( { context: contextMetadata, metrics }, @@ -164,14 +278,20 @@ try { }, }); if (!passed) { - throw new Error(`Lab performance failed: ${JSON.stringify(metrics)}`); + throw new Error( + `Lab performance failed (median of ${runs}): ${JSON.stringify(metrics)}; ` + + `interaction samples ${interaction.samples.join(",")}`, + ); } process.stdout.write( - `Lab performance: PASS (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms)\n`, + `Lab performance: PASS, median of ${runs} (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms ` + + `[${interaction.min}-${interaction.max}])\n`, ); } finally { await browser.close(); } } finally { - server.kill("SIGTERM"); + if (server.pid !== undefined && serverExit === null) { + process.kill(-server.pid, "SIGTERM"); + } } diff --git a/tests/unit/lab-performance.test.ts b/tests/unit/lab-performance.test.ts new file mode 100644 index 00000000..c3ea9d82 --- /dev/null +++ b/tests/unit/lab-performance.test.ts @@ -0,0 +1,106 @@ +import { spawn } from "node:child_process"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + assertPortUnused, + assertServedRelease, + previewArguments, + summarizeSamples, +} from "../../scripts/lib/lab-performance.ts"; + +/** + * 11-07. The lab performance gate measured once, and measured whatever server + * held its port: `vite preview` without `--strictPort` moved elsewhere and the + * script never checked which build answered. + */ +const servers: Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => new Promise((resolve) => server.close(() => resolve())), + ), + ); +}); + +async function foreignServer(manifest: unknown): Promise { + const server = createServer((_request, response) => { + response + .writeHead(200, { "content-type": "application/json" }) + .end(JSON.stringify(manifest)); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +describe("lab performance gate (11-07)", () => { + it("starts the preview server on exactly the requested port", () => { + expect(previewArguments(4173)).toEqual([ + "pnpm", + "preview", + "--host", + "127.0.0.1", + "--port", + "4173", + "--strictPort", + ]); + }); + + it("summarizes repeated runs by their median and keeps every sample", () => { + expect(summarizeSamples([326, 1103, 410, 390, 505])).toEqual({ + median: 410, + min: 326, + max: 1103, + samples: [326, 1103, 410, 390, 505], + }); + expect(summarizeSamples([1, 4, 2, 3]).median).toBe(2.5); + expect(() => summarizeSamples([])).toThrow(RangeError); + }); + + it("refuses a port that another server already answers on", async () => { + const baseUrl = await foreignServer({}); + await expect(assertPortUnused(baseUrl)).rejects.toThrow(/another server/u); + await expect(assertPortUnused("http://127.0.0.1:9")).resolves.toBeUndefined(); + }); + + it("refuses a server that serves a different build", async () => { + const baseUrl = await foreignServer({ buildId: "other", releaseId: "r1" }); + await expect( + assertServedRelease(baseUrl, { buildId: "measured", releaseId: "r1" }), + ).rejects.toThrow(/serves build other\/r1, not the measured measured\/r1/u); + await expect( + assertServedRelease(baseUrl, { buildId: "other", releaseId: "r1" }), + ).resolves.toBeUndefined(); + }); + + it("fails the gate before measuring when its port is taken", async () => { + const baseUrl = await foreignServer({ buildId: "other", releaseId: "r1" }); + const child = spawn( + process.execPath, + [path.resolve("scripts/test-performance.ts")], + { + env: { ...process.env, LAB_PERFORMANCE_PORT: new URL(baseUrl).port }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + const code = await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 20_000); + child.on("exit", (exitCode) => { + clearTimeout(timer); + resolve(exitCode); + }); + }); + + expect(code).not.toBe(0); + expect(stderr).toMatch(/another server holds the port/u); + }, 30_000); +});