perf(lab): repeat lab runs, judge the median and pin the measured server (11-07)

One run decided the release gate, the interaction included Playwright round
trips, and a server already on 4173 was measured instead of the build. Runs
now repeat with every sample kept, time the interaction in the page, use
--strictPort and check the served build, and stop the whole preview group.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
donghyeon-ka
2026-09-27 13:14:03 +09:00
co-authored by Claude Opus 5.5
parent 34c97885ea
commit 8e5c51e813
4 changed files with 380 additions and 47 deletions
+89
View File
@@ -0,0 +1,89 @@
/**
* 11-07. The pieces of the lab performance gate (`scripts/test-performance.ts`)
* that decide what is measured and whether the numbers can be trusted.
*/
export const DEFAULT_LAB_PORT = 4173;
export const DEFAULT_LAB_RUNS = 5;
/**
* `vite preview` arguments. Without `--strictPort` vite moved to the next free
* port when the requested one was taken, and the gate measured whatever server
* already listened there.
*/
export function previewArguments(port: number): readonly string[] {
return [
"pnpm",
"preview",
"--host",
"127.0.0.1",
"--port",
String(port),
"--strictPort",
];
}
export type SampleSummary = Readonly<{
median: number;
min: number;
max: number;
samples: readonly number[];
}>;
export function summarizeSamples(samples: readonly number[]): SampleSummary {
if (samples.length === 0) throw new RangeError("no samples to summarize");
const sorted = [...samples].sort((left, right) => left - right);
const middle = Math.floor(sorted.length / 2);
const median =
sorted.length % 2 === 1
? sorted[middle]!
: (sorted[middle - 1]! + sorted[middle]!) / 2;
return Object.freeze({
median,
min: sorted[0]!,
max: sorted.at(-1)!,
samples: Object.freeze([...samples]),
});
}
/** Fails when anything already answers where the preview server will listen. */
export async function assertPortUnused(
baseUrl: string,
fetchImpl: typeof fetch = fetch,
): Promise<void> {
let answered = false;
try {
await fetchImpl(baseUrl, { signal: AbortSignal.timeout(1_000) });
answered = true;
} catch {
// Nothing is listening: the expected state.
}
if (answered) {
throw new Error(
`${baseUrl} already answers before the preview server starts; another server holds the port.`,
);
}
}
/**
* Compares the release the server actually serves with the local build that
* is being measured, so the numbers belong to that build.
*/
export async function assertServedRelease(
baseUrl: string,
expected: Readonly<{ buildId: string; releaseId: string }>,
fetchImpl: typeof fetch = fetch,
): Promise<void> {
const response = await fetchImpl(new URL("/release-manifest.json", baseUrl), {
cache: "no-store",
});
const served = response.ok
? ((await response.json()) as Partial<Record<"buildId" | "releaseId", unknown>>)
: {};
if (served.buildId !== expected.buildId || served.releaseId !== expected.releaseId) {
throw new Error(
`${baseUrl} serves build ${String(served.buildId)}/${String(served.releaseId)}, ` +
`not the measured ${expected.buildId}/${expected.releaseId}.`,
);
}
}
+163 -43
View File
@@ -1,10 +1,18 @@
import { spawn } from "node:child_process";
import { mkdir, readFile } from "node:fs/promises";
import { performance } from "node:perf_hooks";
import { cpus, loadavg } from "node:os";
import process from "node:process";
import { chromium } from "@playwright/test";
import { chromium, type Browser } from "@playwright/test";
import {
assertPortUnused,
assertServedRelease,
DEFAULT_LAB_PORT,
DEFAULT_LAB_RUNS,
previewArguments,
summarizeSamples,
} from "./lib/lab-performance.ts";
import { evaluateLabBudget } from "./lib/performance-budgets.ts";
import { ROUTE_REGISTRY } from "../src/features/installed-feature-contracts.ts";
import { labPerformanceArtifactSchema } from "./contracts/release-artifacts.ts";
@@ -13,6 +21,9 @@ import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
type ContractPerformanceEvidence = {
lcpMs: number;
cls: number;
clickAt: number | null;
headingAt: number | null;
eventDurationMs: number | null;
};
type ContractPerformanceWindow = Window & {
@@ -24,15 +35,37 @@ type LayoutShiftEntry = PerformanceEntry & {
value: number;
};
const server = spawn(
"corepack",
["pnpm", "preview", "--host", "127.0.0.1", "--port", "4173"],
{ stdio: "ignore" },
);
const baseUrl = "http://127.0.0.1:4173";
const argument = (name: string) =>
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3);
const port = Number(process.env.LAB_PERFORMANCE_PORT ?? DEFAULT_LAB_PORT);
const runs = Number(argument("runs") ?? DEFAULT_LAB_RUNS);
if (!Number.isSafeInteger(port) || port <= 0 || !Number.isSafeInteger(runs) || runs < 1) {
throw new TypeError("LAB_PERFORMANCE_PORT must be a port and --runs a positive integer.");
}
const baseUrl = `http://127.0.0.1:${port}`;
const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM;
const targetLabel = target.navigationLabel;
if (!targetLabel) {
throw new Error("Performance route must be present in navigation.");
}
// 11-07. A server already on the port used to be measured in place of this
// build: vite moved to another port and the gate never noticed.
await assertPortUnused(baseUrl);
// Its own process group: `corepack` starts pnpm, which starts vite, and a
// signal to `corepack` alone left vite serving the port after the gate ended.
const server = spawn("corepack", [...previewArguments(port)], {
stdio: "ignore",
detached: true,
});
let serverExit: string | null = null;
server.on("exit", (code, signal) => {
serverExit = `preview server exited (${signal ?? code})`;
});
async function waitForServer() {
for (let attempt = 0; attempt < 50; attempt += 1) {
for (let attempt = 0; attempt < 100; attempt += 1) {
if (serverExit) throw new Error(serverExit);
try {
const response = await fetch(baseUrl);
if (response.ok) return;
@@ -44,19 +77,14 @@ async function waitForServer() {
throw new Error("Preview server did not become ready.");
}
try {
await waitForServer();
const release = JSON.parse(
await readFile("dist/release-manifest.json", "utf8"),
);
const thresholds = JSON.parse(
await readFile("config/performance/budgets.json", "utf8"),
).lab;
const browser = await chromium.launch();
type RunMetrics = { lcpMs: number; cls: number; namedInteractionMs: number; eventDurationMs: number | null };
async function measureOnce(browser: Browser): Promise<RunMetrics> {
// Every run is a cold, isolated context.
const context = await browser.newContext({
viewport: { width: 1280, height: 720 },
});
try {
const context = await browser.newContext({
viewport: { width: 1280, height: 720 },
});
const page = await context.newPage();
const cdp = await context.newCDPSession(page);
await cdp.send("Network.enable");
@@ -68,8 +96,14 @@ try {
connectionType: "cellular4g",
});
await cdp.send("Emulation.setCPUThrottlingRate", { rate: 4 });
await page.addInitScript(() => {
const evidence = { lcpMs: 0, cls: 0 };
await page.addInitScript((title: string) => {
const evidence: ContractPerformanceEvidence = {
lcpMs: 0,
cls: 0,
clickAt: null,
headingAt: null,
eventDurationMs: null,
};
(window as ContractPerformanceWindow).__contractPerformance = evidence;
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) evidence.lcpMs = entry.startTime;
@@ -82,34 +116,97 @@ try {
}
}
}).observe({ type: "layout-shift", buffered: true });
});
// 11-07. The interaction is timed by the page's own clock, from the
// click event to the target heading entering the DOM, so Playwright's
// round trips are not in it. The Event Timing entry of the click (the
// INP input) is recorded next to it.
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
if (entry.name === "click") {
evidence.eventDurationMs = Math.max(evidence.eventDurationMs ?? 0, entry.duration);
}
}
}).observe({ type: "event", buffered: true, durationThreshold: 16 } as PerformanceObserverInit);
document.addEventListener(
"click",
(event) => {
evidence.clickAt ??= event.timeStamp;
},
{ capture: true },
);
new MutationObserver((_records, observer) => {
if (evidence.clickAt === null) return;
const found = [...document.querySelectorAll("h1, h2")].some(
(heading) => heading.textContent?.trim() === title,
);
if (found) {
evidence.headingAt = performance.now();
observer.disconnect();
}
}).observe(document, { childList: true, subtree: true, characterData: true });
}, target.title);
await page.goto(baseUrl, { waitUntil: "networkidle" });
const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM;
const targetLabel = target.navigationLabel;
if (!targetLabel) {
throw new Error("Performance route must be present in navigation.");
}
const interactionStarted = performance.now();
// Playwright matches accessible names by substring, so the navigation entry
// "플랫폼 구성" also matched the home page's "플랫폼 구성 보기" call to
// action and the locator resolved to two links. That is a strict-mode
// violation before the first measurement is taken, so no lab performance
// evidence could be produced at all — the run failed for an ambiguous
// selector rather than for anything about performance.
// action; `exact` keeps the locator on the navigation link.
await page.getByRole("link", { name: targetLabel, exact: true }).click();
await page.getByRole("heading", { name: target.title }).waitFor();
const namedInteractionMs = performance.now() - interactionStarted;
await page.waitForFunction(
() => (window as ContractPerformanceWindow).__contractPerformance?.headingAt !== null,
);
const paint = await page.evaluate(
() => (window as ContractPerformanceWindow).__contractPerformance,
);
if (!paint) {
if (!paint || paint.clickAt === null || paint.headingAt === null) {
throw new Error("Browser performance evidence was not initialized.");
}
return {
lcpMs: Math.round(paint.lcpMs),
cls: Number(paint.cls.toFixed(4)),
namedInteractionMs: Math.round(paint.headingAt - paint.clickAt),
eventDurationMs:
paint.eventDurationMs === null ? null : Math.round(paint.eventDurationMs),
};
} finally {
await context.close();
}
}
const load = () => loadavg().map((value) => Math.round(value * 100) / 100);
try {
await waitForServer();
const release = JSON.parse(
await readFile("dist/release-manifest.json", "utf8"),
);
await assertServedRelease(baseUrl, release);
const thresholds = JSON.parse(
await readFile("config/performance/budgets.json", "utf8"),
).lab;
const browser = await chromium.launch();
try {
const loadBefore = load();
const samples: RunMetrics[] = [];
for (let run = 0; run < runs; run += 1) samples.push(await measureOnce(browser));
const loadAfter = load();
// The served build must not have changed under the runs either.
await assertServedRelease(baseUrl, release);
const lcp = summarizeSamples(samples.map((sample) => sample.lcpMs));
const cls = summarizeSamples(samples.map((sample) => sample.cls));
const interaction = summarizeSamples(
samples.map((sample) => sample.namedInteractionMs),
);
const eventDurations = samples.flatMap((sample) =>
sample.eventDurationMs === null ? [] : [sample.eventDurationMs],
);
const contextMetadata = {
runner: {
platform: process.platform,
architecture: process.arch,
nodeVersion: process.version,
cpuModel: cpus()[0]?.model.trim() ?? "unknown",
logicalCpus: cpus().length,
loadAverage: { before: loadBefore, after: loadAfter },
},
browser: { name: "chromium", version: await browser.version() },
viewport: { width: 1280, height: 720 },
@@ -121,12 +218,29 @@ try {
},
cpu: { throttlingRate: 4 },
cache: { state: "cold", isolation: "new-browser-context" },
build: { buildId: release.buildId, releaseId: release.releaseId },
build: {
buildId: release.buildId,
releaseId: release.releaseId,
served: `${baseUrl}/release-manifest.json`,
},
// 11-07. The budget is judged on the median of cold runs; every run and
// the spread stay in the evidence.
sampling: {
runs,
statistic: "median",
interactionClock: "page: click event to target heading in DOM",
lcpMs: lcp,
cls,
namedInteractionMs: interaction,
clickEventDurationMs: eventDurations.length
? summarizeSamples(eventDurations)
: null,
},
};
const metrics = {
lcpMs: Math.round(paint.lcpMs),
cls: Number(paint.cls.toFixed(4)),
namedInteractionMs: Math.round(namedInteractionMs),
lcpMs: lcp.median,
cls: cls.median,
namedInteractionMs: interaction.median,
};
const result = evaluateLabBudget(
{ context: contextMetadata, metrics },
@@ -164,14 +278,20 @@ try {
},
});
if (!passed) {
throw new Error(`Lab performance failed: ${JSON.stringify(metrics)}`);
throw new Error(
`Lab performance failed (median of ${runs}): ${JSON.stringify(metrics)}; ` +
`interaction samples ${interaction.samples.join(",")}`,
);
}
process.stdout.write(
`Lab performance: PASS (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms)\n`,
`Lab performance: PASS, median of ${runs} (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms ` +
`[${interaction.min}-${interaction.max}])\n`,
);
} finally {
await browser.close();
}
} finally {
server.kill("SIGTERM");
if (server.pid !== undefined && serverExit === null) {
process.kill(-server.pid, "SIGTERM");
}
}