Merge lane/J: test system (10-02, 10-04~10-07, 11-04~11-08, F-2, F-4, F-12, 00-03)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
donghyeon-ka
2026-09-27 13:28:55 +09:00
co-authored by Claude Opus 5.5
43 changed files with 1908 additions and 172 deletions
@@ -41,7 +41,10 @@ import {
releaseCandidateManifestSchema,
} from "./lib/release-candidate.ts";
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
import { linkFixtureNodeModules } from "./lib/fixture-node-modules.ts";
import {
FIXTURE_PNPM_ENVIRONMENT,
linkFixtureNodeModules,
} from "./lib/fixture-node-modules.ts";
const repositoryRoot = process.cwd();
const fixtureRoot = await mkdtemp(path.join(tmpdir(), "provider-exact-five-fixture-"));
@@ -74,6 +77,7 @@ try {
maxBuffer: 32 * 1024 * 1024,
env: {
...process.env,
...FIXTURE_PNPM_ENVIRONMENT,
CI: "true",
VITE_BUILD_ID: "provider-exact-five-fixture",
VITE_COMMIT_SHA: revision,
+24 -3
View File
@@ -281,17 +281,27 @@ if (!scenarioOnly) {
const sourceFiles = (await filesBelow(sourceRoot)).filter(
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
);
const today = new Date().toISOString().slice(0, 10);
for (const file of sourceFiles) {
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
const source = await readFile(file, "utf8");
facts.scannedFiles += 1;
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
}
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
if (skipPattern.test(source)) {
const quarantine =
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
if (!quarantine.test(source)) {
const quarantines = [
...source.matchAll(
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
),
];
if (quarantines.length === 0) {
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
}
for (const [, expires = ""] of quarantines) {
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
}
}
const wholeUiMask =
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
@@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) {
failures.push(`playwright.config.ts missing release evidence token ${token}`);
}
}
// playwright.dev.config.ts spreads playwright.config.ts.
for (const config of [
"playwright.config.ts",
"playwright.visual.config.ts",
"playwright.storybook.config.ts",
"playwright.capabilities.config.ts",
]) {
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
}
}
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
/\.spec\.ts$/.test(file),
+14
View File
@@ -27,6 +27,20 @@ import path from "node:path";
* the linked directories. It is rewritten to name the fixture root instead:
* the packages are the repository's own, already validated.
*/
/**
* F-4. The environment for every `pnpm` a linked fixture runs. pnpm verifies
* the install before a script and, when the fixture's copied state disagrees
* with its own settings (for example `CI=true` against an install made
* without it), installs. `.pnpm` and `.bin` are links into the repository's
* node_modules, so that install rewrote the shared `.pnpm/lock.yaml` and bin
* shims. A fixture started through `pnpm <script>` inherits this setting from
* pnpm; one started directly (`node scripts/run-vitest.ts run <file>`) did not.
* The fixture uses the repository's packages as they are; it never installs.
*/
export const FIXTURE_PNPM_ENVIRONMENT = Object.freeze({
pnpm_config_verify_deps_before_run: "false",
});
export async function linkFixtureNodeModules(
fixtureRoot: string,
sourceRoot: string = process.cwd(),
+89
View File
@@ -0,0 +1,89 @@
/**
* 11-07. The pieces of the lab performance gate (`scripts/test-performance.ts`)
* that decide what is measured and whether the numbers can be trusted.
*/
export const DEFAULT_LAB_PORT = 4173;
export const DEFAULT_LAB_RUNS = 5;
/**
* `vite preview` arguments. Without `--strictPort` vite moved to the next free
* port when the requested one was taken, and the gate measured whatever server
* already listened there.
*/
export function previewArguments(port: number): readonly string[] {
return [
"pnpm",
"preview",
"--host",
"127.0.0.1",
"--port",
String(port),
"--strictPort",
];
}
export type SampleSummary = Readonly<{
median: number;
min: number;
max: number;
samples: readonly number[];
}>;
export function summarizeSamples(samples: readonly number[]): SampleSummary {
if (samples.length === 0) throw new RangeError("no samples to summarize");
const sorted = [...samples].sort((left, right) => left - right);
const middle = Math.floor(sorted.length / 2);
const median =
sorted.length % 2 === 1
? sorted[middle]!
: (sorted[middle - 1]! + sorted[middle]!) / 2;
return Object.freeze({
median,
min: sorted[0]!,
max: sorted.at(-1)!,
samples: Object.freeze([...samples]),
});
}
/** Fails when anything already answers where the preview server will listen. */
export async function assertPortUnused(
baseUrl: string,
fetchImpl: typeof fetch = fetch,
): Promise<void> {
let answered = false;
try {
await fetchImpl(baseUrl, { signal: AbortSignal.timeout(1_000) });
answered = true;
} catch {
// Nothing is listening: the expected state.
}
if (answered) {
throw new Error(
`${baseUrl} already answers before the preview server starts; another server holds the port.`,
);
}
}
/**
* Compares the release the server actually serves with the local build that
* is being measured, so the numbers belong to that build.
*/
export async function assertServedRelease(
baseUrl: string,
expected: Readonly<{ buildId: string; releaseId: string }>,
fetchImpl: typeof fetch = fetch,
): Promise<void> {
const response = await fetchImpl(new URL("/release-manifest.json", baseUrl), {
cache: "no-store",
});
const served = response.ok
? ((await response.json()) as Partial<Record<"buildId" | "releaseId", unknown>>)
: {};
if (served.buildId !== expected.buildId || served.releaseId !== expected.releaseId) {
throw new Error(
`${baseUrl} serves build ${String(served.buildId)}/${String(served.releaseId)}, ` +
`not the measured ${expected.buildId}/${expected.releaseId}.`,
);
}
}
+6 -3
View File
@@ -15,7 +15,10 @@ import {
loadCiGateContract,
parseCiGateContract,
} from "../contracts/ci-gates.ts";
import { linkFixtureNodeModules } from "./fixture-node-modules.ts";
import {
FIXTURE_PNPM_ENVIRONMENT,
linkFixtureNodeModules,
} from "./fixture-node-modules.ts";
import { generateCiWorkflow } from "../generate-ci-workflow.ts";
import { updateCiContractBaseline } from "../update-ci-contract-baseline.ts";
@@ -189,7 +192,7 @@ export function runRemovalFixtureCheck(
const status = spawnSync(process.execPath, [pnpmCli, script, ...extra], {
cwd: root,
stdio: "inherit",
env: process.env,
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT },
}).status;
return status === 0
? { name, passed: true }
@@ -243,7 +246,7 @@ export function runRemovalFixturePnpm(
return spawnSync(process.execPath, [pnpmCli, script, ...extra], {
cwd: root,
stdio: "inherit",
env: { ...process.env, ...env },
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, ...env },
}).status === 0;
}
@@ -0,0 +1,257 @@
import { spawnSync } from "node:child_process";
import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { availableParallelism, cpus, loadavg, tmpdir } from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { gzipSync } from "node:zlib";
import { classifyViteJavascript } from "../lib/classify-vite-bundle.ts";
import { linkFixtureNodeModules } from "../lib/fixture-node-modules.ts";
import { parseTscExtendedDiagnostics, type TscDiagnostics } from "./collect-tsc-diagnostics.ts";
/**
* 11-04 (D-18). Cost of one app as its feature count grows, measured with
* features shaped like the real one: `src/features/reference-feature` copied N
* times with every feature-owned identifier renamed (feature id, route ids and
* paths, API operations, schemas, query namespace, invalidation topic,
* messages). The copies live in contexts of their own (200 features each, the
* context ceiling) so the reference context is unchanged.
*
* For each N the script runs the product generator, `tsc --extendedDiagnostics`
* over tsconfig.app.json and `vite build`, and records compiler counts, wall
* times and initial/lazy JavaScript gzip bytes. Deterministic counts repeat
* exactly; times depend on the host, so the CPU and load average are recorded
* next to them. A typecheck that fails (for example TS2590, a union too complex
* to represent) is recorded as a result with its first error.
*
* node scripts/scale/measure-feature-shape-growth.ts [--sizes=0,25,100] [--repeats=2] [--write]
*
* `--write` stores the result as config/scale/feature-shape-growth.json.
*/
export const FEATURE_SHAPE_BASELINE_PATH = "config/scale/feature-shape-growth.json";
const SOURCE_FEATURE = "src/features/reference-feature";
const FEATURES_PER_CONTEXT = 200;
const BUILD_INPUTS = [
"src",
"public",
"index.html",
"package.json",
"vite.config.ts",
"tsconfig.base.json",
"tsconfig.app.json",
"scripts/lib/vite-module-inventory.ts",
] as const;
// Platform-owned names that only happen to contain "reference".
const PLATFORM_TOKENS = [
"REFERENCE_EXTERNAL_BEARER",
"REFERENCE_FEATURE_V1",
"supportReference",
] as const;
/** Renames every feature-owned identifier of the reference feature for copy `id`. */
export function renameReferenceFeature(
text: string,
id: string,
context: Readonly<{ id: string; routeRoot: string }>,
): string {
const upper = id.toUpperCase();
let renamed = text;
PLATFORM_TOKENS.forEach((token, index) => {
renamed = renamed.replaceAll(token, `\u0000${index}\u0000`);
});
renamed = renamed
.replaceAll("/examples/reference-resources", `/${context.routeRoot}/${id}-resources`)
.replaceAll("qinv.01k10f7m3w9p6r2c8v5n4x", `qinv.shape.${id}`)
.replaceAll("NoRequest", `${upper}NoRequest`)
.replaceAll("reference-context", context.id)
.replaceAll("REFERENCE", upper)
.replaceAll("Reference", upper)
.replaceAll("reference", id);
PLATFORM_TOKENS.forEach((token, index) => {
renamed = renamed.replaceAll(`\u0000${index}\u0000`, token);
});
return renamed;
}
async function copyRenamed(from: string, to: string, rename: (text: string) => string) {
await mkdir(to, { recursive: true });
for (const entry of await readdir(from, { withFileTypes: true })) {
const source = path.join(from, entry.name);
const target = path.join(to, rename(entry.name));
if (entry.isDirectory()) await copyRenamed(source, target, rename);
else await writeFile(target, rename(await readFile(source, "utf8")));
}
}
async function createFixture(features: number): Promise<string> {
const root = await mkdtemp(path.join(tmpdir(), `feature-shape-${features}-`));
for (const input of BUILD_INPUTS) {
await mkdir(path.dirname(path.join(root, input)), { recursive: true });
await cp(path.resolve(input), path.join(root, input), { recursive: true });
}
await rm(path.join(root, "src/.generated"), { recursive: true, force: true });
await linkFixtureNodeModules(root);
for (let index = 0; index < features; index += 1) {
const contextNumber = Math.floor(index / FEATURES_PER_CONTEXT);
const context = { id: `shape-context-${contextNumber}`, routeRoot: `shape-${contextNumber}` };
if (index % FEATURES_PER_CONTEXT === 0) {
const contextFile = path.join(root, "src/contexts", context.id, "context-module.ts");
await mkdir(path.dirname(contextFile), { recursive: true });
await writeFile(
contextFile,
`export const CONTEXT_MODULE_ID = "${context.id}";\n` +
`export const CONTEXT_APP_ID = "template-app";\n` +
`export const CONTEXT_ROUTE_PATH = "${context.routeRoot}/*";\n` +
`export const CONTEXT_MAX_FEATURES = ${FEATURES_PER_CONTEXT};\n`,
);
}
const id = `s${String(index).padStart(3, "0")}`;
await copyRenamed(
path.resolve(SOURCE_FEATURE),
path.join(root, "src/features", `${id}-feature`),
(text) => renameReferenceFeature(text, id, context),
);
}
return root;
}
function run(
command: string,
args: readonly string[],
cwd: string,
options: Readonly<{ allowFailure?: boolean }> = {},
) {
const started = performance.now();
const result = spawnSync(command, args, {
cwd,
encoding: "utf8",
maxBuffer: 64 * 1024 * 1024,
});
const ms = Math.round(performance.now() - started);
if (result.status !== 0 && !(options.allowFailure && result.stdout.includes("Instantiations:"))) {
throw new Error(
`${path.basename(command)} ${args.join(" ")} failed in ${cwd}:\n${(result.stdout + result.stderr).slice(0, 4_000)}`,
);
}
return { ms, passed: result.status === 0, stdout: result.stdout };
}
const load = () => loadavg().map((value) => Math.round(value * 100) / 100);
async function javascriptGzipBytes(root: string, files: readonly string[]) {
let bytes = 0;
for (const file of files) bytes += gzipSync(await readFile(path.join(root, "dist", file))).byteLength;
return bytes;
}
async function measureOnce(root: string, features: number) {
const loadBefore = load();
run(process.execPath, [path.resolve("scripts/generate-product-feature-index.ts"), `--root=${root}`], root);
// The composed registries throw at module evaluation on a duplicate route,
// operation, schema, namespace or message key: prove the copies compose.
const contracts = (await import(
`${pathToFileURL(path.join(root, "src/features/installed-feature-contracts.ts")).href}?features=${features}`
)) as typeof import("../../src/features/installed-feature-contracts.ts");
const routes = Object.keys(contracts.ROUTE_REGISTRY).length;
const tsc = run(
path.resolve("node_modules/.bin/tsc"),
["--project", path.join(root, "tsconfig.app.json"), "--noEmit", "--extendedDiagnostics"],
root,
// A type error is a result here (the typecheck stops scaling), not a crash.
{ allowFailure: true },
);
await rm(path.join(root, "dist"), { recursive: true, force: true });
const build = run(path.resolve("node_modules/.bin/vite"), ["build", "--logLevel", "error"], root);
const manifest = JSON.parse(await readFile(path.join(root, "dist/.vite/manifest.json"), "utf8"));
const javascript = classifyViteJavascript(manifest);
const diagnostics: TscDiagnostics = parseTscExtendedDiagnostics(tsc.stdout);
return {
addedFeatures: features,
routes,
tsc: {
passed: tsc.passed,
firstError: tsc.stdout.split("\n").find((line) => line.includes("error TS")) ?? null,
files: diagnostics.files,
types: diagnostics.types,
symbols: diagnostics.symbols,
instantiations: diagnostics.instantiations,
memoryKb: diagnostics.memoryKb ?? null,
wallMs: tsc.ms,
},
build: {
wallMs: build.ms,
initialJavascriptFiles: javascript.initialFiles.length,
initialJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.initialFiles),
lazyJavascriptFiles: javascript.lazyFiles.length,
lazyJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.lazyFiles),
},
loadBefore,
loadAfter: load(),
};
}
function version(bin: string): string {
const result = spawnSync(path.resolve("node_modules/.bin", bin), ["--version"], { encoding: "utf8" });
return (result.stdout ?? "").trim();
}
async function main(): Promise<void> {
const argument = (name: string) =>
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3);
const sizes = (argument("sizes") ?? "0,25,100").split(",").map(Number);
const repeats = Number(argument("repeats") ?? "2");
if (!sizes.every((size) => Number.isSafeInteger(size) && size >= 0) || !(repeats >= 1)) {
throw new TypeError("--sizes must be nonnegative integers and --repeats at least 1");
}
const runs = [];
for (const features of sizes) {
const root = await createFixture(features);
try {
for (let repeat = 1; repeat <= repeats; repeat += 1) {
const measured = await measureOnce(root, features);
runs.push({ repeat, ...measured });
process.stdout.write(
`features +${features} (repeat ${repeat}): routes=${measured.routes} ` +
`instantiations=${measured.tsc.instantiations} tsc=${measured.tsc.wallMs}ms ` +
`build=${measured.build.wallMs}ms initial=${measured.build.initialJavascriptGzipBytes}B ` +
`lazy=${measured.build.lazyJavascriptGzipBytes}B load=${measured.loadBefore.join("/")}` +
`${measured.tsc.passed ? "" : ` typecheck FAILED: ${measured.tsc.firstError}`}\n`,
);
// Compiler results repeat exactly; a failed typecheck costs minutes and
// gigabytes again for no new information.
if (!measured.tsc.passed) break;
}
} finally {
await rm(root, { recursive: true, force: true });
}
}
const cpu = cpus();
const result = {
schemaVersion: 1,
measuredAt: new Date().toISOString(),
command: `node scripts/scale/measure-feature-shape-growth.ts --sizes=${sizes.join(",")} --repeats=${repeats}`,
environment: {
node: process.versions.node,
platform: process.platform,
arch: process.arch,
cpuModel: cpu[0]?.model.trim() ?? "unknown",
logicalCpus: cpu.length,
availableParallelism: availableParallelism(),
typescript: version("tsc"),
vite: version("vite"),
},
sourceFeature: SOURCE_FEATURE,
runs,
};
const output = process.argv.includes("--write")
? FEATURE_SHAPE_BASELINE_PATH
: "artifacts/scale/feature-shape-growth.json";
await mkdir(path.dirname(output), { recursive: true });
await writeFile(output, `${JSON.stringify(result, null, 2)}\n`);
process.stdout.write(`feature-shape growth: ${output}\n`);
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
await main();
}
+163 -43
View File
@@ -1,10 +1,18 @@
import { spawn } from "node:child_process";
import { mkdir, readFile } from "node:fs/promises";
import { performance } from "node:perf_hooks";
import { cpus, loadavg } from "node:os";
import process from "node:process";
import { chromium } from "@playwright/test";
import { chromium, type Browser } from "@playwright/test";
import {
assertPortUnused,
assertServedRelease,
DEFAULT_LAB_PORT,
DEFAULT_LAB_RUNS,
previewArguments,
summarizeSamples,
} from "./lib/lab-performance.ts";
import { evaluateLabBudget } from "./lib/performance-budgets.ts";
import { ROUTE_REGISTRY } from "../src/features/installed-feature-contracts.ts";
import { labPerformanceArtifactSchema } from "./contracts/release-artifacts.ts";
@@ -13,6 +21,9 @@ import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
type ContractPerformanceEvidence = {
lcpMs: number;
cls: number;
clickAt: number | null;
headingAt: number | null;
eventDurationMs: number | null;
};
type ContractPerformanceWindow = Window & {
@@ -24,15 +35,37 @@ type LayoutShiftEntry = PerformanceEntry & {
value: number;
};
const server = spawn(
"corepack",
["pnpm", "preview", "--host", "127.0.0.1", "--port", "4173"],
{ stdio: "ignore" },
);
const baseUrl = "http://127.0.0.1:4173";
const argument = (name: string) =>
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3);
const port = Number(process.env.LAB_PERFORMANCE_PORT ?? DEFAULT_LAB_PORT);
const runs = Number(argument("runs") ?? DEFAULT_LAB_RUNS);
if (!Number.isSafeInteger(port) || port <= 0 || !Number.isSafeInteger(runs) || runs < 1) {
throw new TypeError("LAB_PERFORMANCE_PORT must be a port and --runs a positive integer.");
}
const baseUrl = `http://127.0.0.1:${port}`;
const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM;
const targetLabel = target.navigationLabel;
if (!targetLabel) {
throw new Error("Performance route must be present in navigation.");
}
// 11-07. A server already on the port used to be measured in place of this
// build: vite moved to another port and the gate never noticed.
await assertPortUnused(baseUrl);
// Its own process group: `corepack` starts pnpm, which starts vite, and a
// signal to `corepack` alone left vite serving the port after the gate ended.
const server = spawn("corepack", [...previewArguments(port)], {
stdio: "ignore",
detached: true,
});
let serverExit: string | null = null;
server.on("exit", (code, signal) => {
serverExit = `preview server exited (${signal ?? code})`;
});
async function waitForServer() {
for (let attempt = 0; attempt < 50; attempt += 1) {
for (let attempt = 0; attempt < 100; attempt += 1) {
if (serverExit) throw new Error(serverExit);
try {
const response = await fetch(baseUrl);
if (response.ok) return;
@@ -44,19 +77,14 @@ async function waitForServer() {
throw new Error("Preview server did not become ready.");
}
try {
await waitForServer();
const release = JSON.parse(
await readFile("dist/release-manifest.json", "utf8"),
);
const thresholds = JSON.parse(
await readFile("config/performance/budgets.json", "utf8"),
).lab;
const browser = await chromium.launch();
type RunMetrics = { lcpMs: number; cls: number; namedInteractionMs: number; eventDurationMs: number | null };
async function measureOnce(browser: Browser): Promise<RunMetrics> {
// Every run is a cold, isolated context.
const context = await browser.newContext({
viewport: { width: 1280, height: 720 },
});
try {
const context = await browser.newContext({
viewport: { width: 1280, height: 720 },
});
const page = await context.newPage();
const cdp = await context.newCDPSession(page);
await cdp.send("Network.enable");
@@ -68,8 +96,14 @@ try {
connectionType: "cellular4g",
});
await cdp.send("Emulation.setCPUThrottlingRate", { rate: 4 });
await page.addInitScript(() => {
const evidence = { lcpMs: 0, cls: 0 };
await page.addInitScript((title: string) => {
const evidence: ContractPerformanceEvidence = {
lcpMs: 0,
cls: 0,
clickAt: null,
headingAt: null,
eventDurationMs: null,
};
(window as ContractPerformanceWindow).__contractPerformance = evidence;
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) evidence.lcpMs = entry.startTime;
@@ -82,34 +116,97 @@ try {
}
}
}).observe({ type: "layout-shift", buffered: true });
});
// 11-07. The interaction is timed by the page's own clock, from the
// click event to the target heading entering the DOM, so Playwright's
// round trips are not in it. The Event Timing entry of the click (the
// INP input) is recorded next to it.
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
if (entry.name === "click") {
evidence.eventDurationMs = Math.max(evidence.eventDurationMs ?? 0, entry.duration);
}
}
}).observe({ type: "event", buffered: true, durationThreshold: 16 } as PerformanceObserverInit);
document.addEventListener(
"click",
(event) => {
evidence.clickAt ??= event.timeStamp;
},
{ capture: true },
);
new MutationObserver((_records, observer) => {
if (evidence.clickAt === null) return;
const found = [...document.querySelectorAll("h1, h2")].some(
(heading) => heading.textContent?.trim() === title,
);
if (found) {
evidence.headingAt = performance.now();
observer.disconnect();
}
}).observe(document, { childList: true, subtree: true, characterData: true });
}, target.title);
await page.goto(baseUrl, { waitUntil: "networkidle" });
const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM;
const targetLabel = target.navigationLabel;
if (!targetLabel) {
throw new Error("Performance route must be present in navigation.");
}
const interactionStarted = performance.now();
// Playwright matches accessible names by substring, so the navigation entry
// "플랫폼 구성" also matched the home page's "플랫폼 구성 보기" call to
// action and the locator resolved to two links. That is a strict-mode
// violation before the first measurement is taken, so no lab performance
// evidence could be produced at all — the run failed for an ambiguous
// selector rather than for anything about performance.
// action; `exact` keeps the locator on the navigation link.
await page.getByRole("link", { name: targetLabel, exact: true }).click();
await page.getByRole("heading", { name: target.title }).waitFor();
const namedInteractionMs = performance.now() - interactionStarted;
await page.waitForFunction(
() => (window as ContractPerformanceWindow).__contractPerformance?.headingAt !== null,
);
const paint = await page.evaluate(
() => (window as ContractPerformanceWindow).__contractPerformance,
);
if (!paint) {
if (!paint || paint.clickAt === null || paint.headingAt === null) {
throw new Error("Browser performance evidence was not initialized.");
}
return {
lcpMs: Math.round(paint.lcpMs),
cls: Number(paint.cls.toFixed(4)),
namedInteractionMs: Math.round(paint.headingAt - paint.clickAt),
eventDurationMs:
paint.eventDurationMs === null ? null : Math.round(paint.eventDurationMs),
};
} finally {
await context.close();
}
}
const load = () => loadavg().map((value) => Math.round(value * 100) / 100);
try {
await waitForServer();
const release = JSON.parse(
await readFile("dist/release-manifest.json", "utf8"),
);
await assertServedRelease(baseUrl, release);
const thresholds = JSON.parse(
await readFile("config/performance/budgets.json", "utf8"),
).lab;
const browser = await chromium.launch();
try {
const loadBefore = load();
const samples: RunMetrics[] = [];
for (let run = 0; run < runs; run += 1) samples.push(await measureOnce(browser));
const loadAfter = load();
// The served build must not have changed under the runs either.
await assertServedRelease(baseUrl, release);
const lcp = summarizeSamples(samples.map((sample) => sample.lcpMs));
const cls = summarizeSamples(samples.map((sample) => sample.cls));
const interaction = summarizeSamples(
samples.map((sample) => sample.namedInteractionMs),
);
const eventDurations = samples.flatMap((sample) =>
sample.eventDurationMs === null ? [] : [sample.eventDurationMs],
);
const contextMetadata = {
runner: {
platform: process.platform,
architecture: process.arch,
nodeVersion: process.version,
cpuModel: cpus()[0]?.model.trim() ?? "unknown",
logicalCpus: cpus().length,
loadAverage: { before: loadBefore, after: loadAfter },
},
browser: { name: "chromium", version: await browser.version() },
viewport: { width: 1280, height: 720 },
@@ -121,12 +218,29 @@ try {
},
cpu: { throttlingRate: 4 },
cache: { state: "cold", isolation: "new-browser-context" },
build: { buildId: release.buildId, releaseId: release.releaseId },
build: {
buildId: release.buildId,
releaseId: release.releaseId,
served: `${baseUrl}/release-manifest.json`,
},
// 11-07. The budget is judged on the median of cold runs; every run and
// the spread stay in the evidence.
sampling: {
runs,
statistic: "median",
interactionClock: "page: click event to target heading in DOM",
lcpMs: lcp,
cls,
namedInteractionMs: interaction,
clickEventDurationMs: eventDurations.length
? summarizeSamples(eventDurations)
: null,
},
};
const metrics = {
lcpMs: Math.round(paint.lcpMs),
cls: Number(paint.cls.toFixed(4)),
namedInteractionMs: Math.round(namedInteractionMs),
lcpMs: lcp.median,
cls: cls.median,
namedInteractionMs: interaction.median,
};
const result = evaluateLabBudget(
{ context: contextMetadata, metrics },
@@ -164,14 +278,20 @@ try {
},
});
if (!passed) {
throw new Error(`Lab performance failed: ${JSON.stringify(metrics)}`);
throw new Error(
`Lab performance failed (median of ${runs}): ${JSON.stringify(metrics)}; ` +
`interaction samples ${interaction.samples.join(",")}`,
);
}
process.stdout.write(
`Lab performance: PASS (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms)\n`,
`Lab performance: PASS, median of ${runs} (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms ` +
`[${interaction.min}-${interaction.max}])\n`,
);
} finally {
await browser.close();
}
} finally {
server.kill("SIGTERM");
if (server.pid !== undefined && serverExit === null) {
process.kill(-server.pid, "SIGTERM");
}
}
+5
View File
@@ -51,6 +51,11 @@ const SAMPLE_EXTRAS: Readonly<Record<string, FeatureExtras>> = Object.freeze({
"tests/component/product-feature-switch.test.tsx",
"tests/e2e/reference-form.spec.ts",
"tests/e2e/reference-route.spec.ts",
"tests/e2e/reference-save.spec.ts",
// 11-04: the growth measurement copies this sample feature.
"tests/unit/feature-shape-growth.test.ts",
"scripts/scale/measure-feature-shape-growth.ts",
"config/scale/feature-shape-growth.json",
"tests/mocks",
"tests/fixtures/typecheck/invalid-feature-input.ts",
"tests/fixtures/typecheck/invalid-reference-operation.ts",