Merge lane/J: test system (10-02, 10-04~10-07, 11-04~11-08, F-2, F-4, F-12, 00-03)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -41,7 +41,10 @@ import {
|
||||
releaseCandidateManifestSchema,
|
||||
} from "./lib/release-candidate.ts";
|
||||
import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
||||
import { linkFixtureNodeModules } from "./lib/fixture-node-modules.ts";
|
||||
import {
|
||||
FIXTURE_PNPM_ENVIRONMENT,
|
||||
linkFixtureNodeModules,
|
||||
} from "./lib/fixture-node-modules.ts";
|
||||
|
||||
const repositoryRoot = process.cwd();
|
||||
const fixtureRoot = await mkdtemp(path.join(tmpdir(), "provider-exact-five-fixture-"));
|
||||
@@ -74,6 +77,7 @@ try {
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
env: {
|
||||
...process.env,
|
||||
...FIXTURE_PNPM_ENVIRONMENT,
|
||||
CI: "true",
|
||||
VITE_BUILD_ID: "provider-exact-five-fixture",
|
||||
VITE_COMMIT_SHA: revision,
|
||||
|
||||
@@ -281,17 +281,27 @@ if (!scenarioOnly) {
|
||||
const sourceFiles = (await filesBelow(sourceRoot)).filter(
|
||||
(file) => fixtureMode || !file.split(path.sep).includes("fixtures"),
|
||||
);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
for (const file of sourceFiles) {
|
||||
if (!/\.(?:ts|tsx|fixture|txt)$/.test(file)) continue;
|
||||
const source = await readFile(file, "utf8");
|
||||
facts.scannedFiles += 1;
|
||||
if (/\b(?:test|it|describe)(?:\.describe)?\.only\s*\(/.test(source)) {
|
||||
failures.push(`${file}: focused test (.only) runs alone and hides the rest of the suite`);
|
||||
}
|
||||
const skipPattern = /\b(?:test|it|describe)(?:\.describe)?\.(?:skip|fixme)\s*\(/g;
|
||||
if (skipPattern.test(source)) {
|
||||
const quarantine =
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=\d{4}-\d{2}-\d{2}\)/;
|
||||
if (!quarantine.test(source)) {
|
||||
const quarantines = [
|
||||
...source.matchAll(
|
||||
/quarantine\(owner=[^)]+,\s*defect=[^)]+,\s*expires=(\d{4}-\d{2}-\d{2})\)/g,
|
||||
),
|
||||
];
|
||||
if (quarantines.length === 0) {
|
||||
failures.push(`${file}: skip/fixme lacks owned expiring quarantine`);
|
||||
}
|
||||
for (const [, expires = ""] of quarantines) {
|
||||
if (expires < today) failures.push(`${file}: quarantine expired on ${expires}`);
|
||||
}
|
||||
}
|
||||
const wholeUiMask =
|
||||
/\bmask\s*:\s*\[[\s\S]{0,240}(?:locator|getByRole)\s*\(\s*["'](?:html|body|main|application|document)["']/i;
|
||||
@@ -370,6 +380,17 @@ if (!fixtureMode && !scenarioOnly) {
|
||||
failures.push(`playwright.config.ts missing release evidence token ${token}`);
|
||||
}
|
||||
}
|
||||
// playwright.dev.config.ts spreads playwright.config.ts.
|
||||
for (const config of [
|
||||
"playwright.config.ts",
|
||||
"playwright.visual.config.ts",
|
||||
"playwright.storybook.config.ts",
|
||||
"playwright.capabilities.config.ts",
|
||||
]) {
|
||||
if (!(await readFile(config, "utf8")).includes("forbidOnly: true")) {
|
||||
failures.push(`${config}: missing forbidOnly: true, so a committed test.only passes the gate`);
|
||||
}
|
||||
}
|
||||
|
||||
const e2eFiles = (await filesBelow("tests/e2e")).filter((file) =>
|
||||
/\.spec\.ts$/.test(file),
|
||||
|
||||
@@ -27,6 +27,20 @@ import path from "node:path";
|
||||
* the linked directories. It is rewritten to name the fixture root instead:
|
||||
* the packages are the repository's own, already validated.
|
||||
*/
|
||||
/**
|
||||
* F-4. The environment for every `pnpm` a linked fixture runs. pnpm verifies
|
||||
* the install before a script and, when the fixture's copied state disagrees
|
||||
* with its own settings (for example `CI=true` against an install made
|
||||
* without it), installs. `.pnpm` and `.bin` are links into the repository's
|
||||
* node_modules, so that install rewrote the shared `.pnpm/lock.yaml` and bin
|
||||
* shims. A fixture started through `pnpm <script>` inherits this setting from
|
||||
* pnpm; one started directly (`node scripts/run-vitest.ts run <file>`) did not.
|
||||
* The fixture uses the repository's packages as they are; it never installs.
|
||||
*/
|
||||
export const FIXTURE_PNPM_ENVIRONMENT = Object.freeze({
|
||||
pnpm_config_verify_deps_before_run: "false",
|
||||
});
|
||||
|
||||
export async function linkFixtureNodeModules(
|
||||
fixtureRoot: string,
|
||||
sourceRoot: string = process.cwd(),
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* 11-07. The pieces of the lab performance gate (`scripts/test-performance.ts`)
|
||||
* that decide what is measured and whether the numbers can be trusted.
|
||||
*/
|
||||
|
||||
export const DEFAULT_LAB_PORT = 4173;
|
||||
export const DEFAULT_LAB_RUNS = 5;
|
||||
|
||||
/**
|
||||
* `vite preview` arguments. Without `--strictPort` vite moved to the next free
|
||||
* port when the requested one was taken, and the gate measured whatever server
|
||||
* already listened there.
|
||||
*/
|
||||
export function previewArguments(port: number): readonly string[] {
|
||||
return [
|
||||
"pnpm",
|
||||
"preview",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
String(port),
|
||||
"--strictPort",
|
||||
];
|
||||
}
|
||||
|
||||
export type SampleSummary = Readonly<{
|
||||
median: number;
|
||||
min: number;
|
||||
max: number;
|
||||
samples: readonly number[];
|
||||
}>;
|
||||
|
||||
export function summarizeSamples(samples: readonly number[]): SampleSummary {
|
||||
if (samples.length === 0) throw new RangeError("no samples to summarize");
|
||||
const sorted = [...samples].sort((left, right) => left - right);
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
const median =
|
||||
sorted.length % 2 === 1
|
||||
? sorted[middle]!
|
||||
: (sorted[middle - 1]! + sorted[middle]!) / 2;
|
||||
return Object.freeze({
|
||||
median,
|
||||
min: sorted[0]!,
|
||||
max: sorted.at(-1)!,
|
||||
samples: Object.freeze([...samples]),
|
||||
});
|
||||
}
|
||||
|
||||
/** Fails when anything already answers where the preview server will listen. */
|
||||
export async function assertPortUnused(
|
||||
baseUrl: string,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
): Promise<void> {
|
||||
let answered = false;
|
||||
try {
|
||||
await fetchImpl(baseUrl, { signal: AbortSignal.timeout(1_000) });
|
||||
answered = true;
|
||||
} catch {
|
||||
// Nothing is listening: the expected state.
|
||||
}
|
||||
if (answered) {
|
||||
throw new Error(
|
||||
`${baseUrl} already answers before the preview server starts; another server holds the port.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares the release the server actually serves with the local build that
|
||||
* is being measured, so the numbers belong to that build.
|
||||
*/
|
||||
export async function assertServedRelease(
|
||||
baseUrl: string,
|
||||
expected: Readonly<{ buildId: string; releaseId: string }>,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
): Promise<void> {
|
||||
const response = await fetchImpl(new URL("/release-manifest.json", baseUrl), {
|
||||
cache: "no-store",
|
||||
});
|
||||
const served = response.ok
|
||||
? ((await response.json()) as Partial<Record<"buildId" | "releaseId", unknown>>)
|
||||
: {};
|
||||
if (served.buildId !== expected.buildId || served.releaseId !== expected.releaseId) {
|
||||
throw new Error(
|
||||
`${baseUrl} serves build ${String(served.buildId)}/${String(served.releaseId)}, ` +
|
||||
`not the measured ${expected.buildId}/${expected.releaseId}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,10 @@ import {
|
||||
loadCiGateContract,
|
||||
parseCiGateContract,
|
||||
} from "../contracts/ci-gates.ts";
|
||||
import { linkFixtureNodeModules } from "./fixture-node-modules.ts";
|
||||
import {
|
||||
FIXTURE_PNPM_ENVIRONMENT,
|
||||
linkFixtureNodeModules,
|
||||
} from "./fixture-node-modules.ts";
|
||||
import { generateCiWorkflow } from "../generate-ci-workflow.ts";
|
||||
import { updateCiContractBaseline } from "../update-ci-contract-baseline.ts";
|
||||
|
||||
@@ -189,7 +192,7 @@ export function runRemovalFixtureCheck(
|
||||
const status = spawnSync(process.execPath, [pnpmCli, script, ...extra], {
|
||||
cwd: root,
|
||||
stdio: "inherit",
|
||||
env: process.env,
|
||||
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT },
|
||||
}).status;
|
||||
return status === 0
|
||||
? { name, passed: true }
|
||||
@@ -243,7 +246,7 @@ export function runRemovalFixturePnpm(
|
||||
return spawnSync(process.execPath, [pnpmCli, script, ...extra], {
|
||||
cwd: root,
|
||||
stdio: "inherit",
|
||||
env: { ...process.env, ...env },
|
||||
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, ...env },
|
||||
}).status === 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { availableParallelism, cpus, loadavg, tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { gzipSync } from "node:zlib";
|
||||
|
||||
import { classifyViteJavascript } from "../lib/classify-vite-bundle.ts";
|
||||
import { linkFixtureNodeModules } from "../lib/fixture-node-modules.ts";
|
||||
import { parseTscExtendedDiagnostics, type TscDiagnostics } from "./collect-tsc-diagnostics.ts";
|
||||
|
||||
/**
|
||||
* 11-04 (D-18). Cost of one app as its feature count grows, measured with
|
||||
* features shaped like the real one: `src/features/reference-feature` copied N
|
||||
* times with every feature-owned identifier renamed (feature id, route ids and
|
||||
* paths, API operations, schemas, query namespace, invalidation topic,
|
||||
* messages). The copies live in contexts of their own (200 features each, the
|
||||
* context ceiling) so the reference context is unchanged.
|
||||
*
|
||||
* For each N the script runs the product generator, `tsc --extendedDiagnostics`
|
||||
* over tsconfig.app.json and `vite build`, and records compiler counts, wall
|
||||
* times and initial/lazy JavaScript gzip bytes. Deterministic counts repeat
|
||||
* exactly; times depend on the host, so the CPU and load average are recorded
|
||||
* next to them. A typecheck that fails (for example TS2590, a union too complex
|
||||
* to represent) is recorded as a result with its first error.
|
||||
*
|
||||
* node scripts/scale/measure-feature-shape-growth.ts [--sizes=0,25,100] [--repeats=2] [--write]
|
||||
*
|
||||
* `--write` stores the result as config/scale/feature-shape-growth.json.
|
||||
*/
|
||||
export const FEATURE_SHAPE_BASELINE_PATH = "config/scale/feature-shape-growth.json";
|
||||
|
||||
const SOURCE_FEATURE = "src/features/reference-feature";
|
||||
const FEATURES_PER_CONTEXT = 200;
|
||||
const BUILD_INPUTS = [
|
||||
"src",
|
||||
"public",
|
||||
"index.html",
|
||||
"package.json",
|
||||
"vite.config.ts",
|
||||
"tsconfig.base.json",
|
||||
"tsconfig.app.json",
|
||||
"scripts/lib/vite-module-inventory.ts",
|
||||
] as const;
|
||||
// Platform-owned names that only happen to contain "reference".
|
||||
const PLATFORM_TOKENS = [
|
||||
"REFERENCE_EXTERNAL_BEARER",
|
||||
"REFERENCE_FEATURE_V1",
|
||||
"supportReference",
|
||||
] as const;
|
||||
|
||||
/** Renames every feature-owned identifier of the reference feature for copy `id`. */
|
||||
export function renameReferenceFeature(
|
||||
text: string,
|
||||
id: string,
|
||||
context: Readonly<{ id: string; routeRoot: string }>,
|
||||
): string {
|
||||
const upper = id.toUpperCase();
|
||||
let renamed = text;
|
||||
PLATFORM_TOKENS.forEach((token, index) => {
|
||||
renamed = renamed.replaceAll(token, `\u0000${index}\u0000`);
|
||||
});
|
||||
renamed = renamed
|
||||
.replaceAll("/examples/reference-resources", `/${context.routeRoot}/${id}-resources`)
|
||||
.replaceAll("qinv.01k10f7m3w9p6r2c8v5n4x", `qinv.shape.${id}`)
|
||||
.replaceAll("NoRequest", `${upper}NoRequest`)
|
||||
.replaceAll("reference-context", context.id)
|
||||
.replaceAll("REFERENCE", upper)
|
||||
.replaceAll("Reference", upper)
|
||||
.replaceAll("reference", id);
|
||||
PLATFORM_TOKENS.forEach((token, index) => {
|
||||
renamed = renamed.replaceAll(`\u0000${index}\u0000`, token);
|
||||
});
|
||||
return renamed;
|
||||
}
|
||||
|
||||
async function copyRenamed(from: string, to: string, rename: (text: string) => string) {
|
||||
await mkdir(to, { recursive: true });
|
||||
for (const entry of await readdir(from, { withFileTypes: true })) {
|
||||
const source = path.join(from, entry.name);
|
||||
const target = path.join(to, rename(entry.name));
|
||||
if (entry.isDirectory()) await copyRenamed(source, target, rename);
|
||||
else await writeFile(target, rename(await readFile(source, "utf8")));
|
||||
}
|
||||
}
|
||||
|
||||
async function createFixture(features: number): Promise<string> {
|
||||
const root = await mkdtemp(path.join(tmpdir(), `feature-shape-${features}-`));
|
||||
for (const input of BUILD_INPUTS) {
|
||||
await mkdir(path.dirname(path.join(root, input)), { recursive: true });
|
||||
await cp(path.resolve(input), path.join(root, input), { recursive: true });
|
||||
}
|
||||
await rm(path.join(root, "src/.generated"), { recursive: true, force: true });
|
||||
await linkFixtureNodeModules(root);
|
||||
for (let index = 0; index < features; index += 1) {
|
||||
const contextNumber = Math.floor(index / FEATURES_PER_CONTEXT);
|
||||
const context = { id: `shape-context-${contextNumber}`, routeRoot: `shape-${contextNumber}` };
|
||||
if (index % FEATURES_PER_CONTEXT === 0) {
|
||||
const contextFile = path.join(root, "src/contexts", context.id, "context-module.ts");
|
||||
await mkdir(path.dirname(contextFile), { recursive: true });
|
||||
await writeFile(
|
||||
contextFile,
|
||||
`export const CONTEXT_MODULE_ID = "${context.id}";\n` +
|
||||
`export const CONTEXT_APP_ID = "template-app";\n` +
|
||||
`export const CONTEXT_ROUTE_PATH = "${context.routeRoot}/*";\n` +
|
||||
`export const CONTEXT_MAX_FEATURES = ${FEATURES_PER_CONTEXT};\n`,
|
||||
);
|
||||
}
|
||||
const id = `s${String(index).padStart(3, "0")}`;
|
||||
await copyRenamed(
|
||||
path.resolve(SOURCE_FEATURE),
|
||||
path.join(root, "src/features", `${id}-feature`),
|
||||
(text) => renameReferenceFeature(text, id, context),
|
||||
);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function run(
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
cwd: string,
|
||||
options: Readonly<{ allowFailure?: boolean }> = {},
|
||||
) {
|
||||
const started = performance.now();
|
||||
const result = spawnSync(command, args, {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
const ms = Math.round(performance.now() - started);
|
||||
if (result.status !== 0 && !(options.allowFailure && result.stdout.includes("Instantiations:"))) {
|
||||
throw new Error(
|
||||
`${path.basename(command)} ${args.join(" ")} failed in ${cwd}:\n${(result.stdout + result.stderr).slice(0, 4_000)}`,
|
||||
);
|
||||
}
|
||||
return { ms, passed: result.status === 0, stdout: result.stdout };
|
||||
}
|
||||
|
||||
const load = () => loadavg().map((value) => Math.round(value * 100) / 100);
|
||||
|
||||
async function javascriptGzipBytes(root: string, files: readonly string[]) {
|
||||
let bytes = 0;
|
||||
for (const file of files) bytes += gzipSync(await readFile(path.join(root, "dist", file))).byteLength;
|
||||
return bytes;
|
||||
}
|
||||
|
||||
async function measureOnce(root: string, features: number) {
|
||||
const loadBefore = load();
|
||||
run(process.execPath, [path.resolve("scripts/generate-product-feature-index.ts"), `--root=${root}`], root);
|
||||
// The composed registries throw at module evaluation on a duplicate route,
|
||||
// operation, schema, namespace or message key: prove the copies compose.
|
||||
const contracts = (await import(
|
||||
`${pathToFileURL(path.join(root, "src/features/installed-feature-contracts.ts")).href}?features=${features}`
|
||||
)) as typeof import("../../src/features/installed-feature-contracts.ts");
|
||||
const routes = Object.keys(contracts.ROUTE_REGISTRY).length;
|
||||
const tsc = run(
|
||||
path.resolve("node_modules/.bin/tsc"),
|
||||
["--project", path.join(root, "tsconfig.app.json"), "--noEmit", "--extendedDiagnostics"],
|
||||
root,
|
||||
// A type error is a result here (the typecheck stops scaling), not a crash.
|
||||
{ allowFailure: true },
|
||||
);
|
||||
await rm(path.join(root, "dist"), { recursive: true, force: true });
|
||||
const build = run(path.resolve("node_modules/.bin/vite"), ["build", "--logLevel", "error"], root);
|
||||
const manifest = JSON.parse(await readFile(path.join(root, "dist/.vite/manifest.json"), "utf8"));
|
||||
const javascript = classifyViteJavascript(manifest);
|
||||
const diagnostics: TscDiagnostics = parseTscExtendedDiagnostics(tsc.stdout);
|
||||
return {
|
||||
addedFeatures: features,
|
||||
routes,
|
||||
tsc: {
|
||||
passed: tsc.passed,
|
||||
firstError: tsc.stdout.split("\n").find((line) => line.includes("error TS")) ?? null,
|
||||
files: diagnostics.files,
|
||||
types: diagnostics.types,
|
||||
symbols: diagnostics.symbols,
|
||||
instantiations: diagnostics.instantiations,
|
||||
memoryKb: diagnostics.memoryKb ?? null,
|
||||
wallMs: tsc.ms,
|
||||
},
|
||||
build: {
|
||||
wallMs: build.ms,
|
||||
initialJavascriptFiles: javascript.initialFiles.length,
|
||||
initialJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.initialFiles),
|
||||
lazyJavascriptFiles: javascript.lazyFiles.length,
|
||||
lazyJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.lazyFiles),
|
||||
},
|
||||
loadBefore,
|
||||
loadAfter: load(),
|
||||
};
|
||||
}
|
||||
|
||||
function version(bin: string): string {
|
||||
const result = spawnSync(path.resolve("node_modules/.bin", bin), ["--version"], { encoding: "utf8" });
|
||||
return (result.stdout ?? "").trim();
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argument = (name: string) =>
|
||||
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
const sizes = (argument("sizes") ?? "0,25,100").split(",").map(Number);
|
||||
const repeats = Number(argument("repeats") ?? "2");
|
||||
if (!sizes.every((size) => Number.isSafeInteger(size) && size >= 0) || !(repeats >= 1)) {
|
||||
throw new TypeError("--sizes must be nonnegative integers and --repeats at least 1");
|
||||
}
|
||||
const runs = [];
|
||||
for (const features of sizes) {
|
||||
const root = await createFixture(features);
|
||||
try {
|
||||
for (let repeat = 1; repeat <= repeats; repeat += 1) {
|
||||
const measured = await measureOnce(root, features);
|
||||
runs.push({ repeat, ...measured });
|
||||
process.stdout.write(
|
||||
`features +${features} (repeat ${repeat}): routes=${measured.routes} ` +
|
||||
`instantiations=${measured.tsc.instantiations} tsc=${measured.tsc.wallMs}ms ` +
|
||||
`build=${measured.build.wallMs}ms initial=${measured.build.initialJavascriptGzipBytes}B ` +
|
||||
`lazy=${measured.build.lazyJavascriptGzipBytes}B load=${measured.loadBefore.join("/")}` +
|
||||
`${measured.tsc.passed ? "" : ` typecheck FAILED: ${measured.tsc.firstError}`}\n`,
|
||||
);
|
||||
// Compiler results repeat exactly; a failed typecheck costs minutes and
|
||||
// gigabytes again for no new information.
|
||||
if (!measured.tsc.passed) break;
|
||||
}
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
const cpu = cpus();
|
||||
const result = {
|
||||
schemaVersion: 1,
|
||||
measuredAt: new Date().toISOString(),
|
||||
command: `node scripts/scale/measure-feature-shape-growth.ts --sizes=${sizes.join(",")} --repeats=${repeats}`,
|
||||
environment: {
|
||||
node: process.versions.node,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
cpuModel: cpu[0]?.model.trim() ?? "unknown",
|
||||
logicalCpus: cpu.length,
|
||||
availableParallelism: availableParallelism(),
|
||||
typescript: version("tsc"),
|
||||
vite: version("vite"),
|
||||
},
|
||||
sourceFeature: SOURCE_FEATURE,
|
||||
runs,
|
||||
};
|
||||
const output = process.argv.includes("--write")
|
||||
? FEATURE_SHAPE_BASELINE_PATH
|
||||
: "artifacts/scale/feature-shape-growth.json";
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
await writeFile(output, `${JSON.stringify(result, null, 2)}\n`);
|
||||
process.stdout.write(`feature-shape growth: ${output}\n`);
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
await main();
|
||||
}
|
||||
+163
-43
@@ -1,10 +1,18 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, readFile } from "node:fs/promises";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { cpus, loadavg } from "node:os";
|
||||
import process from "node:process";
|
||||
|
||||
import { chromium } from "@playwright/test";
|
||||
import { chromium, type Browser } from "@playwright/test";
|
||||
|
||||
import {
|
||||
assertPortUnused,
|
||||
assertServedRelease,
|
||||
DEFAULT_LAB_PORT,
|
||||
DEFAULT_LAB_RUNS,
|
||||
previewArguments,
|
||||
summarizeSamples,
|
||||
} from "./lib/lab-performance.ts";
|
||||
import { evaluateLabBudget } from "./lib/performance-budgets.ts";
|
||||
import { ROUTE_REGISTRY } from "../src/features/installed-feature-contracts.ts";
|
||||
import { labPerformanceArtifactSchema } from "./contracts/release-artifacts.ts";
|
||||
@@ -13,6 +21,9 @@ import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
|
||||
type ContractPerformanceEvidence = {
|
||||
lcpMs: number;
|
||||
cls: number;
|
||||
clickAt: number | null;
|
||||
headingAt: number | null;
|
||||
eventDurationMs: number | null;
|
||||
};
|
||||
|
||||
type ContractPerformanceWindow = Window & {
|
||||
@@ -24,15 +35,37 @@ type LayoutShiftEntry = PerformanceEntry & {
|
||||
value: number;
|
||||
};
|
||||
|
||||
const server = spawn(
|
||||
"corepack",
|
||||
["pnpm", "preview", "--host", "127.0.0.1", "--port", "4173"],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
const baseUrl = "http://127.0.0.1:4173";
|
||||
const argument = (name: string) =>
|
||||
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
const port = Number(process.env.LAB_PERFORMANCE_PORT ?? DEFAULT_LAB_PORT);
|
||||
const runs = Number(argument("runs") ?? DEFAULT_LAB_RUNS);
|
||||
if (!Number.isSafeInteger(port) || port <= 0 || !Number.isSafeInteger(runs) || runs < 1) {
|
||||
throw new TypeError("LAB_PERFORMANCE_PORT must be a port and --runs a positive integer.");
|
||||
}
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM;
|
||||
const targetLabel = target.navigationLabel;
|
||||
if (!targetLabel) {
|
||||
throw new Error("Performance route must be present in navigation.");
|
||||
}
|
||||
|
||||
// 11-07. A server already on the port used to be measured in place of this
|
||||
// build: vite moved to another port and the gate never noticed.
|
||||
await assertPortUnused(baseUrl);
|
||||
// Its own process group: `corepack` starts pnpm, which starts vite, and a
|
||||
// signal to `corepack` alone left vite serving the port after the gate ended.
|
||||
const server = spawn("corepack", [...previewArguments(port)], {
|
||||
stdio: "ignore",
|
||||
detached: true,
|
||||
});
|
||||
let serverExit: string | null = null;
|
||||
server.on("exit", (code, signal) => {
|
||||
serverExit = `preview server exited (${signal ?? code})`;
|
||||
});
|
||||
|
||||
async function waitForServer() {
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
if (serverExit) throw new Error(serverExit);
|
||||
try {
|
||||
const response = await fetch(baseUrl);
|
||||
if (response.ok) return;
|
||||
@@ -44,19 +77,14 @@ async function waitForServer() {
|
||||
throw new Error("Preview server did not become ready.");
|
||||
}
|
||||
|
||||
try {
|
||||
await waitForServer();
|
||||
const release = JSON.parse(
|
||||
await readFile("dist/release-manifest.json", "utf8"),
|
||||
);
|
||||
const thresholds = JSON.parse(
|
||||
await readFile("config/performance/budgets.json", "utf8"),
|
||||
).lab;
|
||||
const browser = await chromium.launch();
|
||||
type RunMetrics = { lcpMs: number; cls: number; namedInteractionMs: number; eventDurationMs: number | null };
|
||||
|
||||
async function measureOnce(browser: Browser): Promise<RunMetrics> {
|
||||
// Every run is a cold, isolated context.
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
});
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const cdp = await context.newCDPSession(page);
|
||||
await cdp.send("Network.enable");
|
||||
@@ -68,8 +96,14 @@ try {
|
||||
connectionType: "cellular4g",
|
||||
});
|
||||
await cdp.send("Emulation.setCPUThrottlingRate", { rate: 4 });
|
||||
await page.addInitScript(() => {
|
||||
const evidence = { lcpMs: 0, cls: 0 };
|
||||
await page.addInitScript((title: string) => {
|
||||
const evidence: ContractPerformanceEvidence = {
|
||||
lcpMs: 0,
|
||||
cls: 0,
|
||||
clickAt: null,
|
||||
headingAt: null,
|
||||
eventDurationMs: null,
|
||||
};
|
||||
(window as ContractPerformanceWindow).__contractPerformance = evidence;
|
||||
new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) evidence.lcpMs = entry.startTime;
|
||||
@@ -82,34 +116,97 @@ try {
|
||||
}
|
||||
}
|
||||
}).observe({ type: "layout-shift", buffered: true });
|
||||
});
|
||||
// 11-07. The interaction is timed by the page's own clock, from the
|
||||
// click event to the target heading entering the DOM, so Playwright's
|
||||
// round trips are not in it. The Event Timing entry of the click (the
|
||||
// INP input) is recorded next to it.
|
||||
new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) {
|
||||
if (entry.name === "click") {
|
||||
evidence.eventDurationMs = Math.max(evidence.eventDurationMs ?? 0, entry.duration);
|
||||
}
|
||||
}
|
||||
}).observe({ type: "event", buffered: true, durationThreshold: 16 } as PerformanceObserverInit);
|
||||
document.addEventListener(
|
||||
"click",
|
||||
(event) => {
|
||||
evidence.clickAt ??= event.timeStamp;
|
||||
},
|
||||
{ capture: true },
|
||||
);
|
||||
new MutationObserver((_records, observer) => {
|
||||
if (evidence.clickAt === null) return;
|
||||
const found = [...document.querySelectorAll("h1, h2")].some(
|
||||
(heading) => heading.textContent?.trim() === title,
|
||||
);
|
||||
if (found) {
|
||||
evidence.headingAt = performance.now();
|
||||
observer.disconnect();
|
||||
}
|
||||
}).observe(document, { childList: true, subtree: true, characterData: true });
|
||||
}, target.title);
|
||||
await page.goto(baseUrl, { waitUntil: "networkidle" });
|
||||
const target = ROUTE_REGISTRY.EXAMPLES_PLATFORM;
|
||||
const targetLabel = target.navigationLabel;
|
||||
if (!targetLabel) {
|
||||
throw new Error("Performance route must be present in navigation.");
|
||||
}
|
||||
const interactionStarted = performance.now();
|
||||
// Playwright matches accessible names by substring, so the navigation entry
|
||||
// "플랫폼 구성" also matched the home page's "플랫폼 구성 보기" call to
|
||||
// action and the locator resolved to two links. That is a strict-mode
|
||||
// violation before the first measurement is taken, so no lab performance
|
||||
// evidence could be produced at all — the run failed for an ambiguous
|
||||
// selector rather than for anything about performance.
|
||||
// action; `exact` keeps the locator on the navigation link.
|
||||
await page.getByRole("link", { name: targetLabel, exact: true }).click();
|
||||
await page.getByRole("heading", { name: target.title }).waitFor();
|
||||
const namedInteractionMs = performance.now() - interactionStarted;
|
||||
await page.waitForFunction(
|
||||
() => (window as ContractPerformanceWindow).__contractPerformance?.headingAt !== null,
|
||||
);
|
||||
const paint = await page.evaluate(
|
||||
() => (window as ContractPerformanceWindow).__contractPerformance,
|
||||
);
|
||||
if (!paint) {
|
||||
if (!paint || paint.clickAt === null || paint.headingAt === null) {
|
||||
throw new Error("Browser performance evidence was not initialized.");
|
||||
}
|
||||
return {
|
||||
lcpMs: Math.round(paint.lcpMs),
|
||||
cls: Number(paint.cls.toFixed(4)),
|
||||
namedInteractionMs: Math.round(paint.headingAt - paint.clickAt),
|
||||
eventDurationMs:
|
||||
paint.eventDurationMs === null ? null : Math.round(paint.eventDurationMs),
|
||||
};
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
|
||||
const load = () => loadavg().map((value) => Math.round(value * 100) / 100);
|
||||
|
||||
try {
|
||||
await waitForServer();
|
||||
const release = JSON.parse(
|
||||
await readFile("dist/release-manifest.json", "utf8"),
|
||||
);
|
||||
await assertServedRelease(baseUrl, release);
|
||||
const thresholds = JSON.parse(
|
||||
await readFile("config/performance/budgets.json", "utf8"),
|
||||
).lab;
|
||||
const browser = await chromium.launch();
|
||||
try {
|
||||
const loadBefore = load();
|
||||
const samples: RunMetrics[] = [];
|
||||
for (let run = 0; run < runs; run += 1) samples.push(await measureOnce(browser));
|
||||
const loadAfter = load();
|
||||
// The served build must not have changed under the runs either.
|
||||
await assertServedRelease(baseUrl, release);
|
||||
const lcp = summarizeSamples(samples.map((sample) => sample.lcpMs));
|
||||
const cls = summarizeSamples(samples.map((sample) => sample.cls));
|
||||
const interaction = summarizeSamples(
|
||||
samples.map((sample) => sample.namedInteractionMs),
|
||||
);
|
||||
const eventDurations = samples.flatMap((sample) =>
|
||||
sample.eventDurationMs === null ? [] : [sample.eventDurationMs],
|
||||
);
|
||||
const contextMetadata = {
|
||||
runner: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
nodeVersion: process.version,
|
||||
cpuModel: cpus()[0]?.model.trim() ?? "unknown",
|
||||
logicalCpus: cpus().length,
|
||||
loadAverage: { before: loadBefore, after: loadAfter },
|
||||
},
|
||||
browser: { name: "chromium", version: await browser.version() },
|
||||
viewport: { width: 1280, height: 720 },
|
||||
@@ -121,12 +218,29 @@ try {
|
||||
},
|
||||
cpu: { throttlingRate: 4 },
|
||||
cache: { state: "cold", isolation: "new-browser-context" },
|
||||
build: { buildId: release.buildId, releaseId: release.releaseId },
|
||||
build: {
|
||||
buildId: release.buildId,
|
||||
releaseId: release.releaseId,
|
||||
served: `${baseUrl}/release-manifest.json`,
|
||||
},
|
||||
// 11-07. The budget is judged on the median of cold runs; every run and
|
||||
// the spread stay in the evidence.
|
||||
sampling: {
|
||||
runs,
|
||||
statistic: "median",
|
||||
interactionClock: "page: click event to target heading in DOM",
|
||||
lcpMs: lcp,
|
||||
cls,
|
||||
namedInteractionMs: interaction,
|
||||
clickEventDurationMs: eventDurations.length
|
||||
? summarizeSamples(eventDurations)
|
||||
: null,
|
||||
},
|
||||
};
|
||||
const metrics = {
|
||||
lcpMs: Math.round(paint.lcpMs),
|
||||
cls: Number(paint.cls.toFixed(4)),
|
||||
namedInteractionMs: Math.round(namedInteractionMs),
|
||||
lcpMs: lcp.median,
|
||||
cls: cls.median,
|
||||
namedInteractionMs: interaction.median,
|
||||
};
|
||||
const result = evaluateLabBudget(
|
||||
{ context: contextMetadata, metrics },
|
||||
@@ -164,14 +278,20 @@ try {
|
||||
},
|
||||
});
|
||||
if (!passed) {
|
||||
throw new Error(`Lab performance failed: ${JSON.stringify(metrics)}`);
|
||||
throw new Error(
|
||||
`Lab performance failed (median of ${runs}): ${JSON.stringify(metrics)}; ` +
|
||||
`interaction samples ${interaction.samples.join(",")}`,
|
||||
);
|
||||
}
|
||||
process.stdout.write(
|
||||
`Lab performance: PASS (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms)\n`,
|
||||
`Lab performance: PASS, median of ${runs} (LCP ${metrics.lcpMs}ms, CLS ${metrics.cls}, interaction ${metrics.namedInteractionMs}ms ` +
|
||||
`[${interaction.min}-${interaction.max}])\n`,
|
||||
);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
} finally {
|
||||
server.kill("SIGTERM");
|
||||
if (server.pid !== undefined && serverExit === null) {
|
||||
process.kill(-server.pid, "SIGTERM");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,11 @@ const SAMPLE_EXTRAS: Readonly<Record<string, FeatureExtras>> = Object.freeze({
|
||||
"tests/component/product-feature-switch.test.tsx",
|
||||
"tests/e2e/reference-form.spec.ts",
|
||||
"tests/e2e/reference-route.spec.ts",
|
||||
"tests/e2e/reference-save.spec.ts",
|
||||
// 11-04: the growth measurement copies this sample feature.
|
||||
"tests/unit/feature-shape-growth.test.ts",
|
||||
"scripts/scale/measure-feature-shape-growth.ts",
|
||||
"config/scale/feature-shape-growth.json",
|
||||
"tests/mocks",
|
||||
"tests/fixtures/typecheck/invalid-feature-input.ts",
|
||||
"tests/fixtures/typecheck/invalid-reference-operation.ts",
|
||||
|
||||
Reference in New Issue
Block a user