refactor: 도메인이 100개 1000개씩 늘어나도 유지보수 쉬운 구조로 변경

This commit is contained in:
donghyeon-ka
2026-09-24 00:42:21 +09:00
parent 91ae42a2a7
commit 6f6c48ebb7
1274 changed files with 290812 additions and 0 deletions
+69
View File
@@ -0,0 +1,69 @@
import { mkdir, writeFile } from "node:fs/promises";
import { dirname } from "node:path";
/**
* Spec §48. Machine-readable benchmark result.
*/
export type BenchmarkProfile = "S0" | "S1" | "S2" | "S3" | "S4";
export interface BenchmarkResult {
readonly name: string;
readonly profile: BenchmarkProfile;
readonly cardinality: number;
readonly samples: readonly number[];
readonly medianMs: number;
readonly heapBytes?: number;
readonly metadata: Readonly<Record<string, string | number | boolean>>;
}
export type BenchmarkArtifact = Readonly<{
schemaVersion: 1;
suite: string;
generatedAt: string;
node: string;
passed: boolean;
results: readonly BenchmarkResult[];
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
}>;
const PROFILES: ReadonlySet<string> = new Set(["S0", "S1", "S2", "S3", "S4"]);
export function median(values: readonly number[]): number {
if (values.length === 0) throw new TypeError("median of an empty series");
const sorted = [...values].sort((left, right) => left - right);
const middle = Math.floor(sorted.length / 2);
return sorted.length % 2 === 1
? (sorted[middle] as number)
: ((sorted[middle - 1] as number) + (sorted[middle] as number)) / 2;
}
export function assertBenchmarkResult(value: BenchmarkResult): BenchmarkResult {
if (typeof value.name !== "string" || value.name.length === 0) {
throw new TypeError("benchmark name required");
}
if (!PROFILES.has(value.profile)) {
throw new TypeError(`unknown benchmark profile ${String(value.profile)}`);
}
if (!Number.isSafeInteger(value.cardinality) || value.cardinality < 0) {
throw new TypeError("benchmark cardinality must be a non-negative integer");
}
if (
value.samples.length === 0 ||
value.samples.some((sample) => !Number.isFinite(sample) || sample < 0)
) {
throw new TypeError("benchmark samples must be finite non-negative numbers");
}
if (value.medianMs !== median(value.samples)) {
throw new TypeError("benchmark medianMs must be the median of its samples");
}
return value;
}
export async function writeBenchmarkArtifact(
path: string,
artifact: BenchmarkArtifact,
): Promise<void> {
for (const result of artifact.results) assertBenchmarkResult(result);
await mkdir(dirname(path), { recursive: true });
await writeFile(path, `${JSON.stringify(artifact, null, 2)}\n`);
}
+64
View File
@@ -0,0 +1,64 @@
import { performance } from "node:perf_hooks";
import {
assertBenchmarkResult,
median,
type BenchmarkProfile,
type BenchmarkResult,
} from "./benchmark-result.ts";
/**
* Spec §49. Repeated samples with a median, so one noisy scheduler slice can
* neither fail a linear implementation nor hide a slow one.
*
* `run` either does the work (and is timed here) or returns a duration it
* measured itself, which lets tests inject exact durations.
*/
export type BenchmarkOptions = Readonly<{
samples: number;
warmup?: number;
profile?: BenchmarkProfile;
cardinality?: number;
metadata?: Readonly<Record<string, string | number | boolean>>;
measureHeap?: boolean;
run: () => unknown;
}>;
function heapUsed(): number {
return process.memoryUsage().heapUsed;
}
export async function benchmark(
name: string,
options: BenchmarkOptions,
): Promise<BenchmarkResult> {
if (!Number.isSafeInteger(options.samples) || options.samples < 1) {
throw new TypeError("benchmark samples must be a positive integer");
}
for (let index = 0; index < (options.warmup ?? 0); index += 1) {
await options.run();
}
const samples: number[] = [];
let peakHeapDelta = 0;
for (let index = 0; index < options.samples; index += 1) {
const heapBefore = options.measureHeap ? heapUsed() : 0;
const startedAt = performance.now();
const reported = await options.run();
const elapsed = performance.now() - startedAt;
if (options.measureHeap) {
peakHeapDelta = Math.max(peakHeapDelta, heapUsed() - heapBefore);
}
samples.push(typeof reported === "number" && Number.isFinite(reported) ? reported : elapsed);
}
return assertBenchmarkResult(
Object.freeze({
name,
profile: options.profile ?? "S0",
cardinality: options.cardinality ?? 0,
samples: Object.freeze(samples),
medianMs: median(samples),
...(options.measureHeap ? { heapBytes: Math.max(0, peakHeapDelta) } : {}),
metadata: Object.freeze({ ...(options.metadata ?? {}) }),
}),
);
}
+554
View File
@@ -0,0 +1,554 @@
import { createReadStream } from "node:fs";
import { access, mkdir, readFile, stat, writeFile } from "node:fs/promises";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import path from "node:path";
import {
classifyCounterSeries,
classifyMemorySlope,
type MemorySlopeClass,
} from "./memory-slope.ts";
/**
* Spec §33 / §44. Browser soak on the existing Playwright runner.
*
* Serves the built `dist/` on an OS-assigned 127.0.0.1 port and keeps one
* long-lived page cycling through the app's routes by client-side navigation.
* The served runtime config turns telemetry on (pointed at a local collector)
* and points the API at a stub that always answers 503, so every cycle also
* exercises bounded retries and telemetry under a sustained backend failure.
*
* Collected per sample: JS heap, DOM nodes, event listeners, live timers,
* in-flight fetches, Web Storage size. Collected overall: cycles, navigation
* failures, API requests, telemetry POSTs/events. Without a build or browser
* the result is `skipped` (never a pass); too little evidence fails.
*/
export type SoakSample = Readonly<{
elapsedMs: number;
cycles: number;
heapBytes: number;
domNodes: number;
listeners: number;
activeTimers: number;
inFlightRequests: number;
storageKeys: number;
storageBytes: number;
}>;
export type SoakSummary = Readonly<{
cycles: number;
navigationFailures: number;
maxHeapBytes: number;
heapClassification: MemorySlopeClass;
listenerDelta: number;
listenerClassification: MemorySlopeClass;
timerDelta: number;
maxActiveTimers: number;
domNodeDelta: number;
maxInFlightRequests: number;
finalInFlightRequests: number;
storageKeyDelta: number;
storageByteDelta: number;
apiRequests: number;
referenceVisits: number;
settledReferenceVisits: number;
apiRequestsPerReferenceVisit: number;
telemetryPosts: number;
telemetryEvents: number;
maxTelemetryPostsPerMinute: number;
}>;
export type SoakCheck = Readonly<{ name: string; passed: boolean; detail: string }>;
export type SoakArtifact = Readonly<{
schemaVersion: 2;
status: "passed" | "failed" | "skipped";
reason?: string;
durationMs: number;
classification?: MemorySlopeClass;
summary?: SoakSummary;
checks?: readonly SoakCheck[];
/** Identity of the soaked build (from dist/release-manifest.json). */
build?: Readonly<{ buildId?: string; releaseId?: string; builtAt?: string }>;
browser?: string;
samples: readonly SoakSample[];
}>;
/** Thresholds a soak must meet; each is checked and recorded by name. */
export const SOAK_LIMITS = Object.freeze({
minCycles: 200,
minPostWarmupSamples: 3,
maxListenerDelta: 50,
maxTimerDelta: 10,
maxStorageKeyDelta: 2,
/** One logical execution: 1 attempt + at most 2 retries. */
maxApiRequestsPerReferenceVisit: 3,
/** Two bounded pipelines (platform events + feature events) x 20/min. */
maxTelemetryPostsPerMinute: 40,
});
export function summarizeSoak(
samples: readonly Pick<SoakSample, "elapsedMs" | "heapBytes">[],
warmupMs: number,
): MemorySlopeClass {
return classifyMemorySlope(
samples.map((sample) => ({
minute: sample.elapsedMs / 60_000,
heapMb: sample.heapBytes / (1024 * 1024),
})),
{ warmupMinutes: warmupMs / 60_000, maxSlopeMbPerMinute: 1, maxRelativeGrowth: 0.2 },
);
}
export type SoakTotals = Readonly<{
cycles: number;
navigationFailures: number;
referenceVisits: number;
settledReferenceVisits: number;
finalInFlightRequests: number;
apiRequests: number;
telemetryPosts: number;
telemetryEvents: number;
telemetryPostTimes: readonly number[];
}>;
export function evaluateSoak(
samples: readonly SoakSample[],
totals: SoakTotals,
warmupMs: number,
): Readonly<{ summary: SoakSummary; checks: readonly SoakCheck[] }> {
const measured = samples.filter((sample) => sample.elapsedMs >= warmupMs);
const first = measured[0];
const last = measured[measured.length - 1];
const delta = (pick: (sample: SoakSample) => number) =>
first && last ? pick(last) - pick(first) : 0;
let maxTelemetryPostsPerMinute = 0;
for (const time of totals.telemetryPostTimes) {
const inWindow = totals.telemetryPostTimes.filter(
(other) => other >= time && other < time + 60_000,
).length;
maxTelemetryPostsPerMinute = Math.max(maxTelemetryPostsPerMinute, inWindow);
}
const summary: SoakSummary = {
cycles: totals.cycles,
navigationFailures: totals.navigationFailures,
maxHeapBytes: Math.max(0, ...samples.map((sample) => sample.heapBytes)),
heapClassification: summarizeSoak(samples, warmupMs),
listenerDelta: delta((sample) => sample.listeners),
listenerClassification: classifyCounterSeries(
measured.map((sample) => sample.listeners),
{ maxSlopePerSample: 1 },
),
timerDelta: delta((sample) => sample.activeTimers),
maxActiveTimers: Math.max(0, ...samples.map((sample) => sample.activeTimers)),
domNodeDelta: delta((sample) => sample.domNodes),
maxInFlightRequests: Math.max(0, ...samples.map((sample) => sample.inFlightRequests)),
finalInFlightRequests: totals.finalInFlightRequests,
storageKeyDelta: delta((sample) => sample.storageKeys),
storageByteDelta: delta((sample) => sample.storageBytes),
apiRequests: totals.apiRequests,
referenceVisits: totals.referenceVisits,
settledReferenceVisits: totals.settledReferenceVisits,
apiRequestsPerReferenceVisit:
totals.referenceVisits === 0
? 0
: Math.round((totals.apiRequests / totals.referenceVisits) * 1_000) / 1_000,
telemetryPosts: totals.telemetryPosts,
telemetryEvents: totals.telemetryEvents,
maxTelemetryPostsPerMinute,
};
const checks: SoakCheck[] = [
{ name: "cycles", passed: summary.cycles >= SOAK_LIMITS.minCycles, detail: `${summary.cycles} >= ${SOAK_LIMITS.minCycles}` },
{
name: "post-warmup samples",
passed: measured.length >= SOAK_LIMITS.minPostWarmupSamples,
detail: `${measured.length} >= ${SOAK_LIMITS.minPostWarmupSamples}`,
},
{ name: "navigation", passed: summary.navigationFailures === 0, detail: `${summary.navigationFailures} failures` },
{ name: "heap slope", passed: summary.heapClassification === "stable", detail: summary.heapClassification },
{
name: "listeners",
passed:
summary.listenerClassification === "stable" &&
summary.listenerDelta <= SOAK_LIMITS.maxListenerDelta,
detail: `${summary.listenerClassification}, delta ${summary.listenerDelta}`,
},
{
name: "timers",
passed: summary.timerDelta <= SOAK_LIMITS.maxTimerDelta,
detail: `delta ${summary.timerDelta}, max ${summary.maxActiveTimers}`,
},
{
name: "in-flight settles",
passed: summary.finalInFlightRequests === 0,
detail: `final ${summary.finalInFlightRequests}, max ${summary.maxInFlightRequests}`,
},
{
name: "storage growth",
passed: summary.storageKeyDelta <= SOAK_LIMITS.maxStorageKeyDelta,
detail: `keys ${summary.storageKeyDelta}, bytes ${summary.storageByteDelta}`,
},
{
name: "backend failure exercised",
passed:
summary.settledReferenceVisits > 0 && summary.apiRequests > 0 && summary.telemetryEvents > 0,
detail: `${summary.settledReferenceVisits} settled visits, ${summary.apiRequests} API requests, ${summary.telemetryEvents} telemetry events`,
},
{
name: "request amplification",
passed:
summary.referenceVisits > 0 &&
summary.apiRequestsPerReferenceVisit <= SOAK_LIMITS.maxApiRequestsPerReferenceVisit,
detail: `${summary.apiRequests} requests / ${summary.referenceVisits} visits`,
},
{
name: "telemetry rate",
passed: summary.maxTelemetryPostsPerMinute <= SOAK_LIMITS.maxTelemetryPostsPerMinute,
detail: `max ${summary.maxTelemetryPostsPerMinute}/min, ${summary.telemetryPosts} posts, ${summary.telemetryEvents} events`,
},
];
return { summary, checks };
}
const CONTENT_TYPES: Readonly<Record<string, string>> = {
".css": "text/css; charset=utf-8",
".html": "text/html; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".json": "application/json; charset=utf-8",
".svg": "image/svg+xml",
".png": "image/png",
".webmanifest": "application/manifest+json",
};
type Counters = {
apiRequests: number;
telemetryPosts: number;
telemetryEvents: number;
telemetryPostTimes: number[];
};
async function serveSoakTarget(
root: string,
counters: Counters,
startedAt: () => number,
): Promise<Readonly<{ server: Server; baseUrl: string }>> {
let baseUrl = "";
const baseConfig = JSON.parse(
await readFile(path.join(root, "config.json"), "utf8"),
) as Record<string, unknown>;
const server = createServer(async (request, response) => {
try {
const url = new URL(request.url ?? "/", "http://127.0.0.1");
if (url.pathname === "/__telemetry") {
let body = "";
request.on("data", (chunk: Buffer) => {
body += chunk.toString("utf8");
});
request.on("end", () => {
counters.telemetryPosts += 1;
counters.telemetryPostTimes.push(Date.now() - startedAt());
try {
const parsed = JSON.parse(body) as { events?: unknown[] };
counters.telemetryEvents += Array.isArray(parsed.events) ? parsed.events.length : 0;
} catch {
// Counted as a post regardless of its body.
}
response.writeHead(204).end();
});
return;
}
if (url.pathname.startsWith("/api/")) {
counters.apiRequests += 1;
response.writeHead(503, { "Content-Type": "application/json" }).end("{}");
return;
}
if (url.pathname === "/config.json") {
response
.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" })
.end(
JSON.stringify({
...baseConfig,
API_BASE_URL: `${baseUrl}/api/`,
TELEMETRY_ENABLED: true,
TELEMETRY_ENDPOINT: `${baseUrl}/__telemetry`,
}),
);
return;
}
const requested = path.resolve(root, `.${decodeURIComponent(url.pathname)}`);
if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) {
response.writeHead(403).end();
return;
}
const details = await stat(requested).catch(() => null);
const file = details?.isFile() ? requested : path.join(root, "index.html");
response.writeHead(200, {
"Content-Type": CONTENT_TYPES[path.extname(file)] ?? "application/octet-stream",
"Cache-Control": "no-store",
});
createReadStream(file).pipe(response);
} catch {
response.writeHead(404).end();
}
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const { port } = server.address() as AddressInfo;
baseUrl = `http://127.0.0.1:${port}`;
return { server, baseUrl };
}
const PAGE_PROBES = `
(() => {
const live = new Set();
const setT = window.setTimeout, clearT = window.clearTimeout;
const setI = window.setInterval, clearI = window.clearInterval;
window.setTimeout = (fn, ms, ...rest) => {
const id = setT(() => { live.delete(id); typeof fn === "function" && fn(...rest); }, ms);
live.add(id); return id;
};
window.clearTimeout = (id) => { live.delete(id); clearT(id); };
window.setInterval = (fn, ms, ...rest) => { const id = setI(fn, ms, ...rest); live.add(id); return id; };
window.clearInterval = (id) => { live.delete(id); clearI(id); };
let inFlight = 0;
const nativeFetch = window.fetch.bind(window);
window.fetch = (...args) => {
inFlight += 1;
return nativeFetch(...args).finally(() => { inFlight -= 1; });
};
const storageSize = (storage) => {
let bytes = 0;
for (let i = 0; i < storage.length; i += 1) {
const key = storage.key(i) ?? "";
bytes += key.length + (storage.getItem(key) ?? "").length;
}
return { keys: storage.length, bytes };
};
Object.defineProperty(window, "__soakProbe", {
value: () => {
const local = storageSize(window.localStorage);
const session = storageSize(window.sessionStorage);
return {
activeTimers: live.size,
inFlight,
storageKeys: local.keys + session.keys,
storageBytes: local.bytes + session.bytes,
};
},
});
})();
`;
type Probe = { activeTimers: number; inFlight: number; storageKeys: number; storageBytes: number };
export async function runBrowserSoak(
options: Readonly<{
distRoot: string;
durationMs: number;
intervalMs: number;
warmupMs: number;
routes: readonly string[];
}>,
): Promise<SoakArtifact> {
try {
await access(path.join(options.distRoot, "index.html"));
await access(path.join(options.distRoot, "config.json"));
} catch {
return {
schemaVersion: 2,
status: "skipped",
reason: "no built dist/index.html + config.json",
durationMs: 0,
samples: [],
};
}
let chromium: (typeof import("@playwright/test"))["chromium"];
try {
({ chromium } = await import("@playwright/test"));
} catch {
return { schemaVersion: 2, status: "skipped", reason: "playwright unavailable", durationMs: 0, samples: [] };
}
const counters: Counters = {
apiRequests: 0,
telemetryPosts: 0,
telemetryEvents: 0,
telemetryPostTimes: [],
};
let started = Date.now();
const { server, baseUrl } = await serveSoakTarget(
path.resolve(options.distRoot),
counters,
() => started,
);
let browser: Awaited<ReturnType<typeof chromium.launch>> | undefined;
try {
try {
browser = await chromium.launch();
} catch (error) {
return {
schemaVersion: 2,
status: "skipped",
reason: `browser unavailable: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`,
durationMs: 0,
samples: [],
};
}
const page = await browser.newPage();
await page.addInitScript(PAGE_PROBES);
const cdp = await page.context().newCDPSession(page);
await cdp.send("Performance.enable");
await page.goto(`${baseUrl}/examples/reference-resources`, { waitUntil: "load" });
await page.waitForSelector("#root > *", { timeout: 15_000 });
// Protected routes need a session: start the local demo session once, so
// the reference feature really dispatches its (failing) API requests.
const login = page.getByRole("button", { name: /로그인 시작|Sign in|Start sign-in/u });
if (await login.count()) {
await login.first().click();
await page.waitForTimeout(500);
}
const samples: SoakSample[] = [];
started = Date.now();
let nextSample = 0;
let cycles = 0;
let navigationFailures = 0;
let referenceVisits = 0;
let settledReferenceVisits = 0;
const probe = async () =>
(await page.evaluate(() =>
(window as unknown as { __soakProbe: () => Probe }).__soakProbe(),
)) as Probe;
while (Date.now() - started < options.durationMs) {
const route = options.routes[cycles % options.routes.length] ?? "/";
cycles += 1;
if (route === "/examples/reference-resources") referenceVisits += 1;
// Client-side navigation keeps one long-lived document, like a real session.
await page.evaluate((target) => {
window.history.pushState({}, "", target);
window.dispatchEvent(new PopStateEvent("popstate"));
}, route);
await page.waitForTimeout(50);
const rendered = await page.evaluate(
(target) =>
window.location.pathname === target &&
(document.getElementById("root")?.childElementCount ?? 0) > 0,
route,
);
if (!rendered) navigationFailures += 1;
// Most visits leave after 50ms, which cancels the query (a cancelled
// request is intentionally not reported). Every 5th reference visit
// dwells until its interaction settles, so the failing backend is really
// observed end to end: bounded retries, then feature/platform telemetry.
if (route === "/examples/reference-resources" && referenceVisits % 5 === 1) {
await page.waitForTimeout(200);
await page
.waitForFunction(
() => (window as unknown as { __soakProbe: () => Probe }).__soakProbe().inFlight === 0,
undefined,
{ timeout: 5_000 },
)
.catch(() => undefined);
settledReferenceVisits += 1;
}
const elapsedMs = Date.now() - started;
if (elapsedMs >= nextSample) {
nextSample = elapsedMs + options.intervalMs;
await cdp.send("HeapProfiler.collectGarbage").catch(() => undefined);
const { metrics } = (await cdp.send("Performance.getMetrics")) as {
metrics: { name: string; value: number }[];
};
const metric = (name: string) => metrics.find((entry) => entry.name === name)?.value ?? 0;
const probed = await probe();
samples.push({
elapsedMs,
cycles,
heapBytes: metric("JSHeapUsedSize"),
domNodes: metric("Nodes"),
listeners: metric("JSEventListeners"),
activeTimers: probed.activeTimers,
inFlightRequests: probed.inFlight,
storageKeys: probed.storageKeys,
storageBytes: probed.storageBytes,
});
}
}
// Let the last interaction's bounded retries and deadlines settle.
let finalInFlightRequests = (await probe()).inFlight;
for (let wait = 0; wait < 30 && finalInFlightRequests > 0; wait += 1) {
await page.waitForTimeout(500);
finalInFlightRequests = (await probe()).inFlight;
}
const evaluation = evaluateSoak(
samples,
{
cycles,
navigationFailures,
referenceVisits,
settledReferenceVisits,
finalInFlightRequests,
apiRequests: counters.apiRequests,
telemetryPosts: counters.telemetryPosts,
telemetryEvents: counters.telemetryEvents,
telemetryPostTimes: counters.telemetryPostTimes,
},
options.warmupMs,
);
const build = await readFile(path.join(options.distRoot, "release-manifest.json"), "utf8")
.then((raw) => {
const manifest = JSON.parse(raw) as Record<string, unknown>;
const pick = (key: string) =>
typeof manifest[key] === "string" ? (manifest[key] as string) : undefined;
return {
...(pick("buildId") ? { buildId: pick("buildId") } : {}),
...(pick("releaseId") ? { releaseId: pick("releaseId") } : {}),
...(pick("builtAt") ? { builtAt: pick("builtAt") } : {}),
};
})
.catch(() => ({}));
return {
schemaVersion: 2,
status: evaluation.checks.every((check) => check.passed) ? "passed" : "failed",
durationMs: options.durationMs,
classification: evaluation.summary.heapClassification,
summary: evaluation.summary,
checks: evaluation.checks,
build,
browser: `chromium ${browser.version()}`,
samples,
};
} finally {
await browser?.close().catch(() => undefined);
await new Promise<void>((resolve) => server.close(() => resolve()));
}
}
async function main(): Promise<void> {
const argument = (name: string, fallback: string) =>
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;
const artifact = path.resolve(argument("artifact", "artifacts/scale/browser-soak.json"));
const result = await runBrowserSoak({
distRoot: argument("dist", "dist"),
durationMs: Number(argument("duration-ms", "60000")),
intervalMs: Number(argument("interval-ms", "5000")),
warmupMs: Number(argument("warmup-ms", "15000")),
routes: argument(
"routes",
"/,/examples/reference-resources,/examples/platform,/examples/ui,/examples/states,/examples/reference-resources/status",
).split(","),
});
await mkdir(path.dirname(artifact), { recursive: true });
await writeFile(artifact, `${JSON.stringify(result, null, 2)}\n`);
process.stdout.write(
`browser soak: ${result.status}${result.reason ? ` (${result.reason})` : ""}` +
`${result.classification ? ` classification=${result.classification}` : ""} samples=${result.samples.length}` +
`${result.summary ? ` cycles=${result.summary.cycles}` : ""}\n`,
);
for (const check of result.checks ?? []) {
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
}
if (result.status === "failed") process.exitCode = 1;
}
if (import.meta.url === `file://${process.argv[1]}`) {
await main();
}
+113
View File
@@ -0,0 +1,113 @@
import { spawnSync } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { collectTscScale } from "./collect-tsc-diagnostics.ts";
import { runCompositionScale } from "./run-composition-scale.ts";
import {
approveBaseline,
compareToBaseline,
currentEnvironment,
snapshotMetrics,
type ScaleBaseline,
} from "./scale-baseline.ts";
/**
* Spec §48–§49. Relative regression gate against the stored, approved scale
* baseline (`config/scale/benchmark-baseline.json`).
*
* node scripts/scale/check-scale-baseline.ts compare, fail on regression
* SCALE_BASELINE_OWNER=... SCALE_BASELINE_REASON=... \
* node scripts/scale/check-scale-baseline.ts --update re-measure and re-approve
*
* The comparison is written to artifacts/scale/baseline-comparison.json (or
* `--artifact=`). Updating is an explicit, attributed act; the check never
* rewrites the baseline.
*/
export const BASELINE_PATH = "config/scale/benchmark-baseline.json";
export const LOCAL_TSC_SIZES = Object.freeze([25, 100, 400]);
function typescriptVersion(): string {
const result = spawnSync(path.resolve("node_modules/.bin/tsc"), ["-v"], { encoding: "utf8" });
const match = /Version\s+(\S+)/u.exec(result.stdout ?? "");
if (!match?.[1]) throw new Error("cannot determine TypeScript version");
return match[1];
}
async function measure() {
const composition = await runCompositionScale(["S0", "S1", "S2"], { samples: 9, warmup: 2 });
const compiler = await collectTscScale(LOCAL_TSC_SIZES);
return snapshotMetrics(composition.results, compiler);
}
async function main(): Promise<void> {
const update = process.argv.includes("--update");
const artifact = path.resolve(
process.argv.find((entry) => entry.startsWith("--artifact="))?.slice("--artifact=".length) ??
"artifacts/scale/baseline-comparison.json",
);
const environment = currentEnvironment(typescriptVersion());
const metrics = await measure();
if (update) {
const owner = process.env["SCALE_BASELINE_OWNER"];
const reason = process.env["SCALE_BASELINE_REASON"];
if (!owner || !reason) {
process.stderr.write("SCALE_BASELINE_OWNER and SCALE_BASELINE_REASON are required.\n");
process.exit(1);
}
const baseline = approveBaseline(environment, metrics, {
owner,
reason,
approvedAt: new Date().toISOString(),
});
await mkdir(path.dirname(BASELINE_PATH), { recursive: true });
await writeFile(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`);
process.stdout.write(`scale baseline approved: ${baseline.approval.metricsDigest}\n`);
return;
}
let baseline: ScaleBaseline;
try {
baseline = JSON.parse(await readFile(BASELINE_PATH, "utf8")) as ScaleBaseline;
} catch {
process.stderr.write(`missing or unreadable ${BASELINE_PATH}; run update:scale-baseline\n`);
process.exit(1);
}
const comparison = compareToBaseline(baseline, environment, metrics);
await mkdir(path.dirname(artifact), { recursive: true });
await writeFile(
artifact,
`${JSON.stringify(
{ schemaVersion: 1, generatedAt: new Date().toISOString(), environment, baselineApproval: baseline.approval, ...comparison },
null,
2,
)}\n`,
);
const { compatibility } = comparison;
process.stdout.write(
`compatibility: structural=${compatibility.structural.ok} compiler=${compatibility.compiler.ok} timing=${compatibility.timing.ok}\n`,
);
if (!compatibility.timing.ok) {
process.stdout.write(
`timing regression NOT compared (incomplete): ${compatibility.timing.reasons.join("; ")}\n`,
);
}
process.stdout.write(
`axes: absolute ${comparison.absolute.findings.length} (${comparison.absolute.passed ? "ok" : "FAIL"}), ` +
`growth ${comparison.growth.findings.length} (${comparison.growth.passed ? "ok" : "FAIL"}), ` +
`relative ${comparison.compiler.length} compiler + ${comparison.timing.length} timing\n`,
);
for (const failure of comparison.failures) process.stdout.write(`FAIL ${failure}\n`);
process.stdout.write(
`scale baseline gate: ${comparison.status.toUpperCase()} (${path.relative(process.cwd(), artifact)})\n`,
);
if (comparison.status === "failed") process.exitCode = 1;
else if (comparison.status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") {
process.exitCode = 1;
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
await main();
}
+208
View File
@@ -0,0 +1,208 @@
import { spawnSync } from "node:child_process";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import {
assertLinearOrBetter,
growthExponents,
} from "../../src/platform/cardinality/growth-regression.ts";
/**
* Spec §38. TypeScript scale gate.
*
* Parses `tsc --extendedDiagnostics` and measures bounded local fixtures:
* generated feature modules composed by one context, at increasing local
* cardinality. S3/S4 are never compiled; they are verified as topology.
*/
export type TscDiagnostics = Readonly<{
files: number;
lines?: number;
types: number;
symbols: number;
instantiations: number;
memoryKb?: number;
totalTimeSeconds?: number;
}>;
const FIELD_NAMES: Readonly<Record<string, keyof TscDiagnostics>> = Object.freeze({
Files: "files",
Lines: "lines",
Types: "types",
Symbols: "symbols",
Instantiations: "instantiations",
"Memory used": "memoryKb",
"Total time": "totalTimeSeconds",
});
export function parseTscExtendedDiagnostics(text: string): TscDiagnostics {
const parsed: Partial<Record<keyof TscDiagnostics, number>> = {};
for (const line of text.split(/\r?\n/u)) {
const match = /^\s*([A-Za-z][A-Za-z ]*?):\s+([\d.]+)\s*([KMs]?)\s*$/u.exec(line);
if (!match) continue;
const field = FIELD_NAMES[match[1] as string];
if (!field) continue;
const value = Number(match[2]);
if (!Number.isFinite(value)) continue;
parsed[field] = match[3] === "M" ? value * 1024 : value;
}
for (const required of ["files", "types", "symbols", "instantiations"] as const) {
if (parsed[required] === undefined) {
throw new TypeError(`tsc diagnostics missing ${required}`);
}
}
return Object.freeze(parsed) as TscDiagnostics;
}
export function runTscDiagnostics(project: string, cwd = process.cwd()): TscDiagnostics {
const tsc = resolve(cwd, "node_modules/.bin/tsc");
const result = spawnSync(tsc, ["--project", project, "--noEmit", "--extendedDiagnostics"], {
cwd,
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
});
if (result.status !== 0) {
throw new Error(`tsc failed for ${project}: ${(result.stdout + result.stderr).slice(-2_000)}`);
}
return parseTscExtendedDiagnostics(result.stdout);
}
/**
* Generates one bounded local app: `features` feature modules, each with its
* own typed contract, composed by one generated index. The shape mirrors the
* repository's feature-module/generated-index pattern.
*/
export async function writeLocalTscFixture(
root: string,
features: number,
): Promise<string> {
await rm(root, { recursive: true, force: true });
await mkdir(join(root, "features"), { recursive: true });
const imports: string[] = [];
const members: string[] = [];
for (let index = 0; index < features; index += 1) {
const id = `feature${index}`;
await writeFile(
join(root, "features", `${id}.ts`),
`export type ${id}Input = Readonly<{ id: string; limit: number; tags?: readonly string[] }>;\n` +
`export type ${id}Output = Readonly<{ items: readonly Readonly<{ id: string; name: string }>[] }>;\n` +
`export const ${id} = Object.freeze({\n` +
` id: "${id}" as const,\n` +
` routes: Object.freeze({ ${id.toUpperCase()}_LIST: "/${id}" }),\n` +
` execute(input: ${id}Input): ${id}Output {\n` +
` return { items: [{ id: input.id, name: String(input.limit) }] };\n` +
` },\n` +
`});\n`,
);
imports.push(`import { ${id} } from "./features/${id}.ts";`);
members.push(` ${id},`);
}
await writeFile(
join(root, "index.ts"),
`${imports.join("\n")}\nexport const LOCAL_FEATURES = Object.freeze([\n${members.join("\n")}\n] as const);\n` +
`export type LocalFeatureId = (typeof LOCAL_FEATURES)[number]["id"];\n`,
);
const tsconfig = join(root, "tsconfig.json");
await writeFile(
tsconfig,
JSON.stringify(
{
compilerOptions: {
target: "ES2022",
module: "ESNext",
moduleResolution: "Bundler",
allowImportingTsExtensions: true,
strict: true,
noEmit: true,
skipLibCheck: true,
types: [],
lib: ["ES2022"],
},
include: ["index.ts", "features/*.ts"],
},
null,
2,
),
);
return tsconfig;
}
export type TscScaleReport = Readonly<{
passed: boolean;
app: TscDiagnostics;
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
}>;
export async function collectTscScale(
localSizes: readonly number[],
cwd = process.cwd(),
): Promise<TscScaleReport> {
const app = runTscDiagnostics("tsconfig.app.json", cwd);
const local: { features: number; diagnostics: TscDiagnostics }[] = [];
for (const features of localSizes) {
const tsconfig = await writeLocalTscFixture(
resolve(cwd, ".tmp/scale-tsc", `local-${features}`),
features,
);
local.push({ features, diagnostics: runTscDiagnostics(tsconfig, cwd) });
}
const checks: { name: string; passed: boolean; detail: string }[] = [];
for (const metric of ["types", "symbols", "instantiations"] as const) {
const series = local.map(({ features, diagnostics }) => ({
n: features,
// Library/baseline cost is constant; growth is judged on what the local
// features add on top of the smallest fixture.
cost: Math.max(1, diagnostics[metric]),
}));
try {
assertLinearOrBetter(series, { maxExponent: 1.3 });
checks.push({
name: `tsc:${metric}`,
passed: true,
detail: `exponents ${growthExponents(series).map((value) => value.toFixed(2)).join(", ")}`,
});
} catch (error) {
checks.push({
name: `tsc:${metric}`,
passed: false,
detail: error instanceof Error ? error.message : String(error),
});
}
}
return Object.freeze({
passed: checks.every((check) => check.passed),
app,
local: Object.freeze(local),
checks: Object.freeze(checks),
});
}
async function main(): Promise<void> {
const artifactArg = process.argv.find((argument) => argument.startsWith("--artifact="));
const sizesArg = process.argv.find((argument) => argument.startsWith("--local-sizes="));
const sizes = (sizesArg?.slice("--local-sizes=".length) ?? "25,100,400")
.split(",")
.map(Number)
.filter((value) => Number.isSafeInteger(value) && value > 0);
const artifact = resolve(
artifactArg?.slice("--artifact=".length) ?? "artifacts/scale/tsc-diagnostics.json",
);
const report = await collectTscScale(sizes);
await mkdir(resolve(artifact, ".."), { recursive: true });
await writeFile(
artifact,
`${JSON.stringify({ schemaVersion: 1, generatedAt: new Date().toISOString(), ...report }, null, 2)}\n`,
);
process.stdout.write(
`tsc app: files=${report.app.files} types=${report.app.types} instantiations=${report.app.instantiations}\n`,
);
for (const check of report.checks) {
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
}
process.stdout.write(`tsc diagnostics artifact: ${artifact}\n`);
if (!report.passed) process.exitCode = 1;
}
if (import.meta.url === `file://${process.argv[1]}`) {
await main();
}
+108
View File
@@ -0,0 +1,108 @@
import { composeUniqueRecord } from "../../src/platform/cardinality/compose-unique-record.ts";
import {
resolveProductFeatures,
selectCompiledProductFeatures,
} from "../../src/contracts/product-features.ts";
import { defineBoundedContextModule } from "../../src/contexts/bounded-context-module.ts";
import { defineDeployableAppModule } from "../../src/apps/deployable-app-module.ts";
/**
* Spec §39. Scale-sensitive production composition paths, each parameterized
* by one cardinality `n`. Inputs are built outside the timed region.
*/
export type CompositionWorkload = Readonly<{
name: string;
implementation: string;
prepare(n: number): () => unknown;
}>;
export function createUniqueRegistryParts(
count: number,
): readonly Readonly<Record<string, number>>[] {
return Array.from({ length: count }, (_, index) =>
Object.freeze({ [`route-${index}`]: index }),
);
}
function featureIds(count: number): string[] {
return Array.from(
{ length: count },
(_, index) => `feature-${index.toString().padStart(7, "0")}`,
);
}
const CONTEXT_FEATURES = 200;
export const COMPOSITION_WORKLOADS: readonly CompositionWorkload[] = Object.freeze([
{
name: "registry-composition",
implementation: "composeUniqueRecord",
prepare(n: number) {
const parts = createUniqueRegistryParts(n);
return () => composeUniqueRecord(parts, () => "route");
},
},
{
name: "product-feature-selection",
implementation: "selectCompiledProductFeatures",
prepare(n: number) {
const compiled = featureIds(n).map((featureId) => Object.freeze({ featureId }));
const declared = compiled
.filter((_, index) => index % 2 === 0)
.map((feature) => feature.featureId)
.join(",");
return () => selectCompiledProductFeatures(compiled, declared);
},
},
{
name: "product-feature-resolution",
implementation: "resolveProductFeatures",
prepare(n: number) {
const compiled = featureIds(n);
const installed = compiled.filter((_, index) => index % 3 !== 0);
return () => resolveProductFeatures(compiled, installed);
},
},
{
name: "context-hierarchy-composition",
implementation: "defineBoundedContextModule+defineDeployableAppModule",
prepare(n: number) {
const features = featureIds(n).map((id) =>
Object.freeze({
id,
contracts: Object.freeze({ featureId: id }),
routes: Object.freeze({}),
adapters: Object.freeze([]),
runtimes: Object.freeze([]),
messages: Object.freeze({}),
contractContributions: Object.freeze([]),
}),
);
const contextCount = Math.ceil(n / CONTEXT_FEATURES);
return () => {
const contexts = Array.from({ length: contextCount }, (_, contextIndex) =>
defineBoundedContextModule({
id: `context-${contextIndex}`,
features: features.slice(
contextIndex * CONTEXT_FEATURES,
(contextIndex + 1) * CONTEXT_FEATURES,
),
budget: Object.freeze({ maxFeatures: CONTEXT_FEATURES }),
}),
);
// Apps are sharded by the same direct-child budget.
const apps: unknown[] = [];
for (let index = 0; index < contexts.length; index += 20) {
apps.push(
defineDeployableAppModule({
id: `app-${index / 20}`,
contexts: contexts.slice(index, index + 20),
budget: Object.freeze({ maxContexts: 20 }),
}),
);
}
return apps;
};
},
},
]);
+66
View File
@@ -0,0 +1,66 @@
/**
* Spec §44. Soak classification.
*
* The leak signal is sustained positive growth after warm-up, not an
* arbitrary absolute maximum. The classifier fits a least-squares line over
* the post-warm-up samples and reports `unbounded-growth` only when both the
* slope and the total relative growth are material.
*/
export type SlopeSample = Readonly<{ minute: number; heapMb: number }>;
export type MemorySlopeClass = "stable" | "unbounded-growth" | "insufficient-data";
export type MemorySlopeOptions = Readonly<{
/** Samples before this minute are warm-up. Default 0. */
warmupMinutes?: number;
/** Largest tolerated post-warm-up slope in MB per minute. Default 0.5. */
maxSlopeMbPerMinute?: number;
/** Largest tolerated relative growth first->last fitted value. Default 0.1. */
maxRelativeGrowth?: number;
}>;
export function linearSlope(points: readonly Readonly<{ x: number; y: number }>[]): number {
const count = points.length;
if (count < 2) return 0;
const meanX = points.reduce((sum, point) => sum + point.x, 0) / count;
const meanY = points.reduce((sum, point) => sum + point.y, 0) / count;
let numerator = 0;
let denominator = 0;
for (const point of points) {
numerator += (point.x - meanX) * (point.y - meanY);
denominator += (point.x - meanX) ** 2;
}
return denominator === 0 ? 0 : numerator / denominator;
}
export function classifyMemorySlope(
samples: readonly SlopeSample[],
options: MemorySlopeOptions = {},
): MemorySlopeClass {
const warmup = options.warmupMinutes ?? 0;
const maxSlope = options.maxSlopeMbPerMinute ?? 0.5;
const maxGrowth = options.maxRelativeGrowth ?? 0.1;
const measured = samples
.filter((sample) => sample.minute >= warmup)
.filter((sample) => Number.isFinite(sample.minute) && Number.isFinite(sample.heapMb))
.sort((left, right) => left.minute - right.minute);
if (measured.length < 3) return "insufficient-data";
const slope = linearSlope(measured.map((sample) => ({ x: sample.minute, y: sample.heapMb })));
const first = measured[0] as SlopeSample;
const last = measured[measured.length - 1] as SlopeSample;
const fittedStart = Math.max(first.heapMb, Number.MIN_VALUE);
const growth = (slope * (last.minute - first.minute)) / fittedStart;
return slope > maxSlope && growth > maxGrowth ? "unbounded-growth" : "stable";
}
/** Generic variant for logical counters (timers, listeners, buffered events). */
export function classifyCounterSeries(
values: readonly number[],
options: Readonly<{ warmupSamples?: number; maxSlopePerSample?: number }> = {},
): MemorySlopeClass {
const warmup = options.warmupSamples ?? 0;
const measured = values.slice(warmup);
if (measured.length < 3) return "insufficient-data";
const slope = linearSlope(measured.map((y, x) => ({ x, y })));
return slope > (options.maxSlopePerSample ?? 0.01) ? "unbounded-growth" : "stable";
}
+136
View File
@@ -0,0 +1,136 @@
import { resolve } from "node:path";
import {
assertLinearOrBetter,
growthExponents,
} from "../../src/platform/cardinality/growth-regression.ts";
import { benchmark } from "./benchmark.ts";
import {
writeBenchmarkArtifact,
type BenchmarkProfile,
type BenchmarkResult,
} from "./benchmark-result.ts";
import { COMPOSITION_WORKLOADS } from "./composition-workloads.ts";
/**
* Spec §36 / §39. Runtime composition scale gate for S0-S2.
*
* Every production composition workload runs at each requested profile with
* warm-up and repeated samples; the median series must be linear-or-better.
* Results are written as a machine-readable artifact under artifacts/scale/.
*/
export const COMPOSITION_PROFILES: Readonly<Record<string, number>> = Object.freeze({
S0: 100,
S1: 1_000,
S2: 10_000,
});
/** Scheduler noise below this median is not evidence of growth. */
export const COMPOSITION_NOISE_FLOOR_MS = 0.5;
/**
* n log n paths (sorting, allocation-heavy string work) measure ~1.1-1.4
* between S1 and S2; a quadratic path measures ~2. 1.6 separates the two.
*/
export const COMPOSITION_MAX_EXPONENT = 1.6;
export type CompositionScaleReport = Readonly<{
passed: boolean;
results: readonly BenchmarkResult[];
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
}>;
export async function runCompositionScale(
profiles: readonly string[],
options: Readonly<{
samples?: number;
warmup?: number;
/**
* `median` (default) for standalone gate runs. `min` (best observed) when
* the gate shares the machine with other suites: stolen CPU and foreign
* GC only ever add time, so the minimum keeps the algorithmic shape.
*/
statistic?: "median" | "min";
}> = {},
): Promise<CompositionScaleReport> {
const results: BenchmarkResult[] = [];
const checks: { name: string; passed: boolean; detail: string }[] = [];
for (const workload of COMPOSITION_WORKLOADS) {
const series: { n: number; cost: number }[] = [];
for (const profile of profiles) {
const cardinality = COMPOSITION_PROFILES[profile];
if (cardinality === undefined) throw new TypeError(`unknown profile ${profile}`);
const run = workload.prepare(cardinality);
const result = await benchmark(workload.name, {
samples: options.samples ?? 7,
warmup: options.warmup ?? 2,
profile: profile as BenchmarkProfile,
cardinality,
measureHeap: true,
metadata: { implementation: workload.implementation },
run,
});
results.push(result);
series.push({
n: cardinality,
cost:
options.statistic === "min" ? Math.min(...result.samples) : result.medianMs,
});
}
if (series.length < 2) continue;
try {
assertLinearOrBetter(series, {
maxExponent: COMPOSITION_MAX_EXPONENT,
noiseFloor: COMPOSITION_NOISE_FLOOR_MS,
});
checks.push({
name: `${workload.name}:growth`,
passed: true,
detail: `exponents ${growthExponents(series, COMPOSITION_NOISE_FLOOR_MS)
.map((exponent) => exponent.toFixed(2))
.join(", ")}`,
});
} catch (error) {
checks.push({
name: `${workload.name}:growth`,
passed: false,
detail: error instanceof Error ? error.message : String(error),
});
}
}
return Object.freeze({
passed: checks.every((check) => check.passed),
results: Object.freeze(results),
checks: Object.freeze(checks),
});
}
async function main(): Promise<void> {
const profileArg = process.argv.find((argument) => argument.startsWith("--profiles="));
const artifactArg = process.argv.find((argument) => argument.startsWith("--artifact="));
const profiles = (profileArg?.slice("--profiles=".length) ?? "S0,S1,S2")
.split(",")
.map((entry) => entry.trim())
.filter(Boolean);
const artifact = resolve(
artifactArg?.slice("--artifact=".length) ?? "artifacts/scale/composition-scale.json",
);
const report = await runCompositionScale(profiles);
await writeBenchmarkArtifact(artifact, {
schemaVersion: 1,
suite: "composition-scale",
generatedAt: new Date().toISOString(),
node: process.version,
passed: report.passed,
results: report.results,
checks: report.checks,
});
for (const check of report.checks) {
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
}
process.stdout.write(`composition scale artifact: ${artifact}\n`);
if (!report.passed) process.exitCode = 1;
}
if (import.meta.url === `file://${process.argv[1]}`) {
await main();
}
+190
View File
@@ -0,0 +1,190 @@
import { spawnSync } from "node:child_process";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
/**
* Spec §47. CI execution tiers for the scale/chaos/soak program.
*
* pr S0/S1 architecture, composition and traffic invariants
* main pr + S2 composition, TypeScript diagnostics and the stored
* baseline regression gate (config/scale/benchmark-baseline.json)
* nightly main + S3/S4 topology, telemetry storm, session cycles, browser soak
* release release-coherence matrix + the stable main/nightly subset
*
* Every step's output goes to artifacts/scale/<tier>/<step>.log and the tier
* writes one machine-readable summary.json. A step that could not run (e.g. a
* browser soak without a build or browser) is `skipped`, and a tier with any
* skipped step is `incomplete`: it is never reported as passed and exits
* non-zero. `SCALE_ALLOW_INCOMPLETE=1` keeps the exit code zero for local runs,
* but the summary still says `incomplete`.
*/
export type ScaleTier = "pr" | "main" | "nightly" | "release";
type Step = Readonly<{
id: string;
command: readonly string[];
/** A step that may legitimately report `skipped` in its own artifact. */
optional?: boolean;
}>;
const VITEST = ["scripts/run-vitest.ts", "run", "--reporter=default"];
const PR_STEPS: readonly Step[] = [
{
id: "architecture-topology",
command: [...VITEST, "tests/architecture", "tests/platform/cardinality", "tests/platform/routing"],
},
{
id: "traffic-budgets",
command: [...VITEST, "tests/platform/execution", "tests/platform/traffic", "tests/platform/resilience"],
},
{
id: "scale-units",
command: [
...VITEST,
"tests/scale/growth-regression.test.ts",
"tests/scale/benchmark.test.ts",
"tests/scale/synthetic-topology.test.ts",
"tests/scale/scale-baseline.test.ts",
"tests/scale/traffic",
],
},
{
id: "composition-s0-s1",
command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1"],
},
];
const MAIN_STEPS: readonly Step[] = [
...PR_STEPS,
{
id: "composition-s0-s2",
command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1,S2"],
},
{
id: "composition-scale-suite",
command: [...VITEST, "tests/scale/composition-scale.test.ts", "tests/scale/tsc-diagnostics-parser.test.ts"],
},
{
id: "typescript-diagnostics",
command: ["scripts/scale/collect-tsc-diagnostics.ts", "--local-sizes=25,100,400"],
},
{
id: "baseline-regression",
command: ["scripts/scale/check-scale-baseline.ts"],
},
];
const NIGHTLY_STEPS: readonly Step[] = [
...MAIN_STEPS,
{
id: "topology-s3-s4",
command: [...VITEST, "tests/scale/global-local-independence.test.ts", "tests/platform/cardinality/extreme-scale-registry.test.ts"],
},
{
id: "telemetry-storm",
command: [...VITEST, "tests/scale/telemetry", "tests/platform/telemetry"],
},
{ id: "session-cycles", command: [...VITEST, "tests/scale/session"] },
{
id: "browser-soak",
command: ["scripts/scale/browser-soak.ts", "--duration-ms=120000", "--interval-ms=5000", "--warmup-ms=30000"],
optional: true,
},
];
const RELEASE_STEPS: readonly Step[] = [
{
id: "release-coherence",
command: [...VITEST, "tests/platform/release", "tests/release"],
},
...MAIN_STEPS,
{
id: "topology-s3-s4",
command: [...VITEST, "tests/scale/global-local-independence.test.ts"],
},
{
id: "telemetry-storm",
command: [...VITEST, "tests/scale/telemetry"],
},
];
export const SCALE_TIERS: Readonly<Record<ScaleTier, readonly Step[]>> = Object.freeze({
pr: PR_STEPS,
main: MAIN_STEPS,
nightly: NIGHTLY_STEPS,
release: RELEASE_STEPS,
});
function withArtifact(step: Step, directory: string): readonly string[] {
const script = step.command[0] ?? "";
if (
script.endsWith("run-composition-scale.ts") ||
script.endsWith("collect-tsc-diagnostics.ts") ||
script.endsWith("browser-soak.ts") ||
script.endsWith("check-scale-baseline.ts")
) {
return [...step.command, `--artifact=${path.join(directory, `${step.id}.json`)}`];
}
return step.command;
}
async function main(): Promise<void> {
const tier = process.argv[2] as ScaleTier | undefined;
if (!tier || !(tier in SCALE_TIERS)) {
process.stderr.write(`usage: run-scale-profile.ts <${Object.keys(SCALE_TIERS).join("|")}>\n`);
process.exit(2);
}
const directory = path.resolve("artifacts/scale", tier);
await mkdir(directory, { recursive: true });
const results: { id: string; status: "passed" | "failed" | "skipped"; durationMs: number; log: string }[] = [];
for (const step of SCALE_TIERS[tier]) {
const startedAt = Date.now();
const run = spawnSync(process.execPath, [...withArtifact(step, directory)], {
encoding: "utf8",
maxBuffer: 64 * 1024 * 1024,
env: { ...process.env, SCALE_ARTIFACT_DIR: path.join(directory, "metrics") },
});
const log = path.join(directory, `${step.id}.log`);
await writeFile(log, `${run.stdout ?? ""}${run.stderr ?? ""}`);
const skipped = step.optional === true && /: skipped/u.test(run.stdout ?? "");
const status = run.status === 0 ? (skipped ? "skipped" : "passed") : "failed";
results.push({ id: step.id, status, durationMs: Date.now() - startedAt, log: path.relative(process.cwd(), log) });
process.stdout.write(`${status.toUpperCase()} ${tier}:${step.id} (${Date.now() - startedAt}ms)\n`);
}
const failed = results.some((result) => result.status === "failed");
const skipped = results.filter((result) => result.status === "skipped").map((result) => result.id);
const status = failed ? "failed" : skipped.length > 0 ? "incomplete" : "passed";
await writeFile(
path.join(directory, "summary.json"),
`${JSON.stringify(
{
schemaVersion: 1,
tier,
generatedAt: new Date().toISOString(),
node: process.version,
environment: {
node: process.version,
nodeMajor: Number(process.versions.node.split(".")[0]),
platform: process.platform,
arch: process.arch,
},
status,
passed: status === "passed",
skipped,
steps: results,
},
null,
2,
)}\n`,
);
process.stdout.write(
`scale tier ${tier}: ${status.toUpperCase()}${skipped.length > 0 ? ` (skipped: ${skipped.join(", ")})` : ""} (artifacts/scale/${tier}/summary.json)\n`,
);
if (status === "failed") process.exitCode = 1;
else if (status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") process.exitCode = 1;
}
if (import.meta.url === `file://${process.argv[1]}`) {
await main();
}
+381
View File
@@ -0,0 +1,381 @@
import { createHash } from "node:crypto";
import { cpus } from "node:os";
import { growthExponents } from "../../src/platform/cardinality/growth-regression.ts";
import type { BenchmarkResult } from "./benchmark-result.ts";
import type { TscDiagnostics } from "./collect-tsc-diagnostics.ts";
/**
* Spec §48–§49. Stored benchmark baseline, compatibility check and relative
* regression gate.
*
* The growth gates judge the *shape* of one run. This module compares a run
* against an approved stored baseline so a change that keeps the shape but
* multiplies the cost (or silently adds compiler work) is still caught.
*
* Two kinds of metric are compared differently:
* - deterministic metrics (TypeScript types/symbols/instantiations/files) are
* exact for a given compiler, so they use a tight relative tolerance and
* require a compatible toolchain;
* - timing metrics depend on the machine, so they use a loose tolerance plus an
* absolute noise floor, and are only enforced on a compatible environment.
*/
export const SCALE_BASELINE_SCHEMA_VERSION = 1;
export type ScaleEnvironment = Readonly<{
nodeMajor: number;
platform: string;
arch: string;
cpuModel: string;
cpuCount: number;
typescript: string;
}>;
export type TimingEntry = Readonly<{
workload: string;
profile: string;
cardinality: number;
medianMs: number;
minMs: number;
}>;
export type CompilerEntry = Readonly<{
fixture: string;
files: number;
types: number;
symbols: number;
instantiations: number;
}>;
export type ScaleMetrics = Readonly<{
timing: readonly TimingEntry[];
compiler: readonly CompilerEntry[];
}>;
export type ScaleBaseline = Readonly<{
schemaVersion: typeof SCALE_BASELINE_SCHEMA_VERSION;
environment: ScaleEnvironment;
metrics: ScaleMetrics;
approval: Readonly<{
owner: string;
reason: string;
approvedAt: string;
metricsDigest: string;
}>;
}>;
export const REGRESSION_TOLERANCE = Object.freeze({
/** Deterministic compiler counts may grow at most 10% without re-approval. */
compilerRelative: 0.1,
/** Timings may grow at most 2.5x (scheduler/GC noise lives below that)... */
timingRelative: 1.5,
/** ...and never fail on differences below this absolute noise floor. */
timingNoiseFloorMs: 1,
});
/**
* Axis 1 — absolute ceilings. Machine-agnostic hard limits (roughly 50x the
* approved baseline for timings, ~2x for compiler work) that hold even with
* no comparable baseline.
*/
export const ABSOLUTE_CEILINGS = Object.freeze({
timingMinMsByProfile: Object.freeze({ S0: 5, S1: 25, S2: 150 } as Record<string, number>),
app: Object.freeze({ files: 1_500, types: 600_000, symbols: 1_600_000, instantiations: 1_500_000 }),
/**
* Generated `local-<N>` compiler fixtures: a fixed base (lib/runtime
* overhead present at any N) plus a per-local-feature allowance.
*/
localBase: Object.freeze({ files: 100, types: 5_000, symbols: 15_000, instantiations: 5_000 }),
localPerFeature: Object.freeze({ files: 2, types: 100, symbols: 250, instantiations: 150 }),
});
/** Axis 3 — growth shape (see src/platform/cardinality/growth-regression.ts). */
export const GROWTH_LIMITS = Object.freeze({
timingMaxExponent: 1.6,
timingNoiseFloorMs: 0.5,
compilerMaxExponent: 1.3,
});
export function currentEnvironment(typescript: string): ScaleEnvironment {
const cpu = cpus();
return Object.freeze({
nodeMajor: Number(process.versions.node.split(".")[0]),
platform: process.platform,
arch: process.arch,
cpuModel: cpu[0]?.model.trim() ?? "unknown",
cpuCount: cpu.length,
typescript,
});
}
function canonical(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
if (value && typeof value === "object") {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonical((value as Record<string, unknown>)[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
export function metricsDigest(metrics: ScaleMetrics): string {
return createHash("sha256").update(canonical(metrics)).digest("hex");
}
export function snapshotMetrics(
composition: readonly BenchmarkResult[],
compiler: Readonly<{
app: TscDiagnostics;
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
}>,
): ScaleMetrics {
const round = (value: number) => Math.round(value * 1_000) / 1_000;
const toCompiler = (fixture: string, diagnostics: TscDiagnostics): CompilerEntry => ({
fixture,
files: diagnostics.files,
types: diagnostics.types,
symbols: diagnostics.symbols,
instantiations: diagnostics.instantiations,
});
return Object.freeze({
timing: composition
.map((result) => ({
workload: result.name,
profile: result.profile,
cardinality: result.cardinality,
medianMs: round(result.medianMs),
minMs: round(Math.min(...result.samples)),
}))
.sort((left, right) =>
`${left.workload}/${left.profile}`.localeCompare(`${right.workload}/${right.profile}`),
),
compiler: [
toCompiler("app", compiler.app),
...compiler.local.map((entry) => toCompiler(`local-${entry.features}`, entry.diagnostics)),
].sort((left, right) => left.fixture.localeCompare(right.fixture)),
});
}
export type Compatibility = Readonly<{
/** Baseline format, digest and metric coverage are usable at all. */
structural: Readonly<{ ok: boolean; reasons: readonly string[] }>;
/** Same compiler: deterministic compiler counts are comparable. */
compiler: Readonly<{ ok: boolean; reasons: readonly string[] }>;
/** Same machine class and runtime: timings are comparable. */
timing: Readonly<{ ok: boolean; reasons: readonly string[] }>;
}>;
export function checkCompatibility(
baseline: ScaleBaseline,
environment: ScaleEnvironment,
current: ScaleMetrics,
): Compatibility {
const structural: string[] = [];
if (baseline.schemaVersion !== SCALE_BASELINE_SCHEMA_VERSION) {
structural.push(`baseline schemaVersion ${String(baseline.schemaVersion)} is not ${SCALE_BASELINE_SCHEMA_VERSION}`);
}
if (metricsDigest(baseline.metrics) !== baseline.approval?.metricsDigest) {
structural.push("baseline metrics do not match their approved digest (hand-edited?)");
}
const timingKey = (entry: TimingEntry) => `${entry.workload}/${entry.profile}/${entry.cardinality}`;
const baselineTiming = new Set(baseline.metrics.timing.map(timingKey));
const baselineCompiler = new Set(baseline.metrics.compiler.map((entry) => entry.fixture));
for (const entry of current.timing) {
if (!baselineTiming.has(timingKey(entry))) structural.push(`no baseline for timing ${timingKey(entry)}`);
}
for (const entry of current.compiler) {
if (!baselineCompiler.has(entry.fixture)) structural.push(`no baseline for compiler fixture ${entry.fixture}`);
}
const compiler: string[] = [];
if (baseline.environment.typescript !== environment.typescript) {
compiler.push(`TypeScript ${environment.typescript} differs from baseline ${baseline.environment.typescript}`);
}
const timing: string[] = [];
const base = baseline.environment;
if (base.nodeMajor !== environment.nodeMajor) timing.push(`Node ${environment.nodeMajor} vs baseline ${base.nodeMajor}`);
if (base.platform !== environment.platform || base.arch !== environment.arch) {
timing.push(`${environment.platform}/${environment.arch} vs baseline ${base.platform}/${base.arch}`);
}
if (base.cpuModel !== environment.cpuModel) timing.push(`CPU "${environment.cpuModel}" vs baseline "${base.cpuModel}"`);
return Object.freeze({
structural: { ok: structural.length === 0, reasons: structural },
compiler: { ok: compiler.length === 0, reasons: compiler },
timing: { ok: timing.length === 0, reasons: timing },
});
}
export type RegressionFinding = Readonly<{
metric: string;
baseline: number;
current: number;
limit: number;
passed: boolean;
}>;
export type AxisResult = Readonly<{
passed: boolean;
findings: readonly RegressionFinding[];
}>;
export type BaselineComparison = Readonly<{
/**
* `passed` only when every axis was evaluated and held. `incomplete` when the
* environment is not timing-compatible: timing regression was not compared,
* so it is not evidence of a pass.
*/
status: "passed" | "failed" | "incomplete";
passed: boolean;
compatibility: Compatibility;
absolute: AxisResult;
growth: AxisResult;
compiler: readonly RegressionFinding[];
/** Empty when the environment is not timing-compatible. */
timing: readonly RegressionFinding[];
failures: readonly string[];
}>;
function finding(metric: string, baseline: number, current: number, limit: number): RegressionFinding {
return { metric, baseline, current, limit: Math.round(limit * 1_000) / 1_000, passed: current <= limit };
}
export function checkAbsoluteCeilings(current: ScaleMetrics): AxisResult {
const findings: RegressionFinding[] = [];
for (const entry of current.timing) {
const limit = ABSOLUTE_CEILINGS.timingMinMsByProfile[entry.profile];
if (limit !== undefined) {
findings.push(finding(`ceiling:time:${entry.workload}/${entry.profile}`, 0, entry.minMs, limit));
}
}
for (const entry of current.compiler) {
const local = /^local-(\d+)$/u.exec(entry.fixture);
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
const limit = local
? ABSOLUTE_CEILINGS.localBase[metric] +
ABSOLUTE_CEILINGS.localPerFeature[metric] * Number(local[1])
: entry.fixture === "app"
? ABSOLUTE_CEILINGS.app[metric]
: Number.POSITIVE_INFINITY;
findings.push(finding(`ceiling:tsc:${entry.fixture}:${metric}`, 0, entry[metric], limit));
}
}
return { passed: findings.every((item) => item.passed), findings };
}
export function checkGrowthShape(current: ScaleMetrics): AxisResult {
const findings: RegressionFinding[] = [];
const byWorkload = new Map<string, { n: number; cost: number }[]>();
for (const entry of current.timing) {
const series = byWorkload.get(entry.workload) ?? [];
series.push({ n: entry.cardinality, cost: entry.minMs });
byWorkload.set(entry.workload, series);
}
for (const [workload, series] of byWorkload) {
if (series.length < 2) continue;
const worst = Math.max(...growthExponents(series, GROWTH_LIMITS.timingNoiseFloorMs));
findings.push(finding(`growth:time:${workload}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.timingMaxExponent));
}
const local = current.compiler
.map((entry) => ({ entry, n: Number(/^local-(\d+)$/u.exec(entry.fixture)?.[1]) }))
.filter(({ n }) => Number.isFinite(n));
if (local.length >= 2) {
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
const worst = Math.max(...growthExponents(local.map(({ entry, n }) => ({ n, cost: entry[metric] }))));
findings.push(finding(`growth:tsc:local:${metric}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.compilerMaxExponent));
}
}
return { passed: findings.every((item) => item.passed), findings };
}
export function compareToBaseline(
baseline: ScaleBaseline,
environment: ScaleEnvironment,
current: ScaleMetrics,
tolerance = REGRESSION_TOLERANCE,
): BaselineComparison {
const compatibility = checkCompatibility(baseline, environment, current);
const failures: string[] = [...compatibility.structural.reasons];
// A different compiler changes every deterministic count: the baseline must
// be re-approved, it is never silently compared or silently skipped.
if (!compatibility.compiler.ok) {
failures.push(
...compatibility.compiler.reasons.map((reason) => `${reason}; re-approve with update:scale-baseline`),
);
}
const absolute = checkAbsoluteCeilings(current);
const growth = checkGrowthShape(current);
for (const item of [...absolute.findings, ...growth.findings]) {
if (!item.passed) failures.push(`${item.metric} ${item.current} exceeds ${item.limit}`);
}
const compiler: RegressionFinding[] = [];
if (compatibility.structural.ok && compatibility.compiler.ok) {
const byFixture = new Map(baseline.metrics.compiler.map((entry) => [entry.fixture, entry]));
for (const entry of current.compiler) {
const reference = byFixture.get(entry.fixture);
if (!reference) continue;
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
const item = finding(
`tsc:${entry.fixture}:${metric}`,
reference[metric],
entry[metric],
Math.floor(reference[metric] * (1 + tolerance.compilerRelative)),
);
compiler.push(item);
if (!item.passed) {
failures.push(`${item.metric} ${item.current} exceeds ${item.limit} (baseline ${item.baseline})`);
}
}
}
}
const timing: RegressionFinding[] = [];
if (compatibility.structural.ok && compatibility.timing.ok) {
const key = (entry: TimingEntry) => `${entry.workload}/${entry.profile}`;
const byKey = new Map(baseline.metrics.timing.map((entry) => [key(entry), entry]));
for (const entry of current.timing) {
const reference = byKey.get(key(entry));
if (!reference) continue;
// Best-observed samples: contention only ever adds time.
const item = finding(
`time:${key(entry)}`,
reference.minMs,
entry.minMs,
Math.max(reference.minMs * (1 + tolerance.timingRelative), reference.minMs + tolerance.timingNoiseFloorMs),
);
timing.push(item);
if (!item.passed) {
failures.push(`${item.metric} ${item.current}ms exceeds ${item.limit}ms (baseline ${item.baseline}ms)`);
}
}
}
const status =
failures.length > 0 ? "failed" : compatibility.timing.ok ? "passed" : "incomplete";
return Object.freeze({
status,
passed: status === "passed",
compatibility,
absolute,
growth,
compiler,
timing,
failures,
});
}
export function approveBaseline(
environment: ScaleEnvironment,
metrics: ScaleMetrics,
approval: Readonly<{ owner: string; reason: string; approvedAt: string }>,
): ScaleBaseline {
return Object.freeze({
schemaVersion: SCALE_BASELINE_SCHEMA_VERSION,
environment,
metrics,
approval: { ...approval, metricsDigest: metricsDigest(metrics) },
});
}