refactor: 도메인이 100개 1000개씩 늘어나도 유지보수 쉬운 구조로 변경
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
/**
|
||||
* Spec §48. Machine-readable benchmark result.
|
||||
*/
|
||||
export type BenchmarkProfile = "S0" | "S1" | "S2" | "S3" | "S4";
|
||||
|
||||
export interface BenchmarkResult {
|
||||
readonly name: string;
|
||||
readonly profile: BenchmarkProfile;
|
||||
readonly cardinality: number;
|
||||
readonly samples: readonly number[];
|
||||
readonly medianMs: number;
|
||||
readonly heapBytes?: number;
|
||||
readonly metadata: Readonly<Record<string, string | number | boolean>>;
|
||||
}
|
||||
|
||||
export type BenchmarkArtifact = Readonly<{
|
||||
schemaVersion: 1;
|
||||
suite: string;
|
||||
generatedAt: string;
|
||||
node: string;
|
||||
passed: boolean;
|
||||
results: readonly BenchmarkResult[];
|
||||
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
|
||||
}>;
|
||||
|
||||
const PROFILES: ReadonlySet<string> = new Set(["S0", "S1", "S2", "S3", "S4"]);
|
||||
|
||||
export function median(values: readonly number[]): number {
|
||||
if (values.length === 0) throw new TypeError("median of an empty series");
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 === 1
|
||||
? (sorted[middle] as number)
|
||||
: ((sorted[middle - 1] as number) + (sorted[middle] as number)) / 2;
|
||||
}
|
||||
|
||||
export function assertBenchmarkResult(value: BenchmarkResult): BenchmarkResult {
|
||||
if (typeof value.name !== "string" || value.name.length === 0) {
|
||||
throw new TypeError("benchmark name required");
|
||||
}
|
||||
if (!PROFILES.has(value.profile)) {
|
||||
throw new TypeError(`unknown benchmark profile ${String(value.profile)}`);
|
||||
}
|
||||
if (!Number.isSafeInteger(value.cardinality) || value.cardinality < 0) {
|
||||
throw new TypeError("benchmark cardinality must be a non-negative integer");
|
||||
}
|
||||
if (
|
||||
value.samples.length === 0 ||
|
||||
value.samples.some((sample) => !Number.isFinite(sample) || sample < 0)
|
||||
) {
|
||||
throw new TypeError("benchmark samples must be finite non-negative numbers");
|
||||
}
|
||||
if (value.medianMs !== median(value.samples)) {
|
||||
throw new TypeError("benchmark medianMs must be the median of its samples");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function writeBenchmarkArtifact(
|
||||
path: string,
|
||||
artifact: BenchmarkArtifact,
|
||||
): Promise<void> {
|
||||
for (const result of artifact.results) assertBenchmarkResult(result);
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, `${JSON.stringify(artifact, null, 2)}\n`);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { performance } from "node:perf_hooks";
|
||||
|
||||
import {
|
||||
assertBenchmarkResult,
|
||||
median,
|
||||
type BenchmarkProfile,
|
||||
type BenchmarkResult,
|
||||
} from "./benchmark-result.ts";
|
||||
|
||||
/**
|
||||
* Spec §49. Repeated samples with a median, so one noisy scheduler slice can
|
||||
* neither fail a linear implementation nor hide a slow one.
|
||||
*
|
||||
* `run` either does the work (and is timed here) or returns a duration it
|
||||
* measured itself, which lets tests inject exact durations.
|
||||
*/
|
||||
export type BenchmarkOptions = Readonly<{
|
||||
samples: number;
|
||||
warmup?: number;
|
||||
profile?: BenchmarkProfile;
|
||||
cardinality?: number;
|
||||
metadata?: Readonly<Record<string, string | number | boolean>>;
|
||||
measureHeap?: boolean;
|
||||
run: () => unknown;
|
||||
}>;
|
||||
|
||||
function heapUsed(): number {
|
||||
return process.memoryUsage().heapUsed;
|
||||
}
|
||||
|
||||
export async function benchmark(
|
||||
name: string,
|
||||
options: BenchmarkOptions,
|
||||
): Promise<BenchmarkResult> {
|
||||
if (!Number.isSafeInteger(options.samples) || options.samples < 1) {
|
||||
throw new TypeError("benchmark samples must be a positive integer");
|
||||
}
|
||||
for (let index = 0; index < (options.warmup ?? 0); index += 1) {
|
||||
await options.run();
|
||||
}
|
||||
const samples: number[] = [];
|
||||
let peakHeapDelta = 0;
|
||||
for (let index = 0; index < options.samples; index += 1) {
|
||||
const heapBefore = options.measureHeap ? heapUsed() : 0;
|
||||
const startedAt = performance.now();
|
||||
const reported = await options.run();
|
||||
const elapsed = performance.now() - startedAt;
|
||||
if (options.measureHeap) {
|
||||
peakHeapDelta = Math.max(peakHeapDelta, heapUsed() - heapBefore);
|
||||
}
|
||||
samples.push(typeof reported === "number" && Number.isFinite(reported) ? reported : elapsed);
|
||||
}
|
||||
return assertBenchmarkResult(
|
||||
Object.freeze({
|
||||
name,
|
||||
profile: options.profile ?? "S0",
|
||||
cardinality: options.cardinality ?? 0,
|
||||
samples: Object.freeze(samples),
|
||||
medianMs: median(samples),
|
||||
...(options.measureHeap ? { heapBytes: Math.max(0, peakHeapDelta) } : {}),
|
||||
metadata: Object.freeze({ ...(options.metadata ?? {}) }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,554 @@
|
||||
import { createReadStream } from "node:fs";
|
||||
import { access, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
classifyCounterSeries,
|
||||
classifyMemorySlope,
|
||||
type MemorySlopeClass,
|
||||
} from "./memory-slope.ts";
|
||||
|
||||
/**
|
||||
* Spec §33 / §44. Browser soak on the existing Playwright runner.
|
||||
*
|
||||
* Serves the built `dist/` on an OS-assigned 127.0.0.1 port and keeps one
|
||||
* long-lived page cycling through the app's routes by client-side navigation.
|
||||
* The served runtime config turns telemetry on (pointed at a local collector)
|
||||
* and points the API at a stub that always answers 503, so every cycle also
|
||||
* exercises bounded retries and telemetry under a sustained backend failure.
|
||||
*
|
||||
* Collected per sample: JS heap, DOM nodes, event listeners, live timers,
|
||||
* in-flight fetches, Web Storage size. Collected overall: cycles, navigation
|
||||
* failures, API requests, telemetry POSTs/events. Without a build or browser
|
||||
* the result is `skipped` (never a pass); too little evidence fails.
|
||||
*/
|
||||
export type SoakSample = Readonly<{
|
||||
elapsedMs: number;
|
||||
cycles: number;
|
||||
heapBytes: number;
|
||||
domNodes: number;
|
||||
listeners: number;
|
||||
activeTimers: number;
|
||||
inFlightRequests: number;
|
||||
storageKeys: number;
|
||||
storageBytes: number;
|
||||
}>;
|
||||
|
||||
export type SoakSummary = Readonly<{
|
||||
cycles: number;
|
||||
navigationFailures: number;
|
||||
maxHeapBytes: number;
|
||||
heapClassification: MemorySlopeClass;
|
||||
listenerDelta: number;
|
||||
listenerClassification: MemorySlopeClass;
|
||||
timerDelta: number;
|
||||
maxActiveTimers: number;
|
||||
domNodeDelta: number;
|
||||
maxInFlightRequests: number;
|
||||
finalInFlightRequests: number;
|
||||
storageKeyDelta: number;
|
||||
storageByteDelta: number;
|
||||
apiRequests: number;
|
||||
referenceVisits: number;
|
||||
settledReferenceVisits: number;
|
||||
apiRequestsPerReferenceVisit: number;
|
||||
telemetryPosts: number;
|
||||
telemetryEvents: number;
|
||||
maxTelemetryPostsPerMinute: number;
|
||||
}>;
|
||||
|
||||
export type SoakCheck = Readonly<{ name: string; passed: boolean; detail: string }>;
|
||||
|
||||
export type SoakArtifact = Readonly<{
|
||||
schemaVersion: 2;
|
||||
status: "passed" | "failed" | "skipped";
|
||||
reason?: string;
|
||||
durationMs: number;
|
||||
classification?: MemorySlopeClass;
|
||||
summary?: SoakSummary;
|
||||
checks?: readonly SoakCheck[];
|
||||
/** Identity of the soaked build (from dist/release-manifest.json). */
|
||||
build?: Readonly<{ buildId?: string; releaseId?: string; builtAt?: string }>;
|
||||
browser?: string;
|
||||
samples: readonly SoakSample[];
|
||||
}>;
|
||||
|
||||
/** Thresholds a soak must meet; each is checked and recorded by name. */
|
||||
export const SOAK_LIMITS = Object.freeze({
|
||||
minCycles: 200,
|
||||
minPostWarmupSamples: 3,
|
||||
maxListenerDelta: 50,
|
||||
maxTimerDelta: 10,
|
||||
maxStorageKeyDelta: 2,
|
||||
/** One logical execution: 1 attempt + at most 2 retries. */
|
||||
maxApiRequestsPerReferenceVisit: 3,
|
||||
/** Two bounded pipelines (platform events + feature events) x 20/min. */
|
||||
maxTelemetryPostsPerMinute: 40,
|
||||
});
|
||||
|
||||
export function summarizeSoak(
|
||||
samples: readonly Pick<SoakSample, "elapsedMs" | "heapBytes">[],
|
||||
warmupMs: number,
|
||||
): MemorySlopeClass {
|
||||
return classifyMemorySlope(
|
||||
samples.map((sample) => ({
|
||||
minute: sample.elapsedMs / 60_000,
|
||||
heapMb: sample.heapBytes / (1024 * 1024),
|
||||
})),
|
||||
{ warmupMinutes: warmupMs / 60_000, maxSlopeMbPerMinute: 1, maxRelativeGrowth: 0.2 },
|
||||
);
|
||||
}
|
||||
|
||||
export type SoakTotals = Readonly<{
|
||||
cycles: number;
|
||||
navigationFailures: number;
|
||||
referenceVisits: number;
|
||||
settledReferenceVisits: number;
|
||||
finalInFlightRequests: number;
|
||||
apiRequests: number;
|
||||
telemetryPosts: number;
|
||||
telemetryEvents: number;
|
||||
telemetryPostTimes: readonly number[];
|
||||
}>;
|
||||
|
||||
export function evaluateSoak(
|
||||
samples: readonly SoakSample[],
|
||||
totals: SoakTotals,
|
||||
warmupMs: number,
|
||||
): Readonly<{ summary: SoakSummary; checks: readonly SoakCheck[] }> {
|
||||
const measured = samples.filter((sample) => sample.elapsedMs >= warmupMs);
|
||||
const first = measured[0];
|
||||
const last = measured[measured.length - 1];
|
||||
const delta = (pick: (sample: SoakSample) => number) =>
|
||||
first && last ? pick(last) - pick(first) : 0;
|
||||
let maxTelemetryPostsPerMinute = 0;
|
||||
for (const time of totals.telemetryPostTimes) {
|
||||
const inWindow = totals.telemetryPostTimes.filter(
|
||||
(other) => other >= time && other < time + 60_000,
|
||||
).length;
|
||||
maxTelemetryPostsPerMinute = Math.max(maxTelemetryPostsPerMinute, inWindow);
|
||||
}
|
||||
const summary: SoakSummary = {
|
||||
cycles: totals.cycles,
|
||||
navigationFailures: totals.navigationFailures,
|
||||
maxHeapBytes: Math.max(0, ...samples.map((sample) => sample.heapBytes)),
|
||||
heapClassification: summarizeSoak(samples, warmupMs),
|
||||
listenerDelta: delta((sample) => sample.listeners),
|
||||
listenerClassification: classifyCounterSeries(
|
||||
measured.map((sample) => sample.listeners),
|
||||
{ maxSlopePerSample: 1 },
|
||||
),
|
||||
timerDelta: delta((sample) => sample.activeTimers),
|
||||
maxActiveTimers: Math.max(0, ...samples.map((sample) => sample.activeTimers)),
|
||||
domNodeDelta: delta((sample) => sample.domNodes),
|
||||
maxInFlightRequests: Math.max(0, ...samples.map((sample) => sample.inFlightRequests)),
|
||||
finalInFlightRequests: totals.finalInFlightRequests,
|
||||
storageKeyDelta: delta((sample) => sample.storageKeys),
|
||||
storageByteDelta: delta((sample) => sample.storageBytes),
|
||||
apiRequests: totals.apiRequests,
|
||||
referenceVisits: totals.referenceVisits,
|
||||
settledReferenceVisits: totals.settledReferenceVisits,
|
||||
apiRequestsPerReferenceVisit:
|
||||
totals.referenceVisits === 0
|
||||
? 0
|
||||
: Math.round((totals.apiRequests / totals.referenceVisits) * 1_000) / 1_000,
|
||||
telemetryPosts: totals.telemetryPosts,
|
||||
telemetryEvents: totals.telemetryEvents,
|
||||
maxTelemetryPostsPerMinute,
|
||||
};
|
||||
const checks: SoakCheck[] = [
|
||||
{ name: "cycles", passed: summary.cycles >= SOAK_LIMITS.minCycles, detail: `${summary.cycles} >= ${SOAK_LIMITS.minCycles}` },
|
||||
{
|
||||
name: "post-warmup samples",
|
||||
passed: measured.length >= SOAK_LIMITS.minPostWarmupSamples,
|
||||
detail: `${measured.length} >= ${SOAK_LIMITS.minPostWarmupSamples}`,
|
||||
},
|
||||
{ name: "navigation", passed: summary.navigationFailures === 0, detail: `${summary.navigationFailures} failures` },
|
||||
{ name: "heap slope", passed: summary.heapClassification === "stable", detail: summary.heapClassification },
|
||||
{
|
||||
name: "listeners",
|
||||
passed:
|
||||
summary.listenerClassification === "stable" &&
|
||||
summary.listenerDelta <= SOAK_LIMITS.maxListenerDelta,
|
||||
detail: `${summary.listenerClassification}, delta ${summary.listenerDelta}`,
|
||||
},
|
||||
{
|
||||
name: "timers",
|
||||
passed: summary.timerDelta <= SOAK_LIMITS.maxTimerDelta,
|
||||
detail: `delta ${summary.timerDelta}, max ${summary.maxActiveTimers}`,
|
||||
},
|
||||
{
|
||||
name: "in-flight settles",
|
||||
passed: summary.finalInFlightRequests === 0,
|
||||
detail: `final ${summary.finalInFlightRequests}, max ${summary.maxInFlightRequests}`,
|
||||
},
|
||||
{
|
||||
name: "storage growth",
|
||||
passed: summary.storageKeyDelta <= SOAK_LIMITS.maxStorageKeyDelta,
|
||||
detail: `keys ${summary.storageKeyDelta}, bytes ${summary.storageByteDelta}`,
|
||||
},
|
||||
{
|
||||
name: "backend failure exercised",
|
||||
passed:
|
||||
summary.settledReferenceVisits > 0 && summary.apiRequests > 0 && summary.telemetryEvents > 0,
|
||||
detail: `${summary.settledReferenceVisits} settled visits, ${summary.apiRequests} API requests, ${summary.telemetryEvents} telemetry events`,
|
||||
},
|
||||
{
|
||||
name: "request amplification",
|
||||
passed:
|
||||
summary.referenceVisits > 0 &&
|
||||
summary.apiRequestsPerReferenceVisit <= SOAK_LIMITS.maxApiRequestsPerReferenceVisit,
|
||||
detail: `${summary.apiRequests} requests / ${summary.referenceVisits} visits`,
|
||||
},
|
||||
{
|
||||
name: "telemetry rate",
|
||||
passed: summary.maxTelemetryPostsPerMinute <= SOAK_LIMITS.maxTelemetryPostsPerMinute,
|
||||
detail: `max ${summary.maxTelemetryPostsPerMinute}/min, ${summary.telemetryPosts} posts, ${summary.telemetryEvents} events`,
|
||||
},
|
||||
];
|
||||
return { summary, checks };
|
||||
}
|
||||
|
||||
const CONTENT_TYPES: Readonly<Record<string, string>> = {
|
||||
".css": "text/css; charset=utf-8",
|
||||
".html": "text/html; charset=utf-8",
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".json": "application/json; charset=utf-8",
|
||||
".svg": "image/svg+xml",
|
||||
".png": "image/png",
|
||||
".webmanifest": "application/manifest+json",
|
||||
};
|
||||
|
||||
type Counters = {
|
||||
apiRequests: number;
|
||||
telemetryPosts: number;
|
||||
telemetryEvents: number;
|
||||
telemetryPostTimes: number[];
|
||||
};
|
||||
|
||||
async function serveSoakTarget(
|
||||
root: string,
|
||||
counters: Counters,
|
||||
startedAt: () => number,
|
||||
): Promise<Readonly<{ server: Server; baseUrl: string }>> {
|
||||
let baseUrl = "";
|
||||
const baseConfig = JSON.parse(
|
||||
await readFile(path.join(root, "config.json"), "utf8"),
|
||||
) as Record<string, unknown>;
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (url.pathname === "/__telemetry") {
|
||||
let body = "";
|
||||
request.on("data", (chunk: Buffer) => {
|
||||
body += chunk.toString("utf8");
|
||||
});
|
||||
request.on("end", () => {
|
||||
counters.telemetryPosts += 1;
|
||||
counters.telemetryPostTimes.push(Date.now() - startedAt());
|
||||
try {
|
||||
const parsed = JSON.parse(body) as { events?: unknown[] };
|
||||
counters.telemetryEvents += Array.isArray(parsed.events) ? parsed.events.length : 0;
|
||||
} catch {
|
||||
// Counted as a post regardless of its body.
|
||||
}
|
||||
response.writeHead(204).end();
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.pathname.startsWith("/api/")) {
|
||||
counters.apiRequests += 1;
|
||||
response.writeHead(503, { "Content-Type": "application/json" }).end("{}");
|
||||
return;
|
||||
}
|
||||
if (url.pathname === "/config.json") {
|
||||
response
|
||||
.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" })
|
||||
.end(
|
||||
JSON.stringify({
|
||||
...baseConfig,
|
||||
API_BASE_URL: `${baseUrl}/api/`,
|
||||
TELEMETRY_ENABLED: true,
|
||||
TELEMETRY_ENDPOINT: `${baseUrl}/__telemetry`,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const requested = path.resolve(root, `.${decodeURIComponent(url.pathname)}`);
|
||||
if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) {
|
||||
response.writeHead(403).end();
|
||||
return;
|
||||
}
|
||||
const details = await stat(requested).catch(() => null);
|
||||
const file = details?.isFile() ? requested : path.join(root, "index.html");
|
||||
response.writeHead(200, {
|
||||
"Content-Type": CONTENT_TYPES[path.extname(file)] ?? "application/octet-stream",
|
||||
"Cache-Control": "no-store",
|
||||
});
|
||||
createReadStream(file).pipe(response);
|
||||
} catch {
|
||||
response.writeHead(404).end();
|
||||
}
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
baseUrl = `http://127.0.0.1:${port}`;
|
||||
return { server, baseUrl };
|
||||
}
|
||||
|
||||
const PAGE_PROBES = `
|
||||
(() => {
|
||||
const live = new Set();
|
||||
const setT = window.setTimeout, clearT = window.clearTimeout;
|
||||
const setI = window.setInterval, clearI = window.clearInterval;
|
||||
window.setTimeout = (fn, ms, ...rest) => {
|
||||
const id = setT(() => { live.delete(id); typeof fn === "function" && fn(...rest); }, ms);
|
||||
live.add(id); return id;
|
||||
};
|
||||
window.clearTimeout = (id) => { live.delete(id); clearT(id); };
|
||||
window.setInterval = (fn, ms, ...rest) => { const id = setI(fn, ms, ...rest); live.add(id); return id; };
|
||||
window.clearInterval = (id) => { live.delete(id); clearI(id); };
|
||||
let inFlight = 0;
|
||||
const nativeFetch = window.fetch.bind(window);
|
||||
window.fetch = (...args) => {
|
||||
inFlight += 1;
|
||||
return nativeFetch(...args).finally(() => { inFlight -= 1; });
|
||||
};
|
||||
const storageSize = (storage) => {
|
||||
let bytes = 0;
|
||||
for (let i = 0; i < storage.length; i += 1) {
|
||||
const key = storage.key(i) ?? "";
|
||||
bytes += key.length + (storage.getItem(key) ?? "").length;
|
||||
}
|
||||
return { keys: storage.length, bytes };
|
||||
};
|
||||
Object.defineProperty(window, "__soakProbe", {
|
||||
value: () => {
|
||||
const local = storageSize(window.localStorage);
|
||||
const session = storageSize(window.sessionStorage);
|
||||
return {
|
||||
activeTimers: live.size,
|
||||
inFlight,
|
||||
storageKeys: local.keys + session.keys,
|
||||
storageBytes: local.bytes + session.bytes,
|
||||
};
|
||||
},
|
||||
});
|
||||
})();
|
||||
`;
|
||||
|
||||
type Probe = { activeTimers: number; inFlight: number; storageKeys: number; storageBytes: number };
|
||||
|
||||
export async function runBrowserSoak(
|
||||
options: Readonly<{
|
||||
distRoot: string;
|
||||
durationMs: number;
|
||||
intervalMs: number;
|
||||
warmupMs: number;
|
||||
routes: readonly string[];
|
||||
}>,
|
||||
): Promise<SoakArtifact> {
|
||||
try {
|
||||
await access(path.join(options.distRoot, "index.html"));
|
||||
await access(path.join(options.distRoot, "config.json"));
|
||||
} catch {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: "skipped",
|
||||
reason: "no built dist/index.html + config.json",
|
||||
durationMs: 0,
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
let chromium: (typeof import("@playwright/test"))["chromium"];
|
||||
try {
|
||||
({ chromium } = await import("@playwright/test"));
|
||||
} catch {
|
||||
return { schemaVersion: 2, status: "skipped", reason: "playwright unavailable", durationMs: 0, samples: [] };
|
||||
}
|
||||
const counters: Counters = {
|
||||
apiRequests: 0,
|
||||
telemetryPosts: 0,
|
||||
telemetryEvents: 0,
|
||||
telemetryPostTimes: [],
|
||||
};
|
||||
let started = Date.now();
|
||||
const { server, baseUrl } = await serveSoakTarget(
|
||||
path.resolve(options.distRoot),
|
||||
counters,
|
||||
() => started,
|
||||
);
|
||||
let browser: Awaited<ReturnType<typeof chromium.launch>> | undefined;
|
||||
try {
|
||||
try {
|
||||
browser = await chromium.launch();
|
||||
} catch (error) {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: "skipped",
|
||||
reason: `browser unavailable: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`,
|
||||
durationMs: 0,
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
const page = await browser.newPage();
|
||||
await page.addInitScript(PAGE_PROBES);
|
||||
const cdp = await page.context().newCDPSession(page);
|
||||
await cdp.send("Performance.enable");
|
||||
await page.goto(`${baseUrl}/examples/reference-resources`, { waitUntil: "load" });
|
||||
await page.waitForSelector("#root > *", { timeout: 15_000 });
|
||||
// Protected routes need a session: start the local demo session once, so
|
||||
// the reference feature really dispatches its (failing) API requests.
|
||||
const login = page.getByRole("button", { name: /로그인 시작|Sign in|Start sign-in/u });
|
||||
if (await login.count()) {
|
||||
await login.first().click();
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
|
||||
const samples: SoakSample[] = [];
|
||||
started = Date.now();
|
||||
let nextSample = 0;
|
||||
let cycles = 0;
|
||||
let navigationFailures = 0;
|
||||
let referenceVisits = 0;
|
||||
let settledReferenceVisits = 0;
|
||||
const probe = async () =>
|
||||
(await page.evaluate(() =>
|
||||
(window as unknown as { __soakProbe: () => Probe }).__soakProbe(),
|
||||
)) as Probe;
|
||||
while (Date.now() - started < options.durationMs) {
|
||||
const route = options.routes[cycles % options.routes.length] ?? "/";
|
||||
cycles += 1;
|
||||
if (route === "/examples/reference-resources") referenceVisits += 1;
|
||||
// Client-side navigation keeps one long-lived document, like a real session.
|
||||
await page.evaluate((target) => {
|
||||
window.history.pushState({}, "", target);
|
||||
window.dispatchEvent(new PopStateEvent("popstate"));
|
||||
}, route);
|
||||
await page.waitForTimeout(50);
|
||||
const rendered = await page.evaluate(
|
||||
(target) =>
|
||||
window.location.pathname === target &&
|
||||
(document.getElementById("root")?.childElementCount ?? 0) > 0,
|
||||
route,
|
||||
);
|
||||
if (!rendered) navigationFailures += 1;
|
||||
// Most visits leave after 50ms, which cancels the query (a cancelled
|
||||
// request is intentionally not reported). Every 5th reference visit
|
||||
// dwells until its interaction settles, so the failing backend is really
|
||||
// observed end to end: bounded retries, then feature/platform telemetry.
|
||||
if (route === "/examples/reference-resources" && referenceVisits % 5 === 1) {
|
||||
await page.waitForTimeout(200);
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => (window as unknown as { __soakProbe: () => Probe }).__soakProbe().inFlight === 0,
|
||||
undefined,
|
||||
{ timeout: 5_000 },
|
||||
)
|
||||
.catch(() => undefined);
|
||||
settledReferenceVisits += 1;
|
||||
}
|
||||
const elapsedMs = Date.now() - started;
|
||||
if (elapsedMs >= nextSample) {
|
||||
nextSample = elapsedMs + options.intervalMs;
|
||||
await cdp.send("HeapProfiler.collectGarbage").catch(() => undefined);
|
||||
const { metrics } = (await cdp.send("Performance.getMetrics")) as {
|
||||
metrics: { name: string; value: number }[];
|
||||
};
|
||||
const metric = (name: string) => metrics.find((entry) => entry.name === name)?.value ?? 0;
|
||||
const probed = await probe();
|
||||
samples.push({
|
||||
elapsedMs,
|
||||
cycles,
|
||||
heapBytes: metric("JSHeapUsedSize"),
|
||||
domNodes: metric("Nodes"),
|
||||
listeners: metric("JSEventListeners"),
|
||||
activeTimers: probed.activeTimers,
|
||||
inFlightRequests: probed.inFlight,
|
||||
storageKeys: probed.storageKeys,
|
||||
storageBytes: probed.storageBytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
// Let the last interaction's bounded retries and deadlines settle.
|
||||
let finalInFlightRequests = (await probe()).inFlight;
|
||||
for (let wait = 0; wait < 30 && finalInFlightRequests > 0; wait += 1) {
|
||||
await page.waitForTimeout(500);
|
||||
finalInFlightRequests = (await probe()).inFlight;
|
||||
}
|
||||
const evaluation = evaluateSoak(
|
||||
samples,
|
||||
{
|
||||
cycles,
|
||||
navigationFailures,
|
||||
referenceVisits,
|
||||
settledReferenceVisits,
|
||||
finalInFlightRequests,
|
||||
apiRequests: counters.apiRequests,
|
||||
telemetryPosts: counters.telemetryPosts,
|
||||
telemetryEvents: counters.telemetryEvents,
|
||||
telemetryPostTimes: counters.telemetryPostTimes,
|
||||
},
|
||||
options.warmupMs,
|
||||
);
|
||||
const build = await readFile(path.join(options.distRoot, "release-manifest.json"), "utf8")
|
||||
.then((raw) => {
|
||||
const manifest = JSON.parse(raw) as Record<string, unknown>;
|
||||
const pick = (key: string) =>
|
||||
typeof manifest[key] === "string" ? (manifest[key] as string) : undefined;
|
||||
return {
|
||||
...(pick("buildId") ? { buildId: pick("buildId") } : {}),
|
||||
...(pick("releaseId") ? { releaseId: pick("releaseId") } : {}),
|
||||
...(pick("builtAt") ? { builtAt: pick("builtAt") } : {}),
|
||||
};
|
||||
})
|
||||
.catch(() => ({}));
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: evaluation.checks.every((check) => check.passed) ? "passed" : "failed",
|
||||
durationMs: options.durationMs,
|
||||
classification: evaluation.summary.heapClassification,
|
||||
summary: evaluation.summary,
|
||||
checks: evaluation.checks,
|
||||
build,
|
||||
browser: `chromium ${browser.version()}`,
|
||||
samples,
|
||||
};
|
||||
} finally {
|
||||
await browser?.close().catch(() => undefined);
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argument = (name: string, fallback: string) =>
|
||||
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;
|
||||
const artifact = path.resolve(argument("artifact", "artifacts/scale/browser-soak.json"));
|
||||
const result = await runBrowserSoak({
|
||||
distRoot: argument("dist", "dist"),
|
||||
durationMs: Number(argument("duration-ms", "60000")),
|
||||
intervalMs: Number(argument("interval-ms", "5000")),
|
||||
warmupMs: Number(argument("warmup-ms", "15000")),
|
||||
routes: argument(
|
||||
"routes",
|
||||
"/,/examples/reference-resources,/examples/platform,/examples/ui,/examples/states,/examples/reference-resources/status",
|
||||
).split(","),
|
||||
});
|
||||
await mkdir(path.dirname(artifact), { recursive: true });
|
||||
await writeFile(artifact, `${JSON.stringify(result, null, 2)}\n`);
|
||||
process.stdout.write(
|
||||
`browser soak: ${result.status}${result.reason ? ` (${result.reason})` : ""}` +
|
||||
`${result.classification ? ` classification=${result.classification}` : ""} samples=${result.samples.length}` +
|
||||
`${result.summary ? ` cycles=${result.summary.cycles}` : ""}\n`,
|
||||
);
|
||||
for (const check of result.checks ?? []) {
|
||||
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
|
||||
}
|
||||
if (result.status === "failed") process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { collectTscScale } from "./collect-tsc-diagnostics.ts";
|
||||
import { runCompositionScale } from "./run-composition-scale.ts";
|
||||
import {
|
||||
approveBaseline,
|
||||
compareToBaseline,
|
||||
currentEnvironment,
|
||||
snapshotMetrics,
|
||||
type ScaleBaseline,
|
||||
} from "./scale-baseline.ts";
|
||||
|
||||
/**
|
||||
* Spec §48–§49. Relative regression gate against the stored, approved scale
|
||||
* baseline (`config/scale/benchmark-baseline.json`).
|
||||
*
|
||||
* node scripts/scale/check-scale-baseline.ts compare, fail on regression
|
||||
* SCALE_BASELINE_OWNER=... SCALE_BASELINE_REASON=... \
|
||||
* node scripts/scale/check-scale-baseline.ts --update re-measure and re-approve
|
||||
*
|
||||
* The comparison is written to artifacts/scale/baseline-comparison.json (or
|
||||
* `--artifact=`). Updating is an explicit, attributed act; the check never
|
||||
* rewrites the baseline.
|
||||
*/
|
||||
export const BASELINE_PATH = "config/scale/benchmark-baseline.json";
|
||||
export const LOCAL_TSC_SIZES = Object.freeze([25, 100, 400]);
|
||||
|
||||
function typescriptVersion(): string {
|
||||
const result = spawnSync(path.resolve("node_modules/.bin/tsc"), ["-v"], { encoding: "utf8" });
|
||||
const match = /Version\s+(\S+)/u.exec(result.stdout ?? "");
|
||||
if (!match?.[1]) throw new Error("cannot determine TypeScript version");
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function measure() {
|
||||
const composition = await runCompositionScale(["S0", "S1", "S2"], { samples: 9, warmup: 2 });
|
||||
const compiler = await collectTscScale(LOCAL_TSC_SIZES);
|
||||
return snapshotMetrics(composition.results, compiler);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const update = process.argv.includes("--update");
|
||||
const artifact = path.resolve(
|
||||
process.argv.find((entry) => entry.startsWith("--artifact="))?.slice("--artifact=".length) ??
|
||||
"artifacts/scale/baseline-comparison.json",
|
||||
);
|
||||
const environment = currentEnvironment(typescriptVersion());
|
||||
const metrics = await measure();
|
||||
|
||||
if (update) {
|
||||
const owner = process.env["SCALE_BASELINE_OWNER"];
|
||||
const reason = process.env["SCALE_BASELINE_REASON"];
|
||||
if (!owner || !reason) {
|
||||
process.stderr.write("SCALE_BASELINE_OWNER and SCALE_BASELINE_REASON are required.\n");
|
||||
process.exit(1);
|
||||
}
|
||||
const baseline = approveBaseline(environment, metrics, {
|
||||
owner,
|
||||
reason,
|
||||
approvedAt: new Date().toISOString(),
|
||||
});
|
||||
await mkdir(path.dirname(BASELINE_PATH), { recursive: true });
|
||||
await writeFile(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`);
|
||||
process.stdout.write(`scale baseline approved: ${baseline.approval.metricsDigest}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
let baseline: ScaleBaseline;
|
||||
try {
|
||||
baseline = JSON.parse(await readFile(BASELINE_PATH, "utf8")) as ScaleBaseline;
|
||||
} catch {
|
||||
process.stderr.write(`missing or unreadable ${BASELINE_PATH}; run update:scale-baseline\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
const comparison = compareToBaseline(baseline, environment, metrics);
|
||||
await mkdir(path.dirname(artifact), { recursive: true });
|
||||
await writeFile(
|
||||
artifact,
|
||||
`${JSON.stringify(
|
||||
{ schemaVersion: 1, generatedAt: new Date().toISOString(), environment, baselineApproval: baseline.approval, ...comparison },
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
const { compatibility } = comparison;
|
||||
process.stdout.write(
|
||||
`compatibility: structural=${compatibility.structural.ok} compiler=${compatibility.compiler.ok} timing=${compatibility.timing.ok}\n`,
|
||||
);
|
||||
if (!compatibility.timing.ok) {
|
||||
process.stdout.write(
|
||||
`timing regression NOT compared (incomplete): ${compatibility.timing.reasons.join("; ")}\n`,
|
||||
);
|
||||
}
|
||||
process.stdout.write(
|
||||
`axes: absolute ${comparison.absolute.findings.length} (${comparison.absolute.passed ? "ok" : "FAIL"}), ` +
|
||||
`growth ${comparison.growth.findings.length} (${comparison.growth.passed ? "ok" : "FAIL"}), ` +
|
||||
`relative ${comparison.compiler.length} compiler + ${comparison.timing.length} timing\n`,
|
||||
);
|
||||
for (const failure of comparison.failures) process.stdout.write(`FAIL ${failure}\n`);
|
||||
process.stdout.write(
|
||||
`scale baseline gate: ${comparison.status.toUpperCase()} (${path.relative(process.cwd(), artifact)})\n`,
|
||||
);
|
||||
if (comparison.status === "failed") process.exitCode = 1;
|
||||
else if (comparison.status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import {
|
||||
assertLinearOrBetter,
|
||||
growthExponents,
|
||||
} from "../../src/platform/cardinality/growth-regression.ts";
|
||||
|
||||
/**
|
||||
* Spec §38. TypeScript scale gate.
|
||||
*
|
||||
* Parses `tsc --extendedDiagnostics` and measures bounded local fixtures:
|
||||
* generated feature modules composed by one context, at increasing local
|
||||
* cardinality. S3/S4 are never compiled; they are verified as topology.
|
||||
*/
|
||||
export type TscDiagnostics = Readonly<{
|
||||
files: number;
|
||||
lines?: number;
|
||||
types: number;
|
||||
symbols: number;
|
||||
instantiations: number;
|
||||
memoryKb?: number;
|
||||
totalTimeSeconds?: number;
|
||||
}>;
|
||||
|
||||
const FIELD_NAMES: Readonly<Record<string, keyof TscDiagnostics>> = Object.freeze({
|
||||
Files: "files",
|
||||
Lines: "lines",
|
||||
Types: "types",
|
||||
Symbols: "symbols",
|
||||
Instantiations: "instantiations",
|
||||
"Memory used": "memoryKb",
|
||||
"Total time": "totalTimeSeconds",
|
||||
});
|
||||
|
||||
export function parseTscExtendedDiagnostics(text: string): TscDiagnostics {
|
||||
const parsed: Partial<Record<keyof TscDiagnostics, number>> = {};
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const match = /^\s*([A-Za-z][A-Za-z ]*?):\s+([\d.]+)\s*([KMs]?)\s*$/u.exec(line);
|
||||
if (!match) continue;
|
||||
const field = FIELD_NAMES[match[1] as string];
|
||||
if (!field) continue;
|
||||
const value = Number(match[2]);
|
||||
if (!Number.isFinite(value)) continue;
|
||||
parsed[field] = match[3] === "M" ? value * 1024 : value;
|
||||
}
|
||||
for (const required of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
if (parsed[required] === undefined) {
|
||||
throw new TypeError(`tsc diagnostics missing ${required}`);
|
||||
}
|
||||
}
|
||||
return Object.freeze(parsed) as TscDiagnostics;
|
||||
}
|
||||
|
||||
export function runTscDiagnostics(project: string, cwd = process.cwd()): TscDiagnostics {
|
||||
const tsc = resolve(cwd, "node_modules/.bin/tsc");
|
||||
const result = spawnSync(tsc, ["--project", project, "--noEmit", "--extendedDiagnostics"], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`tsc failed for ${project}: ${(result.stdout + result.stderr).slice(-2_000)}`);
|
||||
}
|
||||
return parseTscExtendedDiagnostics(result.stdout);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates one bounded local app: `features` feature modules, each with its
|
||||
* own typed contract, composed by one generated index. The shape mirrors the
|
||||
* repository's feature-module/generated-index pattern.
|
||||
*/
|
||||
export async function writeLocalTscFixture(
|
||||
root: string,
|
||||
features: number,
|
||||
): Promise<string> {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
await mkdir(join(root, "features"), { recursive: true });
|
||||
const imports: string[] = [];
|
||||
const members: string[] = [];
|
||||
for (let index = 0; index < features; index += 1) {
|
||||
const id = `feature${index}`;
|
||||
await writeFile(
|
||||
join(root, "features", `${id}.ts`),
|
||||
`export type ${id}Input = Readonly<{ id: string; limit: number; tags?: readonly string[] }>;\n` +
|
||||
`export type ${id}Output = Readonly<{ items: readonly Readonly<{ id: string; name: string }>[] }>;\n` +
|
||||
`export const ${id} = Object.freeze({\n` +
|
||||
` id: "${id}" as const,\n` +
|
||||
` routes: Object.freeze({ ${id.toUpperCase()}_LIST: "/${id}" }),\n` +
|
||||
` execute(input: ${id}Input): ${id}Output {\n` +
|
||||
` return { items: [{ id: input.id, name: String(input.limit) }] };\n` +
|
||||
` },\n` +
|
||||
`});\n`,
|
||||
);
|
||||
imports.push(`import { ${id} } from "./features/${id}.ts";`);
|
||||
members.push(` ${id},`);
|
||||
}
|
||||
await writeFile(
|
||||
join(root, "index.ts"),
|
||||
`${imports.join("\n")}\nexport const LOCAL_FEATURES = Object.freeze([\n${members.join("\n")}\n] as const);\n` +
|
||||
`export type LocalFeatureId = (typeof LOCAL_FEATURES)[number]["id"];\n`,
|
||||
);
|
||||
const tsconfig = join(root, "tsconfig.json");
|
||||
await writeFile(
|
||||
tsconfig,
|
||||
JSON.stringify(
|
||||
{
|
||||
compilerOptions: {
|
||||
target: "ES2022",
|
||||
module: "ESNext",
|
||||
moduleResolution: "Bundler",
|
||||
allowImportingTsExtensions: true,
|
||||
strict: true,
|
||||
noEmit: true,
|
||||
skipLibCheck: true,
|
||||
types: [],
|
||||
lib: ["ES2022"],
|
||||
},
|
||||
include: ["index.ts", "features/*.ts"],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
return tsconfig;
|
||||
}
|
||||
|
||||
export type TscScaleReport = Readonly<{
|
||||
passed: boolean;
|
||||
app: TscDiagnostics;
|
||||
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
|
||||
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
|
||||
}>;
|
||||
|
||||
export async function collectTscScale(
|
||||
localSizes: readonly number[],
|
||||
cwd = process.cwd(),
|
||||
): Promise<TscScaleReport> {
|
||||
const app = runTscDiagnostics("tsconfig.app.json", cwd);
|
||||
const local: { features: number; diagnostics: TscDiagnostics }[] = [];
|
||||
for (const features of localSizes) {
|
||||
const tsconfig = await writeLocalTscFixture(
|
||||
resolve(cwd, ".tmp/scale-tsc", `local-${features}`),
|
||||
features,
|
||||
);
|
||||
local.push({ features, diagnostics: runTscDiagnostics(tsconfig, cwd) });
|
||||
}
|
||||
const checks: { name: string; passed: boolean; detail: string }[] = [];
|
||||
for (const metric of ["types", "symbols", "instantiations"] as const) {
|
||||
const series = local.map(({ features, diagnostics }) => ({
|
||||
n: features,
|
||||
// Library/baseline cost is constant; growth is judged on what the local
|
||||
// features add on top of the smallest fixture.
|
||||
cost: Math.max(1, diagnostics[metric]),
|
||||
}));
|
||||
try {
|
||||
assertLinearOrBetter(series, { maxExponent: 1.3 });
|
||||
checks.push({
|
||||
name: `tsc:${metric}`,
|
||||
passed: true,
|
||||
detail: `exponents ${growthExponents(series).map((value) => value.toFixed(2)).join(", ")}`,
|
||||
});
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
name: `tsc:${metric}`,
|
||||
passed: false,
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
passed: checks.every((check) => check.passed),
|
||||
app,
|
||||
local: Object.freeze(local),
|
||||
checks: Object.freeze(checks),
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const artifactArg = process.argv.find((argument) => argument.startsWith("--artifact="));
|
||||
const sizesArg = process.argv.find((argument) => argument.startsWith("--local-sizes="));
|
||||
const sizes = (sizesArg?.slice("--local-sizes=".length) ?? "25,100,400")
|
||||
.split(",")
|
||||
.map(Number)
|
||||
.filter((value) => Number.isSafeInteger(value) && value > 0);
|
||||
const artifact = resolve(
|
||||
artifactArg?.slice("--artifact=".length) ?? "artifacts/scale/tsc-diagnostics.json",
|
||||
);
|
||||
const report = await collectTscScale(sizes);
|
||||
await mkdir(resolve(artifact, ".."), { recursive: true });
|
||||
await writeFile(
|
||||
artifact,
|
||||
`${JSON.stringify({ schemaVersion: 1, generatedAt: new Date().toISOString(), ...report }, null, 2)}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`tsc app: files=${report.app.files} types=${report.app.types} instantiations=${report.app.instantiations}\n`,
|
||||
);
|
||||
for (const check of report.checks) {
|
||||
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
|
||||
}
|
||||
process.stdout.write(`tsc diagnostics artifact: ${artifact}\n`);
|
||||
if (!report.passed) process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { composeUniqueRecord } from "../../src/platform/cardinality/compose-unique-record.ts";
|
||||
import {
|
||||
resolveProductFeatures,
|
||||
selectCompiledProductFeatures,
|
||||
} from "../../src/contracts/product-features.ts";
|
||||
import { defineBoundedContextModule } from "../../src/contexts/bounded-context-module.ts";
|
||||
import { defineDeployableAppModule } from "../../src/apps/deployable-app-module.ts";
|
||||
|
||||
/**
|
||||
* Spec §39. Scale-sensitive production composition paths, each parameterized
|
||||
* by one cardinality `n`. Inputs are built outside the timed region.
|
||||
*/
|
||||
export type CompositionWorkload = Readonly<{
|
||||
name: string;
|
||||
implementation: string;
|
||||
prepare(n: number): () => unknown;
|
||||
}>;
|
||||
|
||||
export function createUniqueRegistryParts(
|
||||
count: number,
|
||||
): readonly Readonly<Record<string, number>>[] {
|
||||
return Array.from({ length: count }, (_, index) =>
|
||||
Object.freeze({ [`route-${index}`]: index }),
|
||||
);
|
||||
}
|
||||
|
||||
function featureIds(count: number): string[] {
|
||||
return Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `feature-${index.toString().padStart(7, "0")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const CONTEXT_FEATURES = 200;
|
||||
|
||||
export const COMPOSITION_WORKLOADS: readonly CompositionWorkload[] = Object.freeze([
|
||||
{
|
||||
name: "registry-composition",
|
||||
implementation: "composeUniqueRecord",
|
||||
prepare(n: number) {
|
||||
const parts = createUniqueRegistryParts(n);
|
||||
return () => composeUniqueRecord(parts, () => "route");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "product-feature-selection",
|
||||
implementation: "selectCompiledProductFeatures",
|
||||
prepare(n: number) {
|
||||
const compiled = featureIds(n).map((featureId) => Object.freeze({ featureId }));
|
||||
const declared = compiled
|
||||
.filter((_, index) => index % 2 === 0)
|
||||
.map((feature) => feature.featureId)
|
||||
.join(",");
|
||||
return () => selectCompiledProductFeatures(compiled, declared);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "product-feature-resolution",
|
||||
implementation: "resolveProductFeatures",
|
||||
prepare(n: number) {
|
||||
const compiled = featureIds(n);
|
||||
const installed = compiled.filter((_, index) => index % 3 !== 0);
|
||||
return () => resolveProductFeatures(compiled, installed);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "context-hierarchy-composition",
|
||||
implementation: "defineBoundedContextModule+defineDeployableAppModule",
|
||||
prepare(n: number) {
|
||||
const features = featureIds(n).map((id) =>
|
||||
Object.freeze({
|
||||
id,
|
||||
contracts: Object.freeze({ featureId: id }),
|
||||
routes: Object.freeze({}),
|
||||
adapters: Object.freeze([]),
|
||||
runtimes: Object.freeze([]),
|
||||
messages: Object.freeze({}),
|
||||
contractContributions: Object.freeze([]),
|
||||
}),
|
||||
);
|
||||
const contextCount = Math.ceil(n / CONTEXT_FEATURES);
|
||||
return () => {
|
||||
const contexts = Array.from({ length: contextCount }, (_, contextIndex) =>
|
||||
defineBoundedContextModule({
|
||||
id: `context-${contextIndex}`,
|
||||
features: features.slice(
|
||||
contextIndex * CONTEXT_FEATURES,
|
||||
(contextIndex + 1) * CONTEXT_FEATURES,
|
||||
),
|
||||
budget: Object.freeze({ maxFeatures: CONTEXT_FEATURES }),
|
||||
}),
|
||||
);
|
||||
// Apps are sharded by the same direct-child budget.
|
||||
const apps: unknown[] = [];
|
||||
for (let index = 0; index < contexts.length; index += 20) {
|
||||
apps.push(
|
||||
defineDeployableAppModule({
|
||||
id: `app-${index / 20}`,
|
||||
contexts: contexts.slice(index, index + 20),
|
||||
budget: Object.freeze({ maxContexts: 20 }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
return apps;
|
||||
};
|
||||
},
|
||||
},
|
||||
]);
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Spec §44. Soak classification.
|
||||
*
|
||||
* The leak signal is sustained positive growth after warm-up, not an
|
||||
* arbitrary absolute maximum. The classifier fits a least-squares line over
|
||||
* the post-warm-up samples and reports `unbounded-growth` only when both the
|
||||
* slope and the total relative growth are material.
|
||||
*/
|
||||
export type SlopeSample = Readonly<{ minute: number; heapMb: number }>;
|
||||
|
||||
export type MemorySlopeClass = "stable" | "unbounded-growth" | "insufficient-data";
|
||||
|
||||
export type MemorySlopeOptions = Readonly<{
|
||||
/** Samples before this minute are warm-up. Default 0. */
|
||||
warmupMinutes?: number;
|
||||
/** Largest tolerated post-warm-up slope in MB per minute. Default 0.5. */
|
||||
maxSlopeMbPerMinute?: number;
|
||||
/** Largest tolerated relative growth first->last fitted value. Default 0.1. */
|
||||
maxRelativeGrowth?: number;
|
||||
}>;
|
||||
|
||||
export function linearSlope(points: readonly Readonly<{ x: number; y: number }>[]): number {
|
||||
const count = points.length;
|
||||
if (count < 2) return 0;
|
||||
const meanX = points.reduce((sum, point) => sum + point.x, 0) / count;
|
||||
const meanY = points.reduce((sum, point) => sum + point.y, 0) / count;
|
||||
let numerator = 0;
|
||||
let denominator = 0;
|
||||
for (const point of points) {
|
||||
numerator += (point.x - meanX) * (point.y - meanY);
|
||||
denominator += (point.x - meanX) ** 2;
|
||||
}
|
||||
return denominator === 0 ? 0 : numerator / denominator;
|
||||
}
|
||||
|
||||
export function classifyMemorySlope(
|
||||
samples: readonly SlopeSample[],
|
||||
options: MemorySlopeOptions = {},
|
||||
): MemorySlopeClass {
|
||||
const warmup = options.warmupMinutes ?? 0;
|
||||
const maxSlope = options.maxSlopeMbPerMinute ?? 0.5;
|
||||
const maxGrowth = options.maxRelativeGrowth ?? 0.1;
|
||||
const measured = samples
|
||||
.filter((sample) => sample.minute >= warmup)
|
||||
.filter((sample) => Number.isFinite(sample.minute) && Number.isFinite(sample.heapMb))
|
||||
.sort((left, right) => left.minute - right.minute);
|
||||
if (measured.length < 3) return "insufficient-data";
|
||||
const slope = linearSlope(measured.map((sample) => ({ x: sample.minute, y: sample.heapMb })));
|
||||
const first = measured[0] as SlopeSample;
|
||||
const last = measured[measured.length - 1] as SlopeSample;
|
||||
const fittedStart = Math.max(first.heapMb, Number.MIN_VALUE);
|
||||
const growth = (slope * (last.minute - first.minute)) / fittedStart;
|
||||
return slope > maxSlope && growth > maxGrowth ? "unbounded-growth" : "stable";
|
||||
}
|
||||
|
||||
/** Generic variant for logical counters (timers, listeners, buffered events). */
|
||||
export function classifyCounterSeries(
|
||||
values: readonly number[],
|
||||
options: Readonly<{ warmupSamples?: number; maxSlopePerSample?: number }> = {},
|
||||
): MemorySlopeClass {
|
||||
const warmup = options.warmupSamples ?? 0;
|
||||
const measured = values.slice(warmup);
|
||||
if (measured.length < 3) return "insufficient-data";
|
||||
const slope = linearSlope(measured.map((y, x) => ({ x, y })));
|
||||
return slope > (options.maxSlopePerSample ?? 0.01) ? "unbounded-growth" : "stable";
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import {
|
||||
assertLinearOrBetter,
|
||||
growthExponents,
|
||||
} from "../../src/platform/cardinality/growth-regression.ts";
|
||||
import { benchmark } from "./benchmark.ts";
|
||||
import {
|
||||
writeBenchmarkArtifact,
|
||||
type BenchmarkProfile,
|
||||
type BenchmarkResult,
|
||||
} from "./benchmark-result.ts";
|
||||
import { COMPOSITION_WORKLOADS } from "./composition-workloads.ts";
|
||||
|
||||
/**
|
||||
* Spec §36 / §39. Runtime composition scale gate for S0-S2.
|
||||
*
|
||||
* Every production composition workload runs at each requested profile with
|
||||
* warm-up and repeated samples; the median series must be linear-or-better.
|
||||
* Results are written as a machine-readable artifact under artifacts/scale/.
|
||||
*/
|
||||
export const COMPOSITION_PROFILES: Readonly<Record<string, number>> = Object.freeze({
|
||||
S0: 100,
|
||||
S1: 1_000,
|
||||
S2: 10_000,
|
||||
});
|
||||
|
||||
/** Scheduler noise below this median is not evidence of growth. */
|
||||
export const COMPOSITION_NOISE_FLOOR_MS = 0.5;
|
||||
/**
|
||||
* n log n paths (sorting, allocation-heavy string work) measure ~1.1-1.4
|
||||
* between S1 and S2; a quadratic path measures ~2. 1.6 separates the two.
|
||||
*/
|
||||
export const COMPOSITION_MAX_EXPONENT = 1.6;
|
||||
|
||||
export type CompositionScaleReport = Readonly<{
|
||||
passed: boolean;
|
||||
results: readonly BenchmarkResult[];
|
||||
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
|
||||
}>;
|
||||
|
||||
export async function runCompositionScale(
|
||||
profiles: readonly string[],
|
||||
options: Readonly<{
|
||||
samples?: number;
|
||||
warmup?: number;
|
||||
/**
|
||||
* `median` (default) for standalone gate runs. `min` (best observed) when
|
||||
* the gate shares the machine with other suites: stolen CPU and foreign
|
||||
* GC only ever add time, so the minimum keeps the algorithmic shape.
|
||||
*/
|
||||
statistic?: "median" | "min";
|
||||
}> = {},
|
||||
): Promise<CompositionScaleReport> {
|
||||
const results: BenchmarkResult[] = [];
|
||||
const checks: { name: string; passed: boolean; detail: string }[] = [];
|
||||
for (const workload of COMPOSITION_WORKLOADS) {
|
||||
const series: { n: number; cost: number }[] = [];
|
||||
for (const profile of profiles) {
|
||||
const cardinality = COMPOSITION_PROFILES[profile];
|
||||
if (cardinality === undefined) throw new TypeError(`unknown profile ${profile}`);
|
||||
const run = workload.prepare(cardinality);
|
||||
const result = await benchmark(workload.name, {
|
||||
samples: options.samples ?? 7,
|
||||
warmup: options.warmup ?? 2,
|
||||
profile: profile as BenchmarkProfile,
|
||||
cardinality,
|
||||
measureHeap: true,
|
||||
metadata: { implementation: workload.implementation },
|
||||
run,
|
||||
});
|
||||
results.push(result);
|
||||
series.push({
|
||||
n: cardinality,
|
||||
cost:
|
||||
options.statistic === "min" ? Math.min(...result.samples) : result.medianMs,
|
||||
});
|
||||
}
|
||||
if (series.length < 2) continue;
|
||||
try {
|
||||
assertLinearOrBetter(series, {
|
||||
maxExponent: COMPOSITION_MAX_EXPONENT,
|
||||
noiseFloor: COMPOSITION_NOISE_FLOOR_MS,
|
||||
});
|
||||
checks.push({
|
||||
name: `${workload.name}:growth`,
|
||||
passed: true,
|
||||
detail: `exponents ${growthExponents(series, COMPOSITION_NOISE_FLOOR_MS)
|
||||
.map((exponent) => exponent.toFixed(2))
|
||||
.join(", ")}`,
|
||||
});
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
name: `${workload.name}:growth`,
|
||||
passed: false,
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
passed: checks.every((check) => check.passed),
|
||||
results: Object.freeze(results),
|
||||
checks: Object.freeze(checks),
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const profileArg = process.argv.find((argument) => argument.startsWith("--profiles="));
|
||||
const artifactArg = process.argv.find((argument) => argument.startsWith("--artifact="));
|
||||
const profiles = (profileArg?.slice("--profiles=".length) ?? "S0,S1,S2")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const artifact = resolve(
|
||||
artifactArg?.slice("--artifact=".length) ?? "artifacts/scale/composition-scale.json",
|
||||
);
|
||||
const report = await runCompositionScale(profiles);
|
||||
await writeBenchmarkArtifact(artifact, {
|
||||
schemaVersion: 1,
|
||||
suite: "composition-scale",
|
||||
generatedAt: new Date().toISOString(),
|
||||
node: process.version,
|
||||
passed: report.passed,
|
||||
results: report.results,
|
||||
checks: report.checks,
|
||||
});
|
||||
for (const check of report.checks) {
|
||||
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
|
||||
}
|
||||
process.stdout.write(`composition scale artifact: ${artifact}\n`);
|
||||
if (!report.passed) process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
/**
|
||||
* Spec §47. CI execution tiers for the scale/chaos/soak program.
|
||||
*
|
||||
* pr S0/S1 architecture, composition and traffic invariants
|
||||
* main pr + S2 composition, TypeScript diagnostics and the stored
|
||||
* baseline regression gate (config/scale/benchmark-baseline.json)
|
||||
* nightly main + S3/S4 topology, telemetry storm, session cycles, browser soak
|
||||
* release release-coherence matrix + the stable main/nightly subset
|
||||
*
|
||||
* Every step's output goes to artifacts/scale/<tier>/<step>.log and the tier
|
||||
* writes one machine-readable summary.json. A step that could not run (e.g. a
|
||||
* browser soak without a build or browser) is `skipped`, and a tier with any
|
||||
* skipped step is `incomplete`: it is never reported as passed and exits
|
||||
* non-zero. `SCALE_ALLOW_INCOMPLETE=1` keeps the exit code zero for local runs,
|
||||
* but the summary still says `incomplete`.
|
||||
*/
|
||||
export type ScaleTier = "pr" | "main" | "nightly" | "release";
|
||||
|
||||
type Step = Readonly<{
|
||||
id: string;
|
||||
command: readonly string[];
|
||||
/** A step that may legitimately report `skipped` in its own artifact. */
|
||||
optional?: boolean;
|
||||
}>;
|
||||
|
||||
const VITEST = ["scripts/run-vitest.ts", "run", "--reporter=default"];
|
||||
|
||||
const PR_STEPS: readonly Step[] = [
|
||||
{
|
||||
id: "architecture-topology",
|
||||
command: [...VITEST, "tests/architecture", "tests/platform/cardinality", "tests/platform/routing"],
|
||||
},
|
||||
{
|
||||
id: "traffic-budgets",
|
||||
command: [...VITEST, "tests/platform/execution", "tests/platform/traffic", "tests/platform/resilience"],
|
||||
},
|
||||
{
|
||||
id: "scale-units",
|
||||
command: [
|
||||
...VITEST,
|
||||
"tests/scale/growth-regression.test.ts",
|
||||
"tests/scale/benchmark.test.ts",
|
||||
"tests/scale/synthetic-topology.test.ts",
|
||||
"tests/scale/scale-baseline.test.ts",
|
||||
"tests/scale/traffic",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "composition-s0-s1",
|
||||
command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1"],
|
||||
},
|
||||
];
|
||||
|
||||
const MAIN_STEPS: readonly Step[] = [
|
||||
...PR_STEPS,
|
||||
{
|
||||
id: "composition-s0-s2",
|
||||
command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1,S2"],
|
||||
},
|
||||
{
|
||||
id: "composition-scale-suite",
|
||||
command: [...VITEST, "tests/scale/composition-scale.test.ts", "tests/scale/tsc-diagnostics-parser.test.ts"],
|
||||
},
|
||||
{
|
||||
id: "typescript-diagnostics",
|
||||
command: ["scripts/scale/collect-tsc-diagnostics.ts", "--local-sizes=25,100,400"],
|
||||
},
|
||||
{
|
||||
id: "baseline-regression",
|
||||
command: ["scripts/scale/check-scale-baseline.ts"],
|
||||
},
|
||||
];
|
||||
|
||||
const NIGHTLY_STEPS: readonly Step[] = [
|
||||
...MAIN_STEPS,
|
||||
{
|
||||
id: "topology-s3-s4",
|
||||
command: [...VITEST, "tests/scale/global-local-independence.test.ts", "tests/platform/cardinality/extreme-scale-registry.test.ts"],
|
||||
},
|
||||
{
|
||||
id: "telemetry-storm",
|
||||
command: [...VITEST, "tests/scale/telemetry", "tests/platform/telemetry"],
|
||||
},
|
||||
{ id: "session-cycles", command: [...VITEST, "tests/scale/session"] },
|
||||
{
|
||||
id: "browser-soak",
|
||||
command: ["scripts/scale/browser-soak.ts", "--duration-ms=120000", "--interval-ms=5000", "--warmup-ms=30000"],
|
||||
optional: true,
|
||||
},
|
||||
];
|
||||
|
||||
const RELEASE_STEPS: readonly Step[] = [
|
||||
{
|
||||
id: "release-coherence",
|
||||
command: [...VITEST, "tests/platform/release", "tests/release"],
|
||||
},
|
||||
...MAIN_STEPS,
|
||||
{
|
||||
id: "topology-s3-s4",
|
||||
command: [...VITEST, "tests/scale/global-local-independence.test.ts"],
|
||||
},
|
||||
{
|
||||
id: "telemetry-storm",
|
||||
command: [...VITEST, "tests/scale/telemetry"],
|
||||
},
|
||||
];
|
||||
|
||||
export const SCALE_TIERS: Readonly<Record<ScaleTier, readonly Step[]>> = Object.freeze({
|
||||
pr: PR_STEPS,
|
||||
main: MAIN_STEPS,
|
||||
nightly: NIGHTLY_STEPS,
|
||||
release: RELEASE_STEPS,
|
||||
});
|
||||
|
||||
function withArtifact(step: Step, directory: string): readonly string[] {
|
||||
const script = step.command[0] ?? "";
|
||||
if (
|
||||
script.endsWith("run-composition-scale.ts") ||
|
||||
script.endsWith("collect-tsc-diagnostics.ts") ||
|
||||
script.endsWith("browser-soak.ts") ||
|
||||
script.endsWith("check-scale-baseline.ts")
|
||||
) {
|
||||
return [...step.command, `--artifact=${path.join(directory, `${step.id}.json`)}`];
|
||||
}
|
||||
return step.command;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const tier = process.argv[2] as ScaleTier | undefined;
|
||||
if (!tier || !(tier in SCALE_TIERS)) {
|
||||
process.stderr.write(`usage: run-scale-profile.ts <${Object.keys(SCALE_TIERS).join("|")}>\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const directory = path.resolve("artifacts/scale", tier);
|
||||
await mkdir(directory, { recursive: true });
|
||||
const results: { id: string; status: "passed" | "failed" | "skipped"; durationMs: number; log: string }[] = [];
|
||||
for (const step of SCALE_TIERS[tier]) {
|
||||
const startedAt = Date.now();
|
||||
const run = spawnSync(process.execPath, [...withArtifact(step, directory)], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
env: { ...process.env, SCALE_ARTIFACT_DIR: path.join(directory, "metrics") },
|
||||
});
|
||||
const log = path.join(directory, `${step.id}.log`);
|
||||
await writeFile(log, `${run.stdout ?? ""}${run.stderr ?? ""}`);
|
||||
const skipped = step.optional === true && /: skipped/u.test(run.stdout ?? "");
|
||||
const status = run.status === 0 ? (skipped ? "skipped" : "passed") : "failed";
|
||||
results.push({ id: step.id, status, durationMs: Date.now() - startedAt, log: path.relative(process.cwd(), log) });
|
||||
process.stdout.write(`${status.toUpperCase()} ${tier}:${step.id} (${Date.now() - startedAt}ms)\n`);
|
||||
}
|
||||
const failed = results.some((result) => result.status === "failed");
|
||||
const skipped = results.filter((result) => result.status === "skipped").map((result) => result.id);
|
||||
const status = failed ? "failed" : skipped.length > 0 ? "incomplete" : "passed";
|
||||
await writeFile(
|
||||
path.join(directory, "summary.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
tier,
|
||||
generatedAt: new Date().toISOString(),
|
||||
node: process.version,
|
||||
environment: {
|
||||
node: process.version,
|
||||
nodeMajor: Number(process.versions.node.split(".")[0]),
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
},
|
||||
status,
|
||||
passed: status === "passed",
|
||||
skipped,
|
||||
steps: results,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`scale tier ${tier}: ${status.toUpperCase()}${skipped.length > 0 ? ` (skipped: ${skipped.join(", ")})` : ""} (artifacts/scale/${tier}/summary.json)\n`,
|
||||
);
|
||||
if (status === "failed") process.exitCode = 1;
|
||||
else if (status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { cpus } from "node:os";
|
||||
|
||||
import { growthExponents } from "../../src/platform/cardinality/growth-regression.ts";
|
||||
import type { BenchmarkResult } from "./benchmark-result.ts";
|
||||
import type { TscDiagnostics } from "./collect-tsc-diagnostics.ts";
|
||||
|
||||
/**
|
||||
* Spec §48–§49. Stored benchmark baseline, compatibility check and relative
|
||||
* regression gate.
|
||||
*
|
||||
* The growth gates judge the *shape* of one run. This module compares a run
|
||||
* against an approved stored baseline so a change that keeps the shape but
|
||||
* multiplies the cost (or silently adds compiler work) is still caught.
|
||||
*
|
||||
* Two kinds of metric are compared differently:
|
||||
* - deterministic metrics (TypeScript types/symbols/instantiations/files) are
|
||||
* exact for a given compiler, so they use a tight relative tolerance and
|
||||
* require a compatible toolchain;
|
||||
* - timing metrics depend on the machine, so they use a loose tolerance plus an
|
||||
* absolute noise floor, and are only enforced on a compatible environment.
|
||||
*/
|
||||
export const SCALE_BASELINE_SCHEMA_VERSION = 1;
|
||||
|
||||
export type ScaleEnvironment = Readonly<{
|
||||
nodeMajor: number;
|
||||
platform: string;
|
||||
arch: string;
|
||||
cpuModel: string;
|
||||
cpuCount: number;
|
||||
typescript: string;
|
||||
}>;
|
||||
|
||||
export type TimingEntry = Readonly<{
|
||||
workload: string;
|
||||
profile: string;
|
||||
cardinality: number;
|
||||
medianMs: number;
|
||||
minMs: number;
|
||||
}>;
|
||||
|
||||
export type CompilerEntry = Readonly<{
|
||||
fixture: string;
|
||||
files: number;
|
||||
types: number;
|
||||
symbols: number;
|
||||
instantiations: number;
|
||||
}>;
|
||||
|
||||
export type ScaleMetrics = Readonly<{
|
||||
timing: readonly TimingEntry[];
|
||||
compiler: readonly CompilerEntry[];
|
||||
}>;
|
||||
|
||||
export type ScaleBaseline = Readonly<{
|
||||
schemaVersion: typeof SCALE_BASELINE_SCHEMA_VERSION;
|
||||
environment: ScaleEnvironment;
|
||||
metrics: ScaleMetrics;
|
||||
approval: Readonly<{
|
||||
owner: string;
|
||||
reason: string;
|
||||
approvedAt: string;
|
||||
metricsDigest: string;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export const REGRESSION_TOLERANCE = Object.freeze({
|
||||
/** Deterministic compiler counts may grow at most 10% without re-approval. */
|
||||
compilerRelative: 0.1,
|
||||
/** Timings may grow at most 2.5x (scheduler/GC noise lives below that)... */
|
||||
timingRelative: 1.5,
|
||||
/** ...and never fail on differences below this absolute noise floor. */
|
||||
timingNoiseFloorMs: 1,
|
||||
});
|
||||
|
||||
/**
|
||||
* Axis 1 — absolute ceilings. Machine-agnostic hard limits (roughly 50x the
|
||||
* approved baseline for timings, ~2x for compiler work) that hold even with
|
||||
* no comparable baseline.
|
||||
*/
|
||||
export const ABSOLUTE_CEILINGS = Object.freeze({
|
||||
timingMinMsByProfile: Object.freeze({ S0: 5, S1: 25, S2: 150 } as Record<string, number>),
|
||||
app: Object.freeze({ files: 1_500, types: 600_000, symbols: 1_600_000, instantiations: 1_500_000 }),
|
||||
/**
|
||||
* Generated `local-<N>` compiler fixtures: a fixed base (lib/runtime
|
||||
* overhead present at any N) plus a per-local-feature allowance.
|
||||
*/
|
||||
localBase: Object.freeze({ files: 100, types: 5_000, symbols: 15_000, instantiations: 5_000 }),
|
||||
localPerFeature: Object.freeze({ files: 2, types: 100, symbols: 250, instantiations: 150 }),
|
||||
});
|
||||
|
||||
/** Axis 3 — growth shape (see src/platform/cardinality/growth-regression.ts). */
|
||||
export const GROWTH_LIMITS = Object.freeze({
|
||||
timingMaxExponent: 1.6,
|
||||
timingNoiseFloorMs: 0.5,
|
||||
compilerMaxExponent: 1.3,
|
||||
});
|
||||
|
||||
export function currentEnvironment(typescript: string): ScaleEnvironment {
|
||||
const cpu = cpus();
|
||||
return Object.freeze({
|
||||
nodeMajor: Number(process.versions.node.split(".")[0]),
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
cpuModel: cpu[0]?.model.trim() ?? "unknown",
|
||||
cpuCount: cpu.length,
|
||||
typescript,
|
||||
});
|
||||
}
|
||||
|
||||
function canonical(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonical((value as Record<string, unknown>)[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function metricsDigest(metrics: ScaleMetrics): string {
|
||||
return createHash("sha256").update(canonical(metrics)).digest("hex");
|
||||
}
|
||||
|
||||
export function snapshotMetrics(
|
||||
composition: readonly BenchmarkResult[],
|
||||
compiler: Readonly<{
|
||||
app: TscDiagnostics;
|
||||
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
|
||||
}>,
|
||||
): ScaleMetrics {
|
||||
const round = (value: number) => Math.round(value * 1_000) / 1_000;
|
||||
const toCompiler = (fixture: string, diagnostics: TscDiagnostics): CompilerEntry => ({
|
||||
fixture,
|
||||
files: diagnostics.files,
|
||||
types: diagnostics.types,
|
||||
symbols: diagnostics.symbols,
|
||||
instantiations: diagnostics.instantiations,
|
||||
});
|
||||
return Object.freeze({
|
||||
timing: composition
|
||||
.map((result) => ({
|
||||
workload: result.name,
|
||||
profile: result.profile,
|
||||
cardinality: result.cardinality,
|
||||
medianMs: round(result.medianMs),
|
||||
minMs: round(Math.min(...result.samples)),
|
||||
}))
|
||||
.sort((left, right) =>
|
||||
`${left.workload}/${left.profile}`.localeCompare(`${right.workload}/${right.profile}`),
|
||||
),
|
||||
compiler: [
|
||||
toCompiler("app", compiler.app),
|
||||
...compiler.local.map((entry) => toCompiler(`local-${entry.features}`, entry.diagnostics)),
|
||||
].sort((left, right) => left.fixture.localeCompare(right.fixture)),
|
||||
});
|
||||
}
|
||||
|
||||
export type Compatibility = Readonly<{
|
||||
/** Baseline format, digest and metric coverage are usable at all. */
|
||||
structural: Readonly<{ ok: boolean; reasons: readonly string[] }>;
|
||||
/** Same compiler: deterministic compiler counts are comparable. */
|
||||
compiler: Readonly<{ ok: boolean; reasons: readonly string[] }>;
|
||||
/** Same machine class and runtime: timings are comparable. */
|
||||
timing: Readonly<{ ok: boolean; reasons: readonly string[] }>;
|
||||
}>;
|
||||
|
||||
export function checkCompatibility(
|
||||
baseline: ScaleBaseline,
|
||||
environment: ScaleEnvironment,
|
||||
current: ScaleMetrics,
|
||||
): Compatibility {
|
||||
const structural: string[] = [];
|
||||
if (baseline.schemaVersion !== SCALE_BASELINE_SCHEMA_VERSION) {
|
||||
structural.push(`baseline schemaVersion ${String(baseline.schemaVersion)} is not ${SCALE_BASELINE_SCHEMA_VERSION}`);
|
||||
}
|
||||
if (metricsDigest(baseline.metrics) !== baseline.approval?.metricsDigest) {
|
||||
structural.push("baseline metrics do not match their approved digest (hand-edited?)");
|
||||
}
|
||||
const timingKey = (entry: TimingEntry) => `${entry.workload}/${entry.profile}/${entry.cardinality}`;
|
||||
const baselineTiming = new Set(baseline.metrics.timing.map(timingKey));
|
||||
const baselineCompiler = new Set(baseline.metrics.compiler.map((entry) => entry.fixture));
|
||||
for (const entry of current.timing) {
|
||||
if (!baselineTiming.has(timingKey(entry))) structural.push(`no baseline for timing ${timingKey(entry)}`);
|
||||
}
|
||||
for (const entry of current.compiler) {
|
||||
if (!baselineCompiler.has(entry.fixture)) structural.push(`no baseline for compiler fixture ${entry.fixture}`);
|
||||
}
|
||||
|
||||
const compiler: string[] = [];
|
||||
if (baseline.environment.typescript !== environment.typescript) {
|
||||
compiler.push(`TypeScript ${environment.typescript} differs from baseline ${baseline.environment.typescript}`);
|
||||
}
|
||||
|
||||
const timing: string[] = [];
|
||||
const base = baseline.environment;
|
||||
if (base.nodeMajor !== environment.nodeMajor) timing.push(`Node ${environment.nodeMajor} vs baseline ${base.nodeMajor}`);
|
||||
if (base.platform !== environment.platform || base.arch !== environment.arch) {
|
||||
timing.push(`${environment.platform}/${environment.arch} vs baseline ${base.platform}/${base.arch}`);
|
||||
}
|
||||
if (base.cpuModel !== environment.cpuModel) timing.push(`CPU "${environment.cpuModel}" vs baseline "${base.cpuModel}"`);
|
||||
return Object.freeze({
|
||||
structural: { ok: structural.length === 0, reasons: structural },
|
||||
compiler: { ok: compiler.length === 0, reasons: compiler },
|
||||
timing: { ok: timing.length === 0, reasons: timing },
|
||||
});
|
||||
}
|
||||
|
||||
export type RegressionFinding = Readonly<{
|
||||
metric: string;
|
||||
baseline: number;
|
||||
current: number;
|
||||
limit: number;
|
||||
passed: boolean;
|
||||
}>;
|
||||
|
||||
export type AxisResult = Readonly<{
|
||||
passed: boolean;
|
||||
findings: readonly RegressionFinding[];
|
||||
}>;
|
||||
|
||||
export type BaselineComparison = Readonly<{
|
||||
/**
|
||||
* `passed` only when every axis was evaluated and held. `incomplete` when the
|
||||
* environment is not timing-compatible: timing regression was not compared,
|
||||
* so it is not evidence of a pass.
|
||||
*/
|
||||
status: "passed" | "failed" | "incomplete";
|
||||
passed: boolean;
|
||||
compatibility: Compatibility;
|
||||
absolute: AxisResult;
|
||||
growth: AxisResult;
|
||||
compiler: readonly RegressionFinding[];
|
||||
/** Empty when the environment is not timing-compatible. */
|
||||
timing: readonly RegressionFinding[];
|
||||
failures: readonly string[];
|
||||
}>;
|
||||
|
||||
function finding(metric: string, baseline: number, current: number, limit: number): RegressionFinding {
|
||||
return { metric, baseline, current, limit: Math.round(limit * 1_000) / 1_000, passed: current <= limit };
|
||||
}
|
||||
|
||||
export function checkAbsoluteCeilings(current: ScaleMetrics): AxisResult {
|
||||
const findings: RegressionFinding[] = [];
|
||||
for (const entry of current.timing) {
|
||||
const limit = ABSOLUTE_CEILINGS.timingMinMsByProfile[entry.profile];
|
||||
if (limit !== undefined) {
|
||||
findings.push(finding(`ceiling:time:${entry.workload}/${entry.profile}`, 0, entry.minMs, limit));
|
||||
}
|
||||
}
|
||||
for (const entry of current.compiler) {
|
||||
const local = /^local-(\d+)$/u.exec(entry.fixture);
|
||||
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
const limit = local
|
||||
? ABSOLUTE_CEILINGS.localBase[metric] +
|
||||
ABSOLUTE_CEILINGS.localPerFeature[metric] * Number(local[1])
|
||||
: entry.fixture === "app"
|
||||
? ABSOLUTE_CEILINGS.app[metric]
|
||||
: Number.POSITIVE_INFINITY;
|
||||
findings.push(finding(`ceiling:tsc:${entry.fixture}:${metric}`, 0, entry[metric], limit));
|
||||
}
|
||||
}
|
||||
return { passed: findings.every((item) => item.passed), findings };
|
||||
}
|
||||
|
||||
export function checkGrowthShape(current: ScaleMetrics): AxisResult {
|
||||
const findings: RegressionFinding[] = [];
|
||||
const byWorkload = new Map<string, { n: number; cost: number }[]>();
|
||||
for (const entry of current.timing) {
|
||||
const series = byWorkload.get(entry.workload) ?? [];
|
||||
series.push({ n: entry.cardinality, cost: entry.minMs });
|
||||
byWorkload.set(entry.workload, series);
|
||||
}
|
||||
for (const [workload, series] of byWorkload) {
|
||||
if (series.length < 2) continue;
|
||||
const worst = Math.max(...growthExponents(series, GROWTH_LIMITS.timingNoiseFloorMs));
|
||||
findings.push(finding(`growth:time:${workload}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.timingMaxExponent));
|
||||
}
|
||||
const local = current.compiler
|
||||
.map((entry) => ({ entry, n: Number(/^local-(\d+)$/u.exec(entry.fixture)?.[1]) }))
|
||||
.filter(({ n }) => Number.isFinite(n));
|
||||
if (local.length >= 2) {
|
||||
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
const worst = Math.max(...growthExponents(local.map(({ entry, n }) => ({ n, cost: entry[metric] }))));
|
||||
findings.push(finding(`growth:tsc:local:${metric}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.compilerMaxExponent));
|
||||
}
|
||||
}
|
||||
return { passed: findings.every((item) => item.passed), findings };
|
||||
}
|
||||
|
||||
export function compareToBaseline(
|
||||
baseline: ScaleBaseline,
|
||||
environment: ScaleEnvironment,
|
||||
current: ScaleMetrics,
|
||||
tolerance = REGRESSION_TOLERANCE,
|
||||
): BaselineComparison {
|
||||
const compatibility = checkCompatibility(baseline, environment, current);
|
||||
const failures: string[] = [...compatibility.structural.reasons];
|
||||
// A different compiler changes every deterministic count: the baseline must
|
||||
// be re-approved, it is never silently compared or silently skipped.
|
||||
if (!compatibility.compiler.ok) {
|
||||
failures.push(
|
||||
...compatibility.compiler.reasons.map((reason) => `${reason}; re-approve with update:scale-baseline`),
|
||||
);
|
||||
}
|
||||
|
||||
const absolute = checkAbsoluteCeilings(current);
|
||||
const growth = checkGrowthShape(current);
|
||||
for (const item of [...absolute.findings, ...growth.findings]) {
|
||||
if (!item.passed) failures.push(`${item.metric} ${item.current} exceeds ${item.limit}`);
|
||||
}
|
||||
|
||||
const compiler: RegressionFinding[] = [];
|
||||
if (compatibility.structural.ok && compatibility.compiler.ok) {
|
||||
const byFixture = new Map(baseline.metrics.compiler.map((entry) => [entry.fixture, entry]));
|
||||
for (const entry of current.compiler) {
|
||||
const reference = byFixture.get(entry.fixture);
|
||||
if (!reference) continue;
|
||||
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
const item = finding(
|
||||
`tsc:${entry.fixture}:${metric}`,
|
||||
reference[metric],
|
||||
entry[metric],
|
||||
Math.floor(reference[metric] * (1 + tolerance.compilerRelative)),
|
||||
);
|
||||
compiler.push(item);
|
||||
if (!item.passed) {
|
||||
failures.push(`${item.metric} ${item.current} exceeds ${item.limit} (baseline ${item.baseline})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const timing: RegressionFinding[] = [];
|
||||
if (compatibility.structural.ok && compatibility.timing.ok) {
|
||||
const key = (entry: TimingEntry) => `${entry.workload}/${entry.profile}`;
|
||||
const byKey = new Map(baseline.metrics.timing.map((entry) => [key(entry), entry]));
|
||||
for (const entry of current.timing) {
|
||||
const reference = byKey.get(key(entry));
|
||||
if (!reference) continue;
|
||||
// Best-observed samples: contention only ever adds time.
|
||||
const item = finding(
|
||||
`time:${key(entry)}`,
|
||||
reference.minMs,
|
||||
entry.minMs,
|
||||
Math.max(reference.minMs * (1 + tolerance.timingRelative), reference.minMs + tolerance.timingNoiseFloorMs),
|
||||
);
|
||||
timing.push(item);
|
||||
if (!item.passed) {
|
||||
failures.push(`${item.metric} ${item.current}ms exceeds ${item.limit}ms (baseline ${item.baseline}ms)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const status =
|
||||
failures.length > 0 ? "failed" : compatibility.timing.ok ? "passed" : "incomplete";
|
||||
return Object.freeze({
|
||||
status,
|
||||
passed: status === "passed",
|
||||
compatibility,
|
||||
absolute,
|
||||
growth,
|
||||
compiler,
|
||||
timing,
|
||||
failures,
|
||||
});
|
||||
}
|
||||
|
||||
export function approveBaseline(
|
||||
environment: ScaleEnvironment,
|
||||
metrics: ScaleMetrics,
|
||||
approval: Readonly<{ owner: string; reason: string; approvedAt: string }>,
|
||||
): ScaleBaseline {
|
||||
return Object.freeze({
|
||||
schemaVersion: SCALE_BASELINE_SCHEMA_VERSION,
|
||||
environment,
|
||||
metrics,
|
||||
approval: { ...approval, metricsDigest: metricsDigest(metrics) },
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user