chore: sync the frontend template from ebea300 to af3d8e6
Carries 359 template commits, tag v0.2.0 plus one documentation commit: the generator-built feature index with its app, bounded-context and feature declarations, the src/platform kernel and scale tooling, the 2026-09-24 src review fixes, and the 2026-09-25 maturity review round with its CHANGELOG [보안]/[계약] rows. 691 of the 824 template paths are applied, 85 of them merged with product changes; 127 are not carried. Product identity is unchanged: name, version 0.1.0, routes, design and the TechLog domain. Three decisions worth naming. TechLog stays composed directly, as the last two syncs recorded; its directory carries a NOT_INSTALLED marker that says so, and every installed-* composer takes the template's projection with TechLog added on top. The reference sample is no longer installed: the template now installs a feature only through feature-module.ts and context-routes.ts, which load the screens this product deleted, so its three operations, six schemas and one topic leave the registries, recorded as registry change evidence. And AppRouter mounts the platform and TechLog routes only; the template's bounded-context route gateway waits for a generator-installed feature. Required fixes: readSubject on the BFF session owner (without it every HTTP Studio session would close as integration-failed), zod through the facade in nine TechLog files, noUnusedLocals, and public/config.json written out to match the local profile. Derived baselines were recomputed after reproducing both sides' values: CI contract 101 commands / 106 references / 141 artifacts, gate shape c9adcf85; registry approval 2dc6c3a3; the runtime config schema baseline equals the schema at8210d48. Verified here: types, lint, architecture, adapter inventory, CI contract, i18n, design system, diagnostics, browser security, dev release manifest, TechLog contract and build pass; 220 component, 155 integration, 15 contract, 418 TechLog, 43 reference-feature, 18 recipe and 46 release-coherence tests pass, and 2381 of 2383 unit tests. Reproduced at8210d48first and unchanged: check:registries (byte-identical), test:runtime-schema, verify:documentation. Brought by the template and escalated rather than suppressed: check:request-fanout on TechLog's cursor pagination, check:remediation-ledger (fails upstream too), the shared-chunk hash test, and the registry-dependent sample removal and second-feature tests. Playwright suites were not run. Full record, the five open questions and the follow-ups in docs/operations/template-sync-2026-09-29.md. (cherry picked from commit b2170cf265dea45dc9cc05d3c61e627e97f31cdb)
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
/**
|
||||
* Spec §48. Machine-readable benchmark result.
|
||||
*/
|
||||
export type BenchmarkProfile = "S0" | "S1" | "S2" | "S3" | "S4";
|
||||
|
||||
export interface BenchmarkResult {
|
||||
readonly name: string;
|
||||
readonly profile: BenchmarkProfile;
|
||||
readonly cardinality: number;
|
||||
readonly samples: readonly number[];
|
||||
readonly medianMs: number;
|
||||
readonly heapBytes?: number;
|
||||
readonly metadata: Readonly<Record<string, string | number | boolean>>;
|
||||
}
|
||||
|
||||
export type BenchmarkArtifact = Readonly<{
|
||||
schemaVersion: 1;
|
||||
suite: string;
|
||||
generatedAt: string;
|
||||
node: string;
|
||||
passed: boolean;
|
||||
results: readonly BenchmarkResult[];
|
||||
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
|
||||
}>;
|
||||
|
||||
const PROFILES: ReadonlySet<string> = new Set(["S0", "S1", "S2", "S3", "S4"]);
|
||||
|
||||
export function median(values: readonly number[]): number {
|
||||
if (values.length === 0) throw new TypeError("median of an empty series");
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 === 1
|
||||
? (sorted[middle] as number)
|
||||
: ((sorted[middle - 1] as number) + (sorted[middle] as number)) / 2;
|
||||
}
|
||||
|
||||
export function assertBenchmarkResult(value: BenchmarkResult): BenchmarkResult {
|
||||
if (typeof value.name !== "string" || value.name.length === 0) {
|
||||
throw new TypeError("benchmark name required");
|
||||
}
|
||||
if (!PROFILES.has(value.profile)) {
|
||||
throw new TypeError(`unknown benchmark profile ${String(value.profile)}`);
|
||||
}
|
||||
if (!Number.isSafeInteger(value.cardinality) || value.cardinality < 0) {
|
||||
throw new TypeError("benchmark cardinality must be a non-negative integer");
|
||||
}
|
||||
if (
|
||||
value.samples.length === 0 ||
|
||||
value.samples.some((sample) => !Number.isFinite(sample) || sample < 0)
|
||||
) {
|
||||
throw new TypeError("benchmark samples must be finite non-negative numbers");
|
||||
}
|
||||
if (value.medianMs !== median(value.samples)) {
|
||||
throw new TypeError("benchmark medianMs must be the median of its samples");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function writeBenchmarkArtifact(
|
||||
path: string,
|
||||
artifact: BenchmarkArtifact,
|
||||
): Promise<void> {
|
||||
for (const result of artifact.results) assertBenchmarkResult(result);
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, `${JSON.stringify(artifact, null, 2)}\n`);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { performance } from "node:perf_hooks";
|
||||
|
||||
import {
|
||||
assertBenchmarkResult,
|
||||
median,
|
||||
type BenchmarkProfile,
|
||||
type BenchmarkResult,
|
||||
} from "./benchmark-result.ts";
|
||||
|
||||
/**
|
||||
* Spec §49. Repeated samples with a median, so one noisy scheduler slice can
|
||||
* neither fail a linear implementation nor hide a slow one.
|
||||
*
|
||||
* `run` either does the work (and is timed here) or returns a duration it
|
||||
* measured itself, which lets tests inject exact durations.
|
||||
*/
|
||||
export type BenchmarkOptions = Readonly<{
|
||||
samples: number;
|
||||
warmup?: number;
|
||||
profile?: BenchmarkProfile;
|
||||
cardinality?: number;
|
||||
metadata?: Readonly<Record<string, string | number | boolean>>;
|
||||
measureHeap?: boolean;
|
||||
run: () => unknown;
|
||||
}>;
|
||||
|
||||
function heapUsed(): number {
|
||||
return process.memoryUsage().heapUsed;
|
||||
}
|
||||
|
||||
export async function benchmark(
|
||||
name: string,
|
||||
options: BenchmarkOptions,
|
||||
): Promise<BenchmarkResult> {
|
||||
if (!Number.isSafeInteger(options.samples) || options.samples < 1) {
|
||||
throw new TypeError("benchmark samples must be a positive integer");
|
||||
}
|
||||
for (let index = 0; index < (options.warmup ?? 0); index += 1) {
|
||||
await options.run();
|
||||
}
|
||||
const samples: number[] = [];
|
||||
let peakHeapDelta = 0;
|
||||
for (let index = 0; index < options.samples; index += 1) {
|
||||
const heapBefore = options.measureHeap ? heapUsed() : 0;
|
||||
const startedAt = performance.now();
|
||||
const reported = await options.run();
|
||||
const elapsed = performance.now() - startedAt;
|
||||
if (options.measureHeap) {
|
||||
peakHeapDelta = Math.max(peakHeapDelta, heapUsed() - heapBefore);
|
||||
}
|
||||
samples.push(typeof reported === "number" && Number.isFinite(reported) ? reported : elapsed);
|
||||
}
|
||||
return assertBenchmarkResult(
|
||||
Object.freeze({
|
||||
name,
|
||||
profile: options.profile ?? "S0",
|
||||
cardinality: options.cardinality ?? 0,
|
||||
samples: Object.freeze(samples),
|
||||
medianMs: median(samples),
|
||||
...(options.measureHeap ? { heapBytes: Math.max(0, peakHeapDelta) } : {}),
|
||||
metadata: Object.freeze({ ...(options.metadata ?? {}) }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,554 @@
|
||||
import { createReadStream } from "node:fs";
|
||||
import { access, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
classifyCounterSeries,
|
||||
classifyMemorySlope,
|
||||
type MemorySlopeClass,
|
||||
} from "./memory-slope.ts";
|
||||
|
||||
/**
|
||||
* Spec §33 / §44. Browser soak on the existing Playwright runner.
|
||||
*
|
||||
* Serves the built `dist/` on an OS-assigned 127.0.0.1 port and keeps one
|
||||
* long-lived page cycling through the app's routes by client-side navigation.
|
||||
* The served runtime config turns telemetry on (pointed at a local collector)
|
||||
* and points the API at a stub that always answers 503, so every cycle also
|
||||
* exercises bounded retries and telemetry under a sustained backend failure.
|
||||
*
|
||||
* Collected per sample: JS heap, DOM nodes, event listeners, live timers,
|
||||
* in-flight fetches, Web Storage size. Collected overall: cycles, navigation
|
||||
* failures, API requests, telemetry POSTs/events. Without a build or browser
|
||||
* the result is `skipped` (never a pass); too little evidence fails.
|
||||
*/
|
||||
export type SoakSample = Readonly<{
|
||||
elapsedMs: number;
|
||||
cycles: number;
|
||||
heapBytes: number;
|
||||
domNodes: number;
|
||||
listeners: number;
|
||||
activeTimers: number;
|
||||
inFlightRequests: number;
|
||||
storageKeys: number;
|
||||
storageBytes: number;
|
||||
}>;
|
||||
|
||||
export type SoakSummary = Readonly<{
|
||||
cycles: number;
|
||||
navigationFailures: number;
|
||||
maxHeapBytes: number;
|
||||
heapClassification: MemorySlopeClass;
|
||||
listenerDelta: number;
|
||||
listenerClassification: MemorySlopeClass;
|
||||
timerDelta: number;
|
||||
maxActiveTimers: number;
|
||||
domNodeDelta: number;
|
||||
maxInFlightRequests: number;
|
||||
finalInFlightRequests: number;
|
||||
storageKeyDelta: number;
|
||||
storageByteDelta: number;
|
||||
apiRequests: number;
|
||||
referenceVisits: number;
|
||||
settledReferenceVisits: number;
|
||||
apiRequestsPerReferenceVisit: number;
|
||||
telemetryPosts: number;
|
||||
telemetryEvents: number;
|
||||
maxTelemetryPostsPerMinute: number;
|
||||
}>;
|
||||
|
||||
export type SoakCheck = Readonly<{ name: string; passed: boolean; detail: string }>;
|
||||
|
||||
export type SoakArtifact = Readonly<{
|
||||
schemaVersion: 2;
|
||||
status: "passed" | "failed" | "skipped";
|
||||
reason?: string;
|
||||
durationMs: number;
|
||||
classification?: MemorySlopeClass;
|
||||
summary?: SoakSummary;
|
||||
checks?: readonly SoakCheck[];
|
||||
/** Identity of the soaked build (from dist/release-manifest.json). */
|
||||
build?: Readonly<{ buildId?: string; releaseId?: string; builtAt?: string }>;
|
||||
browser?: string;
|
||||
samples: readonly SoakSample[];
|
||||
}>;
|
||||
|
||||
/** Thresholds a soak must meet; each is checked and recorded by name. */
|
||||
export const SOAK_LIMITS = Object.freeze({
|
||||
minCycles: 200,
|
||||
minPostWarmupSamples: 3,
|
||||
maxListenerDelta: 50,
|
||||
maxTimerDelta: 10,
|
||||
maxStorageKeyDelta: 2,
|
||||
/** One logical execution: 1 attempt + at most 2 retries. */
|
||||
maxApiRequestsPerReferenceVisit: 3,
|
||||
/** Two bounded pipelines (platform events + feature events) x 20/min. */
|
||||
maxTelemetryPostsPerMinute: 40,
|
||||
});
|
||||
|
||||
export function summarizeSoak(
|
||||
samples: readonly Pick<SoakSample, "elapsedMs" | "heapBytes">[],
|
||||
warmupMs: number,
|
||||
): MemorySlopeClass {
|
||||
return classifyMemorySlope(
|
||||
samples.map((sample) => ({
|
||||
minute: sample.elapsedMs / 60_000,
|
||||
heapMb: sample.heapBytes / (1024 * 1024),
|
||||
})),
|
||||
{ warmupMinutes: warmupMs / 60_000, maxSlopeMbPerMinute: 1, maxRelativeGrowth: 0.2 },
|
||||
);
|
||||
}
|
||||
|
||||
export type SoakTotals = Readonly<{
|
||||
cycles: number;
|
||||
navigationFailures: number;
|
||||
referenceVisits: number;
|
||||
settledReferenceVisits: number;
|
||||
finalInFlightRequests: number;
|
||||
apiRequests: number;
|
||||
telemetryPosts: number;
|
||||
telemetryEvents: number;
|
||||
telemetryPostTimes: readonly number[];
|
||||
}>;
|
||||
|
||||
export function evaluateSoak(
|
||||
samples: readonly SoakSample[],
|
||||
totals: SoakTotals,
|
||||
warmupMs: number,
|
||||
): Readonly<{ summary: SoakSummary; checks: readonly SoakCheck[] }> {
|
||||
const measured = samples.filter((sample) => sample.elapsedMs >= warmupMs);
|
||||
const first = measured[0];
|
||||
const last = measured[measured.length - 1];
|
||||
const delta = (pick: (sample: SoakSample) => number) =>
|
||||
first && last ? pick(last) - pick(first) : 0;
|
||||
let maxTelemetryPostsPerMinute = 0;
|
||||
for (const time of totals.telemetryPostTimes) {
|
||||
const inWindow = totals.telemetryPostTimes.filter(
|
||||
(other) => other >= time && other < time + 60_000,
|
||||
).length;
|
||||
maxTelemetryPostsPerMinute = Math.max(maxTelemetryPostsPerMinute, inWindow);
|
||||
}
|
||||
const summary: SoakSummary = {
|
||||
cycles: totals.cycles,
|
||||
navigationFailures: totals.navigationFailures,
|
||||
maxHeapBytes: Math.max(0, ...samples.map((sample) => sample.heapBytes)),
|
||||
heapClassification: summarizeSoak(samples, warmupMs),
|
||||
listenerDelta: delta((sample) => sample.listeners),
|
||||
listenerClassification: classifyCounterSeries(
|
||||
measured.map((sample) => sample.listeners),
|
||||
{ maxSlopePerSample: 1 },
|
||||
),
|
||||
timerDelta: delta((sample) => sample.activeTimers),
|
||||
maxActiveTimers: Math.max(0, ...samples.map((sample) => sample.activeTimers)),
|
||||
domNodeDelta: delta((sample) => sample.domNodes),
|
||||
maxInFlightRequests: Math.max(0, ...samples.map((sample) => sample.inFlightRequests)),
|
||||
finalInFlightRequests: totals.finalInFlightRequests,
|
||||
storageKeyDelta: delta((sample) => sample.storageKeys),
|
||||
storageByteDelta: delta((sample) => sample.storageBytes),
|
||||
apiRequests: totals.apiRequests,
|
||||
referenceVisits: totals.referenceVisits,
|
||||
settledReferenceVisits: totals.settledReferenceVisits,
|
||||
apiRequestsPerReferenceVisit:
|
||||
totals.referenceVisits === 0
|
||||
? 0
|
||||
: Math.round((totals.apiRequests / totals.referenceVisits) * 1_000) / 1_000,
|
||||
telemetryPosts: totals.telemetryPosts,
|
||||
telemetryEvents: totals.telemetryEvents,
|
||||
maxTelemetryPostsPerMinute,
|
||||
};
|
||||
const checks: SoakCheck[] = [
|
||||
{ name: "cycles", passed: summary.cycles >= SOAK_LIMITS.minCycles, detail: `${summary.cycles} >= ${SOAK_LIMITS.minCycles}` },
|
||||
{
|
||||
name: "post-warmup samples",
|
||||
passed: measured.length >= SOAK_LIMITS.minPostWarmupSamples,
|
||||
detail: `${measured.length} >= ${SOAK_LIMITS.minPostWarmupSamples}`,
|
||||
},
|
||||
{ name: "navigation", passed: summary.navigationFailures === 0, detail: `${summary.navigationFailures} failures` },
|
||||
{ name: "heap slope", passed: summary.heapClassification === "stable", detail: summary.heapClassification },
|
||||
{
|
||||
name: "listeners",
|
||||
passed:
|
||||
summary.listenerClassification === "stable" &&
|
||||
summary.listenerDelta <= SOAK_LIMITS.maxListenerDelta,
|
||||
detail: `${summary.listenerClassification}, delta ${summary.listenerDelta}`,
|
||||
},
|
||||
{
|
||||
name: "timers",
|
||||
passed: summary.timerDelta <= SOAK_LIMITS.maxTimerDelta,
|
||||
detail: `delta ${summary.timerDelta}, max ${summary.maxActiveTimers}`,
|
||||
},
|
||||
{
|
||||
name: "in-flight settles",
|
||||
passed: summary.finalInFlightRequests === 0,
|
||||
detail: `final ${summary.finalInFlightRequests}, max ${summary.maxInFlightRequests}`,
|
||||
},
|
||||
{
|
||||
name: "storage growth",
|
||||
passed: summary.storageKeyDelta <= SOAK_LIMITS.maxStorageKeyDelta,
|
||||
detail: `keys ${summary.storageKeyDelta}, bytes ${summary.storageByteDelta}`,
|
||||
},
|
||||
{
|
||||
name: "backend failure exercised",
|
||||
passed:
|
||||
summary.settledReferenceVisits > 0 && summary.apiRequests > 0 && summary.telemetryEvents > 0,
|
||||
detail: `${summary.settledReferenceVisits} settled visits, ${summary.apiRequests} API requests, ${summary.telemetryEvents} telemetry events`,
|
||||
},
|
||||
{
|
||||
name: "request amplification",
|
||||
passed:
|
||||
summary.referenceVisits > 0 &&
|
||||
summary.apiRequestsPerReferenceVisit <= SOAK_LIMITS.maxApiRequestsPerReferenceVisit,
|
||||
detail: `${summary.apiRequests} requests / ${summary.referenceVisits} visits`,
|
||||
},
|
||||
{
|
||||
name: "telemetry rate",
|
||||
passed: summary.maxTelemetryPostsPerMinute <= SOAK_LIMITS.maxTelemetryPostsPerMinute,
|
||||
detail: `max ${summary.maxTelemetryPostsPerMinute}/min, ${summary.telemetryPosts} posts, ${summary.telemetryEvents} events`,
|
||||
},
|
||||
];
|
||||
return { summary, checks };
|
||||
}
|
||||
|
||||
const CONTENT_TYPES: Readonly<Record<string, string>> = {
|
||||
".css": "text/css; charset=utf-8",
|
||||
".html": "text/html; charset=utf-8",
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".json": "application/json; charset=utf-8",
|
||||
".svg": "image/svg+xml",
|
||||
".png": "image/png",
|
||||
".webmanifest": "application/manifest+json",
|
||||
};
|
||||
|
||||
type Counters = {
|
||||
apiRequests: number;
|
||||
telemetryPosts: number;
|
||||
telemetryEvents: number;
|
||||
telemetryPostTimes: number[];
|
||||
};
|
||||
|
||||
async function serveSoakTarget(
|
||||
root: string,
|
||||
counters: Counters,
|
||||
startedAt: () => number,
|
||||
): Promise<Readonly<{ server: Server; baseUrl: string }>> {
|
||||
let baseUrl = "";
|
||||
const baseConfig = JSON.parse(
|
||||
await readFile(path.join(root, "config.json"), "utf8"),
|
||||
) as Record<string, unknown>;
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (url.pathname === "/__telemetry") {
|
||||
let body = "";
|
||||
request.on("data", (chunk: Buffer) => {
|
||||
body += chunk.toString("utf8");
|
||||
});
|
||||
request.on("end", () => {
|
||||
counters.telemetryPosts += 1;
|
||||
counters.telemetryPostTimes.push(Date.now() - startedAt());
|
||||
try {
|
||||
const parsed = JSON.parse(body) as { events?: unknown[] };
|
||||
counters.telemetryEvents += Array.isArray(parsed.events) ? parsed.events.length : 0;
|
||||
} catch {
|
||||
// Counted as a post regardless of its body.
|
||||
}
|
||||
response.writeHead(204).end();
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.pathname.startsWith("/api/")) {
|
||||
counters.apiRequests += 1;
|
||||
response.writeHead(503, { "Content-Type": "application/json" }).end("{}");
|
||||
return;
|
||||
}
|
||||
if (url.pathname === "/config.json") {
|
||||
response
|
||||
.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" })
|
||||
.end(
|
||||
JSON.stringify({
|
||||
...baseConfig,
|
||||
API_BASE_URL: `${baseUrl}/api/`,
|
||||
TELEMETRY_ENABLED: true,
|
||||
TELEMETRY_ENDPOINT: `${baseUrl}/__telemetry`,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const requested = path.resolve(root, `.${decodeURIComponent(url.pathname)}`);
|
||||
if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) {
|
||||
response.writeHead(403).end();
|
||||
return;
|
||||
}
|
||||
const details = await stat(requested).catch(() => null);
|
||||
const file = details?.isFile() ? requested : path.join(root, "index.html");
|
||||
response.writeHead(200, {
|
||||
"Content-Type": CONTENT_TYPES[path.extname(file)] ?? "application/octet-stream",
|
||||
"Cache-Control": "no-store",
|
||||
});
|
||||
createReadStream(file).pipe(response);
|
||||
} catch {
|
||||
response.writeHead(404).end();
|
||||
}
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
baseUrl = `http://127.0.0.1:${port}`;
|
||||
return { server, baseUrl };
|
||||
}
|
||||
|
||||
const PAGE_PROBES = `
|
||||
(() => {
|
||||
const live = new Set();
|
||||
const setT = window.setTimeout, clearT = window.clearTimeout;
|
||||
const setI = window.setInterval, clearI = window.clearInterval;
|
||||
window.setTimeout = (fn, ms, ...rest) => {
|
||||
const id = setT(() => { live.delete(id); typeof fn === "function" && fn(...rest); }, ms);
|
||||
live.add(id); return id;
|
||||
};
|
||||
window.clearTimeout = (id) => { live.delete(id); clearT(id); };
|
||||
window.setInterval = (fn, ms, ...rest) => { const id = setI(fn, ms, ...rest); live.add(id); return id; };
|
||||
window.clearInterval = (id) => { live.delete(id); clearI(id); };
|
||||
let inFlight = 0;
|
||||
const nativeFetch = window.fetch.bind(window);
|
||||
window.fetch = (...args) => {
|
||||
inFlight += 1;
|
||||
return nativeFetch(...args).finally(() => { inFlight -= 1; });
|
||||
};
|
||||
const storageSize = (storage) => {
|
||||
let bytes = 0;
|
||||
for (let i = 0; i < storage.length; i += 1) {
|
||||
const key = storage.key(i) ?? "";
|
||||
bytes += key.length + (storage.getItem(key) ?? "").length;
|
||||
}
|
||||
return { keys: storage.length, bytes };
|
||||
};
|
||||
Object.defineProperty(window, "__soakProbe", {
|
||||
value: () => {
|
||||
const local = storageSize(window.localStorage);
|
||||
const session = storageSize(window.sessionStorage);
|
||||
return {
|
||||
activeTimers: live.size,
|
||||
inFlight,
|
||||
storageKeys: local.keys + session.keys,
|
||||
storageBytes: local.bytes + session.bytes,
|
||||
};
|
||||
},
|
||||
});
|
||||
})();
|
||||
`;
|
||||
|
||||
type Probe = { activeTimers: number; inFlight: number; storageKeys: number; storageBytes: number };
|
||||
|
||||
export async function runBrowserSoak(
|
||||
options: Readonly<{
|
||||
distRoot: string;
|
||||
durationMs: number;
|
||||
intervalMs: number;
|
||||
warmupMs: number;
|
||||
routes: readonly string[];
|
||||
}>,
|
||||
): Promise<SoakArtifact> {
|
||||
try {
|
||||
await access(path.join(options.distRoot, "index.html"));
|
||||
await access(path.join(options.distRoot, "config.json"));
|
||||
} catch {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: "skipped",
|
||||
reason: "no built dist/index.html + config.json",
|
||||
durationMs: 0,
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
let chromium: (typeof import("@playwright/test"))["chromium"];
|
||||
try {
|
||||
({ chromium } = await import("@playwright/test"));
|
||||
} catch {
|
||||
return { schemaVersion: 2, status: "skipped", reason: "playwright unavailable", durationMs: 0, samples: [] };
|
||||
}
|
||||
const counters: Counters = {
|
||||
apiRequests: 0,
|
||||
telemetryPosts: 0,
|
||||
telemetryEvents: 0,
|
||||
telemetryPostTimes: [],
|
||||
};
|
||||
let started = Date.now();
|
||||
const { server, baseUrl } = await serveSoakTarget(
|
||||
path.resolve(options.distRoot),
|
||||
counters,
|
||||
() => started,
|
||||
);
|
||||
let browser: Awaited<ReturnType<typeof chromium.launch>> | undefined;
|
||||
try {
|
||||
try {
|
||||
browser = await chromium.launch();
|
||||
} catch (error) {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: "skipped",
|
||||
reason: `browser unavailable: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`,
|
||||
durationMs: 0,
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
const page = await browser.newPage();
|
||||
await page.addInitScript(PAGE_PROBES);
|
||||
const cdp = await page.context().newCDPSession(page);
|
||||
await cdp.send("Performance.enable");
|
||||
await page.goto(`${baseUrl}/examples/reference-resources`, { waitUntil: "load" });
|
||||
await page.waitForSelector("#root > *", { timeout: 15_000 });
|
||||
// Protected routes need a session: start the local demo session once, so
|
||||
// the reference feature really dispatches its (failing) API requests.
|
||||
const login = page.getByRole("button", { name: /로그인 시작|Sign in|Start sign-in/u });
|
||||
if (await login.count()) {
|
||||
await login.first().click();
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
|
||||
const samples: SoakSample[] = [];
|
||||
started = Date.now();
|
||||
let nextSample = 0;
|
||||
let cycles = 0;
|
||||
let navigationFailures = 0;
|
||||
let referenceVisits = 0;
|
||||
let settledReferenceVisits = 0;
|
||||
const probe = async () =>
|
||||
(await page.evaluate(() =>
|
||||
(window as unknown as { __soakProbe: () => Probe }).__soakProbe(),
|
||||
)) as Probe;
|
||||
while (Date.now() - started < options.durationMs) {
|
||||
const route = options.routes[cycles % options.routes.length] ?? "/";
|
||||
cycles += 1;
|
||||
if (route === "/examples/reference-resources") referenceVisits += 1;
|
||||
// Client-side navigation keeps one long-lived document, like a real session.
|
||||
await page.evaluate((target) => {
|
||||
window.history.pushState({}, "", target);
|
||||
window.dispatchEvent(new PopStateEvent("popstate"));
|
||||
}, route);
|
||||
await page.waitForTimeout(50);
|
||||
const rendered = await page.evaluate(
|
||||
(target) =>
|
||||
window.location.pathname === target &&
|
||||
(document.getElementById("root")?.childElementCount ?? 0) > 0,
|
||||
route,
|
||||
);
|
||||
if (!rendered) navigationFailures += 1;
|
||||
// Most visits leave after 50ms, which cancels the query (a cancelled
|
||||
// request is intentionally not reported). Every 5th reference visit
|
||||
// dwells until its interaction settles, so the failing backend is really
|
||||
// observed end to end: bounded retries, then feature/platform telemetry.
|
||||
if (route === "/examples/reference-resources" && referenceVisits % 5 === 1) {
|
||||
await page.waitForTimeout(200);
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => (window as unknown as { __soakProbe: () => Probe }).__soakProbe().inFlight === 0,
|
||||
undefined,
|
||||
{ timeout: 5_000 },
|
||||
)
|
||||
.catch(() => undefined);
|
||||
settledReferenceVisits += 1;
|
||||
}
|
||||
const elapsedMs = Date.now() - started;
|
||||
if (elapsedMs >= nextSample) {
|
||||
nextSample = elapsedMs + options.intervalMs;
|
||||
await cdp.send("HeapProfiler.collectGarbage").catch(() => undefined);
|
||||
const { metrics } = (await cdp.send("Performance.getMetrics")) as {
|
||||
metrics: { name: string; value: number }[];
|
||||
};
|
||||
const metric = (name: string) => metrics.find((entry) => entry.name === name)?.value ?? 0;
|
||||
const probed = await probe();
|
||||
samples.push({
|
||||
elapsedMs,
|
||||
cycles,
|
||||
heapBytes: metric("JSHeapUsedSize"),
|
||||
domNodes: metric("Nodes"),
|
||||
listeners: metric("JSEventListeners"),
|
||||
activeTimers: probed.activeTimers,
|
||||
inFlightRequests: probed.inFlight,
|
||||
storageKeys: probed.storageKeys,
|
||||
storageBytes: probed.storageBytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
// Let the last interaction's bounded retries and deadlines settle.
|
||||
let finalInFlightRequests = (await probe()).inFlight;
|
||||
for (let wait = 0; wait < 30 && finalInFlightRequests > 0; wait += 1) {
|
||||
await page.waitForTimeout(500);
|
||||
finalInFlightRequests = (await probe()).inFlight;
|
||||
}
|
||||
const evaluation = evaluateSoak(
|
||||
samples,
|
||||
{
|
||||
cycles,
|
||||
navigationFailures,
|
||||
referenceVisits,
|
||||
settledReferenceVisits,
|
||||
finalInFlightRequests,
|
||||
apiRequests: counters.apiRequests,
|
||||
telemetryPosts: counters.telemetryPosts,
|
||||
telemetryEvents: counters.telemetryEvents,
|
||||
telemetryPostTimes: counters.telemetryPostTimes,
|
||||
},
|
||||
options.warmupMs,
|
||||
);
|
||||
const build = await readFile(path.join(options.distRoot, "release-manifest.json"), "utf8")
|
||||
.then((raw) => {
|
||||
const manifest = JSON.parse(raw) as Record<string, unknown>;
|
||||
const pick = (key: string) =>
|
||||
typeof manifest[key] === "string" ? (manifest[key] as string) : undefined;
|
||||
return {
|
||||
...(pick("buildId") ? { buildId: pick("buildId") } : {}),
|
||||
...(pick("releaseId") ? { releaseId: pick("releaseId") } : {}),
|
||||
...(pick("builtAt") ? { builtAt: pick("builtAt") } : {}),
|
||||
};
|
||||
})
|
||||
.catch(() => ({}));
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: evaluation.checks.every((check) => check.passed) ? "passed" : "failed",
|
||||
durationMs: options.durationMs,
|
||||
classification: evaluation.summary.heapClassification,
|
||||
summary: evaluation.summary,
|
||||
checks: evaluation.checks,
|
||||
build,
|
||||
browser: `chromium ${browser.version()}`,
|
||||
samples,
|
||||
};
|
||||
} finally {
|
||||
await browser?.close().catch(() => undefined);
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argument = (name: string, fallback: string) =>
|
||||
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;
|
||||
const artifact = path.resolve(argument("artifact", "artifacts/scale/browser-soak.json"));
|
||||
const result = await runBrowserSoak({
|
||||
distRoot: argument("dist", "dist"),
|
||||
durationMs: Number(argument("duration-ms", "60000")),
|
||||
intervalMs: Number(argument("interval-ms", "5000")),
|
||||
warmupMs: Number(argument("warmup-ms", "15000")),
|
||||
routes: argument(
|
||||
"routes",
|
||||
"/,/examples/reference-resources,/examples/platform,/examples/ui,/examples/states,/examples/reference-resources/status",
|
||||
).split(","),
|
||||
});
|
||||
await mkdir(path.dirname(artifact), { recursive: true });
|
||||
await writeFile(artifact, `${JSON.stringify(result, null, 2)}\n`);
|
||||
process.stdout.write(
|
||||
`browser soak: ${result.status}${result.reason ? ` (${result.reason})` : ""}` +
|
||||
`${result.classification ? ` classification=${result.classification}` : ""} samples=${result.samples.length}` +
|
||||
`${result.summary ? ` cycles=${result.summary.cycles}` : ""}\n`,
|
||||
);
|
||||
for (const check of result.checks ?? []) {
|
||||
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
|
||||
}
|
||||
if (result.status === "failed") process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { collectTscScale } from "./collect-tsc-diagnostics.ts";
|
||||
import { runCompositionScale } from "./run-composition-scale.ts";
|
||||
import {
|
||||
approveBaseline,
|
||||
compareToBaseline,
|
||||
currentEnvironment,
|
||||
snapshotMetrics,
|
||||
type ScaleBaseline,
|
||||
} from "./scale-baseline.ts";
|
||||
|
||||
/**
|
||||
* Spec §48–§49. Relative regression gate against the stored, approved scale
|
||||
* baseline (`config/scale/benchmark-baseline.json`).
|
||||
*
|
||||
* node scripts/scale/check-scale-baseline.ts compare, fail on regression
|
||||
* SCALE_BASELINE_OWNER=... SCALE_BASELINE_REASON=... \
|
||||
* node scripts/scale/check-scale-baseline.ts --update re-measure and re-approve
|
||||
*
|
||||
* The comparison is written to artifacts/scale/baseline-comparison.json (or
|
||||
* `--artifact=`). Updating is an explicit, attributed act; the check never
|
||||
* rewrites the baseline.
|
||||
*/
|
||||
export const BASELINE_PATH = "config/scale/benchmark-baseline.json";
|
||||
export const LOCAL_TSC_SIZES = Object.freeze([25, 100, 400]);
|
||||
|
||||
function typescriptVersion(): string {
|
||||
const result = spawnSync(path.resolve("node_modules/.bin/tsc"), ["-v"], { encoding: "utf8" });
|
||||
const match = /Version\s+(\S+)/u.exec(result.stdout ?? "");
|
||||
if (!match?.[1]) throw new Error("cannot determine TypeScript version");
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function measure() {
|
||||
const composition = await runCompositionScale(["S0", "S1", "S2"], { samples: 9, warmup: 2 });
|
||||
const compiler = await collectTscScale(LOCAL_TSC_SIZES);
|
||||
return snapshotMetrics(composition.results, compiler);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const update = process.argv.includes("--update");
|
||||
const artifact = path.resolve(
|
||||
process.argv.find((entry) => entry.startsWith("--artifact="))?.slice("--artifact=".length) ??
|
||||
"artifacts/scale/baseline-comparison.json",
|
||||
);
|
||||
const environment = currentEnvironment(typescriptVersion());
|
||||
const metrics = await measure();
|
||||
|
||||
if (update) {
|
||||
const owner = process.env["SCALE_BASELINE_OWNER"];
|
||||
const reason = process.env["SCALE_BASELINE_REASON"];
|
||||
if (!owner || !reason) {
|
||||
process.stderr.write("SCALE_BASELINE_OWNER and SCALE_BASELINE_REASON are required.\n");
|
||||
process.exit(1);
|
||||
}
|
||||
const baseline = approveBaseline(environment, metrics, {
|
||||
owner,
|
||||
reason,
|
||||
approvedAt: new Date().toISOString(),
|
||||
});
|
||||
await mkdir(path.dirname(BASELINE_PATH), { recursive: true });
|
||||
await writeFile(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`);
|
||||
process.stdout.write(`scale baseline approved: ${baseline.approval.metricsDigest}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
let baseline: ScaleBaseline;
|
||||
try {
|
||||
baseline = JSON.parse(await readFile(BASELINE_PATH, "utf8")) as ScaleBaseline;
|
||||
} catch {
|
||||
process.stderr.write(`missing or unreadable ${BASELINE_PATH}; run update:scale-baseline\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
const comparison = compareToBaseline(baseline, environment, metrics);
|
||||
await mkdir(path.dirname(artifact), { recursive: true });
|
||||
await writeFile(
|
||||
artifact,
|
||||
`${JSON.stringify(
|
||||
{ schemaVersion: 1, generatedAt: new Date().toISOString(), environment, baselineApproval: baseline.approval, ...comparison },
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
const { compatibility } = comparison;
|
||||
process.stdout.write(
|
||||
`compatibility: structural=${compatibility.structural.ok} compiler=${compatibility.compiler.ok} timing=${compatibility.timing.ok}\n`,
|
||||
);
|
||||
if (!compatibility.timing.ok) {
|
||||
process.stdout.write(
|
||||
`timing regression NOT compared (incomplete): ${compatibility.timing.reasons.join("; ")}\n`,
|
||||
);
|
||||
}
|
||||
process.stdout.write(
|
||||
`axes: absolute ${comparison.absolute.findings.length} (${comparison.absolute.passed ? "ok" : "FAIL"}), ` +
|
||||
`growth ${comparison.growth.findings.length} (${comparison.growth.passed ? "ok" : "FAIL"}), ` +
|
||||
`relative ${comparison.compiler.length} compiler + ${comparison.timing.length} timing\n`,
|
||||
);
|
||||
for (const failure of comparison.failures) process.stdout.write(`FAIL ${failure}\n`);
|
||||
process.stdout.write(
|
||||
`scale baseline gate: ${comparison.status.toUpperCase()} (${path.relative(process.cwd(), artifact)})\n`,
|
||||
);
|
||||
if (comparison.status === "failed") process.exitCode = 1;
|
||||
else if (comparison.status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import {
|
||||
assertLinearOrBetter,
|
||||
growthExponents,
|
||||
} from "../../src/platform/cardinality/growth-regression.ts";
|
||||
|
||||
/**
|
||||
* Spec §38. TypeScript scale gate.
|
||||
*
|
||||
* Parses `tsc --extendedDiagnostics` and measures bounded local fixtures:
|
||||
* generated feature modules composed by one context, at increasing local
|
||||
* cardinality. S3/S4 are never compiled; they are verified as topology.
|
||||
*/
|
||||
export type TscDiagnostics = Readonly<{
|
||||
files: number;
|
||||
lines?: number;
|
||||
types: number;
|
||||
symbols: number;
|
||||
instantiations: number;
|
||||
memoryKb?: number;
|
||||
totalTimeSeconds?: number;
|
||||
}>;
|
||||
|
||||
const FIELD_NAMES: Readonly<Record<string, keyof TscDiagnostics>> = Object.freeze({
|
||||
Files: "files",
|
||||
Lines: "lines",
|
||||
Types: "types",
|
||||
Symbols: "symbols",
|
||||
Instantiations: "instantiations",
|
||||
"Memory used": "memoryKb",
|
||||
"Total time": "totalTimeSeconds",
|
||||
});
|
||||
|
||||
export function parseTscExtendedDiagnostics(text: string): TscDiagnostics {
|
||||
const parsed: Partial<Record<keyof TscDiagnostics, number>> = {};
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const match = /^\s*([A-Za-z][A-Za-z ]*?):\s+([\d.]+)\s*([KMs]?)\s*$/u.exec(line);
|
||||
if (!match) continue;
|
||||
const field = FIELD_NAMES[match[1] as string];
|
||||
if (!field) continue;
|
||||
const value = Number(match[2]);
|
||||
if (!Number.isFinite(value)) continue;
|
||||
parsed[field] = match[3] === "M" ? value * 1024 : value;
|
||||
}
|
||||
for (const required of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
if (parsed[required] === undefined) {
|
||||
throw new TypeError(`tsc diagnostics missing ${required}`);
|
||||
}
|
||||
}
|
||||
return Object.freeze(parsed) as TscDiagnostics;
|
||||
}
|
||||
|
||||
export function runTscDiagnostics(project: string, cwd = process.cwd()): TscDiagnostics {
|
||||
const tsc = resolve(cwd, "node_modules/.bin/tsc");
|
||||
const result = spawnSync(tsc, ["--project", project, "--noEmit", "--extendedDiagnostics"], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`tsc failed for ${project}: ${(result.stdout + result.stderr).slice(-2_000)}`);
|
||||
}
|
||||
return parseTscExtendedDiagnostics(result.stdout);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates one bounded local app: `features` feature modules, each with its
|
||||
* own typed contract, composed by one generated index. The shape mirrors the
|
||||
* repository's feature-module/generated-index pattern.
|
||||
*/
|
||||
export async function writeLocalTscFixture(
|
||||
root: string,
|
||||
features: number,
|
||||
): Promise<string> {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
await mkdir(join(root, "features"), { recursive: true });
|
||||
const imports: string[] = [];
|
||||
const members: string[] = [];
|
||||
for (let index = 0; index < features; index += 1) {
|
||||
const id = `feature${index}`;
|
||||
await writeFile(
|
||||
join(root, "features", `${id}.ts`),
|
||||
`export type ${id}Input = Readonly<{ id: string; limit: number; tags?: readonly string[] }>;\n` +
|
||||
`export type ${id}Output = Readonly<{ items: readonly Readonly<{ id: string; name: string }>[] }>;\n` +
|
||||
`export const ${id} = Object.freeze({\n` +
|
||||
` id: "${id}" as const,\n` +
|
||||
` routes: Object.freeze({ ${id.toUpperCase()}_LIST: "/${id}" }),\n` +
|
||||
` execute(input: ${id}Input): ${id}Output {\n` +
|
||||
` return { items: [{ id: input.id, name: String(input.limit) }] };\n` +
|
||||
` },\n` +
|
||||
`});\n`,
|
||||
);
|
||||
imports.push(`import { ${id} } from "./features/${id}.ts";`);
|
||||
members.push(` ${id},`);
|
||||
}
|
||||
await writeFile(
|
||||
join(root, "index.ts"),
|
||||
`${imports.join("\n")}\nexport const LOCAL_FEATURES = Object.freeze([\n${members.join("\n")}\n] as const);\n` +
|
||||
`export type LocalFeatureId = (typeof LOCAL_FEATURES)[number]["id"];\n`,
|
||||
);
|
||||
const tsconfig = join(root, "tsconfig.json");
|
||||
await writeFile(
|
||||
tsconfig,
|
||||
JSON.stringify(
|
||||
{
|
||||
compilerOptions: {
|
||||
target: "ES2022",
|
||||
module: "ESNext",
|
||||
moduleResolution: "Bundler",
|
||||
allowImportingTsExtensions: true,
|
||||
strict: true,
|
||||
noEmit: true,
|
||||
skipLibCheck: true,
|
||||
types: [],
|
||||
lib: ["ES2022"],
|
||||
},
|
||||
include: ["index.ts", "features/*.ts"],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
return tsconfig;
|
||||
}
|
||||
|
||||
export type TscScaleReport = Readonly<{
|
||||
passed: boolean;
|
||||
app: TscDiagnostics;
|
||||
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
|
||||
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
|
||||
}>;
|
||||
|
||||
export async function collectTscScale(
|
||||
localSizes: readonly number[],
|
||||
cwd = process.cwd(),
|
||||
): Promise<TscScaleReport> {
|
||||
const app = runTscDiagnostics("tsconfig.app.json", cwd);
|
||||
const local: { features: number; diagnostics: TscDiagnostics }[] = [];
|
||||
for (const features of localSizes) {
|
||||
const tsconfig = await writeLocalTscFixture(
|
||||
resolve(cwd, ".tmp/scale-tsc", `local-${features}`),
|
||||
features,
|
||||
);
|
||||
local.push({ features, diagnostics: runTscDiagnostics(tsconfig, cwd) });
|
||||
}
|
||||
const checks: { name: string; passed: boolean; detail: string }[] = [];
|
||||
for (const metric of ["types", "symbols", "instantiations"] as const) {
|
||||
const series = local.map(({ features, diagnostics }) => ({
|
||||
n: features,
|
||||
// Library/baseline cost is constant; growth is judged on what the local
|
||||
// features add on top of the smallest fixture.
|
||||
cost: Math.max(1, diagnostics[metric]),
|
||||
}));
|
||||
try {
|
||||
assertLinearOrBetter(series, { maxExponent: 1.3 });
|
||||
checks.push({
|
||||
name: `tsc:${metric}`,
|
||||
passed: true,
|
||||
detail: `exponents ${growthExponents(series).map((value) => value.toFixed(2)).join(", ")}`,
|
||||
});
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
name: `tsc:${metric}`,
|
||||
passed: false,
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
passed: checks.every((check) => check.passed),
|
||||
app,
|
||||
local: Object.freeze(local),
|
||||
checks: Object.freeze(checks),
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const artifactArg = process.argv.find((argument) => argument.startsWith("--artifact="));
|
||||
const sizesArg = process.argv.find((argument) => argument.startsWith("--local-sizes="));
|
||||
const sizes = (sizesArg?.slice("--local-sizes=".length) ?? "25,100,400")
|
||||
.split(",")
|
||||
.map(Number)
|
||||
.filter((value) => Number.isSafeInteger(value) && value > 0);
|
||||
const artifact = resolve(
|
||||
artifactArg?.slice("--artifact=".length) ?? "artifacts/scale/tsc-diagnostics.json",
|
||||
);
|
||||
const report = await collectTscScale(sizes);
|
||||
await mkdir(resolve(artifact, ".."), { recursive: true });
|
||||
await writeFile(
|
||||
artifact,
|
||||
`${JSON.stringify({ schemaVersion: 1, generatedAt: new Date().toISOString(), ...report }, null, 2)}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`tsc app: files=${report.app.files} types=${report.app.types} instantiations=${report.app.instantiations}\n`,
|
||||
);
|
||||
for (const check of report.checks) {
|
||||
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
|
||||
}
|
||||
process.stdout.write(`tsc diagnostics artifact: ${artifact}\n`);
|
||||
if (!report.passed) process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { composeUniqueRecord } from "../../src/platform/cardinality/compose-unique-record.ts";
|
||||
import {
|
||||
resolveProductFeatures,
|
||||
selectCompiledProductFeatures,
|
||||
} from "../../src/contracts/product-features.ts";
|
||||
import { defineBoundedContextModule } from "../../src/contexts/bounded-context-module.ts";
|
||||
import { defineDeployableAppModule } from "../../src/apps/deployable-app-module.ts";
|
||||
|
||||
/**
|
||||
* Spec §39. Scale-sensitive production composition paths, each parameterized
|
||||
* by one cardinality `n`. Inputs are built outside the timed region.
|
||||
*/
|
||||
export type CompositionWorkload = Readonly<{
|
||||
name: string;
|
||||
implementation: string;
|
||||
prepare(n: number): () => unknown;
|
||||
}>;
|
||||
|
||||
export function createUniqueRegistryParts(
|
||||
count: number,
|
||||
): readonly Readonly<Record<string, number>>[] {
|
||||
return Array.from({ length: count }, (_, index) =>
|
||||
Object.freeze({ [`route-${index}`]: index }),
|
||||
);
|
||||
}
|
||||
|
||||
function featureIds(count: number): string[] {
|
||||
return Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `feature-${index.toString().padStart(7, "0")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const CONTEXT_FEATURES = 200;
|
||||
|
||||
export const COMPOSITION_WORKLOADS: readonly CompositionWorkload[] = Object.freeze([
|
||||
{
|
||||
name: "registry-composition",
|
||||
implementation: "composeUniqueRecord",
|
||||
prepare(n: number) {
|
||||
const parts = createUniqueRegistryParts(n);
|
||||
return () => composeUniqueRecord(parts, () => "route");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "product-feature-selection",
|
||||
implementation: "selectCompiledProductFeatures",
|
||||
prepare(n: number) {
|
||||
const compiled = featureIds(n).map((featureId) => Object.freeze({ featureId }));
|
||||
const declared = compiled
|
||||
.filter((_, index) => index % 2 === 0)
|
||||
.map((feature) => feature.featureId)
|
||||
.join(",");
|
||||
return () => selectCompiledProductFeatures(compiled, declared);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "product-feature-resolution",
|
||||
implementation: "resolveProductFeatures",
|
||||
prepare(n: number) {
|
||||
const compiled = featureIds(n);
|
||||
const installed = compiled.filter((_, index) => index % 3 !== 0);
|
||||
return () => resolveProductFeatures(compiled, installed);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "context-hierarchy-composition",
|
||||
implementation: "defineBoundedContextModule+defineDeployableAppModule",
|
||||
prepare(n: number) {
|
||||
const features = featureIds(n).map((id) =>
|
||||
Object.freeze({
|
||||
id,
|
||||
contracts: Object.freeze({ featureId: id }),
|
||||
routes: Object.freeze({}),
|
||||
adapters: Object.freeze([]),
|
||||
runtimes: Object.freeze([]),
|
||||
messages: Object.freeze({}),
|
||||
contractContributions: Object.freeze([]),
|
||||
}),
|
||||
);
|
||||
const contextCount = Math.ceil(n / CONTEXT_FEATURES);
|
||||
return () => {
|
||||
const contexts = Array.from({ length: contextCount }, (_, contextIndex) =>
|
||||
defineBoundedContextModule({
|
||||
id: `context-${contextIndex}`,
|
||||
features: features.slice(
|
||||
contextIndex * CONTEXT_FEATURES,
|
||||
(contextIndex + 1) * CONTEXT_FEATURES,
|
||||
),
|
||||
budget: Object.freeze({ maxFeatures: CONTEXT_FEATURES }),
|
||||
}),
|
||||
);
|
||||
// Apps are sharded by the same direct-child budget.
|
||||
const apps: unknown[] = [];
|
||||
for (let index = 0; index < contexts.length; index += 20) {
|
||||
apps.push(
|
||||
defineDeployableAppModule({
|
||||
id: `app-${index / 20}`,
|
||||
contexts: contexts.slice(index, index + 20),
|
||||
budget: Object.freeze({ maxContexts: 20 }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
return apps;
|
||||
};
|
||||
},
|
||||
},
|
||||
]);
|
||||
@@ -0,0 +1,259 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { availableParallelism, cpus, loadavg, tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { gzipSync } from "node:zlib";
|
||||
|
||||
import { classifyViteJavascript } from "../lib/classify-vite-bundle.ts";
|
||||
import { linkFixtureNodeModules } from "../lib/fixture-node-modules.ts";
|
||||
import { parseTscExtendedDiagnostics, type TscDiagnostics } from "./collect-tsc-diagnostics.ts";
|
||||
|
||||
/**
|
||||
* 11-04 (D-18). Cost of one app as its feature count grows, measured with
|
||||
* features shaped like the real one: `src/features/reference-feature` copied N
|
||||
* times with every feature-owned identifier renamed (feature id, route ids and
|
||||
* paths, API operations, schemas, query namespace, invalidation topic,
|
||||
* messages). The copies live in contexts of their own (200 features each, the
|
||||
* context ceiling) so the reference context is unchanged.
|
||||
*
|
||||
* For each N the script runs the product generator, `tsc --extendedDiagnostics`
|
||||
* over tsconfig.app.json and `vite build`, and records compiler counts, wall
|
||||
* times and initial/lazy JavaScript gzip bytes. Deterministic counts repeat
|
||||
* exactly; times depend on the host, so the CPU and load average are recorded
|
||||
* next to them. A typecheck that fails (for example TS2590, a union too complex
|
||||
* to represent) is recorded as a result with its first error.
|
||||
*
|
||||
* node scripts/scale/measure-feature-shape-growth.ts [--sizes=0,25,100] [--repeats=2] [--write]
|
||||
*
|
||||
* `--write` stores the result as config/scale/feature-shape-growth.json.
|
||||
*/
|
||||
export const FEATURE_SHAPE_BASELINE_PATH = "config/scale/feature-shape-growth.json";
|
||||
/** What the declared ceilings rest on, derived by hand from the baseline. */
|
||||
export const FEATURE_SHAPE_CEILINGS_PATH = "config/scale/feature-shape-ceilings.json";
|
||||
|
||||
const SOURCE_FEATURE = "src/features/reference-feature";
|
||||
const FEATURES_PER_CONTEXT = 200;
|
||||
const BUILD_INPUTS = [
|
||||
"src",
|
||||
"public",
|
||||
"index.html",
|
||||
"package.json",
|
||||
"vite.config.ts",
|
||||
"tsconfig.base.json",
|
||||
"tsconfig.app.json",
|
||||
"scripts/lib/vite-module-inventory.ts",
|
||||
] as const;
|
||||
// Platform-owned names that only happen to contain "reference".
|
||||
const PLATFORM_TOKENS = [
|
||||
"REFERENCE_EXTERNAL_BEARER",
|
||||
"REFERENCE_FEATURE_V1",
|
||||
"supportReference",
|
||||
] as const;
|
||||
|
||||
/** Renames every feature-owned identifier of the reference feature for copy `id`. */
|
||||
export function renameReferenceFeature(
|
||||
text: string,
|
||||
id: string,
|
||||
context: Readonly<{ id: string; routeRoot: string }>,
|
||||
): string {
|
||||
const upper = id.toUpperCase();
|
||||
let renamed = text;
|
||||
PLATFORM_TOKENS.forEach((token, index) => {
|
||||
renamed = renamed.replaceAll(token, `\u0000${index}\u0000`);
|
||||
});
|
||||
renamed = renamed
|
||||
.replaceAll("/examples/reference-resources", `/${context.routeRoot}/${id}-resources`)
|
||||
.replaceAll("qinv.01k10f7m3w9p6r2c8v5n4x", `qinv.shape.${id}`)
|
||||
.replaceAll("NoRequest", `${upper}NoRequest`)
|
||||
.replaceAll("reference-context", context.id)
|
||||
.replaceAll("REFERENCE", upper)
|
||||
.replaceAll("Reference", upper)
|
||||
.replaceAll("reference", id);
|
||||
PLATFORM_TOKENS.forEach((token, index) => {
|
||||
renamed = renamed.replaceAll(`\u0000${index}\u0000`, token);
|
||||
});
|
||||
return renamed;
|
||||
}
|
||||
|
||||
async function copyRenamed(from: string, to: string, rename: (text: string) => string) {
|
||||
await mkdir(to, { recursive: true });
|
||||
for (const entry of await readdir(from, { withFileTypes: true })) {
|
||||
const source = path.join(from, entry.name);
|
||||
const target = path.join(to, rename(entry.name));
|
||||
if (entry.isDirectory()) await copyRenamed(source, target, rename);
|
||||
else await writeFile(target, rename(await readFile(source, "utf8")));
|
||||
}
|
||||
}
|
||||
|
||||
async function createFixture(features: number): Promise<string> {
|
||||
const root = await mkdtemp(path.join(tmpdir(), `feature-shape-${features}-`));
|
||||
for (const input of BUILD_INPUTS) {
|
||||
await mkdir(path.dirname(path.join(root, input)), { recursive: true });
|
||||
await cp(path.resolve(input), path.join(root, input), { recursive: true });
|
||||
}
|
||||
await rm(path.join(root, "src/.generated"), { recursive: true, force: true });
|
||||
await linkFixtureNodeModules(root);
|
||||
for (let index = 0; index < features; index += 1) {
|
||||
const contextNumber = Math.floor(index / FEATURES_PER_CONTEXT);
|
||||
const context = { id: `shape-context-${contextNumber}`, routeRoot: `shape-${contextNumber}` };
|
||||
if (index % FEATURES_PER_CONTEXT === 0) {
|
||||
const contextFile = path.join(root, "src/contexts", context.id, "context-module.ts");
|
||||
await mkdir(path.dirname(contextFile), { recursive: true });
|
||||
await writeFile(
|
||||
contextFile,
|
||||
`export const CONTEXT_MODULE_ID = "${context.id}";\n` +
|
||||
`export const CONTEXT_APP_ID = "template-app";\n` +
|
||||
`export const CONTEXT_ROUTE_PATH = "${context.routeRoot}/*";\n` +
|
||||
`export const CONTEXT_MAX_FEATURES = ${FEATURES_PER_CONTEXT};\n`,
|
||||
);
|
||||
}
|
||||
const id = `s${String(index).padStart(3, "0")}`;
|
||||
await copyRenamed(
|
||||
path.resolve(SOURCE_FEATURE),
|
||||
path.join(root, "src/features", `${id}-feature`),
|
||||
(text) => renameReferenceFeature(text, id, context),
|
||||
);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function run(
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
cwd: string,
|
||||
options: Readonly<{ allowFailure?: boolean }> = {},
|
||||
) {
|
||||
const started = performance.now();
|
||||
const result = spawnSync(command, args, {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
const ms = Math.round(performance.now() - started);
|
||||
if (result.status !== 0 && !(options.allowFailure && result.stdout.includes("Instantiations:"))) {
|
||||
throw new Error(
|
||||
`${path.basename(command)} ${args.join(" ")} failed in ${cwd}:\n${(result.stdout + result.stderr).slice(0, 4_000)}`,
|
||||
);
|
||||
}
|
||||
return { ms, passed: result.status === 0, stdout: result.stdout };
|
||||
}
|
||||
|
||||
const load = () => loadavg().map((value) => Math.round(value * 100) / 100);
|
||||
|
||||
async function javascriptGzipBytes(root: string, files: readonly string[]) {
|
||||
let bytes = 0;
|
||||
for (const file of files) bytes += gzipSync(await readFile(path.join(root, "dist", file))).byteLength;
|
||||
return bytes;
|
||||
}
|
||||
|
||||
async function measureOnce(root: string, features: number) {
|
||||
const loadBefore = load();
|
||||
run(process.execPath, [path.resolve("scripts/generate-product-feature-index.ts"), `--root=${root}`], root);
|
||||
// The composed registries throw at module evaluation on a duplicate route,
|
||||
// operation, schema, namespace or message key: prove the copies compose.
|
||||
const contracts = (await import(
|
||||
`${pathToFileURL(path.join(root, "src/features/installed-feature-contracts.ts")).href}?features=${features}`
|
||||
)) as typeof import("../../src/features/installed-feature-contracts.ts");
|
||||
const routes = Object.keys(contracts.ROUTE_REGISTRY).length;
|
||||
const tsc = run(
|
||||
path.resolve("node_modules/.bin/tsc"),
|
||||
["--project", path.join(root, "tsconfig.app.json"), "--noEmit", "--extendedDiagnostics"],
|
||||
root,
|
||||
// A type error is a result here (the typecheck stops scaling), not a crash.
|
||||
{ allowFailure: true },
|
||||
);
|
||||
await rm(path.join(root, "dist"), { recursive: true, force: true });
|
||||
const build = run(path.resolve("node_modules/.bin/vite"), ["build", "--logLevel", "error"], root);
|
||||
const manifest = JSON.parse(await readFile(path.join(root, "dist/.vite/manifest.json"), "utf8"));
|
||||
const javascript = classifyViteJavascript(manifest);
|
||||
const diagnostics: TscDiagnostics = parseTscExtendedDiagnostics(tsc.stdout);
|
||||
return {
|
||||
addedFeatures: features,
|
||||
routes,
|
||||
tsc: {
|
||||
passed: tsc.passed,
|
||||
firstError: tsc.stdout.split("\n").find((line) => line.includes("error TS")) ?? null,
|
||||
files: diagnostics.files,
|
||||
types: diagnostics.types,
|
||||
symbols: diagnostics.symbols,
|
||||
instantiations: diagnostics.instantiations,
|
||||
memoryKb: diagnostics.memoryKb ?? null,
|
||||
wallMs: tsc.ms,
|
||||
},
|
||||
build: {
|
||||
wallMs: build.ms,
|
||||
initialJavascriptFiles: javascript.initialFiles.length,
|
||||
initialJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.initialFiles),
|
||||
lazyJavascriptFiles: javascript.lazyFiles.length,
|
||||
lazyJavascriptGzipBytes: await javascriptGzipBytes(root, javascript.lazyFiles),
|
||||
},
|
||||
loadBefore,
|
||||
loadAfter: load(),
|
||||
};
|
||||
}
|
||||
|
||||
function version(bin: string): string {
|
||||
const result = spawnSync(path.resolve("node_modules/.bin", bin), ["--version"], { encoding: "utf8" });
|
||||
return (result.stdout ?? "").trim();
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argument = (name: string) =>
|
||||
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
const sizes = (argument("sizes") ?? "0,25,100").split(",").map(Number);
|
||||
const repeats = Number(argument("repeats") ?? "2");
|
||||
if (!sizes.every((size) => Number.isSafeInteger(size) && size >= 0) || !(repeats >= 1)) {
|
||||
throw new TypeError("--sizes must be nonnegative integers and --repeats at least 1");
|
||||
}
|
||||
const runs = [];
|
||||
for (const features of sizes) {
|
||||
const root = await createFixture(features);
|
||||
try {
|
||||
for (let repeat = 1; repeat <= repeats; repeat += 1) {
|
||||
const measured = await measureOnce(root, features);
|
||||
runs.push({ repeat, ...measured });
|
||||
process.stdout.write(
|
||||
`features +${features} (repeat ${repeat}): routes=${measured.routes} ` +
|
||||
`instantiations=${measured.tsc.instantiations} tsc=${measured.tsc.wallMs}ms ` +
|
||||
`build=${measured.build.wallMs}ms initial=${measured.build.initialJavascriptGzipBytes}B ` +
|
||||
`lazy=${measured.build.lazyJavascriptGzipBytes}B load=${measured.loadBefore.join("/")}` +
|
||||
`${measured.tsc.passed ? "" : ` typecheck FAILED: ${measured.tsc.firstError}`}\n`,
|
||||
);
|
||||
// Compiler results repeat exactly; a failed typecheck costs minutes and
|
||||
// gigabytes again for no new information.
|
||||
if (!measured.tsc.passed) break;
|
||||
}
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
const cpu = cpus();
|
||||
const result = {
|
||||
schemaVersion: 1,
|
||||
measuredAt: new Date().toISOString(),
|
||||
command: `node scripts/scale/measure-feature-shape-growth.ts --sizes=${sizes.join(",")} --repeats=${repeats}`,
|
||||
environment: {
|
||||
node: process.versions.node,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
cpuModel: cpu[0]?.model.trim() ?? "unknown",
|
||||
logicalCpus: cpu.length,
|
||||
availableParallelism: availableParallelism(),
|
||||
typescript: version("tsc"),
|
||||
vite: version("vite"),
|
||||
},
|
||||
sourceFeature: SOURCE_FEATURE,
|
||||
runs,
|
||||
};
|
||||
const output = process.argv.includes("--write")
|
||||
? FEATURE_SHAPE_BASELINE_PATH
|
||||
: "artifacts/scale/feature-shape-growth.json";
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
await writeFile(output, `${JSON.stringify(result, null, 2)}\n`);
|
||||
process.stdout.write(`feature-shape growth: ${output}\n`);
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Spec §44. Soak classification.
|
||||
*
|
||||
* The leak signal is sustained positive growth after warm-up, not an
|
||||
* arbitrary absolute maximum. The classifier fits a least-squares line over
|
||||
* the post-warm-up samples and reports `unbounded-growth` only when both the
|
||||
* slope and the total relative growth are material.
|
||||
*/
|
||||
export type SlopeSample = Readonly<{ minute: number; heapMb: number }>;
|
||||
|
||||
export type MemorySlopeClass = "stable" | "unbounded-growth" | "insufficient-data";
|
||||
|
||||
export type MemorySlopeOptions = Readonly<{
|
||||
/** Samples before this minute are warm-up. Default 0. */
|
||||
warmupMinutes?: number;
|
||||
/** Largest tolerated post-warm-up slope in MB per minute. Default 0.5. */
|
||||
maxSlopeMbPerMinute?: number;
|
||||
/** Largest tolerated relative growth first->last fitted value. Default 0.1. */
|
||||
maxRelativeGrowth?: number;
|
||||
}>;
|
||||
|
||||
export function linearSlope(points: readonly Readonly<{ x: number; y: number }>[]): number {
|
||||
const count = points.length;
|
||||
if (count < 2) return 0;
|
||||
const meanX = points.reduce((sum, point) => sum + point.x, 0) / count;
|
||||
const meanY = points.reduce((sum, point) => sum + point.y, 0) / count;
|
||||
let numerator = 0;
|
||||
let denominator = 0;
|
||||
for (const point of points) {
|
||||
numerator += (point.x - meanX) * (point.y - meanY);
|
||||
denominator += (point.x - meanX) ** 2;
|
||||
}
|
||||
return denominator === 0 ? 0 : numerator / denominator;
|
||||
}
|
||||
|
||||
export function classifyMemorySlope(
|
||||
samples: readonly SlopeSample[],
|
||||
options: MemorySlopeOptions = {},
|
||||
): MemorySlopeClass {
|
||||
const warmup = options.warmupMinutes ?? 0;
|
||||
const maxSlope = options.maxSlopeMbPerMinute ?? 0.5;
|
||||
const maxGrowth = options.maxRelativeGrowth ?? 0.1;
|
||||
const measured = samples
|
||||
.filter((sample) => sample.minute >= warmup)
|
||||
.filter((sample) => Number.isFinite(sample.minute) && Number.isFinite(sample.heapMb))
|
||||
.sort((left, right) => left.minute - right.minute);
|
||||
if (measured.length < 3) return "insufficient-data";
|
||||
const slope = linearSlope(measured.map((sample) => ({ x: sample.minute, y: sample.heapMb })));
|
||||
const first = measured[0] as SlopeSample;
|
||||
const last = measured[measured.length - 1] as SlopeSample;
|
||||
const fittedStart = Math.max(first.heapMb, Number.MIN_VALUE);
|
||||
const growth = (slope * (last.minute - first.minute)) / fittedStart;
|
||||
return slope > maxSlope && growth > maxGrowth ? "unbounded-growth" : "stable";
|
||||
}
|
||||
|
||||
/** Generic variant for logical counters (timers, listeners, buffered events). */
|
||||
export function classifyCounterSeries(
|
||||
values: readonly number[],
|
||||
options: Readonly<{ warmupSamples?: number; maxSlopePerSample?: number }> = {},
|
||||
): MemorySlopeClass {
|
||||
const warmup = options.warmupSamples ?? 0;
|
||||
const measured = values.slice(warmup);
|
||||
if (measured.length < 3) return "insufficient-data";
|
||||
const slope = linearSlope(measured.map((y, x) => ({ x, y })));
|
||||
return slope > (options.maxSlopePerSample ?? 0.01) ? "unbounded-growth" : "stable";
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import {
|
||||
assertLinearOrBetter,
|
||||
growthExponents,
|
||||
} from "../../src/platform/cardinality/growth-regression.ts";
|
||||
import { benchmark } from "./benchmark.ts";
|
||||
import {
|
||||
writeBenchmarkArtifact,
|
||||
type BenchmarkProfile,
|
||||
type BenchmarkResult,
|
||||
} from "./benchmark-result.ts";
|
||||
import { COMPOSITION_WORKLOADS } from "./composition-workloads.ts";
|
||||
|
||||
/**
|
||||
* Spec §36 / §39. Runtime composition scale gate for S0-S2.
|
||||
*
|
||||
* Every production composition workload runs at each requested profile with
|
||||
* warm-up and repeated samples; the median series must be linear-or-better.
|
||||
* Results are written as a machine-readable artifact under artifacts/scale/.
|
||||
*/
|
||||
export const COMPOSITION_PROFILES: Readonly<Record<string, number>> = Object.freeze({
|
||||
S0: 100,
|
||||
S1: 1_000,
|
||||
S2: 10_000,
|
||||
});
|
||||
|
||||
/** Scheduler noise below this median is not evidence of growth. */
|
||||
export const COMPOSITION_NOISE_FLOOR_MS = 0.5;
|
||||
/**
|
||||
* n log n paths (sorting, allocation-heavy string work) measure ~1.1-1.4
|
||||
* between S1 and S2; a quadratic path measures ~2. 1.6 separates the two.
|
||||
*/
|
||||
export const COMPOSITION_MAX_EXPONENT = 1.6;
|
||||
|
||||
export type CompositionScaleReport = Readonly<{
|
||||
passed: boolean;
|
||||
results: readonly BenchmarkResult[];
|
||||
checks: readonly Readonly<{ name: string; passed: boolean; detail: string }>[];
|
||||
}>;
|
||||
|
||||
export async function runCompositionScale(
|
||||
profiles: readonly string[],
|
||||
options: Readonly<{
|
||||
samples?: number;
|
||||
warmup?: number;
|
||||
/**
|
||||
* `median` (default) for standalone gate runs. `min` (best observed) when
|
||||
* the gate shares the machine with other suites: stolen CPU and foreign
|
||||
* GC only ever add time, so the minimum keeps the algorithmic shape.
|
||||
*/
|
||||
statistic?: "median" | "min";
|
||||
}> = {},
|
||||
): Promise<CompositionScaleReport> {
|
||||
const results: BenchmarkResult[] = [];
|
||||
const checks: { name: string; passed: boolean; detail: string }[] = [];
|
||||
for (const workload of COMPOSITION_WORKLOADS) {
|
||||
const series: { n: number; cost: number }[] = [];
|
||||
for (const profile of profiles) {
|
||||
const cardinality = COMPOSITION_PROFILES[profile];
|
||||
if (cardinality === undefined) throw new TypeError(`unknown profile ${profile}`);
|
||||
const run = workload.prepare(cardinality);
|
||||
const result = await benchmark(workload.name, {
|
||||
samples: options.samples ?? 7,
|
||||
warmup: options.warmup ?? 2,
|
||||
profile: profile as BenchmarkProfile,
|
||||
cardinality,
|
||||
measureHeap: true,
|
||||
metadata: { implementation: workload.implementation },
|
||||
run,
|
||||
});
|
||||
results.push(result);
|
||||
series.push({
|
||||
n: cardinality,
|
||||
cost:
|
||||
options.statistic === "min" ? Math.min(...result.samples) : result.medianMs,
|
||||
});
|
||||
}
|
||||
if (series.length < 2) continue;
|
||||
try {
|
||||
assertLinearOrBetter(series, {
|
||||
maxExponent: COMPOSITION_MAX_EXPONENT,
|
||||
noiseFloor: COMPOSITION_NOISE_FLOOR_MS,
|
||||
});
|
||||
checks.push({
|
||||
name: `${workload.name}:growth`,
|
||||
passed: true,
|
||||
detail: `exponents ${growthExponents(series, COMPOSITION_NOISE_FLOOR_MS)
|
||||
.map((exponent) => exponent.toFixed(2))
|
||||
.join(", ")}`,
|
||||
});
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
name: `${workload.name}:growth`,
|
||||
passed: false,
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
passed: checks.every((check) => check.passed),
|
||||
results: Object.freeze(results),
|
||||
checks: Object.freeze(checks),
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const profileArg = process.argv.find((argument) => argument.startsWith("--profiles="));
|
||||
const artifactArg = process.argv.find((argument) => argument.startsWith("--artifact="));
|
||||
const profiles = (profileArg?.slice("--profiles=".length) ?? "S0,S1,S2")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const artifact = resolve(
|
||||
artifactArg?.slice("--artifact=".length) ?? "artifacts/scale/composition-scale.json",
|
||||
);
|
||||
const report = await runCompositionScale(profiles);
|
||||
await writeBenchmarkArtifact(artifact, {
|
||||
schemaVersion: 1,
|
||||
suite: "composition-scale",
|
||||
generatedAt: new Date().toISOString(),
|
||||
node: process.version,
|
||||
passed: report.passed,
|
||||
results: report.results,
|
||||
checks: report.checks,
|
||||
});
|
||||
for (const check of report.checks) {
|
||||
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
|
||||
}
|
||||
process.stdout.write(`composition scale artifact: ${artifact}\n`);
|
||||
if (!report.passed) process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
/**
|
||||
* Spec §47. Execution tiers for the scale/chaos/soak program.
|
||||
*
|
||||
* pr architecture and traffic invariants, S0-S2 composition growth
|
||||
* main pr + TypeScript diagnostics and the stored baseline regression
|
||||
* gate (config/scale/benchmark-baseline.json)
|
||||
* nightly main + S3/S4 topology, telemetry storm, session cycles, browser soak
|
||||
* release release-coherence matrix + the stable main/nightly subset
|
||||
*
|
||||
* Only `pr` runs in CI (merge gate FE-GATE-005, 11-03). Its checks are counts,
|
||||
* bounds and growth shape, so they hold on any CPU; the composition timings it
|
||||
* writes are advisory. `main`, `nightly` and `release` include
|
||||
* `baseline-regression`, which compares timings only on the CPU model that
|
||||
* recorded config/scale/benchmark-baseline.json and is `incomplete` elsewhere,
|
||||
* so no CI job runs them; they are local commands for that machine.
|
||||
*
|
||||
* Every step's output goes to artifacts/scale/<tier>/<step>.log and the tier
|
||||
* writes one machine-readable summary.json. A step that could not run (e.g. a
|
||||
* browser soak without a build or browser) is `skipped`, and a tier with any
|
||||
* skipped step is `incomplete`: it is never reported as passed and exits
|
||||
* non-zero. `SCALE_ALLOW_INCOMPLETE=1` keeps the exit code zero for local runs,
|
||||
* but the summary still says `incomplete`.
|
||||
*/
|
||||
export type ScaleTier = "pr" | "main" | "nightly" | "release";
|
||||
|
||||
type Step = Readonly<{
|
||||
id: string;
|
||||
command: readonly string[];
|
||||
/** A step that may legitimately report `skipped` in its own artifact. */
|
||||
optional?: boolean;
|
||||
}>;
|
||||
|
||||
const VITEST = ["scripts/run-vitest.ts", "run", "--reporter=default"];
|
||||
|
||||
const PR_STEPS: readonly Step[] = [
|
||||
{
|
||||
id: "architecture-topology",
|
||||
command: [...VITEST, "tests/architecture", "tests/platform/cardinality", "tests/platform/routing"],
|
||||
},
|
||||
{
|
||||
id: "traffic-budgets",
|
||||
command: [...VITEST, "tests/platform/execution", "tests/platform/traffic", "tests/platform/resilience"],
|
||||
},
|
||||
{
|
||||
id: "scale-units",
|
||||
command: [
|
||||
...VITEST,
|
||||
"tests/scale/growth-regression.test.ts",
|
||||
"tests/scale/benchmark.test.ts",
|
||||
"tests/scale/synthetic-topology.test.ts",
|
||||
"tests/scale/scale-baseline.test.ts",
|
||||
"tests/scale/traffic",
|
||||
],
|
||||
},
|
||||
// 11-03: S2 is included because below it every workload stays under the
|
||||
// noise floor, so a quadratic accumulation passed S0/S1 unnoticed.
|
||||
{
|
||||
id: "composition-s0-s2",
|
||||
command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1,S2"],
|
||||
},
|
||||
];
|
||||
|
||||
const MAIN_STEPS: readonly Step[] = [
|
||||
...PR_STEPS,
|
||||
{
|
||||
id: "composition-scale-suite",
|
||||
command: [...VITEST, "tests/scale/composition-scale.test.ts", "tests/scale/tsc-diagnostics-parser.test.ts"],
|
||||
},
|
||||
{
|
||||
id: "typescript-diagnostics",
|
||||
command: ["scripts/scale/collect-tsc-diagnostics.ts", "--local-sizes=25,100,400"],
|
||||
},
|
||||
{
|
||||
id: "baseline-regression",
|
||||
command: ["scripts/scale/check-scale-baseline.ts"],
|
||||
},
|
||||
];
|
||||
|
||||
const NIGHTLY_STEPS: readonly Step[] = [
|
||||
...MAIN_STEPS,
|
||||
{
|
||||
id: "topology-s3-s4",
|
||||
command: [...VITEST, "tests/scale/global-local-independence.test.ts", "tests/platform/cardinality/extreme-scale-registry.test.ts"],
|
||||
},
|
||||
{
|
||||
id: "telemetry-storm",
|
||||
command: [...VITEST, "tests/scale/telemetry", "tests/platform/telemetry"],
|
||||
},
|
||||
{ id: "session-cycles", command: [...VITEST, "tests/scale/session"] },
|
||||
{
|
||||
id: "browser-soak",
|
||||
command: ["scripts/scale/browser-soak.ts", "--duration-ms=120000", "--interval-ms=5000", "--warmup-ms=30000"],
|
||||
optional: true,
|
||||
},
|
||||
];
|
||||
|
||||
const RELEASE_STEPS: readonly Step[] = [
|
||||
{
|
||||
id: "release-coherence",
|
||||
command: [...VITEST, "tests/platform/release", "tests/release"],
|
||||
},
|
||||
...MAIN_STEPS,
|
||||
{
|
||||
id: "topology-s3-s4",
|
||||
command: [...VITEST, "tests/scale/global-local-independence.test.ts"],
|
||||
},
|
||||
{
|
||||
id: "telemetry-storm",
|
||||
command: [...VITEST, "tests/scale/telemetry"],
|
||||
},
|
||||
];
|
||||
|
||||
export const SCALE_TIERS: Readonly<Record<ScaleTier, readonly Step[]>> = Object.freeze({
|
||||
pr: PR_STEPS,
|
||||
main: MAIN_STEPS,
|
||||
nightly: NIGHTLY_STEPS,
|
||||
release: RELEASE_STEPS,
|
||||
});
|
||||
|
||||
function withArtifact(step: Step, directory: string): readonly string[] {
|
||||
const script = step.command[0] ?? "";
|
||||
if (
|
||||
script.endsWith("run-composition-scale.ts") ||
|
||||
script.endsWith("collect-tsc-diagnostics.ts") ||
|
||||
script.endsWith("browser-soak.ts") ||
|
||||
script.endsWith("check-scale-baseline.ts")
|
||||
) {
|
||||
return [...step.command, `--artifact=${path.join(directory, `${step.id}.json`)}`];
|
||||
}
|
||||
return step.command;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const tier = process.argv[2] as ScaleTier | undefined;
|
||||
if (!tier || !(tier in SCALE_TIERS)) {
|
||||
process.stderr.write(`usage: run-scale-profile.ts <${Object.keys(SCALE_TIERS).join("|")}>\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const directory = path.resolve("artifacts/scale", tier);
|
||||
await mkdir(directory, { recursive: true });
|
||||
const results: { id: string; status: "passed" | "failed" | "skipped"; durationMs: number; log: string }[] = [];
|
||||
for (const step of SCALE_TIERS[tier]) {
|
||||
const startedAt = Date.now();
|
||||
const run = spawnSync(process.execPath, [...withArtifact(step, directory)], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
env: { ...process.env, SCALE_ARTIFACT_DIR: path.join(directory, "metrics") },
|
||||
});
|
||||
const log = path.join(directory, `${step.id}.log`);
|
||||
await writeFile(log, `${run.stdout ?? ""}${run.stderr ?? ""}`);
|
||||
const skipped = step.optional === true && /: skipped/u.test(run.stdout ?? "");
|
||||
const status = run.status === 0 ? (skipped ? "skipped" : "passed") : "failed";
|
||||
results.push({ id: step.id, status, durationMs: Date.now() - startedAt, log: path.relative(process.cwd(), log) });
|
||||
process.stdout.write(`${status.toUpperCase()} ${tier}:${step.id} (${Date.now() - startedAt}ms)\n`);
|
||||
}
|
||||
const failed = results.some((result) => result.status === "failed");
|
||||
const skipped = results.filter((result) => result.status === "skipped").map((result) => result.id);
|
||||
const status = failed ? "failed" : skipped.length > 0 ? "incomplete" : "passed";
|
||||
await writeFile(
|
||||
path.join(directory, "summary.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
tier,
|
||||
generatedAt: new Date().toISOString(),
|
||||
node: process.version,
|
||||
environment: {
|
||||
node: process.version,
|
||||
nodeMajor: Number(process.versions.node.split(".")[0]),
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
},
|
||||
status,
|
||||
passed: status === "passed",
|
||||
skipped,
|
||||
steps: results,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`scale tier ${tier}: ${status.toUpperCase()}${skipped.length > 0 ? ` (skipped: ${skipped.join(", ")})` : ""} (artifacts/scale/${tier}/summary.json)\n`,
|
||||
);
|
||||
if (status === "failed") process.exitCode = 1;
|
||||
else if (status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { cpus } from "node:os";
|
||||
|
||||
import { growthExponents } from "../../src/platform/cardinality/growth-regression.ts";
|
||||
import type { BenchmarkResult } from "./benchmark-result.ts";
|
||||
import type { TscDiagnostics } from "./collect-tsc-diagnostics.ts";
|
||||
|
||||
/**
|
||||
* Spec §48–§49. Stored benchmark baseline, compatibility check and relative
|
||||
* regression gate.
|
||||
*
|
||||
* The growth gates judge the *shape* of one run. This module compares a run
|
||||
* against an approved stored baseline so a change that keeps the shape but
|
||||
* multiplies the cost (or silently adds compiler work) is still caught.
|
||||
*
|
||||
* Two kinds of metric are compared differently:
|
||||
* - deterministic metrics (TypeScript types/symbols/instantiations/files) are
|
||||
* exact for a given compiler, so they use a tight relative tolerance and
|
||||
* require a compatible toolchain;
|
||||
* - timing metrics depend on the machine, so they use a loose tolerance plus an
|
||||
* absolute noise floor, and are only enforced on a compatible environment.
|
||||
*/
|
||||
export const SCALE_BASELINE_SCHEMA_VERSION = 1;
|
||||
|
||||
export type ScaleEnvironment = Readonly<{
|
||||
nodeMajor: number;
|
||||
platform: string;
|
||||
arch: string;
|
||||
cpuModel: string;
|
||||
cpuCount: number;
|
||||
typescript: string;
|
||||
}>;
|
||||
|
||||
export type TimingEntry = Readonly<{
|
||||
workload: string;
|
||||
profile: string;
|
||||
cardinality: number;
|
||||
medianMs: number;
|
||||
minMs: number;
|
||||
}>;
|
||||
|
||||
export type CompilerEntry = Readonly<{
|
||||
fixture: string;
|
||||
files: number;
|
||||
types: number;
|
||||
symbols: number;
|
||||
instantiations: number;
|
||||
}>;
|
||||
|
||||
export type ScaleMetrics = Readonly<{
|
||||
timing: readonly TimingEntry[];
|
||||
compiler: readonly CompilerEntry[];
|
||||
}>;
|
||||
|
||||
export type ScaleBaseline = Readonly<{
|
||||
schemaVersion: typeof SCALE_BASELINE_SCHEMA_VERSION;
|
||||
environment: ScaleEnvironment;
|
||||
metrics: ScaleMetrics;
|
||||
approval: Readonly<{
|
||||
owner: string;
|
||||
reason: string;
|
||||
approvedAt: string;
|
||||
metricsDigest: string;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export const REGRESSION_TOLERANCE = Object.freeze({
|
||||
/** Deterministic compiler counts may grow at most 10% without re-approval. */
|
||||
compilerRelative: 0.1,
|
||||
/** Timings may grow at most 2.5x (scheduler/GC noise lives below that)... */
|
||||
timingRelative: 1.5,
|
||||
/** ...and never fail on differences below this absolute noise floor. */
|
||||
timingNoiseFloorMs: 1,
|
||||
});
|
||||
|
||||
/**
|
||||
* Axis 1 — absolute ceilings. Machine-agnostic hard limits (roughly 50x the
|
||||
* approved baseline for timings, ~2x for compiler work) that hold even with
|
||||
* no comparable baseline.
|
||||
*/
|
||||
export const ABSOLUTE_CEILINGS = Object.freeze({
|
||||
timingMinMsByProfile: Object.freeze({ S0: 5, S1: 25, S2: 150 } as Record<string, number>),
|
||||
app: Object.freeze({ files: 1_500, types: 600_000, symbols: 1_600_000, instantiations: 1_500_000 }),
|
||||
/**
|
||||
* Generated `local-<N>` compiler fixtures: a fixed base (lib/runtime
|
||||
* overhead present at any N) plus a per-local-feature allowance.
|
||||
*/
|
||||
localBase: Object.freeze({ files: 100, types: 5_000, symbols: 15_000, instantiations: 5_000 }),
|
||||
localPerFeature: Object.freeze({ files: 2, types: 100, symbols: 250, instantiations: 150 }),
|
||||
});
|
||||
|
||||
/** Axis 3 — growth shape (see src/platform/cardinality/growth-regression.ts). */
|
||||
export const GROWTH_LIMITS = Object.freeze({
|
||||
timingMaxExponent: 1.6,
|
||||
timingNoiseFloorMs: 0.5,
|
||||
compilerMaxExponent: 1.3,
|
||||
});
|
||||
|
||||
export function currentEnvironment(typescript: string): ScaleEnvironment {
|
||||
const cpu = cpus();
|
||||
return Object.freeze({
|
||||
nodeMajor: Number(process.versions.node.split(".")[0]),
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
cpuModel: cpu[0]?.model.trim() ?? "unknown",
|
||||
cpuCount: cpu.length,
|
||||
typescript,
|
||||
});
|
||||
}
|
||||
|
||||
function canonical(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonical((value as Record<string, unknown>)[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function metricsDigest(metrics: ScaleMetrics): string {
|
||||
return createHash("sha256").update(canonical(metrics)).digest("hex");
|
||||
}
|
||||
|
||||
export function snapshotMetrics(
|
||||
composition: readonly BenchmarkResult[],
|
||||
compiler: Readonly<{
|
||||
app: TscDiagnostics;
|
||||
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
|
||||
}>,
|
||||
): ScaleMetrics {
|
||||
const round = (value: number) => Math.round(value * 1_000) / 1_000;
|
||||
const toCompiler = (fixture: string, diagnostics: TscDiagnostics): CompilerEntry => ({
|
||||
fixture,
|
||||
files: diagnostics.files,
|
||||
types: diagnostics.types,
|
||||
symbols: diagnostics.symbols,
|
||||
instantiations: diagnostics.instantiations,
|
||||
});
|
||||
return Object.freeze({
|
||||
timing: composition
|
||||
.map((result) => ({
|
||||
workload: result.name,
|
||||
profile: result.profile,
|
||||
cardinality: result.cardinality,
|
||||
medianMs: round(result.medianMs),
|
||||
minMs: round(Math.min(...result.samples)),
|
||||
}))
|
||||
.sort((left, right) =>
|
||||
`${left.workload}/${left.profile}`.localeCompare(`${right.workload}/${right.profile}`),
|
||||
),
|
||||
compiler: [
|
||||
toCompiler("app", compiler.app),
|
||||
...compiler.local.map((entry) => toCompiler(`local-${entry.features}`, entry.diagnostics)),
|
||||
].sort((left, right) => left.fixture.localeCompare(right.fixture)),
|
||||
});
|
||||
}
|
||||
|
||||
export type Compatibility = Readonly<{
|
||||
/** Baseline format, digest and metric coverage are usable at all. */
|
||||
structural: Readonly<{ ok: boolean; reasons: readonly string[] }>;
|
||||
/** Same compiler: deterministic compiler counts are comparable. */
|
||||
compiler: Readonly<{ ok: boolean; reasons: readonly string[] }>;
|
||||
/** Same machine class and runtime: timings are comparable. */
|
||||
timing: Readonly<{ ok: boolean; reasons: readonly string[] }>;
|
||||
}>;
|
||||
|
||||
export function checkCompatibility(
|
||||
baseline: ScaleBaseline,
|
||||
environment: ScaleEnvironment,
|
||||
current: ScaleMetrics,
|
||||
): Compatibility {
|
||||
const structural: string[] = [];
|
||||
if (baseline.schemaVersion !== SCALE_BASELINE_SCHEMA_VERSION) {
|
||||
structural.push(`baseline schemaVersion ${String(baseline.schemaVersion)} is not ${SCALE_BASELINE_SCHEMA_VERSION}`);
|
||||
}
|
||||
if (metricsDigest(baseline.metrics) !== baseline.approval?.metricsDigest) {
|
||||
structural.push("baseline metrics do not match their approved digest (hand-edited?)");
|
||||
}
|
||||
const timingKey = (entry: TimingEntry) => `${entry.workload}/${entry.profile}/${entry.cardinality}`;
|
||||
const baselineTiming = new Set(baseline.metrics.timing.map(timingKey));
|
||||
const baselineCompiler = new Set(baseline.metrics.compiler.map((entry) => entry.fixture));
|
||||
for (const entry of current.timing) {
|
||||
if (!baselineTiming.has(timingKey(entry))) structural.push(`no baseline for timing ${timingKey(entry)}`);
|
||||
}
|
||||
for (const entry of current.compiler) {
|
||||
if (!baselineCompiler.has(entry.fixture)) structural.push(`no baseline for compiler fixture ${entry.fixture}`);
|
||||
}
|
||||
|
||||
const compiler: string[] = [];
|
||||
if (baseline.environment.typescript !== environment.typescript) {
|
||||
compiler.push(`TypeScript ${environment.typescript} differs from baseline ${baseline.environment.typescript}`);
|
||||
}
|
||||
|
||||
const timing: string[] = [];
|
||||
const base = baseline.environment;
|
||||
if (base.nodeMajor !== environment.nodeMajor) timing.push(`Node ${environment.nodeMajor} vs baseline ${base.nodeMajor}`);
|
||||
if (base.platform !== environment.platform || base.arch !== environment.arch) {
|
||||
timing.push(`${environment.platform}/${environment.arch} vs baseline ${base.platform}/${base.arch}`);
|
||||
}
|
||||
if (base.cpuModel !== environment.cpuModel) timing.push(`CPU "${environment.cpuModel}" vs baseline "${base.cpuModel}"`);
|
||||
return Object.freeze({
|
||||
structural: { ok: structural.length === 0, reasons: structural },
|
||||
compiler: { ok: compiler.length === 0, reasons: compiler },
|
||||
timing: { ok: timing.length === 0, reasons: timing },
|
||||
});
|
||||
}
|
||||
|
||||
export type RegressionFinding = Readonly<{
|
||||
metric: string;
|
||||
baseline: number;
|
||||
current: number;
|
||||
limit: number;
|
||||
passed: boolean;
|
||||
}>;
|
||||
|
||||
export type AxisResult = Readonly<{
|
||||
passed: boolean;
|
||||
findings: readonly RegressionFinding[];
|
||||
}>;
|
||||
|
||||
export type BaselineComparison = Readonly<{
|
||||
/**
|
||||
* `passed` only when every axis was evaluated and held. `incomplete` when the
|
||||
* environment is not timing-compatible: timing regression was not compared,
|
||||
* so it is not evidence of a pass.
|
||||
*/
|
||||
status: "passed" | "failed" | "incomplete";
|
||||
passed: boolean;
|
||||
compatibility: Compatibility;
|
||||
absolute: AxisResult;
|
||||
growth: AxisResult;
|
||||
compiler: readonly RegressionFinding[];
|
||||
/** Empty when the environment is not timing-compatible. */
|
||||
timing: readonly RegressionFinding[];
|
||||
failures: readonly string[];
|
||||
}>;
|
||||
|
||||
function finding(metric: string, baseline: number, current: number, limit: number): RegressionFinding {
|
||||
return { metric, baseline, current, limit: Math.round(limit * 1_000) / 1_000, passed: current <= limit };
|
||||
}
|
||||
|
||||
export function checkAbsoluteCeilings(current: ScaleMetrics): AxisResult {
|
||||
const findings: RegressionFinding[] = [];
|
||||
for (const entry of current.timing) {
|
||||
const limit = ABSOLUTE_CEILINGS.timingMinMsByProfile[entry.profile];
|
||||
if (limit !== undefined) {
|
||||
findings.push(finding(`ceiling:time:${entry.workload}/${entry.profile}`, 0, entry.minMs, limit));
|
||||
}
|
||||
}
|
||||
for (const entry of current.compiler) {
|
||||
const local = /^local-(\d+)$/u.exec(entry.fixture);
|
||||
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
const limit = local
|
||||
? ABSOLUTE_CEILINGS.localBase[metric] +
|
||||
ABSOLUTE_CEILINGS.localPerFeature[metric] * Number(local[1])
|
||||
: entry.fixture === "app"
|
||||
? ABSOLUTE_CEILINGS.app[metric]
|
||||
: Number.POSITIVE_INFINITY;
|
||||
findings.push(finding(`ceiling:tsc:${entry.fixture}:${metric}`, 0, entry[metric], limit));
|
||||
}
|
||||
}
|
||||
return { passed: findings.every((item) => item.passed), findings };
|
||||
}
|
||||
|
||||
export function checkGrowthShape(current: ScaleMetrics): AxisResult {
|
||||
const findings: RegressionFinding[] = [];
|
||||
const byWorkload = new Map<string, { n: number; cost: number }[]>();
|
||||
for (const entry of current.timing) {
|
||||
const series = byWorkload.get(entry.workload) ?? [];
|
||||
series.push({ n: entry.cardinality, cost: entry.minMs });
|
||||
byWorkload.set(entry.workload, series);
|
||||
}
|
||||
for (const [workload, series] of byWorkload) {
|
||||
if (series.length < 2) continue;
|
||||
const worst = Math.max(...growthExponents(series, GROWTH_LIMITS.timingNoiseFloorMs));
|
||||
findings.push(finding(`growth:time:${workload}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.timingMaxExponent));
|
||||
}
|
||||
const local = current.compiler
|
||||
.map((entry) => ({ entry, n: Number(/^local-(\d+)$/u.exec(entry.fixture)?.[1]) }))
|
||||
.filter(({ n }) => Number.isFinite(n));
|
||||
if (local.length >= 2) {
|
||||
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
const worst = Math.max(...growthExponents(local.map(({ entry, n }) => ({ n, cost: entry[metric] }))));
|
||||
findings.push(finding(`growth:tsc:local:${metric}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.compilerMaxExponent));
|
||||
}
|
||||
}
|
||||
return { passed: findings.every((item) => item.passed), findings };
|
||||
}
|
||||
|
||||
export function compareToBaseline(
|
||||
baseline: ScaleBaseline,
|
||||
environment: ScaleEnvironment,
|
||||
current: ScaleMetrics,
|
||||
tolerance = REGRESSION_TOLERANCE,
|
||||
): BaselineComparison {
|
||||
const compatibility = checkCompatibility(baseline, environment, current);
|
||||
const failures: string[] = [...compatibility.structural.reasons];
|
||||
// A different compiler changes every deterministic count: the baseline must
|
||||
// be re-approved, it is never silently compared or silently skipped.
|
||||
if (!compatibility.compiler.ok) {
|
||||
failures.push(
|
||||
...compatibility.compiler.reasons.map((reason) => `${reason}; re-approve with update:scale-baseline`),
|
||||
);
|
||||
}
|
||||
|
||||
const absolute = checkAbsoluteCeilings(current);
|
||||
const growth = checkGrowthShape(current);
|
||||
for (const item of [...absolute.findings, ...growth.findings]) {
|
||||
if (!item.passed) failures.push(`${item.metric} ${item.current} exceeds ${item.limit}`);
|
||||
}
|
||||
|
||||
const compiler: RegressionFinding[] = [];
|
||||
if (compatibility.structural.ok && compatibility.compiler.ok) {
|
||||
const byFixture = new Map(baseline.metrics.compiler.map((entry) => [entry.fixture, entry]));
|
||||
for (const entry of current.compiler) {
|
||||
const reference = byFixture.get(entry.fixture);
|
||||
if (!reference) continue;
|
||||
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
|
||||
const item = finding(
|
||||
`tsc:${entry.fixture}:${metric}`,
|
||||
reference[metric],
|
||||
entry[metric],
|
||||
Math.floor(reference[metric] * (1 + tolerance.compilerRelative)),
|
||||
);
|
||||
compiler.push(item);
|
||||
if (!item.passed) {
|
||||
failures.push(`${item.metric} ${item.current} exceeds ${item.limit} (baseline ${item.baseline})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const timing: RegressionFinding[] = [];
|
||||
if (compatibility.structural.ok && compatibility.timing.ok) {
|
||||
const key = (entry: TimingEntry) => `${entry.workload}/${entry.profile}`;
|
||||
const byKey = new Map(baseline.metrics.timing.map((entry) => [key(entry), entry]));
|
||||
for (const entry of current.timing) {
|
||||
const reference = byKey.get(key(entry));
|
||||
if (!reference) continue;
|
||||
// Best-observed samples: contention only ever adds time.
|
||||
const item = finding(
|
||||
`time:${key(entry)}`,
|
||||
reference.minMs,
|
||||
entry.minMs,
|
||||
Math.max(reference.minMs * (1 + tolerance.timingRelative), reference.minMs + tolerance.timingNoiseFloorMs),
|
||||
);
|
||||
timing.push(item);
|
||||
if (!item.passed) {
|
||||
failures.push(`${item.metric} ${item.current}ms exceeds ${item.limit}ms (baseline ${item.baseline}ms)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const status =
|
||||
failures.length > 0 ? "failed" : compatibility.timing.ok ? "passed" : "incomplete";
|
||||
return Object.freeze({
|
||||
status,
|
||||
passed: status === "passed",
|
||||
compatibility,
|
||||
absolute,
|
||||
growth,
|
||||
compiler,
|
||||
timing,
|
||||
failures,
|
||||
});
|
||||
}
|
||||
|
||||
export function approveBaseline(
|
||||
environment: ScaleEnvironment,
|
||||
metrics: ScaleMetrics,
|
||||
approval: Readonly<{ owner: string; reason: string; approvedAt: string }>,
|
||||
): ScaleBaseline {
|
||||
return Object.freeze({
|
||||
schemaVersion: SCALE_BASELINE_SCHEMA_VERSION,
|
||||
environment,
|
||||
metrics,
|
||||
approval: { ...approval, metricsDigest: metricsDigest(metrics) },
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user