import { createHash } from "node:crypto"; import { cpus } from "node:os"; import { growthExponents } from "../../src/platform/cardinality/growth-regression.ts"; import type { BenchmarkResult } from "./benchmark-result.ts"; import type { TscDiagnostics } from "./collect-tsc-diagnostics.ts"; /** * scale-architecture.md §48–§49. Stored benchmark baseline, compatibility check and relative * regression gate. * * The growth gates judge the *shape* of one run. This module compares a run * against an approved stored baseline so a change that keeps the shape but * multiplies the cost (or silently adds compiler work) is still caught. * * Two kinds of metric are compared differently: * - deterministic metrics (TypeScript types/symbols/instantiations/files) are * exact for a given compiler, so they use a tight relative tolerance and * require a compatible toolchain; * - timing metrics depend on the machine, so they use a loose tolerance plus an * absolute noise floor, and are only enforced on a compatible environment. */ export const SCALE_BASELINE_SCHEMA_VERSION = 1; export type ScaleEnvironment = Readonly<{ nodeMajor: number; platform: string; arch: string; cpuModel: string; cpuCount: number; typescript: string; }>; export type TimingEntry = Readonly<{ workload: string; profile: string; cardinality: number; medianMs: number; minMs: number; }>; export type CompilerEntry = Readonly<{ fixture: string; files: number; types: number; symbols: number; instantiations: number; }>; export type ScaleMetrics = Readonly<{ timing: readonly TimingEntry[]; compiler: readonly CompilerEntry[]; }>; export type ScaleBaseline = Readonly<{ schemaVersion: typeof SCALE_BASELINE_SCHEMA_VERSION; environment: ScaleEnvironment; metrics: ScaleMetrics; approval: Readonly<{ owner: string; reason: string; approvedAt: string; metricsDigest: string; }>; }>; export const REGRESSION_TOLERANCE = Object.freeze({ /** Deterministic compiler counts may grow at most 10% without re-approval. */ compilerRelative: 0.1, /** Timings may grow at most 2.5x (scheduler/GC noise lives below that)... */ timingRelative: 1.5, /** ...and never fail on differences below this absolute noise floor. */ timingNoiseFloorMs: 1, }); /** * Axis 1 — absolute ceilings. Machine-agnostic hard limits (roughly 50x the * approved baseline for timings, ~2x for compiler work) that hold even with * no comparable baseline. */ export const ABSOLUTE_CEILINGS = Object.freeze({ timingMinMsByProfile: Object.freeze({ S0: 5, S1: 25, S2: 150 } as Record), app: Object.freeze({ files: 1_500, types: 600_000, symbols: 1_600_000, instantiations: 1_500_000 }), /** * Generated `local-` compiler fixtures: a fixed base (lib/runtime * overhead present at any N) plus a per-local-feature allowance. */ localBase: Object.freeze({ files: 100, types: 5_000, symbols: 15_000, instantiations: 5_000 }), localPerFeature: Object.freeze({ files: 2, types: 100, symbols: 250, instantiations: 150 }), }); /** Axis 3 — growth shape (see src/platform/cardinality/growth-regression.ts). */ export const GROWTH_LIMITS = Object.freeze({ timingMaxExponent: 1.6, timingNoiseFloorMs: 0.5, compilerMaxExponent: 1.3, }); export function currentEnvironment(typescript: string): ScaleEnvironment { const cpu = cpus(); return Object.freeze({ nodeMajor: Number(process.versions.node.split(".")[0]), platform: process.platform, arch: process.arch, cpuModel: cpu[0]?.model.trim() ?? "unknown", cpuCount: cpu.length, typescript, }); } function canonical(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; if (value && typeof value === "object") { return `{${Object.keys(value) .sort() .map((key) => `${JSON.stringify(key)}:${canonical((value as Record)[key])}`) .join(",")}}`; } return JSON.stringify(value); } export function metricsDigest(metrics: ScaleMetrics): string { return createHash("sha256").update(canonical(metrics)).digest("hex"); } export function snapshotMetrics( composition: readonly BenchmarkResult[], compiler: Readonly<{ app: TscDiagnostics; local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[]; }>, ): ScaleMetrics { const round = (value: number) => Math.round(value * 1_000) / 1_000; const toCompiler = (fixture: string, diagnostics: TscDiagnostics): CompilerEntry => ({ fixture, files: diagnostics.files, types: diagnostics.types, symbols: diagnostics.symbols, instantiations: diagnostics.instantiations, }); return Object.freeze({ timing: composition .map((result) => ({ workload: result.name, profile: result.profile, cardinality: result.cardinality, medianMs: round(result.medianMs), minMs: round(Math.min(...result.samples)), })) .sort((left, right) => `${left.workload}/${left.profile}`.localeCompare(`${right.workload}/${right.profile}`), ), compiler: [ toCompiler("app", compiler.app), ...compiler.local.map((entry) => toCompiler(`local-${entry.features}`, entry.diagnostics)), ].sort((left, right) => left.fixture.localeCompare(right.fixture)), }); } export type Compatibility = Readonly<{ /** Baseline format, digest and metric coverage are usable at all. */ structural: Readonly<{ ok: boolean; reasons: readonly string[] }>; /** Same compiler: deterministic compiler counts are comparable. */ compiler: Readonly<{ ok: boolean; reasons: readonly string[] }>; /** Same machine class and runtime: timings are comparable. */ timing: Readonly<{ ok: boolean; reasons: readonly string[] }>; }>; export function checkCompatibility( baseline: ScaleBaseline, environment: ScaleEnvironment, current: ScaleMetrics, ): Compatibility { const structural: string[] = []; if (baseline.schemaVersion !== SCALE_BASELINE_SCHEMA_VERSION) { structural.push(`baseline schemaVersion ${String(baseline.schemaVersion)} is not ${SCALE_BASELINE_SCHEMA_VERSION}`); } if (metricsDigest(baseline.metrics) !== baseline.approval?.metricsDigest) { structural.push("baseline metrics do not match their approved digest (hand-edited?)"); } const timingKey = (entry: TimingEntry) => `${entry.workload}/${entry.profile}/${entry.cardinality}`; const baselineTiming = new Set(baseline.metrics.timing.map(timingKey)); const baselineCompiler = new Set(baseline.metrics.compiler.map((entry) => entry.fixture)); for (const entry of current.timing) { if (!baselineTiming.has(timingKey(entry))) structural.push(`no baseline for timing ${timingKey(entry)}`); } for (const entry of current.compiler) { if (!baselineCompiler.has(entry.fixture)) structural.push(`no baseline for compiler fixture ${entry.fixture}`); } const compiler: string[] = []; if (baseline.environment.typescript !== environment.typescript) { compiler.push(`TypeScript ${environment.typescript} differs from baseline ${baseline.environment.typescript}`); } const timing: string[] = []; const base = baseline.environment; if (base.nodeMajor !== environment.nodeMajor) timing.push(`Node ${environment.nodeMajor} vs baseline ${base.nodeMajor}`); if (base.platform !== environment.platform || base.arch !== environment.arch) { timing.push(`${environment.platform}/${environment.arch} vs baseline ${base.platform}/${base.arch}`); } if (base.cpuModel !== environment.cpuModel) timing.push(`CPU "${environment.cpuModel}" vs baseline "${base.cpuModel}"`); return Object.freeze({ structural: { ok: structural.length === 0, reasons: structural }, compiler: { ok: compiler.length === 0, reasons: compiler }, timing: { ok: timing.length === 0, reasons: timing }, }); } export type RegressionFinding = Readonly<{ metric: string; baseline: number; current: number; limit: number; passed: boolean; }>; export type AxisResult = Readonly<{ passed: boolean; findings: readonly RegressionFinding[]; }>; export type BaselineComparison = Readonly<{ /** * `passed` only when every axis was evaluated and held. `incomplete` when the * environment is not timing-compatible: timing regression was not compared, * so it is not evidence of a pass. */ status: "passed" | "failed" | "incomplete"; passed: boolean; compatibility: Compatibility; absolute: AxisResult; growth: AxisResult; compiler: readonly RegressionFinding[]; /** Empty when the environment is not timing-compatible. */ timing: readonly RegressionFinding[]; failures: readonly string[]; }>; function finding(metric: string, baseline: number, current: number, limit: number): RegressionFinding { return { metric, baseline, current, limit: Math.round(limit * 1_000) / 1_000, passed: current <= limit }; } export function checkAbsoluteCeilings(current: ScaleMetrics): AxisResult { const findings: RegressionFinding[] = []; for (const entry of current.timing) { const limit = ABSOLUTE_CEILINGS.timingMinMsByProfile[entry.profile]; if (limit !== undefined) { findings.push(finding(`ceiling:time:${entry.workload}/${entry.profile}`, 0, entry.minMs, limit)); } } for (const entry of current.compiler) { const local = /^local-(\d+)$/u.exec(entry.fixture); for (const metric of ["files", "types", "symbols", "instantiations"] as const) { const limit = local ? ABSOLUTE_CEILINGS.localBase[metric] + ABSOLUTE_CEILINGS.localPerFeature[metric] * Number(local[1]) : entry.fixture === "app" ? ABSOLUTE_CEILINGS.app[metric] : Number.POSITIVE_INFINITY; findings.push(finding(`ceiling:tsc:${entry.fixture}:${metric}`, 0, entry[metric], limit)); } } return { passed: findings.every((item) => item.passed), findings }; } export function checkGrowthShape(current: ScaleMetrics): AxisResult { const findings: RegressionFinding[] = []; const byWorkload = new Map(); for (const entry of current.timing) { const series = byWorkload.get(entry.workload) ?? []; series.push({ n: entry.cardinality, cost: entry.minMs }); byWorkload.set(entry.workload, series); } for (const [workload, series] of byWorkload) { if (series.length < 2) continue; const worst = Math.max(...growthExponents(series, GROWTH_LIMITS.timingNoiseFloorMs)); findings.push(finding(`growth:time:${workload}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.timingMaxExponent)); } const local = current.compiler .map((entry) => ({ entry, n: Number(/^local-(\d+)$/u.exec(entry.fixture)?.[1]) })) .filter(({ n }) => Number.isFinite(n)); if (local.length >= 2) { for (const metric of ["files", "types", "symbols", "instantiations"] as const) { const worst = Math.max(...growthExponents(local.map(({ entry, n }) => ({ n, cost: entry[metric] })))); findings.push(finding(`growth:tsc:local:${metric}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.compilerMaxExponent)); } } return { passed: findings.every((item) => item.passed), findings }; } export function compareToBaseline( baseline: ScaleBaseline, environment: ScaleEnvironment, current: ScaleMetrics, tolerance = REGRESSION_TOLERANCE, ): BaselineComparison { const compatibility = checkCompatibility(baseline, environment, current); const failures: string[] = [...compatibility.structural.reasons]; // A different compiler changes every deterministic count: the baseline must // be re-approved, it is never silently compared or silently skipped. if (!compatibility.compiler.ok) { failures.push( ...compatibility.compiler.reasons.map((reason) => `${reason}; re-approve with update:scale-baseline`), ); } const absolute = checkAbsoluteCeilings(current); const growth = checkGrowthShape(current); for (const item of [...absolute.findings, ...growth.findings]) { if (!item.passed) failures.push(`${item.metric} ${item.current} exceeds ${item.limit}`); } const compiler: RegressionFinding[] = []; if (compatibility.structural.ok && compatibility.compiler.ok) { const byFixture = new Map(baseline.metrics.compiler.map((entry) => [entry.fixture, entry])); for (const entry of current.compiler) { const reference = byFixture.get(entry.fixture); if (!reference) continue; for (const metric of ["files", "types", "symbols", "instantiations"] as const) { const item = finding( `tsc:${entry.fixture}:${metric}`, reference[metric], entry[metric], Math.floor(reference[metric] * (1 + tolerance.compilerRelative)), ); compiler.push(item); if (!item.passed) { failures.push(`${item.metric} ${item.current} exceeds ${item.limit} (baseline ${item.baseline})`); } } } } const timing: RegressionFinding[] = []; if (compatibility.structural.ok && compatibility.timing.ok) { const key = (entry: TimingEntry) => `${entry.workload}/${entry.profile}`; const byKey = new Map(baseline.metrics.timing.map((entry) => [key(entry), entry])); for (const entry of current.timing) { const reference = byKey.get(key(entry)); if (!reference) continue; // Best-observed samples: contention only ever adds time. const item = finding( `time:${key(entry)}`, reference.minMs, entry.minMs, Math.max(reference.minMs * (1 + tolerance.timingRelative), reference.minMs + tolerance.timingNoiseFloorMs), ); timing.push(item); if (!item.passed) { failures.push(`${item.metric} ${item.current}ms exceeds ${item.limit}ms (baseline ${item.baseline}ms)`); } } } const status = failures.length > 0 ? "failed" : compatibility.timing.ok ? "passed" : "incomplete"; return Object.freeze({ status, passed: status === "passed", compatibility, absolute, growth, compiler, timing, failures, }); } export function approveBaseline( environment: ScaleEnvironment, metrics: ScaleMetrics, approval: Readonly<{ owner: string; reason: string; approvedAt: string }>, ): ScaleBaseline { return Object.freeze({ schemaVersion: SCALE_BASELINE_SCHEMA_VERSION, environment, metrics, approval: { ...approval, metricsDigest: metricsDigest(metrics) }, }); }