Files
clean-architecture-frontend…/scripts/scale/scale-baseline.ts
T

382 lines
14 KiB
TypeScript

import { createHash } from "node:crypto";
import { cpus } from "node:os";
import { growthExponents } from "../../src/platform/cardinality/growth-regression.ts";
import type { BenchmarkResult } from "./benchmark-result.ts";
import type { TscDiagnostics } from "./collect-tsc-diagnostics.ts";
/**
* Spec §48–§49. Stored benchmark baseline, compatibility check and relative
* regression gate.
*
* The growth gates judge the *shape* of one run. This module compares a run
* against an approved stored baseline so a change that keeps the shape but
* multiplies the cost (or silently adds compiler work) is still caught.
*
* Two kinds of metric are compared differently:
* - deterministic metrics (TypeScript types/symbols/instantiations/files) are
* exact for a given compiler, so they use a tight relative tolerance and
* require a compatible toolchain;
* - timing metrics depend on the machine, so they use a loose tolerance plus an
* absolute noise floor, and are only enforced on a compatible environment.
*/
export const SCALE_BASELINE_SCHEMA_VERSION = 1;
export type ScaleEnvironment = Readonly<{
nodeMajor: number;
platform: string;
arch: string;
cpuModel: string;
cpuCount: number;
typescript: string;
}>;
export type TimingEntry = Readonly<{
workload: string;
profile: string;
cardinality: number;
medianMs: number;
minMs: number;
}>;
export type CompilerEntry = Readonly<{
fixture: string;
files: number;
types: number;
symbols: number;
instantiations: number;
}>;
export type ScaleMetrics = Readonly<{
timing: readonly TimingEntry[];
compiler: readonly CompilerEntry[];
}>;
export type ScaleBaseline = Readonly<{
schemaVersion: typeof SCALE_BASELINE_SCHEMA_VERSION;
environment: ScaleEnvironment;
metrics: ScaleMetrics;
approval: Readonly<{
owner: string;
reason: string;
approvedAt: string;
metricsDigest: string;
}>;
}>;
export const REGRESSION_TOLERANCE = Object.freeze({
/** Deterministic compiler counts may grow at most 10% without re-approval. */
compilerRelative: 0.1,
/** Timings may grow at most 2.5x (scheduler/GC noise lives below that)... */
timingRelative: 1.5,
/** ...and never fail on differences below this absolute noise floor. */
timingNoiseFloorMs: 1,
});
/**
* Axis 1 — absolute ceilings. Machine-agnostic hard limits (roughly 50x the
* approved baseline for timings, ~2x for compiler work) that hold even with
* no comparable baseline.
*/
export const ABSOLUTE_CEILINGS = Object.freeze({
timingMinMsByProfile: Object.freeze({ S0: 5, S1: 25, S2: 150 } as Record<string, number>),
app: Object.freeze({ files: 1_500, types: 600_000, symbols: 1_600_000, instantiations: 1_500_000 }),
/**
* Generated `local-<N>` compiler fixtures: a fixed base (lib/runtime
* overhead present at any N) plus a per-local-feature allowance.
*/
localBase: Object.freeze({ files: 100, types: 5_000, symbols: 15_000, instantiations: 5_000 }),
localPerFeature: Object.freeze({ files: 2, types: 100, symbols: 250, instantiations: 150 }),
});
/** Axis 3 — growth shape (see src/platform/cardinality/growth-regression.ts). */
export const GROWTH_LIMITS = Object.freeze({
timingMaxExponent: 1.6,
timingNoiseFloorMs: 0.5,
compilerMaxExponent: 1.3,
});
export function currentEnvironment(typescript: string): ScaleEnvironment {
const cpu = cpus();
return Object.freeze({
nodeMajor: Number(process.versions.node.split(".")[0]),
platform: process.platform,
arch: process.arch,
cpuModel: cpu[0]?.model.trim() ?? "unknown",
cpuCount: cpu.length,
typescript,
});
}
function canonical(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
if (value && typeof value === "object") {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonical((value as Record<string, unknown>)[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
export function metricsDigest(metrics: ScaleMetrics): string {
return createHash("sha256").update(canonical(metrics)).digest("hex");
}
export function snapshotMetrics(
composition: readonly BenchmarkResult[],
compiler: Readonly<{
app: TscDiagnostics;
local: readonly Readonly<{ features: number; diagnostics: TscDiagnostics }>[];
}>,
): ScaleMetrics {
const round = (value: number) => Math.round(value * 1_000) / 1_000;
const toCompiler = (fixture: string, diagnostics: TscDiagnostics): CompilerEntry => ({
fixture,
files: diagnostics.files,
types: diagnostics.types,
symbols: diagnostics.symbols,
instantiations: diagnostics.instantiations,
});
return Object.freeze({
timing: composition
.map((result) => ({
workload: result.name,
profile: result.profile,
cardinality: result.cardinality,
medianMs: round(result.medianMs),
minMs: round(Math.min(...result.samples)),
}))
.sort((left, right) =>
`${left.workload}/${left.profile}`.localeCompare(`${right.workload}/${right.profile}`),
),
compiler: [
toCompiler("app", compiler.app),
...compiler.local.map((entry) => toCompiler(`local-${entry.features}`, entry.diagnostics)),
].sort((left, right) => left.fixture.localeCompare(right.fixture)),
});
}
export type Compatibility = Readonly<{
/** Baseline format, digest and metric coverage are usable at all. */
structural: Readonly<{ ok: boolean; reasons: readonly string[] }>;
/** Same compiler: deterministic compiler counts are comparable. */
compiler: Readonly<{ ok: boolean; reasons: readonly string[] }>;
/** Same machine class and runtime: timings are comparable. */
timing: Readonly<{ ok: boolean; reasons: readonly string[] }>;
}>;
export function checkCompatibility(
baseline: ScaleBaseline,
environment: ScaleEnvironment,
current: ScaleMetrics,
): Compatibility {
const structural: string[] = [];
if (baseline.schemaVersion !== SCALE_BASELINE_SCHEMA_VERSION) {
structural.push(`baseline schemaVersion ${String(baseline.schemaVersion)} is not ${SCALE_BASELINE_SCHEMA_VERSION}`);
}
if (metricsDigest(baseline.metrics) !== baseline.approval?.metricsDigest) {
structural.push("baseline metrics do not match their approved digest (hand-edited?)");
}
const timingKey = (entry: TimingEntry) => `${entry.workload}/${entry.profile}/${entry.cardinality}`;
const baselineTiming = new Set(baseline.metrics.timing.map(timingKey));
const baselineCompiler = new Set(baseline.metrics.compiler.map((entry) => entry.fixture));
for (const entry of current.timing) {
if (!baselineTiming.has(timingKey(entry))) structural.push(`no baseline for timing ${timingKey(entry)}`);
}
for (const entry of current.compiler) {
if (!baselineCompiler.has(entry.fixture)) structural.push(`no baseline for compiler fixture ${entry.fixture}`);
}
const compiler: string[] = [];
if (baseline.environment.typescript !== environment.typescript) {
compiler.push(`TypeScript ${environment.typescript} differs from baseline ${baseline.environment.typescript}`);
}
const timing: string[] = [];
const base = baseline.environment;
if (base.nodeMajor !== environment.nodeMajor) timing.push(`Node ${environment.nodeMajor} vs baseline ${base.nodeMajor}`);
if (base.platform !== environment.platform || base.arch !== environment.arch) {
timing.push(`${environment.platform}/${environment.arch} vs baseline ${base.platform}/${base.arch}`);
}
if (base.cpuModel !== environment.cpuModel) timing.push(`CPU "${environment.cpuModel}" vs baseline "${base.cpuModel}"`);
return Object.freeze({
structural: { ok: structural.length === 0, reasons: structural },
compiler: { ok: compiler.length === 0, reasons: compiler },
timing: { ok: timing.length === 0, reasons: timing },
});
}
export type RegressionFinding = Readonly<{
metric: string;
baseline: number;
current: number;
limit: number;
passed: boolean;
}>;
export type AxisResult = Readonly<{
passed: boolean;
findings: readonly RegressionFinding[];
}>;
export type BaselineComparison = Readonly<{
/**
* `passed` only when every axis was evaluated and held. `incomplete` when the
* environment is not timing-compatible: timing regression was not compared,
* so it is not evidence of a pass.
*/
status: "passed" | "failed" | "incomplete";
passed: boolean;
compatibility: Compatibility;
absolute: AxisResult;
growth: AxisResult;
compiler: readonly RegressionFinding[];
/** Empty when the environment is not timing-compatible. */
timing: readonly RegressionFinding[];
failures: readonly string[];
}>;
function finding(metric: string, baseline: number, current: number, limit: number): RegressionFinding {
return { metric, baseline, current, limit: Math.round(limit * 1_000) / 1_000, passed: current <= limit };
}
export function checkAbsoluteCeilings(current: ScaleMetrics): AxisResult {
const findings: RegressionFinding[] = [];
for (const entry of current.timing) {
const limit = ABSOLUTE_CEILINGS.timingMinMsByProfile[entry.profile];
if (limit !== undefined) {
findings.push(finding(`ceiling:time:${entry.workload}/${entry.profile}`, 0, entry.minMs, limit));
}
}
for (const entry of current.compiler) {
const local = /^local-(\d+)$/u.exec(entry.fixture);
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
const limit = local
? ABSOLUTE_CEILINGS.localBase[metric] +
ABSOLUTE_CEILINGS.localPerFeature[metric] * Number(local[1])
: entry.fixture === "app"
? ABSOLUTE_CEILINGS.app[metric]
: Number.POSITIVE_INFINITY;
findings.push(finding(`ceiling:tsc:${entry.fixture}:${metric}`, 0, entry[metric], limit));
}
}
return { passed: findings.every((item) => item.passed), findings };
}
export function checkGrowthShape(current: ScaleMetrics): AxisResult {
const findings: RegressionFinding[] = [];
const byWorkload = new Map<string, { n: number; cost: number }[]>();
for (const entry of current.timing) {
const series = byWorkload.get(entry.workload) ?? [];
series.push({ n: entry.cardinality, cost: entry.minMs });
byWorkload.set(entry.workload, series);
}
for (const [workload, series] of byWorkload) {
if (series.length < 2) continue;
const worst = Math.max(...growthExponents(series, GROWTH_LIMITS.timingNoiseFloorMs));
findings.push(finding(`growth:time:${workload}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.timingMaxExponent));
}
const local = current.compiler
.map((entry) => ({ entry, n: Number(/^local-(\d+)$/u.exec(entry.fixture)?.[1]) }))
.filter(({ n }) => Number.isFinite(n));
if (local.length >= 2) {
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
const worst = Math.max(...growthExponents(local.map(({ entry, n }) => ({ n, cost: entry[metric] }))));
findings.push(finding(`growth:tsc:local:${metric}`, 1, Math.round(worst * 1_000) / 1_000, GROWTH_LIMITS.compilerMaxExponent));
}
}
return { passed: findings.every((item) => item.passed), findings };
}
export function compareToBaseline(
baseline: ScaleBaseline,
environment: ScaleEnvironment,
current: ScaleMetrics,
tolerance = REGRESSION_TOLERANCE,
): BaselineComparison {
const compatibility = checkCompatibility(baseline, environment, current);
const failures: string[] = [...compatibility.structural.reasons];
// A different compiler changes every deterministic count: the baseline must
// be re-approved, it is never silently compared or silently skipped.
if (!compatibility.compiler.ok) {
failures.push(
...compatibility.compiler.reasons.map((reason) => `${reason}; re-approve with update:scale-baseline`),
);
}
const absolute = checkAbsoluteCeilings(current);
const growth = checkGrowthShape(current);
for (const item of [...absolute.findings, ...growth.findings]) {
if (!item.passed) failures.push(`${item.metric} ${item.current} exceeds ${item.limit}`);
}
const compiler: RegressionFinding[] = [];
if (compatibility.structural.ok && compatibility.compiler.ok) {
const byFixture = new Map(baseline.metrics.compiler.map((entry) => [entry.fixture, entry]));
for (const entry of current.compiler) {
const reference = byFixture.get(entry.fixture);
if (!reference) continue;
for (const metric of ["files", "types", "symbols", "instantiations"] as const) {
const item = finding(
`tsc:${entry.fixture}:${metric}`,
reference[metric],
entry[metric],
Math.floor(reference[metric] * (1 + tolerance.compilerRelative)),
);
compiler.push(item);
if (!item.passed) {
failures.push(`${item.metric} ${item.current} exceeds ${item.limit} (baseline ${item.baseline})`);
}
}
}
}
const timing: RegressionFinding[] = [];
if (compatibility.structural.ok && compatibility.timing.ok) {
const key = (entry: TimingEntry) => `${entry.workload}/${entry.profile}`;
const byKey = new Map(baseline.metrics.timing.map((entry) => [key(entry), entry]));
for (const entry of current.timing) {
const reference = byKey.get(key(entry));
if (!reference) continue;
// Best-observed samples: contention only ever adds time.
const item = finding(
`time:${key(entry)}`,
reference.minMs,
entry.minMs,
Math.max(reference.minMs * (1 + tolerance.timingRelative), reference.minMs + tolerance.timingNoiseFloorMs),
);
timing.push(item);
if (!item.passed) {
failures.push(`${item.metric} ${item.current}ms exceeds ${item.limit}ms (baseline ${item.baseline}ms)`);
}
}
}
const status =
failures.length > 0 ? "failed" : compatibility.timing.ok ? "passed" : "incomplete";
return Object.freeze({
status,
passed: status === "passed",
compatibility,
absolute,
growth,
compiler,
timing,
failures,
});
}
export function approveBaseline(
environment: ScaleEnvironment,
metrics: ScaleMetrics,
approval: Readonly<{ owner: string; reason: string; approvedAt: string }>,
): ScaleBaseline {
return Object.freeze({
schemaVersion: SCALE_BASELINE_SCHEMA_VERSION,
environment,
metrics,
approval: { ...approval, metricsDigest: metricsDigest(metrics) },
});
}