import { readFile } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { approveBaseline, compareToBaseline, metricsDigest, type ScaleBaseline, type ScaleEnvironment, type ScaleMetrics, } from "../../scripts/scale/scale-baseline.ts"; import { COMPOSITION_WORKLOADS } from "../../scripts/scale/composition-workloads.ts"; import { BASELINE_PATH, LOCAL_TSC_SIZES } from "../../scripts/scale/check-scale-baseline.ts"; const ENVIRONMENT: ScaleEnvironment = { nodeMajor: 24, platform: "linux", arch: "x64", cpuModel: "Test CPU", cpuCount: 8, typescript: "7.0.2", }; const METRICS: ScaleMetrics = { timing: [ { workload: "registry-composition", profile: "S1", cardinality: 1_000, medianMs: 0.5, minMs: 0.4 }, { workload: "registry-composition", profile: "S2", cardinality: 10_000, medianMs: 5, minMs: 4 }, ], compiler: [{ fixture: "app", files: 800, types: 270_000, symbols: 770_000, instantiations: 690_000 }], }; function baseline(metrics: ScaleMetrics = METRICS, environment = ENVIRONMENT): ScaleBaseline { return approveBaseline(environment, metrics, { owner: "frontend-platform", reason: "test", approvedAt: "2026-09-23T00:00:00.000Z", }); } function withTiming(minMs: number): ScaleMetrics { return { ...METRICS, timing: METRICS.timing.map((entry) => entry.profile === "S2" ? { ...entry, minMs, medianMs: minMs } : entry, ), }; } function withTypes(types: number): ScaleMetrics { return { ...METRICS, compiler: [{ ...METRICS.compiler[0]!, types }] }; } describe("relative regression gate against the stored baseline", () => { it("passes an unchanged run on all three axes", () => { const result = compareToBaseline(baseline(), ENVIRONMENT, METRICS); expect(result.status).toBe("passed"); expect(result.passed).toBe(true); expect(result.absolute.passed).toBe(true); expect(result.growth.passed).toBe(true); expect(result.compiler.length).toBe(4); expect(result.timing.length).toBe(2); }); it("fails a compiler-cost regression above 10% even when the growth shape is fine", () => { expect(compareToBaseline(baseline(), ENVIRONMENT, withTypes(290_000)).passed).toBe(true); const regressed = compareToBaseline(baseline(), ENVIRONMENT, withTypes(300_000)); expect(regressed.passed).toBe(false); expect(regressed.failures.join("\n")).toMatch(/tsc:app:types 300000 exceeds 297000/); }); it("fails a timing regression beyond 2.5x on a compatible machine", () => { expect(compareToBaseline(baseline(), ENVIRONMENT, withTiming(9.9)).passed).toBe(true); const regressed = compareToBaseline(baseline(), ENVIRONMENT, withTiming(12)); expect(regressed.passed).toBe(false); expect(regressed.failures.join("\n")).toMatch(/time:registry-composition\/S2/); }); it("never fails timing differences inside the absolute noise floor", () => { const tiny: ScaleMetrics = { ...METRICS, timing: [{ workload: "w", profile: "S0", cardinality: 100, medianMs: 0.02, minMs: 0.02 }], }; const noisy: ScaleMetrics = { ...tiny, timing: [{ ...tiny.timing[0]!, minMs: 0.9 }] }; expect(compareToBaseline(baseline(tiny), ENVIRONMENT, noisy).passed).toBe(true); }); it("reports timings from a different machine as incomplete, never as passed, but still gates compiler counts", () => { const otherMachine = { ...ENVIRONMENT, cpuModel: "Other CPU" }; const slow = compareToBaseline(baseline(), otherMachine, withTiming(12)); expect(slow.status).toBe("incomplete"); expect(slow.passed).toBe(false); expect(slow.compatibility.timing.ok).toBe(false); expect(slow.timing).toEqual([]); const heavier = compareToBaseline(baseline(), otherMachine, withTypes(400_000)); expect(heavier.status).toBe("failed"); }); it("enforces machine-agnostic absolute ceilings on every run", () => { const tooSlow = compareToBaseline(baseline(), { ...ENVIRONMENT, cpuModel: "Other CPU" }, withTiming(151)); expect(tooSlow.status).toBe("failed"); expect(tooSlow.failures.join("\n")).toMatch(/ceiling:time:registry-composition\/S2 151 exceeds 150/); const tooHeavy = compareToBaseline(baseline(), ENVIRONMENT, withTypes(600_001)); expect(tooHeavy.absolute.passed).toBe(false); }); it("fails a super-linear growth shape even when relative and absolute limits hold", () => { const quadratic: ScaleMetrics = { ...METRICS, timing: [ { workload: "registry-composition", profile: "S1", cardinality: 1_000, medianMs: 0.6, minMs: 0.6 }, { workload: "registry-composition", profile: "S2", cardinality: 10_000, medianMs: 60, minMs: 60 }, ], }; const result = compareToBaseline(baseline(quadratic), ENVIRONMENT, quadratic); expect(result.growth.passed).toBe(false); expect(result.status).toBe("failed"); expect(result.failures.join("\n")).toMatch(/growth:time:registry-composition/); }); it("requires re-approval when the compiler changes", () => { const result = compareToBaseline(baseline(), { ...ENVIRONMENT, typescript: "7.1.0" }, METRICS); expect(result.passed).toBe(false); expect(result.compatibility.compiler.ok).toBe(false); expect(result.failures.join("\n")).toMatch(/re-approve with update:scale-baseline/); }); it("rejects a hand-edited baseline whose metrics no longer match the approved digest", () => { const tampered: ScaleBaseline = { ...baseline(), metrics: withTypes(900_000), }; const result = compareToBaseline(tampered, ENVIRONMENT, METRICS); expect(result.passed).toBe(false); expect(result.compatibility.structural.reasons.join("\n")).toMatch(/digest/); }); it("rejects a run measuring something the baseline does not cover", () => { const extra: ScaleMetrics = { ...METRICS, compiler: [...METRICS.compiler, { fixture: "local-9999", files: 1, types: 1, symbols: 1, instantiations: 1 }], }; const result = compareToBaseline(baseline(), ENVIRONMENT, extra); expect(result.passed).toBe(false); expect(result.failures.join("\n")).toMatch(/no baseline for compiler fixture local-9999/); }); }); describe("stored benchmark baseline", () => { it("is approved, digest-consistent and covers every measured workload and fixture", async () => { const stored = JSON.parse( await readFile(path.resolve(BASELINE_PATH), "utf8"), ) as ScaleBaseline; expect(stored.schemaVersion).toBe(1); expect(stored.approval.owner).toBeTruthy(); expect(stored.approval.reason).toBeTruthy(); expect(metricsDigest(stored.metrics)).toBe(stored.approval.metricsDigest); const timing = new Set(stored.metrics.timing.map((entry) => `${entry.workload}/${entry.profile}`)); for (const workload of COMPOSITION_WORKLOADS) { for (const profile of ["S0", "S1", "S2"]) { expect(timing.has(`${workload.name}/${profile}`), `${workload.name}/${profile}`).toBe(true); } } const fixtures = new Set(stored.metrics.compiler.map((entry) => entry.fixture)); expect(fixtures).toEqual( new Set(["app", ...LOCAL_TSC_SIZES.map((size) => `local-${size}`)]), ); }); });