Files

174 lines
7.0 KiB
TypeScript

import { readFile } from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
approveBaseline,
compareToBaseline,
metricsDigest,
type ScaleBaseline,
type ScaleEnvironment,
type ScaleMetrics,
} from "../../scripts/scale/scale-baseline.ts";
import { COMPOSITION_WORKLOADS } from "../../scripts/scale/composition-workloads.ts";
import { BASELINE_PATH, LOCAL_TSC_SIZES } from "../../scripts/scale/check-scale-baseline.ts";
const ENVIRONMENT: ScaleEnvironment = {
nodeMajor: 24,
platform: "linux",
arch: "x64",
cpuModel: "Test CPU",
cpuCount: 8,
typescript: "7.0.2",
};
const METRICS: ScaleMetrics = {
timing: [
{ workload: "registry-composition", profile: "S1", cardinality: 1_000, medianMs: 0.5, minMs: 0.4 },
{ workload: "registry-composition", profile: "S2", cardinality: 10_000, medianMs: 5, minMs: 4 },
],
compiler: [{ fixture: "app", files: 800, types: 270_000, symbols: 770_000, instantiations: 690_000 }],
};
function baseline(metrics: ScaleMetrics = METRICS, environment = ENVIRONMENT): ScaleBaseline {
return approveBaseline(environment, metrics, {
owner: "frontend-platform",
reason: "test",
approvedAt: "2026-09-23T00:00:00.000Z",
});
}
function withTiming(minMs: number): ScaleMetrics {
return {
...METRICS,
timing: METRICS.timing.map((entry) =>
entry.profile === "S2" ? { ...entry, minMs, medianMs: minMs } : entry,
),
};
}
function withTypes(types: number): ScaleMetrics {
return { ...METRICS, compiler: [{ ...METRICS.compiler[0]!, types }] };
}
describe("relative regression gate against the stored baseline", () => {
it("passes an unchanged run on all three axes", () => {
const result = compareToBaseline(baseline(), ENVIRONMENT, METRICS);
expect(result.status).toBe("passed");
expect(result.passed).toBe(true);
expect(result.absolute.passed).toBe(true);
expect(result.growth.passed).toBe(true);
expect(result.compiler.length).toBe(4);
expect(result.timing.length).toBe(2);
});
it("fails a compiler-cost regression above 10% even when the growth shape is fine", () => {
expect(compareToBaseline(baseline(), ENVIRONMENT, withTypes(290_000)).passed).toBe(true);
const regressed = compareToBaseline(baseline(), ENVIRONMENT, withTypes(300_000));
expect(regressed.passed).toBe(false);
expect(regressed.failures.join("\n")).toMatch(/tsc:app:types 300000 exceeds 297000/);
});
it("fails a timing regression beyond 2.5x on a compatible machine", () => {
expect(compareToBaseline(baseline(), ENVIRONMENT, withTiming(9.9)).passed).toBe(true);
const regressed = compareToBaseline(baseline(), ENVIRONMENT, withTiming(12));
expect(regressed.passed).toBe(false);
expect(regressed.failures.join("\n")).toMatch(/time:registry-composition\/S2/);
});
it("never fails timing differences inside the absolute noise floor", () => {
const tiny: ScaleMetrics = {
...METRICS,
timing: [{ workload: "w", profile: "S0", cardinality: 100, medianMs: 0.02, minMs: 0.02 }],
};
const noisy: ScaleMetrics = { ...tiny, timing: [{ ...tiny.timing[0]!, minMs: 0.9 }] };
expect(compareToBaseline(baseline(tiny), ENVIRONMENT, noisy).passed).toBe(true);
});
it("reports timings from a different machine as incomplete, never as passed, but still gates compiler counts", () => {
const otherMachine = { ...ENVIRONMENT, cpuModel: "Other CPU" };
const slow = compareToBaseline(baseline(), otherMachine, withTiming(12));
expect(slow.status).toBe("incomplete");
expect(slow.passed).toBe(false);
expect(slow.compatibility.timing.ok).toBe(false);
expect(slow.timing).toEqual([]);
const heavier = compareToBaseline(baseline(), otherMachine, withTypes(400_000));
expect(heavier.status).toBe("failed");
});
it("enforces machine-agnostic absolute ceilings on every run", () => {
const tooSlow = compareToBaseline(baseline(), { ...ENVIRONMENT, cpuModel: "Other CPU" }, withTiming(151));
expect(tooSlow.status).toBe("failed");
expect(tooSlow.failures.join("\n")).toMatch(/ceiling:time:registry-composition\/S2 151 exceeds 150/);
const tooHeavy = compareToBaseline(baseline(), ENVIRONMENT, withTypes(600_001));
expect(tooHeavy.absolute.passed).toBe(false);
});
it("fails a super-linear growth shape even when relative and absolute limits hold", () => {
const quadratic: ScaleMetrics = {
...METRICS,
timing: [
{ workload: "registry-composition", profile: "S1", cardinality: 1_000, medianMs: 0.6, minMs: 0.6 },
{ workload: "registry-composition", profile: "S2", cardinality: 10_000, medianMs: 60, minMs: 60 },
],
};
const result = compareToBaseline(baseline(quadratic), ENVIRONMENT, quadratic);
expect(result.growth.passed).toBe(false);
expect(result.status).toBe("failed");
expect(result.failures.join("\n")).toMatch(/growth:time:registry-composition/);
});
it("requires re-approval when the compiler changes", () => {
const result = compareToBaseline(baseline(), { ...ENVIRONMENT, typescript: "7.1.0" }, METRICS);
expect(result.passed).toBe(false);
expect(result.compatibility.compiler.ok).toBe(false);
expect(result.failures.join("\n")).toMatch(/re-approve with update:scale-baseline/);
});
it("rejects a hand-edited baseline whose metrics no longer match the approved digest", () => {
const tampered: ScaleBaseline = {
...baseline(),
metrics: withTypes(900_000),
};
const result = compareToBaseline(tampered, ENVIRONMENT, METRICS);
expect(result.passed).toBe(false);
expect(result.compatibility.structural.reasons.join("\n")).toMatch(/digest/);
});
it("rejects a run measuring something the baseline does not cover", () => {
const extra: ScaleMetrics = {
...METRICS,
compiler: [...METRICS.compiler, { fixture: "local-9999", files: 1, types: 1, symbols: 1, instantiations: 1 }],
};
const result = compareToBaseline(baseline(), ENVIRONMENT, extra);
expect(result.passed).toBe(false);
expect(result.failures.join("\n")).toMatch(/no baseline for compiler fixture local-9999/);
});
});
describe("stored benchmark baseline", () => {
it("is approved, digest-consistent and covers every measured workload and fixture", async () => {
const stored = JSON.parse(
await readFile(path.resolve(BASELINE_PATH), "utf8"),
) as ScaleBaseline;
expect(stored.schemaVersion).toBe(1);
expect(stored.approval.owner).toBeTruthy();
expect(stored.approval.reason).toBeTruthy();
expect(metricsDigest(stored.metrics)).toBe(stored.approval.metricsDigest);
const timing = new Set(stored.metrics.timing.map((entry) => `${entry.workload}/${entry.profile}`));
for (const workload of COMPOSITION_WORKLOADS) {
for (const profile of ["S0", "S1", "S2"]) {
expect(timing.has(`${workload.name}/${profile}`), `${workload.name}/${profile}`).toBe(true);
}
}
const fixtures = new Set(stored.metrics.compiler.map((entry) => entry.fixture));
expect(fixtures).toEqual(
new Set(["app", ...LOCAL_TSC_SIZES.map((size) => `local-${size}`)]),
);
});
});