114 lines
4.4 KiB
TypeScript
114 lines
4.4 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
import { collectTscScale } from "./collect-tsc-diagnostics.ts";
|
|
import { runCompositionScale } from "./run-composition-scale.ts";
|
|
import {
|
|
approveBaseline,
|
|
compareToBaseline,
|
|
currentEnvironment,
|
|
snapshotMetrics,
|
|
type ScaleBaseline,
|
|
} from "./scale-baseline.ts";
|
|
|
|
/**
|
|
* Spec §48–§49. Relative regression gate against the stored, approved scale
|
|
* baseline (`config/scale/benchmark-baseline.json`).
|
|
*
|
|
* node scripts/scale/check-scale-baseline.ts compare, fail on regression
|
|
* SCALE_BASELINE_OWNER=... SCALE_BASELINE_REASON=... \
|
|
* node scripts/scale/check-scale-baseline.ts --update re-measure and re-approve
|
|
*
|
|
* The comparison is written to artifacts/scale/baseline-comparison.json (or
|
|
* `--artifact=`). Updating is an explicit, attributed act; the check never
|
|
* rewrites the baseline.
|
|
*/
|
|
export const BASELINE_PATH = "config/scale/benchmark-baseline.json";
|
|
export const LOCAL_TSC_SIZES = Object.freeze([25, 100, 400]);
|
|
|
|
function typescriptVersion(): string {
|
|
const result = spawnSync(path.resolve("node_modules/.bin/tsc"), ["-v"], { encoding: "utf8" });
|
|
const match = /Version\s+(\S+)/u.exec(result.stdout ?? "");
|
|
if (!match?.[1]) throw new Error("cannot determine TypeScript version");
|
|
return match[1];
|
|
}
|
|
|
|
async function measure() {
|
|
const composition = await runCompositionScale(["S0", "S1", "S2"], { samples: 9, warmup: 2 });
|
|
const compiler = await collectTscScale(LOCAL_TSC_SIZES);
|
|
return snapshotMetrics(composition.results, compiler);
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const update = process.argv.includes("--update");
|
|
const artifact = path.resolve(
|
|
process.argv.find((entry) => entry.startsWith("--artifact="))?.slice("--artifact=".length) ??
|
|
"artifacts/scale/baseline-comparison.json",
|
|
);
|
|
const environment = currentEnvironment(typescriptVersion());
|
|
const metrics = await measure();
|
|
|
|
if (update) {
|
|
const owner = process.env["SCALE_BASELINE_OWNER"];
|
|
const reason = process.env["SCALE_BASELINE_REASON"];
|
|
if (!owner || !reason) {
|
|
process.stderr.write("SCALE_BASELINE_OWNER and SCALE_BASELINE_REASON are required.\n");
|
|
process.exit(1);
|
|
}
|
|
const baseline = approveBaseline(environment, metrics, {
|
|
owner,
|
|
reason,
|
|
approvedAt: new Date().toISOString(),
|
|
});
|
|
await mkdir(path.dirname(BASELINE_PATH), { recursive: true });
|
|
await writeFile(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`);
|
|
process.stdout.write(`scale baseline approved: ${baseline.approval.metricsDigest}\n`);
|
|
return;
|
|
}
|
|
|
|
let baseline: ScaleBaseline;
|
|
try {
|
|
baseline = JSON.parse(await readFile(BASELINE_PATH, "utf8")) as ScaleBaseline;
|
|
} catch {
|
|
process.stderr.write(`missing or unreadable ${BASELINE_PATH}; run update:scale-baseline\n`);
|
|
process.exit(1);
|
|
}
|
|
const comparison = compareToBaseline(baseline, environment, metrics);
|
|
await mkdir(path.dirname(artifact), { recursive: true });
|
|
await writeFile(
|
|
artifact,
|
|
`${JSON.stringify(
|
|
{ schemaVersion: 1, generatedAt: new Date().toISOString(), environment, baselineApproval: baseline.approval, ...comparison },
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
const { compatibility } = comparison;
|
|
process.stdout.write(
|
|
`compatibility: structural=${compatibility.structural.ok} compiler=${compatibility.compiler.ok} timing=${compatibility.timing.ok}\n`,
|
|
);
|
|
if (!compatibility.timing.ok) {
|
|
process.stdout.write(
|
|
`timing regression NOT compared (incomplete): ${compatibility.timing.reasons.join("; ")}\n`,
|
|
);
|
|
}
|
|
process.stdout.write(
|
|
`axes: absolute ${comparison.absolute.findings.length} (${comparison.absolute.passed ? "ok" : "FAIL"}), ` +
|
|
`growth ${comparison.growth.findings.length} (${comparison.growth.passed ? "ok" : "FAIL"}), ` +
|
|
`relative ${comparison.compiler.length} compiler + ${comparison.timing.length} timing\n`,
|
|
);
|
|
for (const failure of comparison.failures) process.stdout.write(`FAIL ${failure}\n`);
|
|
process.stdout.write(
|
|
`scale baseline gate: ${comparison.status.toUpperCase()} (${path.relative(process.cwd(), artifact)})\n`,
|
|
);
|
|
if (comparison.status === "failed") process.exitCode = 1;
|
|
else if (comparison.status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
await main();
|
|
}
|