No CI job ran any scale tier, and every tier past pr compares timings only on the baseline CPU. The pr tier checks counts, bounds and growth shape; it now includes S2 composition, since S0/S1 stay under the noise floor and let a quadratic accumulation through. The ungated-test check reads tier paths. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
196 lines
7.0 KiB
TypeScript
196 lines
7.0 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
/**
|
|
* Spec §47. Execution tiers for the scale/chaos/soak program.
|
|
*
|
|
* pr architecture and traffic invariants, S0-S2 composition growth
|
|
* main pr + TypeScript diagnostics and the stored baseline regression
|
|
* gate (config/scale/benchmark-baseline.json)
|
|
* nightly main + S3/S4 topology, telemetry storm, session cycles, browser soak
|
|
* release release-coherence matrix + the stable main/nightly subset
|
|
*
|
|
* Only `pr` runs in CI (merge gate FE-GATE-005, 11-03). Its checks are counts,
|
|
* bounds and growth shape, so they hold on any CPU; the composition timings it
|
|
* writes are advisory. `main`, `nightly` and `release` include
|
|
* `baseline-regression`, which compares timings only on the CPU model that
|
|
* recorded config/scale/benchmark-baseline.json and is `incomplete` elsewhere,
|
|
* so no CI job runs them; they are local commands for that machine.
|
|
*
|
|
* Every step's output goes to artifacts/scale/<tier>/<step>.log and the tier
|
|
* writes one machine-readable summary.json. A step that could not run (e.g. a
|
|
* browser soak without a build or browser) is `skipped`, and a tier with any
|
|
* skipped step is `incomplete`: it is never reported as passed and exits
|
|
* non-zero. `SCALE_ALLOW_INCOMPLETE=1` keeps the exit code zero for local runs,
|
|
* but the summary still says `incomplete`.
|
|
*/
|
|
export type ScaleTier = "pr" | "main" | "nightly" | "release";
|
|
|
|
type Step = Readonly<{
|
|
id: string;
|
|
command: readonly string[];
|
|
/** A step that may legitimately report `skipped` in its own artifact. */
|
|
optional?: boolean;
|
|
}>;
|
|
|
|
const VITEST = ["scripts/run-vitest.ts", "run", "--reporter=default"];
|
|
|
|
const PR_STEPS: readonly Step[] = [
|
|
{
|
|
id: "architecture-topology",
|
|
command: [...VITEST, "tests/architecture", "tests/platform/cardinality", "tests/platform/routing"],
|
|
},
|
|
{
|
|
id: "traffic-budgets",
|
|
command: [...VITEST, "tests/platform/execution", "tests/platform/traffic", "tests/platform/resilience"],
|
|
},
|
|
{
|
|
id: "scale-units",
|
|
command: [
|
|
...VITEST,
|
|
"tests/scale/growth-regression.test.ts",
|
|
"tests/scale/benchmark.test.ts",
|
|
"tests/scale/synthetic-topology.test.ts",
|
|
"tests/scale/scale-baseline.test.ts",
|
|
"tests/scale/traffic",
|
|
],
|
|
},
|
|
// 11-03: S2 is included because below it every workload stays under the
|
|
// noise floor, so a quadratic accumulation passed S0/S1 unnoticed.
|
|
{
|
|
id: "composition-s0-s2",
|
|
command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1,S2"],
|
|
},
|
|
];
|
|
|
|
const MAIN_STEPS: readonly Step[] = [
|
|
...PR_STEPS,
|
|
{
|
|
id: "composition-scale-suite",
|
|
command: [...VITEST, "tests/scale/composition-scale.test.ts", "tests/scale/tsc-diagnostics-parser.test.ts"],
|
|
},
|
|
{
|
|
id: "typescript-diagnostics",
|
|
command: ["scripts/scale/collect-tsc-diagnostics.ts", "--local-sizes=25,100,400"],
|
|
},
|
|
{
|
|
id: "baseline-regression",
|
|
command: ["scripts/scale/check-scale-baseline.ts"],
|
|
},
|
|
];
|
|
|
|
const NIGHTLY_STEPS: readonly Step[] = [
|
|
...MAIN_STEPS,
|
|
{
|
|
id: "topology-s3-s4",
|
|
command: [...VITEST, "tests/scale/global-local-independence.test.ts", "tests/platform/cardinality/extreme-scale-registry.test.ts"],
|
|
},
|
|
{
|
|
id: "telemetry-storm",
|
|
command: [...VITEST, "tests/scale/telemetry", "tests/platform/telemetry"],
|
|
},
|
|
{ id: "session-cycles", command: [...VITEST, "tests/scale/session"] },
|
|
{
|
|
id: "browser-soak",
|
|
command: ["scripts/scale/browser-soak.ts", "--duration-ms=120000", "--interval-ms=5000", "--warmup-ms=30000"],
|
|
optional: true,
|
|
},
|
|
];
|
|
|
|
const RELEASE_STEPS: readonly Step[] = [
|
|
{
|
|
id: "release-coherence",
|
|
command: [...VITEST, "tests/platform/release", "tests/release"],
|
|
},
|
|
...MAIN_STEPS,
|
|
{
|
|
id: "topology-s3-s4",
|
|
command: [...VITEST, "tests/scale/global-local-independence.test.ts"],
|
|
},
|
|
{
|
|
id: "telemetry-storm",
|
|
command: [...VITEST, "tests/scale/telemetry"],
|
|
},
|
|
];
|
|
|
|
export const SCALE_TIERS: Readonly<Record<ScaleTier, readonly Step[]>> = Object.freeze({
|
|
pr: PR_STEPS,
|
|
main: MAIN_STEPS,
|
|
nightly: NIGHTLY_STEPS,
|
|
release: RELEASE_STEPS,
|
|
});
|
|
|
|
function withArtifact(step: Step, directory: string): readonly string[] {
|
|
const script = step.command[0] ?? "";
|
|
if (
|
|
script.endsWith("run-composition-scale.ts") ||
|
|
script.endsWith("collect-tsc-diagnostics.ts") ||
|
|
script.endsWith("browser-soak.ts") ||
|
|
script.endsWith("check-scale-baseline.ts")
|
|
) {
|
|
return [...step.command, `--artifact=${path.join(directory, `${step.id}.json`)}`];
|
|
}
|
|
return step.command;
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const tier = process.argv[2] as ScaleTier | undefined;
|
|
if (!tier || !(tier in SCALE_TIERS)) {
|
|
process.stderr.write(`usage: run-scale-profile.ts <${Object.keys(SCALE_TIERS).join("|")}>\n`);
|
|
process.exit(2);
|
|
}
|
|
const directory = path.resolve("artifacts/scale", tier);
|
|
await mkdir(directory, { recursive: true });
|
|
const results: { id: string; status: "passed" | "failed" | "skipped"; durationMs: number; log: string }[] = [];
|
|
for (const step of SCALE_TIERS[tier]) {
|
|
const startedAt = Date.now();
|
|
const run = spawnSync(process.execPath, [...withArtifact(step, directory)], {
|
|
encoding: "utf8",
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
env: { ...process.env, SCALE_ARTIFACT_DIR: path.join(directory, "metrics") },
|
|
});
|
|
const log = path.join(directory, `${step.id}.log`);
|
|
await writeFile(log, `${run.stdout ?? ""}${run.stderr ?? ""}`);
|
|
const skipped = step.optional === true && /: skipped/u.test(run.stdout ?? "");
|
|
const status = run.status === 0 ? (skipped ? "skipped" : "passed") : "failed";
|
|
results.push({ id: step.id, status, durationMs: Date.now() - startedAt, log: path.relative(process.cwd(), log) });
|
|
process.stdout.write(`${status.toUpperCase()} ${tier}:${step.id} (${Date.now() - startedAt}ms)\n`);
|
|
}
|
|
const failed = results.some((result) => result.status === "failed");
|
|
const skipped = results.filter((result) => result.status === "skipped").map((result) => result.id);
|
|
const status = failed ? "failed" : skipped.length > 0 ? "incomplete" : "passed";
|
|
await writeFile(
|
|
path.join(directory, "summary.json"),
|
|
`${JSON.stringify(
|
|
{
|
|
schemaVersion: 1,
|
|
tier,
|
|
generatedAt: new Date().toISOString(),
|
|
node: process.version,
|
|
environment: {
|
|
node: process.version,
|
|
nodeMajor: Number(process.versions.node.split(".")[0]),
|
|
platform: process.platform,
|
|
arch: process.arch,
|
|
},
|
|
status,
|
|
passed: status === "passed",
|
|
skipped,
|
|
steps: results,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
process.stdout.write(
|
|
`scale tier ${tier}: ${status.toUpperCase()}${skipped.length > 0 ? ` (skipped: ${skipped.join(", ")})` : ""} (artifacts/scale/${tier}/summary.json)\n`,
|
|
);
|
|
if (status === "failed") process.exitCode = 1;
|
|
else if (status === "incomplete" && process.env["SCALE_ALLOW_INCOMPLETE"] !== "1") process.exitCode = 1;
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
await main();
|
|
}
|