diff --git a/config/ci/contract-baseline.json b/config/ci/contract-baseline.json index b33f3528..8134b889 100644 --- a/config/ci/contract-baseline.json +++ b/config/ci/contract-baseline.json @@ -1,11 +1,11 @@ { "schemaVersion": 1, "owner": "frontend-platform", - "reason": "11-01, 13-03 (D-14): FE-GATE-016 retired; it ran the same FE-RB-005 drill as FE-GATE-025", + "reason": "11-03: FE-GATE-005 runs the CPU-independent scale PR tier (check:scale:pr)", "gates": 27, - "commands": 94, - "commandReferences": 104, + "commands": 95, + "commandReferences": 105, "artifacts": 109, "stages": 5, - "gateShapeSha256": "3ad2407c5929a7ec43dadebc68a3dc69e265793bf1c80c355b6d2fdeae019821" + "gateShapeSha256": "58e5be2bdf92c9ee355fa93ced794b6cc948a960905e80919d95087e56262d94" } diff --git a/config/ci/gates.json b/config/ci/gates.json index 0e41a294..3d75cb03 100644 --- a/config/ci/gates.json +++ b/config/ci/gates.json @@ -215,6 +215,11 @@ "expectedExitCode": 1, "expectedDiagnosticId": "Risk coverage failed:" }, + { + "id": "check-scale-pr", + "script": "check:scale:pr", + "expect": "pass" + }, { "id": "test-component", "script": "test:component", @@ -1791,7 +1796,8 @@ "test-unit", "test-contract", "test-coverage", - "check-coverage-fixture" + "check-coverage-fixture", + "check-scale-pr" ], "logArtifactId": "artifact-artifacts-quality-gates-FE-GATE-005-txt", "evidenceArtifactIds": [ diff --git a/config/ci/ungated-tests.json b/config/ci/ungated-tests.json index 8d34b040..f9612c48 100644 --- a/config/ci/ungated-tests.json +++ b/config/ci/ungated-tests.json @@ -1,57 +1,29 @@ { "schemaVersion": 1, "entries": [ - { - "path": "tests/scale/benchmark.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, { "path": "tests/scale/composition-scale.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." + "reason": "Scale suite in the main or nightly tier (scripts/scale/run-scale-profile.ts). No CI job runs those tiers: both include baseline-regression, which compares timings only on the CPU model that recorded config/scale/benchmark-baseline.json and reports incomplete elsewhere (review 11-03). Run check:scale:main or check:scale:nightly on that machine. The CPU-independent pr tier runs in FE-GATE-005." }, { "path": "tests/scale/global-local-independence.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, - { - "path": "tests/scale/growth-regression.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, - { - "path": "tests/scale/scale-baseline.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." + "reason": "Scale suite in the main or nightly tier (scripts/scale/run-scale-profile.ts). No CI job runs those tiers: both include baseline-regression, which compares timings only on the CPU model that recorded config/scale/benchmark-baseline.json and reports incomplete elsewhere (review 11-03). Run check:scale:main or check:scale:nightly on that machine. The CPU-independent pr tier runs in FE-GATE-005." }, { "path": "tests/scale/session/long-session.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." + "reason": "Scale suite in the main or nightly tier (scripts/scale/run-scale-profile.ts). No CI job runs those tiers: both include baseline-regression, which compares timings only on the CPU model that recorded config/scale/benchmark-baseline.json and reports incomplete elsewhere (review 11-03). Run check:scale:main or check:scale:nightly on that machine. The CPU-independent pr tier runs in FE-GATE-005." }, { "path": "tests/scale/session/network-flap.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, - { - "path": "tests/scale/synthetic-topology.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." + "reason": "Scale suite in the main or nightly tier (scripts/scale/run-scale-profile.ts). No CI job runs those tiers: both include baseline-regression, which compares timings only on the CPU model that recorded config/scale/benchmark-baseline.json and reports incomplete elsewhere (review 11-03). Run check:scale:main or check:scale:nightly on that machine. The CPU-independent pr tier runs in FE-GATE-005." }, { "path": "tests/scale/telemetry/telemetry-storm.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, - { - "path": "tests/scale/traffic/amplification-metrics.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, - { - "path": "tests/scale/traffic/failure-scenarios.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." - }, - { - "path": "tests/scale/traffic/retry-after-jitter.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." + "reason": "Scale suite in the main or nightly tier (scripts/scale/run-scale-profile.ts). No CI job runs those tiers: both include baseline-regression, which compares timings only on the CPU model that recorded config/scale/benchmark-baseline.json and reports incomplete elsewhere (review 11-03). Run check:scale:main or check:scale:nightly on that machine. The CPU-independent pr tier runs in FE-GATE-005." }, { "path": "tests/scale/tsc-diagnostics-parser.test.ts", - "reason": "Scale suite. It runs through check:scale:* (scripts/scale/run-scale-profile.ts) and verify:extreme-scale, which no CI gate calls, because its timing baseline passes only on the CPU that recorded it. Review 11-03 moves it into a PR tier; drop the entry when a gate runs it." + "reason": "Scale suite in the main or nightly tier (scripts/scale/run-scale-profile.ts). No CI job runs those tiers: both include baseline-regression, which compares timings only on the CPU model that recorded config/scale/benchmark-baseline.json and reports incomplete elsewhere (review 11-03). Run check:scale:main or check:scale:nightly on that machine. The CPU-independent pr tier runs in FE-GATE-005." } ] } diff --git a/docs/operations/ci-quality-gates.md b/docs/operations/ci-quality-gates.md index b0f8a1fc..d8c5afa6 100644 --- a/docs/operations/ci-quality-gates.md +++ b/docs/operations/ci-quality-gates.md @@ -35,9 +35,20 @@ canonical `check:ci`를 돌린다(`scripts/lib/removal-fixture.ts`의 `check:ci`는 `tests/` 아래 테스트 파일(`tests/fixtures` 제외)이 모두 CI 게이트 명령에서 실행되는지도 확인한다. 게이트 명령이 닿는 package script의 Vitest 경로와 Playwright 설정의 `testDir`를 모아 비교한다(`scripts/lib/ungated-tests.ts`). +`run-scale-profile.ts `를 부르는 명령은 그 tier의 Vitest 경로로 센다. 게이트가 돌리지 않는 테스트는 `config/ci/ungated-tests.json`에 이유와 함께 적어야 -한다. 지금은 `tests/scale`의 규모 테스트 13개만 있다(리뷰 11-03에서 PR tier로 -옮길 때 뺀다). 게이트가 이미 돌리는 파일을 이 목록에 남겨 두면 실패한다. +한다. 게이트가 이미 돌리는 파일을 이 목록에 남겨 두면 실패한다. + +규모 테스트는 이렇게 나뉜다(리뷰 11-03, `scripts/scale/run-scale-profile.ts`). + +- `check:scale:pr`은 merge gate `FE-GATE-005`에서 돈다. 개수·상한·증가 모양만 + 검사하므로 CPU가 달라도 결과가 같다. S0~S2 조합 증가 검사가 여기 있다. S0/S1만 + 재면 모든 workload가 noise floor 아래라 이차 누적을 놓친다. 조합 timing은 + 산출물(`artifacts/scale/pr/composition-s0-s2.json`)에 기록만 한다. +- `main`·`nightly`·`release` tier는 CI에서 돌지 않는다. 이 tier들은 + `baseline-regression`을 포함한다. 이 단계는 `config/scale/benchmark-baseline.json`을 + 기록한 CPU 모델에서만 timing을 비교하고, 다른 CPU에서는 `incomplete`로 끝난다. + 이 tier에만 있는 테스트 6개는 `ungated-tests.json`에 이유와 함께 남아 있다. 템플릿을 복사해 새 프로젝트를 시작하면 아래 기준선은 템플릿의 값이다. 샘플을 지우거나 구조를 바꾸면 프로젝트 값으로 다시 찍는다. diff --git a/scripts/lib/ungated-tests.ts b/scripts/lib/ungated-tests.ts index b87cd600..cdaeff35 100644 --- a/scripts/lib/ungated-tests.ts +++ b/scripts/lib/ungated-tests.ts @@ -3,12 +3,14 @@ import path from "node:path"; import { z } from "zod"; +import { SCALE_TIERS, type ScaleTier } from "../scale/run-scale-profile.ts"; import { reachablePackageScripts } from "./package-script-graph.ts"; /** * 12-04. A test file protects nothing unless a CI gate runs it. The gated set * is read from the package scripts the CI contract's commands reach: Vitest - * path filters (`run-vitest.ts run `) and Playwright config `testDir`s. + * path filters (`run-vitest.ts run `), the Vitest steps of a scale tier + * (`run-scale-profile.ts `) and Playwright config `testDir`s. * A test outside that set needs a reasoned entry in config/ci/ungated-tests.json. */ export const UNGATED_TESTS_PATH = "config/ci/ungated-tests.json"; @@ -53,17 +55,22 @@ export function gatedTestPaths( readConfig: (file: string) => string, ): string[] { const paths = new Set(); + const addVitestPaths = (tokens: readonly string[]) => { + const vitestRun = tokens.findIndex( + (token, index) => token === "run" && /(?:run-vitest\.ts|vitest)$/u.test(tokens[index - 1] ?? ""), + ); + if (vitestRun < 0) return; + for (const token of tokens.slice(vitestRun + 1)) { + if (!token.startsWith("-")) paths.add(token.replace(/\/$/u, "")); + } + }; for (const script of reachablePackageScripts(scripts, entryScripts)) { for (const segment of scripts[script]!.split("&&")) { const tokens = segment.trim().split(/\s+/u); - const vitestRun = tokens.findIndex( - (token, index) => token === "run" && /(?:run-vitest\.ts|vitest)$/u.test(tokens[index - 1] ?? ""), - ); - if (vitestRun >= 0) { - for (const token of tokens.slice(vitestRun + 1)) { - if (token.startsWith("-")) break; - paths.add(token.replace(/\/$/u, "")); - } + addVitestPaths(tokens); + const scaleProfile = tokens.findIndex((token) => /run-scale-profile\.ts$/u.test(token)); + if (scaleProfile >= 0) { + for (const step of SCALE_TIERS[tokens[scaleProfile + 1] as ScaleTier] ?? []) addVitestPaths(step.command); } const playwrightTest = tokens.findIndex( (token, index) => token === "test" && tokens[index - 1] === "playwright", diff --git a/scripts/scale/run-scale-profile.ts b/scripts/scale/run-scale-profile.ts index c22b6919..ac360ff1 100644 --- a/scripts/scale/run-scale-profile.ts +++ b/scripts/scale/run-scale-profile.ts @@ -3,14 +3,21 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; /** - * Spec §47. CI execution tiers for the scale/chaos/soak program. + * Spec §47. Execution tiers for the scale/chaos/soak program. * - * pr S0/S1 architecture, composition and traffic invariants - * main pr + S2 composition, TypeScript diagnostics and the stored - * baseline regression gate (config/scale/benchmark-baseline.json) + * pr architecture and traffic invariants, S0-S2 composition growth + * main pr + TypeScript diagnostics and the stored baseline regression + * gate (config/scale/benchmark-baseline.json) * nightly main + S3/S4 topology, telemetry storm, session cycles, browser soak * release release-coherence matrix + the stable main/nightly subset * + * Only `pr` runs in CI (merge gate FE-GATE-005, 11-03). Its checks are counts, + * bounds and growth shape, so they hold on any CPU; the composition timings it + * writes are advisory. `main`, `nightly` and `release` include + * `baseline-regression`, which compares timings only on the CPU model that + * recorded config/scale/benchmark-baseline.json and is `incomplete` elsewhere, + * so no CI job runs them; they are local commands for that machine. + * * Every step's output goes to artifacts/scale//.log and the tier * writes one machine-readable summary.json. A step that could not run (e.g. a * browser soak without a build or browser) is `skipped`, and a tier with any @@ -49,18 +56,16 @@ const PR_STEPS: readonly Step[] = [ "tests/scale/traffic", ], }, + // 11-03: S2 is included because below it every workload stays under the + // noise floor, so a quadratic accumulation passed S0/S1 unnoticed. { - id: "composition-s0-s1", - command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1"], + id: "composition-s0-s2", + command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1,S2"], }, ]; const MAIN_STEPS: readonly Step[] = [ ...PR_STEPS, - { - id: "composition-s0-s2", - command: ["scripts/scale/run-composition-scale.ts", "--profiles=S0,S1,S2"], - }, { id: "composition-scale-suite", command: [...VITEST, "tests/scale/composition-scale.test.ts", "tests/scale/tsc-diagnostics-parser.test.ts"], diff --git a/tests/unit/ci-workflow-generation.test.ts b/tests/unit/ci-workflow-generation.test.ts index 763a7269..1a112315 100644 --- a/tests/unit/ci-workflow-generation.test.ts +++ b/tests/unit/ci-workflow-generation.test.ts @@ -194,8 +194,8 @@ describe("CI gate contract", () => { ).filter((gateId) => gateId !== "FE-GATE-016"), ); expect(contract.jobs).toHaveLength(9); - expect(contract.commands).toHaveLength(94); - expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(104); + expect(contract.commands).toHaveLength(95); + expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(105); expect(contract.commands.filter(({ expect }) => expect === "fail")).toHaveLength(31); expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(87); expect(contract.artifacts).toHaveLength(109); @@ -221,6 +221,14 @@ describe("CI gate contract", () => { expect(stageOf("FE-GATE-028")).toBe("release"); }); + it("runs the CPU-independent scale PR tier in a merge gate (11-03)", async () => { + const contract = await loadCiGateContract(process.cwd()); + const index = indexCiGateContract(contract); + const scripts = index.gates.get("FE-GATE-005")?.commandIds.map((id) => index.commands.get(id)?.script); + expect(scripts).toContain("check:scale:pr"); + expect(contract.stages.find(({ id }) => id === "merge")?.gateIds).toContain("FE-GATE-005"); + }); + it("keeps the action registry recursively immutable and resolves only known actions", () => { expect(Object.isFrozen(CI_ACTION_REGISTRY)).toBe(true); expect(Object.values(CI_ACTION_REGISTRY).every((action) => Object.isFrozen(action))).toBe(true); diff --git a/tests/unit/ungated-tests.test.ts b/tests/unit/ungated-tests.test.ts index be2daf62..32a788fe 100644 --- a/tests/unit/ungated-tests.test.ts +++ b/tests/unit/ungated-tests.test.ts @@ -46,12 +46,24 @@ describe("ungated test files", () => { "tests/scripts/verify-extreme-scale.test.ts", "tests/scale/browser-soak.contract.test.ts", "tests/e2e/app-shell.spec.ts", + // 11-03: the CPU-independent scale PR tier runs in FE-GATE-005. + "tests/scale/growth-regression.test.ts", + "tests/scale/traffic/amplification-metrics.test.ts", ])("gates %s", async (file) => { const inputs = await repositoryInputs(); expect(ungatedTestFailures({ ...inputs, testFiles: [file] })).toEqual([]); expect(inputs.exclusions.some(({ path }) => path === file)).toBe(false); }); + it("reads the Vitest paths of a scale tier the gate command runs (11-03)", () => { + expect( + gatedTestPaths({ gate: "node scripts/scale/run-scale-profile.ts pr" }, ["gate"], () => ""), + ).toEqual(expect.arrayContaining(["tests/scale/growth-regression.test.ts", "tests/scale/traffic"])); + expect( + gatedTestPaths({ gate: "node scripts/scale/run-scale-profile.ts pr" }, ["gate"], () => ""), + ).not.toContain("tests/scale/session"); + }); + it("rejects a new test directory that no gate runs", async () => { const inputs = await repositoryInputs(); expect(