Files
clean-architecture-frontend…/tests/system/ci-runner/ci-evidence-freshness.test.ts
T
donghyeon.kang f6b2b96c53 ci(gates): 검사기 자기 시험과 gating 검사를 main 단계로 내리고 계약 기준선을 지운다
merge 단계(=PR)가 변경이 아니라 검사기 자신을 시험하는 명령까지 돈다.
검사기가 나쁜 입력을 거부하는지 보는 `*:fixture` 명령 15개, 모든 시험
파일과 검증 script가 게이트에 걸렸는지 보는 ungated 검사, CI 계약의
개수·해시 기준선(`config/ci/contract-baseline.json`)이다. 기준선은 게이트
명령 하나를 더하거나 빼도 OWNER·REASON을 붙여 다시 받아야 했다.
소유자 결정(2026-09-16)은 개수·배선 검사를 main 단계에 두는 것이다.

고른 방법(design-docs repo-structure/03-ci.md 7.3절 F2, README.md 3.3절,
소유자 승인 2026-11-07):
- 새 게이트 FE-GATE-029 `checker-self-tests`, 새 단계 `main`
  (`MAIN_READY`, 승격을 막지 않는다), 새 job `main_gate`. 브랜치 push와
  `workflow_dispatch`(stage=main)에서만 돈다. PR과 태그에서는 돌지 않는다.
- FE-GATE-001·005·008·010·013의 `*:fixture` 명령 15개와 그 증거
  산출물 10개를 FE-GATE-029로 옮긴다.
- `check:ci`의 ungated 검사를 `scripts/check-ci-gating.ts`
  (`check:ci:gating`)로 떼어 FE-GATE-029에 둔다. `check:ci`는 계약·
  workflow 검사만 PR에서 돈다.
- 기준선 파일, `update:ci-contract-baseline`, 그 검사를 지운다(소유자
  결정). 계약 모양은 ci-gates.ts의 의미 검사가 지킨다. 개수를 고정하던
  단위 시험의 개수 단언도 뺐다.

남긴 것: 음성 타입 fixture(`check:types:fixtures`, FE-GATE-003)는
이름이 `*:fixture`여도 merge에 둔다. 검사기가 아니라 src의 포트·계약
타입을 시험한다. `TelemetryPort.emit`을 넓히는 변경을 PR에서 막는다(정리
3단계 9.3절 변이로 확인). merge 단계 명령 참조 64 → 49.

위험: 검사기를 망가뜨린 PR은 merge를 통과하고 main에서 빨개진다. 그
대가로 PR이 검사기 자기 시험을 기다리지 않는다(설계 7.5절의 같은 교환).
FE-GATE-029의 명령이 앞 게이트 산출물 없이 새 checkout에서 도는지는
desktop에서 따로 확인했다(PR 본문).

검증(desktop): check:ci, check:ci:gating, check:types, CI 시험 통과.
merge 단계·FE-GATE-029 실행 결과는 PR 본문에 적는다.

Refs: design-docs repo-structure/03-ci.md 7.3·7.4절, README.md 3.3절
2026-11-08 21:56:37 +09:00

234 lines
9.0 KiB
TypeScript

import { spawnSync } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { loadCiGateContract } from "../../../scripts/contracts/ci-gates.ts";
import { DEPENDENCY_ADVISORIES_RECORD_PATH } from "../../../scripts/lib/dependency-advisories.ts";
import { FIXTURE_PNPM_ENVIRONMENT } from "../../../scripts/lib/fixture-node-modules.ts";
// F-12. The temp roots have a package.json but no node_modules or lockfile.
// Run directly (not under pnpm), pnpm's pre-script install check started an
// install there and failed with ERR_PNPM_IGNORED_BUILDS; the gate runner has
// to run the scripts as they are.
const temporaryRoots: string[] = [];
afterEach(async () => {
await Promise.all(
temporaryRoots.splice(0).map((root) =>
rm(root, { recursive: true, force: true }),
),
);
});
describe("CI-runner command-generated evidence freshness", () => {
it("rejects stale command-generated evidence from a successful no-op producer", async () => {
const root = await mkdtemp(path.join(tmpdir(), "ci-gate-stale-evidence-"));
temporaryRoots.push(root);
await mkdir(path.join(root, "config/ci"), { recursive: true });
await mkdir(path.join(root, "artifacts/tests"), { recursive: true });
const contract = JSON.parse(
JSON.stringify(await loadCiGateContract(process.cwd())),
) as Record<string, any>;
const command = contract.commands.find(
(entry: Record<string, any>) => entry.id === "test-runtime-schema",
);
command.script = "test:stale-evidence-noop";
const evidence = contract.artifacts.find(
(entry: Record<string, any>) =>
entry.path === "artifacts/tests/runtime-schema.xml",
);
const packageDocument = JSON.parse(
await readFile("package.json", "utf8"),
) as { scripts: Record<string, string> };
packageDocument.scripts[command.script] = "true";
await writeFile(
path.join(root, "config/ci/gates.json"),
`${JSON.stringify(contract)}\n`,
);
await writeFile(
path.join(root, "package.json"),
`${JSON.stringify(packageDocument)}\n`,
);
await writeFile(
path.join(root, evidence.path),
'<testsuite name="stale" tests="0" failures="0"/>\n',
);
const result = spawnSync(
process.execPath,
[path.resolve("scripts/run-ci-gate.ts"), "FE-GATE-004"],
{ cwd: root, encoding: "utf8", env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, CI: "false" } },
);
expect(result.status).toBe(1);
expect(
await readFile(
path.join(root, "artifacts/quality/gates/FE-GATE-004.txt"),
"utf8",
),
).toMatch(/not freshly produced/i);
});
it("accepts a fresh deterministic rewrite with identical evidence bytes", async () => {
const root = await mkdtemp(
path.join(tmpdir(), "ci-gate-identical-rewrite-"),
);
temporaryRoots.push(root);
await mkdir(path.join(root, "config/ci"), { recursive: true });
await mkdir(path.join(root, "artifacts/tests"), { recursive: true });
const contract = JSON.parse(
JSON.stringify(await loadCiGateContract(process.cwd())),
) as Record<string, any>;
const command = contract.commands.find(
(entry: Record<string, any>) => entry.id === "test-runtime-schema",
);
command.script = "test:identical-evidence-rewrite";
const evidence = contract.artifacts.find(
(entry: Record<string, any>) =>
entry.path === "artifacts/tests/runtime-schema.xml",
);
const evidenceBytes =
'<testsuite name="deterministic" tests="0" failures="0"/>\n';
const packageDocument = JSON.parse(
await readFile("package.json", "utf8"),
) as { scripts: Record<string, string> };
packageDocument.scripts[command.script] =
`node -e 'require("node:fs").writeFileSync("${evidence.path}", Buffer.from("${Buffer.from(evidenceBytes).toString("base64")}", "base64"))'`;
await writeFile(
path.join(root, "config/ci/gates.json"),
`${JSON.stringify(contract)}\n`,
);
await writeFile(
path.join(root, "package.json"),
`${JSON.stringify(packageDocument)}\n`,
);
await writeFile(path.join(root, evidence.path), evidenceBytes);
const result = spawnSync(
process.execPath,
[path.resolve("scripts/run-ci-gate.ts"), "FE-GATE-004"],
{ cwd: root, encoding: "utf8", env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, CI: "false" } },
);
expect(result.status).toBe(0);
expect(result.stdout).toMatch(/FE-GATE-004 runtime-schema: PASS/);
});
});
describe("CI-runner gate output budget", () => {
it("caps aggregate gate output at the log schema before later commands can accumulate", async () => {
const root = await mkdtemp(path.join(tmpdir(), "ci-gate-output-budget-"));
temporaryRoots.push(root);
await mkdir(path.join(root, "config/ci"), { recursive: true });
const contract = JSON.parse(
JSON.stringify(await loadCiGateContract(process.cwd())),
) as Record<string, any>;
const gate = contract.gates.find(
(entry: Record<string, any>) => entry.id === "FE-GATE-001",
);
const command = contract.commands.find(
(entry: Record<string, any>) => entry.id === gate.commandIds[0],
);
command.script = "test:huge-output";
const logArtifact = contract.artifacts.find(
(entry: Record<string, any>) => entry.id === gate.logArtifactId,
);
const logSchema = contract.artifactSchemas.find(
(entry: Record<string, any>) => entry.id === logArtifact.schemaId,
);
logSchema.maxBytes = 8_192;
const packageDocument = JSON.parse(
await readFile("package.json", "utf8"),
) as { scripts: Record<string, string> };
packageDocument.scripts["test:huge-output"] =
"node -e \"process.stdout.write('x'.repeat(20000))\"";
await writeFile(
path.join(root, "config/ci/gates.json"),
`${JSON.stringify(contract)}\n`,
);
await writeFile(
path.join(root, "package.json"),
`${JSON.stringify(packageDocument)}\n`,
);
const result = spawnSync(
process.execPath,
[path.resolve("scripts/run-ci-gate.ts"), "FE-GATE-001"],
{
cwd: root,
encoding: "utf8",
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, CI: "false" },
timeout: 15_000,
},
);
expect(result.status).toBe(1);
const log = await readFile(path.join(root, logArtifact.path));
expect(log.byteLength).toBeLessThanOrEqual(8_192);
expect(log.toString("utf8")).toMatch(
/aggregate output|INFRASTRUCTURE_FAILURE/i,
);
}, 20_000);
});
// 1106/13-01. ci:gate stops at the first command that fails. FE-GATE-027 ran
// the admission first, so a refused candidate left no advisory record, and the
// release evidence the operations guide points to had none. The two package
// scripts are stand-ins: the admission refuses, the scan writes its record.
describe("CI-runner FE-GATE-027 advisory record", () => {
it("writes the advisory record when the release admission refuses the candidate", async () => {
const root = await mkdtemp(path.join(tmpdir(), "ci-gate-refused-admission-"));
temporaryRoots.push(root);
await mkdir(path.join(root, "config/ci"), { recursive: true });
await mkdir(path.join(root, path.dirname(DEPENDENCY_ADVISORIES_RECORD_PATH)), {
recursive: true,
});
const contract = JSON.parse(
JSON.stringify(await loadCiGateContract(process.cwd())),
) as Record<string, any>;
const gate = contract.gates.find(
(entry: Record<string, any>) => entry.id === "FE-GATE-027",
);
const logArtifact = contract.artifacts.find(
(entry: Record<string, any>) => entry.id === gate.logArtifactId,
);
const packageDocument = JSON.parse(
await readFile("package.json", "utf8"),
) as { scripts: Record<string, string> };
packageDocument.scripts["check:release-admission"] =
`node -e "process.stderr.write('release admission refused for production: fixture'),process.exit(1)"`;
packageDocument.scripts["scan:dependency-advisories"] =
`node -e 'require("node:fs").writeFileSync(${JSON.stringify(DEPENDENCY_ADVISORIES_RECORD_PATH)}, "fixture record")'`;
await writeFile(
path.join(root, "config/ci/gates.json"),
`${JSON.stringify(contract)}\n`,
);
await writeFile(
path.join(root, "package.json"),
`${JSON.stringify(packageDocument)}\n`,
);
const result = spawnSync(
process.execPath,
[path.resolve("scripts/run-ci-gate.ts"), "FE-GATE-027"],
{
cwd: root,
encoding: "utf8",
env: { ...process.env, ...FIXTURE_PNPM_ENVIRONMENT, CI: "false" },
timeout: 15_000,
},
);
expect(result.status, result.stderr).toBe(1);
expect(
await readFile(path.join(root, logArtifact.path), "utf8"),
).toContain("release admission refused for production: fixture");
expect(
await readFile(path.join(root, DEPENDENCY_ADVISORIES_RECORD_PATH), "utf8"),
).toBe("fixture record");
}, 20_000);
});