Files
clean-architecture-frontend…/tests/unit/runbook-drill.test.ts
T
DongHyeonka 47a5957233 feat(drill): provider 증거로 로컬 drill이 못 보는 항목을 닫는다
문제: provider 확인이 필요한 drill(FE-RB-002~005)은 입력이 runbook ID와
`--release-root`뿐이라 어떤 입력으로도 PASS가 될 수 없었다. 문서는
"provider 증거가 없으면 production 단계는 닫히지 않는다"고 했지만 증거를
넘길 길이 없었다.

고친 것: `--provider-evidence <파일>`을 받는다. 파일 꼴은
`providerDrillEvidenceSchema`(runbook ID, release ID, 관찰 시각, provider
run ID, 항목마다 PASS/FAIL과 근거)다. 같은 runbook·같은 release이고, 관찰이
7일 안이며 5분 넘게 앞서지 않고, 로컬 drill이 UNVERIFIED로 남긴 항목만
답한 증거만 받는다. 받은 증거는 그 항목의 UNVERIFIED만 바꾸고 로컬의
PASS·FAIL은 그대로 둔다. `provider verification` 항목도 증거가 답해야
한다. 받지 않은 증거는 이유와 함께 record의 `providerEvidence`에 남고,
record schema는 그 경우 PASS를 거절한다.

버린 대안: production·field 단계를 "provider 연결 전 기본 꺼짐"으로
문서화하는 (b)는 job 그래프(`config/ci/**`)를 바꾸는 일이라 하지 않았다.

위험·한계: 템플릿의 production 게이트 명령은 아직 이 인자를 넘기지
않는다. 증거 파일을 만들고 넘기는 자리는 provider를 연결한 제품 몫이다.
유효 기간 7일은 정한 기본값이다.

검증: desktop `node scripts/run-vitest.ts run tests/unit/runbook-drill.test.ts`
통과(유효 증거 PASS, FAIL 증거, 빠진 항목, 다른 release·runbook, 만료,
미래 시각, 로컬이 본 항목, 다른 꼴 거절, PASS를 주장하는 거절 record).

Refs: 0929/11-02
2026-11-13 21:49:58 +09:00

252 lines
11 KiB
TypeScript

import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { runbookRecordArtifactSchema } from "../../scripts/contracts/release-artifacts.ts";
import { PROVIDER_EVIDENCE_MAX_AGE_MS, runRunbookDrill } from "../../scripts/drill-runbook.ts";
import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts";
import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts";
// 11-01 / 13-03 (D-14). A drill reports PASS only for what it observed on the
// local build. Boot, live traffic, smoke and the provider switch are not
// observable here, so a drill with provider scope ends UNVERIFIED, never PASS.
const contractPackages = [
{
packageId: "@example/accounts",
version: "1.2.3",
digest: `sha256:${"1".repeat(64)}`,
runtimeProtocolVersion: 1,
sourceRevision: "a".repeat(40),
},
] as const satisfies readonly InstalledContractPackageIdentity[];
const RUNBOOK_IDS = ["FE-RB-001", "FE-RB-002", "FE-RB-003", "FE-RB-004", "FE-RB-005"] as const;
const roots: string[] = [];
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
});
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
roots.push(root);
if (options.withoutDist) return root;
const write = async (file: string, content: string) => {
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
await writeFile(path.join(root, file), content);
};
await write("dist/index.html", "<!doctype html><script type=module src=/assets/index-a.js></script>");
await write(
"dist/.vite/manifest.json",
JSON.stringify({
"index.html": { file: "assets/index-a.js", isEntry: true, css: ["assets/index-a.css"], imports: ["_shared-b.js"] },
"_shared-b.js": { file: "assets/shared-b.js" },
"src/pages/lazy.tsx": { file: "assets/lazy-c.js", isDynamicEntry: true },
}),
);
await write("dist/assets/index-a.css", "body{}");
if (!options.withoutJs) {
for (const file of ["assets/index-a.js", "assets/shared-b.js", "assets/lazy-c.js"]) {
await write(`dist/${file}`, "export {};");
}
}
await write(
"dist/release-manifest.json",
JSON.stringify({
schemaVersion: 2,
appVersion: "1.0.0",
buildId: "build-a",
commitSha: "abc1234",
configSchemaVersion: "2.0",
assetManifestHash: "assets-a",
releaseId: "release-a",
builtAt: "2026-08-01T00:00:00.000Z",
routeChunks: {},
contractSet: {
setAlgorithm: "CA_CONTRACT_SET_V1",
setDigest: await computeContractSetDigest(contractPackages),
packages: [...contractPackages],
},
}),
);
await write(
"dist/config.json",
JSON.stringify({
APP_ENV: "local",
API_BASE_URL: "http://localhost:8080/",
REQUEST_TIMEOUT_MS: 10_000,
MAX_RETRY_ATTEMPTS: 2,
TELEMETRY_ENABLED: false,
AUTH_MODE: "external",
CONFIG_SCHEMA_VERSION: "2.0",
RELEASE_MANIFEST_URL: "/release-manifest.json",
BUILD_ID: "build-a",
RELEASE_ID: "release-a",
CAPABILITY_OVERRIDES: {
REALTIME: "DEFAULT",
WEB_WORKER: "DEFAULT",
SERVICE_WORKER: "DEFAULT",
OFFLINE_COMMANDS: "DEFAULT",
},
FEATURE_OVERRIDES: {},
}),
);
return root;
}
describe("runbook drill verdicts", () => {
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
expect(record.result).toBe("UNVERIFIED");
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
});
// A drill's recovery assertions are the runbook's recoveryEvidence items, named
// as config/runbooks/runbooks.json names them and in its order, so the record
// and the UNVERIFIED line name the same checks the runbook lists.
it.each(RUNBOOK_IDS)("%s names its recovery assertions after the runbook's recoveryEvidence", async (runbookId) => {
const document = JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8")) as {
runbooks: Record<string, { recoveryEvidence: string[] }>;
};
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
expect(record.recoveryAssertions.map(({ assertion }) => assertion)).toEqual(
document.runbooks[runbookId]?.recoveryEvidence,
);
});
it.each(["FE-RB-002", "FE-RB-005"] as const)(
"%s fails when the build lost its JavaScript assets",
async (runbookId) => {
const record = await runRunbookDrill(runbookId, {
releaseRoot: await builtDist({ withoutJs: true }),
contractPackages,
});
expect(record.result).toBe("FAIL");
},
);
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
await expect(
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist({ withoutDist: true }), contractPackages }),
).rejects.toThrow(/dist\/release-manifest\.json/u);
});
it("keeps no constant recovery verdict in the drill source", async () => {
const source = await readFile("scripts/drill-runbook.ts", "utf8");
expect(source).not.toMatch(/assertion\([^;]*,\s*true\s*\)/u);
expect(source).not.toMatch(/Asserted:\s*true\b/u);
expect(source).not.toMatch(/injectedMountFailure\s*=\s*true/u);
// 0929/11-08. The drill records string verdicts now; the patterns above
// only knew the old boolean shape, so a constant "PASS" went unnoticed.
expect(source).not.toMatch(/assertion\([^;]*,\s*["']PASS["']\s*,?\s*\)/u);
expect(source).not.toMatch(/(?:Asserted|AsExpected):\s*["']PASS["']/u);
expect(source).not.toMatch(/verdict\(\s*true\s*\)/u);
});
});
// 0929/11-02. The drills with provider scope took only a runbook ID and a
// release root, so no input could ever make them PASS and the production
// stage could not close. Provider evidence now answers the items the local
// drill cannot observe, for this runbook and release only, and while recent.
describe("runbook drill with provider evidence (0929/11-02)", () => {
const NOW = new Date("2026-11-13T09:00:00.000Z");
const evidence = (overrides: Record<string, unknown> = {}) => ({
schemaVersion: 1,
artifactType: "provider-drill-evidence",
runbookId: "FE-RB-005",
releaseId: "release-a",
observedAt: "2026-11-13T08:30:00.000Z",
runId: "provider-run-41",
observations: [
{ item: "trigger", result: "PASS", evidence: "incident INC-7 declared the release-blocking defect" },
{ item: "핵심 smoke 시험", result: "PASS", evidence: "smoke against the prior release after the switch" },
{ item: "provider verification", result: "PASS", evidence: "provider serves release-a again" },
],
...overrides,
});
const drill = async (providerEvidence: unknown) =>
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist(), contractPackages, providerEvidence, now: NOW });
it("passes FE-RB-005 when the provider observed every item the local drill could not", async () => {
const record = await drill(evidence());
expect(record.result).toBe("PASS");
expect(record.triggerAsserted).toBe("PASS");
expect(record.recoveryAssertions.find(({ assertion }) => assertion === "핵심 smoke 시험")).toEqual({
assertion: "핵심 smoke 시험",
evidence: "provider evidence provider-run-41 at 2026-11-13T08:30:00.000Z: smoke against the prior release after the switch",
result: "PASS",
});
expect(record.providerEvidence).toEqual({
accepted: true,
runId: "provider-run-41",
observedAt: "2026-11-13T08:30:00.000Z",
providerVerification: "PASS",
});
const { gateId: _gateId, ...artifact } = record;
expect(runbookRecordArtifactSchema.safeParse(artifact).success).toBe(true);
});
it("fails when the provider observed a failure", async () => {
const observations = evidence().observations.map((entry) =>
entry.item === "핵심 smoke 시험" ? { ...entry, result: "FAIL" } : entry);
expect((await drill(evidence({ observations }))).result).toBe("FAIL");
});
it("stays UNVERIFIED while an item the local drill could not observe has no answer", async () => {
const record = await drill(evidence({ observations: evidence().observations.slice(0, 2) }));
expect(record.result).toBe("UNVERIFIED");
expect(record.providerEvidence?.providerVerification).toBe("UNVERIFIED");
});
it.each([
["another release", { releaseId: "release-b" }, "evidence is for release release-b, not release-a"],
["another runbook", { runbookId: "FE-RB-002" }, "evidence is for FE-RB-002, not FE-RB-005"],
[
"an expired observation",
{ observedAt: new Date(NOW.getTime() - PROVIDER_EVIDENCE_MAX_AGE_MS - 1_000).toISOString() },
"is older than 7 days",
],
["a future observation", { observedAt: "2026-11-13T10:00:00.000Z" }, "is in the future"],
[
"an answer for a check the local drill made",
{ observations: [...evidence().observations, { item: "호환성 gate", result: "PASS", evidence: "x" }] },
"evidence names items the local drill observed or does not have: 호환성 gate",
],
["a record that is not provider evidence", { artifactType: "runbook-record" }, "not provider drill evidence"],
])("does not pass on evidence for %s", async (_case, overrides, reason) => {
const record = await drill(evidence(overrides));
expect(record.result).toBe("UNVERIFIED");
expect(record.triggerAsserted).toBe("UNVERIFIED");
expect(record.providerEvidence?.accepted).toBe(false);
expect(record.providerEvidence?.reason).toContain(reason);
const { gateId: _gateId, ...artifact } = record;
expect(runbookRecordArtifactSchema.safeParse(artifact).success).toBe(true);
});
it("rejects a record that claims PASS without accepted provider evidence", () => {
const record = {
schemaVersion: 2,
runbookId: "FE-RB-005",
releaseId: "release-a",
drillTimestamp: NOW.toISOString(),
triggerInjected: "RELEASE_BLOCKING_DEFECT",
triggerAsserted: "PASS",
containmentAsserted: "PASS",
escalationPathAsserted: "PASS",
recoveryAssertions: [],
negativeFixtureFailedAsExpected: "PASS",
windowObservedBucket: "TBD",
providerVerificationRequired: true,
providerEvidence: { accepted: false, reason: "expired", providerVerification: "UNVERIFIED" },
result: "PASS",
};
expect(runbookRecordArtifactSchema.safeParse(record).success).toBe(false);
expect(runbookRecordArtifactSchema.safeParse({ ...record, result: "UNVERIFIED" }).success).toBe(true);
});
});