fix(runbook): drill 복구 항목 이름을 runbooks.json과 같게 한다

문제: scripts/drill-runbook.ts의 복구 확인 항목 이름은 영어 리터럴이고,
config/runbooks/runbooks.json의 recoveryEvidence는 2026-11-08에 한국어로
옮겼다. 둘을 비교하는 곳이 없어 record.json과 UNVERIFIED 줄이 runbook에
없는 이름을 쓴다. 앞으로 한쪽만 고쳐도 아무 신호가 없다.

고른 방법: 다섯 drill의 항목 이름 20개를 recoveryEvidence 문자열로 바꾸고,
이름과 순서가 recoveryEvidence와 같은지 보는 시험을 runbook-drill.test.ts에
더한다. 판정 근거 문구(evidence)와 오류 메시지는 영어 그대로다.
버린 대안: drill이 recoveryEvidence를 위치로 읽어 이름을 붙이는 방법은
코드에서 항목과 판정의 짝이 보이지 않고, 설정 순서를 바꾸면 조용히
어긋난다.

위험: artifacts/runbooks/<id>/record.json의 assertion 값이 한국어가 된다.
record schema는 빈 문자열만 막으니 통과하지만, 그 값을 영어로 읽는 바깥
도구가 있으면 바꿔야 한다. 이 repo에는 없다.

검증: desktop에서 node scripts/run-vitest.ts run tests/unit/runbook-drill.test.ts
— 고치기 전 새 시험 5개 실패(RED), 고친 뒤 runbook-drill 14개·
runbook-contract 2개 통과(GREEN).
This commit is contained in:
DongHyeonka
2026-11-12 09:40:51 +09:00
parent 374af98640
commit 6571d8cb9d
2 changed files with 37 additions and 20 deletions
+24 -20
View File
@@ -32,6 +32,10 @@ import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
// it on the local build. What needs a browser boot, live traffic or the hosting
// provider is UNVERIFIED, and so is every drill whose runbook needs provider
// verification: a local drill never claims the release can be recovered.
//
// Each recovery assertion is named after one of the runbook's recoveryEvidence
// items in config/runbooks/runbooks.json, word for word and in its order
// (tests/unit/runbook-drill.test.ts), so the record names what the runbook lists.
type RecoveryAssertion = Readonly<{
assertion: string;
@@ -181,14 +185,14 @@ async function drillBoot(): Promise<DrillResult> {
triggerAsserted: verdict(!invalid.success),
containmentAsserted: verdict(!invalid.success),
recoveryAssertions: [
notObserved("clean-session boot", "a browser boot of the promoted release"),
notObserved("product root mount", "a browser boot of the promoted release"),
notObserved("깨끗한 세션에서 boot", "a browser boot of the promoted release"),
notObserved("제품 root mount", "a browser boot of the promoted release"),
assertion(
"config validation",
"config 검증",
"invalid fixture rejected, valid fixture accepted",
verdict(!invalid.success && recovered.success),
),
notObserved("no repeated boot error", "boot telemetry from the promoted release"),
notObserved("boot 오류 반복 없음", "boot telemetry from the promoted release"),
],
// The runbook's negative fixture injects a mount failure after a valid
// config; that needs a mounted app, which a Node drill does not have.
@@ -233,13 +237,13 @@ async function drillChunkMismatch(context: DrillContext): Promise<DrillResult> {
),
recoveryAssertions: [
assertion(
"entry and lazy assets reachable",
"entry asset과 lazy asset에 접근 가능",
"every Vite manifest file exists in the local dist",
verdict(await builtAssetsPresent(context.root)),
),
assertion("release tuple coherent", "build/config/manifest tuple", verdict(coherence.compatible)),
assertion("second reload blocked", "reload guard decision", verdict(second.action === "support")),
notObserved("critical route smoke", "a browser smoke against the hosted release"),
assertion("release tuple 일치", "build/config/manifest tuple", verdict(coherence.compatible)),
assertion("두 번째 reload 차단", "reload guard decision", verdict(second.action === "support")),
notObserved("핵심 route smoke 시험", "a browser smoke against the hosted release"),
],
negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"),
providerVerificationRequired: true,
@@ -338,11 +342,11 @@ async function drillApiDegradation(): Promise<DrillResult> {
triggerAsserted: "UNVERIFIED",
containmentAsserted: verdict(!unkeyedRetry && safeRetry),
recoveryAssertions: [
notObserved("terminal failure rate at baseline", "API failure telemetry from the promoted release"),
assertion("no retry amplification", "unkeyed retry policy", verdict(!unkeyedRetry)),
notObserved("critical read/write smoke", "a read and a write against the live API"),
notObserved("최종(terminal) 실패율이 기준선 수준", "API failure telemetry from the promoted release"),
assertion("재시도 증폭 없음", "unkeyed retry policy", verdict(!unkeyedRetry)),
notObserved("핵심 읽기/쓰기 smoke 시험", "a read and a write against the live API"),
assertion(
"schema fixtures",
"스키마 fixture",
"schema mismatch is not retryable",
verdict(!schemaRetry),
),
@@ -384,10 +388,10 @@ async function drillTelemetry(): Promise<DrillResult> {
triggerAsserted: verdict(adapter.droppedCount() === 1),
containmentAsserted: verdict(adapter.pendingCount() === 0),
recoveryAssertions: [
assertion("product flow unaffected", "adapter flush resolves despite the sink failure", verdict(flushResolved)),
assertion("delivery self-check", "sink failure counted", verdict(adapter.droppedCount() === 1)),
assertion("queue drained within bound", "pending queue count", verdict(adapter.pendingCount() === 0)),
assertion("forbidden attributes absent", "default-deny projection", verdict(redacted)),
assertion("제품 흐름 영향 없음", "adapter flush resolves despite the sink failure", verdict(flushResolved)),
assertion("전송 자체 점검", "sink failure counted", verdict(adapter.droppedCount() === 1)),
assertion("queue가 상한 안에서 비워짐", "pending queue count", verdict(adapter.pendingCount() === 0)),
assertion("금지 속성 없음", "default-deny projection", verdict(redacted)),
],
negativeFixtureFailedAsExpected: verdict(redacted),
providerVerificationRequired: true,
@@ -421,14 +425,14 @@ async function drillRollback(context: DrillContext): Promise<DrillResult> {
triggerAsserted: "UNVERIFIED",
containmentAsserted: verdict(coherent.compatible && assetsPresent),
recoveryAssertions: [
assertion("compatibility gate", "typed version comparison", verdict(coherent.compatible)),
assertion("호환성 gate", "typed version comparison", verdict(coherent.compatible)),
assertion(
"release coherence gate",
"release 일관성 gate",
"build/config/manifest tuple and every Vite manifest file in the local dist",
verdict(coherent.compatible && assetsPresent),
),
notObserved("critical smoke", "a browser smoke after the provider switches to the prior release"),
assertion("release ID in timeline", "drill record releaseId", verdict(Boolean(release.releaseId))),
notObserved("핵심 smoke 시험", "a browser smoke after the provider switches to the prior release"),
assertion("장애 타임라인에 release ID 기록", "drill record releaseId", verdict(Boolean(release.releaseId))),
],
negativeFixtureFailedAsExpected: verdict(!mixed.compatible),
providerVerificationRequired: true,
+13
View File
@@ -103,6 +103,19 @@ describe("runbook drill verdicts", () => {
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
});
// A drill's recovery assertions are the runbook's recoveryEvidence items, named
// as config/runbooks/runbooks.json names them and in its order, so the record
// and the UNVERIFIED line name the same checks the runbook lists.
it.each(RUNBOOK_IDS)("%s names its recovery assertions after the runbook's recoveryEvidence", async (runbookId) => {
const document = JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8")) as {
runbooks: Record<string, { recoveryEvidence: string[] }>;
};
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
expect(record.recoveryAssertions.map(({ assertion }) => assertion)).toEqual(
document.runbooks[runbookId]?.recoveryEvidence,
);
});
it.each(["FE-RB-002", "FE-RB-005"] as const)(
"%s fails when the build lost its JavaScript assets",
async (runbookId) => {