문제: scripts/drill-runbook.ts의 복구 확인 항목 이름은 영어 리터럴이고, config/runbooks/runbooks.json의 recoveryEvidence는 2026-11-08에 한국어로 옮겼다. 둘을 비교하는 곳이 없어 record.json과 UNVERIFIED 줄이 runbook에 없는 이름을 쓴다. 앞으로 한쪽만 고쳐도 아무 신호가 없다. 고른 방법: 다섯 drill의 항목 이름 20개를 recoveryEvidence 문자열로 바꾸고, 이름과 순서가 recoveryEvidence와 같은지 보는 시험을 runbook-drill.test.ts에 더한다. 판정 근거 문구(evidence)와 오류 메시지는 영어 그대로다. 버린 대안: drill이 recoveryEvidence를 위치로 읽어 이름을 붙이는 방법은 코드에서 항목과 판정의 짝이 보이지 않고, 설정 순서를 바꾸면 조용히 어긋난다. 위험: artifacts/runbooks/<id>/record.json의 assertion 값이 한국어가 된다. record schema는 빈 문자열만 막으니 통과하지만, 그 값을 영어로 읽는 바깥 도구가 있으면 바꿔야 한다. 이 repo에는 없다. 검증: desktop에서 node scripts/run-vitest.ts run tests/unit/runbook-drill.test.ts — 고치기 전 새 시험 5개 실패(RED), 고친 뒤 runbook-drill 14개· runbook-contract 2개 통과(GREEN).
148 lines
5.8 KiB
TypeScript
148 lines
5.8 KiB
TypeScript
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
|
|
import { runRunbookDrill } from "../../scripts/drill-runbook.ts";
|
|
import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts";
|
|
import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts";
|
|
|
|
// 11-01 / 13-03 (D-14). A drill reports PASS only for what it observed on the
|
|
// local build. Boot, live traffic, smoke and the provider switch are not
|
|
// observable here, so a drill with provider scope ends UNVERIFIED, never PASS.
|
|
|
|
const contractPackages = [
|
|
{
|
|
packageId: "@example/accounts",
|
|
version: "1.2.3",
|
|
digest: `sha256:${"1".repeat(64)}`,
|
|
runtimeProtocolVersion: 1,
|
|
sourceRevision: "a".repeat(40),
|
|
},
|
|
] as const satisfies readonly InstalledContractPackageIdentity[];
|
|
|
|
const RUNBOOK_IDS = ["FE-RB-001", "FE-RB-002", "FE-RB-003", "FE-RB-004", "FE-RB-005"] as const;
|
|
const roots: string[] = [];
|
|
|
|
afterEach(async () => {
|
|
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
|
});
|
|
|
|
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
|
|
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
|
|
roots.push(root);
|
|
if (options.withoutDist) return root;
|
|
const write = async (file: string, content: string) => {
|
|
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
|
|
await writeFile(path.join(root, file), content);
|
|
};
|
|
await write("dist/index.html", "<!doctype html><script type=module src=/assets/index-a.js></script>");
|
|
await write(
|
|
"dist/.vite/manifest.json",
|
|
JSON.stringify({
|
|
"index.html": { file: "assets/index-a.js", isEntry: true, css: ["assets/index-a.css"], imports: ["_shared-b.js"] },
|
|
"_shared-b.js": { file: "assets/shared-b.js" },
|
|
"src/pages/lazy.tsx": { file: "assets/lazy-c.js", isDynamicEntry: true },
|
|
}),
|
|
);
|
|
await write("dist/assets/index-a.css", "body{}");
|
|
if (!options.withoutJs) {
|
|
for (const file of ["assets/index-a.js", "assets/shared-b.js", "assets/lazy-c.js"]) {
|
|
await write(`dist/${file}`, "export {};");
|
|
}
|
|
}
|
|
await write(
|
|
"dist/release-manifest.json",
|
|
JSON.stringify({
|
|
schemaVersion: 2,
|
|
appVersion: "1.0.0",
|
|
buildId: "build-a",
|
|
commitSha: "abc1234",
|
|
configSchemaVersion: "2.0",
|
|
assetManifestHash: "assets-a",
|
|
releaseId: "release-a",
|
|
builtAt: "2026-08-01T00:00:00.000Z",
|
|
routeChunks: {},
|
|
contractSet: {
|
|
setAlgorithm: "CA_CONTRACT_SET_V1",
|
|
setDigest: await computeContractSetDigest(contractPackages),
|
|
packages: [...contractPackages],
|
|
},
|
|
}),
|
|
);
|
|
await write(
|
|
"dist/config.json",
|
|
JSON.stringify({
|
|
APP_ENV: "local",
|
|
API_BASE_URL: "http://localhost:8080/",
|
|
REQUEST_TIMEOUT_MS: 10_000,
|
|
MAX_RETRY_ATTEMPTS: 2,
|
|
TELEMETRY_ENABLED: false,
|
|
AUTH_MODE: "external",
|
|
CONFIG_SCHEMA_VERSION: "2.0",
|
|
RELEASE_MANIFEST_URL: "/release-manifest.json",
|
|
BUILD_ID: "build-a",
|
|
RELEASE_ID: "release-a",
|
|
CAPABILITY_OVERRIDES: {
|
|
REALTIME: "DEFAULT",
|
|
WEB_WORKER: "DEFAULT",
|
|
SERVICE_WORKER: "DEFAULT",
|
|
OFFLINE_COMMANDS: "DEFAULT",
|
|
},
|
|
FEATURE_OVERRIDES: {},
|
|
}),
|
|
);
|
|
return root;
|
|
}
|
|
|
|
describe("runbook drill verdicts", () => {
|
|
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
|
|
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
|
|
expect(record.result).toBe("UNVERIFIED");
|
|
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
|
|
});
|
|
|
|
// A drill's recovery assertions are the runbook's recoveryEvidence items, named
|
|
// as config/runbooks/runbooks.json names them and in its order, so the record
|
|
// and the UNVERIFIED line name the same checks the runbook lists.
|
|
it.each(RUNBOOK_IDS)("%s names its recovery assertions after the runbook's recoveryEvidence", async (runbookId) => {
|
|
const document = JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8")) as {
|
|
runbooks: Record<string, { recoveryEvidence: string[] }>;
|
|
};
|
|
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
|
|
expect(record.recoveryAssertions.map(({ assertion }) => assertion)).toEqual(
|
|
document.runbooks[runbookId]?.recoveryEvidence,
|
|
);
|
|
});
|
|
|
|
it.each(["FE-RB-002", "FE-RB-005"] as const)(
|
|
"%s fails when the build lost its JavaScript assets",
|
|
async (runbookId) => {
|
|
const record = await runRunbookDrill(runbookId, {
|
|
releaseRoot: await builtDist({ withoutJs: true }),
|
|
contractPackages,
|
|
});
|
|
expect(record.result).toBe("FAIL");
|
|
},
|
|
);
|
|
|
|
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
|
|
await expect(
|
|
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist({ withoutDist: true }), contractPackages }),
|
|
).rejects.toThrow(/dist\/release-manifest\.json/u);
|
|
});
|
|
|
|
it("keeps no constant recovery verdict in the drill source", async () => {
|
|
const source = await readFile("scripts/drill-runbook.ts", "utf8");
|
|
expect(source).not.toMatch(/assertion\([^;]*,\s*true\s*\)/u);
|
|
expect(source).not.toMatch(/Asserted:\s*true\b/u);
|
|
expect(source).not.toMatch(/injectedMountFailure\s*=\s*true/u);
|
|
// 0929/11-08. The drill records string verdicts now; the patterns above
|
|
// only knew the old boolean shape, so a constant "PASS" went unnoticed.
|
|
expect(source).not.toMatch(/assertion\([^;]*,\s*["']PASS["']\s*,?\s*\)/u);
|
|
expect(source).not.toMatch(/(?:Asserted|AsExpected):\s*["']PASS["']/u);
|
|
expect(source).not.toMatch(/verdict\(\s*true\s*\)/u);
|
|
});
|
|
});
|