문제: provider 확인이 필요한 drill(FE-RB-002~005)은 입력이 runbook ID와 `--release-root`뿐이라 어떤 입력으로도 PASS가 될 수 없었다. 문서는 "provider 증거가 없으면 production 단계는 닫히지 않는다"고 했지만 증거를 넘길 길이 없었다. 고친 것: `--provider-evidence <파일>`을 받는다. 파일 꼴은 `providerDrillEvidenceSchema`(runbook ID, release ID, 관찰 시각, provider run ID, 항목마다 PASS/FAIL과 근거)다. 같은 runbook·같은 release이고, 관찰이 7일 안이며 5분 넘게 앞서지 않고, 로컬 drill이 UNVERIFIED로 남긴 항목만 답한 증거만 받는다. 받은 증거는 그 항목의 UNVERIFIED만 바꾸고 로컬의 PASS·FAIL은 그대로 둔다. `provider verification` 항목도 증거가 답해야 한다. 받지 않은 증거는 이유와 함께 record의 `providerEvidence`에 남고, record schema는 그 경우 PASS를 거절한다. 버린 대안: production·field 단계를 "provider 연결 전 기본 꺼짐"으로 문서화하는 (b)는 job 그래프(`config/ci/**`)를 바꾸는 일이라 하지 않았다. 위험·한계: 템플릿의 production 게이트 명령은 아직 이 인자를 넘기지 않는다. 증거 파일을 만들고 넘기는 자리는 provider를 연결한 제품 몫이다. 유효 기간 7일은 정한 기본값이다. 검증: desktop `node scripts/run-vitest.ts run tests/unit/runbook-drill.test.ts` 통과(유효 증거 PASS, FAIL 증거, 빠진 항목, 다른 release·runbook, 만료, 미래 시각, 로컬이 본 항목, 다른 꼴 거절, PASS를 주장하는 거절 record). Refs: 0929/11-02
252 lines
11 KiB
TypeScript
252 lines
11 KiB
TypeScript
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
|
|
import { runbookRecordArtifactSchema } from "../../scripts/contracts/release-artifacts.ts";
|
|
import { PROVIDER_EVIDENCE_MAX_AGE_MS, runRunbookDrill } from "../../scripts/drill-runbook.ts";
|
|
import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts";
|
|
import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts";
|
|
|
|
// 11-01 / 13-03 (D-14). A drill reports PASS only for what it observed on the
|
|
// local build. Boot, live traffic, smoke and the provider switch are not
|
|
// observable here, so a drill with provider scope ends UNVERIFIED, never PASS.
|
|
|
|
const contractPackages = [
|
|
{
|
|
packageId: "@example/accounts",
|
|
version: "1.2.3",
|
|
digest: `sha256:${"1".repeat(64)}`,
|
|
runtimeProtocolVersion: 1,
|
|
sourceRevision: "a".repeat(40),
|
|
},
|
|
] as const satisfies readonly InstalledContractPackageIdentity[];
|
|
|
|
const RUNBOOK_IDS = ["FE-RB-001", "FE-RB-002", "FE-RB-003", "FE-RB-004", "FE-RB-005"] as const;
|
|
const roots: string[] = [];
|
|
|
|
afterEach(async () => {
|
|
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
|
});
|
|
|
|
async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) {
|
|
const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-"));
|
|
roots.push(root);
|
|
if (options.withoutDist) return root;
|
|
const write = async (file: string, content: string) => {
|
|
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
|
|
await writeFile(path.join(root, file), content);
|
|
};
|
|
await write("dist/index.html", "<!doctype html><script type=module src=/assets/index-a.js></script>");
|
|
await write(
|
|
"dist/.vite/manifest.json",
|
|
JSON.stringify({
|
|
"index.html": { file: "assets/index-a.js", isEntry: true, css: ["assets/index-a.css"], imports: ["_shared-b.js"] },
|
|
"_shared-b.js": { file: "assets/shared-b.js" },
|
|
"src/pages/lazy.tsx": { file: "assets/lazy-c.js", isDynamicEntry: true },
|
|
}),
|
|
);
|
|
await write("dist/assets/index-a.css", "body{}");
|
|
if (!options.withoutJs) {
|
|
for (const file of ["assets/index-a.js", "assets/shared-b.js", "assets/lazy-c.js"]) {
|
|
await write(`dist/${file}`, "export {};");
|
|
}
|
|
}
|
|
await write(
|
|
"dist/release-manifest.json",
|
|
JSON.stringify({
|
|
schemaVersion: 2,
|
|
appVersion: "1.0.0",
|
|
buildId: "build-a",
|
|
commitSha: "abc1234",
|
|
configSchemaVersion: "2.0",
|
|
assetManifestHash: "assets-a",
|
|
releaseId: "release-a",
|
|
builtAt: "2026-08-01T00:00:00.000Z",
|
|
routeChunks: {},
|
|
contractSet: {
|
|
setAlgorithm: "CA_CONTRACT_SET_V1",
|
|
setDigest: await computeContractSetDigest(contractPackages),
|
|
packages: [...contractPackages],
|
|
},
|
|
}),
|
|
);
|
|
await write(
|
|
"dist/config.json",
|
|
JSON.stringify({
|
|
APP_ENV: "local",
|
|
API_BASE_URL: "http://localhost:8080/",
|
|
REQUEST_TIMEOUT_MS: 10_000,
|
|
MAX_RETRY_ATTEMPTS: 2,
|
|
TELEMETRY_ENABLED: false,
|
|
AUTH_MODE: "external",
|
|
CONFIG_SCHEMA_VERSION: "2.0",
|
|
RELEASE_MANIFEST_URL: "/release-manifest.json",
|
|
BUILD_ID: "build-a",
|
|
RELEASE_ID: "release-a",
|
|
CAPABILITY_OVERRIDES: {
|
|
REALTIME: "DEFAULT",
|
|
WEB_WORKER: "DEFAULT",
|
|
SERVICE_WORKER: "DEFAULT",
|
|
OFFLINE_COMMANDS: "DEFAULT",
|
|
},
|
|
FEATURE_OVERRIDES: {},
|
|
}),
|
|
);
|
|
return root;
|
|
}
|
|
|
|
describe("runbook drill verdicts", () => {
|
|
it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => {
|
|
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
|
|
expect(record.result).toBe("UNVERIFIED");
|
|
expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false);
|
|
});
|
|
|
|
// A drill's recovery assertions are the runbook's recoveryEvidence items, named
|
|
// as config/runbooks/runbooks.json names them and in its order, so the record
|
|
// and the UNVERIFIED line name the same checks the runbook lists.
|
|
it.each(RUNBOOK_IDS)("%s names its recovery assertions after the runbook's recoveryEvidence", async (runbookId) => {
|
|
const document = JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8")) as {
|
|
runbooks: Record<string, { recoveryEvidence: string[] }>;
|
|
};
|
|
const record = await runRunbookDrill(runbookId, { releaseRoot: await builtDist(), contractPackages });
|
|
expect(record.recoveryAssertions.map(({ assertion }) => assertion)).toEqual(
|
|
document.runbooks[runbookId]?.recoveryEvidence,
|
|
);
|
|
});
|
|
|
|
it.each(["FE-RB-002", "FE-RB-005"] as const)(
|
|
"%s fails when the build lost its JavaScript assets",
|
|
async (runbookId) => {
|
|
const record = await runRunbookDrill(runbookId, {
|
|
releaseRoot: await builtDist({ withoutJs: true }),
|
|
contractPackages,
|
|
});
|
|
expect(record.result).toBe("FAIL");
|
|
},
|
|
);
|
|
|
|
it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => {
|
|
await expect(
|
|
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist({ withoutDist: true }), contractPackages }),
|
|
).rejects.toThrow(/dist\/release-manifest\.json/u);
|
|
});
|
|
|
|
it("keeps no constant recovery verdict in the drill source", async () => {
|
|
const source = await readFile("scripts/drill-runbook.ts", "utf8");
|
|
expect(source).not.toMatch(/assertion\([^;]*,\s*true\s*\)/u);
|
|
expect(source).not.toMatch(/Asserted:\s*true\b/u);
|
|
expect(source).not.toMatch(/injectedMountFailure\s*=\s*true/u);
|
|
// 0929/11-08. The drill records string verdicts now; the patterns above
|
|
// only knew the old boolean shape, so a constant "PASS" went unnoticed.
|
|
expect(source).not.toMatch(/assertion\([^;]*,\s*["']PASS["']\s*,?\s*\)/u);
|
|
expect(source).not.toMatch(/(?:Asserted|AsExpected):\s*["']PASS["']/u);
|
|
expect(source).not.toMatch(/verdict\(\s*true\s*\)/u);
|
|
});
|
|
});
|
|
|
|
// 0929/11-02. The drills with provider scope took only a runbook ID and a
|
|
// release root, so no input could ever make them PASS and the production
|
|
// stage could not close. Provider evidence now answers the items the local
|
|
// drill cannot observe, for this runbook and release only, and while recent.
|
|
describe("runbook drill with provider evidence (0929/11-02)", () => {
|
|
const NOW = new Date("2026-11-13T09:00:00.000Z");
|
|
const evidence = (overrides: Record<string, unknown> = {}) => ({
|
|
schemaVersion: 1,
|
|
artifactType: "provider-drill-evidence",
|
|
runbookId: "FE-RB-005",
|
|
releaseId: "release-a",
|
|
observedAt: "2026-11-13T08:30:00.000Z",
|
|
runId: "provider-run-41",
|
|
observations: [
|
|
{ item: "trigger", result: "PASS", evidence: "incident INC-7 declared the release-blocking defect" },
|
|
{ item: "핵심 smoke 시험", result: "PASS", evidence: "smoke against the prior release after the switch" },
|
|
{ item: "provider verification", result: "PASS", evidence: "provider serves release-a again" },
|
|
],
|
|
...overrides,
|
|
});
|
|
const drill = async (providerEvidence: unknown) =>
|
|
runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist(), contractPackages, providerEvidence, now: NOW });
|
|
|
|
it("passes FE-RB-005 when the provider observed every item the local drill could not", async () => {
|
|
const record = await drill(evidence());
|
|
|
|
expect(record.result).toBe("PASS");
|
|
expect(record.triggerAsserted).toBe("PASS");
|
|
expect(record.recoveryAssertions.find(({ assertion }) => assertion === "핵심 smoke 시험")).toEqual({
|
|
assertion: "핵심 smoke 시험",
|
|
evidence: "provider evidence provider-run-41 at 2026-11-13T08:30:00.000Z: smoke against the prior release after the switch",
|
|
result: "PASS",
|
|
});
|
|
expect(record.providerEvidence).toEqual({
|
|
accepted: true,
|
|
runId: "provider-run-41",
|
|
observedAt: "2026-11-13T08:30:00.000Z",
|
|
providerVerification: "PASS",
|
|
});
|
|
const { gateId: _gateId, ...artifact } = record;
|
|
expect(runbookRecordArtifactSchema.safeParse(artifact).success).toBe(true);
|
|
});
|
|
|
|
it("fails when the provider observed a failure", async () => {
|
|
const observations = evidence().observations.map((entry) =>
|
|
entry.item === "핵심 smoke 시험" ? { ...entry, result: "FAIL" } : entry);
|
|
expect((await drill(evidence({ observations }))).result).toBe("FAIL");
|
|
});
|
|
|
|
it("stays UNVERIFIED while an item the local drill could not observe has no answer", async () => {
|
|
const record = await drill(evidence({ observations: evidence().observations.slice(0, 2) }));
|
|
expect(record.result).toBe("UNVERIFIED");
|
|
expect(record.providerEvidence?.providerVerification).toBe("UNVERIFIED");
|
|
});
|
|
|
|
it.each([
|
|
["another release", { releaseId: "release-b" }, "evidence is for release release-b, not release-a"],
|
|
["another runbook", { runbookId: "FE-RB-002" }, "evidence is for FE-RB-002, not FE-RB-005"],
|
|
[
|
|
"an expired observation",
|
|
{ observedAt: new Date(NOW.getTime() - PROVIDER_EVIDENCE_MAX_AGE_MS - 1_000).toISOString() },
|
|
"is older than 7 days",
|
|
],
|
|
["a future observation", { observedAt: "2026-11-13T10:00:00.000Z" }, "is in the future"],
|
|
[
|
|
"an answer for a check the local drill made",
|
|
{ observations: [...evidence().observations, { item: "호환성 gate", result: "PASS", evidence: "x" }] },
|
|
"evidence names items the local drill observed or does not have: 호환성 gate",
|
|
],
|
|
["a record that is not provider evidence", { artifactType: "runbook-record" }, "not provider drill evidence"],
|
|
])("does not pass on evidence for %s", async (_case, overrides, reason) => {
|
|
const record = await drill(evidence(overrides));
|
|
|
|
expect(record.result).toBe("UNVERIFIED");
|
|
expect(record.triggerAsserted).toBe("UNVERIFIED");
|
|
expect(record.providerEvidence?.accepted).toBe(false);
|
|
expect(record.providerEvidence?.reason).toContain(reason);
|
|
const { gateId: _gateId, ...artifact } = record;
|
|
expect(runbookRecordArtifactSchema.safeParse(artifact).success).toBe(true);
|
|
});
|
|
|
|
it("rejects a record that claims PASS without accepted provider evidence", () => {
|
|
const record = {
|
|
schemaVersion: 2,
|
|
runbookId: "FE-RB-005",
|
|
releaseId: "release-a",
|
|
drillTimestamp: NOW.toISOString(),
|
|
triggerInjected: "RELEASE_BLOCKING_DEFECT",
|
|
triggerAsserted: "PASS",
|
|
containmentAsserted: "PASS",
|
|
escalationPathAsserted: "PASS",
|
|
recoveryAssertions: [],
|
|
negativeFixtureFailedAsExpected: "PASS",
|
|
windowObservedBucket: "TBD",
|
|
providerVerificationRequired: true,
|
|
providerEvidence: { accepted: false, reason: "expired", providerVerification: "UNVERIFIED" },
|
|
result: "PASS",
|
|
};
|
|
expect(runbookRecordArtifactSchema.safeParse(record).success).toBe(false);
|
|
expect(runbookRecordArtifactSchema.safeParse({ ...record, result: "UNVERIFIED" }).success).toBe(true);
|
|
});
|
|
});
|