diff --git a/docs/operations/ci-quality-gates.md b/docs/operations/ci-quality-gates.md index 61e5d39b..42988613 100644 --- a/docs/operations/ci-quality-gates.md +++ b/docs/operations/ci-quality-gates.md @@ -452,7 +452,18 @@ provider 확인이 필요하다고 표시한 것은 모두 `UNVERIFIED`다. `FAI drill은 `FAIL`(exit 1)이다. 그렇지 않고 `UNVERIFIED`가 하나라도 있으면 `UNVERIFIED`(exit 3)이고, 게이트는 `PASS`가 아니다. 그래서 provider 증거가 없으면 production 단계는 닫히지 않는다. 의도한 결과다. 로컬 drill은 승격한 release를 rollback할 -수 있다는 것을 보여 줄 수 없다. FE-GATE-016(`rollback-drill`)은 FE-GATE-025와 같은 +수 있다는 것을 보여 줄 수 없다. + +provider 증거는 `--provider-evidence <파일>`로 넘긴다(0929/11-02). 파일은 +`providerDrillEvidenceSchema`(`scripts/contracts/release-artifacts.ts`) 꼴이다: runbook ID, +release ID, 관찰 시각, provider 쪽 run ID, 그리고 항목마다 `PASS`/`FAIL`과 근거다. 항목 +이름은 `UNVERIFIED` 줄이 적는 이름(`trigger`, recovery assertion 이름, `provider +verification` 등)이다. drill은 runbook과 release가 같고, 관찰이 7일 안이고, 로컬 drill이 +보지 못한 항목만 답한 증거만 받는다. 받은 증거는 그 항목의 `UNVERIFIED`만 바꾸고, 로컬에서 +본 `PASS`·`FAIL`은 그대로 둔다. 모든 항목이 답을 얻고 `FAIL`이 없을 때만 drill이 `PASS`다. +받지 않은 증거는 이유와 함께 record의 `providerEvidence`에 남고 drill은 `UNVERIFIED`다. +템플릿의 production 게이트 명령은 아직 이 인자를 넘기지 않는다. provider를 연결한 제품이 +증거 파일을 만들고 drill 명령에 넘겨야 그 단계가 닫힌다. FE-GATE-016(`rollback-drill`)은 FE-GATE-025와 같은 FE-RB-005 drill을 돌렸으므로 폐기했다(`scripts/contracts/ci-gates.ts`의 `RETIRED_GATE_IDS`). diff --git a/scripts/contracts/release-artifacts.ts b/scripts/contracts/release-artifacts.ts index 13d30bdd..2b98f9df 100644 --- a/scripts/contracts/release-artifacts.ts +++ b/scripts/contracts/release-artifacts.ts @@ -746,16 +746,52 @@ const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]); export type DrillVerdict = z.infer; -/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */ +/** + * FAIL beats UNVERIFIED beats PASS. A drill with provider scope stays + * UNVERIFIED until accepted provider evidence says the provider side passed + * (0929/11-02). + */ export function combineDrillVerdicts( verdicts: readonly DrillVerdict[], providerVerificationRequired: boolean, + providerVerification: DrillVerdict = "UNVERIFIED", ): DrillVerdict { - if (verdicts.includes("FAIL")) return "FAIL"; - if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED"; + const all = providerVerificationRequired ? [...verdicts, providerVerification] : verdicts; + if (all.includes("FAIL")) return "FAIL"; + if (all.includes("UNVERIFIED")) return "UNVERIFIED"; return "PASS"; } +/** + * 0929/11-02. What the hosting provider observed for one drill of one + * release: the provider-side items a local drill cannot see (a smoke after the + * switch, live traffic, the trigger), named as the drill's UNVERIFIED line + * names them, plus `provider verification` itself. + */ +export const providerDrillEvidenceSchema = z + .object({ + schemaVersion: z.literal(1), + artifactType: z.literal("provider-drill-evidence"), + runbookId: z.string().regex(/^FE-RB-00[1-5]$/u), + releaseId: nonEmptyString, + observedAt: timestamp, + runId: nonEmptyString, + observations: z + .array( + z + .object({ item: nonEmptyString, result: z.enum(["PASS", "FAIL"]), evidence: nonEmptyString }) + .strict(), + ) + .min(1) + .max(32) + .refine((items) => new Set(items.map(({ item }) => item)).size === items.length, { + message: "must name each item once", + }), + }) + .strict(); + +export type ProviderDrillEvidence = z.infer; + export const runbookRecordArtifactSchema = z .object({ schemaVersion: z.literal(2), @@ -778,6 +814,19 @@ export const runbookRecordArtifactSchema = z negativeFixtureFailedAsExpected: drillVerdict, windowObservedBucket: nonEmptyString, providerVerificationRequired: z.boolean(), + providerEvidence: z + .object({ + accepted: z.boolean(), + reason: nonEmptyString.optional(), + runId: nonEmptyString.optional(), + observedAt: timestamp.optional(), + providerVerification: drillVerdict, + }) + .strict() + .refine((evidence) => evidence.accepted || (evidence.reason !== undefined && evidence.providerVerification === "UNVERIFIED"), { + message: "rejected provider evidence must say why and verify nothing", + }) + .optional(), result: drillVerdict, }) .strict() @@ -791,6 +840,7 @@ export const runbookRecordArtifactSchema = z record.negativeFixtureFailedAsExpected, ], record.providerVerificationRequired, + record.providerEvidence?.providerVerification, ); if (record.result !== expected) { context.addIssue({ diff --git a/scripts/drill-runbook.ts b/scripts/drill-runbook.ts index c1b03c2d..c610a8a8 100644 --- a/scripts/drill-runbook.ts +++ b/scripts/drill-runbook.ts @@ -22,8 +22,10 @@ import { projectTelemetryEvent } from "../src/contracts/telemetry.ts"; import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts"; import { combineDrillVerdicts, + providerDrillEvidenceSchema, runbookRecordArtifactSchema, type DrillVerdict, + type ProviderDrillEvidence, } from "./contracts/release-artifacts.ts"; import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; @@ -87,8 +89,25 @@ export type DrillOptions = Readonly<{ */ releaseRoot?: string; contractPackages?: readonly InstalledContractPackageIdentity[]; + /** + * 0929/11-02. The provider's record for this drill (`--provider-evidence`), + * parsed JSON. Without it a drill with provider scope cannot pass. + */ + providerEvidence?: unknown; + /** The clock evidence age is judged against; a test passes a fixed one. */ + now?: Date; }>; +/** Provider evidence older than this no longer describes the provider as it is. */ +export const PROVIDER_EVIDENCE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1_000; +/** How far ahead of this clock a provider's timestamp may be. */ +const PROVIDER_EVIDENCE_CLOCK_SKEW_MS = 5 * 60 * 1_000; +/** The item names a drill's UNVERIFIED line uses, and provider evidence answers. */ +const ITEM_TRIGGER = "trigger"; +const ITEM_CONTAINMENT = "containment"; +const ITEM_NEGATIVE_FIXTURE = "negative fixture"; +export const ITEM_PROVIDER_VERIFICATION = "provider verification"; + type DrillContext = Readonly<{ root: string; contractPackages: readonly InstalledContractPackageIdentity[]; @@ -490,14 +509,29 @@ export async function runRunbookDrill(runbookId: string, options: DrillOptions = const release = parseReleaseArtifact( await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release), ); - const result = await drill(context); + const local = await drill(context); + const now = options.now ?? new Date(); + const judged = options.providerEvidence === undefined + ? undefined + : judgeProviderEvidence(options.providerEvidence, runbookId, release.releaseId, openItems(local), now); + const result = judged?.accepted ? applyProviderEvidence(local, judged.evidence) : local; + const providerEvidence = judged === undefined + ? undefined + : judged.accepted + ? { + accepted: true as const, + runId: judged.evidence.runId, + observedAt: judged.evidence.observedAt, + providerVerification: observed(judged.evidence, ITEM_PROVIDER_VERIFICATION) ?? "UNVERIFIED", + } + : { accepted: false as const, reason: judged.reason, providerVerification: "UNVERIFIED" as const }; const escalationPathAsserted = verdict(specification.escalation.length >= 2); return { schemaVersion: 2 as const, runbookId, gateId: specification.gateId, releaseId: release.releaseId, - drillTimestamp: new Date().toISOString(), + drillTimestamp: now.toISOString(), triggerInjected: specification.triggerKinds[0] ?? "", triggerAsserted: result.triggerAsserted, containmentAsserted: result.containmentAsserted, @@ -506,6 +540,7 @@ export async function runRunbookDrill(runbookId: string, options: DrillOptions = negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected, windowObservedBucket: specification.window, providerVerificationRequired: result.providerVerificationRequired, + ...(providerEvidence === undefined ? {} : { providerEvidence }), result: combineDrillVerdicts( [ result.triggerAsserted, @@ -515,24 +550,117 @@ export async function runRunbookDrill(runbookId: string, options: DrillOptions = result.negativeFixtureFailedAsExpected, ], result.providerVerificationRequired, + providerEvidence?.providerVerification, ), }; } +/** The items the local drill left UNVERIFIED, by the names provider evidence uses. */ +function openItems(result: DrillResult): string[] { + return [ + ...(result.triggerAsserted === "UNVERIFIED" ? [ITEM_TRIGGER] : []), + ...(result.containmentAsserted === "UNVERIFIED" ? [ITEM_CONTAINMENT] : []), + ...result.recoveryAssertions.filter((item) => item.result === "UNVERIFIED").map((item) => item.assertion), + ...(result.negativeFixtureFailedAsExpected === "UNVERIFIED" ? [ITEM_NEGATIVE_FIXTURE] : []), + ...(result.providerVerificationRequired ? [ITEM_PROVIDER_VERIFICATION] : []), + ]; +} + +type JudgedProviderEvidence = + | Readonly<{ accepted: true; evidence: ProviderDrillEvidence }> + | Readonly<{ accepted: false; reason: string }>; + +/** + * 0929/11-02. Provider evidence counts only for this runbook, this release, + * a recent observation, and the items the local drill could not observe. + */ +export function judgeProviderEvidence( + value: unknown, + runbookId: string, + releaseId: string, + open: readonly string[], + now: Date, +): JudgedProviderEvidence { + const parsed = providerDrillEvidenceSchema.safeParse(value); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + return { accepted: false, reason: `not provider drill evidence (${issue?.path.join(".") || "root"}: ${issue?.message ?? "invalid"})` }; + } + const evidence = parsed.data; + if (evidence.runbookId !== runbookId) { + return { accepted: false, reason: `evidence is for ${evidence.runbookId}, not ${runbookId}` }; + } + if (evidence.releaseId !== releaseId) { + return { accepted: false, reason: `evidence is for release ${evidence.releaseId}, not ${releaseId}` }; + } + const age = now.getTime() - Date.parse(evidence.observedAt); + if (age > PROVIDER_EVIDENCE_MAX_AGE_MS) { + return { accepted: false, reason: `evidence observed at ${evidence.observedAt} is older than ${PROVIDER_EVIDENCE_MAX_AGE_MS / 86_400_000} days` }; + } + if (age < -PROVIDER_EVIDENCE_CLOCK_SKEW_MS) { + return { accepted: false, reason: `evidence observed at ${evidence.observedAt} is in the future` }; + } + const extra = evidence.observations.map(({ item }) => item).filter((item) => !open.includes(item)); + if (extra.length > 0) { + return { accepted: false, reason: `evidence names items the local drill observed or does not have: ${extra.join(", ")}` }; + } + return { accepted: true, evidence }; +} + +function observed(evidence: ProviderDrillEvidence, item: string): DrillVerdict | undefined { + return evidence.observations.find((observation) => observation.item === item)?.result; +} + +/** Replaces each UNVERIFIED item the evidence answers; local PASS and FAIL stay. */ +function applyProviderEvidence(result: DrillResult, evidence: ProviderDrillEvidence): DrillResult { + const answer = (item: string, local: DrillVerdict): DrillVerdict => + local === "UNVERIFIED" ? observed(evidence, item) ?? local : local; + const source = (item: string) => + `provider evidence ${evidence.runId} at ${evidence.observedAt}: ${ + evidence.observations.find((observation) => observation.item === item)?.evidence ?? "" + }`; + return { + ...result, + triggerAsserted: answer(ITEM_TRIGGER, result.triggerAsserted), + containmentAsserted: answer(ITEM_CONTAINMENT, result.containmentAsserted), + recoveryAssertions: result.recoveryAssertions.map((item) => + item.result === "UNVERIFIED" && observed(evidence, item.assertion) !== undefined + ? { ...item, evidence: source(item.assertion), result: observed(evidence, item.assertion)! } + : item, + ), + negativeFixtureFailedAsExpected: answer(ITEM_NEGATIVE_FIXTURE, result.negativeFixtureFailedAsExpected), + }; +} + async function main(): Promise { const runbookId = process.argv .slice(2) .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); const releaseRootFlag = process.argv.indexOf("--release-root"); const releaseRoot = releaseRootFlag >= 0 ? process.argv[releaseRootFlag + 1] : undefined; - if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot)) { + const evidenceFlag = process.argv.indexOf("--provider-evidence"); + const evidencePath = evidenceFlag >= 0 ? process.argv[evidenceFlag + 1] : undefined; + if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot) || (evidenceFlag >= 0 && !evidencePath)) { process.stderr.write( - "Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root ]\n", + "Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root ]" + + " [--provider-evidence ]\n", ); process.exit(2); } + let providerEvidence: unknown; + if (evidencePath !== undefined) { + try { + providerEvidence = JSON.parse(await readFile(evidencePath, "utf8")); + } catch (error) { + process.stderr.write(`${runbookId} drill: cannot read provider evidence ${evidencePath}: ${String(error)}\n`); + process.exit(2); + } + } - const { gateId, ...record } = await runRunbookDrill(runbookId, releaseRoot ? { releaseRoot } : {}); + const { gateId, ...record } = await runRunbookDrill(runbookId, { + ...(releaseRoot ? { releaseRoot } : {}), + ...(evidencePath !== undefined ? { providerEvidence } : {}), + }); const artifactDirectory = `artifacts/runbooks/${runbookId}`; await mkdir(artifactDirectory, { recursive: true }); await writeValidatedJsonArtifact({ @@ -547,14 +675,18 @@ async function main(): Promise { if (record.result === "UNVERIFIED") { // Not a pass: the gate stays red until provider evidence exists (D-14). const open = [ - ["trigger", record.triggerAsserted], - ["containment", record.containmentAsserted], + [ITEM_TRIGGER, record.triggerAsserted], + [ITEM_CONTAINMENT, record.containmentAsserted], ...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const), - ["negative fixture", record.negativeFixtureFailedAsExpected], - ["provider verification", record.providerVerificationRequired ? "UNVERIFIED" : "PASS"], + [ITEM_NEGATIVE_FIXTURE, record.negativeFixtureFailedAsExpected], + [ + ITEM_PROVIDER_VERIFICATION, + record.providerVerificationRequired ? record.providerEvidence?.providerVerification ?? "UNVERIFIED" : "PASS", + ], ].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name); + const rejected = record.providerEvidence?.accepted === false ? `; provider evidence rejected: ${record.providerEvidence.reason}` : ""; process.stderr.write( - `${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")})\n`, + `${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")}${rejected})\n`, ); process.exit(3); } diff --git a/tests/unit/runbook-drill.test.ts b/tests/unit/runbook-drill.test.ts index 611d4598..afc78619 100644 --- a/tests/unit/runbook-drill.test.ts +++ b/tests/unit/runbook-drill.test.ts @@ -4,7 +4,8 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { runRunbookDrill } from "../../scripts/drill-runbook.ts"; +import { runbookRecordArtifactSchema } from "../../scripts/contracts/release-artifacts.ts"; +import { PROVIDER_EVIDENCE_MAX_AGE_MS, runRunbookDrill } from "../../scripts/drill-runbook.ts"; import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts"; import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts"; @@ -145,3 +146,106 @@ describe("runbook drill verdicts", () => { expect(source).not.toMatch(/verdict\(\s*true\s*\)/u); }); }); + +// 0929/11-02. The drills with provider scope took only a runbook ID and a +// release root, so no input could ever make them PASS and the production +// stage could not close. Provider evidence now answers the items the local +// drill cannot observe, for this runbook and release only, and while recent. +describe("runbook drill with provider evidence (0929/11-02)", () => { + const NOW = new Date("2026-11-13T09:00:00.000Z"); + const evidence = (overrides: Record = {}) => ({ + schemaVersion: 1, + artifactType: "provider-drill-evidence", + runbookId: "FE-RB-005", + releaseId: "release-a", + observedAt: "2026-11-13T08:30:00.000Z", + runId: "provider-run-41", + observations: [ + { item: "trigger", result: "PASS", evidence: "incident INC-7 declared the release-blocking defect" }, + { item: "핵심 smoke 시험", result: "PASS", evidence: "smoke against the prior release after the switch" }, + { item: "provider verification", result: "PASS", evidence: "provider serves release-a again" }, + ], + ...overrides, + }); + const drill = async (providerEvidence: unknown) => + runRunbookDrill("FE-RB-005", { releaseRoot: await builtDist(), contractPackages, providerEvidence, now: NOW }); + + it("passes FE-RB-005 when the provider observed every item the local drill could not", async () => { + const record = await drill(evidence()); + + expect(record.result).toBe("PASS"); + expect(record.triggerAsserted).toBe("PASS"); + expect(record.recoveryAssertions.find(({ assertion }) => assertion === "핵심 smoke 시험")).toEqual({ + assertion: "핵심 smoke 시험", + evidence: "provider evidence provider-run-41 at 2026-11-13T08:30:00.000Z: smoke against the prior release after the switch", + result: "PASS", + }); + expect(record.providerEvidence).toEqual({ + accepted: true, + runId: "provider-run-41", + observedAt: "2026-11-13T08:30:00.000Z", + providerVerification: "PASS", + }); + const { gateId: _gateId, ...artifact } = record; + expect(runbookRecordArtifactSchema.safeParse(artifact).success).toBe(true); + }); + + it("fails when the provider observed a failure", async () => { + const observations = evidence().observations.map((entry) => + entry.item === "핵심 smoke 시험" ? { ...entry, result: "FAIL" } : entry); + expect((await drill(evidence({ observations }))).result).toBe("FAIL"); + }); + + it("stays UNVERIFIED while an item the local drill could not observe has no answer", async () => { + const record = await drill(evidence({ observations: evidence().observations.slice(0, 2) })); + expect(record.result).toBe("UNVERIFIED"); + expect(record.providerEvidence?.providerVerification).toBe("UNVERIFIED"); + }); + + it.each([ + ["another release", { releaseId: "release-b" }, "evidence is for release release-b, not release-a"], + ["another runbook", { runbookId: "FE-RB-002" }, "evidence is for FE-RB-002, not FE-RB-005"], + [ + "an expired observation", + { observedAt: new Date(NOW.getTime() - PROVIDER_EVIDENCE_MAX_AGE_MS - 1_000).toISOString() }, + "is older than 7 days", + ], + ["a future observation", { observedAt: "2026-11-13T10:00:00.000Z" }, "is in the future"], + [ + "an answer for a check the local drill made", + { observations: [...evidence().observations, { item: "호환성 gate", result: "PASS", evidence: "x" }] }, + "evidence names items the local drill observed or does not have: 호환성 gate", + ], + ["a record that is not provider evidence", { artifactType: "runbook-record" }, "not provider drill evidence"], + ])("does not pass on evidence for %s", async (_case, overrides, reason) => { + const record = await drill(evidence(overrides)); + + expect(record.result).toBe("UNVERIFIED"); + expect(record.triggerAsserted).toBe("UNVERIFIED"); + expect(record.providerEvidence?.accepted).toBe(false); + expect(record.providerEvidence?.reason).toContain(reason); + const { gateId: _gateId, ...artifact } = record; + expect(runbookRecordArtifactSchema.safeParse(artifact).success).toBe(true); + }); + + it("rejects a record that claims PASS without accepted provider evidence", () => { + const record = { + schemaVersion: 2, + runbookId: "FE-RB-005", + releaseId: "release-a", + drillTimestamp: NOW.toISOString(), + triggerInjected: "RELEASE_BLOCKING_DEFECT", + triggerAsserted: "PASS", + containmentAsserted: "PASS", + escalationPathAsserted: "PASS", + recoveryAssertions: [], + negativeFixtureFailedAsExpected: "PASS", + windowObservedBucket: "TBD", + providerVerificationRequired: true, + providerEvidence: { accepted: false, reason: "expired", providerVerification: "UNVERIFIED" }, + result: "PASS", + }; + expect(runbookRecordArtifactSchema.safeParse(record).success).toBe(false); + expect(runbookRecordArtifactSchema.safeParse({ ...record, result: "UNVERIFIED" }).success).toBe(true); + }); +});