From 7f889a6d8365cdb47fbaa2a0484a5db1ddde8398 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 25 Sep 2026 22:15:59 +0900 Subject: [PATCH] fix(release): record unobserved runbook drill verdicts as UNVERIFIED (11-01, 13-03) Drills held constant true verdicts and read public/ placeholders, so they passed with no build or with JS assets deleted. They now check the local dist, mark boot, live traffic and provider steps UNVERIFIED (exit 3), and FE-GATE-016 is retired because it repeated FE-GATE-025's FE-RB-005 drill. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/quality-gates.yml | 1 - README.md | 7 +- config/ci/contract-baseline.json | 10 +- config/ci/gates.json | 21 - .../schemas/runbook-drill-record.schema.json | 28 +- docs/operations/ci-quality-gates.md | 17 +- scripts/check-ci-contract.ts | 8 +- scripts/contracts/ci-gates.ts | 22 +- scripts/contracts/release-artifacts.ts | 52 ++- scripts/drill-runbook.ts | 423 +++++++++--------- scripts/lib/ci-contract-report.ts | 2 +- scripts/lib/promotion-readiness.ts | 2 +- .../ci-workflow-generation.test.ts.snap | 1 - tests/unit/ci-workflow-generation.test.ts | 8 +- tests/unit/promotion-readiness.test.ts | 2 +- tests/unit/release-coherence.test.ts | 78 +--- tests/unit/runbook-drill.test.ts | 134 ++++++ 17 files changed, 482 insertions(+), 334 deletions(-) create mode 100644 tests/unit/runbook-drill.test.ts diff --git a/.gitea/workflows/quality-gates.yml b/.gitea/workflows/quality-gates.yml index b979b3dd..195b9b55 100644 --- a/.gitea/workflows/quality-gates.yml +++ b/.gitea/workflows/quality-gates.yml @@ -379,7 +379,6 @@ jobs: fail-fast: false matrix: include: - - { gate: FE-GATE-016, name: rollback-drill } - { gate: FE-GATE-021, name: runbook-boot-config } - { gate: FE-GATE-022, name: runbook-chunk-mismatch } - { gate: FE-GATE-023, name: runbook-api-degradation } diff --git a/README.md b/README.md index 57cbe239..eefc9f96 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,11 @@ corepack pnpm drill:runbooks corepack pnpm check:ci ``` +`drill:runbooks` stops at the first drill that is not `PASS`. Without provider +evidence every drill ends `UNVERIFIED` (exit 3), because a drill never reports +a check it could not run locally as `PASS`; see "Runbook jobs" in +[`docs/operations/ci-quality-gates.md`](docs/operations/ci-quality-gates.md). + `test:all` is the ordinary product-development loop and intentionally excludes host-level CI-runner assurance. Run `corepack pnpm test:system` only on the compatible Linux assurance host described in @@ -168,7 +173,7 @@ Live release verification additionally requires `HOSTING_BASE_URL`. ## CI and evidence -The 28-gate registry is `config/ci/gates.json`; the Gitea workflow is +The 27-gate registry is `config/ci/gates.json`; the Gitea workflow is `.gitea/workflows/quality-gates.yml`. It follows: ```text diff --git a/config/ci/contract-baseline.json b/config/ci/contract-baseline.json index 66869356..b33f3528 100644 --- a/config/ci/contract-baseline.json +++ b/config/ci/contract-baseline.json @@ -1,11 +1,11 @@ { "schemaVersion": 1, "owner": "frontend-platform", - "reason": "08-06: the signed manual accessibility review moves to release gate FE-GATE-028; merge gate FE-GATE-009 checks only the record format", - "gates": 28, + "reason": "11-01, 13-03 (D-14): FE-GATE-016 retired; it ran the same FE-RB-005 drill as FE-GATE-025", + "gates": 27, "commands": 94, - "commandReferences": 106, - "artifacts": 110, + "commandReferences": 104, + "artifacts": 109, "stages": 5, - "gateShapeSha256": "111e82fa1fd8341f8ee1f3d10d14365de382d83da5f8f9c5a1e3fcfecc5cafae" + "gateShapeSha256": "3ad2407c5929a7ec43dadebc68a3dc69e265793bf1c80c355b6d2fdeae019821" } diff --git a/config/ci/gates.json b/config/ci/gates.json index 509bc4cb..0e41a294 100644 --- a/config/ci/gates.json +++ b/config/ci/gates.json @@ -1502,12 +1502,6 @@ "build-release-candidate" ] }, - { - "id": "artifact-artifacts-quality-gates-FE-GATE-016-txt", - "path": "artifacts/quality/gates/FE-GATE-016.txt", - "schemaId": "text", - "production": "runner-generated" - }, { "id": "artifact-artifacts-runbooks-FE-RB-005-record-json", "path": "artifacts/runbooks/FE-RB-005/record.json", @@ -2015,19 +2009,6 @@ ], "retentionClassId": "release-coherence" }, - { - "id": "FE-GATE-016", - "name": "rollback-drill", - "commandIds": [ - "build", - "drill-runbook" - ], - "logArtifactId": "artifact-artifacts-quality-gates-FE-GATE-016-txt", - "evidenceArtifactIds": [ - "artifact-artifacts-runbooks-FE-RB-005-record-json" - ], - "retentionClassId": "prod-drill" - }, { "id": "FE-GATE-017", "name": "diagram-review", @@ -2243,7 +2224,6 @@ "release" ], "gateIds": [ - "FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", @@ -2774,7 +2754,6 @@ "condition": "production", "timeoutMinutes": 45, "gateIds": [ - "FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", diff --git a/config/schemas/runbook-drill-record.schema.json b/config/schemas/runbook-drill-record.schema.json index d58daba3..9affe249 100644 --- a/config/schemas/runbook-drill-record.schema.json +++ b/config/schemas/runbook-drill-record.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "ART-FE-RUNBOOK-DRILL@1", + "$id": "ART-FE-RUNBOOK-DRILL@2", "type": "object", "required": [ "schemaVersion", @@ -14,29 +14,39 @@ "recoveryAssertions", "negativeFixtureFailedAsExpected", "windowObservedBucket", - "passed" + "providerVerificationRequired", + "result" ], + "$defs": { + "verdict": { "enum": ["PASS", "FAIL", "UNVERIFIED"] } + }, "properties": { - "schemaVersion": { "const": 1 }, + "schemaVersion": { "const": 2 }, "runbookId": { "pattern": "^FE-RB-00[1-5]$" }, "releaseId": { "type": "string", "minLength": 1 }, "drillTimestamp": { "type": "string", "format": "date-time" }, "triggerInjected": { "type": "string" }, - "triggerAsserted": { "type": "boolean" }, - "containmentAsserted": { "type": "boolean" }, - "escalationPathAsserted": { "type": "boolean" }, + "triggerAsserted": { "$ref": "#/$defs/verdict" }, + "containmentAsserted": { "$ref": "#/$defs/verdict" }, + "escalationPathAsserted": { "$ref": "#/$defs/verdict" }, "recoveryAssertions": { "type": "array", "minItems": 4, "items": { "type": "object", - "required": ["assertion", "evidence", "passed"] + "required": ["assertion", "evidence", "result"], + "properties": { + "assertion": { "type": "string", "minLength": 1 }, + "evidence": { "type": "string", "minLength": 1 }, + "result": { "$ref": "#/$defs/verdict" } + }, + "additionalProperties": false } }, - "negativeFixtureFailedAsExpected": { "type": "boolean" }, + "negativeFixtureFailedAsExpected": { "$ref": "#/$defs/verdict" }, "windowObservedBucket": { "type": "string" }, "providerVerificationRequired": { "type": "boolean" }, - "passed": { "type": "boolean" } + "result": { "$ref": "#/$defs/verdict" } }, "additionalProperties": false } diff --git a/docs/operations/ci-quality-gates.md b/docs/operations/ci-quality-gates.md index 4edb2fb5..b0f8a1fc 100644 --- a/docs/operations/ci-quality-gates.md +++ b/docs/operations/ci-quality-gates.md @@ -1,6 +1,6 @@ # CI quality-gate orchestration -`config/ci/gates.json` is the executable registry for all 28 gates and the sole +`config/ci/gates.json` is the executable registry for all 27 gates and the sole workflow model. `.gitea/workflows/quality-gates.yml` is generated from that strict v2 contract; it is not an independent configuration authority. Update the registry, run `corepack pnpm generate:ci-workflow`, and commit the generated @@ -269,7 +269,20 @@ promotion, and field evidence through aggregation. Runbook jobs write to the stable path `artifacts/runbooks//record.json`; the dynamic release identity is -stored inside the record. This keeps gate evidence lookup independent from +stored inside the record. + +A runbook drill (`drill:runbook`, FE-GATE-021..025) records each verdict as +`PASS`, `FAIL` or `UNVERIFIED` (`scripts/drill-runbook.ts`, record schema in +`scripts/contracts/release-artifacts.ts`). `PASS`/`FAIL` means the drill checked +it against the local `dist/` build; it reads no `public/` placeholder, so a +missing build fails. Browser boot, live API traffic, smoke after a provider +switch, and anything the runbook marks as needing provider verification are +`UNVERIFIED`. One `FAIL` makes the drill `FAIL` (exit 1). Otherwise any +`UNVERIFIED` makes it `UNVERIFIED` (exit 3), and the gate is not `PASS`. So +without provider evidence the production stage stays open. That is the intended +result: a local drill cannot show that the promoted release can be rolled back. +FE-GATE-016 (`rollback-drill`) is retired because it ran the same FE-RB-005 +drill as FE-GATE-025 (`RETIRED_GATE_IDS` in `scripts/contracts/ci-gates.ts`). This keeps gate evidence lookup independent from slashes or other provider-specific characters in `RELEASE_ID`, while each workflow artifact remains scoped to its own run. diff --git a/scripts/check-ci-contract.ts b/scripts/check-ci-contract.ts index 5fb777ea..c35dfbca 100644 --- a/scripts/check-ci-contract.ts +++ b/scripts/check-ci-contract.ts @@ -8,6 +8,7 @@ import { type GateResult, } from "./lib/promotion-readiness.ts"; import { + CANONICAL_GATE_IDS, loadCiGateContract, indexCiGateContract, } from "./contracts/ci-gates.ts"; @@ -104,10 +105,7 @@ if (!architecture?.evidenceArtifactIds.some((id) => index.artifacts.get(id)?.pat failures.push("FE-GATE-010 must publish the typed CI contract report"); } -const expectedGateIds = Array.from( - { length: 28 }, - (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`, -); +const expectedGateIds = CANONICAL_GATE_IDS; const passingResults: Record = Object.fromEntries( expectedGateIds.map((gateId) => [gateId, "PASS"]), ); @@ -168,4 +166,4 @@ if (failures.length > 0) { process.stderr.write(`CI contract failed:\n${failures.join("\n")}\n`); process.exit(1); } -process.stdout.write("CI contract: 28 gates, strict v2 graph and generated workflow model PASS\n"); +process.stdout.write(`CI contract: ${CANONICAL_GATE_IDS.length} gates, strict v2 graph and generated workflow model PASS\n`); diff --git a/scripts/contracts/ci-gates.ts b/scripts/contracts/ci-gates.ts index 18f8bc85..648e6429 100644 --- a/scripts/contracts/ci-gates.ts +++ b/scripts/contracts/ci-gates.ts @@ -438,6 +438,19 @@ export type CiGateContractIndex = Readonly<{ jobs: ReadonlyMap; retentionClasses: ReadonlyMap; }>; + +/** + * D-14. FE-GATE-016 (rollback-drill) ran the same FE-RB-005 drill as + * FE-GATE-025 and counted it twice toward production readiness. Its id stays + * retired so no later gate reuses it with a different meaning. + */ +export const RETIRED_GATE_IDS: readonly string[] = Object.freeze(["FE-GATE-016"]); + +export const CANONICAL_GATE_IDS: readonly string[] = Object.freeze( + Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`) + .filter((gateId) => !RETIRED_GATE_IDS.includes(gateId)), +); + /** * D-13. The contract's size and gate shape are a reviewed baseline, not code * constants: a project that removes a sample or adds a gate records the new @@ -789,12 +802,9 @@ function validateContractSemantics( issue(`stage dependency cycle: ${cycle.join(" -> ")}`); } - const expectedGateIds = Array.from( - { length: 28 }, - (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`, - ); + const expectedGateIds = CANONICAL_GATE_IDS; if (JSON.stringify(contract.gates.map(({ id }) => id)) !== JSON.stringify(expectedGateIds)) { - issue("gate registry must contain FE-GATE-001..028 in canonical order"); + issue(`gate registry must contain FE-GATE-001..028 without retired ${RETIRED_GATE_IDS.join(", ")} in canonical order`); } const expectedStages: ReadonlyArray = [ ["merge", "MERGE_READY", [], PROMOTION_FORMULA.MERGE_READY], @@ -869,7 +879,7 @@ function validateContractSemantics( vulnerability_provider: [], provenance_provider: [], promotion: [], - production_gate: ["FE-GATE-016", "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"], + production_gate: ["FE-GATE-021", "FE-GATE-022", "FE-GATE-023", "FE-GATE-024", "FE-GATE-025"], field_gate: ["FE-GATE-018"], documentation_gate: ["FE-GATE-017"], }; diff --git a/scripts/contracts/release-artifacts.ts b/scripts/contracts/release-artifacts.ts index f459f995..2bfae290 100644 --- a/scripts/contracts/release-artifacts.ts +++ b/scripts/contracts/release-artifacts.ts @@ -718,31 +718,67 @@ export const releaseVerificationArtifactSchema = z }) .strict(); +// 11-01 / 13-03 (D-14): each verdict is PASS or FAIL only when observed on the +// local build; anything that needs a boot, live traffic or the provider is +// UNVERIFIED, and so is a drill with provider scope. +const drillVerdict = z.enum(["PASS", "FAIL", "UNVERIFIED"]); + +export type DrillVerdict = z.infer; + +/** FAIL beats UNVERIFIED beats PASS; provider scope alone keeps a drill UNVERIFIED. */ +export function combineDrillVerdicts( + verdicts: readonly DrillVerdict[], + providerVerificationRequired: boolean, +): DrillVerdict { + if (verdicts.includes("FAIL")) return "FAIL"; + if (verdicts.includes("UNVERIFIED") || providerVerificationRequired) return "UNVERIFIED"; + return "PASS"; +} + export const runbookRecordArtifactSchema = z .object({ - schemaVersion: z.literal(1), + schemaVersion: z.literal(2), runbookId: z.string().regex(/^FE-RB-00[1-5]$/u), releaseId: nonEmptyString, drillTimestamp: timestamp, triggerInjected: nonEmptyString, - triggerAsserted: z.boolean(), - containmentAsserted: z.boolean(), - escalationPathAsserted: z.boolean(), + triggerAsserted: drillVerdict, + containmentAsserted: drillVerdict, + escalationPathAsserted: drillVerdict, recoveryAssertions: z.array( z .object({ assertion: nonEmptyString, evidence: nonEmptyString, - passed: z.boolean(), + result: drillVerdict, }) .strict(), ), - negativeFixtureFailedAsExpected: z.boolean(), + negativeFixtureFailedAsExpected: drillVerdict, windowObservedBucket: nonEmptyString, providerVerificationRequired: z.boolean(), - passed: z.boolean(), + result: drillVerdict, }) - .strict(); + .strict() + .superRefine((record, context) => { + const expected = combineDrillVerdicts( + [ + record.triggerAsserted, + record.containmentAsserted, + record.escalationPathAsserted, + ...record.recoveryAssertions.map(({ result }) => result), + record.negativeFixtureFailedAsExpected, + ], + record.providerVerificationRequired, + ); + if (record.result !== expected) { + context.addIssue({ + code: "custom", + path: ["result"], + message: `result must be ${expected} for these verdicts`, + }); + } + }); const failureList = z.array(nonEmptyString).max(4_096); const sourceOrFixtureMode = z.enum(["source", "negative-fixture"]); diff --git a/scripts/drill-runbook.ts b/scripts/drill-runbook.ts index 1c1a5a62..43444f7d 100644 --- a/scripts/drill-runbook.ts +++ b/scripts/drill-runbook.ts @@ -1,4 +1,5 @@ import { access, mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; import { pathToFileURL } from "node:url"; import { shouldRetry } from "../src/adapters/http/retry-policy.ts"; @@ -15,21 +16,30 @@ import { } from "../src/contracts/release-artifacts.ts"; import { projectTelemetryEvent } from "../src/contracts/telemetry.ts"; import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts"; -import { runbookRecordArtifactSchema } from "./contracts/release-artifacts.ts"; +import { + combineDrillVerdicts, + runbookRecordArtifactSchema, + type DrillVerdict, +} from "./contracts/release-artifacts.ts"; import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; +// 11-01 / 13-03 (D-14). A verdict is PASS or FAIL only when the drill observed +// it on the local build. What needs a browser boot, live traffic or the hosting +// provider is UNVERIFIED, and so is every drill whose runbook needs provider +// verification: a local drill never claims the release can be recovered. + type RecoveryAssertion = Readonly<{ assertion: string; evidence: string; - passed: boolean; + result: DrillVerdict; }>; type DrillResult = Readonly<{ - triggerAsserted: boolean; - containmentAsserted: boolean; + triggerAsserted: DrillVerdict; + containmentAsserted: DrillVerdict; recoveryAssertions: RecoveryAssertion[]; - negativeFixtureFailedAsExpected: boolean; + negativeFixtureFailedAsExpected: DrillVerdict; providerVerificationRequired: boolean; }>; @@ -51,10 +61,8 @@ type RunbookDocument = Readonly<{ export type JsonArtifactReader = (path: string) => Promise; export type RollbackArtifactPaths = Readonly<{ - primaryRelease: string; - fallbackRelease: string; - primaryRuntime: string; - fallbackRuntime: string; + release: string; + runtime: string; }>; export type RollbackCoherenceOptions = Readonly<{ @@ -63,27 +71,37 @@ export type RollbackCoherenceOptions = Readonly<{ paths?: RollbackArtifactPaths; }>; +export type DrillOptions = Readonly<{ + /** Directory holding `dist/` and `config/runbooks/`. Defaults to the cwd. */ + root?: string; + contractPackages?: readonly InstalledContractPackageIdentity[]; +}>; + +type DrillContext = Readonly<{ + root: string; + contractPackages: readonly InstalledContractPackageIdentity[]; +}>; + +// 13-03: the drill reads the built release only. It used to fall back to the +// placeholders in public/, so it passed without any build at all. const DEFAULT_ROLLBACK_PATHS = Object.freeze({ - primaryRelease: "dist/release-manifest.json", - fallbackRelease: "public/release-manifest.json", - primaryRuntime: "dist/config.json", - fallbackRuntime: "public/config.json", + release: "dist/release-manifest.json", + runtime: "dist/config.json", }); async function readJsonArtifact(path: string): Promise { - return JSON.parse(await readFile(path, "utf8")); + try { + return JSON.parse(await readFile(path, "utf8")); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) { + throw new Error(`${path} is missing; build the release before running the drill`, { cause: error }); + } + throw error; + } } -async function releaseManifest( - readArtifact: JsonArtifactReader = readJsonArtifact, - paths: RollbackArtifactPaths = DEFAULT_ROLLBACK_PATHS, -): Promise { - const value = await readPrimaryOrFallback( - readArtifact, - paths.primaryRelease, - paths.fallbackRelease, - ); - return parseReleaseArtifact(value); +function rootedReader(root: string): JsonArtifactReader { + return (file) => readJsonArtifact(path.join(root, file)); } const validConfig = { @@ -100,12 +118,48 @@ const validConfig = { RELEASE_ID: "local-release", }; +function verdict(passed: boolean): DrillVerdict { + return passed ? "PASS" : "FAIL"; +} + function assertion( assertion: string, evidence: string, - passed: boolean, + result: DrillVerdict, ): RecoveryAssertion { - return { assertion, evidence, passed }; + return { assertion, evidence, result }; +} + +/** An assertion the local drill cannot observe; `needs` names what can. */ +function notObserved(name: string, needs: string): RecoveryAssertion { + return assertion(name, `not observed locally; needs ${needs}`, "UNVERIFIED"); +} + +/** + * Every file the Vite manifest names (entries, lazy chunks, CSS, assets) exists + * in the local dist. Hosting reachability still needs the provider smoke. + */ +async function builtAssetsPresent(root: string): Promise { + const manifest = await readJsonArtifact(path.join(root, "dist/.vite/manifest.json")); + if (!manifest || typeof manifest !== "object") return false; + const files = new Set(); + for (const chunk of Object.values(manifest as Record>)) { + for (const key of ["file", "css", "assets"] as const) { + const value = chunk[key]; + for (const file of Array.isArray(value) ? value : [value]) { + if (typeof file === "string") files.add(file); + } + } + } + if (![...files].some((file) => file.endsWith(".js"))) return false; + try { + await Promise.all( + ["index.html", ...files].map((file) => access(path.join(root, "dist", file))), + ); + return true; + } catch { + return false; + } } async function drillBoot(): Promise { @@ -115,19 +169,22 @@ async function drillBoot(): Promise { API_BASE_URL: "http://insecure.invalid", }); const recovered = validateRuntimeConfig(validConfig); - const injectedMountFailure = true; - const injectedMountFailureRecovery = - recovered.success && !injectedMountFailure; return { - triggerAsserted: !invalid.success, - containmentAsserted: !invalid.success, + triggerAsserted: verdict(!invalid.success), + containmentAsserted: verdict(!invalid.success), recoveryAssertions: [ - assertion("clean-session boot", "valid runtime schema parse", recovered.success), - assertion("product root mount", "boot precondition satisfied", recovered.success), - assertion("config validation", "invalid fixture rejected", !invalid.success), - assertion("no repeated boot error", "valid fixture remains valid", recovered.success), + notObserved("clean-session boot", "a browser boot of the promoted release"), + notObserved("product root mount", "a browser boot of the promoted release"), + assertion( + "config validation", + "invalid fixture rejected, valid fixture accepted", + verdict(!invalid.success && recovered.success), + ), + notObserved("no repeated boot error", "boot telemetry from the promoted release"), ], - negativeFixtureFailedAsExpected: !injectedMountFailureRecovery, + // The runbook's negative fixture injects a mount failure after a valid + // config; that needs a mounted app, which a Node drill does not have. + negativeFixtureFailedAsExpected: "UNVERIFIED", providerVerificationRequired: false, }; } @@ -144,7 +201,7 @@ function memoryStorage(): StoragePort { }; } -async function drillChunkMismatch(): Promise { +async function drillChunkMismatch(context: DrillContext): Promise { const storage = memoryStorage(); const input: Parameters[0] = { failureKind: "DEPLOY_MISMATCH", @@ -157,29 +214,26 @@ async function drillChunkMismatch(): Promise { }; const first = decideChunkRecovery(input); const second = decideChunkRecovery(input); - const manifest = await releaseManifest(); - let assetsReachable = true; - try { - await access("dist/index.html"); - await access("dist/.vite/manifest.json"); - } catch { - assetsReachable = false; - } + const { coherence } = await verifyRollbackReleaseCoherence({ + readArtifact: rootedReader(context.root), + contractPackages: context.contractPackages, + }); return { - triggerAsserted: first.action === "reload-once", - containmentAsserted: + triggerAsserted: verdict(first.action === "reload-once"), + containmentAsserted: verdict( first.action === "reload-once" && second.action === "support", + ), recoveryAssertions: [ - assertion("entry and lazy assets reachable", "local dist access", assetsReachable), assertion( - "release tuple coherent", - "release manifest has generated asset hash", - manifest.assetManifestHash !== "generated-during-build", + "entry and lazy assets reachable", + "every Vite manifest file exists in the local dist", + verdict(await builtAssetsPresent(context.root)), ), - assertion("second reload blocked", "reload guard decision", second.action === "support"), - assertion("critical route smoke", "built index available", assetsReachable), + assertion("release tuple coherent", "build/config/manifest tuple", verdict(coherence.compatible)), + assertion("second reload blocked", "reload guard decision", verdict(second.action === "support")), + notObserved("critical route smoke", "a browser smoke against the hosted release"), ], - negativeFixtureFailedAsExpected: second.action !== "reload-once", + negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"), providerVerificationRequired: true, }; } @@ -196,15 +250,20 @@ async function drillApiDegradation(): Promise { 0, ); return { - triggerAsserted: true, - containmentAsserted: !unkeyedRetry, + // The trigger is a failure rate over live traffic. + triggerAsserted: "UNVERIFIED", + containmentAsserted: verdict(!unkeyedRetry && safeRetry), recoveryAssertions: [ - assertion("failure rate at baseline", "deterministic recovery window", true), - assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry), - assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry), - assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)), + notObserved("terminal failure rate at baseline", "API failure telemetry from the promoted release"), + assertion("no retry amplification", "unkeyed retry policy", verdict(!unkeyedRetry)), + notObserved("critical read/write smoke", "a read and a write against the live API"), + assertion( + "schema fixtures", + "schema mismatch is not retryable", + verdict(!shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)), + ), ], - negativeFixtureFailedAsExpected: !unkeyedRetry, + negativeFixtureFailedAsExpected: verdict(!unkeyedRetry), providerVerificationRequired: true, }; } @@ -224,7 +283,10 @@ async function drillTelemetry(): Promise { attempt_count_bucket: "1", route_id: "APP_HOME", }); - await adapter.flush(); + const flushResolved = await adapter.flush().then( + () => true, + () => false, + ); const projected = projectTelemetryEvent("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", @@ -235,22 +297,25 @@ async function drillTelemetry(): Promise { const redacted = projected.success && !JSON.stringify(projected).includes("raw_url"); return { - triggerAsserted: adapter.droppedCount() === 1, - containmentAsserted: adapter.pendingCount() === 0, + triggerAsserted: verdict(adapter.droppedCount() === 1), + containmentAsserted: verdict(adapter.pendingCount() === 0), recoveryAssertions: [ - assertion("product flow unaffected", "adapter flush resolves", true), - assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1), - assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0), - assertion("forbidden attributes absent", "default-deny projection", redacted), + assertion("product flow unaffected", "adapter flush resolves despite the sink failure", verdict(flushResolved)), + assertion("delivery self-check", "sink failure counted", verdict(adapter.droppedCount() === 1)), + assertion("queue drained within bound", "pending queue count", verdict(adapter.pendingCount() === 0)), + assertion("forbidden attributes absent", "default-deny projection", verdict(redacted)), ], - negativeFixtureFailedAsExpected: redacted, + negativeFixtureFailedAsExpected: verdict(redacted), providerVerificationRequired: true, }; } -async function drillRollback(): Promise { - const verified = await verifyRollbackReleaseCoherence(); - const { release, coherence: coherent } = verified; +async function drillRollback(context: DrillContext): Promise { + const { release, coherence: coherent } = await verifyRollbackReleaseCoherence({ + readArtifact: rootedReader(context.root), + contractPackages: context.contractPackages, + }); + const assetsPresent = await builtAssetsPresent(context.root); const mixed = verifyCompatibilityTuple({ frontend: { buildId: "build-a", @@ -268,20 +333,25 @@ async function drillRollback(): Promise { }, }); return { - triggerAsserted: true, - containmentAsserted: coherent.compatible, + // A release-blocking defect is declared by a person during an incident. + triggerAsserted: "UNVERIFIED", + containmentAsserted: verdict(coherent.compatible && assetsPresent), recoveryAssertions: [ - assertion("compatibility gate", "typed version comparison", coherent.compatible), - assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible), - assertion("critical smoke", "built or public runtime set parsed", true), - assertion("release ID in timeline", "drill record releaseId", Boolean(release.releaseId)), + assertion("compatibility gate", "typed version comparison", verdict(coherent.compatible)), + assertion( + "release coherence gate", + "build/config/manifest tuple and every Vite manifest file in the local dist", + verdict(coherent.compatible && assetsPresent), + ), + notObserved("critical smoke", "a browser smoke after the provider switches to the prior release"), + assertion("release ID in timeline", "drill record releaseId", verdict(Boolean(release.releaseId))), ], - negativeFixtureFailedAsExpected: !mixed.compatible, + negativeFixtureFailedAsExpected: verdict(!mixed.compatible), providerVerificationRequired: true, }; } -const drillById: Record Promise> = { +const drillById: Record Promise> = { "FE-RB-001": drillBoot, "FE-RB-002": drillChunkMismatch, "FE-RB-003": drillApiDegradation, @@ -294,12 +364,12 @@ export async function verifyRollbackReleaseCoherence( ) { const readArtifact = options.readArtifact ?? readJsonArtifact; const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS; - const artifactPair = await selectRollbackArtifactPair( - readArtifact, - paths, - ); - const release = parseReleaseArtifact(artifactPair.release); - const runtimeArtifact = parseRuntimeConfigArtifact(artifactPair.runtime); + const [releaseValue, runtimeValue] = await Promise.all([ + readArtifact(paths.release), + readArtifact(paths.runtime), + ]); + const release = parseReleaseArtifact(releaseValue); + const runtimeArtifact = parseRuntimeConfigArtifact(runtimeValue); const runtime = { ...runtimeArtifact, BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"), @@ -317,43 +387,60 @@ export async function verifyRollbackReleaseCoherence( return Object.freeze({ release, runtime, coherence }); } +export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) { + const context: DrillContext = { + root: options.root ?? process.cwd(), + contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES, + }; + const document = JSON.parse( + await readFile(path.join(context.root, "config/runbooks/runbooks.json"), "utf8"), + ) as RunbookDocument; + const specification = document.runbooks[runbookId]; + const drill = drillById[runbookId]; + if (!specification || !drill) throw new RangeError(`unknown runbook: ${runbookId}`); + + const release = parseReleaseArtifact( + await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release), + ); + const result = await drill(context); + const escalationPathAsserted = verdict(specification.escalation.length >= 2); + return { + schemaVersion: 2 as const, + runbookId, + gateId: specification.gateId, + releaseId: release.releaseId, + drillTimestamp: new Date().toISOString(), + triggerInjected: specification.triggerKinds[0] ?? "", + triggerAsserted: result.triggerAsserted, + containmentAsserted: result.containmentAsserted, + escalationPathAsserted, + recoveryAssertions: result.recoveryAssertions, + negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected, + windowObservedBucket: specification.window, + providerVerificationRequired: result.providerVerificationRequired, + result: combineDrillVerdicts( + [ + result.triggerAsserted, + result.containmentAsserted, + escalationPathAsserted, + ...result.recoveryAssertions.map((item) => item.result), + result.negativeFixtureFailedAsExpected, + ], + result.providerVerificationRequired, + ), + }; +} + async function main(): Promise { const runbookId = process.argv .slice(2) .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); - const document = JSON.parse( - await readFile("config/runbooks/runbooks.json", "utf8"), - ) as RunbookDocument; - const specification = runbookId ? document.runbooks[runbookId] : undefined; - if (!runbookId || !specification) { + if (!runbookId) { process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n"); process.exit(2); } - const drill = await drillById[runbookId](); - const escalationPathAsserted = specification.escalation.length >= 2; - const passed = - drill.triggerAsserted && - drill.containmentAsserted && - escalationPathAsserted && - drill.recoveryAssertions.every((item) => item.passed) && - drill.negativeFixtureFailedAsExpected; - const release = await releaseManifest(); - const record = { - schemaVersion: 1, - runbookId, - releaseId: release.releaseId, - drillTimestamp: new Date().toISOString(), - triggerInjected: specification.triggerKinds[0], - triggerAsserted: drill.triggerAsserted, - containmentAsserted: drill.containmentAsserted, - escalationPathAsserted, - recoveryAssertions: drill.recoveryAssertions, - negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected, - windowObservedBucket: specification.window, - providerVerificationRequired: drill.providerVerificationRequired, - passed, - }; + const { gateId, ...record } = await runRunbookDrill(runbookId); const artifactDirectory = `artifacts/runbooks/${runbookId}`; await mkdir(artifactDirectory, { recursive: true }); await writeValidatedJsonArtifact({ @@ -361,15 +448,25 @@ async function main(): Promise { schema: runbookRecordArtifactSchema, value: record, }); - if (!passed) { - process.stderr.write(`${runbookId} drill failed.\n`); + if (record.result === "FAIL") { + process.stderr.write(`${runbookId} drill: FAIL (${gateId})\n`); process.exit(1); } - process.stdout.write( - `${runbookId} drill: PASS (${specification.gateId}; provider verification ${ - drill.providerVerificationRequired ? "still required" : "not required" - })\n`, - ); + if (record.result === "UNVERIFIED") { + // Not a pass: the gate stays red until provider evidence exists (D-14). + const open = [ + ["trigger", record.triggerAsserted], + ["containment", record.containmentAsserted], + ...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const), + ["negative fixture", record.negativeFixtureFailedAsExpected], + ["provider verification", record.providerVerificationRequired ? "UNVERIFIED" : "PASS"], + ].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name); + process.stderr.write( + `${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")})\n`, + ); + process.exit(3); + } + process.stdout.write(`${runbookId} drill: PASS (${gateId})\n`); } function requireIdentity(value: string | undefined, label: string): string { @@ -379,88 +476,6 @@ function requireIdentity(value: string | undefined, label: string): string { return value; } -async function selectRollbackArtifactPair( - readArtifact: JsonArtifactReader, - paths: RollbackArtifactPaths, -): Promise> { - const primary = await readArtifactPair( - readArtifact, - paths.primaryRelease, - paths.primaryRuntime, - ); - if ( - primary.release.status === "fulfilled" && - primary.runtime.status === "fulfilled" - ) { - return Object.freeze({ - release: primary.release.value, - runtime: primary.runtime.value, - }); - } - - const releaseMissing = - primary.release.status === "rejected" && - hasErrorCode(primary.release.reason, "ENOENT"); - const runtimeMissing = - primary.runtime.status === "rejected" && - hasErrorCode(primary.runtime.reason, "ENOENT"); - if (releaseMissing && runtimeMissing) { - const fallback = await readArtifactPair( - readArtifact, - paths.fallbackRelease, - paths.fallbackRuntime, - ); - if (fallback.release.status === "rejected") { - throw fallback.release.reason; - } - if (fallback.runtime.status === "rejected") { - throw fallback.runtime.reason; - } - return Object.freeze({ - release: fallback.release.value, - runtime: fallback.runtime.value, - }); - } - - if (primary.release.status === "rejected" && !releaseMissing) { - throw primary.release.reason; - } - if (primary.runtime.status === "rejected" && !runtimeMissing) { - throw primary.runtime.reason; - } - throw new Error("primary rollback artifact pair is incomplete"); -} - -async function readArtifactPair( - readArtifact: JsonArtifactReader, - releasePath: string, - runtimePath: string, -): Promise< - Readonly<{ - release: PromiseSettledResult; - runtime: PromiseSettledResult; - }> -> { - const [release, runtime] = await Promise.allSettled([ - Promise.resolve().then(() => readArtifact(releasePath)), - Promise.resolve().then(() => readArtifact(runtimePath)), - ]); - return Object.freeze({ release, runtime }); -} - -async function readPrimaryOrFallback( - readArtifact: JsonArtifactReader, - primary: string, - fallback: string, -): Promise { - try { - return await readArtifact(primary); - } catch (error) { - if (!hasErrorCode(error, "ENOENT")) throw error; - return readArtifact(fallback); - } -} - function hasErrorCode(error: unknown, code: string): boolean { return Boolean( error && diff --git a/scripts/lib/ci-contract-report.ts b/scripts/lib/ci-contract-report.ts index e208d668..3aaf1d68 100644 --- a/scripts/lib/ci-contract-report.ts +++ b/scripts/lib/ci-contract-report.ts @@ -4,7 +4,7 @@ export const ciContractReportSchema = z .object({ schemaVersion: z.literal(2), nodeVersion: z.string().regex(/^\d+\.\d+\.\d+$/u), - gateCount: z.literal(28), + gateCount: z.literal(27), commandDefinitionCount: z.number().int().positive(), commandReferenceCount: z.number().int().positive(), artifactCount: z.number().int().positive(), diff --git a/scripts/lib/promotion-readiness.ts b/scripts/lib/promotion-readiness.ts index 9d265741..607f1649 100644 --- a/scripts/lib/promotion-readiness.ts +++ b/scripts/lib/promotion-readiness.ts @@ -28,7 +28,7 @@ export const PROMOTION_FORMULA = Object.freeze({ "FE-GATE-028", ]), PROD_PROMOTION_READY: Object.freeze([ - "FE-GATE-016", + // FE-GATE-016 is retired (D-14): it repeated FE-GATE-025's FE-RB-005 drill. "FE-GATE-021", "FE-GATE-022", "FE-GATE-023", diff --git a/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap b/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap index a77fc42d..d85bd21f 100644 --- a/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap +++ b/tests/unit/__snapshots__/ci-workflow-generation.test.ts.snap @@ -382,7 +382,6 @@ jobs: fail-fast: false matrix: include: - - { gate: FE-GATE-016, name: rollback-drill } - { gate: FE-GATE-021, name: runbook-boot-config } - { gate: FE-GATE-022, name: runbook-chunk-mismatch } - { gate: FE-GATE-023, name: runbook-api-degradation } diff --git a/tests/unit/ci-workflow-generation.test.ts b/tests/unit/ci-workflow-generation.test.ts index 4c4e3105..763a7269 100644 --- a/tests/unit/ci-workflow-generation.test.ts +++ b/tests/unit/ci-workflow-generation.test.ts @@ -191,14 +191,14 @@ describe("CI gate contract", () => { expect(contract.gates.map(({ id }) => id)).toEqual( Array.from({ length: 28 }, (_, index) => `FE-GATE-${String(index + 1).padStart(3, "0")}`, - ), + ).filter((gateId) => gateId !== "FE-GATE-016"), ); expect(contract.jobs).toHaveLength(9); expect(contract.commands).toHaveLength(94); - expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(106); + expect(contract.gates.reduce((total, gate) => total + gate.commandIds.length, 0)).toBe(104); expect(contract.commands.filter(({ expect }) => expect === "fail")).toHaveLength(31); - expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(88); - expect(contract.artifacts).toHaveLength(110); + expect(contract.gates.reduce((total, gate) => total + gate.evidenceArtifactIds.length, 0)).toBe(87); + expect(contract.artifacts).toHaveLength(109); expect(contract.stages).toHaveLength(5); expect(contract.retention.classes).toHaveLength(5); expect(index.gates.get("FE-GATE-015")?.commandIds).toHaveLength(2); diff --git a/tests/unit/promotion-readiness.test.ts b/tests/unit/promotion-readiness.test.ts index 0984ae69..1d6e6b35 100644 --- a/tests/unit/promotion-readiness.test.ts +++ b/tests/unit/promotion-readiness.test.ts @@ -25,7 +25,7 @@ describe("promotion readiness formula", () => { it.each([ ["FE-GATE-001", "MERGE_READY"], ["FE-GATE-012", "RELEASE_READY"], - ["FE-GATE-016", "PROD_PROMOTION_READY"], + ["FE-GATE-021", "PROD_PROMOTION_READY"], ["FE-GATE-018", "FIELD_SLO_READY"], ["FE-GATE-017", "DOCUMENTATION_READY"], ] as const)("fails closed when %s fails", (failedGate, readiness) => { diff --git a/tests/unit/release-coherence.test.ts b/tests/unit/release-coherence.test.ts index 96160c78..d4edab6c 100644 --- a/tests/unit/release-coherence.test.ts +++ b/tests/unit/release-coherence.test.ts @@ -281,32 +281,32 @@ describe("release coherence", () => { } }); - it("falls back to public rollback artifacts only when primary dist is absent", async () => { + // 13-03: the rollback drill checks the built release only. Reading the + // public/ placeholders when dist was absent let it pass without any build. + it("reads only the built dist pair and never falls back to public placeholders", async () => { const exactV2 = await releaseV2With(contractPackages); - const tamperedPrimary = await releaseV2With(contractPackages.slice(0, 1)); const validPublic = { "public/release-manifest.json": exactV2, "public/config.json": runtimeV2, }; - const unavailablePrimary = [ - new SyntaxError("invalid primary JSON"), - Object.assign(new Error("unreadable primary"), { code: "EACCES" }), - { ...exactV2, unexpected: "tampered" }, - ]; - - for (const primaryFailure of unavailablePrimary) { + for (const primary of [ + {}, + { "dist/config.json": runtimeV2 }, + { "dist/release-manifest.json": exactV2 }, + { "dist/release-manifest.json": new SyntaxError("invalid primary JSON"), "dist/config.json": runtimeV2 }, + { "dist/release-manifest.json": { ...exactV2, unexpected: "tampered" }, "dist/config.json": runtimeV2 }, + ]) { + const reads: string[] = []; await expect( verifyRollbackReleaseCoherence({ - readArtifact: artifactReader({ - "dist/release-manifest.json": primaryFailure, - "dist/config.json": runtimeV2, - ...validPublic, - }), + readArtifact: artifactReader({ ...primary, ...validPublic }, reads), contractPackages, }), ).rejects.toBeDefined(); + expect(reads.some((read) => read.startsWith("public/"))).toBe(false); } + const tamperedPrimary = await releaseV2With(contractPackages.slice(0, 1)); await expect( verifyRollbackReleaseCoherence({ readArtifact: artifactReader({ @@ -317,55 +317,5 @@ describe("release coherence", () => { contractPackages, }), ).resolves.toMatchObject({ coherence: { compatible: false } }); - - const releaseOnlyMissingReads: string[] = []; - await expect( - verifyRollbackReleaseCoherence({ - readArtifact: artifactReader( - { - "dist/config.json": runtimeV2, - ...validPublic, - }, - releaseOnlyMissingReads, - ), - contractPackages, - }), - ).rejects.toThrow("primary rollback artifact pair"); - expect(releaseOnlyMissingReads).toEqual([ - "dist/release-manifest.json", - "dist/config.json", - ]); - - const runtimeOnlyMissingReads: string[] = []; - await expect( - verifyRollbackReleaseCoherence({ - readArtifact: artifactReader( - { - "dist/release-manifest.json": exactV2, - ...validPublic, - }, - runtimeOnlyMissingReads, - ), - contractPackages, - }), - ).rejects.toThrow("primary rollback artifact pair"); - expect(runtimeOnlyMissingReads).toEqual([ - "dist/release-manifest.json", - "dist/config.json", - ]); - - const missingPairReads: string[] = []; - await expect( - verifyRollbackReleaseCoherence({ - readArtifact: artifactReader(validPublic, missingPairReads), - contractPackages, - }), - ).resolves.toMatchObject({ coherence: { compatible: true } }); - expect(missingPairReads).toEqual([ - "dist/release-manifest.json", - "dist/config.json", - "public/release-manifest.json", - "public/config.json", - ]); }); }); diff --git a/tests/unit/runbook-drill.test.ts b/tests/unit/runbook-drill.test.ts new file mode 100644 index 00000000..0a474d6f --- /dev/null +++ b/tests/unit/runbook-drill.test.ts @@ -0,0 +1,134 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { runRunbookDrill } from "../../scripts/drill-runbook.ts"; +import { computeContractSetDigest } from "../../src/contracts/contract-set-canonical.ts"; +import type { InstalledContractPackageIdentity } from "../../src/contracts/external-contract-runtime.ts"; + +// 11-01 / 13-03 (D-14). A drill reports PASS only for what it observed on the +// local build. Boot, live traffic, smoke and the provider switch are not +// observable here, so a drill with provider scope ends UNVERIFIED, never PASS. + +const contractPackages = [ + { + packageId: "@example/accounts", + version: "1.2.3", + digest: `sha256:${"1".repeat(64)}`, + runtimeProtocolVersion: 1, + sourceRevision: "a".repeat(40), + }, +] as const satisfies readonly InstalledContractPackageIdentity[]; + +const RUNBOOK_IDS = ["FE-RB-001", "FE-RB-002", "FE-RB-003", "FE-RB-004", "FE-RB-005"] as const; +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function builtDist(options: Readonly<{ withoutJs?: boolean; withoutDist?: boolean }> = {}) { + const root = await mkdtemp(path.join(tmpdir(), "runbook-drill-")); + roots.push(root); + await mkdir(path.join(root, "config/runbooks"), { recursive: true }); + await writeFile( + path.join(root, "config/runbooks/runbooks.json"), + await readFile("config/runbooks/runbooks.json"), + ); + if (options.withoutDist) return root; + const write = async (file: string, content: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }); + await writeFile(path.join(root, file), content); + }; + await write("dist/index.html", ""); + await write( + "dist/.vite/manifest.json", + JSON.stringify({ + "index.html": { file: "assets/index-a.js", isEntry: true, css: ["assets/index-a.css"], imports: ["_shared-b.js"] }, + "_shared-b.js": { file: "assets/shared-b.js" }, + "src/pages/lazy.tsx": { file: "assets/lazy-c.js", isDynamicEntry: true }, + }), + ); + await write("dist/assets/index-a.css", "body{}"); + if (!options.withoutJs) { + for (const file of ["assets/index-a.js", "assets/shared-b.js", "assets/lazy-c.js"]) { + await write(`dist/${file}`, "export {};"); + } + } + await write( + "dist/release-manifest.json", + JSON.stringify({ + schemaVersion: 2, + appVersion: "1.0.0", + buildId: "build-a", + commitSha: "abc1234", + configSchemaVersion: "2.0", + assetManifestHash: "assets-a", + releaseId: "release-a", + builtAt: "2026-08-01T00:00:00.000Z", + routeChunks: {}, + contractSet: { + setAlgorithm: "CA_CONTRACT_SET_V1", + setDigest: await computeContractSetDigest(contractPackages), + packages: [...contractPackages], + }, + }), + ); + await write( + "dist/config.json", + JSON.stringify({ + APP_ENV: "local", + API_BASE_URL: "http://localhost:8080/", + REQUEST_TIMEOUT_MS: 10_000, + MAX_RETRY_ATTEMPTS: 2, + TELEMETRY_ENABLED: false, + AUTH_MODE: "external", + CONFIG_SCHEMA_VERSION: "2.0", + RELEASE_MANIFEST_URL: "/release-manifest.json", + BUILD_ID: "build-a", + RELEASE_ID: "release-a", + CAPABILITY_OVERRIDES: { + REALTIME: "DEFAULT", + WEB_WORKER: "DEFAULT", + SERVICE_WORKER: "DEFAULT", + OFFLINE_COMMANDS: "DEFAULT", + }, + FEATURE_OVERRIDES: {}, + }), + ); + return root; +} + +describe("runbook drill verdicts", () => { + it.each(RUNBOOK_IDS)("%s on a complete local build is UNVERIFIED, not PASS", async (runbookId) => { + const record = await runRunbookDrill(runbookId, { root: await builtDist(), contractPackages }); + expect(record.result).toBe("UNVERIFIED"); + expect(record.recoveryAssertions.some(({ result }) => result === "FAIL")).toBe(false); + }); + + it.each(["FE-RB-002", "FE-RB-005"] as const)( + "%s fails when the build lost its JavaScript assets", + async (runbookId) => { + const record = await runRunbookDrill(runbookId, { + root: await builtDist({ withoutJs: true }), + contractPackages, + }); + expect(record.result).toBe("FAIL"); + }, + ); + + it("FE-RB-005 fails without a build instead of reading public/ placeholders", async () => { + await expect( + runRunbookDrill("FE-RB-005", { root: await builtDist({ withoutDist: true }), contractPackages }), + ).rejects.toThrow(/dist\/release-manifest\.json/u); + }); + + it("keeps no constant recovery verdict in the drill source", async () => { + const source = await readFile("scripts/drill-runbook.ts", "utf8"); + expect(source).not.toMatch(/assertion\([^;]*,\s*true\s*\)/u); + expect(source).not.toMatch(/Asserted:\s*true\b/u); + expect(source).not.toMatch(/injectedMountFailure\s*=\s*true/u); + }); +});