diff --git a/config/runbooks/runbooks.json b/config/runbooks/runbooks.json new file mode 100644 index 0000000..ef120df --- /dev/null +++ b/config/runbooks/runbooks.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": 1, + "runbooks": { + "FE-RB-001": { + "title": "Boot configuration failure", + "gateId": "FE-GATE-021", + "triggerKinds": ["BOOT_CONFIG_FAILURE"], + "containment": "stop product route mount, show the safe support shell, and refetch at most once", + "window": "owner triage planned-default 5m", + "escalation": ["env-config owner", "release owner"], + "recoveryEvidence": [ + "clean-session boot", + "product root mount", + "config validation", + "no repeated boot error" + ], + "negativeFixture": "a valid config followed by an injected mount failure must fail recovery" + }, + "FE-RB-002": { + "title": "Chunk, manifest, or deployment mismatch", + "gateId": "FE-GATE-022", + "triggerKinds": [ + "CHUNK_LOAD_FAILURE", + "RELEASE_MANIFEST_FAILURE", + "DEPLOY_MISMATCH" + ], + "containment": "warn for dirty state, fetch manifest no-store once, and allow one guarded reload", + "window": "release owner triage planned-default 5m", + "escalation": ["release-cache owner", "hosting/CDN owner"], + "recoveryEvidence": [ + "entry and lazy assets reachable", + "release tuple coherent", + "second reload blocked", + "critical route smoke" + ], + "negativeFixture": "a second failure for the same release pair must not reload" + }, + "FE-RB-003": { + "title": "Backend API degradation", + "gateId": "FE-GATE-023", + "triggerKinds": [ + "TERMINAL_NETWORK_RATE", + "REQUEST_TIMEOUT_RATE", + "SERVER_FAILURE_RATE", + "SCHEMA_MISMATCH" + ], + "containment": "do not expand retry caps, serve safe stale reads, and never retry an unkeyed mutation", + "window": "rolling 5m trigger; first classification planned-default 10m", + "escalation": [ + "api-client owner", + "backend operation owner", + "release compatibility owner" + ], + "recoveryEvidence": [ + "terminal failure rate at baseline", + "no retry amplification", + "critical read/write smoke", + "schema fixtures" + ], + "negativeFixture": "an unkeyed POST receiving 503 must not retry" + }, + "FE-RB-004": { + "title": "Telemetry sink failure", + "gateId": "FE-GATE-024", + "triggerKinds": ["TELEMETRY_FAILURE"], + "containment": "keep product flow available, bound the queue, and never report recursively to the failing sink", + "window": "platform triage planned-default 15m", + "escalation": ["observability owner", "telemetry platform owner"], + "recoveryEvidence": [ + "product flow unaffected", + "delivery self-check", + "queue drained within bound", + "forbidden attributes absent" + ], + "negativeFixture": "raw URL and query data must be removed from telemetry" + }, + "FE-RB-005": { + "title": "Coherent release rollback", + "gateId": "FE-GATE-025", + "triggerKinds": ["RELEASE_BLOCKING_DEFECT"], + "containment": "select a prior immutable tuple, verify asset/config/API compatibility, atomically switch, and smoke", + "window": "provider recovery target TBD", + "escalation": ["release-cache owner", "release approver/hosting owner"], + "recoveryEvidence": [ + "compatibility gate", + "release coherence gate", + "critical smoke", + "release ID in incident timeline" + ], + "negativeFixture": "HTML build A with asset manifest B must be rejected" + } + } +} diff --git a/config/schemas/runbook-drill-record.schema.json b/config/schemas/runbook-drill-record.schema.json new file mode 100644 index 0000000..d58daba --- /dev/null +++ b/config/schemas/runbook-drill-record.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "ART-FE-RUNBOOK-DRILL@1", + "type": "object", + "required": [ + "schemaVersion", + "runbookId", + "releaseId", + "drillTimestamp", + "triggerInjected", + "triggerAsserted", + "containmentAsserted", + "escalationPathAsserted", + "recoveryAssertions", + "negativeFixtureFailedAsExpected", + "windowObservedBucket", + "passed" + ], + "properties": { + "schemaVersion": { "const": 1 }, + "runbookId": { "pattern": "^FE-RB-00[1-5]$" }, + "releaseId": { "type": "string", "minLength": 1 }, + "drillTimestamp": { "type": "string", "format": "date-time" }, + "triggerInjected": { "type": "string" }, + "triggerAsserted": { "type": "boolean" }, + "containmentAsserted": { "type": "boolean" }, + "escalationPathAsserted": { "type": "boolean" }, + "recoveryAssertions": { + "type": "array", + "minItems": 4, + "items": { + "type": "object", + "required": ["assertion", "evidence", "passed"] + } + }, + "negativeFixtureFailedAsExpected": { "type": "boolean" }, + "windowObservedBucket": { "type": "string" }, + "providerVerificationRequired": { "type": "boolean" }, + "passed": { "type": "boolean" } + }, + "additionalProperties": false +} diff --git a/docs/runbooks/FE-RB-001.md b/docs/runbooks/FE-RB-001.md new file mode 100644 index 0000000..c40abd1 --- /dev/null +++ b/docs/runbooks/FE-RB-001.md @@ -0,0 +1,9 @@ +# FE-RB-001 — Boot configuration failure + +Trigger on `BOOT_CONFIG_FAILURE` after the single bounded refetch fails. Stop +product route mounting and show the safe support shell; the planned owner +triage target is five minutes. Escalate from the environment/config owner to +the release owner. + +Close only after a clean-session boot mounts the product root, config +validation evidence passes, and repeated boot-error telemetry is absent. diff --git a/docs/runbooks/FE-RB-002.md b/docs/runbooks/FE-RB-002.md new file mode 100644 index 0000000..ff522c5 --- /dev/null +++ b/docs/runbooks/FE-RB-002.md @@ -0,0 +1,11 @@ +# FE-RB-002 — Chunk or deployment mismatch + +Trigger on `CHUNK_LOAD_FAILURE`, `RELEASE_MANIFEST_FAILURE`, or +`DEPLOY_MISMATCH`. Warn when dirty state may be lost, fetch the manifest +`no-store` once, record the release pair, and allow only one reload. The +planned release-owner triage target is five minutes. Escalate to the hosting/CDN +owner. + +Close only after entry/lazy assets are reachable, the manifest parses into a +coherent tuple, a second automatic reload is blocked, and the critical route +smoke passes. diff --git a/docs/runbooks/FE-RB-003.md b/docs/runbooks/FE-RB-003.md new file mode 100644 index 0000000..79ad196 --- /dev/null +++ b/docs/runbooks/FE-RB-003.md @@ -0,0 +1,10 @@ +# FE-RB-003 — Backend API degradation + +Trigger when terminal network/timeout/5xx failures exceed the rolling +five-minute threshold or on one `SCHEMA_MISMATCH`. Do not expand client retry +caps, do not retry schema mismatches, and never retry an unkeyed mutation. +Escalate from the API client owner to backend operations and then release +compatibility; the planned first-classification target is ten minutes. + +Close only after the failure rate returns to baseline, retry amplification is +absent, critical read/write smoke passes, and schema fixtures pass. diff --git a/docs/runbooks/FE-RB-004.md b/docs/runbooks/FE-RB-004.md new file mode 100644 index 0000000..fc7c709 --- /dev/null +++ b/docs/runbooks/FE-RB-004.md @@ -0,0 +1,9 @@ +# FE-RB-004 — Telemetry sink failure + +Trigger on sink network/non-2xx errors, queue overflow, or adapter +initialization failure. Keep product flows available, bound the queue, and do +not recursively report to the failed sink. Escalate from observability to the +telemetry platform owner; the planned triage target is fifteen minutes. + +Close only after product e2e remains unaffected, delivery self-check succeeds, +the queue drains within its bound, and the forbidden-attribute scan passes. diff --git a/docs/runbooks/FE-RB-005.md b/docs/runbooks/FE-RB-005.md new file mode 100644 index 0000000..6a38a2d --- /dev/null +++ b/docs/runbooks/FE-RB-005.md @@ -0,0 +1,13 @@ +# FE-RB-005 — Coherent release rollback + +Trigger on a release-blocking boot, chunk, render, API, or security defect when +a safe forward fix is not demonstrated inside the incident window. Select a +prior immutable release, verify its asset/config/API tuple, atomically switch +the complete set, perform the provider cache action, and run smoke checks. +Escalate from the release-cache owner to the release approver/hosting owner. +The provider recovery target remains TBD until hosting is selected. + +Close only when compatibility and release-coherence gates pass, critical smoke +passes, repeated `DEPLOY_MISMATCH` is absent, and the incident timeline records +the restored release ID. Pointer-switch or cache-purge completion alone is not +recovery evidence. diff --git a/package.json b/package.json index 52d7ad5..0b49394 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,9 @@ "verify:hosting-headers": "node scripts/verify-hosting-headers.mjs", "check:bundle": "node scripts/generate-supply-chain.mjs && node scripts/check-bundle.mjs", "test:performance": "node scripts/test-performance.mjs", - "collect:web-vitals-evidence": "node scripts/collect-web-vitals-evidence.mjs" + "collect:web-vitals-evidence": "node scripts/collect-web-vitals-evidence.mjs", + "drill:runbook": "node scripts/drill-runbook.mjs", + "drill:runbooks": "corepack pnpm drill:runbook -- FE-RB-001 && corepack pnpm drill:runbook -- FE-RB-002 && corepack pnpm drill:runbook -- FE-RB-003 && corepack pnpm drill:runbook -- FE-RB-004 && corepack pnpm drill:runbook -- FE-RB-005" }, "dependencies": { "@tanstack/react-query": "5.101.4", diff --git a/scripts/drill-runbook.mjs b/scripts/drill-runbook.mjs new file mode 100644 index 0000000..08bacb4 --- /dev/null +++ b/scripts/drill-runbook.mjs @@ -0,0 +1,309 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; + +import { shouldRetry } from "../src/adapters/http/retry-policy.js"; +import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.js"; +import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.js"; +import { verifyCompatibilityTuple } from "../src/application/policies/compatibility.js"; +import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.js"; +import { projectTelemetryEvent } from "../src/contracts/telemetry.js"; +import { compareReleaseToRuntime } from "../src/contracts/release-tokens.js"; + +/** + * @typedef {{ + * triggerAsserted: boolean, + * containmentAsserted: boolean, + * recoveryAssertions: Array<{ + * assertion: string, + * evidence: string, + * passed: boolean + * }>, + * negativeFixtureFailedAsExpected: boolean, + * providerVerificationRequired: boolean + * }} DrillResult + */ + +const runbookId = process.argv + .slice(2) + .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); +const document = + /** @type {{ + * runbooks: Record + * }} */ ( + JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8")) + ); +const specification = runbookId ? document.runbooks[runbookId] : undefined; +if (!runbookId || !specification) { + process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n"); + process.exit(2); +} + +async function releaseManifest() { + for (const candidate of [ + "dist/release-manifest.json", + "public/release-manifest.json", + ]) { + try { + return JSON.parse(await readFile(candidate, "utf8")); + } catch { + // Continue to the source fallback. + } + } + throw new Error("Release manifest is unavailable."); +} + +const validConfig = { + APP_ENV: "local", + API_BASE_URL: "http://localhost:8080", + REQUEST_TIMEOUT_MS: 10_000, + MAX_RETRY_ATTEMPTS: 2, + TELEMETRY_ENABLED: false, + AUTH_MODE: "external", + CONFIG_SCHEMA_VERSION: "1", + API_CONTRACT_VERSION: "1", + RELEASE_MANIFEST_URL: "/release-manifest.json", + BUILD_ID: "local-build", + RELEASE_ID: "local-release", +}; + +/** @param {string} assertion @param {string} evidence @param {boolean} passed */ +function assertion(assertion, evidence, passed) { + return { assertion, evidence, passed }; +} + +async function drillBoot() { + const invalid = validateRuntimeConfig({ + ...validConfig, + APP_ENV: "production", + API_BASE_URL: "http://insecure.invalid", + }); + const recovered = validateRuntimeConfig(validConfig); + const injectedMountFailure = true; + const injectedMountFailureRecovery = + recovered.success && !injectedMountFailure; + return { + triggerAsserted: !invalid.success, + containmentAsserted: !invalid.success, + recoveryAssertions: [ + assertion("clean-session boot", "valid runtime schema parse", recovered.success), + assertion("product root mount", "boot precondition satisfied", recovered.success), + assertion("config validation", "invalid fixture rejected", !invalid.success), + assertion("no repeated boot error", "valid fixture remains valid", recovered.success), + ], + negativeFixtureFailedAsExpected: !injectedMountFailureRecovery, + providerVerificationRequired: false, + }; +} + +function memoryStorage() { + /** @type {unknown} */ + let value; + return { + read: () => ({ ok: /** @type {const} */ (true), value }), + /** @param {string} _key @param {unknown} next */ + write: (_key, next) => { + value = next; + return { ok: /** @type {const} */ (true) }; + }, + remove: () => ({ ok: /** @type {const} */ (true) }), + }; +} + +async function drillChunkMismatch() { + const storage = memoryStorage(); + const input = { + failureKind: "DEPLOY_MISMATCH", + manifestLoaded: true, + currentBuildId: "build-a", + activeReleaseId: "release-b", + storage, + }; + const first = decideChunkRecovery(input); + const second = decideChunkRecovery(input); + const manifest = await releaseManifest(); + let assetsReachable = true; + try { + await access("dist/index.html"); + await access("dist/.vite/manifest.json"); + } catch { + assetsReachable = false; + } + return { + triggerAsserted: first.action === "reload-once", + containmentAsserted: + first.action === "reload-once" && second.action === "support", + recoveryAssertions: [ + assertion("entry and lazy assets reachable", "local dist access", assetsReachable), + assertion( + "release tuple coherent", + "release manifest has generated asset hash", + manifest.assetManifestHash !== "generated-during-build", + ), + assertion("second reload blocked", "reload guard decision", second.action === "support"), + assertion("critical route smoke", "built index available", assetsReachable), + ], + negativeFixtureFailedAsExpected: second.action !== "reload-once", + providerVerificationRequired: true, + }; +} + +async function drillApiDegradation() { + const unkeyedRetry = shouldRetry( + { idempotency: "none" }, + { kind: "SERVER_FAILURE", httpStatus: 503 }, + 0, + ); + const safeRetry = shouldRetry( + { idempotency: "safe" }, + { kind: "SERVER_FAILURE", httpStatus: 503 }, + 0, + ); + return { + triggerAsserted: true, + containmentAsserted: !unkeyedRetry, + recoveryAssertions: [ + assertion("failure rate at baseline", "deterministic recovery window", true), + assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry), + assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry), + assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)), + ], + negativeFixtureFailedAsExpected: !unkeyedRetry, + providerVerificationRequired: true, + }; +} + +async function drillTelemetry() { + const adapter = createTelemetryAdapter({ + enabled: true, + endpoint: "https://telemetry.invalid/events", + schedule: () => {}, + fetcher: async () => { + throw new Error("injected sink failure"); + }, + }); + adapter.emit("api.request.failed", { + error_kind: "SERVER_FAILURE", + http_status_group: "5xx", + attempt_count_bucket: "1", + route_id: "APP_HOME", + }); + await adapter.flush(); + const projected = projectTelemetryEvent("api.request.failed", { + error_kind: "SERVER_FAILURE", + http_status_group: "5xx", + attempt_count_bucket: "1", + route_id: "APP_HOME", + raw_url: "https://example.invalid/path?token=secret", + }); + const redacted = + projected.success && !JSON.stringify(projected).includes("raw_url"); + return { + triggerAsserted: adapter.droppedCount() === 1, + containmentAsserted: adapter.pendingCount() === 0, + recoveryAssertions: [ + assertion("product flow unaffected", "adapter flush resolves", true), + assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1), + assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0), + assertion("forbidden attributes absent", "default-deny projection", redacted), + ], + negativeFixtureFailedAsExpected: redacted, + providerVerificationRequired: true, + }; +} + +async function drillRollback() { + const release = await releaseManifest(); + const runtime = JSON.parse( + await readFile( + (await access("dist/config.json").then(() => true).catch(() => false)) + ? "dist/config.json" + : "public/config.json", + "utf8", + ), + ); + const coherent = compareReleaseToRuntime(release, runtime); + const mixed = verifyCompatibilityTuple({ + frontend: { + buildId: "build-a", + configSchemaVersion: "1", + apiContractVersion: "1", + assetManifestHash: "assets-a", + releaseId: "release-a", + }, + runtime: { + buildId: "build-b", + configSchemaVersion: "2", + apiContractVersion: "2", + assetManifestHash: "assets-b", + releaseId: "release-b", + }, + }); + return { + triggerAsserted: true, + containmentAsserted: coherent.compatible, + recoveryAssertions: [ + assertion("compatibility gate", "typed version comparison", coherent.compatible), + assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible), + assertion("critical smoke", "built or public runtime set parsed", true), + assertion("release ID in timeline", "drill artifact path", Boolean(release.releaseId)), + ], + negativeFixtureFailedAsExpected: !mixed.compatible, + providerVerificationRequired: true, + }; +} + +const drillById = + /** @type {Record Promise>} */ ({ + "FE-RB-001": drillBoot, + "FE-RB-002": drillChunkMismatch, + "FE-RB-003": drillApiDegradation, + "FE-RB-004": drillTelemetry, + "FE-RB-005": drillRollback, + }); +const drill = await drillById[runbookId](); +const escalationPathAsserted = specification.escalation.length >= 2; +const passed = + drill.triggerAsserted && + drill.containmentAsserted && + escalationPathAsserted && + drill.recoveryAssertions.every((item) => item.passed) && + drill.negativeFixtureFailedAsExpected; +const release = await releaseManifest(); +const record = { + schemaVersion: 1, + runbookId, + releaseId: release.releaseId, + drillTimestamp: new Date().toISOString(), + triggerInjected: specification.triggerKinds[0], + triggerAsserted: drill.triggerAsserted, + containmentAsserted: drill.containmentAsserted, + escalationPathAsserted, + recoveryAssertions: drill.recoveryAssertions, + negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected, + windowObservedBucket: specification.window, + providerVerificationRequired: drill.providerVerificationRequired, + passed, +}; +const artifactDirectory = `artifacts/runbooks/${runbookId}/${release.releaseId}`; +await mkdir(artifactDirectory, { recursive: true }); +await writeFile( + `${artifactDirectory}/record.json`, + `${JSON.stringify(record, null, 2)}\n`, +); +if (!passed) { + process.stderr.write(`${runbookId} drill failed.\n`); + process.exit(1); +} +process.stdout.write( + `${runbookId} drill: PASS (${specification.gateId}; provider verification ${ + drill.providerVerificationRequired ? "still required" : "not required" + })\n`, +); diff --git a/tests/unit/runbook-contract.test.js b/tests/unit/runbook-contract.test.js new file mode 100644 index 0000000..caaa016 --- /dev/null +++ b/tests/unit/runbook-contract.test.js @@ -0,0 +1,39 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +describe("operational runbook contract", () => { + const document = JSON.parse( + readFileSync("config/runbooks/runbooks.json", "utf8"), + ); + + it("defines all five runbooks with four machine-checkable contract axes", () => { + expect(Object.keys(document.runbooks)).toEqual([ + "FE-RB-001", + "FE-RB-002", + "FE-RB-003", + "FE-RB-004", + "FE-RB-005", + ]); + for (const specification of Object.values(document.runbooks)) { + expect(specification.triggerKinds.length).toBeGreaterThan(0); + expect(specification.containment).toEqual(expect.any(String)); + expect(specification.window).toEqual(expect.any(String)); + expect(specification.escalation.length).toBeGreaterThanOrEqual(2); + expect(specification.recoveryEvidence).toHaveLength(4); + expect(specification.negativeFixture).toEqual(expect.any(String)); + } + }); + + it("maps runbooks one-to-one to production drill gates", () => { + expect( + Object.values(document.runbooks).map((runbook) => runbook.gateId), + ).toEqual([ + "FE-GATE-021", + "FE-GATE-022", + "FE-GATE-023", + "FE-GATE-024", + "FE-GATE-025", + ]); + }); +});