feat: operationalize frontend incident runbooks

2026-07-25 21:30:40 +09:00
parent 15ddb8d474
commit 75c3f5b08c
10 changed files with 538 additions and 1 deletions
@@ -0,0 +1,93 @@
+{
+  "schemaVersion": 1,
+  "runbooks": {
+    "FE-RB-001": {
+      "title": "Boot configuration failure",
+      "gateId": "FE-GATE-021",
+      "triggerKinds": ["BOOT_CONFIG_FAILURE"],
+      "containment": "stop product route mount, show the safe support shell, and refetch at most once",
+      "window": "owner triage planned-default 5m",
+      "escalation": ["env-config owner", "release owner"],
+      "recoveryEvidence": [
+        "clean-session boot",
+        "product root mount",
+        "config validation",
+        "no repeated boot error"
+      ],
+      "negativeFixture": "a valid config followed by an injected mount failure must fail recovery"
+    },
+    "FE-RB-002": {
+      "title": "Chunk, manifest, or deployment mismatch",
+      "gateId": "FE-GATE-022",
+      "triggerKinds": [
+        "CHUNK_LOAD_FAILURE",
+        "RELEASE_MANIFEST_FAILURE",
+        "DEPLOY_MISMATCH"
+      ],
+      "containment": "warn for dirty state, fetch manifest no-store once, and allow one guarded reload",
+      "window": "release owner triage planned-default 5m",
+      "escalation": ["release-cache owner", "hosting/CDN owner"],
+      "recoveryEvidence": [
+        "entry and lazy assets reachable",
+        "release tuple coherent",
+        "second reload blocked",
+        "critical route smoke"
+      ],
+      "negativeFixture": "a second failure for the same release pair must not reload"
+    },
+    "FE-RB-003": {
+      "title": "Backend API degradation",
+      "gateId": "FE-GATE-023",
+      "triggerKinds": [
+        "TERMINAL_NETWORK_RATE",
+        "REQUEST_TIMEOUT_RATE",
+        "SERVER_FAILURE_RATE",
+        "SCHEMA_MISMATCH"
+      ],
+      "containment": "do not expand retry caps, serve safe stale reads, and never retry an unkeyed mutation",
+      "window": "rolling 5m trigger; first classification planned-default 10m",
+      "escalation": [
+        "api-client owner",
+        "backend operation owner",
+        "release compatibility owner"
+      ],
+      "recoveryEvidence": [
+        "terminal failure rate at baseline",
+        "no retry amplification",
+        "critical read/write smoke",
+        "schema fixtures"
+      ],
+      "negativeFixture": "an unkeyed POST receiving 503 must not retry"
+    },
+    "FE-RB-004": {
+      "title": "Telemetry sink failure",
+      "gateId": "FE-GATE-024",
+      "triggerKinds": ["TELEMETRY_FAILURE"],
+      "containment": "keep product flow available, bound the queue, and never report recursively to the failing sink",
+      "window": "platform triage planned-default 15m",
+      "escalation": ["observability owner", "telemetry platform owner"],
+      "recoveryEvidence": [
+        "product flow unaffected",
+        "delivery self-check",
+        "queue drained within bound",
+        "forbidden attributes absent"
+      ],
+      "negativeFixture": "raw URL and query data must be removed from telemetry"
+    },
+    "FE-RB-005": {
+      "title": "Coherent release rollback",
+      "gateId": "FE-GATE-025",
+      "triggerKinds": ["RELEASE_BLOCKING_DEFECT"],
+      "containment": "select a prior immutable tuple, verify asset/config/API compatibility, atomically switch, and smoke",
+      "window": "provider recovery target TBD",
+      "escalation": ["release-cache owner", "release approver/hosting owner"],
+      "recoveryEvidence": [
+        "compatibility gate",
+        "release coherence gate",
+        "critical smoke",
+        "release ID in incident timeline"
+      ],
+      "negativeFixture": "HTML build A with asset manifest B must be rejected"
+    }
+  }
+}
@@ -0,0 +1,42 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "ART-FE-RUNBOOK-DRILL@1",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "runbookId",
+    "releaseId",
+    "drillTimestamp",
+    "triggerInjected",
+    "triggerAsserted",
+    "containmentAsserted",
+    "escalationPathAsserted",
+    "recoveryAssertions",
+    "negativeFixtureFailedAsExpected",
+    "windowObservedBucket",
+    "passed"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "runbookId": { "pattern": "^FE-RB-00[1-5]$" },
+    "releaseId": { "type": "string", "minLength": 1 },
+    "drillTimestamp": { "type": "string", "format": "date-time" },
+    "triggerInjected": { "type": "string" },
+    "triggerAsserted": { "type": "boolean" },
+    "containmentAsserted": { "type": "boolean" },
+    "escalationPathAsserted": { "type": "boolean" },
+    "recoveryAssertions": {
+      "type": "array",
+      "minItems": 4,
+      "items": {
+        "type": "object",
+        "required": ["assertion", "evidence", "passed"]
+      }
+    },
+    "negativeFixtureFailedAsExpected": { "type": "boolean" },
+    "windowObservedBucket": { "type": "string" },
+    "providerVerificationRequired": { "type": "boolean" },
+    "passed": { "type": "boolean" }
+  },
+  "additionalProperties": false
+}