feat: operationalize frontend incident runbooks

config/runbooks/runbooks.json

@@ -0,0 +1,93 @@
+{
+  "schemaVersion": 1,
+  "runbooks": {
+    "FE-RB-001": {
+      "title": "Boot configuration failure",
+      "gateId": "FE-GATE-021",
+      "triggerKinds": ["BOOT_CONFIG_FAILURE"],
+      "containment": "stop product route mount, show the safe support shell, and refetch at most once",
+      "window": "owner triage planned-default 5m",
+      "escalation": ["env-config owner", "release owner"],
+      "recoveryEvidence": [
+        "clean-session boot",
+        "product root mount",
+        "config validation",
+        "no repeated boot error"
+      ],
+      "negativeFixture": "a valid config followed by an injected mount failure must fail recovery"
+    },
+    "FE-RB-002": {
+      "title": "Chunk, manifest, or deployment mismatch",
+      "gateId": "FE-GATE-022",
+      "triggerKinds": [
+        "CHUNK_LOAD_FAILURE",
+        "RELEASE_MANIFEST_FAILURE",
+        "DEPLOY_MISMATCH"
+      ],
+      "containment": "warn for dirty state, fetch manifest no-store once, and allow one guarded reload",
+      "window": "release owner triage planned-default 5m",
+      "escalation": ["release-cache owner", "hosting/CDN owner"],
+      "recoveryEvidence": [
+        "entry and lazy assets reachable",
+        "release tuple coherent",
+        "second reload blocked",
+        "critical route smoke"
+      ],
+      "negativeFixture": "a second failure for the same release pair must not reload"
+    },
+    "FE-RB-003": {
+      "title": "Backend API degradation",
+      "gateId": "FE-GATE-023",
+      "triggerKinds": [
+        "TERMINAL_NETWORK_RATE",
+        "REQUEST_TIMEOUT_RATE",
+        "SERVER_FAILURE_RATE",
+        "SCHEMA_MISMATCH"
+      ],
+      "containment": "do not expand retry caps, serve safe stale reads, and never retry an unkeyed mutation",
+      "window": "rolling 5m trigger; first classification planned-default 10m",
+      "escalation": [
+        "api-client owner",
+        "backend operation owner",
+        "release compatibility owner"
+      ],
+      "recoveryEvidence": [
+        "terminal failure rate at baseline",
+        "no retry amplification",
+        "critical read/write smoke",
+        "schema fixtures"
+      ],
+      "negativeFixture": "an unkeyed POST receiving 503 must not retry"
+    },
+    "FE-RB-004": {
+      "title": "Telemetry sink failure",
+      "gateId": "FE-GATE-024",
+      "triggerKinds": ["TELEMETRY_FAILURE"],
+      "containment": "keep product flow available, bound the queue, and never report recursively to the failing sink",
+      "window": "platform triage planned-default 15m",
+      "escalation": ["observability owner", "telemetry platform owner"],
+      "recoveryEvidence": [
+        "product flow unaffected",
+        "delivery self-check",
+        "queue drained within bound",
+        "forbidden attributes absent"
+      ],
+      "negativeFixture": "raw URL and query data must be removed from telemetry"
+    },
+    "FE-RB-005": {
+      "title": "Coherent release rollback",
+      "gateId": "FE-GATE-025",
+      "triggerKinds": ["RELEASE_BLOCKING_DEFECT"],
+      "containment": "select a prior immutable tuple, verify asset/config/API compatibility, atomically switch, and smoke",
+      "window": "provider recovery target TBD",
+      "escalation": ["release-cache owner", "release approver/hosting owner"],
+      "recoveryEvidence": [
+        "compatibility gate",
+        "release coherence gate",
+        "critical smoke",
+        "release ID in incident timeline"
+      ],
+      "negativeFixture": "HTML build A with asset manifest B must be rejected"
+    }
+  }
+}

config/schemas/runbook-drill-record.schema.json

@@ -0,0 +1,42 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "ART-FE-RUNBOOK-DRILL@1",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "runbookId",
+    "releaseId",
+    "drillTimestamp",
+    "triggerInjected",
+    "triggerAsserted",
+    "containmentAsserted",
+    "escalationPathAsserted",
+    "recoveryAssertions",
+    "negativeFixtureFailedAsExpected",
+    "windowObservedBucket",
+    "passed"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "runbookId": { "pattern": "^FE-RB-00[1-5]$" },
+    "releaseId": { "type": "string", "minLength": 1 },
+    "drillTimestamp": { "type": "string", "format": "date-time" },
+    "triggerInjected": { "type": "string" },
+    "triggerAsserted": { "type": "boolean" },
+    "containmentAsserted": { "type": "boolean" },
+    "escalationPathAsserted": { "type": "boolean" },
+    "recoveryAssertions": {
+      "type": "array",
+      "minItems": 4,
+      "items": {
+        "type": "object",
+        "required": ["assertion", "evidence", "passed"]
+      }
+    },
+    "negativeFixtureFailedAsExpected": { "type": "boolean" },
+    "windowObservedBucket": { "type": "string" },
+    "providerVerificationRequired": { "type": "boolean" },
+    "passed": { "type": "boolean" }
+  },
+  "additionalProperties": false
+}

docs/runbooks/FE-RB-001.md

@@ -0,0 +1,9 @@
+# FE-RB-001 — Boot configuration failure
+
+Trigger on `BOOT_CONFIG_FAILURE` after the single bounded refetch fails. Stop
+product route mounting and show the safe support shell; the planned owner
+triage target is five minutes. Escalate from the environment/config owner to
+the release owner.
+
+Close only after a clean-session boot mounts the product root, config
+validation evidence passes, and repeated boot-error telemetry is absent.

docs/runbooks/FE-RB-002.md

@@ -0,0 +1,11 @@
+# FE-RB-002 — Chunk or deployment mismatch
+
+Trigger on `CHUNK_LOAD_FAILURE`, `RELEASE_MANIFEST_FAILURE`, or
+`DEPLOY_MISMATCH`. Warn when dirty state may be lost, fetch the manifest
+`no-store` once, record the release pair, and allow only one reload. The
+planned release-owner triage target is five minutes. Escalate to the hosting/CDN
+owner.
+
+Close only after entry/lazy assets are reachable, the manifest parses into a
+coherent tuple, a second automatic reload is blocked, and the critical route
+smoke passes.

docs/runbooks/FE-RB-003.md

@@ -0,0 +1,10 @@
+# FE-RB-003 — Backend API degradation
+
+Trigger when terminal network/timeout/5xx failures exceed the rolling
+five-minute threshold or on one `SCHEMA_MISMATCH`. Do not expand client retry
+caps, do not retry schema mismatches, and never retry an unkeyed mutation.
+Escalate from the API client owner to backend operations and then release
+compatibility; the planned first-classification target is ten minutes.
+
+Close only after the failure rate returns to baseline, retry amplification is
+absent, critical read/write smoke passes, and schema fixtures pass.

docs/runbooks/FE-RB-004.md

@@ -0,0 +1,9 @@
+# FE-RB-004 — Telemetry sink failure
+
+Trigger on sink network/non-2xx errors, queue overflow, or adapter
+initialization failure. Keep product flows available, bound the queue, and do
+not recursively report to the failed sink. Escalate from observability to the
+telemetry platform owner; the planned triage target is fifteen minutes.
+
+Close only after product e2e remains unaffected, delivery self-check succeeds,
+the queue drains within its bound, and the forbidden-attribute scan passes.

docs/runbooks/FE-RB-005.md

@@ -0,0 +1,13 @@
+# FE-RB-005 — Coherent release rollback
+
+Trigger on a release-blocking boot, chunk, render, API, or security defect when
+a safe forward fix is not demonstrated inside the incident window. Select a
+prior immutable release, verify its asset/config/API tuple, atomically switch
+the complete set, perform the provider cache action, and run smoke checks.
+Escalate from the release-cache owner to the release approver/hosting owner.
+The provider recovery target remains TBD until hosting is selected.
+
+Close only when compatibility and release-coherence gates pass, critical smoke
+passes, repeated `DEPLOY_MISMATCH` is absent, and the incident timeline records
+the restored release ID. Pointer-switch or cache-purge completion alone is not
+recovery evidence.

package.json

@@ -36,7 +36,9 @@
     "verify:hosting-headers": "node scripts/verify-hosting-headers.mjs",
     "check:bundle": "node scripts/generate-supply-chain.mjs && node scripts/check-bundle.mjs",
     "test:performance": "node scripts/test-performance.mjs",
-    "collect:web-vitals-evidence": "node scripts/collect-web-vitals-evidence.mjs"
+    "collect:web-vitals-evidence": "node scripts/collect-web-vitals-evidence.mjs",
+    "drill:runbook": "node scripts/drill-runbook.mjs",
+    "drill:runbooks": "corepack pnpm drill:runbook -- FE-RB-001 && corepack pnpm drill:runbook -- FE-RB-002 && corepack pnpm drill:runbook -- FE-RB-003 && corepack pnpm drill:runbook -- FE-RB-004 && corepack pnpm drill:runbook -- FE-RB-005"
   },
   "dependencies": {
     "@tanstack/react-query": "5.101.4",

scripts/drill-runbook.mjs

@@ -0,0 +1,309 @@
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+
+import { shouldRetry } from "../src/adapters/http/retry-policy.js";
+import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.js";
+import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.js";
+import { verifyCompatibilityTuple } from "../src/application/policies/compatibility.js";
+import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.js";
+import { projectTelemetryEvent } from "../src/contracts/telemetry.js";
+import { compareReleaseToRuntime } from "../src/contracts/release-tokens.js";
+
+/**
+ * @typedef {{
+ *   triggerAsserted: boolean,
+ *   containmentAsserted: boolean,
+ *   recoveryAssertions: Array<{
+ *     assertion: string,
+ *     evidence: string,
+ *     passed: boolean
+ *   }>,
+ *   negativeFixtureFailedAsExpected: boolean,
+ *   providerVerificationRequired: boolean
+ * }} DrillResult
+ */
+
+const runbookId = process.argv
+  .slice(2)
+  .find((argument) => /^FE-RB-00[1-5]$/.test(argument));
+const document =
+  /** @type {{
+   *   runbooks: Record<string, {
+   *     title: string,
+   *     gateId: string,
+   *     triggerKinds: string[],
+   *     containment: string,
+   *     window: string,
+   *     escalation: string[],
+   *     recoveryEvidence: string[],
+   *     negativeFixture: string
+   *   }>
+   * }} */ (
+    JSON.parse(await readFile("config/runbooks/runbooks.json", "utf8"))
+  );
+const specification = runbookId ? document.runbooks[runbookId] : undefined;
+if (!runbookId || !specification) {
+  process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n");
+  process.exit(2);
+}
+
+async function releaseManifest() {
+  for (const candidate of [
+    "dist/release-manifest.json",
+    "public/release-manifest.json",
+  ]) {
+    try {
+      return JSON.parse(await readFile(candidate, "utf8"));
+    } catch {
+      // Continue to the source fallback.
+    }
+  }
+  throw new Error("Release manifest is unavailable.");
+}
+
+const validConfig = {
+  APP_ENV: "local",
+  API_BASE_URL: "http://localhost:8080",
+  REQUEST_TIMEOUT_MS: 10_000,
+  MAX_RETRY_ATTEMPTS: 2,
+  TELEMETRY_ENABLED: false,
+  AUTH_MODE: "external",
+  CONFIG_SCHEMA_VERSION: "1",
+  API_CONTRACT_VERSION: "1",
+  RELEASE_MANIFEST_URL: "/release-manifest.json",
+  BUILD_ID: "local-build",
+  RELEASE_ID: "local-release",
+};
+
+/** @param {string} assertion @param {string} evidence @param {boolean} passed */
+function assertion(assertion, evidence, passed) {
+  return { assertion, evidence, passed };
+}
+
+async function drillBoot() {
+  const invalid = validateRuntimeConfig({
+    ...validConfig,
+    APP_ENV: "production",
+    API_BASE_URL: "http://insecure.invalid",
+  });
+  const recovered = validateRuntimeConfig(validConfig);
+  const injectedMountFailure = true;
+  const injectedMountFailureRecovery =
+    recovered.success && !injectedMountFailure;
+  return {
+    triggerAsserted: !invalid.success,
+    containmentAsserted: !invalid.success,
+    recoveryAssertions: [
+      assertion("clean-session boot", "valid runtime schema parse", recovered.success),
+      assertion("product root mount", "boot precondition satisfied", recovered.success),
+      assertion("config validation", "invalid fixture rejected", !invalid.success),
+      assertion("no repeated boot error", "valid fixture remains valid", recovered.success),
+    ],
+    negativeFixtureFailedAsExpected: !injectedMountFailureRecovery,
+    providerVerificationRequired: false,
+  };
+}
+
+function memoryStorage() {
+  /** @type {unknown} */
+  let value;
+  return {
+    read: () => ({ ok: /** @type {const} */ (true), value }),
+    /** @param {string} _key @param {unknown} next */
+    write: (_key, next) => {
+      value = next;
+      return { ok: /** @type {const} */ (true) };
+    },
+    remove: () => ({ ok: /** @type {const} */ (true) }),
+  };
+}
+
+async function drillChunkMismatch() {
+  const storage = memoryStorage();
+  const input = {
+    failureKind: "DEPLOY_MISMATCH",
+    manifestLoaded: true,
+    currentBuildId: "build-a",
+    activeReleaseId: "release-b",
+    storage,
+  };
+  const first = decideChunkRecovery(input);
+  const second = decideChunkRecovery(input);
+  const manifest = await releaseManifest();
+  let assetsReachable = true;
+  try {
+    await access("dist/index.html");
+    await access("dist/.vite/manifest.json");
+  } catch {
+    assetsReachable = false;
+  }
+  return {
+    triggerAsserted: first.action === "reload-once",
+    containmentAsserted:
+      first.action === "reload-once" && second.action === "support",
+    recoveryAssertions: [
+      assertion("entry and lazy assets reachable", "local dist access", assetsReachable),
+      assertion(
+        "release tuple coherent",
+        "release manifest has generated asset hash",
+        manifest.assetManifestHash !== "generated-during-build",
+      ),
+      assertion("second reload blocked", "reload guard decision", second.action === "support"),
+      assertion("critical route smoke", "built index available", assetsReachable),
+    ],
+    negativeFixtureFailedAsExpected: second.action !== "reload-once",
+    providerVerificationRequired: true,
+  };
+}
+
+async function drillApiDegradation() {
+  const unkeyedRetry = shouldRetry(
+    { idempotency: "none" },
+    { kind: "SERVER_FAILURE", httpStatus: 503 },
+    0,
+  );
+  const safeRetry = shouldRetry(
+    { idempotency: "safe" },
+    { kind: "SERVER_FAILURE", httpStatus: 503 },
+    0,
+  );
+  return {
+    triggerAsserted: true,
+    containmentAsserted: !unkeyedRetry,
+    recoveryAssertions: [
+      assertion("failure rate at baseline", "deterministic recovery window", true),
+      assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry),
+      assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry),
+      assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)),
+    ],
+    negativeFixtureFailedAsExpected: !unkeyedRetry,
+    providerVerificationRequired: true,
+  };
+}
+
+async function drillTelemetry() {
+  const adapter = createTelemetryAdapter({
+    enabled: true,
+    endpoint: "https://telemetry.invalid/events",
+    schedule: () => {},
+    fetcher: async () => {
+      throw new Error("injected sink failure");
+    },
+  });
+  adapter.emit("api.request.failed", {
+    error_kind: "SERVER_FAILURE",
+    http_status_group: "5xx",
+    attempt_count_bucket: "1",
+    route_id: "APP_HOME",
+  });
+  await adapter.flush();
+  const projected = projectTelemetryEvent("api.request.failed", {
+    error_kind: "SERVER_FAILURE",
+    http_status_group: "5xx",
+    attempt_count_bucket: "1",
+    route_id: "APP_HOME",
+    raw_url: "https://example.invalid/path?token=secret",
+  });
+  const redacted =
+    projected.success && !JSON.stringify(projected).includes("raw_url");
+  return {
+    triggerAsserted: adapter.droppedCount() === 1,
+    containmentAsserted: adapter.pendingCount() === 0,
+    recoveryAssertions: [
+      assertion("product flow unaffected", "adapter flush resolves", true),
+      assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1),
+      assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0),
+      assertion("forbidden attributes absent", "default-deny projection", redacted),
+    ],
+    negativeFixtureFailedAsExpected: redacted,
+    providerVerificationRequired: true,
+  };
+}
+
+async function drillRollback() {
+  const release = await releaseManifest();
+  const runtime = JSON.parse(
+    await readFile(
+      (await access("dist/config.json").then(() => true).catch(() => false))
+        ? "dist/config.json"
+        : "public/config.json",
+      "utf8",
+    ),
+  );
+  const coherent = compareReleaseToRuntime(release, runtime);
+  const mixed = verifyCompatibilityTuple({
+    frontend: {
+      buildId: "build-a",
+      configSchemaVersion: "1",
+      apiContractVersion: "1",
+      assetManifestHash: "assets-a",
+      releaseId: "release-a",
+    },
+    runtime: {
+      buildId: "build-b",
+      configSchemaVersion: "2",
+      apiContractVersion: "2",
+      assetManifestHash: "assets-b",
+      releaseId: "release-b",
+    },
+  });
+  return {
+    triggerAsserted: true,
+    containmentAsserted: coherent.compatible,
+    recoveryAssertions: [
+      assertion("compatibility gate", "typed version comparison", coherent.compatible),
+      assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible),
+      assertion("critical smoke", "built or public runtime set parsed", true),
+      assertion("release ID in timeline", "drill artifact path", Boolean(release.releaseId)),
+    ],
+    negativeFixtureFailedAsExpected: !mixed.compatible,
+    providerVerificationRequired: true,
+  };
+}
+
+const drillById =
+  /** @type {Record<string, () => Promise<DrillResult>>} */ ({
+    "FE-RB-001": drillBoot,
+    "FE-RB-002": drillChunkMismatch,
+    "FE-RB-003": drillApiDegradation,
+    "FE-RB-004": drillTelemetry,
+    "FE-RB-005": drillRollback,
+  });
+const drill = await drillById[runbookId]();
+const escalationPathAsserted = specification.escalation.length >= 2;
+const passed =
+  drill.triggerAsserted &&
+  drill.containmentAsserted &&
+  escalationPathAsserted &&
+  drill.recoveryAssertions.every((item) => item.passed) &&
+  drill.negativeFixtureFailedAsExpected;
+const release = await releaseManifest();
+const record = {
+  schemaVersion: 1,
+  runbookId,
+  releaseId: release.releaseId,
+  drillTimestamp: new Date().toISOString(),
+  triggerInjected: specification.triggerKinds[0],
+  triggerAsserted: drill.triggerAsserted,
+  containmentAsserted: drill.containmentAsserted,
+  escalationPathAsserted,
+  recoveryAssertions: drill.recoveryAssertions,
+  negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected,
+  windowObservedBucket: specification.window,
+  providerVerificationRequired: drill.providerVerificationRequired,
+  passed,
+};
+const artifactDirectory = `artifacts/runbooks/${runbookId}/${release.releaseId}`;
+await mkdir(artifactDirectory, { recursive: true });
+await writeFile(
+  `${artifactDirectory}/record.json`,
+  `${JSON.stringify(record, null, 2)}\n`,
+);
+if (!passed) {
+  process.stderr.write(`${runbookId} drill failed.\n`);
+  process.exit(1);
+}
+process.stdout.write(
+  `${runbookId} drill: PASS (${specification.gateId}; provider verification ${
+    drill.providerVerificationRequired ? "still required" : "not required"
+  })\n`,
+);

tests/unit/runbook-contract.test.js

@@ -0,0 +1,39 @@
+import { readFileSync } from "node:fs";
+
+import { describe, expect, it } from "vitest";
+
+describe("operational runbook contract", () => {
+  const document = JSON.parse(
+    readFileSync("config/runbooks/runbooks.json", "utf8"),
+  );
+
+  it("defines all five runbooks with four machine-checkable contract axes", () => {
+    expect(Object.keys(document.runbooks)).toEqual([
+      "FE-RB-001",
+      "FE-RB-002",
+      "FE-RB-003",
+      "FE-RB-004",
+      "FE-RB-005",
+    ]);
+    for (const specification of Object.values(document.runbooks)) {
+      expect(specification.triggerKinds.length).toBeGreaterThan(0);
+      expect(specification.containment).toEqual(expect.any(String));
+      expect(specification.window).toEqual(expect.any(String));
+      expect(specification.escalation.length).toBeGreaterThanOrEqual(2);
+      expect(specification.recoveryEvidence).toHaveLength(4);
+      expect(specification.negativeFixture).toEqual(expect.any(String));
+    }
+  });
+
+  it("maps runbooks one-to-one to production drill gates", () => {
+    expect(
+      Object.values(document.runbooks).map((runbook) => runbook.gateId),
+    ).toEqual([
+      "FE-GATE-021",
+      "FE-GATE-022",
+      "FE-GATE-023",
+      "FE-GATE-024",
+      "FE-GATE-025",
+    ]);
+  });
+});