import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { shouldRetry } from "../src/adapters/http/retry-policy.ts"; import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.ts"; import { verifyCompatibilityTuple } from "../src/application/policies/compatibility.ts"; import type { StoragePort } from "../src/application/ports/storage-port.ts"; import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.ts"; import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.ts"; import { compareReleaseToRuntime } from "../src/contracts/release-tokens.ts"; import { projectTelemetryEvent } from "../src/contracts/telemetry.ts"; type RecoveryAssertion = Readonly<{ assertion: string; evidence: string; passed: boolean; }>; type DrillResult = Readonly<{ triggerAsserted: boolean; containmentAsserted: boolean; recoveryAssertions: RecoveryAssertion[]; negativeFixtureFailedAsExpected: boolean; providerVerificationRequired: boolean; }>; type RunbookSpecification = Readonly<{ title: string; gateId: string; triggerKinds: string[]; containment: string; window: string; escalation: string[]; recoveryEvidence: string[]; negativeFixture: string; }>; type RunbookDocument = Readonly<{ runbooks: Record; }>; type ReleaseManifest = Record & { buildId: string; configSchemaVersion: string; apiContractVersion: string; assetManifestHash: string; releaseId: string; }; const runbookId = process.argv .slice(2) .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); const document = JSON.parse( await readFile("config/runbooks/runbooks.json", "utf8"), ) as RunbookDocument; const specification = runbookId ? document.runbooks[runbookId] : undefined; if (!runbookId || !specification) { process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n"); process.exit(2); } async function releaseManifest(): Promise { for (const candidate of [ "dist/release-manifest.json", "public/release-manifest.json", ]) { try { return JSON.parse( await readFile(candidate, "utf8"), ) as ReleaseManifest; } catch { // Continue to the source fallback. } } throw new Error("Release manifest is unavailable."); } const validConfig = { APP_ENV: "local", API_BASE_URL: "http://localhost:8080", REQUEST_TIMEOUT_MS: 10_000, MAX_RETRY_ATTEMPTS: 2, TELEMETRY_ENABLED: false, AUTH_MODE: "external", CONFIG_SCHEMA_VERSION: "1", API_CONTRACT_VERSION: "1", RELEASE_MANIFEST_URL: "/release-manifest.json", BUILD_ID: "local-build", RELEASE_ID: "local-release", }; function assertion( assertion: string, evidence: string, passed: boolean, ): RecoveryAssertion { return { assertion, evidence, passed }; } async function drillBoot(): Promise { const invalid = validateRuntimeConfig({ ...validConfig, APP_ENV: "production", API_BASE_URL: "http://insecure.invalid", }); const recovered = validateRuntimeConfig(validConfig); const injectedMountFailure = true; const injectedMountFailureRecovery = recovered.success && !injectedMountFailure; return { triggerAsserted: !invalid.success, containmentAsserted: !invalid.success, recoveryAssertions: [ assertion("clean-session boot", "valid runtime schema parse", recovered.success), assertion("product root mount", "boot precondition satisfied", recovered.success), assertion("config validation", "invalid fixture rejected", !invalid.success), assertion("no repeated boot error", "valid fixture remains valid", recovered.success), ], negativeFixtureFailedAsExpected: !injectedMountFailureRecovery, providerVerificationRequired: false, }; } function memoryStorage(): StoragePort { let value: unknown; return { read: () => ({ ok: true, value }), write: (_key: string, next: unknown) => { value = next; return { ok: true }; }, remove: () => ({ ok: true }), }; } async function drillChunkMismatch(): Promise { const storage = memoryStorage(); const input: Parameters[0] = { failureKind: "DEPLOY_MISMATCH", manifestLoaded: true, currentBuildId: "build-a", currentReleaseId: "release-a", activeBuildId: "build-b", activeReleaseId: "release-b", storage, }; const first = decideChunkRecovery(input); const second = decideChunkRecovery(input); const manifest = await releaseManifest(); let assetsReachable = true; try { await access("dist/index.html"); await access("dist/.vite/manifest.json"); } catch { assetsReachable = false; } return { triggerAsserted: first.action === "reload-once", containmentAsserted: first.action === "reload-once" && second.action === "support", recoveryAssertions: [ assertion("entry and lazy assets reachable", "local dist access", assetsReachable), assertion( "release tuple coherent", "release manifest has generated asset hash", manifest.assetManifestHash !== "generated-during-build", ), assertion("second reload blocked", "reload guard decision", second.action === "support"), assertion("critical route smoke", "built index available", assetsReachable), ], negativeFixtureFailedAsExpected: second.action !== "reload-once", providerVerificationRequired: true, }; } async function drillApiDegradation(): Promise { const unkeyedRetry = shouldRetry( { idempotency: "none" }, { kind: "SERVER_FAILURE", httpStatus: 503 }, 0, ); const safeRetry = shouldRetry( { idempotency: "safe" }, { kind: "SERVER_FAILURE", httpStatus: 503 }, 0, ); return { triggerAsserted: true, containmentAsserted: !unkeyedRetry, recoveryAssertions: [ assertion("failure rate at baseline", "deterministic recovery window", true), assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry), assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry), assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)), ], negativeFixtureFailedAsExpected: !unkeyedRetry, providerVerificationRequired: true, }; } async function drillTelemetry(): Promise { const adapter = createTelemetryAdapter({ enabled: true, endpoint: "https://telemetry.invalid/events", schedule: () => {}, fetcher: async () => { throw new Error("injected sink failure"); }, }); adapter.emit("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", attempt_count_bucket: "1", route_id: "APP_HOME", }); await adapter.flush(); const projected = projectTelemetryEvent("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", attempt_count_bucket: "1", route_id: "APP_HOME", raw_url: "https://example.invalid/path?token=secret", }); const redacted = projected.success && !JSON.stringify(projected).includes("raw_url"); return { triggerAsserted: adapter.droppedCount() === 1, containmentAsserted: adapter.pendingCount() === 0, recoveryAssertions: [ assertion("product flow unaffected", "adapter flush resolves", true), assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1), assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0), assertion("forbidden attributes absent", "default-deny projection", redacted), ], negativeFixtureFailedAsExpected: redacted, providerVerificationRequired: true, }; } async function drillRollback(): Promise { const release = await releaseManifest(); const runtime = JSON.parse( await readFile( (await access("dist/config.json").then(() => true).catch(() => false)) ? "dist/config.json" : "public/config.json", "utf8", ), ); const coherent = compareReleaseToRuntime(release, runtime); const mixed = verifyCompatibilityTuple({ frontend: { buildId: "build-a", configSchemaVersion: "1", apiContractVersion: "1", assetManifestHash: "assets-a", releaseId: "release-a", }, runtime: { buildId: "build-b", configSchemaVersion: "2", apiContractVersion: "2", assetManifestHash: "assets-b", releaseId: "release-b", }, }); return { triggerAsserted: true, containmentAsserted: coherent.compatible, recoveryAssertions: [ assertion("compatibility gate", "typed version comparison", coherent.compatible), assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible), assertion("critical smoke", "built or public runtime set parsed", true), assertion("release ID in timeline", "drill record releaseId", Boolean(release.releaseId)), ], negativeFixtureFailedAsExpected: !mixed.compatible, providerVerificationRequired: true, }; } const drillById: Record Promise> = { "FE-RB-001": drillBoot, "FE-RB-002": drillChunkMismatch, "FE-RB-003": drillApiDegradation, "FE-RB-004": drillTelemetry, "FE-RB-005": drillRollback, }; const drill = await drillById[runbookId](); const escalationPathAsserted = specification.escalation.length >= 2; const passed = drill.triggerAsserted && drill.containmentAsserted && escalationPathAsserted && drill.recoveryAssertions.every((item) => item.passed) && drill.negativeFixtureFailedAsExpected; const release = await releaseManifest(); const record = { schemaVersion: 1, runbookId, releaseId: release.releaseId, drillTimestamp: new Date().toISOString(), triggerInjected: specification.triggerKinds[0], triggerAsserted: drill.triggerAsserted, containmentAsserted: drill.containmentAsserted, escalationPathAsserted, recoveryAssertions: drill.recoveryAssertions, negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected, windowObservedBucket: specification.window, providerVerificationRequired: drill.providerVerificationRequired, passed, }; const artifactDirectory = `artifacts/runbooks/${runbookId}`; await mkdir(artifactDirectory, { recursive: true }); await writeFile( `${artifactDirectory}/record.json`, `${JSON.stringify(record, null, 2)}\n`, ); if (!passed) { process.stderr.write(`${runbookId} drill failed.\n`); process.exit(1); } process.stdout.write( `${runbookId} drill: PASS (${specification.gateId}; provider verification ${ drill.providerVerificationRequired ? "still required" : "not required" })\n`, );