import { access, mkdir, readFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; import { shouldRetry } from "../src/adapters/http/retry-policy.ts"; import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.ts"; import { verifyCompatibilityTuple } from "../src/application/policies/compatibility.ts"; import type { StoragePort } from "../src/application/ports/storage-port.ts"; import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.ts"; import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.ts"; import type { InstalledContractPackageIdentity } from "../src/contracts/external-contract-runtime.ts"; import { parseReleaseArtifact, parseRuntimeConfigArtifact, type ReleaseArtifact, } from "../src/contracts/release-artifacts.ts"; import { projectTelemetryEvent } from "../src/contracts/telemetry.ts"; import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts"; import { runbookRecordArtifactSchema } from "./contracts/release-artifacts.ts"; import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; type RecoveryAssertion = Readonly<{ assertion: string; evidence: string; passed: boolean; }>; type DrillResult = Readonly<{ triggerAsserted: boolean; containmentAsserted: boolean; recoveryAssertions: RecoveryAssertion[]; negativeFixtureFailedAsExpected: boolean; providerVerificationRequired: boolean; }>; type RunbookSpecification = Readonly<{ title: string; gateId: string; triggerKinds: string[]; containment: string; window: string; escalation: string[]; recoveryEvidence: string[]; negativeFixture: string; }>; type RunbookDocument = Readonly<{ runbooks: Record; }>; export type JsonArtifactReader = (path: string) => Promise; export type RollbackArtifactPaths = Readonly<{ primaryRelease: string; fallbackRelease: string; primaryRuntime: string; fallbackRuntime: string; }>; export type RollbackCoherenceOptions = Readonly<{ readArtifact?: JsonArtifactReader; contractPackages?: readonly InstalledContractPackageIdentity[]; paths?: RollbackArtifactPaths; }>; const DEFAULT_ROLLBACK_PATHS = Object.freeze({ primaryRelease: "dist/release-manifest.json", fallbackRelease: "public/release-manifest.json", primaryRuntime: "dist/config.json", fallbackRuntime: "public/config.json", }); async function readJsonArtifact(path: string): Promise { return JSON.parse(await readFile(path, "utf8")); } async function releaseManifest( readArtifact: JsonArtifactReader = readJsonArtifact, paths: RollbackArtifactPaths = DEFAULT_ROLLBACK_PATHS, ): Promise { const value = await readPrimaryOrFallback( readArtifact, paths.primaryRelease, paths.fallbackRelease, ); return parseReleaseArtifact(value); } const validConfig = { APP_ENV: "local", API_BASE_URL: "http://localhost:8080", REQUEST_TIMEOUT_MS: 10_000, MAX_RETRY_ATTEMPTS: 2, TELEMETRY_ENABLED: false, AUTH_MODE: "external", CONFIG_SCHEMA_VERSION: "1", API_CONTRACT_VERSION: "1", RELEASE_MANIFEST_URL: "/release-manifest.json", BUILD_ID: "local-build", RELEASE_ID: "local-release", }; function assertion( assertion: string, evidence: string, passed: boolean, ): RecoveryAssertion { return { assertion, evidence, passed }; } async function drillBoot(): Promise { const invalid = validateRuntimeConfig({ ...validConfig, APP_ENV: "production", API_BASE_URL: "http://insecure.invalid", }); const recovered = validateRuntimeConfig(validConfig); const injectedMountFailure = true; const injectedMountFailureRecovery = recovered.success && !injectedMountFailure; return { triggerAsserted: !invalid.success, containmentAsserted: !invalid.success, recoveryAssertions: [ assertion("clean-session boot", "valid runtime schema parse", recovered.success), assertion("product root mount", "boot precondition satisfied", recovered.success), assertion("config validation", "invalid fixture rejected", !invalid.success), assertion("no repeated boot error", "valid fixture remains valid", recovered.success), ], negativeFixtureFailedAsExpected: !injectedMountFailureRecovery, providerVerificationRequired: false, }; } function memoryStorage(): StoragePort { let value: unknown; return { read: () => ({ ok: true, value }), write: (_key: string, next: unknown) => { value = next; return { ok: true }; }, remove: () => ({ ok: true }), }; } async function drillChunkMismatch(): Promise { const storage = memoryStorage(); const input: Parameters[0] = { failureKind: "DEPLOY_MISMATCH", manifestLoaded: true, currentBuildId: "build-a", currentReleaseId: "release-a", activeBuildId: "build-b", activeReleaseId: "release-b", storage, }; const first = decideChunkRecovery(input); const second = decideChunkRecovery(input); const manifest = await releaseManifest(); let assetsReachable = true; try { await access("dist/index.html"); await access("dist/.vite/manifest.json"); } catch { assetsReachable = false; } return { triggerAsserted: first.action === "reload-once", containmentAsserted: first.action === "reload-once" && second.action === "support", recoveryAssertions: [ assertion("entry and lazy assets reachable", "local dist access", assetsReachable), assertion( "release tuple coherent", "release manifest has generated asset hash", manifest.assetManifestHash !== "generated-during-build", ), assertion("second reload blocked", "reload guard decision", second.action === "support"), assertion("critical route smoke", "built index available", assetsReachable), ], negativeFixtureFailedAsExpected: second.action !== "reload-once", providerVerificationRequired: true, }; } async function drillApiDegradation(): Promise { const unkeyedRetry = shouldRetry( { idempotency: "none" }, { kind: "SERVER_FAILURE", httpStatus: 503 }, 0, ); const safeRetry = shouldRetry( { idempotency: "safe" }, { kind: "SERVER_FAILURE", httpStatus: 503 }, 0, ); return { triggerAsserted: true, containmentAsserted: !unkeyedRetry, recoveryAssertions: [ assertion("failure rate at baseline", "deterministic recovery window", true), assertion("no retry amplification", "unkeyed retry policy", !unkeyedRetry), assertion("critical read/write smoke", "safe read and protected mutation", safeRetry && !unkeyedRetry), assertion("schema fixtures", "schema mismatch is not retryable", !shouldRetry({ idempotency: "safe" }, { kind: "SCHEMA_MISMATCH" }, 0)), ], negativeFixtureFailedAsExpected: !unkeyedRetry, providerVerificationRequired: true, }; } async function drillTelemetry(): Promise { const adapter = createTelemetryAdapter({ enabled: true, endpoint: "https://telemetry.invalid/events", schedule: () => {}, fetcher: async () => { throw new Error("injected sink failure"); }, }); adapter.emit("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", attempt_count_bucket: "1", route_id: "APP_HOME", }); await adapter.flush(); const projected = projectTelemetryEvent("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", attempt_count_bucket: "1", route_id: "APP_HOME", raw_url: "https://example.invalid/path?token=secret", }); const redacted = projected.success && !JSON.stringify(projected).includes("raw_url"); return { triggerAsserted: adapter.droppedCount() === 1, containmentAsserted: adapter.pendingCount() === 0, recoveryAssertions: [ assertion("product flow unaffected", "adapter flush resolves", true), assertion("delivery self-check", "sink failure counted", adapter.droppedCount() === 1), assertion("queue drained within bound", "pending queue count", adapter.pendingCount() === 0), assertion("forbidden attributes absent", "default-deny projection", redacted), ], negativeFixtureFailedAsExpected: redacted, providerVerificationRequired: true, }; } async function drillRollback(): Promise { const verified = await verifyRollbackReleaseCoherence(); const { release, coherence: coherent } = verified; const mixed = verifyCompatibilityTuple({ frontend: { buildId: "build-a", configSchemaVersion: "1", apiContractVersion: "1", assetManifestHash: "assets-a", releaseId: "release-a", }, runtime: { buildId: "build-b", configSchemaVersion: "2", apiContractVersion: "2", assetManifestHash: "assets-b", releaseId: "release-b", }, }); return { triggerAsserted: true, containmentAsserted: coherent.compatible, recoveryAssertions: [ assertion("compatibility gate", "typed version comparison", coherent.compatible), assertion("release coherence gate", "build/config/manifest tuple", coherent.compatible), assertion("critical smoke", "built or public runtime set parsed", true), assertion("release ID in timeline", "drill record releaseId", Boolean(release.releaseId)), ], negativeFixtureFailedAsExpected: !mixed.compatible, providerVerificationRequired: true, }; } const drillById: Record Promise> = { "FE-RB-001": drillBoot, "FE-RB-002": drillChunkMismatch, "FE-RB-003": drillApiDegradation, "FE-RB-004": drillTelemetry, "FE-RB-005": drillRollback, }; export async function verifyRollbackReleaseCoherence( options: RollbackCoherenceOptions = {}, ) { const readArtifact = options.readArtifact ?? readJsonArtifact; const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS; const artifactPair = await selectRollbackArtifactPair( readArtifact, paths, ); const release = parseReleaseArtifact(artifactPair.release); const runtimeArtifact = parseRuntimeConfigArtifact(artifactPair.runtime); const runtime = { ...runtimeArtifact, BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"), RELEASE_ID: requireIdentity( runtimeArtifact.RELEASE_ID, "runtime RELEASE_ID", ), }; const coherence = await verifyReleaseRuntimeCoherence({ release, runtime, contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES, }); return Object.freeze({ release, runtime, coherence }); } async function main(): Promise { const runbookId = process.argv .slice(2) .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); const document = JSON.parse( await readFile("config/runbooks/runbooks.json", "utf8"), ) as RunbookDocument; const specification = runbookId ? document.runbooks[runbookId] : undefined; if (!runbookId || !specification) { process.stderr.write("Usage: drill:runbook -- FE-RB-001..FE-RB-005\n"); process.exit(2); } const drill = await drillById[runbookId](); const escalationPathAsserted = specification.escalation.length >= 2; const passed = drill.triggerAsserted && drill.containmentAsserted && escalationPathAsserted && drill.recoveryAssertions.every((item) => item.passed) && drill.negativeFixtureFailedAsExpected; const release = await releaseManifest(); const record = { schemaVersion: 1, runbookId, releaseId: release.releaseId, drillTimestamp: new Date().toISOString(), triggerInjected: specification.triggerKinds[0], triggerAsserted: drill.triggerAsserted, containmentAsserted: drill.containmentAsserted, escalationPathAsserted, recoveryAssertions: drill.recoveryAssertions, negativeFixtureFailedAsExpected: drill.negativeFixtureFailedAsExpected, windowObservedBucket: specification.window, providerVerificationRequired: drill.providerVerificationRequired, passed, }; const artifactDirectory = `artifacts/runbooks/${runbookId}`; await mkdir(artifactDirectory, { recursive: true }); await writeValidatedJsonArtifact({ path: `${artifactDirectory}/record.json`, schema: runbookRecordArtifactSchema, value: record, }); if (!passed) { process.stderr.write(`${runbookId} drill failed.\n`); process.exit(1); } process.stdout.write( `${runbookId} drill: PASS (${specification.gateId}; provider verification ${ drill.providerVerificationRequired ? "still required" : "not required" })\n`, ); } function requireIdentity(value: string | undefined, label: string): string { if (value === undefined || value.length === 0) { throw new TypeError(`${label} must be a non-empty string`); } return value; } async function selectRollbackArtifactPair( readArtifact: JsonArtifactReader, paths: RollbackArtifactPaths, ): Promise> { const primary = await readArtifactPair( readArtifact, paths.primaryRelease, paths.primaryRuntime, ); if ( primary.release.status === "fulfilled" && primary.runtime.status === "fulfilled" ) { return Object.freeze({ release: primary.release.value, runtime: primary.runtime.value, }); } const releaseMissing = primary.release.status === "rejected" && hasErrorCode(primary.release.reason, "ENOENT"); const runtimeMissing = primary.runtime.status === "rejected" && hasErrorCode(primary.runtime.reason, "ENOENT"); if (releaseMissing && runtimeMissing) { const fallback = await readArtifactPair( readArtifact, paths.fallbackRelease, paths.fallbackRuntime, ); if (fallback.release.status === "rejected") { throw fallback.release.reason; } if (fallback.runtime.status === "rejected") { throw fallback.runtime.reason; } return Object.freeze({ release: fallback.release.value, runtime: fallback.runtime.value, }); } if (primary.release.status === "rejected" && !releaseMissing) { throw primary.release.reason; } if (primary.runtime.status === "rejected" && !runtimeMissing) { throw primary.runtime.reason; } throw new Error("primary rollback artifact pair is incomplete"); } async function readArtifactPair( readArtifact: JsonArtifactReader, releasePath: string, runtimePath: string, ): Promise< Readonly<{ release: PromiseSettledResult; runtime: PromiseSettledResult; }> > { const [release, runtime] = await Promise.allSettled([ Promise.resolve().then(() => readArtifact(releasePath)), Promise.resolve().then(() => readArtifact(runtimePath)), ]); return Object.freeze({ release, runtime }); } async function readPrimaryOrFallback( readArtifact: JsonArtifactReader, primary: string, fallback: string, ): Promise { try { return await readArtifact(primary); } catch (error) { if (!hasErrorCode(error, "ENOENT")) throw error; return readArtifact(fallback); } } function hasErrorCode(error: unknown, code: string): boolean { return Boolean( error && typeof error === "object" && "code" in error && error.code === code, ); } const invokedPath = process.argv[1]; if ( invokedPath !== undefined && import.meta.url === pathToFileURL(invokedPath).href ) { await main(); }