import { access, mkdir, readFile } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { createContractHttpExecutor } from "../src/adapters/http/index.ts"; import { createTelemetryAdapter } from "../src/adapters/telemetry/best-effort-telemetry.ts"; import { verifyCompatibilityTuple } from "../src/contracts/compatibility.ts"; import type { StoragePort } from "../src/application/ports/storage-port.ts"; import { decideChunkRecovery } from "../src/application/use-cases/decide-chunk-recovery.ts"; import { validateRuntimeConfig } from "../src/bootstrap/runtime-config-schema.ts"; import type { InstalledContractPackageIdentity, RetrySemantics, } from "../src/contracts/external-contract-runtime.ts"; import { z } from "../src/contracts/zod.ts"; import { zodValidator } from "../src/contracts/zod-validator.ts"; import { parseReleaseArtifact, parseRuntimeConfigArtifact, } from "../src/contracts/release-artifacts.ts"; import { projectTelemetryEvent } from "../src/contracts/telemetry.ts"; import { EXPECTED_CONTRACT_SET_PACKAGES } from "../src/features/installed-contract-contributions.ts"; import { combineDrillVerdicts, runbookRecordArtifactSchema, type DrillVerdict, } from "./contracts/release-artifacts.ts"; import { verifyReleaseRuntimeCoherence } from "./lib/release-runtime-coherence.ts"; import { writeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; // 11-01 / 13-03 (D-14). A verdict is PASS or FAIL only when the drill observed // it on the local build. What needs a browser boot, live traffic or the hosting // provider is UNVERIFIED, and so is every drill whose runbook needs provider // verification: a local drill never claims the release can be recovered. type RecoveryAssertion = Readonly<{ assertion: string; evidence: string; result: DrillVerdict; }>; type DrillResult = Readonly<{ triggerAsserted: DrillVerdict; containmentAsserted: DrillVerdict; recoveryAssertions: RecoveryAssertion[]; negativeFixtureFailedAsExpected: DrillVerdict; providerVerificationRequired: boolean; }>; type RunbookSpecification = Readonly<{ title: string; gateId: string; triggerKinds: string[]; containment: string; window: string; escalation: string[]; recoveryEvidence: string[]; negativeFixture: string; }>; type RunbookDocument = Readonly<{ runbooks: Record; }>; export type JsonArtifactReader = (path: string) => Promise; export type RollbackArtifactPaths = Readonly<{ release: string; runtime: string; }>; export type RollbackCoherenceOptions = Readonly<{ readArtifact?: JsonArtifactReader; contractPackages?: readonly InstalledContractPackageIdentity[]; paths?: RollbackArtifactPaths; }>; export type DrillOptions = Readonly<{ /** * Directory holding the release's `dist/`. Defaults to the cwd (the local * build). 13-04: pass an extracted candidate or promoted archive to drill * the release that is actually deployed or rolled back to. */ releaseRoot?: string; contractPackages?: readonly InstalledContractPackageIdentity[]; }>; type DrillContext = Readonly<{ root: string; contractPackages: readonly InstalledContractPackageIdentity[]; }>; // 13-03: the drill reads the built release only. It used to fall back to the // placeholders in public/, so it passed without any build at all. const DEFAULT_ROLLBACK_PATHS = Object.freeze({ release: "dist/release-manifest.json", runtime: "dist/config.json", }); async function readJsonArtifact(path: string): Promise { try { return JSON.parse(await readFile(path, "utf8")); } catch (error) { if (hasErrorCode(error, "ENOENT")) { throw new Error(`${path} is missing; build the release before running the drill`, { cause: error }); } throw error; } } function rootedReader(root: string): JsonArtifactReader { return (file) => readJsonArtifact(path.join(root, file)); } const validConfig = { APP_ENV: "local", API_BASE_URL: "http://localhost:8080", REQUEST_TIMEOUT_MS: 10_000, MAX_RETRY_ATTEMPTS: 2, TELEMETRY_ENABLED: false, AUTH_MODE: "external", CONFIG_SCHEMA_VERSION: "1", API_CONTRACT_VERSION: "1", RELEASE_MANIFEST_URL: "/release-manifest.json", BUILD_ID: "local-build", RELEASE_ID: "local-release", }; function verdict(passed: boolean): DrillVerdict { return passed ? "PASS" : "FAIL"; } function assertion( assertion: string, evidence: string, result: DrillVerdict, ): RecoveryAssertion { return { assertion, evidence, result }; } /** An assertion the local drill cannot observe; `needs` names what can. */ function notObserved(name: string, needs: string): RecoveryAssertion { return assertion(name, `not observed locally; needs ${needs}`, "UNVERIFIED"); } /** * Every file the Vite manifest names (entries, lazy chunks, CSS, assets) exists * in the local dist. Hosting reachability still needs the provider smoke. */ async function builtAssetsPresent(root: string): Promise { const manifest = await readJsonArtifact(path.join(root, "dist/.vite/manifest.json")); if (!manifest || typeof manifest !== "object") return false; const files = new Set(); for (const chunk of Object.values(manifest as Record>)) { for (const key of ["file", "css", "assets"] as const) { const value = chunk[key]; for (const file of Array.isArray(value) ? value : [value]) { if (typeof file === "string") files.add(file); } } } if (![...files].some((file) => file.endsWith(".js"))) return false; try { await Promise.all( ["index.html", ...files].map((file) => access(path.join(root, "dist", file))), ); return true; } catch { return false; } } async function drillBoot(): Promise { const invalid = validateRuntimeConfig({ ...validConfig, APP_ENV: "production", API_BASE_URL: "http://insecure.invalid", }); const recovered = validateRuntimeConfig(validConfig); return { triggerAsserted: verdict(!invalid.success), containmentAsserted: verdict(!invalid.success), recoveryAssertions: [ notObserved("clean-session boot", "a browser boot of the promoted release"), notObserved("product root mount", "a browser boot of the promoted release"), assertion( "config validation", "invalid fixture rejected, valid fixture accepted", verdict(!invalid.success && recovered.success), ), notObserved("no repeated boot error", "boot telemetry from the promoted release"), ], // The runbook's negative fixture injects a mount failure after a valid // config; that needs a mounted app, which a Node drill does not have. negativeFixtureFailedAsExpected: "UNVERIFIED", providerVerificationRequired: false, }; } function memoryStorage(): StoragePort { let value: unknown; return { read: () => ({ ok: true, value }), write: (_key: string, next: unknown) => { value = next; return { ok: true }; }, remove: () => ({ ok: true }), }; } async function drillChunkMismatch(context: DrillContext): Promise { const storage = memoryStorage(); const input: Parameters[0] = { failureKind: "DEPLOY_MISMATCH", manifestLoaded: true, currentBuildId: "build-a", currentReleaseId: "release-a", activeBuildId: "build-b", activeReleaseId: "release-b", storage, }; const first = decideChunkRecovery(input); const second = decideChunkRecovery(input); const { coherence } = await verifyRollbackReleaseCoherence({ readArtifact: rootedReader(context.root), contractPackages: context.contractPackages, }); return { triggerAsserted: verdict(first.action === "reload-once"), containmentAsserted: verdict( first.action === "reload-once" && second.action === "support", ), recoveryAssertions: [ assertion( "entry and lazy assets reachable", "every Vite manifest file exists in the local dist", verdict(await builtAssetsPresent(context.root)), ), assertion("release tuple coherent", "build/config/manifest tuple", verdict(coherence.compatible)), assertion("second reload blocked", "reload guard decision", verdict(second.action === "support")), notObserved("critical route smoke", "a browser smoke against the hosted release"), ], negativeFixtureFailedAsExpected: verdict(second.action !== "reload-once"), providerVerificationRequired: true, }; } /** * One operation through the V3 executor against a fixed answer. Returns how * many requests it sent, which is what retry amplification is measured in. */ async function physicalAttempts( retrySemantics: Extract, answer: () => Response, ): Promise { const read = retrySemantics === "SAFE"; let sent = 0; const executor = createContractHttpExecutor({ baseUrl: "https://api.invalid", maxRetryAttempts: 2, attachCredentials: () => ({ kind: "READY" as const, headers: {} }), fetcher: (async () => { sent += 1; return answer(); }) as typeof fetch, sleep: async () => {}, random: () => 0, }); await executor.execute( Object.freeze({ contract: Object.freeze({ operationId: "DRILL_API_DEGRADATION", method: read ? ("GET" as const) : ("POST" as const), pathTemplate: "/api/drill", inputValidator: zodValidator("DrillInput", z.object({}).strict()), outputValidator: zodValidator( "DrillOutput", z.object({ id: z.string() }).strict(), ), problemValidator: zodValidator( "DrillProblem", z.object({ code: z.string() }).strip(), ), acceptedStatuses: Object.freeze([200]), emptyBodyStatuses: Object.freeze([]), retrySemantics, requestBody: read ? ("NONE" as const) : ("JSON" as const), responseBody: "REQUIRED_JSON" as const, commandRecovery: null, commandEffect: read ? null : Object.freeze({ successEffect: "APPLIED_CONFIRMED" as const, classifyProblem: () => "MAYBE_APPLIED" as const, }), projectRequest: () => Object.freeze({ pathValues: Object.freeze({}), queryEntries: Object.freeze([]), body: read ? null : Object.freeze({}), }), }), frontend: Object.freeze({ policyId: "DRILL_API_V1", requestByteLimit: read ? 0 : 1_024, responseByteLimit: 4_096, totalDeadlineMs: 5_000, retryBudget: read ? (2 as const) : (0 as const), authProfileId: "ANONYMOUS", diagnosticsOperation: "drill.api", }), }), Object.freeze({}), { routeId: "DRILL_ROUTE", scope: Object.freeze({ generation: 1, fingerprint: "drill-scope", identities: Object.freeze({}) as never, signal: new AbortController().signal, isCurrent: () => true, }), }, ); return sent; } async function drillApiDegradation(): Promise { const unavailable = () => Response.json({ code: "TEMPORARY" }, { status: 503 }); const unkeyedRetry = (await physicalAttempts("NEVER", unavailable)) > 1; const safeRetry = (await physicalAttempts("SAFE", unavailable)) > 1; const schemaRetry = (await physicalAttempts("SAFE", () => Response.json({ unexpected: true }))) > 1; return { // The trigger is a failure rate over live traffic. triggerAsserted: "UNVERIFIED", containmentAsserted: verdict(!unkeyedRetry && safeRetry), recoveryAssertions: [ notObserved("terminal failure rate at baseline", "API failure telemetry from the promoted release"), assertion("no retry amplification", "unkeyed retry policy", verdict(!unkeyedRetry)), notObserved("critical read/write smoke", "a read and a write against the live API"), assertion( "schema fixtures", "schema mismatch is not retryable", verdict(!schemaRetry), ), ], negativeFixtureFailedAsExpected: verdict(!unkeyedRetry), providerVerificationRequired: true, }; } async function drillTelemetry(): Promise { const adapter = createTelemetryAdapter({ enabled: true, endpoint: "https://telemetry.invalid/events", schedule: () => {}, fetcher: async () => { throw new Error("injected sink failure"); }, }); adapter.emit("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", attempt_count_bucket: "1", route_id: "APP_HOME", }); const flushResolved = await adapter.flush().then( () => true, () => false, ); const projected = projectTelemetryEvent("api.request.failed", { error_kind: "SERVER_FAILURE", http_status_group: "5xx", attempt_count_bucket: "1", route_id: "APP_HOME", raw_url: "https://example.invalid/path?token=secret", }); const redacted = projected.success && !JSON.stringify(projected).includes("raw_url"); return { triggerAsserted: verdict(adapter.droppedCount() === 1), containmentAsserted: verdict(adapter.pendingCount() === 0), recoveryAssertions: [ assertion("product flow unaffected", "adapter flush resolves despite the sink failure", verdict(flushResolved)), assertion("delivery self-check", "sink failure counted", verdict(adapter.droppedCount() === 1)), assertion("queue drained within bound", "pending queue count", verdict(adapter.pendingCount() === 0)), assertion("forbidden attributes absent", "default-deny projection", verdict(redacted)), ], negativeFixtureFailedAsExpected: verdict(redacted), providerVerificationRequired: true, }; } async function drillRollback(context: DrillContext): Promise { const { release, coherence: coherent } = await verifyRollbackReleaseCoherence({ readArtifact: rootedReader(context.root), contractPackages: context.contractPackages, }); const assetsPresent = await builtAssetsPresent(context.root); const mixed = verifyCompatibilityTuple({ frontend: { buildId: "build-a", configSchemaVersion: "1", apiContractVersion: "1", assetManifestHash: "assets-a", releaseId: "release-a", }, runtime: { buildId: "build-b", configSchemaVersion: "2", apiContractVersion: "2", assetManifestHash: "assets-b", releaseId: "release-b", }, }); return { // A release-blocking defect is declared by a person during an incident. triggerAsserted: "UNVERIFIED", containmentAsserted: verdict(coherent.compatible && assetsPresent), recoveryAssertions: [ assertion("compatibility gate", "typed version comparison", verdict(coherent.compatible)), assertion( "release coherence gate", "build/config/manifest tuple and every Vite manifest file in the local dist", verdict(coherent.compatible && assetsPresent), ), notObserved("critical smoke", "a browser smoke after the provider switches to the prior release"), assertion("release ID in timeline", "drill record releaseId", verdict(Boolean(release.releaseId))), ], negativeFixtureFailedAsExpected: verdict(!mixed.compatible), providerVerificationRequired: true, }; } const drillById: Record Promise> = { "FE-RB-001": drillBoot, "FE-RB-002": drillChunkMismatch, "FE-RB-003": drillApiDegradation, "FE-RB-004": drillTelemetry, "FE-RB-005": drillRollback, }; export async function verifyRollbackReleaseCoherence( options: RollbackCoherenceOptions = {}, ) { const readArtifact = options.readArtifact ?? readJsonArtifact; const paths = options.paths ?? DEFAULT_ROLLBACK_PATHS; const [releaseValue, runtimeValue] = await Promise.all([ readArtifact(paths.release), readArtifact(paths.runtime), ]); const release = parseReleaseArtifact(releaseValue); const runtimeArtifact = parseRuntimeConfigArtifact(runtimeValue); const runtime = { ...runtimeArtifact, BUILD_ID: requireIdentity(runtimeArtifact.BUILD_ID, "runtime BUILD_ID"), RELEASE_ID: requireIdentity( runtimeArtifact.RELEASE_ID, "runtime RELEASE_ID", ), }; const coherence = await verifyReleaseRuntimeCoherence({ release, runtime, contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES, }); return Object.freeze({ release, runtime, coherence }); } export async function runRunbookDrill(runbookId: string, options: DrillOptions = {}) { const context: DrillContext = { root: options.releaseRoot ?? process.cwd(), contractPackages: options.contractPackages ?? EXPECTED_CONTRACT_SET_PACKAGES, }; const document = JSON.parse( await readFile("config/runbooks/runbooks.json", "utf8"), ) as RunbookDocument; const specification = document.runbooks[runbookId]; const drill = drillById[runbookId]; if (!specification || !drill) throw new RangeError(`unknown runbook: ${runbookId}`); const release = parseReleaseArtifact( await rootedReader(context.root)(DEFAULT_ROLLBACK_PATHS.release), ); const result = await drill(context); const escalationPathAsserted = verdict(specification.escalation.length >= 2); return { schemaVersion: 2 as const, runbookId, gateId: specification.gateId, releaseId: release.releaseId, drillTimestamp: new Date().toISOString(), triggerInjected: specification.triggerKinds[0] ?? "", triggerAsserted: result.triggerAsserted, containmentAsserted: result.containmentAsserted, escalationPathAsserted, recoveryAssertions: result.recoveryAssertions, negativeFixtureFailedAsExpected: result.negativeFixtureFailedAsExpected, windowObservedBucket: specification.window, providerVerificationRequired: result.providerVerificationRequired, result: combineDrillVerdicts( [ result.triggerAsserted, result.containmentAsserted, escalationPathAsserted, ...result.recoveryAssertions.map((item) => item.result), result.negativeFixtureFailedAsExpected, ], result.providerVerificationRequired, ), }; } async function main(): Promise { const runbookId = process.argv .slice(2) .find((argument) => /^FE-RB-00[1-5]$/.test(argument)); const releaseRootFlag = process.argv.indexOf("--release-root"); const releaseRoot = releaseRootFlag >= 0 ? process.argv[releaseRootFlag + 1] : undefined; if (!runbookId || (releaseRootFlag >= 0 && !releaseRoot)) { process.stderr.write( "Usage: drill:runbook -- FE-RB-001..FE-RB-005 [--release-root ]\n", ); process.exit(2); } const { gateId, ...record } = await runRunbookDrill(runbookId, releaseRoot ? { releaseRoot } : {}); const artifactDirectory = `artifacts/runbooks/${runbookId}`; await mkdir(artifactDirectory, { recursive: true }); await writeValidatedJsonArtifact({ path: `${artifactDirectory}/record.json`, schema: runbookRecordArtifactSchema, value: record, }); if (record.result === "FAIL") { process.stderr.write(`${runbookId} drill: FAIL (${gateId})\n`); process.exit(1); } if (record.result === "UNVERIFIED") { // Not a pass: the gate stays red until provider evidence exists (D-14). const open = [ ["trigger", record.triggerAsserted], ["containment", record.containmentAsserted], ...record.recoveryAssertions.map((item) => [item.assertion, item.result] as const), ["negative fixture", record.negativeFixtureFailedAsExpected], ["provider verification", record.providerVerificationRequired ? "UNVERIFIED" : "PASS"], ].filter(([, result]) => result === "UNVERIFIED").map(([name]) => name); process.stderr.write( `${runbookId} drill: UNVERIFIED (${gateId}; no local check failed; not observed locally: ${open.join(", ")})\n`, ); process.exit(3); } process.stdout.write(`${runbookId} drill: PASS (${gateId})\n`); } function requireIdentity(value: string | undefined, label: string): string { if (value === undefined || value.length === 0) { throw new TypeError(`${label} must be a non-empty string`); } return value; } function hasErrorCode(error: unknown, code: string): boolean { return Boolean( error && typeof error === "object" && "code" in error && error.code === code, ); } const invokedPath = process.argv[1]; if ( invokedPath !== undefined && import.meta.url === pathToFileURL(invokedPath).href ) { await main(); }