import { describe, expect, it, vi } from "vitest"; import type { ClockPort } from "../../../src/application/ports/clock-port.ts"; import type { RealtimeResult, } from "../../../src/application/ports/realtime/shared.ts"; import { REALTIME_TRANSPORT_CONTINUE, type RealtimeRecoveryCheckpoint, } from "../../../src/application/ports/realtime/event-authority.ts"; import { createRealtimeReconnectCoordinator, type RealtimeCommittedRecovery, type RealtimeReconnectAttemptContext, type RealtimeReconnectAttemptSuccess, type RealtimeReconnectEnvironment, type RealtimeReconnectOutcome, type RealtimeReconnectSession, type RealtimeRecoveryReconnectDirective, } from "../../../src/adapters/realtime/index.ts"; import { defineReconnectPolicy, REALTIME_RECONNECT_CEILINGS, type ReconnectPolicy, } from "../../../src/adapters/realtime/reconnect-policy.ts"; import { realtimeFailure, realtimeSuccess, } from "../../../src/adapters/realtime/result.ts"; import { createFetchSseConnection, readSseForReconnect, SSE_CONTINUE, } from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts"; import { createWebSocketConnection, type WebSocketClosedReceipt, type WebSocketFacade, } from "../../../src/adapters/realtime/websocket/websocket-connection.ts"; import { REALTIME_WEBSOCKET_PROTOCOL, } from "../../../src/adapters/realtime/websocket/websocket-protocol.ts"; type Sleeper = Readonly<{ dueAt: number; signal?: AbortSignal; resolve(): void; reject(): void; }>; class ManualClock implements ClockPort { current = 0; readonly sleepCalls: number[] = []; readonly sleepers: Sleeper[] = []; now(): number { return this.current; } sleep( milliseconds: number, signal?: AbortSignal, ): Promise { this.sleepCalls.push(milliseconds); return new Promise((resolve, reject) => { if (signal?.aborted) { reject(new Error("Aborted.")); return; } let onAbort: (() => void) | undefined; const sleeper: Sleeper = { dueAt: this.current + milliseconds, signal, resolve: () => { signal?.removeEventListener("abort", onAbort!); resolve(); }, reject: () => { signal?.removeEventListener("abort", onAbort!); reject(new Error("Aborted.")); }, }; onAbort = () => { this.remove(sleeper); sleeper.reject(); }; signal?.addEventListener("abort", onAbort, { once: true }); this.sleepers.push(sleeper); }); } advance(milliseconds: number): void { this.current += milliseconds; const ready = this.sleepers .filter((sleeper) => sleeper.dueAt <= this.current) .sort((left, right) => left.dueAt - right.dueAt); for (const sleeper of ready) { this.remove(sleeper); sleeper.resolve(); } } private remove(target: Sleeper): void { const index = this.sleepers.indexOf(target); if (index >= 0) this.sleepers.splice(index, 1); } } class MutableOnlineEnvironment implements RealtimeReconnectEnvironment { readonly listeners = new Set<(online: boolean) => void>(); constructor(public connected: boolean) {} online(): boolean { return this.connected; } subscribeOnline( listener: (online: boolean) => void, ): () => void { this.listeners.add(listener); return () => this.listeners.delete(listener); } emit(online: boolean): void { this.connected = online; for (const listener of this.listeners) listener(online); } } function policy( override: Partial = {}, ): ReconnectPolicy { return defineReconnectPolicy({ baseDelayMs: 100, maxDelayMs: 1_000, maxAttempts: 2, maxElapsedMs: 5_000, stableOpenMs: 500, ...override, }); } function failed( result: Extract, { ok: false }>, serverNotBeforeMs?: number, ): RealtimeReconnectOutcome { return Object.freeze({ result, ...(serverNotBeforeMs === undefined ? {} : { serverNotBeforeMs }), }); } function opened( session: RealtimeReconnectSession, establishedRecoveryBarrier?: RealtimeCommittedRecovery, ): RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess > { return Object.freeze({ result: realtimeSuccess( Object.freeze({ session, ...(establishedRecoveryBarrier === undefined ? {} : { establishedRecoveryBarrier }), }), ), }); } function closedSuccessfully(): RealtimeReconnectOutcome { return Object.freeze({ result: realtimeSuccess(undefined), }); } function confirmationSucceeded(): RealtimeResult { return realtimeSuccess(undefined); } function recoveryToken( sequence = "7", ): RealtimeCommittedRecovery { const checkpoint = Object.freeze({ lastAppliedSequence: sequence, recoveryMode: "CURSOR", resumeCursor: `cursor-${sequence}`, streamEpoch: "epoch-1", }) as unknown as RealtimeRecoveryCheckpoint; return Object.freeze({ kind: "RECOVERY_COMMITTED", streamId: "orders", checkpoint, }) as unknown as RealtimeCommittedRecovery; } function recoveryDirective( recovery: RealtimeCommittedRecovery, terminalResult: Extract< RealtimeResult, { ok: false } >, serverNotBeforeMs?: number, ): RealtimeRecoveryReconnectDirective { return Object.freeze({ kind: "RECOVERY_RECONNECT", recovery, terminalResult, ...(serverNotBeforeMs === undefined ? {} : { serverNotBeforeMs }), }); } function deferred(): Readonly<{ promise: Promise; resolve(value: Value): void; }> { let resolvePromise: ((value: Value) => void) | undefined; const promise = new Promise((resolve) => { resolvePromise = resolve; }); return Object.freeze({ promise, resolve(value) { resolvePromise?.(value); }, }); } async function flush(): Promise { // Enough turns for a retry loop iteration: sleep, attempt and close races. for (let turn = 0; turn < 16; turn += 1) { await Promise.resolve(); } } async function flushUntil( condition: () => boolean, ): Promise { for (let turn = 0; turn < 50; turn += 1) { if (condition()) return; await Promise.resolve(); } } describe("realtime reconnect coordinator", () => { it("owns one bounded retry loop and returns the exact last failure", async () => { const clock = new ManualClock(); const environment = new MutableOnlineEnvironment(true); const failures = [ realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true), realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true), realtimeFailure("IDLE_TIMEOUT", "RECEIVE", true), ] as const; let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: () => { const next = failures[attempts++]!; return failed( next, next === failures[1] ? 0 : undefined, ); }, }); const running = coordinator.run(); await flush(); expect(attempts).toBe(1); expect(clock.sleepCalls).toEqual([50]); clock.advance(50); await flush(); expect(attempts).toBe(2); expect(clock.sleepCalls).toEqual([50, 100]); clock.advance(100); const result = await running; expect(attempts).toBe(3); expect(result).toBe(failures[2]); expect(coordinator.getState()).toBe("IDLE"); }); it("does not clamp a server not-before value past elapsed budget", async () => { const clock = new ManualClock(); const terminal = realtimeFailure( "RATE_LIMITED", "CONNECT", true, ); const random = vi.fn(() => 0); const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1, maxElapsedMs: 500, }), environment: new MutableOnlineEnvironment(true), clock, random, confirmTransportBarrier: confirmationSucceeded, attempt: () => failed(terminal, 500), }); const result = await coordinator.run(); expect(result).toBe(terminal); expect(clock.sleepCalls).toEqual([]); expect(random).not.toHaveBeenCalled(); }); it.each([ "RATE_LIMITED", "PROVIDER_UNAVAILABLE", ] as const)( "reconnects a retryable external %s attempt failure without a server hint", async (kind) => { const clock = new ManualClock(); const retryable = realtimeFailure(kind, "CONNECT", true); const terminal = realtimeFailure(kind, "CONNECT", false); let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: () => failed(++attempts === 1 ? retryable : terminal), }); const running = coordinator.run(); await flush(); expect(attempts).toBe(1); expect(clock.sleepCalls).toEqual([50]); clock.advance(50); const result = await running; expect(attempts).toBe(2); expect(result).toBe(terminal); }, ); it("reconnects a retryable external close failure without a server hint", async () => { const clock = new ManualClock(); const retryable = realtimeFailure( "PROVIDER_UNAVAILABLE", "RECEIVE", true, ); const terminal = realtimeFailure( "PROVIDER_UNAVAILABLE", "RECEIVE", false, ); let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: () => { const closedWith = ++attempts === 1 ? retryable : terminal; return opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: vi.fn(), waitClosed: () => failed(closedWith), }), ); }, }); const running = coordinator.run(); await flush(); expect(attempts).toBe(1); expect(clock.sleepCalls).toEqual([50]); clock.advance(50); const result = await running; expect(attempts).toBe(2); expect(result).toBe(terminal); }); it("waits for an explicit online signal without a retry timer", async () => { const clock = new ManualClock(); const environment = new MutableOnlineEnvironment(false); const terminal = realtimeFailure( "FORBIDDEN", "CONNECT", false, ); const attempt = vi.fn(() => failed(terminal)); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt, }); const running = coordinator.run(); await flush(); clock.advance(100); await flush(); expect(attempt).not.toHaveBeenCalled(); expect(clock.sleepCalls).toEqual([]); environment.emit(true); await flush(); expect(clock.sleepCalls).toEqual([50]); expect(attempt).not.toHaveBeenCalled(); clock.advance(50); const result = await running; expect(attempt).toHaveBeenCalledTimes(1); expect(result).toBe(terminal); }); it("does not spend the elapsed budget while waiting offline", async () => { const clock = new ManualClock(); const environment = new MutableOnlineEnvironment(false); const terminal = realtimeFailure( "FORBIDDEN", "CONNECT", false, ); const attempt = vi.fn(() => failed(terminal)); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt, }); const running = coordinator.run(); await flush(); // Offline for longer than maxElapsedMs before the first attempt. clock.advance(6_000); environment.emit(true); await flushUntil(() => clock.sleepCalls.length === 1); expect(clock.sleepCalls).toEqual([50]); clock.advance(50); const result = await running; expect(attempt).toHaveBeenCalledTimes(1); expect(result).toBe(terminal); }); it.each(["an offline blip", "a waitClosed throw"] as const)( "reconnects after %s ends a healthy session older than the elapsed budget", async (ending) => { const clock = new ManualClock(); const environment = new MutableOnlineEnvironment(true); const closing = deferred>(); const terminal = realtimeFailure( "FORBIDDEN", "CONNECT", false, ); let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: (context: RealtimeReconnectAttemptContext) => { attempts += 1; if (attempts > 1) return failed(terminal); context.markValidHeartbeatOrEvent(); return opened>( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: () => closing.resolve( failed( realtimeFailure("OFFLINE", "RECEIVE", true), ), ), waitClosed: () => ending === "an offline blip" ? closing.promise : closing.promise.then(() => { throw new Error("transport failed"); }), }), ); }, }); const running = coordinator.run(); await flushUntil(() => attempts === 1); await flush(); // Healthy for twice maxElapsedMs, then a one second outage. clock.advance(10_000); if (ending === "an offline blip") { environment.emit(false); await flush(); clock.advance(1_000); environment.emit(true); } else { closing.resolve(closedSuccessfully()); } await flushUntil(() => clock.sleepCalls.length === 1); expect(clock.sleepCalls).toEqual([50]); clock.advance(50); const result = await running; expect(attempts).toBe(2); expect(result).toBe(terminal); }, ); it("observes attempts, retries, the session and why the run ended", async () => { const clock = new ManualClock(); const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false); const observations: unknown[] = []; let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, observe: (observation) => { observations.push(observation); throw new Error("sink failure must not change the run"); }, attempt: () => ++attempts === 1 ? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true)) : opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: vi.fn(), waitClosed: () => failed(terminal), }), ), }); const running = coordinator.run(); await flushUntil(() => clock.sleepCalls.length === 1); clock.advance(50); await expect(running).resolves.toBe(terminal); expect(observations).toEqual([ { kind: "ATTEMPT_STARTED" }, { kind: "RETRY_SCHEDULED", failureKind: "CONNECT_TIMEOUT", delayBucket: "UNDER_1S", }, { kind: "ATTEMPT_STARTED" }, { kind: "SESSION_OPENED" }, { kind: "SESSION_CLOSED" }, { kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" }, ]); // A second run that spends its retries ends as EXHAUSTED, not TERMINAL. const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true); const ends: unknown[] = []; const spending = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier: confirmationSucceeded, observe: (observation) => { if (observation.kind === "RUN_ENDED") ends.push(observation); }, attempt: () => failed(exhausted), }); const spent = spending.run(); await flushUntil(() => clock.sleepCalls.length === 2); clock.advance(0); await expect(spent).resolves.toBe(exhausted); expect(ends).toEqual([ { kind: "RUN_ENDED", reason: "EXHAUSTED", failureKind: "PROVIDER_UNAVAILABLE", }, ]); }); it("resets a spent retry only after a valid signal and a stable open", async () => { const runScenario = async (markValid: boolean, openMs: number) => { const clock = new ManualClock(); const firstFailure = realtimeFailure( "CONNECT_TIMEOUT", "CONNECT", true, ); const sessionFailure = realtimeFailure( "PROVIDER_UNAVAILABLE", "RECEIVE", true, ); const terminal = realtimeFailure( "FORBIDDEN", "CONNECT", false, ); const closeSession = vi.fn(); const closing = deferred>(); let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier: confirmationSucceeded, attempt: ( context: RealtimeReconnectAttemptContext, ) => { attempts += 1; if (attempts === 1) return failed(firstFailure); if (attempts === 2) { if (markValid) { context.markValidHeartbeatOrEvent(); } return opened>( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: closeSession, waitClosed: () => closing.promise, }), ); } return failed(terminal); }, }); const running = coordinator.run(); await flush(); clock.advance(0); await flushUntil(() => attempts === 2); await flush(); clock.advance(openMs); closing.resolve(failed(sessionFailure, 0)); await flushUntil(() => clock.sleepCalls.length === 2); clock.advance(0); const result = await running; return { attempts, closeSession, result, sessionFailure, terminal, }; }; // A heartbeat on a short-lived connection, or a quiet long one, keeps // the spent budget; only both together reset it. for (const [markValid, openMs] of [ [false, 0], [true, 499], [false, 500], ] as const) { const spent = await runScenario(markValid, openMs); expect(spent.attempts).toBe(2); expect(spent.result).toBe(spent.sessionFailure); expect(spent.closeSession).toHaveBeenCalledOnce(); } const eligible = await runScenario(true, 500); expect(eligible.attempts).toBe(3); expect(eligible.result).toBe(eligible.terminal); expect(eligible.closeSession).toHaveBeenCalledOnce(); }); it("does not reconnect before the session's authoritative close result", async () => { const clock = new ManualClock(); const closure = deferred>(); const terminal = realtimeFailure( "FORBIDDEN", "RECEIVE", false, ); const closeSession = vi.fn(); const attempt = vi.fn(() => opened>( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: closeSession, waitClosed: () => closure.promise, }), ), ); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, confirmTransportBarrier: confirmationSucceeded, attempt, }); let settled = false; const running = coordinator.run().then((result) => { settled = true; return result; }); await flush(); expect(attempt).toHaveBeenCalledOnce(); expect(clock.sleepCalls).toEqual([]); expect(settled).toBe(false); expect(coordinator.getState()).toBe("RUNNING"); closure.resolve(failed(terminal)); const result = await running; expect(result).toBe(terminal); expect(closeSession).toHaveBeenCalledOnce(); }); it("carries an exact recovery token into the next attempt and opens only after confirmation", async () => { const clock = new ManualClock(); const recovery = recoveryToken(); const terminal = realtimeFailure( "CURSOR_EXPIRED", "RECOVER", false, ); const directive = recoveryDirective(recovery, terminal); const completed = closedSuccessfully(); const confirmation = realtimeSuccess(undefined); const confirmTransportBarrier = vi.fn( (received: RealtimeCommittedRecovery) => { expect(received).toBe(recovery); return confirmation; }, ); let attempts = 0; let gateSettled = false; let gateResult: RealtimeResult | undefined; type RecoveryCloseReceipt = | RealtimeRecoveryReconnectDirective | RealtimeReconnectOutcome; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier, attempt: (context) => { attempts += 1; if (attempts === 1) { expect(context.pendingRecovery).toBeNull(); return opened( Object.freeze({ classifyClosed: ( receipt: RecoveryCloseReceipt, ) => receipt, close: vi.fn(), waitClosed: () => directive, }), ); } expect(context.pendingRecovery).toBe(recovery); const gate = context.waitForRecoveryBarrierConfirmation(); void gate.then((result) => { gateSettled = true; gateResult = result; }); expect(gateSettled).toBe(false); return opened( Object.freeze({ classifyClosed: ( receipt: RecoveryCloseReceipt, ) => receipt, close: vi.fn(), waitClosed: () => completed, }), recovery, ); }, }); const running = coordinator.run(); await flushUntil(() => clock.sleepCalls.length === 1); expect(clock.sleepCalls).toEqual([0]); clock.advance(0); const result = await running; await flush(); expect(attempts).toBe(2); expect(confirmTransportBarrier).toHaveBeenCalledOnce(); expect(gateSettled).toBe(true); expect(gateResult).toBe(confirmation); expect(result).toBe(completed.result); }); it.each([ ["missing", undefined], ["cloned", recoveryToken()], ] as const)( "rejects a %s recovery proof and closes the new session", async (_case, proof) => { const clock = new ManualClock(); const recovery = recoveryToken(); const terminal = realtimeFailure( "CURSOR_EXPIRED", "RECOVER", false, ); const closeSession = vi.fn(); const waitClosed = vi.fn(closedSuccessfully); const confirmTransportBarrier = vi.fn( confirmationSucceeded, ); const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier, attempt: (context) => { expect(context.pendingRecovery).toBe(recovery); const session = Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: closeSession, waitClosed, }); return proof === undefined ? opened(session) : opened(session, proof); }, }); const running = coordinator.run({ initialRecovery: recoveryDirective( recovery, terminal, ), }); await flushUntil(() => clock.sleepCalls.length === 1); clock.advance(0); const result = await running; expect(result).toEqual( realtimeFailure( "PROTOCOL_MISMATCH", "RECOVER", false, ), ); expect(closeSession).toHaveBeenCalledOnce(); expect(waitClosed).not.toHaveBeenCalled(); expect(confirmTransportBarrier).not.toHaveBeenCalled(); }, ); it("settles the readiness gate and run with the exact confirmation failure", async () => { const clock = new ManualClock(); const recovery = recoveryToken(); const terminal = realtimeFailure( "CURSOR_EXPIRED", "RECOVER", false, ); const confirmationFailure = realtimeFailure( "SCOPE_PROTOCOL_VIOLATION", "RECOVER", false, ); const closeSession = vi.fn(); const waitClosed = vi.fn(closedSuccessfully); let gateResult: RealtimeResult | undefined; const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier: () => confirmationFailure, attempt: (context) => { void context .waitForRecoveryBarrierConfirmation() .then((result) => { gateResult = result; }); return opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: closeSession, waitClosed, }), recovery, ); }, }); const running = coordinator.run({ initialRecovery: recoveryDirective( recovery, terminal, ), }); await flushUntil(() => clock.sleepCalls.length === 1); clock.advance(0); const result = await running; await flush(); expect(result).toBe(confirmationFailure); expect(gateResult).toBe(confirmationFailure); expect(closeSession).toHaveBeenCalledOnce(); expect(waitClosed).not.toHaveBeenCalled(); }); it("retains the exact recovery token across retryable attempt failures and returns its terminal result on exhaustion", async () => { const clock = new ManualClock(); const recovery = recoveryToken(); const terminal = realtimeFailure( "CURSOR_EXPIRED", "RECOVER", false, ); const attemptFailure = realtimeFailure( "CONNECT_TIMEOUT", "CONNECT", true, ); const gateResults: RealtimeResult[] = []; const seenRecoveries: Array< RealtimeCommittedRecovery | null > = []; const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 2 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier: confirmationSucceeded, attempt: (context) => { seenRecoveries.push(context.pendingRecovery); void context .waitForRecoveryBarrierConfirmation() .then((result) => gateResults.push(result)); return failed(attemptFailure); }, }); const running = coordinator.run({ initialRecovery: recoveryDirective( recovery, terminal, ), }); await flushUntil(() => clock.sleepCalls.length === 1); clock.advance(0); await flushUntil(() => clock.sleepCalls.length === 2); clock.advance(0); const result = await running; await flush(); expect(seenRecoveries).toEqual([recovery, recovery]); expect(gateResults).toEqual([ attemptFailure, attemptFailure, ]); expect(gateResults[0]).toBe(attemptFailure); expect(gateResults[1]).toBe(attemptFailure); expect(result).toBe(terminal); }); it("fails a no-pending gate immediately and rejects any unsolicited proof", async () => { const proof = recoveryToken(); const closeSession = vi.fn(); const confirmTransportBarrier = vi.fn( confirmationSucceeded, ); let gateResult: RealtimeResult | undefined; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), confirmTransportBarrier, attempt: async (context) => { gateResult = await context.waitForRecoveryBarrierConfirmation(); return opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: closeSession, waitClosed: closedSuccessfully, }), proof, ); }, }); const result = await coordinator.run(); expect(gateResult).toEqual( realtimeFailure( "PROTOCOL_MISMATCH", "RECOVER", false, ), ); expect(result).toBe(gateResult); expect(closeSession).toHaveBeenCalledOnce(); expect(confirmTransportBarrier).not.toHaveBeenCalled(); }); it("applies retry ceilings to an initial recovery and preserves its exact terminal result", async () => { const recovery = recoveryToken(); const terminal = realtimeFailure( "CURSOR_EXPIRED", "RECOVER", false, ); const attempt = vi.fn(); const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1, maxElapsedMs: 500, }), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt, }); const result = await coordinator.run({ initialRecovery: recoveryDirective( recovery, terminal, 500, ), }); expect(result).toBe(terminal); expect(attempt).not.toHaveBeenCalled(); }); it("settles a pending readiness gate when the run is aborted", async () => { const clock = new ManualClock(); const recovery = recoveryToken(); const terminal = realtimeFailure( "CURSOR_EXPIRED", "RECOVER", false, ); const attemptResult = deferred< RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess< RealtimeReconnectOutcome > > >(); let gateResult: RealtimeResult | undefined; const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxAttempts: 1 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0, confirmTransportBarrier: confirmationSucceeded, attempt: (context) => { void context .waitForRecoveryBarrierConfirmation() .then((result) => { gateResult = result; }); return attemptResult.promise; }, }); const controller = new AbortController(); const running = coordinator.run({ signal: controller.signal, initialRecovery: recoveryDirective( recovery, terminal, ), }); await flushUntil(() => clock.sleepCalls.length === 1); clock.advance(0); await flush(); controller.abort(); const result = await running; await flush(); expect(result).toEqual( realtimeFailure("ABORTED", "CONNECT", false), ); expect(gateResult).toEqual( realtimeFailure("ABORTED", "RECOVER", false), ); attemptResult.resolve( opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: vi.fn(), waitClosed: closedSuccessfully, }), recovery, ), ); await flush(); }); it("rejects accessors without invoking them and consumes proxy values only from one descriptor snapshot", async () => { let getterReads = 0; const accessorOutcome = Object.defineProperty({}, "result", { enumerable: true, get() { getterReads += 1; return realtimeSuccess(undefined); }, }); const accessorCoordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt: () => accessorOutcome as unknown as RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess >, }); await expect(accessorCoordinator.run()).resolves.toEqual( realtimeFailure( "PROTOCOL_MISMATCH", "CONNECT", false, ), ); expect(getterReads).toBe(0); const completed = closedSuccessfully(); const session = Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: vi.fn(), waitClosed: () => completed, }); const rawOutcome = opened(session); const proxyReads: PropertyKey[] = []; const proxyOutcome = new Proxy(rawOutcome, { get(_target, key) { proxyReads.push(key); if (key === "then") return undefined; throw new Error("raw property reread"); }, }); const proxyCoordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt: () => proxyOutcome, }); await expect(proxyCoordinator.run()).resolves.toBe( completed.result, ); expect(proxyReads).toEqual(["then"]); }); it("rechecks scope after attempt and waitClosed boundaries before accepting callbacks", async () => { let attemptScopeCurrent = true; const lateAttemptClose = vi.fn(); const lateAttemptWaitClosed = vi.fn(closedSuccessfully); const lateAttemptCoordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), isCurrent: () => attemptScopeCurrent, confirmTransportBarrier: confirmationSucceeded, attempt: () => { attemptScopeCurrent = false; return opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: lateAttemptClose, waitClosed: lateAttemptWaitClosed, }), ); }, }); await expect(lateAttemptCoordinator.run()).resolves.toEqual( realtimeFailure("SCOPE_FENCED", "CONNECT", false), ); expect(lateAttemptClose).toHaveBeenCalledOnce(); expect(lateAttemptWaitClosed).not.toHaveBeenCalled(); let closeScopeCurrent = true; const classifyClosed = vi.fn( (receipt: RealtimeReconnectOutcome) => receipt, ); const closeSession = vi.fn(); const closeCoordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), isCurrent: () => closeScopeCurrent, confirmTransportBarrier: confirmationSucceeded, attempt: () => opened( Object.freeze({ classifyClosed, close: closeSession, waitClosed: () => { closeScopeCurrent = false; return closedSuccessfully(); }, }), ), }); await expect(closeCoordinator.run()).resolves.toEqual( realtimeFailure("SCOPE_FENCED", "CONNECT", false), ); expect(classifyClosed).not.toHaveBeenCalled(); expect(closeSession).toHaveBeenCalledOnce(); }); it("rejects extra and missing outcome keys through the common result codec", async () => { const validSession = Object.freeze({ classifyClosed: closedSuccessfully, close: vi.fn(), waitClosed: closedSuccessfully, }); const invalidOutcomes = [ { extra: true, result: realtimeSuccess(validSession), }, { result: { ok: true }, }, { result: { ok: false, error: { extra: true, kind: "FORBIDDEN", operation: "CONNECT", retryable: false, }, }, }, ]; for (const invalid of invalidOutcomes) { const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt: () => invalid as unknown as RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess >, }); await expect(coordinator.run()).resolves.toEqual( realtimeFailure( "PROTOCOL_MISMATCH", "CONNECT", false, ), ); } }); it("rechecks abort immediately after listener registration", async () => { let abortReads = 0; let listener: (() => void) | undefined; const signal = { get aborted() { abortReads += 1; return abortReads >= 2; }, addEventListener( type: string, nextListener: () => void, ) { if (type === "abort") listener = nextListener; }, removeEventListener() { listener = undefined; }, } as unknown as AbortSignal; const attempt = vi.fn(); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt, }); const result = await coordinator.run({ signal }); expect(listener).toBeUndefined(); expect(attempt).not.toHaveBeenCalled(); expect(result).toEqual( realtimeFailure("ABORTED", "CONNECT", false), ); }); it("bounds a non-cooperative retry sleep after its phase is aborted", async () => { vi.useFakeTimers(); const environment = new MutableOnlineEnvironment(true); const sleep = vi.fn( () => new Promise(() => undefined), ); const clock: ClockPort = Object.freeze({ now: () => 0, sleep, }); const retryable = realtimeFailure( "CONNECT_TIMEOUT", "CONNECT", true, ); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock, random: () => 0, confirmTransportBarrier: confirmationSucceeded, attempt: () => failed(retryable), }); try { const running = coordinator.run(); await flushUntil(() => sleep.mock.calls.length === 1); environment.emit(false); await flush(); await vi.advanceTimersByTimeAsync( REALTIME_RECONNECT_CEILINGS.drainTimeoutMs, ); const result = await running; expect(result).toEqual( realtimeFailure( "PROVIDER_UNAVAILABLE", "CONNECT", false, ), ); expect(coordinator.getState()).toBe("DRAINING"); await expect(coordinator.run()).resolves.toEqual( realtimeFailure( "PROTOCOL_MISMATCH", "CONNECT", false, ), ); expect(environment.listeners.size).toBe(0); expect(vi.getTimerCount()).toBe(0); } finally { coordinator.close(); vi.useRealTimers(); } }); it("bounds a non-cooperative attempt and closes its late successful session before leaving DRAINING", async () => { vi.useFakeTimers(); const environment = new MutableOnlineEnvironment(true); const attemptResult = deferred< RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess< RealtimeReconnectOutcome > > >(); const lateClose = vi.fn(); const lateWaitClosed = vi.fn(closedSuccessfully); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt: () => attemptResult.promise, }); try { const running = coordinator.run(); await flush(); environment.emit(false); await flush(); await vi.advanceTimersByTimeAsync( REALTIME_RECONNECT_CEILINGS.drainTimeoutMs, ); const result = await running; expect(result).toEqual( realtimeFailure( "PROVIDER_UNAVAILABLE", "CONNECT", false, ), ); expect(coordinator.getState()).toBe("DRAINING"); await expect(coordinator.run()).resolves.toEqual( realtimeFailure( "PROTOCOL_MISMATCH", "CONNECT", false, ), ); expect(environment.listeners.size).toBe(0); expect(vi.getTimerCount()).toBe(0); attemptResult.resolve( opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: lateClose, waitClosed: lateWaitClosed, }), ), ); await flush(); expect(lateClose).toHaveBeenCalledOnce(); expect(lateWaitClosed).not.toHaveBeenCalled(); expect(coordinator.getState()).toBe("IDLE"); } finally { coordinator.close(); vi.useRealTimers(); } }); it("cancels the drain timer on abort and keeps the late generation fenced", async () => { vi.useFakeTimers(); const environment = new MutableOnlineEnvironment(true); const attemptResult = deferred< RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess< RealtimeReconnectOutcome > > >(); const lateClose = vi.fn(); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt: () => attemptResult.promise, }); const controller = new AbortController(); try { const running = coordinator.run({ signal: controller.signal, }); await flush(); environment.emit(false); await flush(); expect(vi.getTimerCount()).toBe(1); controller.abort(); const result = await running; expect(result).toEqual( realtimeFailure("ABORTED", "CONNECT", false), ); expect(coordinator.getState()).toBe("DRAINING"); expect(environment.listeners.size).toBe(0); expect(vi.getTimerCount()).toBe(0); attemptResult.resolve( opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: lateClose, waitClosed: closedSuccessfully, }), ), ); await flush(); expect(lateClose).toHaveBeenCalledOnce(); expect(coordinator.getState()).toBe("IDLE"); } finally { coordinator.close(); vi.useRealTimers(); } }); it("keeps a healthy waitClosed unbounded but bounds its post-abort drain", async () => { vi.useFakeTimers(); const environment = new MutableOnlineEnvironment(true); const closure = deferred>(); const classifyClosed = vi.fn( (receipt: RealtimeReconnectOutcome) => receipt, ); const closeSession = vi.fn(); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment, clock: new ManualClock(), confirmTransportBarrier: confirmationSucceeded, attempt: () => opened( Object.freeze({ classifyClosed, close: closeSession, waitClosed: () => closure.promise, }), ), }); try { let settled = false; const running = coordinator.run().then((result) => { settled = true; return result; }); await flush(); await vi.advanceTimersByTimeAsync( REALTIME_RECONNECT_CEILINGS.drainTimeoutMs * 2, ); expect(settled).toBe(false); expect(coordinator.getState()).toBe("RUNNING"); expect(vi.getTimerCount()).toBe(0); environment.emit(false); await flush(); await vi.advanceTimersByTimeAsync( REALTIME_RECONNECT_CEILINGS.drainTimeoutMs, ); const result = await running; expect(result).toEqual( realtimeFailure( "PROVIDER_UNAVAILABLE", "CONNECT", false, ), ); expect(closeSession).toHaveBeenCalled(); expect(classifyClosed).not.toHaveBeenCalled(); expect(coordinator.getState()).toBe("DRAINING"); expect(environment.listeners.size).toBe(0); expect(vi.getTimerCount()).toBe(0); closure.resolve(closedSuccessfully()); await flush(); expect(classifyClosed).not.toHaveBeenCalled(); expect(coordinator.getState()).toBe("IDLE"); } finally { coordinator.close(); vi.useRealTimers(); } }); it("uses waitClosed as authority and fences an aborted late generation", async () => { const clock = new ManualClock(); const attemptResult = deferred< RealtimeReconnectOutcome< RealtimeReconnectAttemptSuccess< RealtimeReconnectOutcome > > >(); let captured: | RealtimeReconnectAttemptContext | undefined; const lateClose = vi.fn(); const lateWaitClosed = vi.fn(closedSuccessfully); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, confirmTransportBarrier: confirmationSucceeded, attempt: (context) => { captured = context; return attemptResult.promise; }, }); const controller = new AbortController(); const running = coordinator.run({ signal: controller.signal }); await flush(); expect(captured?.isCurrent()).toBe(true); controller.abort(); const result = await running; expect(result).toEqual( realtimeFailure("ABORTED", "CONNECT", false), ); expect(coordinator.getState()).toBe("DRAINING"); expect(captured?.isCurrent()).toBe(false); attemptResult.resolve( opened( Object.freeze({ classifyClosed: ( receipt: RealtimeReconnectOutcome, ) => receipt, close: lateClose, waitClosed: lateWaitClosed, }), ), ); await flush(); expect(lateClose).toHaveBeenCalledOnce(); expect(lateWaitClosed).not.toHaveBeenCalled(); expect(coordinator.getState()).toBe("IDLE"); }); }); class FakeWebSocket implements WebSocketFacade { protocol: string = REALTIME_WEBSOCKET_PROTOCOL; readyState = 0; bufferedAmount = 0; readonly listeners = new Map< string, Set<(event: unknown) => void> >(); readonly sent: string[] = []; send(data: string): void { if (this.readyState !== 1) throw new Error("not open"); this.sent.push(data); } close(): void { this.readyState = 3; } addEventListener( type: string, listener: (event: unknown) => void, ): void { const listeners = this.listeners.get(type) ?? new Set(); listeners.add(listener); this.listeners.set(type, listeners); } removeEventListener( type: string, listener: (event: unknown) => void, ): void { this.listeners.get(type)?.delete(listener); } accept(): void { this.readyState = 1; this.emit("open", {}); this.emit("message", { data: JSON.stringify({ type: "WELCOME", protocol: REALTIME_WEBSOCKET_PROTOCOL, connectionId: "connection.0001", heartbeatMs: 5_000, heartbeatAckTimeoutMs: 1_000, limits: { maxFrameBytes: 64 * 1_024, maxSubscriptions: 32, maxInboundQueueCount: 256, maxInboundQueueBytes: 4 * 1_024 * 1_024, maxOutboundQueueCount: 128, maxOutboundQueueBytes: 256 * 1_024, maxBufferedAmountBytes: 256 * 1_024, maxEventsPerSecond: 256, }, }), }); } nativeClose(code: number): void { this.readyState = 3; this.emit("close", { code, reason: "" }); } ackHeartbeats(): void { for (const raw of this.sent.splice(0)) { const frame = JSON.parse(raw) as { type: string; nonce: string }; if (frame.type !== "HEARTBEAT") continue; this.emit("message", { data: JSON.stringify({ type: "HEARTBEAT_ACK", protocol: REALTIME_WEBSOCKET_PROTOCOL, nonce: frame.nonce, }), }); } } private emit(type: string, event: unknown): void { for (const listener of [...(this.listeners.get(type) ?? [])]) { listener(event); } } } describe("transport failures through the reconnect coordinator", () => { it.each([ ["NETWORK_LOST", 1_006], ["RESTART", 1_012], ] as const)( "reconnects a WebSocket %s close that carries no server hint", async (_category, code) => { const clock = new ManualClock(); const sockets: FakeWebSocket[] = []; const coordinator = createRealtimeReconnectCoordinator< WebSocketClosedReceipt >({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: async (context) => { const connection = createWebSocketConnection({ origin: "https://app.example.test", endpointPath: "/_realtime/v1", clock, createNonce: () => "nonce.0001", createSocket: () => { const socket = new FakeWebSocket(); sockets.push(socket); return socket; }, onEvent: async () => realtimeSuccess(REALTIME_TRANSPORT_CONTINUE), onRecoveryRequired: () => undefined, }); const connected = await connection.connect(context.signal); if (!connected.ok) return failed(connected); return opened( Object.freeze({ waitClosed: () => connection.waitClosed(), classifyClosed: (receipt: WebSocketClosedReceipt) => Object.freeze({ result: Object.freeze({ ok: false as const, error: receipt.error, }), }), close: () => connection.close(), }), ); }, }); const running = coordinator.run(); await flushUntil(() => sockets.length === 1); sockets[0]!.accept(); await flush(); sockets[0]!.nativeClose(code); await flushUntil(() => clock.sleepCalls.includes(50)); clock.advance(50); await flushUntil(() => sockets.length === 2); expect(sockets).toHaveLength(2); // OVERLOADED carries no hint either and stays terminal. sockets[1]!.accept(); await flush(); sockets[1]!.nativeClose(1_013); await expect(running).resolves.toEqual({ ok: false, error: { kind: "PROVIDER_UNAVAILABLE", operation: "RECEIVE", retryable: false, }, }); expect(sockets).toHaveLength(2); }, ); it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => { const clock = new ManualClock(); const sockets: FakeWebSocket[] = []; let nonce = 0; const coordinator = createRealtimeReconnectCoordinator< WebSocketClosedReceipt >({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: async (context) => { const connection = createWebSocketConnection({ origin: "https://app.example.test", endpointPath: "/_realtime/v1", clock, createNonce: () => `nonce.${++nonce}`, createSocket: () => { const socket = new FakeWebSocket(); sockets.push(socket); return socket; }, onEvent: async () => realtimeSuccess(REALTIME_TRANSPORT_CONTINUE), onRecoveryRequired: () => undefined, observe: (observation) => { if (observation.kind === "HEARTBEAT_ACK") { context.markValidHeartbeatOrEvent(); } }, }); const connected = await connection.connect(context.signal); if (!connected.ok) return failed(connected); return opened( Object.freeze({ waitClosed: () => connection.waitClosed(), classifyClosed: (receipt: WebSocketClosedReceipt) => Object.freeze({ result: Object.freeze({ ok: false as const, error: receipt.error, }), }), close: () => connection.close(), }), ); }, }); const running = coordinator.run(); await flushUntil(() => sockets.length === 1); sockets[0]!.accept(); await flush(); // No events, only acknowledged heartbeats, for longer than maxElapsedMs. for (let beat = 0; beat < 2; beat += 1) { clock.advance(5_000); await flush(); sockets[0]!.ackHeartbeats(); clock.advance(1_000); await flush(); } sockets[0]!.nativeClose(1_012); await flushUntil(() => clock.sleepCalls.includes(50)); clock.advance(50); await flushUntil(() => sockets.length === 2); expect(sockets).toHaveLength(2); coordinator.close(); await expect(running).resolves.toMatchObject({ ok: false, error: { kind: "CLOSED" }, }); }); it.each([502, 504])( "reconnects SSE HTTP %i that carries no server hint", async (status) => { const clock = new ManualClock(); const fetcher = vi .fn() .mockResolvedValueOnce(new Response(null, { status })) .mockResolvedValueOnce(new Response(null, { status: 403 })); const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: async (context) => { const read = await createFetchSseConnection({ endpoint: "https://app.example.test/events", applicationOrigin: "https://app.example.test", recoveryMode: "CURSOR", fetcher, clock, }).read({ resumeCursor: null, signal: context.signal, onEvent: () => realtimeSuccess(SSE_CONTINUE), }); // Only connect failures are exercised here. return failed( read.ok ? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false) : read, ); }, }); const running = coordinator.run(); await flushUntil(() => clock.sleepCalls.includes(50)); clock.advance(50); await expect(running).resolves.toEqual({ ok: false, error: { kind: "FORBIDDEN", operation: "CONNECT", retryable: false, }, }); expect(fetcher).toHaveBeenCalledTimes(2); }, ); it.each([429, 503])( "waits at least the SSE %i Retry-After through the reconnect bridge", async (status) => { const clock = new ManualClock(); const fetcher = vi .fn() .mockResolvedValueOnce( new Response(null, { status, headers: { "retry-after": "3" }, }), ) .mockResolvedValueOnce( new Response(null, { status, headers: { "retry-after": "10" }, }), ); const coordinator = createRealtimeReconnectCoordinator({ policy: policy({ maxDelayMs: 5_000, maxElapsedMs: 60_000 }), environment: new MutableOnlineEnvironment(true), clock, random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, attempt: async (context) => { const read = await readSseForReconnect( createFetchSseConnection({ endpoint: "https://app.example.test/events", applicationOrigin: "https://app.example.test", recoveryMode: "CURSOR", fetcher, clock, }), { resumeCursor: null, signal: context.signal, onEvent: () => realtimeSuccess(SSE_CONTINUE), }, ); // Only connect failures are exercised here. if (read.result.ok) { return failed( realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false), ); } return read as RealtimeReconnectOutcome; }, }); // 10_000 is each read's connect timeout; 3_000 is the reconnect wait. const running = coordinator.run(); await flushUntil(() => clock.sleepCalls.length === 2); expect(clock.sleepCalls).toEqual([10_000, 3_000]); clock.advance(3_000); // A hint above the policy's maxDelayMs still ends the run. await expect(running).resolves.toMatchObject({ ok: false, error: { operation: "CONNECT", retryable: true }, }); expect(fetcher).toHaveBeenCalledTimes(2); expect(clock.sleepCalls).toEqual([10_000, 3_000, 10_000]); }, ); });