From bcad7b0d12dbf04bce3d5aaaea9549161e16709b Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Sat, 26 Sep 2026 00:01:10 +0900 Subject: [PATCH] fix(realtime): reset the retry budget on every session end and pause it offline (04-01) A long healthy connection that ended offline or with a throwing waitClosed kept a spent elapsed budget and stopped with zero retries. Offline waits no longer spend the budget, and WebSocket HEARTBEAT_ACK is now observable so a quiet stream can earn a fresh budget. Co-Authored-By: Claude Opus 5.5 --- .../realtime/reconnect-coordinator.ts | 51 +++-- .../websocket/websocket-connection.ts | 4 + .../realtime-reconnect-coordinator.test.ts | 192 +++++++++++++++++- 3 files changed, 218 insertions(+), 29 deletions(-) diff --git a/src/adapters/realtime/reconnect-coordinator.ts b/src/adapters/realtime/reconnect-coordinator.ts index 708014dc..0c529a58 100644 --- a/src/adapters/realtime/reconnect-coordinator.ts +++ b/src/adapters/realtime/reconnect-coordinator.ts @@ -55,7 +55,9 @@ export type RealtimeReconnectAttemptContext = Readonly<{ pendingRecovery: RealtimeCommittedRecovery | null; isCurrent(): boolean; /** - * Call only after a protocol-valid heartbeat or event. + * Call only after a protocol-valid heartbeat or event: an applied event, + * an SSE `onComment`, or a WebSocket `HEARTBEAT_ACK` observation. A quiet + * stream that never calls it cannot earn a fresh retry budget. */ markValidHeartbeatOrEvent(): void; /** @@ -493,15 +495,16 @@ export function createRealtimeReconnectCoordinator( ) { return retry.terminalResult; } - if ( - (!online || requiredOnlineVersion !== null) && - !(await waitOnline()) - ) { + const offlineWait = !online || requiredOnlineVersion !== null; + if (offlineWait && !(await waitOnline())) { return failure(currentFailure() ?? "ABORTED"); } now = safeNow(clock); if (now === null) return retry.terminalResult; + // Offline time is not retry time. The elapsed budget restarts when + // the network returns; maxAttempts still bounds the loop. + if (offlineWait) budgetStartedAt = now; const delay = nextDelay( retry, retriesUsed, @@ -815,6 +818,21 @@ export function createRealtimeReconnectCoordinator( phase.controller.signal, ); const closeOffline = wentOffline(phase); + // Every way an opened session ends, including offline and a throwing + // waitClosed, can earn a fresh budget, not only a classified close. + const endedAt = safeNow(clock); + if ( + endedAt !== null && + isReconnectAttemptResetEligible({ + policy, + openedAtMs: openedAt, + nowMs: endedAt, + observedValidHeartbeatOrEvent: validSignalObserved, + }) + ) { + retriesUsed = 0; + budgetStartedAt = endedAt; + } const afterWaitClosed = currentFailure(); if (afterWaitClosed) { @@ -902,18 +920,6 @@ export function createRealtimeReconnectCoordinator( if (closedAt === null) { return classified.outcome.failure.result; } - if ( - isReconnectAttemptResetEligible({ - policy, - openedAtMs: openedAt, - nowMs: closedAt, - observedValidHeartbeatOrEvent: - validSignalObserved, - }) - ) { - retriesUsed = 0; - budgetStartedAt = closedAt; - } retry = captureRetry( classified.outcome.failure.result, classified.outcome.failure.result, @@ -932,17 +938,6 @@ export function createRealtimeReconnectCoordinator( if (closedAt === null) { return classified.directive.terminalFailure.result; } - if ( - isReconnectAttemptResetEligible({ - policy, - openedAtMs: openedAt, - nowMs: closedAt, - observedValidHeartbeatOrEvent: validSignalObserved, - }) - ) { - retriesUsed = 0; - budgetStartedAt = closedAt; - } pendingRecovery = classified.directive; retry = captureRetry( classified.directive.terminalFailure.result, diff --git a/src/adapters/realtime/websocket/websocket-connection.ts b/src/adapters/realtime/websocket/websocket-connection.ts index 6b270ac0..592b328e 100644 --- a/src/adapters/realtime/websocket/websocket-connection.ts +++ b/src/adapters/realtime/websocket/websocket-connection.ts @@ -71,6 +71,7 @@ export type WebSocketConnectionObservation = Readonly<{ kind: | "CONNECTED" | "SUBSCRIBED" + | "HEARTBEAT_ACK" | "FRAME_REJECTED" | "RECOVERY_REQUIRED" | "CLOSED"; @@ -372,6 +373,9 @@ export function createWebSocketConnection( if (decoded.value.nonce === retiredHeartbeat) { retiredHeartbeat = null; } + // A quiet stream's only liveness proof; composition can forward it to + // the reconnect coordinator's markValidHeartbeatOrEvent. + observe({ kind: "HEARTBEAT_ACK" }); return; } const queueLimits = effectiveLimits ?? ceilings; diff --git a/tests/unit/realtime/realtime-reconnect-coordinator.test.ts b/tests/unit/realtime/realtime-reconnect-coordinator.test.ts index 8e3fd832..268304c8 100644 --- a/tests/unit/realtime/realtime-reconnect-coordinator.test.ts +++ b/tests/unit/realtime/realtime-reconnect-coordinator.test.ts @@ -434,6 +434,105 @@ describe("realtime reconnect coordinator", () => { expect(result).toBe(terminal); }); + it("does not spend the elapsed budget while waiting offline", async () => { + const clock = new ManualClock(); + const environment = new MutableOnlineEnvironment(false); + const terminal = realtimeFailure( + "FORBIDDEN", + "CONNECT", + false, + ); + const attempt = vi.fn(() => failed(terminal)); + const coordinator = createRealtimeReconnectCoordinator({ + policy: policy(), + environment, + clock, + random: () => 0.5, + confirmTransportBarrier: confirmationSucceeded, + attempt, + }); + + const running = coordinator.run(); + await flush(); + // Offline for longer than maxElapsedMs before the first attempt. + clock.advance(6_000); + environment.emit(true); + await flushUntil(() => clock.sleepCalls.length === 1); + expect(clock.sleepCalls).toEqual([50]); + + clock.advance(50); + const result = await running; + expect(attempt).toHaveBeenCalledTimes(1); + expect(result).toBe(terminal); + }); + + it.each(["an offline blip", "a waitClosed throw"] as const)( + "reconnects after %s ends a healthy session older than the elapsed budget", + async (ending) => { + const clock = new ManualClock(); + const environment = new MutableOnlineEnvironment(true); + const closing = deferred>(); + const terminal = realtimeFailure( + "FORBIDDEN", + "CONNECT", + false, + ); + let attempts = 0; + const coordinator = createRealtimeReconnectCoordinator({ + policy: policy(), + environment, + clock, + random: () => 0.5, + confirmTransportBarrier: confirmationSucceeded, + attempt: (context: RealtimeReconnectAttemptContext) => { + attempts += 1; + if (attempts > 1) return failed(terminal); + context.markValidHeartbeatOrEvent(); + return opened>( + Object.freeze({ + classifyClosed: ( + receipt: RealtimeReconnectOutcome, + ) => receipt, + close: () => + closing.resolve( + failed( + realtimeFailure("OFFLINE", "RECEIVE", true), + ), + ), + waitClosed: () => + ending === "an offline blip" + ? closing.promise + : closing.promise.then(() => { + throw new Error("transport failed"); + }), + }), + ); + }, + }); + + const running = coordinator.run(); + await flushUntil(() => attempts === 1); + await flush(); + // Healthy for twice maxElapsedMs, then a one second outage. + clock.advance(10_000); + if (ending === "an offline blip") { + environment.emit(false); + await flush(); + clock.advance(1_000); + environment.emit(true); + } else { + closing.resolve(closedSuccessfully()); + } + await flushUntil(() => clock.sleepCalls.length === 1); + expect(clock.sleepCalls).toEqual([50]); + + clock.advance(50); + const result = await running; + expect(attempts).toBe(2); + expect(result).toBe(terminal); + }, + ); + it("resets a spent retry only after a valid signal and a stable open", async () => { const runScenario = async (markValid: boolean, openMs: number) => { const clock = new ManualClock(); @@ -1502,9 +1601,11 @@ class FakeWebSocket implements WebSocketFacade { string, Set<(event: unknown) => void> >(); + readonly sent: string[] = []; - send(): void { + send(data: string): void { if (this.readyState !== 1) throw new Error("not open"); + this.sent.push(data); } close(): void { @@ -1556,6 +1657,20 @@ class FakeWebSocket implements WebSocketFacade { this.emit("close", { code, reason: "" }); } + ackHeartbeats(): void { + for (const raw of this.sent.splice(0)) { + const frame = JSON.parse(raw) as { type: string; nonce: string }; + if (frame.type !== "HEARTBEAT") continue; + this.emit("message", { + data: JSON.stringify({ + type: "HEARTBEAT_ACK", + protocol: REALTIME_WEBSOCKET_PROTOCOL, + nonce: frame.nonce, + }), + }); + } + } + private emit(type: string, event: unknown): void { for (const listener of [...(this.listeners.get(type) ?? [])]) { listener(event); @@ -1639,6 +1754,81 @@ describe("transport failures through the reconnect coordinator", () => { }, ); + it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => { + const clock = new ManualClock(); + const sockets: FakeWebSocket[] = []; + let nonce = 0; + const coordinator = createRealtimeReconnectCoordinator< + WebSocketClosedReceipt + >({ + policy: policy(), + environment: new MutableOnlineEnvironment(true), + clock, + random: () => 0.5, + confirmTransportBarrier: confirmationSucceeded, + attempt: async (context) => { + const connection = createWebSocketConnection({ + origin: "https://app.example.test", + endpointPath: "/_realtime/v1", + clock, + createNonce: () => `nonce.${++nonce}`, + createSocket: () => { + const socket = new FakeWebSocket(); + sockets.push(socket); + return socket; + }, + onEvent: async () => + realtimeSuccess(REALTIME_TRANSPORT_CONTINUE), + onRecoveryRequired: () => undefined, + observe: (observation) => { + if (observation.kind === "HEARTBEAT_ACK") { + context.markValidHeartbeatOrEvent(); + } + }, + }); + const connected = await connection.connect(context.signal); + if (!connected.ok) return failed(connected); + return opened( + Object.freeze({ + waitClosed: () => connection.waitClosed(), + classifyClosed: (receipt: WebSocketClosedReceipt) => + Object.freeze({ + result: Object.freeze({ + ok: false as const, + error: receipt.error, + }), + }), + close: () => connection.close(), + }), + ); + }, + }); + + const running = coordinator.run(); + await flushUntil(() => sockets.length === 1); + sockets[0]!.accept(); + await flush(); + // No events, only acknowledged heartbeats, for longer than maxElapsedMs. + for (let beat = 0; beat < 2; beat += 1) { + clock.advance(5_000); + await flush(); + sockets[0]!.ackHeartbeats(); + clock.advance(1_000); + await flush(); + } + sockets[0]!.nativeClose(1_012); + await flushUntil(() => clock.sleepCalls.includes(50)); + clock.advance(50); + await flushUntil(() => sockets.length === 2); + expect(sockets).toHaveLength(2); + + coordinator.close(); + await expect(running).resolves.toMatchObject({ + ok: false, + error: { kind: "CLOSED" }, + }); + }); + it.each([502, 504])( "reconnects SSE HTTP %i that carries no server hint", async (status) => {