From 87a45893d64fc30c4f12b59a75cf49f385acccd7 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 24 Sep 2026 12:00:36 +0900 Subject: [PATCH] fix(realtime): reconnect retryable failures without a server hint (R-4.2) reconnectable() in the reconnect coordinator returned false for PROVIDER_UNAVAILABLE/RATE_LIMITED whenever serverNotBeforeMs was null, even with retryable:true. The transports send exactly that for WS NETWORK_LOST / RESTART (1006/1001/1012, error event) and SSE 502/504, and WS can never carry a hint, so those streams ended instead of reconnecting. The bounded poll coordinator had the same rule for retryAfterMs. Both now decide on retryable + kind only; a missing hint falls back to policy backoff. Transports that require a hint and lack one already report retryable:false (SSE 429/503 without Retry-After, WS OVERLOADED), which stays terminal. Tests: the reconnect and poll tests that pinned the old behavior ("does not reconnect/retry ... without a server hint") encoded the bug and are rewritten to expect a policy-backoff retry. New seam tests wire the real createWebSocketConnection / createFetchSseConnection into the reconnect coordinator. RED before the fix: "reconnects a WebSocket NETWORK_LOST|RESTART close that carries no server hint" (1 socket, expected 2), "reconnects SSE HTTP 502|504 that carries no server hint" (got PROVIDER_UNAVAILABLE retryable:true instead of the second attempt's FORBIDDEN), "retries retryable RATE_LIMITED|PROVIDER_UNAVAILABLE without a server hint on policy backoff" (execute called once). The WS test also closes with OVERLOADED to show retryable:false stays terminal. Gates: check:architecture, check:types:app, check:types:test, lint, test:component pass. test:unit passes except removal-fixture.test.ts, which needs generated artifacts/release evidence absent from this fresh worktree (unrelated). Co-Authored-By: Claude Opus 5.5 --- .../polling/bounded-poll-coordinator.ts | 6 +- .../realtime/reconnect-coordinator.ts | 30 +- .../realtime/bounded-poll-coordinator.test.ts | 34 ++- .../realtime-reconnect-coordinator.test.ts | 284 ++++++++++++++++-- 4 files changed, 289 insertions(+), 65 deletions(-) diff --git a/src/adapters/realtime/polling/bounded-poll-coordinator.ts b/src/adapters/realtime/polling/bounded-poll-coordinator.ts index 58b4c192..7eff8333 100644 --- a/src/adapters/realtime/polling/bounded-poll-coordinator.ts +++ b/src/adapters/realtime/polling/bounded-poll-coordinator.ts @@ -341,11 +341,11 @@ export function createBoundedPollCoordinator( } result = parsedResult; if (!result.ok) { + // A missing server hint falls back to policy backoff; an executor + // that requires one reports `retryable: false` instead. if ( !result.error.retryable || - !isPollRetryableFailureKind(result.error.kind) || - (isPollRetryAfterFailureKind(result.error.kind) && - result.error.retryAfterMs === undefined) + !isPollRetryableFailureKind(result.error.kind) ) { return pollFailure(result.error.kind); } diff --git a/src/adapters/realtime/reconnect-coordinator.ts b/src/adapters/realtime/reconnect-coordinator.ts index fd68af21..1a2a269d 100644 --- a/src/adapters/realtime/reconnect-coordinator.ts +++ b/src/adapters/realtime/reconnect-coordinator.ts @@ -717,12 +717,7 @@ export function createRealtimeReconnectCoordinator( settleAttemptGate(opened.failure.result); phase.controller.abort(); phase.dispose(); - if ( - !reconnectable( - opened.failure, - opened.serverNotBeforeMs, - ) - ) { + if (!reconnectable(opened.failure)) { return opened.failure.result; } const failedAt = safeNow(clock); @@ -934,12 +929,7 @@ export function createRealtimeReconnectCoordinator( if (classified.outcome.ok) { return classified.outcome.result as RealtimeResult; } - if ( - !reconnectable( - classified.outcome.failure, - classified.outcome.serverNotBeforeMs, - ) - ) { + if (!reconnectable(classified.outcome.failure)) { return classified.outcome.failure.result; } @@ -1658,17 +1648,11 @@ function providerRetry( ); } -function reconnectable( - failureSnapshot: FailureSnapshot, - serverNotBeforeMs: number | null, -): boolean { - if ( - serverNotBeforeMs === null && - (failureSnapshot.kind === "RATE_LIMITED" || - failureSnapshot.kind === "PROVIDER_UNAVAILABLE") - ) { - return false; - } +/** + * A transport that needs a server hint and did not get one already reports + * `retryable: false`, so a missing hint alone never ends the loop. + */ +function reconnectable(failureSnapshot: FailureSnapshot): boolean { return ( failureSnapshot.retryable && RETRYABLE_KINDS.includes( diff --git a/tests/unit/realtime/bounded-poll-coordinator.test.ts b/tests/unit/realtime/bounded-poll-coordinator.test.ts index eacee74b..06cfd849 100644 --- a/tests/unit/realtime/bounded-poll-coordinator.test.ts +++ b/tests/unit/realtime/bounded-poll-coordinator.test.ts @@ -562,17 +562,27 @@ describe("bounded poll coordinator", () => { }); it.each(["RATE_LIMITED", "PROVIDER_UNAVAILABLE"] as const)( - "does not retry %s without a bounded server hint", + "retries retryable %s without a server hint on policy backoff", async (kind) => { const clock = new ManualClock(); - const execute = vi.fn().mockResolvedValue({ - ok: false, - error: { - kind, - operation: "POLL", - retryable: true, - }, - }); + const execute = vi + .fn() + .mockResolvedValueOnce({ + ok: false, + error: { + kind, + operation: "POLL", + retryable: true, + }, + }) + .mockResolvedValue({ + ok: false, + error: { + kind, + operation: "POLL", + retryable: false, + }, + }); const coordinator = createBoundedPollCoordinator({ policy, operation, @@ -585,6 +595,10 @@ describe("bounded poll coordinator", () => { const pending = coordinator.run(); clock.advance(5_000); await flush(); + expect(execute).toHaveBeenCalledTimes(1); + clock.advance(5_000); + await flush(); + await expect(pending).resolves.toEqual({ ok: false, error: { @@ -593,7 +607,7 @@ describe("bounded poll coordinator", () => { retryable: false, }, }); - expect(execute).toHaveBeenCalledTimes(1); + expect(execute).toHaveBeenCalledTimes(2); }, ); diff --git a/tests/unit/realtime/realtime-reconnect-coordinator.test.ts b/tests/unit/realtime/realtime-reconnect-coordinator.test.ts index c84fa3e3..6d337439 100644 --- a/tests/unit/realtime/realtime-reconnect-coordinator.test.ts +++ b/tests/unit/realtime/realtime-reconnect-coordinator.test.ts @@ -5,8 +5,9 @@ import type { RealtimeFailureKind, RealtimeResult, } from "../../../src/application/ports/realtime/shared.ts"; -import type { - RealtimeRecoveryCheckpoint, +import { + REALTIME_TRANSPORT_CONTINUE, + type RealtimeRecoveryCheckpoint, } from "../../../src/application/ports/realtime/event-authority.ts"; import { createRealtimeReconnectCoordinator, @@ -27,6 +28,18 @@ import { realtimeFailure, realtimeSuccess, } from "../../../src/adapters/realtime/result.ts"; +import { + createFetchSseConnection, + SSE_CONTINUE, +} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts"; +import { + createWebSocketConnection, + type WebSocketClosedReceipt, + type WebSocketFacade, +} from "../../../src/adapters/realtime/websocket/websocket-connection.ts"; +import { + REALTIME_WEBSOCKET_PROTOCOL, +} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts"; type Sleeper = Readonly<{ dueAt: number; @@ -309,62 +322,78 @@ describe("realtime reconnect coordinator", () => { "RATE_LIMITED", "PROVIDER_UNAVAILABLE", ] as const)( - "does not reconnect an external %s attempt failure without a server hint", + "reconnects a retryable external %s attempt failure without a server hint", async (kind) => { const clock = new ManualClock(); - const terminal = realtimeFailure( - kind, - "CONNECT", - true, - ); - const attempt = vi.fn(() => failed(terminal)); + const retryable = realtimeFailure(kind, "CONNECT", true); + const terminal = realtimeFailure(kind, "CONNECT", false); + let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, + random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, - attempt, + attempt: () => + failed(++attempts === 1 ? retryable : terminal), }); - const result = await coordinator.run(); + const running = coordinator.run(); + await flush(); + expect(attempts).toBe(1); + expect(clock.sleepCalls).toEqual([50]); + clock.advance(50); + const result = await running; + + expect(attempts).toBe(2); expect(result).toBe(terminal); - expect(attempt).toHaveBeenCalledOnce(); - expect(clock.sleepCalls).toEqual([]); }, ); - it("does not reconnect an external close failure that requires but omits a server hint", async () => { + it("reconnects a retryable external close failure without a server hint", async () => { const clock = new ManualClock(); - const terminal = realtimeFailure( + const retryable = realtimeFailure( "PROVIDER_UNAVAILABLE", "RECEIVE", true, ); - const attempt = vi.fn(() => - opened( - Object.freeze({ - classifyClosed: ( - receipt: RealtimeReconnectOutcome, - ) => receipt, - close: vi.fn(), - waitClosed: () => failed(terminal), - }), - ), + const terminal = realtimeFailure( + "PROVIDER_UNAVAILABLE", + "RECEIVE", + false, ); + let attempts = 0; const coordinator = createRealtimeReconnectCoordinator({ policy: policy(), environment: new MutableOnlineEnvironment(true), clock, + random: () => 0.5, confirmTransportBarrier: confirmationSucceeded, - attempt, + attempt: () => { + const closedWith = ++attempts === 1 ? retryable : terminal; + return opened( + Object.freeze({ + classifyClosed: ( + receipt: RealtimeReconnectOutcome, + ) => receipt, + close: vi.fn(), + waitClosed: () => failed(closedWith), + }), + ); + }, }); - const result = await coordinator.run(); + const running = coordinator.run(); + await flush(); + expect(attempts).toBe(1); + expect(clock.sleepCalls).toEqual([50]); + clock.advance(50); + const result = await running; + + expect(attempts).toBe(2); expect(result).toBe(terminal); - expect(attempt).toHaveBeenCalledOnce(); - expect(clock.sleepCalls).toEqual([]); }); it("waits for an explicit online signal without a retry timer", async () => { @@ -1455,3 +1484,200 @@ describe("realtime reconnect coordinator", () => { expect(coordinator.getState()).toBe("IDLE"); }); }); + +class FakeWebSocket implements WebSocketFacade { + protocol: string = REALTIME_WEBSOCKET_PROTOCOL; + readyState = 0; + bufferedAmount = 0; + readonly listeners = new Map< + string, + Set<(event: unknown) => void> + >(); + + send(): void { + if (this.readyState !== 1) throw new Error("not open"); + } + + close(): void { + this.readyState = 3; + } + + addEventListener( + type: string, + listener: (event: unknown) => void, + ): void { + const listeners = this.listeners.get(type) ?? new Set(); + listeners.add(listener); + this.listeners.set(type, listeners); + } + + removeEventListener( + type: string, + listener: (event: unknown) => void, + ): void { + this.listeners.get(type)?.delete(listener); + } + + accept(): void { + this.readyState = 1; + this.emit("open", {}); + this.emit("message", { + data: JSON.stringify({ + type: "WELCOME", + protocol: REALTIME_WEBSOCKET_PROTOCOL, + connectionId: "connection.0001", + heartbeatMs: 5_000, + heartbeatAckTimeoutMs: 1_000, + limits: { + maxFrameBytes: 64 * 1_024, + maxSubscriptions: 32, + maxInboundQueueCount: 256, + maxInboundQueueBytes: 4 * 1_024 * 1_024, + maxOutboundQueueCount: 128, + maxOutboundQueueBytes: 256 * 1_024, + maxBufferedAmountBytes: 256 * 1_024, + maxEventsPerSecond: 256, + }, + }), + }); + } + + nativeClose(code: number): void { + this.readyState = 3; + this.emit("close", { code, reason: "" }); + } + + private emit(type: string, event: unknown): void { + for (const listener of [...(this.listeners.get(type) ?? [])]) { + listener(event); + } + } +} + +describe("transport failures through the reconnect coordinator", () => { + it.each([ + ["NETWORK_LOST", 1_006], + ["RESTART", 1_012], + ] as const)( + "reconnects a WebSocket %s close that carries no server hint", + async (_category, code) => { + const clock = new ManualClock(); + const sockets: FakeWebSocket[] = []; + const coordinator = createRealtimeReconnectCoordinator< + WebSocketClosedReceipt + >({ + policy: policy(), + environment: new MutableOnlineEnvironment(true), + clock, + random: () => 0.5, + confirmTransportBarrier: confirmationSucceeded, + attempt: async (context) => { + const connection = createWebSocketConnection({ + origin: "https://app.example.test", + endpointPath: "/_realtime/v1", + clock, + createNonce: () => "nonce.0001", + createSocket: () => { + const socket = new FakeWebSocket(); + sockets.push(socket); + return socket; + }, + onEvent: async () => + realtimeSuccess(REALTIME_TRANSPORT_CONTINUE), + onRecoveryRequired: () => undefined, + }); + const connected = await connection.connect(context.signal); + if (!connected.ok) return failed(connected); + return opened( + Object.freeze({ + waitClosed: () => connection.waitClosed(), + classifyClosed: (receipt: WebSocketClosedReceipt) => + Object.freeze({ + result: Object.freeze({ + ok: false as const, + error: receipt.error, + }), + }), + close: () => connection.close(), + }), + ); + }, + }); + + const running = coordinator.run(); + await flushUntil(() => sockets.length === 1); + sockets[0]!.accept(); + await flush(); + sockets[0]!.nativeClose(code); + await flushUntil(() => clock.sleepCalls.includes(50)); + clock.advance(50); + await flushUntil(() => sockets.length === 2); + expect(sockets).toHaveLength(2); + + // OVERLOADED carries no hint either and stays terminal. + sockets[1]!.accept(); + await flush(); + sockets[1]!.nativeClose(1_013); + await expect(running).resolves.toEqual({ + ok: false, + error: { + kind: "PROVIDER_UNAVAILABLE", + operation: "RECEIVE", + retryable: false, + }, + }); + expect(sockets).toHaveLength(2); + }, + ); + + it.each([502, 504])( + "reconnects SSE HTTP %i that carries no server hint", + async (status) => { + const clock = new ManualClock(); + const fetcher = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status })) + .mockResolvedValueOnce(new Response(null, { status: 403 })); + const coordinator = createRealtimeReconnectCoordinator({ + policy: policy(), + environment: new MutableOnlineEnvironment(true), + clock, + random: () => 0.5, + confirmTransportBarrier: confirmationSucceeded, + attempt: async (context) => { + const read = await createFetchSseConnection({ + endpoint: "https://app.example.test/events", + applicationOrigin: "https://app.example.test", + recoveryMode: "CURSOR", + fetcher, + clock, + }).read({ + resumeCursor: null, + signal: context.signal, + onEvent: () => realtimeSuccess(SSE_CONTINUE), + }); + // Only connect failures are exercised here. + return failed( + read.ok + ? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false) + : read, + ); + }, + }); + + const running = coordinator.run(); + await flushUntil(() => clock.sleepCalls.includes(50)); + clock.advance(50); + + await expect(running).resolves.toEqual({ + ok: false, + error: { + kind: "FORBIDDEN", + operation: "CONNECT", + retryable: false, + }, + }); + expect(fetcher).toHaveBeenCalledTimes(2); + }, + ); +});