fix(realtime): reconnect retryable failures without a server hint (R-4.2)

reconnectable() in the reconnect coordinator returned false for
PROVIDER_UNAVAILABLE/RATE_LIMITED whenever serverNotBeforeMs was null, even
with retryable:true. The transports send exactly that for WS NETWORK_LOST /
RESTART (1006/1001/1012, error event) and SSE 502/504, and WS can never carry
a hint, so those streams ended instead of reconnecting. The bounded poll
coordinator had the same rule for retryAfterMs.

Both now decide on retryable + kind only; a missing hint falls back to policy
backoff. Transports that require a hint and lack one already report
retryable:false (SSE 429/503 without Retry-After, WS OVERLOADED), which stays
terminal.

Tests: the reconnect and poll tests that pinned the old behavior ("does not
reconnect/retry ... without a server hint") encoded the bug and are rewritten
to expect a policy-backoff retry. New seam tests wire the real
createWebSocketConnection / createFetchSseConnection into the reconnect
coordinator. RED before the fix: "reconnects a WebSocket NETWORK_LOST|RESTART
close that carries no server hint" (1 socket, expected 2), "reconnects SSE
HTTP 502|504 that carries no server hint" (got PROVIDER_UNAVAILABLE
retryable:true instead of the second attempt's FORBIDDEN), "retries retryable
RATE_LIMITED|PROVIDER_UNAVAILABLE without a server hint on policy backoff"
(execute called once). The WS test also closes with OVERLOADED to show
retryable:false stays terminal.

Gates: check:architecture, check:types:app, check:types:test, lint,
test:component pass. test:unit passes except removal-fixture.test.ts, which
needs generated artifacts/release evidence absent from this fresh worktree
(unrelated).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-24 12:19:48 +09:00
co-authored by Claude Opus 5.5
parent 4e0745c569
commit 87a45893d6
4 changed files with 289 additions and 65 deletions
@@ -341,11 +341,11 @@ export function createBoundedPollCoordinator<Value>(
}
result = parsedResult;
if (!result.ok) {
// A missing server hint falls back to policy backoff; an executor
// that requires one reports `retryable: false` instead.
if (
!result.error.retryable ||
!isPollRetryableFailureKind(result.error.kind) ||
(isPollRetryAfterFailureKind(result.error.kind) &&
result.error.retryAfterMs === undefined)
!isPollRetryableFailureKind(result.error.kind)
) {
return pollFailure(result.error.kind);
}
+7 -23
View File
@@ -717,12 +717,7 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
settleAttemptGate(opened.failure.result);
phase.controller.abort();
phase.dispose();
if (
!reconnectable(
opened.failure,
opened.serverNotBeforeMs,
)
) {
if (!reconnectable(opened.failure)) {
return opened.failure.result;
}
const failedAt = safeNow(clock);
@@ -934,12 +929,7 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
if (classified.outcome.ok) {
return classified.outcome.result as RealtimeResult<void>;
}
if (
!reconnectable(
classified.outcome.failure,
classified.outcome.serverNotBeforeMs,
)
) {
if (!reconnectable(classified.outcome.failure)) {
return classified.outcome.failure.result;
}
@@ -1658,17 +1648,11 @@ function providerRetry(
);
}
function reconnectable(
failureSnapshot: FailureSnapshot,
serverNotBeforeMs: number | null,
): boolean {
if (
serverNotBeforeMs === null &&
(failureSnapshot.kind === "RATE_LIMITED" ||
failureSnapshot.kind === "PROVIDER_UNAVAILABLE")
) {
return false;
}
/**
* A transport that needs a server hint and did not get one already reports
* `retryable: false`, so a missing hint alone never ends the loop.
*/
function reconnectable(failureSnapshot: FailureSnapshot): boolean {
return (
failureSnapshot.retryable &&
RETRYABLE_KINDS.includes(
@@ -562,17 +562,27 @@ describe("bounded poll coordinator", () => {
});
it.each(["RATE_LIMITED", "PROVIDER_UNAVAILABLE"] as const)(
"does not retry %s without a bounded server hint",
"retries retryable %s without a server hint on policy backoff",
async (kind) => {
const clock = new ManualClock();
const execute = vi.fn().mockResolvedValue({
ok: false,
error: {
kind,
operation: "POLL",
retryable: true,
},
});
const execute = vi
.fn()
.mockResolvedValueOnce({
ok: false,
error: {
kind,
operation: "POLL",
retryable: true,
},
})
.mockResolvedValue({
ok: false,
error: {
kind,
operation: "POLL",
retryable: false,
},
});
const coordinator = createBoundedPollCoordinator<string>({
policy,
operation,
@@ -585,6 +595,10 @@ describe("bounded poll coordinator", () => {
const pending = coordinator.run();
clock.advance(5_000);
await flush();
expect(execute).toHaveBeenCalledTimes(1);
clock.advance(5_000);
await flush();
await expect(pending).resolves.toEqual({
ok: false,
error: {
@@ -593,7 +607,7 @@ describe("bounded poll coordinator", () => {
retryable: false,
},
});
expect(execute).toHaveBeenCalledTimes(1);
expect(execute).toHaveBeenCalledTimes(2);
},
);
@@ -5,8 +5,9 @@ import type {
RealtimeFailureKind,
RealtimeResult,
} from "../../../src/application/ports/realtime/shared.ts";
import type {
RealtimeRecoveryCheckpoint,
import {
REALTIME_TRANSPORT_CONTINUE,
type RealtimeRecoveryCheckpoint,
} from "../../../src/application/ports/realtime/event-authority.ts";
import {
createRealtimeReconnectCoordinator,
@@ -27,6 +28,18 @@ import {
realtimeFailure,
realtimeSuccess,
} from "../../../src/adapters/realtime/result.ts";
import {
createFetchSseConnection,
SSE_CONTINUE,
} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts";
import {
createWebSocketConnection,
type WebSocketClosedReceipt,
type WebSocketFacade,
} from "../../../src/adapters/realtime/websocket/websocket-connection.ts";
import {
REALTIME_WEBSOCKET_PROTOCOL,
} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts";
type Sleeper = Readonly<{
dueAt: number;
@@ -309,62 +322,78 @@ describe("realtime reconnect coordinator", () => {
"RATE_LIMITED",
"PROVIDER_UNAVAILABLE",
] as const)(
"does not reconnect an external %s attempt failure without a server hint",
"reconnects a retryable external %s attempt failure without a server hint",
async (kind) => {
const clock = new ManualClock();
const terminal = realtimeFailure(
kind,
"CONNECT",
true,
);
const attempt = vi.fn(() => failed(terminal));
const retryable = realtimeFailure(kind, "CONNECT", true);
const terminal = realtimeFailure(kind, "CONNECT", false);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
attempt: () =>
failed(++attempts === 1 ? retryable : terminal),
});
const result = await coordinator.run();
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
expect(attempt).toHaveBeenCalledOnce();
expect(clock.sleepCalls).toEqual([]);
},
);
it("does not reconnect an external close failure that requires but omits a server hint", async () => {
it("reconnects a retryable external close failure without a server hint", async () => {
const clock = new ManualClock();
const terminal = realtimeFailure(
const retryable = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
true,
);
const attempt = vi.fn(() =>
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(terminal),
}),
),
const terminal = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
false,
);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
attempt: () => {
const closedWith = ++attempts === 1 ? retryable : terminal;
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(closedWith),
}),
);
},
});
const result = await coordinator.run();
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
expect(attempt).toHaveBeenCalledOnce();
expect(clock.sleepCalls).toEqual([]);
});
it("waits for an explicit online signal without a retry timer", async () => {
@@ -1455,3 +1484,200 @@ describe("realtime reconnect coordinator", () => {
expect(coordinator.getState()).toBe("IDLE");
});
});
class FakeWebSocket implements WebSocketFacade {
protocol: string = REALTIME_WEBSOCKET_PROTOCOL;
readyState = 0;
bufferedAmount = 0;
readonly listeners = new Map<
string,
Set<(event: unknown) => void>
>();
send(): void {
if (this.readyState !== 1) throw new Error("not open");
}
close(): void {
this.readyState = 3;
}
addEventListener(
type: string,
listener: (event: unknown) => void,
): void {
const listeners = this.listeners.get(type) ?? new Set();
listeners.add(listener);
this.listeners.set(type, listeners);
}
removeEventListener(
type: string,
listener: (event: unknown) => void,
): void {
this.listeners.get(type)?.delete(listener);
}
accept(): void {
this.readyState = 1;
this.emit("open", {});
this.emit("message", {
data: JSON.stringify({
type: "WELCOME",
protocol: REALTIME_WEBSOCKET_PROTOCOL,
connectionId: "connection.0001",
heartbeatMs: 5_000,
heartbeatAckTimeoutMs: 1_000,
limits: {
maxFrameBytes: 64 * 1_024,
maxSubscriptions: 32,
maxInboundQueueCount: 256,
maxInboundQueueBytes: 4 * 1_024 * 1_024,
maxOutboundQueueCount: 128,
maxOutboundQueueBytes: 256 * 1_024,
maxBufferedAmountBytes: 256 * 1_024,
maxEventsPerSecond: 256,
},
}),
});
}
nativeClose(code: number): void {
this.readyState = 3;
this.emit("close", { code, reason: "" });
}
private emit(type: string, event: unknown): void {
for (const listener of [...(this.listeners.get(type) ?? [])]) {
listener(event);
}
}
}
describe("transport failures through the reconnect coordinator", () => {
it.each([
["NETWORK_LOST", 1_006],
["RESTART", 1_012],
] as const)(
"reconnects a WebSocket %s close that carries no server hint",
async (_category, code) => {
const clock = new ManualClock();
const sockets: FakeWebSocket[] = [];
const coordinator = createRealtimeReconnectCoordinator<
WebSocketClosedReceipt
>({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const connection = createWebSocketConnection({
origin: "https://app.example.test",
endpointPath: "/_realtime/v1",
clock,
createNonce: () => "nonce.0001",
createSocket: () => {
const socket = new FakeWebSocket();
sockets.push(socket);
return socket;
},
onEvent: async () =>
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
onRecoveryRequired: () => undefined,
});
const connected = await connection.connect(context.signal);
if (!connected.ok) return failed(connected);
return opened(
Object.freeze({
waitClosed: () => connection.waitClosed(),
classifyClosed: (receipt: WebSocketClosedReceipt) =>
Object.freeze({
result: Object.freeze({
ok: false as const,
error: receipt.error,
}),
}),
close: () => connection.close(),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => sockets.length === 1);
sockets[0]!.accept();
await flush();
sockets[0]!.nativeClose(code);
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await flushUntil(() => sockets.length === 2);
expect(sockets).toHaveLength(2);
// OVERLOADED carries no hint either and stays terminal.
sockets[1]!.accept();
await flush();
sockets[1]!.nativeClose(1_013);
await expect(running).resolves.toEqual({
ok: false,
error: {
kind: "PROVIDER_UNAVAILABLE",
operation: "RECEIVE",
retryable: false,
},
});
expect(sockets).toHaveLength(2);
},
);
it.each([502, 504])(
"reconnects SSE HTTP %i that carries no server hint",
async (status) => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status }))
.mockResolvedValueOnce(new Response(null, { status: 403 }));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const read = await createFetchSseConnection({
endpoint: "https://app.example.test/events",
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR",
fetcher,
clock,
}).read({
resumeCursor: null,
signal: context.signal,
onEvent: () => realtimeSuccess(SSE_CONTINUE),
});
// Only connect failures are exercised here.
return failed(
read.ok
? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false)
: read,
);
},
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await expect(running).resolves.toEqual({
ok: false,
error: {
kind: "FORBIDDEN",
operation: "CONNECT",
retryable: false,
},
});
expect(fetcher).toHaveBeenCalledTimes(2);
},
);
});