fix(realtime): reconnect retryable failures without a server hint (R-4.2)
reconnectable() in the reconnect coordinator returned false for
PROVIDER_UNAVAILABLE/RATE_LIMITED whenever serverNotBeforeMs was null, even
with retryable:true. The transports send exactly that for WS NETWORK_LOST /
RESTART (1006/1001/1012, error event) and SSE 502/504, and WS can never carry
a hint, so those streams ended instead of reconnecting. The bounded poll
coordinator had the same rule for retryAfterMs.
Both now decide on retryable + kind only; a missing hint falls back to policy
backoff. Transports that require a hint and lack one already report
retryable:false (SSE 429/503 without Retry-After, WS OVERLOADED), which stays
terminal.
Tests: the reconnect and poll tests that pinned the old behavior ("does not
reconnect/retry ... without a server hint") encoded the bug and are rewritten
to expect a policy-backoff retry. New seam tests wire the real
createWebSocketConnection / createFetchSseConnection into the reconnect
coordinator. RED before the fix: "reconnects a WebSocket NETWORK_LOST|RESTART
close that carries no server hint" (1 socket, expected 2), "reconnects SSE
HTTP 502|504 that carries no server hint" (got PROVIDER_UNAVAILABLE
retryable:true instead of the second attempt's FORBIDDEN), "retries retryable
RATE_LIMITED|PROVIDER_UNAVAILABLE without a server hint on policy backoff"
(execute called once). The WS test also closes with OVERLOADED to show
retryable:false stays terminal.
Gates: check:architecture, check:types:app, check:types:test, lint,
test:component pass. test:unit passes except removal-fixture.test.ts, which
needs generated artifacts/release evidence absent from this fresh worktree
(unrelated).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4e0745c569
commit
87a45893d6
@@ -341,11 +341,11 @@ export function createBoundedPollCoordinator<Value>(
|
||||
}
|
||||
result = parsedResult;
|
||||
if (!result.ok) {
|
||||
// A missing server hint falls back to policy backoff; an executor
|
||||
// that requires one reports `retryable: false` instead.
|
||||
if (
|
||||
!result.error.retryable ||
|
||||
!isPollRetryableFailureKind(result.error.kind) ||
|
||||
(isPollRetryAfterFailureKind(result.error.kind) &&
|
||||
result.error.retryAfterMs === undefined)
|
||||
!isPollRetryableFailureKind(result.error.kind)
|
||||
) {
|
||||
return pollFailure(result.error.kind);
|
||||
}
|
||||
|
||||
@@ -717,12 +717,7 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
settleAttemptGate(opened.failure.result);
|
||||
phase.controller.abort();
|
||||
phase.dispose();
|
||||
if (
|
||||
!reconnectable(
|
||||
opened.failure,
|
||||
opened.serverNotBeforeMs,
|
||||
)
|
||||
) {
|
||||
if (!reconnectable(opened.failure)) {
|
||||
return opened.failure.result;
|
||||
}
|
||||
const failedAt = safeNow(clock);
|
||||
@@ -934,12 +929,7 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
if (classified.outcome.ok) {
|
||||
return classified.outcome.result as RealtimeResult<void>;
|
||||
}
|
||||
if (
|
||||
!reconnectable(
|
||||
classified.outcome.failure,
|
||||
classified.outcome.serverNotBeforeMs,
|
||||
)
|
||||
) {
|
||||
if (!reconnectable(classified.outcome.failure)) {
|
||||
return classified.outcome.failure.result;
|
||||
}
|
||||
|
||||
@@ -1658,17 +1648,11 @@ function providerRetry(
|
||||
);
|
||||
}
|
||||
|
||||
function reconnectable(
|
||||
failureSnapshot: FailureSnapshot,
|
||||
serverNotBeforeMs: number | null,
|
||||
): boolean {
|
||||
if (
|
||||
serverNotBeforeMs === null &&
|
||||
(failureSnapshot.kind === "RATE_LIMITED" ||
|
||||
failureSnapshot.kind === "PROVIDER_UNAVAILABLE")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* A transport that needs a server hint and did not get one already reports
|
||||
* `retryable: false`, so a missing hint alone never ends the loop.
|
||||
*/
|
||||
function reconnectable(failureSnapshot: FailureSnapshot): boolean {
|
||||
return (
|
||||
failureSnapshot.retryable &&
|
||||
RETRYABLE_KINDS.includes(
|
||||
|
||||
@@ -562,17 +562,27 @@ describe("bounded poll coordinator", () => {
|
||||
});
|
||||
|
||||
it.each(["RATE_LIMITED", "PROVIDER_UNAVAILABLE"] as const)(
|
||||
"does not retry %s without a bounded server hint",
|
||||
"retries retryable %s without a server hint on policy backoff",
|
||||
async (kind) => {
|
||||
const clock = new ManualClock();
|
||||
const execute = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
error: {
|
||||
kind,
|
||||
operation: "POLL",
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
const execute = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: {
|
||||
kind,
|
||||
operation: "POLL",
|
||||
retryable: true,
|
||||
},
|
||||
})
|
||||
.mockResolvedValue({
|
||||
ok: false,
|
||||
error: {
|
||||
kind,
|
||||
operation: "POLL",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
const coordinator = createBoundedPollCoordinator<string>({
|
||||
policy,
|
||||
operation,
|
||||
@@ -585,6 +595,10 @@ describe("bounded poll coordinator", () => {
|
||||
const pending = coordinator.run();
|
||||
clock.advance(5_000);
|
||||
await flush();
|
||||
expect(execute).toHaveBeenCalledTimes(1);
|
||||
clock.advance(5_000);
|
||||
await flush();
|
||||
|
||||
await expect(pending).resolves.toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
@@ -593,7 +607,7 @@ describe("bounded poll coordinator", () => {
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(execute).toHaveBeenCalledTimes(1);
|
||||
expect(execute).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -5,8 +5,9 @@ import type {
|
||||
RealtimeFailureKind,
|
||||
RealtimeResult,
|
||||
} from "../../../src/application/ports/realtime/shared.ts";
|
||||
import type {
|
||||
RealtimeRecoveryCheckpoint,
|
||||
import {
|
||||
REALTIME_TRANSPORT_CONTINUE,
|
||||
type RealtimeRecoveryCheckpoint,
|
||||
} from "../../../src/application/ports/realtime/event-authority.ts";
|
||||
import {
|
||||
createRealtimeReconnectCoordinator,
|
||||
@@ -27,6 +28,18 @@ import {
|
||||
realtimeFailure,
|
||||
realtimeSuccess,
|
||||
} from "../../../src/adapters/realtime/result.ts";
|
||||
import {
|
||||
createFetchSseConnection,
|
||||
SSE_CONTINUE,
|
||||
} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts";
|
||||
import {
|
||||
createWebSocketConnection,
|
||||
type WebSocketClosedReceipt,
|
||||
type WebSocketFacade,
|
||||
} from "../../../src/adapters/realtime/websocket/websocket-connection.ts";
|
||||
import {
|
||||
REALTIME_WEBSOCKET_PROTOCOL,
|
||||
} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts";
|
||||
|
||||
type Sleeper = Readonly<{
|
||||
dueAt: number;
|
||||
@@ -309,62 +322,78 @@ describe("realtime reconnect coordinator", () => {
|
||||
"RATE_LIMITED",
|
||||
"PROVIDER_UNAVAILABLE",
|
||||
] as const)(
|
||||
"does not reconnect an external %s attempt failure without a server hint",
|
||||
"reconnects a retryable external %s attempt failure without a server hint",
|
||||
async (kind) => {
|
||||
const clock = new ManualClock();
|
||||
const terminal = realtimeFailure(
|
||||
kind,
|
||||
"CONNECT",
|
||||
true,
|
||||
);
|
||||
const attempt = vi.fn(() => failed(terminal));
|
||||
const retryable = realtimeFailure(kind, "CONNECT", true);
|
||||
const terminal = realtimeFailure(kind, "CONNECT", false);
|
||||
let attempts = 0;
|
||||
const coordinator = createRealtimeReconnectCoordinator({
|
||||
policy: policy(),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt,
|
||||
attempt: () =>
|
||||
failed(++attempts === 1 ? retryable : terminal),
|
||||
});
|
||||
|
||||
const result = await coordinator.run();
|
||||
const running = coordinator.run();
|
||||
await flush();
|
||||
expect(attempts).toBe(1);
|
||||
expect(clock.sleepCalls).toEqual([50]);
|
||||
|
||||
clock.advance(50);
|
||||
const result = await running;
|
||||
|
||||
expect(attempts).toBe(2);
|
||||
expect(result).toBe(terminal);
|
||||
expect(attempt).toHaveBeenCalledOnce();
|
||||
expect(clock.sleepCalls).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not reconnect an external close failure that requires but omits a server hint", async () => {
|
||||
it("reconnects a retryable external close failure without a server hint", async () => {
|
||||
const clock = new ManualClock();
|
||||
const terminal = realtimeFailure(
|
||||
const retryable = realtimeFailure(
|
||||
"PROVIDER_UNAVAILABLE",
|
||||
"RECEIVE",
|
||||
true,
|
||||
);
|
||||
const attempt = vi.fn(() =>
|
||||
opened(
|
||||
Object.freeze({
|
||||
classifyClosed: (
|
||||
receipt: RealtimeReconnectOutcome<void>,
|
||||
) => receipt,
|
||||
close: vi.fn(),
|
||||
waitClosed: () => failed(terminal),
|
||||
}),
|
||||
),
|
||||
const terminal = realtimeFailure(
|
||||
"PROVIDER_UNAVAILABLE",
|
||||
"RECEIVE",
|
||||
false,
|
||||
);
|
||||
let attempts = 0;
|
||||
const coordinator = createRealtimeReconnectCoordinator({
|
||||
policy: policy(),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt,
|
||||
attempt: () => {
|
||||
const closedWith = ++attempts === 1 ? retryable : terminal;
|
||||
return opened(
|
||||
Object.freeze({
|
||||
classifyClosed: (
|
||||
receipt: RealtimeReconnectOutcome<void>,
|
||||
) => receipt,
|
||||
close: vi.fn(),
|
||||
waitClosed: () => failed(closedWith),
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const result = await coordinator.run();
|
||||
const running = coordinator.run();
|
||||
await flush();
|
||||
expect(attempts).toBe(1);
|
||||
expect(clock.sleepCalls).toEqual([50]);
|
||||
|
||||
clock.advance(50);
|
||||
const result = await running;
|
||||
|
||||
expect(attempts).toBe(2);
|
||||
expect(result).toBe(terminal);
|
||||
expect(attempt).toHaveBeenCalledOnce();
|
||||
expect(clock.sleepCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it("waits for an explicit online signal without a retry timer", async () => {
|
||||
@@ -1455,3 +1484,200 @@ describe("realtime reconnect coordinator", () => {
|
||||
expect(coordinator.getState()).toBe("IDLE");
|
||||
});
|
||||
});
|
||||
|
||||
class FakeWebSocket implements WebSocketFacade {
|
||||
protocol: string = REALTIME_WEBSOCKET_PROTOCOL;
|
||||
readyState = 0;
|
||||
bufferedAmount = 0;
|
||||
readonly listeners = new Map<
|
||||
string,
|
||||
Set<(event: unknown) => void>
|
||||
>();
|
||||
|
||||
send(): void {
|
||||
if (this.readyState !== 1) throw new Error("not open");
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.readyState = 3;
|
||||
}
|
||||
|
||||
addEventListener(
|
||||
type: string,
|
||||
listener: (event: unknown) => void,
|
||||
): void {
|
||||
const listeners = this.listeners.get(type) ?? new Set();
|
||||
listeners.add(listener);
|
||||
this.listeners.set(type, listeners);
|
||||
}
|
||||
|
||||
removeEventListener(
|
||||
type: string,
|
||||
listener: (event: unknown) => void,
|
||||
): void {
|
||||
this.listeners.get(type)?.delete(listener);
|
||||
}
|
||||
|
||||
accept(): void {
|
||||
this.readyState = 1;
|
||||
this.emit("open", {});
|
||||
this.emit("message", {
|
||||
data: JSON.stringify({
|
||||
type: "WELCOME",
|
||||
protocol: REALTIME_WEBSOCKET_PROTOCOL,
|
||||
connectionId: "connection.0001",
|
||||
heartbeatMs: 5_000,
|
||||
heartbeatAckTimeoutMs: 1_000,
|
||||
limits: {
|
||||
maxFrameBytes: 64 * 1_024,
|
||||
maxSubscriptions: 32,
|
||||
maxInboundQueueCount: 256,
|
||||
maxInboundQueueBytes: 4 * 1_024 * 1_024,
|
||||
maxOutboundQueueCount: 128,
|
||||
maxOutboundQueueBytes: 256 * 1_024,
|
||||
maxBufferedAmountBytes: 256 * 1_024,
|
||||
maxEventsPerSecond: 256,
|
||||
},
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
nativeClose(code: number): void {
|
||||
this.readyState = 3;
|
||||
this.emit("close", { code, reason: "" });
|
||||
}
|
||||
|
||||
private emit(type: string, event: unknown): void {
|
||||
for (const listener of [...(this.listeners.get(type) ?? [])]) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe("transport failures through the reconnect coordinator", () => {
|
||||
it.each([
|
||||
["NETWORK_LOST", 1_006],
|
||||
["RESTART", 1_012],
|
||||
] as const)(
|
||||
"reconnects a WebSocket %s close that carries no server hint",
|
||||
async (_category, code) => {
|
||||
const clock = new ManualClock();
|
||||
const sockets: FakeWebSocket[] = [];
|
||||
const coordinator = createRealtimeReconnectCoordinator<
|
||||
WebSocketClosedReceipt
|
||||
>({
|
||||
policy: policy(),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt: async (context) => {
|
||||
const connection = createWebSocketConnection({
|
||||
origin: "https://app.example.test",
|
||||
endpointPath: "/_realtime/v1",
|
||||
clock,
|
||||
createNonce: () => "nonce.0001",
|
||||
createSocket: () => {
|
||||
const socket = new FakeWebSocket();
|
||||
sockets.push(socket);
|
||||
return socket;
|
||||
},
|
||||
onEvent: async () =>
|
||||
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
|
||||
onRecoveryRequired: () => undefined,
|
||||
});
|
||||
const connected = await connection.connect(context.signal);
|
||||
if (!connected.ok) return failed(connected);
|
||||
return opened(
|
||||
Object.freeze({
|
||||
waitClosed: () => connection.waitClosed(),
|
||||
classifyClosed: (receipt: WebSocketClosedReceipt) =>
|
||||
Object.freeze({
|
||||
result: Object.freeze({
|
||||
ok: false as const,
|
||||
error: receipt.error,
|
||||
}),
|
||||
}),
|
||||
close: () => connection.close(),
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const running = coordinator.run();
|
||||
await flushUntil(() => sockets.length === 1);
|
||||
sockets[0]!.accept();
|
||||
await flush();
|
||||
sockets[0]!.nativeClose(code);
|
||||
await flushUntil(() => clock.sleepCalls.includes(50));
|
||||
clock.advance(50);
|
||||
await flushUntil(() => sockets.length === 2);
|
||||
expect(sockets).toHaveLength(2);
|
||||
|
||||
// OVERLOADED carries no hint either and stays terminal.
|
||||
sockets[1]!.accept();
|
||||
await flush();
|
||||
sockets[1]!.nativeClose(1_013);
|
||||
await expect(running).resolves.toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "PROVIDER_UNAVAILABLE",
|
||||
operation: "RECEIVE",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(sockets).toHaveLength(2);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([502, 504])(
|
||||
"reconnects SSE HTTP %i that carries no server hint",
|
||||
async (status) => {
|
||||
const clock = new ManualClock();
|
||||
const fetcher = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValueOnce(new Response(null, { status }))
|
||||
.mockResolvedValueOnce(new Response(null, { status: 403 }));
|
||||
const coordinator = createRealtimeReconnectCoordinator({
|
||||
policy: policy(),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt: async (context) => {
|
||||
const read = await createFetchSseConnection({
|
||||
endpoint: "https://app.example.test/events",
|
||||
applicationOrigin: "https://app.example.test",
|
||||
recoveryMode: "CURSOR",
|
||||
fetcher,
|
||||
clock,
|
||||
}).read({
|
||||
resumeCursor: null,
|
||||
signal: context.signal,
|
||||
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
||||
});
|
||||
// Only connect failures are exercised here.
|
||||
return failed(
|
||||
read.ok
|
||||
? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false)
|
||||
: read,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const running = coordinator.run();
|
||||
await flushUntil(() => clock.sleepCalls.includes(50));
|
||||
clock.advance(50);
|
||||
|
||||
await expect(running).resolves.toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "FORBIDDEN",
|
||||
operation: "CONNECT",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(fetcher).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user