fix(realtime): reset the retry budget on every session end and pause it offline (04-01)

A long healthy connection that ended offline or with a throwing waitClosed
kept a spent elapsed budget and stopped with zero retries. Offline waits no
longer spend the budget, and WebSocket HEARTBEAT_ACK is now observable so a
quiet stream can earn a fresh budget.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-26 00:01:10 +09:00
co-authored by Claude Opus 5.5
parent 4d17259037
commit bcad7b0d12
3 changed files with 218 additions and 29 deletions
+23 -28
View File
@@ -55,7 +55,9 @@ export type RealtimeReconnectAttemptContext = Readonly<{
pendingRecovery: RealtimeCommittedRecovery | null;
isCurrent(): boolean;
/**
* Call only after a protocol-valid heartbeat or event.
* Call only after a protocol-valid heartbeat or event: an applied event,
* an SSE `onComment`, or a WebSocket `HEARTBEAT_ACK` observation. A quiet
* stream that never calls it cannot earn a fresh retry budget.
*/
markValidHeartbeatOrEvent(): void;
/**
@@ -493,15 +495,16 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
) {
return retry.terminalResult;
}
if (
(!online || requiredOnlineVersion !== null) &&
!(await waitOnline())
) {
const offlineWait = !online || requiredOnlineVersion !== null;
if (offlineWait && !(await waitOnline())) {
return failure(currentFailure() ?? "ABORTED");
}
now = safeNow(clock);
if (now === null) return retry.terminalResult;
// Offline time is not retry time. The elapsed budget restarts when
// the network returns; maxAttempts still bounds the loop.
if (offlineWait) budgetStartedAt = now;
const delay = nextDelay(
retry,
retriesUsed,
@@ -815,6 +818,21 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
phase.controller.signal,
);
const closeOffline = wentOffline(phase);
// Every way an opened session ends, including offline and a throwing
// waitClosed, can earn a fresh budget, not only a classified close.
const endedAt = safeNow(clock);
if (
endedAt !== null &&
isReconnectAttemptResetEligible({
policy,
openedAtMs: openedAt,
nowMs: endedAt,
observedValidHeartbeatOrEvent: validSignalObserved,
})
) {
retriesUsed = 0;
budgetStartedAt = endedAt;
}
const afterWaitClosed = currentFailure();
if (afterWaitClosed) {
@@ -902,18 +920,6 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
if (closedAt === null) {
return classified.outcome.failure.result;
}
if (
isReconnectAttemptResetEligible({
policy,
openedAtMs: openedAt,
nowMs: closedAt,
observedValidHeartbeatOrEvent:
validSignalObserved,
})
) {
retriesUsed = 0;
budgetStartedAt = closedAt;
}
retry = captureRetry(
classified.outcome.failure.result,
classified.outcome.failure.result,
@@ -932,17 +938,6 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
if (closedAt === null) {
return classified.directive.terminalFailure.result;
}
if (
isReconnectAttemptResetEligible({
policy,
openedAtMs: openedAt,
nowMs: closedAt,
observedValidHeartbeatOrEvent: validSignalObserved,
})
) {
retriesUsed = 0;
budgetStartedAt = closedAt;
}
pendingRecovery = classified.directive;
retry = captureRetry(
classified.directive.terminalFailure.result,
@@ -71,6 +71,7 @@ export type WebSocketConnectionObservation = Readonly<{
kind:
| "CONNECTED"
| "SUBSCRIBED"
| "HEARTBEAT_ACK"
| "FRAME_REJECTED"
| "RECOVERY_REQUIRED"
| "CLOSED";
@@ -372,6 +373,9 @@ export function createWebSocketConnection(
if (decoded.value.nonce === retiredHeartbeat) {
retiredHeartbeat = null;
}
// A quiet stream's only liveness proof; composition can forward it to
// the reconnect coordinator's markValidHeartbeatOrEvent.
observe({ kind: "HEARTBEAT_ACK" });
return;
}
const queueLimits = effectiveLimits ?? ceilings;
@@ -434,6 +434,105 @@ describe("realtime reconnect coordinator", () => {
expect(result).toBe(terminal);
});
it("does not spend the elapsed budget while waiting offline", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(false);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const attempt = vi.fn(() => failed(terminal));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const running = coordinator.run();
await flush();
// Offline for longer than maxElapsedMs before the first attempt.
clock.advance(6_000);
environment.emit(true);
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempt).toHaveBeenCalledTimes(1);
expect(result).toBe(terminal);
});
it.each(["an offline blip", "a waitClosed throw"] as const)(
"reconnects after %s ends a healthy session older than the elapsed budget",
async (ending) => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const closing = deferred<RealtimeReconnectOutcome<void>>();
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context: RealtimeReconnectAttemptContext) => {
attempts += 1;
if (attempts > 1) return failed(terminal);
context.markValidHeartbeatOrEvent();
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: () =>
closing.resolve(
failed(
realtimeFailure("OFFLINE", "RECEIVE", true),
),
),
waitClosed: () =>
ending === "an offline blip"
? closing.promise
: closing.promise.then(() => {
throw new Error("transport failed");
}),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => attempts === 1);
await flush();
// Healthy for twice maxElapsedMs, then a one second outage.
clock.advance(10_000);
if (ending === "an offline blip") {
environment.emit(false);
await flush();
clock.advance(1_000);
environment.emit(true);
} else {
closing.resolve(closedSuccessfully());
}
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
},
);
it("resets a spent retry only after a valid signal and a stable open", async () => {
const runScenario = async (markValid: boolean, openMs: number) => {
const clock = new ManualClock();
@@ -1502,9 +1601,11 @@ class FakeWebSocket implements WebSocketFacade {
string,
Set<(event: unknown) => void>
>();
readonly sent: string[] = [];
send(): void {
send(data: string): void {
if (this.readyState !== 1) throw new Error("not open");
this.sent.push(data);
}
close(): void {
@@ -1556,6 +1657,20 @@ class FakeWebSocket implements WebSocketFacade {
this.emit("close", { code, reason: "" });
}
ackHeartbeats(): void {
for (const raw of this.sent.splice(0)) {
const frame = JSON.parse(raw) as { type: string; nonce: string };
if (frame.type !== "HEARTBEAT") continue;
this.emit("message", {
data: JSON.stringify({
type: "HEARTBEAT_ACK",
protocol: REALTIME_WEBSOCKET_PROTOCOL,
nonce: frame.nonce,
}),
});
}
}
private emit(type: string, event: unknown): void {
for (const listener of [...(this.listeners.get(type) ?? [])]) {
listener(event);
@@ -1639,6 +1754,81 @@ describe("transport failures through the reconnect coordinator", () => {
},
);
it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => {
const clock = new ManualClock();
const sockets: FakeWebSocket[] = [];
let nonce = 0;
const coordinator = createRealtimeReconnectCoordinator<
WebSocketClosedReceipt
>({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const connection = createWebSocketConnection({
origin: "https://app.example.test",
endpointPath: "/_realtime/v1",
clock,
createNonce: () => `nonce.${++nonce}`,
createSocket: () => {
const socket = new FakeWebSocket();
sockets.push(socket);
return socket;
},
onEvent: async () =>
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
onRecoveryRequired: () => undefined,
observe: (observation) => {
if (observation.kind === "HEARTBEAT_ACK") {
context.markValidHeartbeatOrEvent();
}
},
});
const connected = await connection.connect(context.signal);
if (!connected.ok) return failed(connected);
return opened(
Object.freeze({
waitClosed: () => connection.waitClosed(),
classifyClosed: (receipt: WebSocketClosedReceipt) =>
Object.freeze({
result: Object.freeze({
ok: false as const,
error: receipt.error,
}),
}),
close: () => connection.close(),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => sockets.length === 1);
sockets[0]!.accept();
await flush();
// No events, only acknowledged heartbeats, for longer than maxElapsedMs.
for (let beat = 0; beat < 2; beat += 1) {
clock.advance(5_000);
await flush();
sockets[0]!.ackHeartbeats();
clock.advance(1_000);
await flush();
}
sockets[0]!.nativeClose(1_012);
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await flushUntil(() => sockets.length === 2);
expect(sockets).toHaveLength(2);
coordinator.close();
await expect(running).resolves.toMatchObject({
ok: false,
error: { kind: "CLOSED" },
});
});
it.each([502, 504])(
"reconnects SSE HTTP %i that carries no server hint",
async (status) => {