fix(realtime): reset the retry budget on every session end and pause it offline (04-01)
A long healthy connection that ended offline or with a throwing waitClosed kept a spent elapsed budget and stopped with zero retries. Offline waits no longer spend the budget, and WebSocket HEARTBEAT_ACK is now observable so a quiet stream can earn a fresh budget. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4d17259037
commit
bcad7b0d12
@@ -55,7 +55,9 @@ export type RealtimeReconnectAttemptContext = Readonly<{
|
||||
pendingRecovery: RealtimeCommittedRecovery | null;
|
||||
isCurrent(): boolean;
|
||||
/**
|
||||
* Call only after a protocol-valid heartbeat or event.
|
||||
* Call only after a protocol-valid heartbeat or event: an applied event,
|
||||
* an SSE `onComment`, or a WebSocket `HEARTBEAT_ACK` observation. A quiet
|
||||
* stream that never calls it cannot earn a fresh retry budget.
|
||||
*/
|
||||
markValidHeartbeatOrEvent(): void;
|
||||
/**
|
||||
@@ -493,15 +495,16 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
) {
|
||||
return retry.terminalResult;
|
||||
}
|
||||
if (
|
||||
(!online || requiredOnlineVersion !== null) &&
|
||||
!(await waitOnline())
|
||||
) {
|
||||
const offlineWait = !online || requiredOnlineVersion !== null;
|
||||
if (offlineWait && !(await waitOnline())) {
|
||||
return failure(currentFailure() ?? "ABORTED");
|
||||
}
|
||||
|
||||
now = safeNow(clock);
|
||||
if (now === null) return retry.terminalResult;
|
||||
// Offline time is not retry time. The elapsed budget restarts when
|
||||
// the network returns; maxAttempts still bounds the loop.
|
||||
if (offlineWait) budgetStartedAt = now;
|
||||
const delay = nextDelay(
|
||||
retry,
|
||||
retriesUsed,
|
||||
@@ -815,6 +818,21 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
phase.controller.signal,
|
||||
);
|
||||
const closeOffline = wentOffline(phase);
|
||||
// Every way an opened session ends, including offline and a throwing
|
||||
// waitClosed, can earn a fresh budget, not only a classified close.
|
||||
const endedAt = safeNow(clock);
|
||||
if (
|
||||
endedAt !== null &&
|
||||
isReconnectAttemptResetEligible({
|
||||
policy,
|
||||
openedAtMs: openedAt,
|
||||
nowMs: endedAt,
|
||||
observedValidHeartbeatOrEvent: validSignalObserved,
|
||||
})
|
||||
) {
|
||||
retriesUsed = 0;
|
||||
budgetStartedAt = endedAt;
|
||||
}
|
||||
|
||||
const afterWaitClosed = currentFailure();
|
||||
if (afterWaitClosed) {
|
||||
@@ -902,18 +920,6 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
if (closedAt === null) {
|
||||
return classified.outcome.failure.result;
|
||||
}
|
||||
if (
|
||||
isReconnectAttemptResetEligible({
|
||||
policy,
|
||||
openedAtMs: openedAt,
|
||||
nowMs: closedAt,
|
||||
observedValidHeartbeatOrEvent:
|
||||
validSignalObserved,
|
||||
})
|
||||
) {
|
||||
retriesUsed = 0;
|
||||
budgetStartedAt = closedAt;
|
||||
}
|
||||
retry = captureRetry(
|
||||
classified.outcome.failure.result,
|
||||
classified.outcome.failure.result,
|
||||
@@ -932,17 +938,6 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
if (closedAt === null) {
|
||||
return classified.directive.terminalFailure.result;
|
||||
}
|
||||
if (
|
||||
isReconnectAttemptResetEligible({
|
||||
policy,
|
||||
openedAtMs: openedAt,
|
||||
nowMs: closedAt,
|
||||
observedValidHeartbeatOrEvent: validSignalObserved,
|
||||
})
|
||||
) {
|
||||
retriesUsed = 0;
|
||||
budgetStartedAt = closedAt;
|
||||
}
|
||||
pendingRecovery = classified.directive;
|
||||
retry = captureRetry(
|
||||
classified.directive.terminalFailure.result,
|
||||
|
||||
@@ -71,6 +71,7 @@ export type WebSocketConnectionObservation = Readonly<{
|
||||
kind:
|
||||
| "CONNECTED"
|
||||
| "SUBSCRIBED"
|
||||
| "HEARTBEAT_ACK"
|
||||
| "FRAME_REJECTED"
|
||||
| "RECOVERY_REQUIRED"
|
||||
| "CLOSED";
|
||||
@@ -372,6 +373,9 @@ export function createWebSocketConnection(
|
||||
if (decoded.value.nonce === retiredHeartbeat) {
|
||||
retiredHeartbeat = null;
|
||||
}
|
||||
// A quiet stream's only liveness proof; composition can forward it to
|
||||
// the reconnect coordinator's markValidHeartbeatOrEvent.
|
||||
observe({ kind: "HEARTBEAT_ACK" });
|
||||
return;
|
||||
}
|
||||
const queueLimits = effectiveLimits ?? ceilings;
|
||||
|
||||
@@ -434,6 +434,105 @@ describe("realtime reconnect coordinator", () => {
|
||||
expect(result).toBe(terminal);
|
||||
});
|
||||
|
||||
it("does not spend the elapsed budget while waiting offline", async () => {
|
||||
const clock = new ManualClock();
|
||||
const environment = new MutableOnlineEnvironment(false);
|
||||
const terminal = realtimeFailure(
|
||||
"FORBIDDEN",
|
||||
"CONNECT",
|
||||
false,
|
||||
);
|
||||
const attempt = vi.fn(() => failed(terminal));
|
||||
const coordinator = createRealtimeReconnectCoordinator({
|
||||
policy: policy(),
|
||||
environment,
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt,
|
||||
});
|
||||
|
||||
const running = coordinator.run();
|
||||
await flush();
|
||||
// Offline for longer than maxElapsedMs before the first attempt.
|
||||
clock.advance(6_000);
|
||||
environment.emit(true);
|
||||
await flushUntil(() => clock.sleepCalls.length === 1);
|
||||
expect(clock.sleepCalls).toEqual([50]);
|
||||
|
||||
clock.advance(50);
|
||||
const result = await running;
|
||||
expect(attempt).toHaveBeenCalledTimes(1);
|
||||
expect(result).toBe(terminal);
|
||||
});
|
||||
|
||||
it.each(["an offline blip", "a waitClosed throw"] as const)(
|
||||
"reconnects after %s ends a healthy session older than the elapsed budget",
|
||||
async (ending) => {
|
||||
const clock = new ManualClock();
|
||||
const environment = new MutableOnlineEnvironment(true);
|
||||
const closing = deferred<RealtimeReconnectOutcome<void>>();
|
||||
const terminal = realtimeFailure(
|
||||
"FORBIDDEN",
|
||||
"CONNECT",
|
||||
false,
|
||||
);
|
||||
let attempts = 0;
|
||||
const coordinator = createRealtimeReconnectCoordinator({
|
||||
policy: policy(),
|
||||
environment,
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt: (context: RealtimeReconnectAttemptContext) => {
|
||||
attempts += 1;
|
||||
if (attempts > 1) return failed(terminal);
|
||||
context.markValidHeartbeatOrEvent();
|
||||
return opened<RealtimeReconnectOutcome<void>>(
|
||||
Object.freeze({
|
||||
classifyClosed: (
|
||||
receipt: RealtimeReconnectOutcome<void>,
|
||||
) => receipt,
|
||||
close: () =>
|
||||
closing.resolve(
|
||||
failed(
|
||||
realtimeFailure("OFFLINE", "RECEIVE", true),
|
||||
),
|
||||
),
|
||||
waitClosed: () =>
|
||||
ending === "an offline blip"
|
||||
? closing.promise
|
||||
: closing.promise.then(() => {
|
||||
throw new Error("transport failed");
|
||||
}),
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const running = coordinator.run();
|
||||
await flushUntil(() => attempts === 1);
|
||||
await flush();
|
||||
// Healthy for twice maxElapsedMs, then a one second outage.
|
||||
clock.advance(10_000);
|
||||
if (ending === "an offline blip") {
|
||||
environment.emit(false);
|
||||
await flush();
|
||||
clock.advance(1_000);
|
||||
environment.emit(true);
|
||||
} else {
|
||||
closing.resolve(closedSuccessfully());
|
||||
}
|
||||
await flushUntil(() => clock.sleepCalls.length === 1);
|
||||
expect(clock.sleepCalls).toEqual([50]);
|
||||
|
||||
clock.advance(50);
|
||||
const result = await running;
|
||||
expect(attempts).toBe(2);
|
||||
expect(result).toBe(terminal);
|
||||
},
|
||||
);
|
||||
|
||||
it("resets a spent retry only after a valid signal and a stable open", async () => {
|
||||
const runScenario = async (markValid: boolean, openMs: number) => {
|
||||
const clock = new ManualClock();
|
||||
@@ -1502,9 +1601,11 @@ class FakeWebSocket implements WebSocketFacade {
|
||||
string,
|
||||
Set<(event: unknown) => void>
|
||||
>();
|
||||
readonly sent: string[] = [];
|
||||
|
||||
send(): void {
|
||||
send(data: string): void {
|
||||
if (this.readyState !== 1) throw new Error("not open");
|
||||
this.sent.push(data);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
@@ -1556,6 +1657,20 @@ class FakeWebSocket implements WebSocketFacade {
|
||||
this.emit("close", { code, reason: "" });
|
||||
}
|
||||
|
||||
ackHeartbeats(): void {
|
||||
for (const raw of this.sent.splice(0)) {
|
||||
const frame = JSON.parse(raw) as { type: string; nonce: string };
|
||||
if (frame.type !== "HEARTBEAT") continue;
|
||||
this.emit("message", {
|
||||
data: JSON.stringify({
|
||||
type: "HEARTBEAT_ACK",
|
||||
protocol: REALTIME_WEBSOCKET_PROTOCOL,
|
||||
nonce: frame.nonce,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private emit(type: string, event: unknown): void {
|
||||
for (const listener of [...(this.listeners.get(type) ?? [])]) {
|
||||
listener(event);
|
||||
@@ -1639,6 +1754,81 @@ describe("transport failures through the reconnect coordinator", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => {
|
||||
const clock = new ManualClock();
|
||||
const sockets: FakeWebSocket[] = [];
|
||||
let nonce = 0;
|
||||
const coordinator = createRealtimeReconnectCoordinator<
|
||||
WebSocketClosedReceipt
|
||||
>({
|
||||
policy: policy(),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
attempt: async (context) => {
|
||||
const connection = createWebSocketConnection({
|
||||
origin: "https://app.example.test",
|
||||
endpointPath: "/_realtime/v1",
|
||||
clock,
|
||||
createNonce: () => `nonce.${++nonce}`,
|
||||
createSocket: () => {
|
||||
const socket = new FakeWebSocket();
|
||||
sockets.push(socket);
|
||||
return socket;
|
||||
},
|
||||
onEvent: async () =>
|
||||
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
|
||||
onRecoveryRequired: () => undefined,
|
||||
observe: (observation) => {
|
||||
if (observation.kind === "HEARTBEAT_ACK") {
|
||||
context.markValidHeartbeatOrEvent();
|
||||
}
|
||||
},
|
||||
});
|
||||
const connected = await connection.connect(context.signal);
|
||||
if (!connected.ok) return failed(connected);
|
||||
return opened(
|
||||
Object.freeze({
|
||||
waitClosed: () => connection.waitClosed(),
|
||||
classifyClosed: (receipt: WebSocketClosedReceipt) =>
|
||||
Object.freeze({
|
||||
result: Object.freeze({
|
||||
ok: false as const,
|
||||
error: receipt.error,
|
||||
}),
|
||||
}),
|
||||
close: () => connection.close(),
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const running = coordinator.run();
|
||||
await flushUntil(() => sockets.length === 1);
|
||||
sockets[0]!.accept();
|
||||
await flush();
|
||||
// No events, only acknowledged heartbeats, for longer than maxElapsedMs.
|
||||
for (let beat = 0; beat < 2; beat += 1) {
|
||||
clock.advance(5_000);
|
||||
await flush();
|
||||
sockets[0]!.ackHeartbeats();
|
||||
clock.advance(1_000);
|
||||
await flush();
|
||||
}
|
||||
sockets[0]!.nativeClose(1_012);
|
||||
await flushUntil(() => clock.sleepCalls.includes(50));
|
||||
clock.advance(50);
|
||||
await flushUntil(() => sockets.length === 2);
|
||||
expect(sockets).toHaveLength(2);
|
||||
|
||||
coordinator.close();
|
||||
await expect(running).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "CLOSED" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([502, 504])(
|
||||
"reconnects SSE HTTP %i that carries no server hint",
|
||||
async (status) => {
|
||||
|
||||
Reference in New Issue
Block a user