Files
clean-architecture-frontend…/tests/unit/realtime/realtime-reconnect-coordinator.test.ts
T
DongHyeonka 0438c3f248 fix(realtime): 던지는 reconnect attempt·waitClosed의 규칙을 적는다
문제: reconnect coordinator는 `attempt`·`waitClosed`가 던지면
PROVIDER_UNAVAILABLE(CONNECT·RECEIVE, retryable)로 보고 backoff로 다시
시도한다. 05330e3 뒤 poll은 같은 던짐을 PROTOCOL_MISMATCH로 끝낸다. 이
차이는 그 커밋 본문에만 있고 타입 주석과 설계 13.1에는 없다. repo의 SSE
bridge 예(`sseAttempt`)는 `attempt` 안에서 연결을 만든다. 그래서
endpoint 설정 오류가 요청 0번, attempt 3번, PROVIDER_UNAVAILABLE
(EXHAUSTED)로 끝나 서버 장애로 보인다(리뷰 실험 04-R2).

고른 방법: 리뷰 선택지 (c). 동작은 그대로 두고 규칙을 적는다.
- `attempt` 주석: 던짐은 transport 실패로 재시도된다. 설정 검사는
  `attempt` 밖, 조합 코드가 transport를 만들 때 한다. poll과 다른 이유도
  적었다. `waitClosed` 주석: RECEIVE로 같은 규칙.
  `classifyClosed`·`confirmTransportBarrier` 주석: 던짐은
  PROTOCOL_MISMATCH로 끝나고 재시도하지 않는다.
- 설계 13.1 PROVIDER_UNAVAILABLE·PROTOCOL_MISMATCH 행에 같은 규칙을 적었다.
- 테스트 bridge `sseAttempt`는 만들어질 때 연결 설정을 한 번 검사한다.
  연결은 attempt마다 새로 만든다. session `close`가 연결을 끝내므로 한
  연결을 여러 attempt에 쓸 수 없다.
- 규칙을 고정하는 테스트를 더했다. attempt·waitClosed 던짐은 3회 재시도 뒤
  PROVIDER_UNAVAILABLE(EXHAUSTED), classifyClosed 던짐은 1회로
  PROTOCOL_MISMATCH, bridge의 http endpoint는 만들 때 TypeError(fetch 0).
버린 대안:
- (b) reconnect도 던짐을 PROTOCOL_MISMATCH로 끝내기. 연결 실패를 던지는
  라이브러리를 감싼 transport가 첫 실패에서 run을 끝내게 되는 공개 동작
  변경(BREAKING)이다. 리뷰도 지금 동작이 회복에 유리하다고 봤다.
  05330e3이 poll에서 (c)를 버린 이유("결과로 구분한다")는 이번 완료
  기준에 없다.
- 연결 하나를 bridge에서 만들고 attempt마다 read만 하기. 위와 같이
  session `close`가 연결을 닫아 다음 read가 CLOSED가 된다.

위험·한계:
- 동작이 같으므로 조합 코드가 `attempt` 안에서 연결을 만들면 설정 오류는
  여전히 서버 장애로 보인다. 주석과 13.1은 모르고 쓰는 경우만 줄인다.
- 매핑 오류 같은 나머지 던짐도 그대로 재시도되고 원인 예외는 남지 않는다
  (닫힌 관측 어휘).
- CHANGELOG 행은 이 커밋에 없다. CHANGELOG 담당이 더한다.

검증: desktop batch에서 확인.
- node scripts/run-vitest.ts run tests/unit/realtime/realtime-reconnect-coordinator.test.ts
- RED 없음. 동작을 바꾸지 않는 커밋이라 새 테스트는 부모 커밋에서도
  통과한다. 규칙을 고정하는 테스트다.
- check:types, lint, verify:documentation

Refs: 1106/04-01
2026-11-06 05:27:35 +09:00

2554 lines
75 KiB
TypeScript

import { describe, expect, it, vi } from "vitest";
import type { ClockPort } from "../../../src/application/ports/clock-port.ts";
import type {
RealtimeResult,
} from "../../../src/application/ports/realtime/shared.ts";
import {
REALTIME_TRANSPORT_CONTINUE,
type RealtimeRecoveryCheckpoint,
} from "../../../src/application/ports/realtime/event-authority.ts";
import {
createRealtimeReconnectCoordinator,
type RealtimeCommittedRecovery,
type RealtimeReconnectAttemptContext,
type RealtimeReconnectAttemptSuccess,
type RealtimeReconnectEnvironment,
type RealtimeReconnectObservation,
type RealtimeReconnectOutcome,
type RealtimeReconnectSession,
type RealtimeRecoveryReconnectDirective,
realtimeFailure as publicRealtimeFailure,
realtimeSuccess as publicRealtimeSuccess,
} from "../../../src/adapters/realtime/index.ts";
import {
defineReconnectPolicy,
REALTIME_RECONNECT_CEILINGS,
type ReconnectPolicy,
} from "../../../src/adapters/realtime/reconnect-policy.ts";
import {
realtimeFailure,
realtimeSuccess,
} from "../../../src/adapters/realtime/result.ts";
import {
createFetchSseConnection,
readSseForReconnect,
SSE_CONTINUE,
type FetchSseClosedOutcome,
} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts";
import {
createWebSocketConnection,
type WebSocketClosedReceipt,
type WebSocketFacade,
} from "../../../src/adapters/realtime/websocket/websocket-connection.ts";
import {
REALTIME_WEBSOCKET_PROTOCOL,
} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts";
type Sleeper = Readonly<{
dueAt: number;
signal?: AbortSignal;
resolve(): void;
reject(): void;
}>;
class ManualClock implements ClockPort {
current = 0;
readonly sleepCalls: number[] = [];
readonly sleepers: Sleeper[] = [];
now(): number {
return this.current;
}
sleep(
milliseconds: number,
signal?: AbortSignal,
): Promise<void> {
this.sleepCalls.push(milliseconds);
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new Error("Aborted."));
return;
}
let onAbort: (() => void) | undefined;
const sleeper: Sleeper = {
dueAt: this.current + milliseconds,
signal,
resolve: () => {
signal?.removeEventListener("abort", onAbort!);
resolve();
},
reject: () => {
signal?.removeEventListener("abort", onAbort!);
reject(new Error("Aborted."));
},
};
onAbort = () => {
this.remove(sleeper);
sleeper.reject();
};
signal?.addEventListener("abort", onAbort, { once: true });
this.sleepers.push(sleeper);
});
}
advance(milliseconds: number): void {
this.current += milliseconds;
const ready = this.sleepers
.filter((sleeper) => sleeper.dueAt <= this.current)
.sort((left, right) => left.dueAt - right.dueAt);
for (const sleeper of ready) {
this.remove(sleeper);
sleeper.resolve();
}
}
private remove(target: Sleeper): void {
const index = this.sleepers.indexOf(target);
if (index >= 0) this.sleepers.splice(index, 1);
}
}
class MutableOnlineEnvironment
implements RealtimeReconnectEnvironment
{
readonly listeners = new Set<(online: boolean) => void>();
constructor(public connected: boolean) {}
online(): boolean {
return this.connected;
}
subscribeOnline(
listener: (online: boolean) => void,
): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
emit(online: boolean): void {
this.connected = online;
for (const listener of this.listeners) listener(online);
}
}
function policy(
override: Partial<ReconnectPolicy> = {},
): ReconnectPolicy {
return defineReconnectPolicy({
baseDelayMs: 100,
maxDelayMs: 1_000,
maxAttempts: 2,
maxElapsedMs: 5_000,
stableOpenMs: 500,
...override,
});
}
function failed(
result: Extract<RealtimeResult<never>, { ok: false }>,
serverNotBeforeMs?: number,
): RealtimeReconnectOutcome<never> {
return Object.freeze({
result,
...(serverNotBeforeMs === undefined
? {}
: { serverNotBeforeMs }),
});
}
function opened<ClosedReceipt>(
session: RealtimeReconnectSession<ClosedReceipt>,
establishedRecoveryBarrier?: RealtimeCommittedRecovery,
): RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<ClosedReceipt>
> {
return Object.freeze({
result: realtimeSuccess(
Object.freeze({
session,
...(establishedRecoveryBarrier === undefined
? {}
: { establishedRecoveryBarrier }),
}),
),
});
}
function closedSuccessfully(): RealtimeReconnectOutcome<void> {
return Object.freeze({
result: realtimeSuccess(undefined),
});
}
function confirmationSucceeded(): RealtimeResult<void> {
return realtimeSuccess(undefined);
}
function recoveryToken(
sequence = "7",
): RealtimeCommittedRecovery {
const checkpoint = Object.freeze({
lastAppliedSequence: sequence,
recoveryMode: "CURSOR",
resumeCursor: `cursor-${sequence}`,
streamEpoch: "epoch-1",
}) as unknown as RealtimeRecoveryCheckpoint;
return Object.freeze({
kind: "RECOVERY_COMMITTED",
streamId: "orders",
checkpoint,
}) as unknown as RealtimeCommittedRecovery;
}
function recoveryDirective(
recovery: RealtimeCommittedRecovery,
terminalResult: Extract<
RealtimeResult<never>,
{ ok: false }
>,
serverNotBeforeMs?: number,
): RealtimeRecoveryReconnectDirective {
return Object.freeze({
kind: "RECOVERY_RECONNECT",
recovery,
terminalResult,
...(serverNotBeforeMs === undefined
? {}
: { serverNotBeforeMs }),
});
}
function deferred<Value>(): Readonly<{
promise: Promise<Value>;
resolve(value: Value): void;
}> {
let resolvePromise: ((value: Value) => void) | undefined;
const promise = new Promise<Value>((resolve) => {
resolvePromise = resolve;
});
return Object.freeze({
promise,
resolve(value) {
resolvePromise?.(value);
},
});
}
async function flush(): Promise<void> {
// Enough turns for a retry loop iteration: sleep, attempt and close races.
for (let turn = 0; turn < 16; turn += 1) {
await Promise.resolve();
}
}
async function flushUntil(
condition: () => boolean,
): Promise<void> {
for (let turn = 0; turn < 50; turn += 1) {
if (condition()) return;
await Promise.resolve();
}
}
describe("realtime reconnect coordinator", () => {
it("owns one bounded retry loop and returns the exact last failure", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const failures = [
realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true),
realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true),
realtimeFailure("IDLE_TIMEOUT", "RECEIVE", true),
] as const;
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
const next = failures[attempts++]!;
return failed(
next,
next === failures[1] ? 0 : undefined,
);
},
});
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
await flush();
expect(attempts).toBe(2);
expect(clock.sleepCalls).toEqual([50, 100]);
clock.advance(100);
const result = await running;
expect(attempts).toBe(3);
expect(result).toBe(failures[2]);
expect(coordinator.getState()).toBe("IDLE");
});
it("does not clamp a server not-before value past elapsed budget", async () => {
const clock = new ManualClock();
const terminal = realtimeFailure(
"RATE_LIMITED",
"CONNECT",
true,
);
const random = vi.fn(() => 0);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({
maxAttempts: 1,
maxElapsedMs: 500,
}),
environment: new MutableOnlineEnvironment(true),
clock,
random,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => failed(terminal, 500),
});
const result = await coordinator.run();
expect(result).toBe(terminal);
expect(clock.sleepCalls).toEqual([]);
expect(random).not.toHaveBeenCalled();
});
it.each([
"RATE_LIMITED",
"PROVIDER_UNAVAILABLE",
] as const)(
"reconnects a retryable external %s attempt failure without a server hint",
async (kind) => {
const clock = new ManualClock();
const retryable = realtimeFailure(kind, "CONNECT", true);
const terminal = realtimeFailure(kind, "CONNECT", false);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
failed(++attempts === 1 ? retryable : terminal),
});
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
},
);
it("reconnects a retryable external close failure without a server hint", async () => {
const clock = new ManualClock();
const retryable = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
true,
);
const terminal = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
false,
);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
const closedWith = ++attempts === 1 ? retryable : terminal;
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(closedWith),
}),
);
},
});
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
});
it("waits for an explicit online signal without a retry timer", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(false);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const attempt = vi.fn(() => failed(terminal));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const running = coordinator.run();
await flush();
clock.advance(100);
await flush();
expect(attempt).not.toHaveBeenCalled();
expect(clock.sleepCalls).toEqual([]);
environment.emit(true);
await flush();
expect(clock.sleepCalls).toEqual([50]);
expect(attempt).not.toHaveBeenCalled();
clock.advance(50);
const result = await running;
expect(attempt).toHaveBeenCalledTimes(1);
expect(result).toBe(terminal);
});
it("does not spend the elapsed budget while waiting offline", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(false);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const attempt = vi.fn(() => failed(terminal));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const running = coordinator.run();
await flush();
// Offline for longer than maxElapsedMs before the first attempt.
clock.advance(6_000);
environment.emit(true);
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempt).toHaveBeenCalledTimes(1);
expect(result).toBe(terminal);
});
it.each(["an offline blip", "a waitClosed throw"] as const)(
"reconnects after %s ends a healthy session older than the elapsed budget",
async (ending) => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const closing = deferred<RealtimeReconnectOutcome<void>>();
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context: RealtimeReconnectAttemptContext) => {
attempts += 1;
if (attempts > 1) return failed(terminal);
context.markValidHeartbeatOrEvent();
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: () =>
closing.resolve(
failed(
realtimeFailure("OFFLINE", "RECEIVE", true),
),
),
waitClosed: () =>
ending === "an offline blip"
? closing.promise
: closing.promise.then(() => {
throw new Error("transport failed");
}),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => attempts === 1);
await flush();
// Healthy for twice maxElapsedMs, then a one second outage.
clock.advance(10_000);
if (ending === "an offline blip") {
environment.emit(false);
await flush();
clock.advance(1_000);
environment.emit(true);
} else {
closing.resolve(closedSuccessfully());
}
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
},
);
it("observes attempts, retries, the session and why the run ended", async () => {
const clock = new ManualClock();
const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false);
const observations: unknown[] = [];
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
observe: (observation) => {
observations.push(observation);
throw new Error("sink failure must not change the run");
},
attempt: () =>
++attempts === 1
? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true))
: opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(terminal),
}),
),
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(50);
await expect(running).resolves.toBe(terminal);
expect(observations).toEqual([
{ kind: "ATTEMPT_STARTED" },
{
kind: "RETRY_SCHEDULED",
failureKind: "CONNECT_TIMEOUT",
delayBucket: "UNDER_1S",
},
{ kind: "ATTEMPT_STARTED" },
{ kind: "SESSION_OPENED" },
{ kind: "SESSION_CLOSED" },
{ kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" },
]);
// A second run that spends its retries ends as EXHAUSTED, not TERMINAL.
const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true);
const ends: unknown[] = [];
const spending = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
observe: (observation) => {
if (observation.kind === "RUN_ENDED") ends.push(observation);
},
attempt: () => failed(exhausted),
});
const spent = spending.run();
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
await expect(spent).resolves.toBe(exhausted);
expect(ends).toEqual([
{
kind: "RUN_ENDED",
reason: "EXHAUSTED",
failureKind: "PROVIDER_UNAVAILABLE",
},
]);
});
it("resets a spent retry only after a valid signal and a stable open", async () => {
const runScenario = async (markValid: boolean, openMs: number) => {
const clock = new ManualClock();
const firstFailure = realtimeFailure(
"CONNECT_TIMEOUT",
"CONNECT",
true,
);
const sessionFailure = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
true,
);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const closeSession = vi.fn();
const closing = deferred<RealtimeReconnectOutcome<void>>();
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: (
context: RealtimeReconnectAttemptContext,
) => {
attempts += 1;
if (attempts === 1) return failed(firstFailure);
if (attempts === 2) {
if (markValid) {
context.markValidHeartbeatOrEvent();
}
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed: () => closing.promise,
}),
);
}
return failed(terminal);
},
});
const running = coordinator.run();
await flush();
clock.advance(0);
await flushUntil(() => attempts === 2);
await flush();
clock.advance(openMs);
closing.resolve(failed(sessionFailure, 0));
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
const result = await running;
return {
attempts,
closeSession,
result,
sessionFailure,
terminal,
};
};
// A heartbeat on a short-lived connection, or a quiet long one, keeps
// the spent budget; only both together reset it.
for (const [markValid, openMs] of [
[false, 0],
[true, 499],
[false, 500],
] as const) {
const spent = await runScenario(markValid, openMs);
expect(spent.attempts).toBe(2);
expect(spent.result).toBe(spent.sessionFailure);
expect(spent.closeSession).toHaveBeenCalledOnce();
}
const eligible = await runScenario(true, 500);
expect(eligible.attempts).toBe(3);
expect(eligible.result).toBe(eligible.terminal);
expect(eligible.closeSession).toHaveBeenCalledOnce();
});
it("does not reconnect before the session's authoritative close result", async () => {
const clock = new ManualClock();
const closure =
deferred<RealtimeReconnectOutcome<void>>();
const terminal = realtimeFailure(
"FORBIDDEN",
"RECEIVE",
false,
);
const closeSession = vi.fn();
const attempt = vi.fn(() =>
opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed: () => closure.promise,
}),
),
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
let settled = false;
const running = coordinator.run().then((result) => {
settled = true;
return result;
});
await flush();
expect(attempt).toHaveBeenCalledOnce();
expect(clock.sleepCalls).toEqual([]);
expect(settled).toBe(false);
expect(coordinator.getState()).toBe("RUNNING");
closure.resolve(failed(terminal));
const result = await running;
expect(result).toBe(terminal);
expect(closeSession).toHaveBeenCalledOnce();
});
it("carries an exact recovery token into the next attempt and opens only after confirmation", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const directive = recoveryDirective(recovery, terminal);
const completed = closedSuccessfully();
const confirmation = realtimeSuccess(undefined);
const confirmTransportBarrier = vi.fn(
(received: RealtimeCommittedRecovery) => {
expect(received).toBe(recovery);
return confirmation;
},
);
let attempts = 0;
let gateSettled = false;
let gateResult: RealtimeResult<void> | undefined;
type RecoveryCloseReceipt =
| RealtimeRecoveryReconnectDirective
| RealtimeReconnectOutcome<void>;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier,
attempt: (context) => {
attempts += 1;
if (attempts === 1) {
expect(context.pendingRecovery).toBeNull();
return opened<RecoveryCloseReceipt>(
Object.freeze({
classifyClosed: (
receipt: RecoveryCloseReceipt,
) => receipt,
close: vi.fn(),
waitClosed: () => directive,
}),
);
}
expect(context.pendingRecovery).toBe(recovery);
const gate =
context.waitForRecoveryBarrierConfirmation();
void gate.then((result) => {
gateSettled = true;
gateResult = result;
});
expect(gateSettled).toBe(false);
return opened<RecoveryCloseReceipt>(
Object.freeze({
classifyClosed: (
receipt: RecoveryCloseReceipt,
) => receipt,
close: vi.fn(),
waitClosed: () => completed,
}),
recovery,
);
},
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([0]);
clock.advance(0);
const result = await running;
await flush();
expect(attempts).toBe(2);
expect(confirmTransportBarrier).toHaveBeenCalledOnce();
expect(gateSettled).toBe(true);
expect(gateResult).toBe(confirmation);
expect(result).toBe(completed.result);
});
it.each([
["missing", undefined],
["cloned", recoveryToken()],
] as const)(
"rejects a %s recovery proof and closes the new session",
async (_case, proof) => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const closeSession = vi.fn();
const waitClosed = vi.fn(closedSuccessfully);
const confirmTransportBarrier = vi.fn(
confirmationSucceeded,
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier,
attempt: (context) => {
expect(context.pendingRecovery).toBe(recovery);
const session = Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed,
});
return proof === undefined
? opened(session)
: opened(session, proof);
},
});
const running = coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"RECOVER",
false,
),
);
expect(closeSession).toHaveBeenCalledOnce();
expect(waitClosed).not.toHaveBeenCalled();
expect(confirmTransportBarrier).not.toHaveBeenCalled();
},
);
it("settles the readiness gate and run with the exact confirmation failure", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const confirmationFailure = realtimeFailure(
"SCOPE_PROTOCOL_VIOLATION",
"RECOVER",
false,
);
const closeSession = vi.fn();
const waitClosed = vi.fn(closedSuccessfully);
let gateResult: RealtimeResult<void> | undefined;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: () => confirmationFailure,
attempt: (context) => {
void context
.waitForRecoveryBarrierConfirmation()
.then((result) => {
gateResult = result;
});
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed,
}),
recovery,
);
},
});
const running = coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
const result = await running;
await flush();
expect(result).toBe(confirmationFailure);
expect(gateResult).toBe(confirmationFailure);
expect(closeSession).toHaveBeenCalledOnce();
expect(waitClosed).not.toHaveBeenCalled();
});
it("retains the exact recovery token across retryable attempt failures and returns its terminal result on exhaustion", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const attemptFailure = realtimeFailure(
"CONNECT_TIMEOUT",
"CONNECT",
true,
);
const gateResults: RealtimeResult<void>[] = [];
const seenRecoveries: Array<
RealtimeCommittedRecovery | null
> = [];
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 2 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context) => {
seenRecoveries.push(context.pendingRecovery);
void context
.waitForRecoveryBarrierConfirmation()
.then((result) => gateResults.push(result));
return failed(attemptFailure);
},
});
const running = coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
const result = await running;
await flush();
expect(seenRecoveries).toEqual([recovery, recovery]);
expect(gateResults).toEqual([
attemptFailure,
attemptFailure,
]);
expect(gateResults[0]).toBe(attemptFailure);
expect(gateResults[1]).toBe(attemptFailure);
expect(result).toBe(terminal);
});
it("fails a no-pending gate immediately and rejects any unsolicited proof", async () => {
const proof = recoveryToken();
const closeSession = vi.fn();
const confirmTransportBarrier = vi.fn(
confirmationSucceeded,
);
let gateResult: RealtimeResult<void> | undefined;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier,
attempt: async (context) => {
gateResult =
await context.waitForRecoveryBarrierConfirmation();
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed: closedSuccessfully,
}),
proof,
);
},
});
const result = await coordinator.run();
expect(gateResult).toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"RECOVER",
false,
),
);
expect(result).toBe(gateResult);
expect(closeSession).toHaveBeenCalledOnce();
expect(confirmTransportBarrier).not.toHaveBeenCalled();
});
it("applies retry ceilings to an initial recovery and preserves its exact terminal result", async () => {
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const attempt = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({
maxAttempts: 1,
maxElapsedMs: 500,
}),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const result = await coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
500,
),
});
expect(result).toBe(terminal);
expect(attempt).not.toHaveBeenCalled();
});
it("settles a pending readiness gate when the run is aborted", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
let gateResult: RealtimeResult<void> | undefined;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context) => {
void context
.waitForRecoveryBarrierConfirmation()
.then((result) => {
gateResult = result;
});
return attemptResult.promise;
},
});
const controller = new AbortController();
const running = coordinator.run({
signal: controller.signal,
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
await flush();
controller.abort();
const result = await running;
await flush();
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
expect(gateResult).toEqual(
realtimeFailure("ABORTED", "RECOVER", false),
);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: closedSuccessfully,
}),
recovery,
),
);
await flush();
});
it("rejects accessors without invoking them and consumes proxy values only from one descriptor snapshot", async () => {
let getterReads = 0;
const accessorOutcome = Object.defineProperty({}, "result", {
enumerable: true,
get() {
getterReads += 1;
return realtimeSuccess(undefined);
},
});
const accessorCoordinator =
createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
accessorOutcome as unknown as RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<unknown>
>,
});
await expect(accessorCoordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
expect(getterReads).toBe(0);
const completed = closedSuccessfully();
const session = Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => completed,
});
const rawOutcome = opened(session);
const proxyReads: PropertyKey[] = [];
const proxyOutcome = new Proxy(rawOutcome, {
get(_target, key) {
proxyReads.push(key);
if (key === "then") return undefined;
throw new Error("raw property reread");
},
});
const proxyCoordinator =
createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () => proxyOutcome,
});
await expect(proxyCoordinator.run()).resolves.toBe(
completed.result,
);
expect(proxyReads).toEqual(["then"]);
});
it("rechecks scope after attempt and waitClosed boundaries before accepting callbacks", async () => {
let attemptScopeCurrent = true;
const lateAttemptClose = vi.fn();
const lateAttemptWaitClosed = vi.fn(closedSuccessfully);
const lateAttemptCoordinator =
createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
isCurrent: () => attemptScopeCurrent,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attemptScopeCurrent = false;
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateAttemptClose,
waitClosed: lateAttemptWaitClosed,
}),
);
},
});
await expect(lateAttemptCoordinator.run()).resolves.toEqual(
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
);
expect(lateAttemptClose).toHaveBeenCalledOnce();
expect(lateAttemptWaitClosed).not.toHaveBeenCalled();
let closeScopeCurrent = true;
const classifyClosed = vi.fn(
(receipt: RealtimeReconnectOutcome<void>) => receipt,
);
const closeSession = vi.fn();
const closeCoordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
isCurrent: () => closeScopeCurrent,
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
opened(
Object.freeze({
classifyClosed,
close: closeSession,
waitClosed: () => {
closeScopeCurrent = false;
return closedSuccessfully();
},
}),
),
});
await expect(closeCoordinator.run()).resolves.toEqual(
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
);
expect(classifyClosed).not.toHaveBeenCalled();
expect(closeSession).toHaveBeenCalledOnce();
});
it("rejects extra and missing outcome keys through the common result codec", async () => {
const validSession = Object.freeze({
classifyClosed: closedSuccessfully,
close: vi.fn(),
waitClosed: closedSuccessfully,
});
const invalidOutcomes = [
{
extra: true,
result: realtimeSuccess(validSession),
},
{
result: { ok: true },
},
{
result: {
ok: false,
error: {
extra: true,
kind: "FORBIDDEN",
operation: "CONNECT",
retryable: false,
},
},
},
];
for (const invalid of invalidOutcomes) {
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
invalid as unknown as RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<unknown>
>,
});
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
}
});
it("rechecks abort immediately after listener registration", async () => {
let abortReads = 0;
let listener: (() => void) | undefined;
const signal = {
get aborted() {
abortReads += 1;
return abortReads >= 2;
},
addEventListener(
type: string,
nextListener: () => void,
) {
if (type === "abort") listener = nextListener;
},
removeEventListener() {
listener = undefined;
},
} as unknown as AbortSignal;
const attempt = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const result = await coordinator.run({ signal });
expect(listener).toBeUndefined();
expect(attempt).not.toHaveBeenCalled();
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
});
it("bounds a non-cooperative retry sleep after its phase is aborted", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const sleep = vi.fn(
() => new Promise<void>(() => undefined),
);
const clock: ClockPort = Object.freeze({
now: () => 0,
sleep,
});
const retryable = realtimeFailure(
"CONNECT_TIMEOUT",
"CONNECT",
true,
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => failed(retryable),
});
try {
const running = coordinator.run();
await flushUntil(() => sleep.mock.calls.length === 1);
environment.emit(false);
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROVIDER_UNAVAILABLE",
"CONNECT",
false,
),
);
expect(coordinator.getState()).toBe("DRAINING");
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("bounds a non-cooperative attempt and closes its late successful session before leaving DRAINING", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
const lateClose = vi.fn();
const lateWaitClosed = vi.fn(closedSuccessfully);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () => attemptResult.promise,
});
try {
const running = coordinator.run();
await flush();
environment.emit(false);
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROVIDER_UNAVAILABLE",
"CONNECT",
false,
),
);
expect(coordinator.getState()).toBe("DRAINING");
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateClose,
waitClosed: lateWaitClosed,
}),
),
);
await flush();
expect(lateClose).toHaveBeenCalledOnce();
expect(lateWaitClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("IDLE");
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("cancels the drain timer on abort and keeps the late generation fenced", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
const lateClose = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () => attemptResult.promise,
});
const controller = new AbortController();
try {
const running = coordinator.run({
signal: controller.signal,
});
await flush();
environment.emit(false);
await flush();
expect(vi.getTimerCount()).toBe(1);
controller.abort();
const result = await running;
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
expect(coordinator.getState()).toBe("DRAINING");
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateClose,
waitClosed: closedSuccessfully,
}),
),
);
await flush();
expect(lateClose).toHaveBeenCalledOnce();
expect(coordinator.getState()).toBe("IDLE");
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("keeps a healthy waitClosed unbounded but bounds its post-abort drain", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const closure =
deferred<RealtimeReconnectOutcome<void>>();
const classifyClosed = vi.fn(
(receipt: RealtimeReconnectOutcome<void>) => receipt,
);
const closeSession = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
opened(
Object.freeze({
classifyClosed,
close: closeSession,
waitClosed: () => closure.promise,
}),
),
});
try {
let settled = false;
const running = coordinator.run().then((result) => {
settled = true;
return result;
});
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs * 2,
);
expect(settled).toBe(false);
expect(coordinator.getState()).toBe("RUNNING");
expect(vi.getTimerCount()).toBe(0);
environment.emit(false);
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROVIDER_UNAVAILABLE",
"CONNECT",
false,
),
);
expect(closeSession).toHaveBeenCalled();
expect(classifyClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("DRAINING");
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
closure.resolve(closedSuccessfully());
await flush();
expect(classifyClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("IDLE");
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("uses waitClosed as authority and fences an aborted late generation", async () => {
const clock = new ManualClock();
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
let captured:
| RealtimeReconnectAttemptContext
| undefined;
const lateClose = vi.fn();
const lateWaitClosed = vi.fn(closedSuccessfully);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context) => {
captured = context;
return attemptResult.promise;
},
});
const controller = new AbortController();
const running = coordinator.run({ signal: controller.signal });
await flush();
expect(captured?.isCurrent()).toBe(true);
controller.abort();
const result = await running;
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
expect(coordinator.getState()).toBe("DRAINING");
expect(captured?.isCurrent()).toBe(false);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateClose,
waitClosed: lateWaitClosed,
}),
),
);
await flush();
expect(lateClose).toHaveBeenCalledOnce();
expect(lateWaitClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("IDLE");
});
});
class FakeWebSocket implements WebSocketFacade {
protocol: string = REALTIME_WEBSOCKET_PROTOCOL;
readyState = 0;
bufferedAmount = 0;
readonly listeners = new Map<
string,
Set<(event: unknown) => void>
>();
readonly sent: string[] = [];
send(data: string): void {
if (this.readyState !== 1) throw new Error("not open");
this.sent.push(data);
}
close(): void {
this.readyState = 3;
}
addEventListener(
type: string,
listener: (event: unknown) => void,
): void {
const listeners = this.listeners.get(type) ?? new Set();
listeners.add(listener);
this.listeners.set(type, listeners);
}
removeEventListener(
type: string,
listener: (event: unknown) => void,
): void {
this.listeners.get(type)?.delete(listener);
}
accept(): void {
this.readyState = 1;
this.emit("open", {});
this.emit("message", {
data: JSON.stringify({
type: "WELCOME",
protocol: REALTIME_WEBSOCKET_PROTOCOL,
connectionId: "connection.0001",
heartbeatMs: 5_000,
heartbeatAckTimeoutMs: 1_000,
limits: {
maxFrameBytes: 64 * 1_024,
maxSubscriptions: 32,
maxInboundQueueCount: 256,
maxInboundQueueBytes: 4 * 1_024 * 1_024,
maxOutboundQueueCount: 128,
maxOutboundQueueBytes: 256 * 1_024,
maxBufferedAmountBytes: 256 * 1_024,
maxEventsPerSecond: 256,
},
}),
});
}
nativeClose(code: number): void {
this.readyState = 3;
this.emit("close", { code, reason: "" });
}
ackHeartbeats(): void {
for (const raw of this.sent.splice(0)) {
const frame = JSON.parse(raw) as { type: string; nonce: string };
if (frame.type !== "HEARTBEAT") continue;
this.emit("message", {
data: JSON.stringify({
type: "HEARTBEAT_ACK",
protocol: REALTIME_WEBSOCKET_PROTOCOL,
nonce: frame.nonce,
}),
});
}
}
private emit(type: string, event: unknown): void {
for (const listener of [...(this.listeners.get(type) ?? [])]) {
listener(event);
}
}
}
/**
* The smallest SSE bridge design 6.3 leaves to composition: the attempt ends
* at `onOpen` and the rest of the read is the session.
*
* 1106/04-01. The coordinator retries a throwing attempt as
* PROVIDER_UNAVAILABLE, so the bridge checks its connection settings here,
* when it is built: a wrong endpoint throws before any run. Each attempt
* still makes its own connection, because the session's `close` ends that
* connection for good.
*/
function sseAttempt(
fetcher: typeof fetch,
clock: ManualClock,
endpoint = "https://app.example.test/events",
) {
const settings = Object.freeze({
endpoint,
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR" as const,
fetcher,
clock,
});
// Creating a connection checks the settings and sends nothing.
createFetchSseConnection(settings).close();
return async (context: RealtimeReconnectAttemptContext) => {
const connection = createFetchSseConnection(settings);
const open = deferred<"OPEN">();
const reading = readSseForReconnect(connection, {
resumeCursor: null,
signal: context.signal,
onOpen: () => {
open.resolve("OPEN");
return realtimeSuccess(undefined);
},
onEvent: () => realtimeSuccess(SSE_CONTINUE),
onComment: () => context.markValidHeartbeatOrEvent(),
});
const first = await Promise.race([open.promise, reading]);
if (first !== "OPEN") {
return first.result.ok
? failed(realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false))
: (first as RealtimeReconnectOutcome<never>);
}
return opened(
Object.freeze({
waitClosed: () => reading,
classifyClosed: (
receipt: RealtimeReconnectOutcome<FetchSseClosedOutcome>,
) =>
receipt.result.ok
? failed(
realtimeFailure("PROTOCOL_MISMATCH", "RECEIVE", false),
)
: (receipt as RealtimeReconnectOutcome<never>),
close: () => connection.close(),
}),
);
};
}
async function advanceUntil(
clock: ManualClock,
settled: () => boolean,
): Promise<void> {
// Ten simulated seconds outlast policy()'s whole elapsed budget.
for (let elapsed = 0; elapsed < 10_000 && !settled(); elapsed += 50) {
clock.advance(50);
await flush();
}
}
function observationLabel({
kind,
failureKind,
reason,
}: RealtimeReconnectObservation): string {
return [kind, failureKind, reason].filter(Boolean).join(":");
}
describe("transport failures through the reconnect coordinator", () => {
it.each([
["NETWORK_LOST", 1_006],
["RESTART", 1_012],
] as const)(
"reconnects a WebSocket %s close that carries no server hint",
async (_category, code) => {
const clock = new ManualClock();
const sockets: FakeWebSocket[] = [];
const coordinator = createRealtimeReconnectCoordinator<
WebSocketClosedReceipt
>({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const connection = createWebSocketConnection({
origin: "https://app.example.test",
endpointPath: "/_realtime/v1",
clock,
createNonce: () => "nonce.0001",
createSocket: () => {
const socket = new FakeWebSocket();
sockets.push(socket);
return socket;
},
onEvent: async () =>
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
onRecoveryRequired: () => undefined,
});
const connected = await connection.connect(context.signal);
if (!connected.ok) return failed(connected);
return opened(
Object.freeze({
waitClosed: () => connection.waitClosed(),
classifyClosed: (receipt: WebSocketClosedReceipt) =>
Object.freeze({
result: Object.freeze({
ok: false as const,
error: receipt.error,
}),
}),
close: () => connection.close(),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => sockets.length === 1);
sockets[0]!.accept();
await flush();
sockets[0]!.nativeClose(code);
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await flushUntil(() => sockets.length === 2);
expect(sockets).toHaveLength(2);
// OVERLOADED carries no hint either and stays terminal.
sockets[1]!.accept();
await flush();
sockets[1]!.nativeClose(1_013);
await expect(running).resolves.toEqual({
ok: false,
error: {
kind: "PROVIDER_UNAVAILABLE",
operation: "RECEIVE",
retryable: false,
},
});
expect(sockets).toHaveLength(2);
},
);
it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => {
const clock = new ManualClock();
const sockets: FakeWebSocket[] = [];
let nonce = 0;
const coordinator = createRealtimeReconnectCoordinator<
WebSocketClosedReceipt
>({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const connection = createWebSocketConnection({
origin: "https://app.example.test",
endpointPath: "/_realtime/v1",
clock,
createNonce: () => `nonce.${++nonce}`,
createSocket: () => {
const socket = new FakeWebSocket();
sockets.push(socket);
return socket;
},
onEvent: async () =>
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
onRecoveryRequired: () => undefined,
observe: (observation) => {
if (observation.kind === "HEARTBEAT_ACK") {
context.markValidHeartbeatOrEvent();
}
},
});
const connected = await connection.connect(context.signal);
if (!connected.ok) return failed(connected);
return opened(
Object.freeze({
waitClosed: () => connection.waitClosed(),
classifyClosed: (receipt: WebSocketClosedReceipt) =>
Object.freeze({
result: Object.freeze({
ok: false as const,
error: receipt.error,
}),
}),
close: () => connection.close(),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => sockets.length === 1);
sockets[0]!.accept();
await flush();
// No events, only acknowledged heartbeats, for longer than maxElapsedMs.
for (let beat = 0; beat < 2; beat += 1) {
clock.advance(5_000);
await flush();
sockets[0]!.ackHeartbeats();
clock.advance(1_000);
await flush();
}
sockets[0]!.nativeClose(1_012);
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await flushUntil(() => sockets.length === 2);
expect(sockets).toHaveLength(2);
coordinator.close();
await expect(running).resolves.toMatchObject({
ok: false,
error: { kind: "CLOSED" },
});
});
it.each([502, 504])(
"reconnects SSE HTTP %i that carries no server hint",
async (status) => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status }))
.mockResolvedValueOnce(new Response(null, { status: 403 }));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const read = await createFetchSseConnection({
endpoint: "https://app.example.test/events",
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR",
fetcher,
clock,
}).read({
resumeCursor: null,
signal: context.signal,
onEvent: () => realtimeSuccess(SSE_CONTINUE),
});
// Only connect failures are exercised here.
return failed(
read.ok
? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false)
: read,
);
},
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await expect(running).resolves.toEqual({
ok: false,
error: {
kind: "FORBIDDEN",
operation: "CONNECT",
retryable: false,
},
});
expect(fetcher).toHaveBeenCalledTimes(2);
},
);
it.each([429, 503])(
"waits at least the SSE %i Retry-After through the reconnect bridge",
async (status) => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(
new Response(null, {
status,
headers: { "retry-after": "3" },
}),
)
.mockResolvedValueOnce(
new Response(null, {
status,
headers: { "retry-after": "10" },
}),
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxDelayMs: 5_000, maxElapsedMs: 60_000 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const read = await readSseForReconnect(
createFetchSseConnection({
endpoint: "https://app.example.test/events",
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR",
fetcher,
clock,
}),
{
resumeCursor: null,
signal: context.signal,
onEvent: () => realtimeSuccess(SSE_CONTINUE),
},
);
// Only connect failures are exercised here.
if (read.result.ok) {
return failed(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
}
return read as RealtimeReconnectOutcome<never>;
},
});
// 10_000 is each read's connect timeout; 3_000 is the reconnect wait.
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 2);
expect(clock.sleepCalls).toEqual([10_000, 3_000]);
clock.advance(3_000);
// A hint above the policy's maxDelayMs still ends the run.
await expect(running).resolves.toMatchObject({
ok: false,
error: { operation: "CONNECT", retryable: true },
});
expect(fetcher).toHaveBeenCalledTimes(2);
expect(clock.sleepCalls).toEqual([10_000, 3_000, 10_000]);
},
);
// navigator.onLine stays true when only the server is down, so no online
// event ever arrives (0929/04-01). The SSE adapter reports a rejected fetch
// or read as PROVIDER_UNAVAILABLE, as WebSocket does; OFFLINE comes only
// from the environment signal (1103/04-04).
it("retries an SSE fetch rejection with backoff while the browser stays online", async () => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockRejectedValue(new TypeError("Failed to fetch"));
const observed: string[] = [];
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: sseAttempt(fetcher, clock),
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await advanceUntil(clock, () => result !== null);
expect(fetcher).toHaveBeenCalledTimes(3);
// Full-jitter backoff; 10_000 is each read's connect timeout.
expect(clock.sleepCalls.filter((ms) => ms !== 10_000)).toEqual([
50, 100,
]);
expect(result).toEqual({
ok: false,
error: {
kind: "PROVIDER_UNAVAILABLE",
operation: "CONNECT",
retryable: true,
},
});
expect(observed).toEqual([
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
"ATTEMPT_STARTED",
"RUN_ENDED:PROVIDER_UNAVAILABLE:EXHAUSTED",
]);
});
it("reconnects after an open SSE stream's reader rejects while the browser stays online", async () => {
const clock = new ManualClock();
let stream: ReadableStreamDefaultController<Uint8Array> | undefined;
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(
new Response(
new ReadableStream<Uint8Array>({
start(controller) {
stream = controller;
controller.enqueue(new TextEncoder().encode(": ping\n\n"));
},
}),
{ status: 200, headers: { "content-type": "text/event-stream" } },
),
)
.mockRejectedValue(new TypeError("Failed to fetch"));
const observed: string[] = [];
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: sseAttempt(fetcher, clock),
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await flushUntil(() => observed.includes("SESSION_OPENED"));
await flush();
// Healthy past stableOpenMs, then a proxy restart drops the stream.
clock.advance(1_000);
await flush();
stream!.error(new TypeError("network error"));
await advanceUntil(clock, () => result !== null);
expect(fetcher).toHaveBeenCalledTimes(3);
expect(result).toEqual({
ok: false,
error: {
kind: "PROVIDER_UNAVAILABLE",
operation: "CONNECT",
retryable: true,
},
});
expect(observed).toEqual([
"ATTEMPT_STARTED",
"SESSION_OPENED",
"SESSION_CLOSED",
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
"ATTEMPT_STARTED",
"RUN_ENDED:PROVIDER_UNAVAILABLE:EXHAUSTED",
]);
});
// 1105/04-01. No template transport reports OFFLINE any more (1103/04-04),
// so the two SSE tests above no longer reach the 0929/04-01 rule. A
// composer's transport (a STOMP one, say) still may. While the environment
// stays online, its OFFLINE is retried with backoff; waiting for an online
// event that never comes would park the run with no retry and no result.
it("retries a composer transport's OFFLINE attempt failure while the browser stays online", async () => {
const clock = new ManualClock();
const offline = realtimeFailure("OFFLINE", "CONNECT", true);
const observed: string[] = [];
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attempts += 1;
return failed(offline);
},
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await advanceUntil(clock, () => result !== null);
expect(attempts).toBe(3);
expect(clock.sleepCalls).toEqual([50, 100]);
expect(result).toEqual({
ok: false,
error: { kind: "OFFLINE", operation: "CONNECT", retryable: true },
});
expect(observed).toEqual([
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:OFFLINE",
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:OFFLINE",
"ATTEMPT_STARTED",
"RUN_ENDED:OFFLINE:EXHAUSTED",
]);
});
it("reconnects after a composer session closes with OFFLINE while the browser stays online", async () => {
const clock = new ManualClock();
const offline = realtimeFailure("OFFLINE", "RECEIVE", true);
const observed: string[] = [];
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attempts += 1;
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(offline),
}),
);
},
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await advanceUntil(clock, () => result !== null);
expect(attempts).toBe(3);
expect(result).toEqual({
ok: false,
error: { kind: "OFFLINE", operation: "RECEIVE", retryable: true },
});
expect(
observed.filter((label) => label === "RETRY_SCHEDULED:OFFLINE"),
).toHaveLength(2);
});
it.each(["before", "after"] as const)(
"waits for the online event when the browser goes offline %s the SSE fetch rejects",
async (order) => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const fetcher = vi
.fn<typeof fetch>()
.mockImplementationOnce(async () => {
if (order === "before") environment.emit(false);
throw new TypeError("Failed to fetch");
})
.mockResolvedValueOnce(new Response(null, { status: 403 }));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: sseAttempt(fetcher, clock),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await flushUntil(() => fetcher.mock.calls.length === 1);
await flush();
if (order === "after") environment.emit(false);
await flush();
// Twenty offline minutes fire any retry timer that was armed.
clock.advance(20 * 60_000);
await flush();
expect(fetcher).toHaveBeenCalledTimes(1);
expect(result).toBeNull();
environment.emit(true);
await advanceUntil(clock, () => result !== null);
expect(fetcher).toHaveBeenCalledTimes(2);
expect(result).toEqual({
ok: false,
error: { kind: "FORBIDDEN", operation: "CONNECT", retryable: false },
});
},
);
});
describe("results that composition code hands the reconnect coordinator", () => {
// Composition code may import only the adapter barrel. Each case passes a
// value that type-checks as RealtimeReconnectOutcome.
it("retries a failure built with the barrel's realtimeFailure", async () => {
const clock = new ManualClock();
const last = publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT");
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: () => publicRealtimeSuccess(undefined),
attempt: () => ({
result:
++attempts === 1
? publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT")
: last,
}),
});
const running = coordinator.run();
await flush();
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
await expect(running).resolves.toBe(last);
expect(attempts).toBe(2);
});
const unfrozenFailure: RealtimeReconnectOutcome<never> = {
result: {
ok: false,
error: {
kind: "CONNECT_TIMEOUT",
operation: "CONNECT",
retryable: true,
},
},
};
const unfrozenError: RealtimeReconnectOutcome<never> = {
result: Object.freeze({
ok: false,
error: {
kind: "CONNECT_TIMEOUT",
operation: "CONNECT",
retryable: true,
},
} as const),
};
it.each([
["a plain failure object", unfrozenFailure],
["a frozen failure around a plain error", unfrozenError],
] as const)(
"ends the run without a retry when an attempt returns %s",
async (_label, outcome) => {
const clock = new ManualClock();
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attempts += 1;
return outcome;
},
});
const running = coordinator.run();
await flush();
expect(clock.sleepCalls).toEqual([]);
await expect(running).resolves.toEqual(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
expect(attempts).toBe(1);
},
);
it("ends the run when classifyClosed returns a plain success object", async () => {
const clock = new ManualClock();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
opened(
Object.freeze({
classifyClosed: (): RealtimeReconnectOutcome<void> => ({
result: { ok: true, value: undefined },
}),
close: vi.fn(),
waitClosed: closedSuccessfully,
}),
),
});
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
expect(clock.sleepCalls).toEqual([]);
});
// 1106/04-01. The rule the attempt and session types state. A throwing
// attempt or waitClosed is a transport failure and is retried; poll reads a
// throwing execute the other way (1105/04-02).
it.each([
["attempt", "CONNECT"],
["waitClosed", "RECEIVE"],
] as const)(
"retries a throwing %s as PROVIDER_UNAVAILABLE on %s",
async (thrower, operation) => {
const clock = new ManualClock();
const observed: string[] = [];
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attempts += 1;
if (thrower === "attempt") throw new TypeError("composer defect");
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => {
throw new TypeError("composer defect");
},
}),
);
},
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await advanceUntil(clock, () => result !== null);
expect(attempts).toBe(3);
expect(clock.sleepCalls).toEqual([50, 100]);
expect(result).toEqual({
ok: false,
error: { kind: "PROVIDER_UNAVAILABLE", operation, retryable: true },
});
expect(
observed.filter((label) => label.startsWith("RETRY_SCHEDULED")),
).toEqual([
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
]);
expect(observed.at(-1)).toBe(
"RUN_ENDED:PROVIDER_UNAVAILABLE:EXHAUSTED",
);
},
);
it("ends the run without a retry when classifyClosed throws", async () => {
const clock = new ManualClock();
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attempts += 1;
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: () => {
throw new TypeError("composer defect");
},
close: vi.fn(),
waitClosed: closedSuccessfully,
}),
);
},
});
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([]);
});
// Review 04-R2 built this endpoint's connection inside the attempt: no
// request, three attempts, then PROVIDER_UNAVAILABLE (EXHAUSTED). The bridge
// now checks its settings when it is built, so the run never starts.
it("refuses a wrong SSE endpoint when the bridge is built", () => {
const fetcher = vi.fn<typeof fetch>();
const build = () =>
sseAttempt(fetcher, new ManualClock(), "http://app.example.test/events");
expect(build).toThrow(TypeError);
expect(build).toThrow("SSE endpoint must be fixed same-origin HTTPS.");
expect(fetcher).not.toHaveBeenCalled();
});
});