Files
clean-architecture-frontend…/tests/unit/realtime/realtime-reconnect-coordinator.test.ts
T
DongHyeonka 3fe28dec2e fix(realtime): 배럴에서 결과 생성 함수를 내보내고 동결 규칙을 적는다 (04-01)
문제: reconnect coordinator, live/poll handoff, SSE onOpen·onEvent,
WS onSubscribed·onEvent는 받은 RealtimeResult를 2단계로 동결된
객체만 받는다. 타입은 Readonly일 뿐이라 같은 모양의 평범한 객체도
컴파일된다. 그런데 동결된 결과를 만드는 realtimeSuccess /
realtimeFailure는 result.ts에만 있고 배럴에 없다. 조합 코드는 배럴만
import할 수 있으므로 결과를 손으로 만들게 되고, 그러면 첫 실패나 첫
종료에서 PROTOCOL_MISMATCH·MAPPING_CONTRACT_VIOLATION으로 재시도
없이 끝난다. 이 규칙은 주석에도 없었고, reconnect 경계의 동결 검사
3곳을 지워도(MU4) 테스트가 모두 통과했다.

고른 방법: 리뷰 6.2절의 (b). 배럴 src/adapters/realtime/index.ts에서
두 함수를 export하고(추가만, 기존 시그니처 변화 없음), 엄격한 경계의
타입 주석(RealtimeReconnectOutcome, RealtimeReconnectSession,
confirmTransportBarrier, handoff apply·establishAuthoritativeCheckpoint,
SSE onOpen·onEvent, WS onEvent·onSubscribed)과 생성 함수 주석에 규칙을
적었다. 동결 규칙 자체와 DEFAULT_RETRYABLE 기본값은 그대로다.

버린 대안:
- (c) 엄격한 경계도 stream coordinator처럼 한 번 복사하기. 규칙이
  하나가 되지만 "검증한 실패 객체를 그대로 돌려준다"(toBe) 계약을
  깬다.
- (d) 생성 함수를 application port(shared.ts)에 두기. feature 코드도
  쓸 수 있게 되지만 application 계층에 런타임 헬퍼가 는다. handoff
  apply를 feature가 구현한다고 정해지면 그때 옮긴다.
- 런타임 오류 이름을 따로 두기(예: RESULT_NOT_FROZEN). failure kind
  어휘가 바뀌는 계약 변경이라 이번 범위 밖이다.

위험·한계:
- 동결하지 않은 결과는 여전히 컴파일된다. 타입으로 막으려면 brand가
  필요하고, 그건 공개 타입을 깨는 변경이다. 주석과 테스트로만 알린다.
- realtime 레시피 번들 40,863 -> 40,879 / 40,900 gzip B(+16 B, 여유
  21 B). 예산은 그대로다.

검증:
- RED: 새 테스트 "retries a failure built with the barrel's
  realtimeFailure"가 수정 전 `TypeError: realtimeFailure is not a
  function`으로 실패(1 failed | 43 passed).
- 새 테스트 4건(시도의 평범한 실패, 동결된 실패 안의 평범한 error,
  classifyClosed의 평범한 성공 -> PROTOCOL_MISMATCH, 재시도 없음)과
  handoff apply의 평범한 성공 -> MAPPING_CONTRACT_VIOLATION 1건.
- MU4(reconnect 동결 검사 3곳 제거)를 다시 넣으면 새 테스트 3건이
  실패한다(수정 전에는 생존).
- GREEN: `node scripts/run-vitest.ts run tests/unit/realtime/` ->
  14파일 230건 통과.
- `npx tsc --project tsconfig.app.json --noEmit`,
  `npx tsc --project tsconfig.test.json --noEmit` -> 오류 0.
- `npx eslint <바꾼 파일 8개> --max-warnings=0` -> 통과.
- `node scripts/check-realtime-boundaries.ts` -> PASS,
  `node scripts/check-adapter-inventory.ts` -> PASS,
  `node scripts/check-optional-recipes.ts` -> PASS(realtime 40879/40900).

Refs: docs/reviews/2026-11-01-maturity-review/04-adapters-realtime.md 04-01, 6.2절
2026-11-01 22:49:38 +09:00

2343 lines
68 KiB
TypeScript

import { describe, expect, it, vi } from "vitest";
import type { ClockPort } from "../../../src/application/ports/clock-port.ts";
import type {
RealtimeResult,
} from "../../../src/application/ports/realtime/shared.ts";
import {
REALTIME_TRANSPORT_CONTINUE,
type RealtimeRecoveryCheckpoint,
} from "../../../src/application/ports/realtime/event-authority.ts";
import {
createRealtimeReconnectCoordinator,
type RealtimeCommittedRecovery,
type RealtimeReconnectAttemptContext,
type RealtimeReconnectAttemptSuccess,
type RealtimeReconnectEnvironment,
type RealtimeReconnectObservation,
type RealtimeReconnectOutcome,
type RealtimeReconnectSession,
type RealtimeRecoveryReconnectDirective,
realtimeFailure as publicRealtimeFailure,
realtimeSuccess as publicRealtimeSuccess,
} from "../../../src/adapters/realtime/index.ts";
import {
defineReconnectPolicy,
REALTIME_RECONNECT_CEILINGS,
type ReconnectPolicy,
} from "../../../src/adapters/realtime/reconnect-policy.ts";
import {
realtimeFailure,
realtimeSuccess,
} from "../../../src/adapters/realtime/result.ts";
import {
createFetchSseConnection,
readSseForReconnect,
SSE_CONTINUE,
type FetchSseClosedOutcome,
} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts";
import {
createWebSocketConnection,
type WebSocketClosedReceipt,
type WebSocketFacade,
} from "../../../src/adapters/realtime/websocket/websocket-connection.ts";
import {
REALTIME_WEBSOCKET_PROTOCOL,
} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts";
type Sleeper = Readonly<{
dueAt: number;
signal?: AbortSignal;
resolve(): void;
reject(): void;
}>;
class ManualClock implements ClockPort {
current = 0;
readonly sleepCalls: number[] = [];
readonly sleepers: Sleeper[] = [];
now(): number {
return this.current;
}
sleep(
milliseconds: number,
signal?: AbortSignal,
): Promise<void> {
this.sleepCalls.push(milliseconds);
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new Error("Aborted."));
return;
}
let onAbort: (() => void) | undefined;
const sleeper: Sleeper = {
dueAt: this.current + milliseconds,
signal,
resolve: () => {
signal?.removeEventListener("abort", onAbort!);
resolve();
},
reject: () => {
signal?.removeEventListener("abort", onAbort!);
reject(new Error("Aborted."));
},
};
onAbort = () => {
this.remove(sleeper);
sleeper.reject();
};
signal?.addEventListener("abort", onAbort, { once: true });
this.sleepers.push(sleeper);
});
}
advance(milliseconds: number): void {
this.current += milliseconds;
const ready = this.sleepers
.filter((sleeper) => sleeper.dueAt <= this.current)
.sort((left, right) => left.dueAt - right.dueAt);
for (const sleeper of ready) {
this.remove(sleeper);
sleeper.resolve();
}
}
private remove(target: Sleeper): void {
const index = this.sleepers.indexOf(target);
if (index >= 0) this.sleepers.splice(index, 1);
}
}
class MutableOnlineEnvironment
implements RealtimeReconnectEnvironment
{
readonly listeners = new Set<(online: boolean) => void>();
constructor(public connected: boolean) {}
online(): boolean {
return this.connected;
}
subscribeOnline(
listener: (online: boolean) => void,
): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
emit(online: boolean): void {
this.connected = online;
for (const listener of this.listeners) listener(online);
}
}
function policy(
override: Partial<ReconnectPolicy> = {},
): ReconnectPolicy {
return defineReconnectPolicy({
baseDelayMs: 100,
maxDelayMs: 1_000,
maxAttempts: 2,
maxElapsedMs: 5_000,
stableOpenMs: 500,
...override,
});
}
function failed(
result: Extract<RealtimeResult<never>, { ok: false }>,
serverNotBeforeMs?: number,
): RealtimeReconnectOutcome<never> {
return Object.freeze({
result,
...(serverNotBeforeMs === undefined
? {}
: { serverNotBeforeMs }),
});
}
function opened<ClosedReceipt>(
session: RealtimeReconnectSession<ClosedReceipt>,
establishedRecoveryBarrier?: RealtimeCommittedRecovery,
): RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<ClosedReceipt>
> {
return Object.freeze({
result: realtimeSuccess(
Object.freeze({
session,
...(establishedRecoveryBarrier === undefined
? {}
: { establishedRecoveryBarrier }),
}),
),
});
}
function closedSuccessfully(): RealtimeReconnectOutcome<void> {
return Object.freeze({
result: realtimeSuccess(undefined),
});
}
function confirmationSucceeded(): RealtimeResult<void> {
return realtimeSuccess(undefined);
}
function recoveryToken(
sequence = "7",
): RealtimeCommittedRecovery {
const checkpoint = Object.freeze({
lastAppliedSequence: sequence,
recoveryMode: "CURSOR",
resumeCursor: `cursor-${sequence}`,
streamEpoch: "epoch-1",
}) as unknown as RealtimeRecoveryCheckpoint;
return Object.freeze({
kind: "RECOVERY_COMMITTED",
streamId: "orders",
checkpoint,
}) as unknown as RealtimeCommittedRecovery;
}
function recoveryDirective(
recovery: RealtimeCommittedRecovery,
terminalResult: Extract<
RealtimeResult<never>,
{ ok: false }
>,
serverNotBeforeMs?: number,
): RealtimeRecoveryReconnectDirective {
return Object.freeze({
kind: "RECOVERY_RECONNECT",
recovery,
terminalResult,
...(serverNotBeforeMs === undefined
? {}
: { serverNotBeforeMs }),
});
}
function deferred<Value>(): Readonly<{
promise: Promise<Value>;
resolve(value: Value): void;
}> {
let resolvePromise: ((value: Value) => void) | undefined;
const promise = new Promise<Value>((resolve) => {
resolvePromise = resolve;
});
return Object.freeze({
promise,
resolve(value) {
resolvePromise?.(value);
},
});
}
async function flush(): Promise<void> {
// Enough turns for a retry loop iteration: sleep, attempt and close races.
for (let turn = 0; turn < 16; turn += 1) {
await Promise.resolve();
}
}
async function flushUntil(
condition: () => boolean,
): Promise<void> {
for (let turn = 0; turn < 50; turn += 1) {
if (condition()) return;
await Promise.resolve();
}
}
describe("realtime reconnect coordinator", () => {
it("owns one bounded retry loop and returns the exact last failure", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const failures = [
realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true),
realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true),
realtimeFailure("IDLE_TIMEOUT", "RECEIVE", true),
] as const;
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
const next = failures[attempts++]!;
return failed(
next,
next === failures[1] ? 0 : undefined,
);
},
});
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
await flush();
expect(attempts).toBe(2);
expect(clock.sleepCalls).toEqual([50, 100]);
clock.advance(100);
const result = await running;
expect(attempts).toBe(3);
expect(result).toBe(failures[2]);
expect(coordinator.getState()).toBe("IDLE");
});
it("does not clamp a server not-before value past elapsed budget", async () => {
const clock = new ManualClock();
const terminal = realtimeFailure(
"RATE_LIMITED",
"CONNECT",
true,
);
const random = vi.fn(() => 0);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({
maxAttempts: 1,
maxElapsedMs: 500,
}),
environment: new MutableOnlineEnvironment(true),
clock,
random,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => failed(terminal, 500),
});
const result = await coordinator.run();
expect(result).toBe(terminal);
expect(clock.sleepCalls).toEqual([]);
expect(random).not.toHaveBeenCalled();
});
it.each([
"RATE_LIMITED",
"PROVIDER_UNAVAILABLE",
] as const)(
"reconnects a retryable external %s attempt failure without a server hint",
async (kind) => {
const clock = new ManualClock();
const retryable = realtimeFailure(kind, "CONNECT", true);
const terminal = realtimeFailure(kind, "CONNECT", false);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
failed(++attempts === 1 ? retryable : terminal),
});
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
},
);
it("reconnects a retryable external close failure without a server hint", async () => {
const clock = new ManualClock();
const retryable = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
true,
);
const terminal = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
false,
);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
const closedWith = ++attempts === 1 ? retryable : terminal;
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(closedWith),
}),
);
},
});
const running = coordinator.run();
await flush();
expect(attempts).toBe(1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
});
it("waits for an explicit online signal without a retry timer", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(false);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const attempt = vi.fn(() => failed(terminal));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const running = coordinator.run();
await flush();
clock.advance(100);
await flush();
expect(attempt).not.toHaveBeenCalled();
expect(clock.sleepCalls).toEqual([]);
environment.emit(true);
await flush();
expect(clock.sleepCalls).toEqual([50]);
expect(attempt).not.toHaveBeenCalled();
clock.advance(50);
const result = await running;
expect(attempt).toHaveBeenCalledTimes(1);
expect(result).toBe(terminal);
});
it("does not spend the elapsed budget while waiting offline", async () => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(false);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const attempt = vi.fn(() => failed(terminal));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const running = coordinator.run();
await flush();
// Offline for longer than maxElapsedMs before the first attempt.
clock.advance(6_000);
environment.emit(true);
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempt).toHaveBeenCalledTimes(1);
expect(result).toBe(terminal);
});
it.each(["an offline blip", "a waitClosed throw"] as const)(
"reconnects after %s ends a healthy session older than the elapsed budget",
async (ending) => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const closing = deferred<RealtimeReconnectOutcome<void>>();
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context: RealtimeReconnectAttemptContext) => {
attempts += 1;
if (attempts > 1) return failed(terminal);
context.markValidHeartbeatOrEvent();
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: () =>
closing.resolve(
failed(
realtimeFailure("OFFLINE", "RECEIVE", true),
),
),
waitClosed: () =>
ending === "an offline blip"
? closing.promise
: closing.promise.then(() => {
throw new Error("transport failed");
}),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => attempts === 1);
await flush();
// Healthy for twice maxElapsedMs, then a one second outage.
clock.advance(10_000);
if (ending === "an offline blip") {
environment.emit(false);
await flush();
clock.advance(1_000);
environment.emit(true);
} else {
closing.resolve(closedSuccessfully());
}
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
const result = await running;
expect(attempts).toBe(2);
expect(result).toBe(terminal);
},
);
it("observes attempts, retries, the session and why the run ended", async () => {
const clock = new ManualClock();
const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false);
const observations: unknown[] = [];
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
observe: (observation) => {
observations.push(observation);
throw new Error("sink failure must not change the run");
},
attempt: () =>
++attempts === 1
? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true))
: opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(terminal),
}),
),
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(50);
await expect(running).resolves.toBe(terminal);
expect(observations).toEqual([
{ kind: "ATTEMPT_STARTED" },
{
kind: "RETRY_SCHEDULED",
failureKind: "CONNECT_TIMEOUT",
delayBucket: "UNDER_1S",
},
{ kind: "ATTEMPT_STARTED" },
{ kind: "SESSION_OPENED" },
{ kind: "SESSION_CLOSED" },
{ kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" },
]);
// A second run that spends its retries ends as EXHAUSTED, not TERMINAL.
const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true);
const ends: unknown[] = [];
const spending = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
observe: (observation) => {
if (observation.kind === "RUN_ENDED") ends.push(observation);
},
attempt: () => failed(exhausted),
});
const spent = spending.run();
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
await expect(spent).resolves.toBe(exhausted);
expect(ends).toEqual([
{
kind: "RUN_ENDED",
reason: "EXHAUSTED",
failureKind: "PROVIDER_UNAVAILABLE",
},
]);
});
it("resets a spent retry only after a valid signal and a stable open", async () => {
const runScenario = async (markValid: boolean, openMs: number) => {
const clock = new ManualClock();
const firstFailure = realtimeFailure(
"CONNECT_TIMEOUT",
"CONNECT",
true,
);
const sessionFailure = realtimeFailure(
"PROVIDER_UNAVAILABLE",
"RECEIVE",
true,
);
const terminal = realtimeFailure(
"FORBIDDEN",
"CONNECT",
false,
);
const closeSession = vi.fn();
const closing = deferred<RealtimeReconnectOutcome<void>>();
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: (
context: RealtimeReconnectAttemptContext,
) => {
attempts += 1;
if (attempts === 1) return failed(firstFailure);
if (attempts === 2) {
if (markValid) {
context.markValidHeartbeatOrEvent();
}
return opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed: () => closing.promise,
}),
);
}
return failed(terminal);
},
});
const running = coordinator.run();
await flush();
clock.advance(0);
await flushUntil(() => attempts === 2);
await flush();
clock.advance(openMs);
closing.resolve(failed(sessionFailure, 0));
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
const result = await running;
return {
attempts,
closeSession,
result,
sessionFailure,
terminal,
};
};
// A heartbeat on a short-lived connection, or a quiet long one, keeps
// the spent budget; only both together reset it.
for (const [markValid, openMs] of [
[false, 0],
[true, 499],
[false, 500],
] as const) {
const spent = await runScenario(markValid, openMs);
expect(spent.attempts).toBe(2);
expect(spent.result).toBe(spent.sessionFailure);
expect(spent.closeSession).toHaveBeenCalledOnce();
}
const eligible = await runScenario(true, 500);
expect(eligible.attempts).toBe(3);
expect(eligible.result).toBe(eligible.terminal);
expect(eligible.closeSession).toHaveBeenCalledOnce();
});
it("does not reconnect before the session's authoritative close result", async () => {
const clock = new ManualClock();
const closure =
deferred<RealtimeReconnectOutcome<void>>();
const terminal = realtimeFailure(
"FORBIDDEN",
"RECEIVE",
false,
);
const closeSession = vi.fn();
const attempt = vi.fn(() =>
opened<RealtimeReconnectOutcome<void>>(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed: () => closure.promise,
}),
),
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
let settled = false;
const running = coordinator.run().then((result) => {
settled = true;
return result;
});
await flush();
expect(attempt).toHaveBeenCalledOnce();
expect(clock.sleepCalls).toEqual([]);
expect(settled).toBe(false);
expect(coordinator.getState()).toBe("RUNNING");
closure.resolve(failed(terminal));
const result = await running;
expect(result).toBe(terminal);
expect(closeSession).toHaveBeenCalledOnce();
});
it("carries an exact recovery token into the next attempt and opens only after confirmation", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const directive = recoveryDirective(recovery, terminal);
const completed = closedSuccessfully();
const confirmation = realtimeSuccess(undefined);
const confirmTransportBarrier = vi.fn(
(received: RealtimeCommittedRecovery) => {
expect(received).toBe(recovery);
return confirmation;
},
);
let attempts = 0;
let gateSettled = false;
let gateResult: RealtimeResult<void> | undefined;
type RecoveryCloseReceipt =
| RealtimeRecoveryReconnectDirective
| RealtimeReconnectOutcome<void>;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier,
attempt: (context) => {
attempts += 1;
if (attempts === 1) {
expect(context.pendingRecovery).toBeNull();
return opened<RecoveryCloseReceipt>(
Object.freeze({
classifyClosed: (
receipt: RecoveryCloseReceipt,
) => receipt,
close: vi.fn(),
waitClosed: () => directive,
}),
);
}
expect(context.pendingRecovery).toBe(recovery);
const gate =
context.waitForRecoveryBarrierConfirmation();
void gate.then((result) => {
gateSettled = true;
gateResult = result;
});
expect(gateSettled).toBe(false);
return opened<RecoveryCloseReceipt>(
Object.freeze({
classifyClosed: (
receipt: RecoveryCloseReceipt,
) => receipt,
close: vi.fn(),
waitClosed: () => completed,
}),
recovery,
);
},
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 1);
expect(clock.sleepCalls).toEqual([0]);
clock.advance(0);
const result = await running;
await flush();
expect(attempts).toBe(2);
expect(confirmTransportBarrier).toHaveBeenCalledOnce();
expect(gateSettled).toBe(true);
expect(gateResult).toBe(confirmation);
expect(result).toBe(completed.result);
});
it.each([
["missing", undefined],
["cloned", recoveryToken()],
] as const)(
"rejects a %s recovery proof and closes the new session",
async (_case, proof) => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const closeSession = vi.fn();
const waitClosed = vi.fn(closedSuccessfully);
const confirmTransportBarrier = vi.fn(
confirmationSucceeded,
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier,
attempt: (context) => {
expect(context.pendingRecovery).toBe(recovery);
const session = Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed,
});
return proof === undefined
? opened(session)
: opened(session, proof);
},
});
const running = coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"RECOVER",
false,
),
);
expect(closeSession).toHaveBeenCalledOnce();
expect(waitClosed).not.toHaveBeenCalled();
expect(confirmTransportBarrier).not.toHaveBeenCalled();
},
);
it("settles the readiness gate and run with the exact confirmation failure", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const confirmationFailure = realtimeFailure(
"SCOPE_PROTOCOL_VIOLATION",
"RECOVER",
false,
);
const closeSession = vi.fn();
const waitClosed = vi.fn(closedSuccessfully);
let gateResult: RealtimeResult<void> | undefined;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: () => confirmationFailure,
attempt: (context) => {
void context
.waitForRecoveryBarrierConfirmation()
.then((result) => {
gateResult = result;
});
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed,
}),
recovery,
);
},
});
const running = coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
const result = await running;
await flush();
expect(result).toBe(confirmationFailure);
expect(gateResult).toBe(confirmationFailure);
expect(closeSession).toHaveBeenCalledOnce();
expect(waitClosed).not.toHaveBeenCalled();
});
it("retains the exact recovery token across retryable attempt failures and returns its terminal result on exhaustion", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const attemptFailure = realtimeFailure(
"CONNECT_TIMEOUT",
"CONNECT",
true,
);
const gateResults: RealtimeResult<void>[] = [];
const seenRecoveries: Array<
RealtimeCommittedRecovery | null
> = [];
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 2 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context) => {
seenRecoveries.push(context.pendingRecovery);
void context
.waitForRecoveryBarrierConfirmation()
.then((result) => gateResults.push(result));
return failed(attemptFailure);
},
});
const running = coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
const result = await running;
await flush();
expect(seenRecoveries).toEqual([recovery, recovery]);
expect(gateResults).toEqual([
attemptFailure,
attemptFailure,
]);
expect(gateResults[0]).toBe(attemptFailure);
expect(gateResults[1]).toBe(attemptFailure);
expect(result).toBe(terminal);
});
it("fails a no-pending gate immediately and rejects any unsolicited proof", async () => {
const proof = recoveryToken();
const closeSession = vi.fn();
const confirmTransportBarrier = vi.fn(
confirmationSucceeded,
);
let gateResult: RealtimeResult<void> | undefined;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier,
attempt: async (context) => {
gateResult =
await context.waitForRecoveryBarrierConfirmation();
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: closeSession,
waitClosed: closedSuccessfully,
}),
proof,
);
},
});
const result = await coordinator.run();
expect(gateResult).toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"RECOVER",
false,
),
);
expect(result).toBe(gateResult);
expect(closeSession).toHaveBeenCalledOnce();
expect(confirmTransportBarrier).not.toHaveBeenCalled();
});
it("applies retry ceilings to an initial recovery and preserves its exact terminal result", async () => {
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const attempt = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({
maxAttempts: 1,
maxElapsedMs: 500,
}),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const result = await coordinator.run({
initialRecovery: recoveryDirective(
recovery,
terminal,
500,
),
});
expect(result).toBe(terminal);
expect(attempt).not.toHaveBeenCalled();
});
it("settles a pending readiness gate when the run is aborted", async () => {
const clock = new ManualClock();
const recovery = recoveryToken();
const terminal = realtimeFailure(
"CURSOR_EXPIRED",
"RECOVER",
false,
);
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
let gateResult: RealtimeResult<void> | undefined;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context) => {
void context
.waitForRecoveryBarrierConfirmation()
.then((result) => {
gateResult = result;
});
return attemptResult.promise;
},
});
const controller = new AbortController();
const running = coordinator.run({
signal: controller.signal,
initialRecovery: recoveryDirective(
recovery,
terminal,
),
});
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(0);
await flush();
controller.abort();
const result = await running;
await flush();
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
expect(gateResult).toEqual(
realtimeFailure("ABORTED", "RECOVER", false),
);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: closedSuccessfully,
}),
recovery,
),
);
await flush();
});
it("rejects accessors without invoking them and consumes proxy values only from one descriptor snapshot", async () => {
let getterReads = 0;
const accessorOutcome = Object.defineProperty({}, "result", {
enumerable: true,
get() {
getterReads += 1;
return realtimeSuccess(undefined);
},
});
const accessorCoordinator =
createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
accessorOutcome as unknown as RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<unknown>
>,
});
await expect(accessorCoordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
expect(getterReads).toBe(0);
const completed = closedSuccessfully();
const session = Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => completed,
});
const rawOutcome = opened(session);
const proxyReads: PropertyKey[] = [];
const proxyOutcome = new Proxy(rawOutcome, {
get(_target, key) {
proxyReads.push(key);
if (key === "then") return undefined;
throw new Error("raw property reread");
},
});
const proxyCoordinator =
createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () => proxyOutcome,
});
await expect(proxyCoordinator.run()).resolves.toBe(
completed.result,
);
expect(proxyReads).toEqual(["then"]);
});
it("rechecks scope after attempt and waitClosed boundaries before accepting callbacks", async () => {
let attemptScopeCurrent = true;
const lateAttemptClose = vi.fn();
const lateAttemptWaitClosed = vi.fn(closedSuccessfully);
const lateAttemptCoordinator =
createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
isCurrent: () => attemptScopeCurrent,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attemptScopeCurrent = false;
return opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateAttemptClose,
waitClosed: lateAttemptWaitClosed,
}),
);
},
});
await expect(lateAttemptCoordinator.run()).resolves.toEqual(
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
);
expect(lateAttemptClose).toHaveBeenCalledOnce();
expect(lateAttemptWaitClosed).not.toHaveBeenCalled();
let closeScopeCurrent = true;
const classifyClosed = vi.fn(
(receipt: RealtimeReconnectOutcome<void>) => receipt,
);
const closeSession = vi.fn();
const closeCoordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
isCurrent: () => closeScopeCurrent,
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
opened(
Object.freeze({
classifyClosed,
close: closeSession,
waitClosed: () => {
closeScopeCurrent = false;
return closedSuccessfully();
},
}),
),
});
await expect(closeCoordinator.run()).resolves.toEqual(
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
);
expect(classifyClosed).not.toHaveBeenCalled();
expect(closeSession).toHaveBeenCalledOnce();
});
it("rejects extra and missing outcome keys through the common result codec", async () => {
const validSession = Object.freeze({
classifyClosed: closedSuccessfully,
close: vi.fn(),
waitClosed: closedSuccessfully,
});
const invalidOutcomes = [
{
extra: true,
result: realtimeSuccess(validSession),
},
{
result: { ok: true },
},
{
result: {
ok: false,
error: {
extra: true,
kind: "FORBIDDEN",
operation: "CONNECT",
retryable: false,
},
},
},
];
for (const invalid of invalidOutcomes) {
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
invalid as unknown as RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<unknown>
>,
});
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
}
});
it("rechecks abort immediately after listener registration", async () => {
let abortReads = 0;
let listener: (() => void) | undefined;
const signal = {
get aborted() {
abortReads += 1;
return abortReads >= 2;
},
addEventListener(
type: string,
nextListener: () => void,
) {
if (type === "abort") listener = nextListener;
},
removeEventListener() {
listener = undefined;
},
} as unknown as AbortSignal;
const attempt = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt,
});
const result = await coordinator.run({ signal });
expect(listener).toBeUndefined();
expect(attempt).not.toHaveBeenCalled();
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
});
it("bounds a non-cooperative retry sleep after its phase is aborted", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const sleep = vi.fn(
() => new Promise<void>(() => undefined),
);
const clock: ClockPort = Object.freeze({
now: () => 0,
sleep,
});
const retryable = realtimeFailure(
"CONNECT_TIMEOUT",
"CONNECT",
true,
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => failed(retryable),
});
try {
const running = coordinator.run();
await flushUntil(() => sleep.mock.calls.length === 1);
environment.emit(false);
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROVIDER_UNAVAILABLE",
"CONNECT",
false,
),
);
expect(coordinator.getState()).toBe("DRAINING");
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("bounds a non-cooperative attempt and closes its late successful session before leaving DRAINING", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
const lateClose = vi.fn();
const lateWaitClosed = vi.fn(closedSuccessfully);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () => attemptResult.promise,
});
try {
const running = coordinator.run();
await flush();
environment.emit(false);
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROVIDER_UNAVAILABLE",
"CONNECT",
false,
),
);
expect(coordinator.getState()).toBe("DRAINING");
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure(
"PROTOCOL_MISMATCH",
"CONNECT",
false,
),
);
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateClose,
waitClosed: lateWaitClosed,
}),
),
);
await flush();
expect(lateClose).toHaveBeenCalledOnce();
expect(lateWaitClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("IDLE");
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("cancels the drain timer on abort and keeps the late generation fenced", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
const lateClose = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () => attemptResult.promise,
});
const controller = new AbortController();
try {
const running = coordinator.run({
signal: controller.signal,
});
await flush();
environment.emit(false);
await flush();
expect(vi.getTimerCount()).toBe(1);
controller.abort();
const result = await running;
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
expect(coordinator.getState()).toBe("DRAINING");
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateClose,
waitClosed: closedSuccessfully,
}),
),
);
await flush();
expect(lateClose).toHaveBeenCalledOnce();
expect(coordinator.getState()).toBe("IDLE");
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("keeps a healthy waitClosed unbounded but bounds its post-abort drain", async () => {
vi.useFakeTimers();
const environment = new MutableOnlineEnvironment(true);
const closure =
deferred<RealtimeReconnectOutcome<void>>();
const classifyClosed = vi.fn(
(receipt: RealtimeReconnectOutcome<void>) => receipt,
);
const closeSession = vi.fn();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock: new ManualClock(),
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
opened(
Object.freeze({
classifyClosed,
close: closeSession,
waitClosed: () => closure.promise,
}),
),
});
try {
let settled = false;
const running = coordinator.run().then((result) => {
settled = true;
return result;
});
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs * 2,
);
expect(settled).toBe(false);
expect(coordinator.getState()).toBe("RUNNING");
expect(vi.getTimerCount()).toBe(0);
environment.emit(false);
await flush();
await vi.advanceTimersByTimeAsync(
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
);
const result = await running;
expect(result).toEqual(
realtimeFailure(
"PROVIDER_UNAVAILABLE",
"CONNECT",
false,
),
);
expect(closeSession).toHaveBeenCalled();
expect(classifyClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("DRAINING");
expect(environment.listeners.size).toBe(0);
expect(vi.getTimerCount()).toBe(0);
closure.resolve(closedSuccessfully());
await flush();
expect(classifyClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("IDLE");
} finally {
coordinator.close();
vi.useRealTimers();
}
});
it("uses waitClosed as authority and fences an aborted late generation", async () => {
const clock = new ManualClock();
const attemptResult =
deferred<
RealtimeReconnectOutcome<
RealtimeReconnectAttemptSuccess<
RealtimeReconnectOutcome<void>
>
>
>();
let captured:
| RealtimeReconnectAttemptContext
| undefined;
const lateClose = vi.fn();
const lateWaitClosed = vi.fn(closedSuccessfully);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: (context) => {
captured = context;
return attemptResult.promise;
},
});
const controller = new AbortController();
const running = coordinator.run({ signal: controller.signal });
await flush();
expect(captured?.isCurrent()).toBe(true);
controller.abort();
const result = await running;
expect(result).toEqual(
realtimeFailure("ABORTED", "CONNECT", false),
);
expect(coordinator.getState()).toBe("DRAINING");
expect(captured?.isCurrent()).toBe(false);
attemptResult.resolve(
opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: lateClose,
waitClosed: lateWaitClosed,
}),
),
);
await flush();
expect(lateClose).toHaveBeenCalledOnce();
expect(lateWaitClosed).not.toHaveBeenCalled();
expect(coordinator.getState()).toBe("IDLE");
});
});
class FakeWebSocket implements WebSocketFacade {
protocol: string = REALTIME_WEBSOCKET_PROTOCOL;
readyState = 0;
bufferedAmount = 0;
readonly listeners = new Map<
string,
Set<(event: unknown) => void>
>();
readonly sent: string[] = [];
send(data: string): void {
if (this.readyState !== 1) throw new Error("not open");
this.sent.push(data);
}
close(): void {
this.readyState = 3;
}
addEventListener(
type: string,
listener: (event: unknown) => void,
): void {
const listeners = this.listeners.get(type) ?? new Set();
listeners.add(listener);
this.listeners.set(type, listeners);
}
removeEventListener(
type: string,
listener: (event: unknown) => void,
): void {
this.listeners.get(type)?.delete(listener);
}
accept(): void {
this.readyState = 1;
this.emit("open", {});
this.emit("message", {
data: JSON.stringify({
type: "WELCOME",
protocol: REALTIME_WEBSOCKET_PROTOCOL,
connectionId: "connection.0001",
heartbeatMs: 5_000,
heartbeatAckTimeoutMs: 1_000,
limits: {
maxFrameBytes: 64 * 1_024,
maxSubscriptions: 32,
maxInboundQueueCount: 256,
maxInboundQueueBytes: 4 * 1_024 * 1_024,
maxOutboundQueueCount: 128,
maxOutboundQueueBytes: 256 * 1_024,
maxBufferedAmountBytes: 256 * 1_024,
maxEventsPerSecond: 256,
},
}),
});
}
nativeClose(code: number): void {
this.readyState = 3;
this.emit("close", { code, reason: "" });
}
ackHeartbeats(): void {
for (const raw of this.sent.splice(0)) {
const frame = JSON.parse(raw) as { type: string; nonce: string };
if (frame.type !== "HEARTBEAT") continue;
this.emit("message", {
data: JSON.stringify({
type: "HEARTBEAT_ACK",
protocol: REALTIME_WEBSOCKET_PROTOCOL,
nonce: frame.nonce,
}),
});
}
}
private emit(type: string, event: unknown): void {
for (const listener of [...(this.listeners.get(type) ?? [])]) {
listener(event);
}
}
}
/**
* The smallest SSE bridge design 6.3 leaves to composition: the attempt ends
* at `onOpen` and the rest of the read is the session.
*/
function sseAttempt(fetcher: typeof fetch, clock: ManualClock) {
return async (context: RealtimeReconnectAttemptContext) => {
const connection = createFetchSseConnection({
endpoint: "https://app.example.test/events",
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR",
fetcher,
clock,
});
const open = deferred<"OPEN">();
const reading = readSseForReconnect(connection, {
resumeCursor: null,
signal: context.signal,
onOpen: () => {
open.resolve("OPEN");
return realtimeSuccess(undefined);
},
onEvent: () => realtimeSuccess(SSE_CONTINUE),
onComment: () => context.markValidHeartbeatOrEvent(),
});
const first = await Promise.race([open.promise, reading]);
if (first !== "OPEN") {
return first.result.ok
? failed(realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false))
: (first as RealtimeReconnectOutcome<never>);
}
return opened(
Object.freeze({
waitClosed: () => reading,
classifyClosed: (
receipt: RealtimeReconnectOutcome<FetchSseClosedOutcome>,
) =>
receipt.result.ok
? failed(
realtimeFailure("PROTOCOL_MISMATCH", "RECEIVE", false),
)
: (receipt as RealtimeReconnectOutcome<never>),
close: () => connection.close(),
}),
);
};
}
async function advanceUntil(
clock: ManualClock,
settled: () => boolean,
): Promise<void> {
// Ten simulated seconds outlast policy()'s whole elapsed budget.
for (let elapsed = 0; elapsed < 10_000 && !settled(); elapsed += 50) {
clock.advance(50);
await flush();
}
}
function observationLabel({
kind,
failureKind,
reason,
}: RealtimeReconnectObservation): string {
return [kind, failureKind, reason].filter(Boolean).join(":");
}
describe("transport failures through the reconnect coordinator", () => {
it.each([
["NETWORK_LOST", 1_006],
["RESTART", 1_012],
] as const)(
"reconnects a WebSocket %s close that carries no server hint",
async (_category, code) => {
const clock = new ManualClock();
const sockets: FakeWebSocket[] = [];
const coordinator = createRealtimeReconnectCoordinator<
WebSocketClosedReceipt
>({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const connection = createWebSocketConnection({
origin: "https://app.example.test",
endpointPath: "/_realtime/v1",
clock,
createNonce: () => "nonce.0001",
createSocket: () => {
const socket = new FakeWebSocket();
sockets.push(socket);
return socket;
},
onEvent: async () =>
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
onRecoveryRequired: () => undefined,
});
const connected = await connection.connect(context.signal);
if (!connected.ok) return failed(connected);
return opened(
Object.freeze({
waitClosed: () => connection.waitClosed(),
classifyClosed: (receipt: WebSocketClosedReceipt) =>
Object.freeze({
result: Object.freeze({
ok: false as const,
error: receipt.error,
}),
}),
close: () => connection.close(),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => sockets.length === 1);
sockets[0]!.accept();
await flush();
sockets[0]!.nativeClose(code);
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await flushUntil(() => sockets.length === 2);
expect(sockets).toHaveLength(2);
// OVERLOADED carries no hint either and stays terminal.
sockets[1]!.accept();
await flush();
sockets[1]!.nativeClose(1_013);
await expect(running).resolves.toEqual({
ok: false,
error: {
kind: "PROVIDER_UNAVAILABLE",
operation: "RECEIVE",
retryable: false,
},
});
expect(sockets).toHaveLength(2);
},
);
it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => {
const clock = new ManualClock();
const sockets: FakeWebSocket[] = [];
let nonce = 0;
const coordinator = createRealtimeReconnectCoordinator<
WebSocketClosedReceipt
>({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const connection = createWebSocketConnection({
origin: "https://app.example.test",
endpointPath: "/_realtime/v1",
clock,
createNonce: () => `nonce.${++nonce}`,
createSocket: () => {
const socket = new FakeWebSocket();
sockets.push(socket);
return socket;
},
onEvent: async () =>
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
onRecoveryRequired: () => undefined,
observe: (observation) => {
if (observation.kind === "HEARTBEAT_ACK") {
context.markValidHeartbeatOrEvent();
}
},
});
const connected = await connection.connect(context.signal);
if (!connected.ok) return failed(connected);
return opened(
Object.freeze({
waitClosed: () => connection.waitClosed(),
classifyClosed: (receipt: WebSocketClosedReceipt) =>
Object.freeze({
result: Object.freeze({
ok: false as const,
error: receipt.error,
}),
}),
close: () => connection.close(),
}),
);
},
});
const running = coordinator.run();
await flushUntil(() => sockets.length === 1);
sockets[0]!.accept();
await flush();
// No events, only acknowledged heartbeats, for longer than maxElapsedMs.
for (let beat = 0; beat < 2; beat += 1) {
clock.advance(5_000);
await flush();
sockets[0]!.ackHeartbeats();
clock.advance(1_000);
await flush();
}
sockets[0]!.nativeClose(1_012);
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await flushUntil(() => sockets.length === 2);
expect(sockets).toHaveLength(2);
coordinator.close();
await expect(running).resolves.toMatchObject({
ok: false,
error: { kind: "CLOSED" },
});
});
it.each([502, 504])(
"reconnects SSE HTTP %i that carries no server hint",
async (status) => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status }))
.mockResolvedValueOnce(new Response(null, { status: 403 }));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const read = await createFetchSseConnection({
endpoint: "https://app.example.test/events",
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR",
fetcher,
clock,
}).read({
resumeCursor: null,
signal: context.signal,
onEvent: () => realtimeSuccess(SSE_CONTINUE),
});
// Only connect failures are exercised here.
return failed(
read.ok
? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false)
: read,
);
},
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.includes(50));
clock.advance(50);
await expect(running).resolves.toEqual({
ok: false,
error: {
kind: "FORBIDDEN",
operation: "CONNECT",
retryable: false,
},
});
expect(fetcher).toHaveBeenCalledTimes(2);
},
);
it.each([429, 503])(
"waits at least the SSE %i Retry-After through the reconnect bridge",
async (status) => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(
new Response(null, {
status,
headers: { "retry-after": "3" },
}),
)
.mockResolvedValueOnce(
new Response(null, {
status,
headers: { "retry-after": "10" },
}),
);
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxDelayMs: 5_000, maxElapsedMs: 60_000 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: async (context) => {
const read = await readSseForReconnect(
createFetchSseConnection({
endpoint: "https://app.example.test/events",
applicationOrigin: "https://app.example.test",
recoveryMode: "CURSOR",
fetcher,
clock,
}),
{
resumeCursor: null,
signal: context.signal,
onEvent: () => realtimeSuccess(SSE_CONTINUE),
},
);
// Only connect failures are exercised here.
if (read.result.ok) {
return failed(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
}
return read as RealtimeReconnectOutcome<never>;
},
});
// 10_000 is each read's connect timeout; 3_000 is the reconnect wait.
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 2);
expect(clock.sleepCalls).toEqual([10_000, 3_000]);
clock.advance(3_000);
// A hint above the policy's maxDelayMs still ends the run.
await expect(running).resolves.toMatchObject({
ok: false,
error: { operation: "CONNECT", retryable: true },
});
expect(fetcher).toHaveBeenCalledTimes(2);
expect(clock.sleepCalls).toEqual([10_000, 3_000, 10_000]);
},
);
// The SSE adapter reports a rejected fetch or read as OFFLINE, but
// navigator.onLine stays true when only the server is down, so no online
// event ever arrives (0929/04-01).
it("retries an SSE fetch rejection with backoff while the browser stays online", async () => {
const clock = new ManualClock();
const fetcher = vi
.fn<typeof fetch>()
.mockRejectedValue(new TypeError("Failed to fetch"));
const observed: string[] = [];
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: sseAttempt(fetcher, clock),
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await advanceUntil(clock, () => result !== null);
expect(fetcher).toHaveBeenCalledTimes(3);
// Full-jitter backoff; 10_000 is each read's connect timeout.
expect(clock.sleepCalls.filter((ms) => ms !== 10_000)).toEqual([
50, 100,
]);
expect(result).toEqual({
ok: false,
error: { kind: "OFFLINE", operation: "CONNECT", retryable: true },
});
expect(observed).toEqual([
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:OFFLINE",
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:OFFLINE",
"ATTEMPT_STARTED",
"RUN_ENDED:OFFLINE:EXHAUSTED",
]);
});
it("reconnects after an open SSE stream's reader rejects while the browser stays online", async () => {
const clock = new ManualClock();
let stream: ReadableStreamDefaultController<Uint8Array> | undefined;
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(
new Response(
new ReadableStream<Uint8Array>({
start(controller) {
stream = controller;
controller.enqueue(new TextEncoder().encode(": ping\n\n"));
},
}),
{ status: 200, headers: { "content-type": "text/event-stream" } },
),
)
.mockRejectedValue(new TypeError("Failed to fetch"));
const observed: string[] = [];
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: sseAttempt(fetcher, clock),
observe: (observation) => observed.push(observationLabel(observation)),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await flushUntil(() => observed.includes("SESSION_OPENED"));
await flush();
// Healthy past stableOpenMs, then a proxy restart drops the stream.
clock.advance(1_000);
await flush();
stream!.error(new TypeError("network error"));
await advanceUntil(clock, () => result !== null);
expect(fetcher).toHaveBeenCalledTimes(3);
expect(result).toEqual({
ok: false,
error: { kind: "OFFLINE", operation: "CONNECT", retryable: true },
});
expect(observed).toEqual([
"ATTEMPT_STARTED",
"SESSION_OPENED",
"SESSION_CLOSED",
"RETRY_SCHEDULED:OFFLINE",
"ATTEMPT_STARTED",
"RETRY_SCHEDULED:OFFLINE",
"ATTEMPT_STARTED",
"RUN_ENDED:OFFLINE:EXHAUSTED",
]);
});
it.each(["before", "after"] as const)(
"waits for the online event when the browser goes offline %s the SSE fetch rejects",
async (order) => {
const clock = new ManualClock();
const environment = new MutableOnlineEnvironment(true);
const fetcher = vi
.fn<typeof fetch>()
.mockImplementationOnce(async () => {
if (order === "before") environment.emit(false);
throw new TypeError("Failed to fetch");
})
.mockResolvedValueOnce(new Response(null, { status: 403 }));
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment,
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
attempt: sseAttempt(fetcher, clock),
});
let result: RealtimeResult<void> | null = null;
void coordinator.run().then((value) => {
result = value;
});
await flushUntil(() => fetcher.mock.calls.length === 1);
await flush();
if (order === "after") environment.emit(false);
await flush();
// Twenty offline minutes fire any retry timer that was armed.
clock.advance(20 * 60_000);
await flush();
expect(fetcher).toHaveBeenCalledTimes(1);
expect(result).toBeNull();
environment.emit(true);
await advanceUntil(clock, () => result !== null);
expect(fetcher).toHaveBeenCalledTimes(2);
expect(result).toEqual({
ok: false,
error: { kind: "FORBIDDEN", operation: "CONNECT", retryable: false },
});
},
);
});
describe("results that composition code hands the reconnect coordinator", () => {
// Composition code may import only the adapter barrel. Each case passes a
// value that type-checks as RealtimeReconnectOutcome.
it("retries a failure built with the barrel's realtimeFailure", async () => {
const clock = new ManualClock();
const last = publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT");
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: () => publicRealtimeSuccess(undefined),
attempt: () => ({
result:
++attempts === 1
? publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT")
: last,
}),
});
const running = coordinator.run();
await flush();
expect(clock.sleepCalls).toEqual([50]);
clock.advance(50);
await expect(running).resolves.toBe(last);
expect(attempts).toBe(2);
});
const unfrozenFailure: RealtimeReconnectOutcome<never> = {
result: {
ok: false,
error: {
kind: "CONNECT_TIMEOUT",
operation: "CONNECT",
retryable: true,
},
},
};
const unfrozenError: RealtimeReconnectOutcome<never> = {
result: Object.freeze({
ok: false,
error: {
kind: "CONNECT_TIMEOUT",
operation: "CONNECT",
retryable: true,
},
} as const),
};
it.each([
["a plain failure object", unfrozenFailure],
["a frozen failure around a plain error", unfrozenError],
] as const)(
"ends the run without a retry when an attempt returns %s",
async (_label, outcome) => {
const clock = new ManualClock();
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: () => {
attempts += 1;
return outcome;
},
});
const running = coordinator.run();
await flush();
expect(clock.sleepCalls).toEqual([]);
await expect(running).resolves.toEqual(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
expect(attempts).toBe(1);
},
);
it("ends the run when classifyClosed returns a plain success object", async () => {
const clock = new ManualClock();
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
confirmTransportBarrier: confirmationSucceeded,
attempt: () =>
opened(
Object.freeze({
classifyClosed: (): RealtimeReconnectOutcome<void> => ({
result: { ok: true, value: undefined },
}),
close: vi.fn(),
waitClosed: closedSuccessfully,
}),
),
});
await expect(coordinator.run()).resolves.toEqual(
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
);
expect(clock.sleepCalls).toEqual([]);
});
});