문제: reconnect coordinator, live/poll handoff, SSE onOpen·onEvent, WS onSubscribed·onEvent는 받은 RealtimeResult를 2단계로 동결된 객체만 받는다. 타입은 Readonly일 뿐이라 같은 모양의 평범한 객체도 컴파일된다. 그런데 동결된 결과를 만드는 realtimeSuccess / realtimeFailure는 result.ts에만 있고 배럴에 없다. 조합 코드는 배럴만 import할 수 있으므로 결과를 손으로 만들게 되고, 그러면 첫 실패나 첫 종료에서 PROTOCOL_MISMATCH·MAPPING_CONTRACT_VIOLATION으로 재시도 없이 끝난다. 이 규칙은 주석에도 없었고, reconnect 경계의 동결 검사 3곳을 지워도(MU4) 테스트가 모두 통과했다. 고른 방법: 리뷰 6.2절의 (b). 배럴 src/adapters/realtime/index.ts에서 두 함수를 export하고(추가만, 기존 시그니처 변화 없음), 엄격한 경계의 타입 주석(RealtimeReconnectOutcome, RealtimeReconnectSession, confirmTransportBarrier, handoff apply·establishAuthoritativeCheckpoint, SSE onOpen·onEvent, WS onEvent·onSubscribed)과 생성 함수 주석에 규칙을 적었다. 동결 규칙 자체와 DEFAULT_RETRYABLE 기본값은 그대로다. 버린 대안: - (c) 엄격한 경계도 stream coordinator처럼 한 번 복사하기. 규칙이 하나가 되지만 "검증한 실패 객체를 그대로 돌려준다"(toBe) 계약을 깬다. - (d) 생성 함수를 application port(shared.ts)에 두기. feature 코드도 쓸 수 있게 되지만 application 계층에 런타임 헬퍼가 는다. handoff apply를 feature가 구현한다고 정해지면 그때 옮긴다. - 런타임 오류 이름을 따로 두기(예: RESULT_NOT_FROZEN). failure kind 어휘가 바뀌는 계약 변경이라 이번 범위 밖이다. 위험·한계: - 동결하지 않은 결과는 여전히 컴파일된다. 타입으로 막으려면 brand가 필요하고, 그건 공개 타입을 깨는 변경이다. 주석과 테스트로만 알린다. - realtime 레시피 번들 40,863 -> 40,879 / 40,900 gzip B(+16 B, 여유 21 B). 예산은 그대로다. 검증: - RED: 새 테스트 "retries a failure built with the barrel's realtimeFailure"가 수정 전 `TypeError: realtimeFailure is not a function`으로 실패(1 failed | 43 passed). - 새 테스트 4건(시도의 평범한 실패, 동결된 실패 안의 평범한 error, classifyClosed의 평범한 성공 -> PROTOCOL_MISMATCH, 재시도 없음)과 handoff apply의 평범한 성공 -> MAPPING_CONTRACT_VIOLATION 1건. - MU4(reconnect 동결 검사 3곳 제거)를 다시 넣으면 새 테스트 3건이 실패한다(수정 전에는 생존). - GREEN: `node scripts/run-vitest.ts run tests/unit/realtime/` -> 14파일 230건 통과. - `npx tsc --project tsconfig.app.json --noEmit`, `npx tsc --project tsconfig.test.json --noEmit` -> 오류 0. - `npx eslint <바꾼 파일 8개> --max-warnings=0` -> 통과. - `node scripts/check-realtime-boundaries.ts` -> PASS, `node scripts/check-adapter-inventory.ts` -> PASS, `node scripts/check-optional-recipes.ts` -> PASS(realtime 40879/40900). Refs: docs/reviews/2026-11-01-maturity-review/04-adapters-realtime.md 04-01, 6.2절
2343 lines
68 KiB
TypeScript
2343 lines
68 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { ClockPort } from "../../../src/application/ports/clock-port.ts";
|
|
import type {
|
|
RealtimeResult,
|
|
} from "../../../src/application/ports/realtime/shared.ts";
|
|
import {
|
|
REALTIME_TRANSPORT_CONTINUE,
|
|
type RealtimeRecoveryCheckpoint,
|
|
} from "../../../src/application/ports/realtime/event-authority.ts";
|
|
import {
|
|
createRealtimeReconnectCoordinator,
|
|
type RealtimeCommittedRecovery,
|
|
type RealtimeReconnectAttemptContext,
|
|
type RealtimeReconnectAttemptSuccess,
|
|
type RealtimeReconnectEnvironment,
|
|
type RealtimeReconnectObservation,
|
|
type RealtimeReconnectOutcome,
|
|
type RealtimeReconnectSession,
|
|
type RealtimeRecoveryReconnectDirective,
|
|
realtimeFailure as publicRealtimeFailure,
|
|
realtimeSuccess as publicRealtimeSuccess,
|
|
} from "../../../src/adapters/realtime/index.ts";
|
|
import {
|
|
defineReconnectPolicy,
|
|
REALTIME_RECONNECT_CEILINGS,
|
|
type ReconnectPolicy,
|
|
} from "../../../src/adapters/realtime/reconnect-policy.ts";
|
|
import {
|
|
realtimeFailure,
|
|
realtimeSuccess,
|
|
} from "../../../src/adapters/realtime/result.ts";
|
|
import {
|
|
createFetchSseConnection,
|
|
readSseForReconnect,
|
|
SSE_CONTINUE,
|
|
type FetchSseClosedOutcome,
|
|
} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts";
|
|
import {
|
|
createWebSocketConnection,
|
|
type WebSocketClosedReceipt,
|
|
type WebSocketFacade,
|
|
} from "../../../src/adapters/realtime/websocket/websocket-connection.ts";
|
|
import {
|
|
REALTIME_WEBSOCKET_PROTOCOL,
|
|
} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts";
|
|
|
|
type Sleeper = Readonly<{
|
|
dueAt: number;
|
|
signal?: AbortSignal;
|
|
resolve(): void;
|
|
reject(): void;
|
|
}>;
|
|
|
|
class ManualClock implements ClockPort {
|
|
current = 0;
|
|
readonly sleepCalls: number[] = [];
|
|
readonly sleepers: Sleeper[] = [];
|
|
|
|
now(): number {
|
|
return this.current;
|
|
}
|
|
|
|
sleep(
|
|
milliseconds: number,
|
|
signal?: AbortSignal,
|
|
): Promise<void> {
|
|
this.sleepCalls.push(milliseconds);
|
|
return new Promise((resolve, reject) => {
|
|
if (signal?.aborted) {
|
|
reject(new Error("Aborted."));
|
|
return;
|
|
}
|
|
let onAbort: (() => void) | undefined;
|
|
const sleeper: Sleeper = {
|
|
dueAt: this.current + milliseconds,
|
|
signal,
|
|
resolve: () => {
|
|
signal?.removeEventListener("abort", onAbort!);
|
|
resolve();
|
|
},
|
|
reject: () => {
|
|
signal?.removeEventListener("abort", onAbort!);
|
|
reject(new Error("Aborted."));
|
|
},
|
|
};
|
|
onAbort = () => {
|
|
this.remove(sleeper);
|
|
sleeper.reject();
|
|
};
|
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
this.sleepers.push(sleeper);
|
|
});
|
|
}
|
|
|
|
advance(milliseconds: number): void {
|
|
this.current += milliseconds;
|
|
const ready = this.sleepers
|
|
.filter((sleeper) => sleeper.dueAt <= this.current)
|
|
.sort((left, right) => left.dueAt - right.dueAt);
|
|
for (const sleeper of ready) {
|
|
this.remove(sleeper);
|
|
sleeper.resolve();
|
|
}
|
|
}
|
|
|
|
private remove(target: Sleeper): void {
|
|
const index = this.sleepers.indexOf(target);
|
|
if (index >= 0) this.sleepers.splice(index, 1);
|
|
}
|
|
}
|
|
|
|
class MutableOnlineEnvironment
|
|
implements RealtimeReconnectEnvironment
|
|
{
|
|
readonly listeners = new Set<(online: boolean) => void>();
|
|
|
|
constructor(public connected: boolean) {}
|
|
|
|
online(): boolean {
|
|
return this.connected;
|
|
}
|
|
|
|
subscribeOnline(
|
|
listener: (online: boolean) => void,
|
|
): () => void {
|
|
this.listeners.add(listener);
|
|
return () => this.listeners.delete(listener);
|
|
}
|
|
|
|
emit(online: boolean): void {
|
|
this.connected = online;
|
|
for (const listener of this.listeners) listener(online);
|
|
}
|
|
}
|
|
|
|
function policy(
|
|
override: Partial<ReconnectPolicy> = {},
|
|
): ReconnectPolicy {
|
|
return defineReconnectPolicy({
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 1_000,
|
|
maxAttempts: 2,
|
|
maxElapsedMs: 5_000,
|
|
stableOpenMs: 500,
|
|
...override,
|
|
});
|
|
}
|
|
|
|
function failed(
|
|
result: Extract<RealtimeResult<never>, { ok: false }>,
|
|
serverNotBeforeMs?: number,
|
|
): RealtimeReconnectOutcome<never> {
|
|
return Object.freeze({
|
|
result,
|
|
...(serverNotBeforeMs === undefined
|
|
? {}
|
|
: { serverNotBeforeMs }),
|
|
});
|
|
}
|
|
|
|
function opened<ClosedReceipt>(
|
|
session: RealtimeReconnectSession<ClosedReceipt>,
|
|
establishedRecoveryBarrier?: RealtimeCommittedRecovery,
|
|
): RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<ClosedReceipt>
|
|
> {
|
|
return Object.freeze({
|
|
result: realtimeSuccess(
|
|
Object.freeze({
|
|
session,
|
|
...(establishedRecoveryBarrier === undefined
|
|
? {}
|
|
: { establishedRecoveryBarrier }),
|
|
}),
|
|
),
|
|
});
|
|
}
|
|
|
|
function closedSuccessfully(): RealtimeReconnectOutcome<void> {
|
|
return Object.freeze({
|
|
result: realtimeSuccess(undefined),
|
|
});
|
|
}
|
|
|
|
function confirmationSucceeded(): RealtimeResult<void> {
|
|
return realtimeSuccess(undefined);
|
|
}
|
|
|
|
function recoveryToken(
|
|
sequence = "7",
|
|
): RealtimeCommittedRecovery {
|
|
const checkpoint = Object.freeze({
|
|
lastAppliedSequence: sequence,
|
|
recoveryMode: "CURSOR",
|
|
resumeCursor: `cursor-${sequence}`,
|
|
streamEpoch: "epoch-1",
|
|
}) as unknown as RealtimeRecoveryCheckpoint;
|
|
return Object.freeze({
|
|
kind: "RECOVERY_COMMITTED",
|
|
streamId: "orders",
|
|
checkpoint,
|
|
}) as unknown as RealtimeCommittedRecovery;
|
|
}
|
|
|
|
function recoveryDirective(
|
|
recovery: RealtimeCommittedRecovery,
|
|
terminalResult: Extract<
|
|
RealtimeResult<never>,
|
|
{ ok: false }
|
|
>,
|
|
serverNotBeforeMs?: number,
|
|
): RealtimeRecoveryReconnectDirective {
|
|
return Object.freeze({
|
|
kind: "RECOVERY_RECONNECT",
|
|
recovery,
|
|
terminalResult,
|
|
...(serverNotBeforeMs === undefined
|
|
? {}
|
|
: { serverNotBeforeMs }),
|
|
});
|
|
}
|
|
|
|
function deferred<Value>(): Readonly<{
|
|
promise: Promise<Value>;
|
|
resolve(value: Value): void;
|
|
}> {
|
|
let resolvePromise: ((value: Value) => void) | undefined;
|
|
const promise = new Promise<Value>((resolve) => {
|
|
resolvePromise = resolve;
|
|
});
|
|
return Object.freeze({
|
|
promise,
|
|
resolve(value) {
|
|
resolvePromise?.(value);
|
|
},
|
|
});
|
|
}
|
|
|
|
async function flush(): Promise<void> {
|
|
// Enough turns for a retry loop iteration: sleep, attempt and close races.
|
|
for (let turn = 0; turn < 16; turn += 1) {
|
|
await Promise.resolve();
|
|
}
|
|
}
|
|
|
|
async function flushUntil(
|
|
condition: () => boolean,
|
|
): Promise<void> {
|
|
for (let turn = 0; turn < 50; turn += 1) {
|
|
if (condition()) return;
|
|
await Promise.resolve();
|
|
}
|
|
}
|
|
|
|
describe("realtime reconnect coordinator", () => {
|
|
it("owns one bounded retry loop and returns the exact last failure", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const failures = [
|
|
realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true),
|
|
realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true),
|
|
realtimeFailure("IDLE_TIMEOUT", "RECEIVE", true),
|
|
] as const;
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
const next = failures[attempts++]!;
|
|
return failed(
|
|
next,
|
|
next === failures[1] ? 0 : undefined,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
await flush();
|
|
expect(attempts).toBe(2);
|
|
expect(clock.sleepCalls).toEqual([50, 100]);
|
|
|
|
clock.advance(100);
|
|
const result = await running;
|
|
|
|
expect(attempts).toBe(3);
|
|
expect(result).toBe(failures[2]);
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
|
|
it("does not clamp a server not-before value past elapsed budget", async () => {
|
|
const clock = new ManualClock();
|
|
const terminal = realtimeFailure(
|
|
"RATE_LIMITED",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const random = vi.fn(() => 0);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({
|
|
maxAttempts: 1,
|
|
maxElapsedMs: 500,
|
|
}),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => failed(terminal, 500),
|
|
});
|
|
|
|
const result = await coordinator.run();
|
|
|
|
expect(result).toBe(terminal);
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
expect(random).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each([
|
|
"RATE_LIMITED",
|
|
"PROVIDER_UNAVAILABLE",
|
|
] as const)(
|
|
"reconnects a retryable external %s attempt failure without a server hint",
|
|
async (kind) => {
|
|
const clock = new ManualClock();
|
|
const retryable = realtimeFailure(kind, "CONNECT", true);
|
|
const terminal = realtimeFailure(kind, "CONNECT", false);
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
failed(++attempts === 1 ? retryable : terminal),
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
|
|
expect(attempts).toBe(2);
|
|
expect(result).toBe(terminal);
|
|
},
|
|
);
|
|
|
|
it("reconnects a retryable external close failure without a server hint", async () => {
|
|
const clock = new ManualClock();
|
|
const retryable = realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"RECEIVE",
|
|
true,
|
|
);
|
|
const terminal = realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"RECEIVE",
|
|
false,
|
|
);
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
const closedWith = ++attempts === 1 ? retryable : terminal;
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => failed(closedWith),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
|
|
expect(attempts).toBe(2);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it("waits for an explicit online signal without a retry timer", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(false);
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
const attempt = vi.fn(() => failed(terminal));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
clock.advance(100);
|
|
await flush();
|
|
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
|
|
environment.emit(true);
|
|
await flush();
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
expect(attempt).toHaveBeenCalledTimes(1);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it("does not spend the elapsed budget while waiting offline", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(false);
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
const attempt = vi.fn(() => failed(terminal));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
// Offline for longer than maxElapsedMs before the first attempt.
|
|
clock.advance(6_000);
|
|
environment.emit(true);
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
expect(attempt).toHaveBeenCalledTimes(1);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it.each(["an offline blip", "a waitClosed throw"] as const)(
|
|
"reconnects after %s ends a healthy session older than the elapsed budget",
|
|
async (ending) => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const closing = deferred<RealtimeReconnectOutcome<void>>();
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context: RealtimeReconnectAttemptContext) => {
|
|
attempts += 1;
|
|
if (attempts > 1) return failed(terminal);
|
|
context.markValidHeartbeatOrEvent();
|
|
return opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: () =>
|
|
closing.resolve(
|
|
failed(
|
|
realtimeFailure("OFFLINE", "RECEIVE", true),
|
|
),
|
|
),
|
|
waitClosed: () =>
|
|
ending === "an offline blip"
|
|
? closing.promise
|
|
: closing.promise.then(() => {
|
|
throw new Error("transport failed");
|
|
}),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => attempts === 1);
|
|
await flush();
|
|
// Healthy for twice maxElapsedMs, then a one second outage.
|
|
clock.advance(10_000);
|
|
if (ending === "an offline blip") {
|
|
environment.emit(false);
|
|
await flush();
|
|
clock.advance(1_000);
|
|
environment.emit(true);
|
|
} else {
|
|
closing.resolve(closedSuccessfully());
|
|
}
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
expect(attempts).toBe(2);
|
|
expect(result).toBe(terminal);
|
|
},
|
|
);
|
|
|
|
it("observes attempts, retries, the session and why the run ended", async () => {
|
|
const clock = new ManualClock();
|
|
const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false);
|
|
const observations: unknown[] = [];
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
observe: (observation) => {
|
|
observations.push(observation);
|
|
throw new Error("sink failure must not change the run");
|
|
},
|
|
attempt: () =>
|
|
++attempts === 1
|
|
? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true))
|
|
: opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => failed(terminal),
|
|
}),
|
|
),
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(50);
|
|
await expect(running).resolves.toBe(terminal);
|
|
expect(observations).toEqual([
|
|
{ kind: "ATTEMPT_STARTED" },
|
|
{
|
|
kind: "RETRY_SCHEDULED",
|
|
failureKind: "CONNECT_TIMEOUT",
|
|
delayBucket: "UNDER_1S",
|
|
},
|
|
{ kind: "ATTEMPT_STARTED" },
|
|
{ kind: "SESSION_OPENED" },
|
|
{ kind: "SESSION_CLOSED" },
|
|
{ kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" },
|
|
]);
|
|
|
|
// A second run that spends its retries ends as EXHAUSTED, not TERMINAL.
|
|
const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true);
|
|
const ends: unknown[] = [];
|
|
const spending = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
observe: (observation) => {
|
|
if (observation.kind === "RUN_ENDED") ends.push(observation);
|
|
},
|
|
attempt: () => failed(exhausted),
|
|
});
|
|
const spent = spending.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
clock.advance(0);
|
|
await expect(spent).resolves.toBe(exhausted);
|
|
expect(ends).toEqual([
|
|
{
|
|
kind: "RUN_ENDED",
|
|
reason: "EXHAUSTED",
|
|
failureKind: "PROVIDER_UNAVAILABLE",
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("resets a spent retry only after a valid signal and a stable open", async () => {
|
|
const runScenario = async (markValid: boolean, openMs: number) => {
|
|
const clock = new ManualClock();
|
|
const firstFailure = realtimeFailure(
|
|
"CONNECT_TIMEOUT",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const sessionFailure = realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"RECEIVE",
|
|
true,
|
|
);
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const closing = deferred<RealtimeReconnectOutcome<void>>();
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (
|
|
context: RealtimeReconnectAttemptContext,
|
|
) => {
|
|
attempts += 1;
|
|
if (attempts === 1) return failed(firstFailure);
|
|
if (attempts === 2) {
|
|
if (markValid) {
|
|
context.markValidHeartbeatOrEvent();
|
|
}
|
|
return opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed: () => closing.promise,
|
|
}),
|
|
);
|
|
}
|
|
return failed(terminal);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
clock.advance(0);
|
|
await flushUntil(() => attempts === 2);
|
|
await flush();
|
|
clock.advance(openMs);
|
|
closing.resolve(failed(sessionFailure, 0));
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
return {
|
|
attempts,
|
|
closeSession,
|
|
result,
|
|
sessionFailure,
|
|
terminal,
|
|
};
|
|
};
|
|
|
|
// A heartbeat on a short-lived connection, or a quiet long one, keeps
|
|
// the spent budget; only both together reset it.
|
|
for (const [markValid, openMs] of [
|
|
[false, 0],
|
|
[true, 499],
|
|
[false, 500],
|
|
] as const) {
|
|
const spent = await runScenario(markValid, openMs);
|
|
expect(spent.attempts).toBe(2);
|
|
expect(spent.result).toBe(spent.sessionFailure);
|
|
expect(spent.closeSession).toHaveBeenCalledOnce();
|
|
}
|
|
|
|
const eligible = await runScenario(true, 500);
|
|
expect(eligible.attempts).toBe(3);
|
|
expect(eligible.result).toBe(eligible.terminal);
|
|
expect(eligible.closeSession).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("does not reconnect before the session's authoritative close result", async () => {
|
|
const clock = new ManualClock();
|
|
const closure =
|
|
deferred<RealtimeReconnectOutcome<void>>();
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"RECEIVE",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const attempt = vi.fn(() =>
|
|
opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed: () => closure.promise,
|
|
}),
|
|
),
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
let settled = false;
|
|
|
|
const running = coordinator.run().then((result) => {
|
|
settled = true;
|
|
return result;
|
|
});
|
|
await flush();
|
|
|
|
expect(attempt).toHaveBeenCalledOnce();
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
expect(settled).toBe(false);
|
|
expect(coordinator.getState()).toBe("RUNNING");
|
|
|
|
closure.resolve(failed(terminal));
|
|
const result = await running;
|
|
expect(result).toBe(terminal);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("carries an exact recovery token into the next attempt and opens only after confirmation", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const directive = recoveryDirective(recovery, terminal);
|
|
const completed = closedSuccessfully();
|
|
const confirmation = realtimeSuccess(undefined);
|
|
const confirmTransportBarrier = vi.fn(
|
|
(received: RealtimeCommittedRecovery) => {
|
|
expect(received).toBe(recovery);
|
|
return confirmation;
|
|
},
|
|
);
|
|
let attempts = 0;
|
|
let gateSettled = false;
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
type RecoveryCloseReceipt =
|
|
| RealtimeRecoveryReconnectDirective
|
|
| RealtimeReconnectOutcome<void>;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier,
|
|
attempt: (context) => {
|
|
attempts += 1;
|
|
if (attempts === 1) {
|
|
expect(context.pendingRecovery).toBeNull();
|
|
return opened<RecoveryCloseReceipt>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RecoveryCloseReceipt,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => directive,
|
|
}),
|
|
);
|
|
}
|
|
|
|
expect(context.pendingRecovery).toBe(recovery);
|
|
const gate =
|
|
context.waitForRecoveryBarrierConfirmation();
|
|
void gate.then((result) => {
|
|
gateSettled = true;
|
|
gateResult = result;
|
|
});
|
|
expect(gateSettled).toBe(false);
|
|
return opened<RecoveryCloseReceipt>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RecoveryCloseReceipt,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => completed,
|
|
}),
|
|
recovery,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
expect(clock.sleepCalls).toEqual([0]);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(attempts).toBe(2);
|
|
expect(confirmTransportBarrier).toHaveBeenCalledOnce();
|
|
expect(gateSettled).toBe(true);
|
|
expect(gateResult).toBe(confirmation);
|
|
expect(result).toBe(completed.result);
|
|
});
|
|
|
|
it.each([
|
|
["missing", undefined],
|
|
["cloned", recoveryToken()],
|
|
] as const)(
|
|
"rejects a %s recovery proof and closes the new session",
|
|
async (_case, proof) => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const waitClosed = vi.fn(closedSuccessfully);
|
|
const confirmTransportBarrier = vi.fn(
|
|
confirmationSucceeded,
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier,
|
|
attempt: (context) => {
|
|
expect(context.pendingRecovery).toBe(recovery);
|
|
const session = Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed,
|
|
});
|
|
return proof === undefined
|
|
? opened(session)
|
|
: opened(session, proof);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"RECOVER",
|
|
false,
|
|
),
|
|
);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
expect(waitClosed).not.toHaveBeenCalled();
|
|
expect(confirmTransportBarrier).not.toHaveBeenCalled();
|
|
},
|
|
);
|
|
|
|
it("settles the readiness gate and run with the exact confirmation failure", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const confirmationFailure = realtimeFailure(
|
|
"SCOPE_PROTOCOL_VIOLATION",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const waitClosed = vi.fn(closedSuccessfully);
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: () => confirmationFailure,
|
|
attempt: (context) => {
|
|
void context
|
|
.waitForRecoveryBarrierConfirmation()
|
|
.then((result) => {
|
|
gateResult = result;
|
|
});
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed,
|
|
}),
|
|
recovery,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(result).toBe(confirmationFailure);
|
|
expect(gateResult).toBe(confirmationFailure);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
expect(waitClosed).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("retains the exact recovery token across retryable attempt failures and returns its terminal result on exhaustion", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const attemptFailure = realtimeFailure(
|
|
"CONNECT_TIMEOUT",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const gateResults: RealtimeResult<void>[] = [];
|
|
const seenRecoveries: Array<
|
|
RealtimeCommittedRecovery | null
|
|
> = [];
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 2 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context) => {
|
|
seenRecoveries.push(context.pendingRecovery);
|
|
void context
|
|
.waitForRecoveryBarrierConfirmation()
|
|
.then((result) => gateResults.push(result));
|
|
return failed(attemptFailure);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(seenRecoveries).toEqual([recovery, recovery]);
|
|
expect(gateResults).toEqual([
|
|
attemptFailure,
|
|
attemptFailure,
|
|
]);
|
|
expect(gateResults[0]).toBe(attemptFailure);
|
|
expect(gateResults[1]).toBe(attemptFailure);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it("fails a no-pending gate immediately and rejects any unsolicited proof", async () => {
|
|
const proof = recoveryToken();
|
|
const closeSession = vi.fn();
|
|
const confirmTransportBarrier = vi.fn(
|
|
confirmationSucceeded,
|
|
);
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier,
|
|
attempt: async (context) => {
|
|
gateResult =
|
|
await context.waitForRecoveryBarrierConfirmation();
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
proof,
|
|
);
|
|
},
|
|
});
|
|
|
|
const result = await coordinator.run();
|
|
|
|
expect(gateResult).toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"RECOVER",
|
|
false,
|
|
),
|
|
);
|
|
expect(result).toBe(gateResult);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
expect(confirmTransportBarrier).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("applies retry ceilings to an initial recovery and preserves its exact terminal result", async () => {
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const attempt = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({
|
|
maxAttempts: 1,
|
|
maxElapsedMs: 500,
|
|
}),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const result = await coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
500,
|
|
),
|
|
});
|
|
|
|
expect(result).toBe(terminal);
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("settles a pending readiness gate when the run is aborted", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context) => {
|
|
void context
|
|
.waitForRecoveryBarrierConfirmation()
|
|
.then((result) => {
|
|
gateResult = result;
|
|
});
|
|
return attemptResult.promise;
|
|
},
|
|
});
|
|
const controller = new AbortController();
|
|
|
|
const running = coordinator.run({
|
|
signal: controller.signal,
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
await flush();
|
|
controller.abort();
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
expect(gateResult).toEqual(
|
|
realtimeFailure("ABORTED", "RECOVER", false),
|
|
);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
recovery,
|
|
),
|
|
);
|
|
await flush();
|
|
});
|
|
|
|
it("rejects accessors without invoking them and consumes proxy values only from one descriptor snapshot", async () => {
|
|
let getterReads = 0;
|
|
const accessorOutcome = Object.defineProperty({}, "result", {
|
|
enumerable: true,
|
|
get() {
|
|
getterReads += 1;
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
});
|
|
const accessorCoordinator =
|
|
createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
accessorOutcome as unknown as RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<unknown>
|
|
>,
|
|
});
|
|
|
|
await expect(accessorCoordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(getterReads).toBe(0);
|
|
|
|
const completed = closedSuccessfully();
|
|
const session = Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => completed,
|
|
});
|
|
const rawOutcome = opened(session);
|
|
const proxyReads: PropertyKey[] = [];
|
|
const proxyOutcome = new Proxy(rawOutcome, {
|
|
get(_target, key) {
|
|
proxyReads.push(key);
|
|
if (key === "then") return undefined;
|
|
throw new Error("raw property reread");
|
|
},
|
|
});
|
|
const proxyCoordinator =
|
|
createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => proxyOutcome,
|
|
});
|
|
|
|
await expect(proxyCoordinator.run()).resolves.toBe(
|
|
completed.result,
|
|
);
|
|
expect(proxyReads).toEqual(["then"]);
|
|
});
|
|
|
|
it("rechecks scope after attempt and waitClosed boundaries before accepting callbacks", async () => {
|
|
let attemptScopeCurrent = true;
|
|
const lateAttemptClose = vi.fn();
|
|
const lateAttemptWaitClosed = vi.fn(closedSuccessfully);
|
|
const lateAttemptCoordinator =
|
|
createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
isCurrent: () => attemptScopeCurrent,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attemptScopeCurrent = false;
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateAttemptClose,
|
|
waitClosed: lateAttemptWaitClosed,
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
await expect(lateAttemptCoordinator.run()).resolves.toEqual(
|
|
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
|
|
);
|
|
expect(lateAttemptClose).toHaveBeenCalledOnce();
|
|
expect(lateAttemptWaitClosed).not.toHaveBeenCalled();
|
|
|
|
let closeScopeCurrent = true;
|
|
const classifyClosed = vi.fn(
|
|
(receipt: RealtimeReconnectOutcome<void>) => receipt,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const closeCoordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
isCurrent: () => closeScopeCurrent,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed,
|
|
close: closeSession,
|
|
waitClosed: () => {
|
|
closeScopeCurrent = false;
|
|
return closedSuccessfully();
|
|
},
|
|
}),
|
|
),
|
|
});
|
|
|
|
await expect(closeCoordinator.run()).resolves.toEqual(
|
|
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
|
|
);
|
|
expect(classifyClosed).not.toHaveBeenCalled();
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("rejects extra and missing outcome keys through the common result codec", async () => {
|
|
const validSession = Object.freeze({
|
|
classifyClosed: closedSuccessfully,
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
});
|
|
const invalidOutcomes = [
|
|
{
|
|
extra: true,
|
|
result: realtimeSuccess(validSession),
|
|
},
|
|
{
|
|
result: { ok: true },
|
|
},
|
|
{
|
|
result: {
|
|
ok: false,
|
|
error: {
|
|
extra: true,
|
|
kind: "FORBIDDEN",
|
|
operation: "CONNECT",
|
|
retryable: false,
|
|
},
|
|
},
|
|
},
|
|
];
|
|
|
|
for (const invalid of invalidOutcomes) {
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
invalid as unknown as RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<unknown>
|
|
>,
|
|
});
|
|
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
}
|
|
});
|
|
|
|
it("rechecks abort immediately after listener registration", async () => {
|
|
let abortReads = 0;
|
|
let listener: (() => void) | undefined;
|
|
const signal = {
|
|
get aborted() {
|
|
abortReads += 1;
|
|
return abortReads >= 2;
|
|
},
|
|
addEventListener(
|
|
type: string,
|
|
nextListener: () => void,
|
|
) {
|
|
if (type === "abort") listener = nextListener;
|
|
},
|
|
removeEventListener() {
|
|
listener = undefined;
|
|
},
|
|
} as unknown as AbortSignal;
|
|
const attempt = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const result = await coordinator.run({ signal });
|
|
|
|
expect(listener).toBeUndefined();
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
});
|
|
|
|
it("bounds a non-cooperative retry sleep after its phase is aborted", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const sleep = vi.fn(
|
|
() => new Promise<void>(() => undefined),
|
|
);
|
|
const clock: ClockPort = Object.freeze({
|
|
now: () => 0,
|
|
sleep,
|
|
});
|
|
const retryable = realtimeFailure(
|
|
"CONNECT_TIMEOUT",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => failed(retryable),
|
|
});
|
|
|
|
try {
|
|
const running = coordinator.run();
|
|
await flushUntil(() => sleep.mock.calls.length === 1);
|
|
environment.emit(false);
|
|
await flush();
|
|
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
|
|
);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("bounds a non-cooperative attempt and closes its late successful session before leaving DRAINING", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
const lateClose = vi.fn();
|
|
const lateWaitClosed = vi.fn(closedSuccessfully);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => attemptResult.promise,
|
|
});
|
|
|
|
try {
|
|
const running = coordinator.run();
|
|
await flush();
|
|
environment.emit(false);
|
|
await flush();
|
|
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
|
|
);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateClose,
|
|
waitClosed: lateWaitClosed,
|
|
}),
|
|
),
|
|
);
|
|
await flush();
|
|
|
|
expect(lateClose).toHaveBeenCalledOnce();
|
|
expect(lateWaitClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("cancels the drain timer on abort and keeps the late generation fenced", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
const lateClose = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => attemptResult.promise,
|
|
});
|
|
const controller = new AbortController();
|
|
|
|
try {
|
|
const running = coordinator.run({
|
|
signal: controller.signal,
|
|
});
|
|
await flush();
|
|
environment.emit(false);
|
|
await flush();
|
|
expect(vi.getTimerCount()).toBe(1);
|
|
|
|
controller.abort();
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateClose,
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
),
|
|
);
|
|
await flush();
|
|
|
|
expect(lateClose).toHaveBeenCalledOnce();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("keeps a healthy waitClosed unbounded but bounds its post-abort drain", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const closure =
|
|
deferred<RealtimeReconnectOutcome<void>>();
|
|
const classifyClosed = vi.fn(
|
|
(receipt: RealtimeReconnectOutcome<void>) => receipt,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed,
|
|
close: closeSession,
|
|
waitClosed: () => closure.promise,
|
|
}),
|
|
),
|
|
});
|
|
|
|
try {
|
|
let settled = false;
|
|
const running = coordinator.run().then((result) => {
|
|
settled = true;
|
|
return result;
|
|
});
|
|
await flush();
|
|
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs * 2,
|
|
);
|
|
expect(settled).toBe(false);
|
|
expect(coordinator.getState()).toBe("RUNNING");
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
environment.emit(false);
|
|
await flush();
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
|
|
);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(closeSession).toHaveBeenCalled();
|
|
expect(classifyClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
closure.resolve(closedSuccessfully());
|
|
await flush();
|
|
|
|
expect(classifyClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("uses waitClosed as authority and fences an aborted late generation", async () => {
|
|
const clock = new ManualClock();
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
let captured:
|
|
| RealtimeReconnectAttemptContext
|
|
| undefined;
|
|
const lateClose = vi.fn();
|
|
const lateWaitClosed = vi.fn(closedSuccessfully);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context) => {
|
|
captured = context;
|
|
return attemptResult.promise;
|
|
},
|
|
});
|
|
const controller = new AbortController();
|
|
|
|
const running = coordinator.run({ signal: controller.signal });
|
|
await flush();
|
|
expect(captured?.isCurrent()).toBe(true);
|
|
|
|
controller.abort();
|
|
const result = await running;
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
expect(captured?.isCurrent()).toBe(false);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateClose,
|
|
waitClosed: lateWaitClosed,
|
|
}),
|
|
),
|
|
);
|
|
await flush();
|
|
|
|
expect(lateClose).toHaveBeenCalledOnce();
|
|
expect(lateWaitClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
});
|
|
|
|
class FakeWebSocket implements WebSocketFacade {
|
|
protocol: string = REALTIME_WEBSOCKET_PROTOCOL;
|
|
readyState = 0;
|
|
bufferedAmount = 0;
|
|
readonly listeners = new Map<
|
|
string,
|
|
Set<(event: unknown) => void>
|
|
>();
|
|
readonly sent: string[] = [];
|
|
|
|
send(data: string): void {
|
|
if (this.readyState !== 1) throw new Error("not open");
|
|
this.sent.push(data);
|
|
}
|
|
|
|
close(): void {
|
|
this.readyState = 3;
|
|
}
|
|
|
|
addEventListener(
|
|
type: string,
|
|
listener: (event: unknown) => void,
|
|
): void {
|
|
const listeners = this.listeners.get(type) ?? new Set();
|
|
listeners.add(listener);
|
|
this.listeners.set(type, listeners);
|
|
}
|
|
|
|
removeEventListener(
|
|
type: string,
|
|
listener: (event: unknown) => void,
|
|
): void {
|
|
this.listeners.get(type)?.delete(listener);
|
|
}
|
|
|
|
accept(): void {
|
|
this.readyState = 1;
|
|
this.emit("open", {});
|
|
this.emit("message", {
|
|
data: JSON.stringify({
|
|
type: "WELCOME",
|
|
protocol: REALTIME_WEBSOCKET_PROTOCOL,
|
|
connectionId: "connection.0001",
|
|
heartbeatMs: 5_000,
|
|
heartbeatAckTimeoutMs: 1_000,
|
|
limits: {
|
|
maxFrameBytes: 64 * 1_024,
|
|
maxSubscriptions: 32,
|
|
maxInboundQueueCount: 256,
|
|
maxInboundQueueBytes: 4 * 1_024 * 1_024,
|
|
maxOutboundQueueCount: 128,
|
|
maxOutboundQueueBytes: 256 * 1_024,
|
|
maxBufferedAmountBytes: 256 * 1_024,
|
|
maxEventsPerSecond: 256,
|
|
},
|
|
}),
|
|
});
|
|
}
|
|
|
|
nativeClose(code: number): void {
|
|
this.readyState = 3;
|
|
this.emit("close", { code, reason: "" });
|
|
}
|
|
|
|
ackHeartbeats(): void {
|
|
for (const raw of this.sent.splice(0)) {
|
|
const frame = JSON.parse(raw) as { type: string; nonce: string };
|
|
if (frame.type !== "HEARTBEAT") continue;
|
|
this.emit("message", {
|
|
data: JSON.stringify({
|
|
type: "HEARTBEAT_ACK",
|
|
protocol: REALTIME_WEBSOCKET_PROTOCOL,
|
|
nonce: frame.nonce,
|
|
}),
|
|
});
|
|
}
|
|
}
|
|
|
|
private emit(type: string, event: unknown): void {
|
|
for (const listener of [...(this.listeners.get(type) ?? [])]) {
|
|
listener(event);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The smallest SSE bridge design 6.3 leaves to composition: the attempt ends
|
|
* at `onOpen` and the rest of the read is the session.
|
|
*/
|
|
function sseAttempt(fetcher: typeof fetch, clock: ManualClock) {
|
|
return async (context: RealtimeReconnectAttemptContext) => {
|
|
const connection = createFetchSseConnection({
|
|
endpoint: "https://app.example.test/events",
|
|
applicationOrigin: "https://app.example.test",
|
|
recoveryMode: "CURSOR",
|
|
fetcher,
|
|
clock,
|
|
});
|
|
const open = deferred<"OPEN">();
|
|
const reading = readSseForReconnect(connection, {
|
|
resumeCursor: null,
|
|
signal: context.signal,
|
|
onOpen: () => {
|
|
open.resolve("OPEN");
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
|
onComment: () => context.markValidHeartbeatOrEvent(),
|
|
});
|
|
const first = await Promise.race([open.promise, reading]);
|
|
if (first !== "OPEN") {
|
|
return first.result.ok
|
|
? failed(realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false))
|
|
: (first as RealtimeReconnectOutcome<never>);
|
|
}
|
|
return opened(
|
|
Object.freeze({
|
|
waitClosed: () => reading,
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<FetchSseClosedOutcome>,
|
|
) =>
|
|
receipt.result.ok
|
|
? failed(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECEIVE", false),
|
|
)
|
|
: (receipt as RealtimeReconnectOutcome<never>),
|
|
close: () => connection.close(),
|
|
}),
|
|
);
|
|
};
|
|
}
|
|
|
|
async function advanceUntil(
|
|
clock: ManualClock,
|
|
settled: () => boolean,
|
|
): Promise<void> {
|
|
// Ten simulated seconds outlast policy()'s whole elapsed budget.
|
|
for (let elapsed = 0; elapsed < 10_000 && !settled(); elapsed += 50) {
|
|
clock.advance(50);
|
|
await flush();
|
|
}
|
|
}
|
|
|
|
function observationLabel({
|
|
kind,
|
|
failureKind,
|
|
reason,
|
|
}: RealtimeReconnectObservation): string {
|
|
return [kind, failureKind, reason].filter(Boolean).join(":");
|
|
}
|
|
|
|
describe("transport failures through the reconnect coordinator", () => {
|
|
it.each([
|
|
["NETWORK_LOST", 1_006],
|
|
["RESTART", 1_012],
|
|
] as const)(
|
|
"reconnects a WebSocket %s close that carries no server hint",
|
|
async (_category, code) => {
|
|
const clock = new ManualClock();
|
|
const sockets: FakeWebSocket[] = [];
|
|
const coordinator = createRealtimeReconnectCoordinator<
|
|
WebSocketClosedReceipt
|
|
>({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const connection = createWebSocketConnection({
|
|
origin: "https://app.example.test",
|
|
endpointPath: "/_realtime/v1",
|
|
clock,
|
|
createNonce: () => "nonce.0001",
|
|
createSocket: () => {
|
|
const socket = new FakeWebSocket();
|
|
sockets.push(socket);
|
|
return socket;
|
|
},
|
|
onEvent: async () =>
|
|
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
|
|
onRecoveryRequired: () => undefined,
|
|
});
|
|
const connected = await connection.connect(context.signal);
|
|
if (!connected.ok) return failed(connected);
|
|
return opened(
|
|
Object.freeze({
|
|
waitClosed: () => connection.waitClosed(),
|
|
classifyClosed: (receipt: WebSocketClosedReceipt) =>
|
|
Object.freeze({
|
|
result: Object.freeze({
|
|
ok: false as const,
|
|
error: receipt.error,
|
|
}),
|
|
}),
|
|
close: () => connection.close(),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => sockets.length === 1);
|
|
sockets[0]!.accept();
|
|
await flush();
|
|
sockets[0]!.nativeClose(code);
|
|
await flushUntil(() => clock.sleepCalls.includes(50));
|
|
clock.advance(50);
|
|
await flushUntil(() => sockets.length === 2);
|
|
expect(sockets).toHaveLength(2);
|
|
|
|
// OVERLOADED carries no hint either and stays terminal.
|
|
sockets[1]!.accept();
|
|
await flush();
|
|
sockets[1]!.nativeClose(1_013);
|
|
await expect(running).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "RECEIVE",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(sockets).toHaveLength(2);
|
|
},
|
|
);
|
|
|
|
it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => {
|
|
const clock = new ManualClock();
|
|
const sockets: FakeWebSocket[] = [];
|
|
let nonce = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator<
|
|
WebSocketClosedReceipt
|
|
>({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const connection = createWebSocketConnection({
|
|
origin: "https://app.example.test",
|
|
endpointPath: "/_realtime/v1",
|
|
clock,
|
|
createNonce: () => `nonce.${++nonce}`,
|
|
createSocket: () => {
|
|
const socket = new FakeWebSocket();
|
|
sockets.push(socket);
|
|
return socket;
|
|
},
|
|
onEvent: async () =>
|
|
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
|
|
onRecoveryRequired: () => undefined,
|
|
observe: (observation) => {
|
|
if (observation.kind === "HEARTBEAT_ACK") {
|
|
context.markValidHeartbeatOrEvent();
|
|
}
|
|
},
|
|
});
|
|
const connected = await connection.connect(context.signal);
|
|
if (!connected.ok) return failed(connected);
|
|
return opened(
|
|
Object.freeze({
|
|
waitClosed: () => connection.waitClosed(),
|
|
classifyClosed: (receipt: WebSocketClosedReceipt) =>
|
|
Object.freeze({
|
|
result: Object.freeze({
|
|
ok: false as const,
|
|
error: receipt.error,
|
|
}),
|
|
}),
|
|
close: () => connection.close(),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => sockets.length === 1);
|
|
sockets[0]!.accept();
|
|
await flush();
|
|
// No events, only acknowledged heartbeats, for longer than maxElapsedMs.
|
|
for (let beat = 0; beat < 2; beat += 1) {
|
|
clock.advance(5_000);
|
|
await flush();
|
|
sockets[0]!.ackHeartbeats();
|
|
clock.advance(1_000);
|
|
await flush();
|
|
}
|
|
sockets[0]!.nativeClose(1_012);
|
|
await flushUntil(() => clock.sleepCalls.includes(50));
|
|
clock.advance(50);
|
|
await flushUntil(() => sockets.length === 2);
|
|
expect(sockets).toHaveLength(2);
|
|
|
|
coordinator.close();
|
|
await expect(running).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "CLOSED" },
|
|
});
|
|
});
|
|
|
|
it.each([502, 504])(
|
|
"reconnects SSE HTTP %i that carries no server hint",
|
|
async (status) => {
|
|
const clock = new ManualClock();
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(null, { status }))
|
|
.mockResolvedValueOnce(new Response(null, { status: 403 }));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const read = await createFetchSseConnection({
|
|
endpoint: "https://app.example.test/events",
|
|
applicationOrigin: "https://app.example.test",
|
|
recoveryMode: "CURSOR",
|
|
fetcher,
|
|
clock,
|
|
}).read({
|
|
resumeCursor: null,
|
|
signal: context.signal,
|
|
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
|
});
|
|
// Only connect failures are exercised here.
|
|
return failed(
|
|
read.ok
|
|
? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false)
|
|
: read,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.includes(50));
|
|
clock.advance(50);
|
|
|
|
await expect(running).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "FORBIDDEN",
|
|
operation: "CONNECT",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(fetcher).toHaveBeenCalledTimes(2);
|
|
},
|
|
);
|
|
|
|
it.each([429, 503])(
|
|
"waits at least the SSE %i Retry-After through the reconnect bridge",
|
|
async (status) => {
|
|
const clock = new ManualClock();
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status,
|
|
headers: { "retry-after": "3" },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status,
|
|
headers: { "retry-after": "10" },
|
|
}),
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxDelayMs: 5_000, maxElapsedMs: 60_000 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const read = await readSseForReconnect(
|
|
createFetchSseConnection({
|
|
endpoint: "https://app.example.test/events",
|
|
applicationOrigin: "https://app.example.test",
|
|
recoveryMode: "CURSOR",
|
|
fetcher,
|
|
clock,
|
|
}),
|
|
{
|
|
resumeCursor: null,
|
|
signal: context.signal,
|
|
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
|
},
|
|
);
|
|
// Only connect failures are exercised here.
|
|
if (read.result.ok) {
|
|
return failed(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
}
|
|
return read as RealtimeReconnectOutcome<never>;
|
|
},
|
|
});
|
|
|
|
// 10_000 is each read's connect timeout; 3_000 is the reconnect wait.
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
expect(clock.sleepCalls).toEqual([10_000, 3_000]);
|
|
clock.advance(3_000);
|
|
|
|
// A hint above the policy's maxDelayMs still ends the run.
|
|
await expect(running).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { operation: "CONNECT", retryable: true },
|
|
});
|
|
expect(fetcher).toHaveBeenCalledTimes(2);
|
|
expect(clock.sleepCalls).toEqual([10_000, 3_000, 10_000]);
|
|
},
|
|
);
|
|
|
|
// The SSE adapter reports a rejected fetch or read as OFFLINE, but
|
|
// navigator.onLine stays true when only the server is down, so no online
|
|
// event ever arrives (0929/04-01).
|
|
it("retries an SSE fetch rejection with backoff while the browser stays online", async () => {
|
|
const clock = new ManualClock();
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockRejectedValue(new TypeError("Failed to fetch"));
|
|
const observed: string[] = [];
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: sseAttempt(fetcher, clock),
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(fetcher).toHaveBeenCalledTimes(3);
|
|
// Full-jitter backoff; 10_000 is each read's connect timeout.
|
|
expect(clock.sleepCalls.filter((ms) => ms !== 10_000)).toEqual([
|
|
50, 100,
|
|
]);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "OFFLINE", operation: "CONNECT", retryable: true },
|
|
});
|
|
expect(observed).toEqual([
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:OFFLINE",
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:OFFLINE",
|
|
"ATTEMPT_STARTED",
|
|
"RUN_ENDED:OFFLINE:EXHAUSTED",
|
|
]);
|
|
});
|
|
|
|
it("reconnects after an open SSE stream's reader rejects while the browser stays online", async () => {
|
|
const clock = new ManualClock();
|
|
let stream: ReadableStreamDefaultController<Uint8Array> | undefined;
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
stream = controller;
|
|
controller.enqueue(new TextEncoder().encode(": ping\n\n"));
|
|
},
|
|
}),
|
|
{ status: 200, headers: { "content-type": "text/event-stream" } },
|
|
),
|
|
)
|
|
.mockRejectedValue(new TypeError("Failed to fetch"));
|
|
const observed: string[] = [];
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: sseAttempt(fetcher, clock),
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await flushUntil(() => observed.includes("SESSION_OPENED"));
|
|
await flush();
|
|
// Healthy past stableOpenMs, then a proxy restart drops the stream.
|
|
clock.advance(1_000);
|
|
await flush();
|
|
stream!.error(new TypeError("network error"));
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(fetcher).toHaveBeenCalledTimes(3);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "OFFLINE", operation: "CONNECT", retryable: true },
|
|
});
|
|
expect(observed).toEqual([
|
|
"ATTEMPT_STARTED",
|
|
"SESSION_OPENED",
|
|
"SESSION_CLOSED",
|
|
"RETRY_SCHEDULED:OFFLINE",
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:OFFLINE",
|
|
"ATTEMPT_STARTED",
|
|
"RUN_ENDED:OFFLINE:EXHAUSTED",
|
|
]);
|
|
});
|
|
|
|
it.each(["before", "after"] as const)(
|
|
"waits for the online event when the browser goes offline %s the SSE fetch rejects",
|
|
async (order) => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockImplementationOnce(async () => {
|
|
if (order === "before") environment.emit(false);
|
|
throw new TypeError("Failed to fetch");
|
|
})
|
|
.mockResolvedValueOnce(new Response(null, { status: 403 }));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: sseAttempt(fetcher, clock),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await flushUntil(() => fetcher.mock.calls.length === 1);
|
|
await flush();
|
|
if (order === "after") environment.emit(false);
|
|
await flush();
|
|
// Twenty offline minutes fire any retry timer that was armed.
|
|
clock.advance(20 * 60_000);
|
|
await flush();
|
|
expect(fetcher).toHaveBeenCalledTimes(1);
|
|
expect(result).toBeNull();
|
|
|
|
environment.emit(true);
|
|
await advanceUntil(clock, () => result !== null);
|
|
expect(fetcher).toHaveBeenCalledTimes(2);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "FORBIDDEN", operation: "CONNECT", retryable: false },
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("results that composition code hands the reconnect coordinator", () => {
|
|
// Composition code may import only the adapter barrel. Each case passes a
|
|
// value that type-checks as RealtimeReconnectOutcome.
|
|
it("retries a failure built with the barrel's realtimeFailure", async () => {
|
|
const clock = new ManualClock();
|
|
const last = publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT");
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: () => publicRealtimeSuccess(undefined),
|
|
attempt: () => ({
|
|
result:
|
|
++attempts === 1
|
|
? publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT")
|
|
: last,
|
|
}),
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
clock.advance(50);
|
|
|
|
await expect(running).resolves.toBe(last);
|
|
expect(attempts).toBe(2);
|
|
});
|
|
|
|
const unfrozenFailure: RealtimeReconnectOutcome<never> = {
|
|
result: {
|
|
ok: false,
|
|
error: {
|
|
kind: "CONNECT_TIMEOUT",
|
|
operation: "CONNECT",
|
|
retryable: true,
|
|
},
|
|
},
|
|
};
|
|
const unfrozenError: RealtimeReconnectOutcome<never> = {
|
|
result: Object.freeze({
|
|
ok: false,
|
|
error: {
|
|
kind: "CONNECT_TIMEOUT",
|
|
operation: "CONNECT",
|
|
retryable: true,
|
|
},
|
|
} as const),
|
|
};
|
|
|
|
it.each([
|
|
["a plain failure object", unfrozenFailure],
|
|
["a frozen failure around a plain error", unfrozenError],
|
|
] as const)(
|
|
"ends the run without a retry when an attempt returns %s",
|
|
async (_label, outcome) => {
|
|
const clock = new ManualClock();
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attempts += 1;
|
|
return outcome;
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
await expect(running).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
expect(attempts).toBe(1);
|
|
},
|
|
);
|
|
|
|
it("ends the run when classifyClosed returns a plain success object", async () => {
|
|
const clock = new ManualClock();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (): RealtimeReconnectOutcome<void> => ({
|
|
result: { ok: true, value: undefined },
|
|
}),
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
),
|
|
});
|
|
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
});
|
|
});
|