문제: reconnect coordinator는 `attempt`·`waitClosed`가 던지면
PROVIDER_UNAVAILABLE(CONNECT·RECEIVE, retryable)로 보고 backoff로 다시
시도한다. 05330e3 뒤 poll은 같은 던짐을 PROTOCOL_MISMATCH로 끝낸다. 이
차이는 그 커밋 본문에만 있고 타입 주석과 설계 13.1에는 없다. repo의 SSE
bridge 예(`sseAttempt`)는 `attempt` 안에서 연결을 만든다. 그래서
endpoint 설정 오류가 요청 0번, attempt 3번, PROVIDER_UNAVAILABLE
(EXHAUSTED)로 끝나 서버 장애로 보인다(리뷰 실험 04-R2).
고른 방법: 리뷰 선택지 (c). 동작은 그대로 두고 규칙을 적는다.
- `attempt` 주석: 던짐은 transport 실패로 재시도된다. 설정 검사는
`attempt` 밖, 조합 코드가 transport를 만들 때 한다. poll과 다른 이유도
적었다. `waitClosed` 주석: RECEIVE로 같은 규칙.
`classifyClosed`·`confirmTransportBarrier` 주석: 던짐은
PROTOCOL_MISMATCH로 끝나고 재시도하지 않는다.
- 설계 13.1 PROVIDER_UNAVAILABLE·PROTOCOL_MISMATCH 행에 같은 규칙을 적었다.
- 테스트 bridge `sseAttempt`는 만들어질 때 연결 설정을 한 번 검사한다.
연결은 attempt마다 새로 만든다. session `close`가 연결을 끝내므로 한
연결을 여러 attempt에 쓸 수 없다.
- 규칙을 고정하는 테스트를 더했다. attempt·waitClosed 던짐은 3회 재시도 뒤
PROVIDER_UNAVAILABLE(EXHAUSTED), classifyClosed 던짐은 1회로
PROTOCOL_MISMATCH, bridge의 http endpoint는 만들 때 TypeError(fetch 0).
버린 대안:
- (b) reconnect도 던짐을 PROTOCOL_MISMATCH로 끝내기. 연결 실패를 던지는
라이브러리를 감싼 transport가 첫 실패에서 run을 끝내게 되는 공개 동작
변경(BREAKING)이다. 리뷰도 지금 동작이 회복에 유리하다고 봤다.
05330e3이 poll에서 (c)를 버린 이유("결과로 구분한다")는 이번 완료
기준에 없다.
- 연결 하나를 bridge에서 만들고 attempt마다 read만 하기. 위와 같이
session `close`가 연결을 닫아 다음 read가 CLOSED가 된다.
위험·한계:
- 동작이 같으므로 조합 코드가 `attempt` 안에서 연결을 만들면 설정 오류는
여전히 서버 장애로 보인다. 주석과 13.1은 모르고 쓰는 경우만 줄인다.
- 매핑 오류 같은 나머지 던짐도 그대로 재시도되고 원인 예외는 남지 않는다
(닫힌 관측 어휘).
- CHANGELOG 행은 이 커밋에 없다. CHANGELOG 담당이 더한다.
검증: desktop batch에서 확인.
- node scripts/run-vitest.ts run tests/unit/realtime/realtime-reconnect-coordinator.test.ts
- RED 없음. 동작을 바꾸지 않는 커밋이라 새 테스트는 부모 커밋에서도
통과한다. 규칙을 고정하는 테스트다.
- check:types, lint, verify:documentation
Refs: 1106/04-01
2554 lines
75 KiB
TypeScript
2554 lines
75 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { ClockPort } from "../../../src/application/ports/clock-port.ts";
|
|
import type {
|
|
RealtimeResult,
|
|
} from "../../../src/application/ports/realtime/shared.ts";
|
|
import {
|
|
REALTIME_TRANSPORT_CONTINUE,
|
|
type RealtimeRecoveryCheckpoint,
|
|
} from "../../../src/application/ports/realtime/event-authority.ts";
|
|
import {
|
|
createRealtimeReconnectCoordinator,
|
|
type RealtimeCommittedRecovery,
|
|
type RealtimeReconnectAttemptContext,
|
|
type RealtimeReconnectAttemptSuccess,
|
|
type RealtimeReconnectEnvironment,
|
|
type RealtimeReconnectObservation,
|
|
type RealtimeReconnectOutcome,
|
|
type RealtimeReconnectSession,
|
|
type RealtimeRecoveryReconnectDirective,
|
|
realtimeFailure as publicRealtimeFailure,
|
|
realtimeSuccess as publicRealtimeSuccess,
|
|
} from "../../../src/adapters/realtime/index.ts";
|
|
import {
|
|
defineReconnectPolicy,
|
|
REALTIME_RECONNECT_CEILINGS,
|
|
type ReconnectPolicy,
|
|
} from "../../../src/adapters/realtime/reconnect-policy.ts";
|
|
import {
|
|
realtimeFailure,
|
|
realtimeSuccess,
|
|
} from "../../../src/adapters/realtime/result.ts";
|
|
import {
|
|
createFetchSseConnection,
|
|
readSseForReconnect,
|
|
SSE_CONTINUE,
|
|
type FetchSseClosedOutcome,
|
|
} from "../../../src/adapters/realtime/sse/fetch-sse-connection.ts";
|
|
import {
|
|
createWebSocketConnection,
|
|
type WebSocketClosedReceipt,
|
|
type WebSocketFacade,
|
|
} from "../../../src/adapters/realtime/websocket/websocket-connection.ts";
|
|
import {
|
|
REALTIME_WEBSOCKET_PROTOCOL,
|
|
} from "../../../src/adapters/realtime/websocket/websocket-protocol.ts";
|
|
|
|
type Sleeper = Readonly<{
|
|
dueAt: number;
|
|
signal?: AbortSignal;
|
|
resolve(): void;
|
|
reject(): void;
|
|
}>;
|
|
|
|
class ManualClock implements ClockPort {
|
|
current = 0;
|
|
readonly sleepCalls: number[] = [];
|
|
readonly sleepers: Sleeper[] = [];
|
|
|
|
now(): number {
|
|
return this.current;
|
|
}
|
|
|
|
sleep(
|
|
milliseconds: number,
|
|
signal?: AbortSignal,
|
|
): Promise<void> {
|
|
this.sleepCalls.push(milliseconds);
|
|
return new Promise((resolve, reject) => {
|
|
if (signal?.aborted) {
|
|
reject(new Error("Aborted."));
|
|
return;
|
|
}
|
|
let onAbort: (() => void) | undefined;
|
|
const sleeper: Sleeper = {
|
|
dueAt: this.current + milliseconds,
|
|
signal,
|
|
resolve: () => {
|
|
signal?.removeEventListener("abort", onAbort!);
|
|
resolve();
|
|
},
|
|
reject: () => {
|
|
signal?.removeEventListener("abort", onAbort!);
|
|
reject(new Error("Aborted."));
|
|
},
|
|
};
|
|
onAbort = () => {
|
|
this.remove(sleeper);
|
|
sleeper.reject();
|
|
};
|
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
this.sleepers.push(sleeper);
|
|
});
|
|
}
|
|
|
|
advance(milliseconds: number): void {
|
|
this.current += milliseconds;
|
|
const ready = this.sleepers
|
|
.filter((sleeper) => sleeper.dueAt <= this.current)
|
|
.sort((left, right) => left.dueAt - right.dueAt);
|
|
for (const sleeper of ready) {
|
|
this.remove(sleeper);
|
|
sleeper.resolve();
|
|
}
|
|
}
|
|
|
|
private remove(target: Sleeper): void {
|
|
const index = this.sleepers.indexOf(target);
|
|
if (index >= 0) this.sleepers.splice(index, 1);
|
|
}
|
|
}
|
|
|
|
class MutableOnlineEnvironment
|
|
implements RealtimeReconnectEnvironment
|
|
{
|
|
readonly listeners = new Set<(online: boolean) => void>();
|
|
|
|
constructor(public connected: boolean) {}
|
|
|
|
online(): boolean {
|
|
return this.connected;
|
|
}
|
|
|
|
subscribeOnline(
|
|
listener: (online: boolean) => void,
|
|
): () => void {
|
|
this.listeners.add(listener);
|
|
return () => this.listeners.delete(listener);
|
|
}
|
|
|
|
emit(online: boolean): void {
|
|
this.connected = online;
|
|
for (const listener of this.listeners) listener(online);
|
|
}
|
|
}
|
|
|
|
function policy(
|
|
override: Partial<ReconnectPolicy> = {},
|
|
): ReconnectPolicy {
|
|
return defineReconnectPolicy({
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 1_000,
|
|
maxAttempts: 2,
|
|
maxElapsedMs: 5_000,
|
|
stableOpenMs: 500,
|
|
...override,
|
|
});
|
|
}
|
|
|
|
function failed(
|
|
result: Extract<RealtimeResult<never>, { ok: false }>,
|
|
serverNotBeforeMs?: number,
|
|
): RealtimeReconnectOutcome<never> {
|
|
return Object.freeze({
|
|
result,
|
|
...(serverNotBeforeMs === undefined
|
|
? {}
|
|
: { serverNotBeforeMs }),
|
|
});
|
|
}
|
|
|
|
function opened<ClosedReceipt>(
|
|
session: RealtimeReconnectSession<ClosedReceipt>,
|
|
establishedRecoveryBarrier?: RealtimeCommittedRecovery,
|
|
): RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<ClosedReceipt>
|
|
> {
|
|
return Object.freeze({
|
|
result: realtimeSuccess(
|
|
Object.freeze({
|
|
session,
|
|
...(establishedRecoveryBarrier === undefined
|
|
? {}
|
|
: { establishedRecoveryBarrier }),
|
|
}),
|
|
),
|
|
});
|
|
}
|
|
|
|
function closedSuccessfully(): RealtimeReconnectOutcome<void> {
|
|
return Object.freeze({
|
|
result: realtimeSuccess(undefined),
|
|
});
|
|
}
|
|
|
|
function confirmationSucceeded(): RealtimeResult<void> {
|
|
return realtimeSuccess(undefined);
|
|
}
|
|
|
|
function recoveryToken(
|
|
sequence = "7",
|
|
): RealtimeCommittedRecovery {
|
|
const checkpoint = Object.freeze({
|
|
lastAppliedSequence: sequence,
|
|
recoveryMode: "CURSOR",
|
|
resumeCursor: `cursor-${sequence}`,
|
|
streamEpoch: "epoch-1",
|
|
}) as unknown as RealtimeRecoveryCheckpoint;
|
|
return Object.freeze({
|
|
kind: "RECOVERY_COMMITTED",
|
|
streamId: "orders",
|
|
checkpoint,
|
|
}) as unknown as RealtimeCommittedRecovery;
|
|
}
|
|
|
|
function recoveryDirective(
|
|
recovery: RealtimeCommittedRecovery,
|
|
terminalResult: Extract<
|
|
RealtimeResult<never>,
|
|
{ ok: false }
|
|
>,
|
|
serverNotBeforeMs?: number,
|
|
): RealtimeRecoveryReconnectDirective {
|
|
return Object.freeze({
|
|
kind: "RECOVERY_RECONNECT",
|
|
recovery,
|
|
terminalResult,
|
|
...(serverNotBeforeMs === undefined
|
|
? {}
|
|
: { serverNotBeforeMs }),
|
|
});
|
|
}
|
|
|
|
function deferred<Value>(): Readonly<{
|
|
promise: Promise<Value>;
|
|
resolve(value: Value): void;
|
|
}> {
|
|
let resolvePromise: ((value: Value) => void) | undefined;
|
|
const promise = new Promise<Value>((resolve) => {
|
|
resolvePromise = resolve;
|
|
});
|
|
return Object.freeze({
|
|
promise,
|
|
resolve(value) {
|
|
resolvePromise?.(value);
|
|
},
|
|
});
|
|
}
|
|
|
|
async function flush(): Promise<void> {
|
|
// Enough turns for a retry loop iteration: sleep, attempt and close races.
|
|
for (let turn = 0; turn < 16; turn += 1) {
|
|
await Promise.resolve();
|
|
}
|
|
}
|
|
|
|
async function flushUntil(
|
|
condition: () => boolean,
|
|
): Promise<void> {
|
|
for (let turn = 0; turn < 50; turn += 1) {
|
|
if (condition()) return;
|
|
await Promise.resolve();
|
|
}
|
|
}
|
|
|
|
describe("realtime reconnect coordinator", () => {
|
|
it("owns one bounded retry loop and returns the exact last failure", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const failures = [
|
|
realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true),
|
|
realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true),
|
|
realtimeFailure("IDLE_TIMEOUT", "RECEIVE", true),
|
|
] as const;
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
const next = failures[attempts++]!;
|
|
return failed(
|
|
next,
|
|
next === failures[1] ? 0 : undefined,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
await flush();
|
|
expect(attempts).toBe(2);
|
|
expect(clock.sleepCalls).toEqual([50, 100]);
|
|
|
|
clock.advance(100);
|
|
const result = await running;
|
|
|
|
expect(attempts).toBe(3);
|
|
expect(result).toBe(failures[2]);
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
|
|
it("does not clamp a server not-before value past elapsed budget", async () => {
|
|
const clock = new ManualClock();
|
|
const terminal = realtimeFailure(
|
|
"RATE_LIMITED",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const random = vi.fn(() => 0);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({
|
|
maxAttempts: 1,
|
|
maxElapsedMs: 500,
|
|
}),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => failed(terminal, 500),
|
|
});
|
|
|
|
const result = await coordinator.run();
|
|
|
|
expect(result).toBe(terminal);
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
expect(random).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each([
|
|
"RATE_LIMITED",
|
|
"PROVIDER_UNAVAILABLE",
|
|
] as const)(
|
|
"reconnects a retryable external %s attempt failure without a server hint",
|
|
async (kind) => {
|
|
const clock = new ManualClock();
|
|
const retryable = realtimeFailure(kind, "CONNECT", true);
|
|
const terminal = realtimeFailure(kind, "CONNECT", false);
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
failed(++attempts === 1 ? retryable : terminal),
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
|
|
expect(attempts).toBe(2);
|
|
expect(result).toBe(terminal);
|
|
},
|
|
);
|
|
|
|
it("reconnects a retryable external close failure without a server hint", async () => {
|
|
const clock = new ManualClock();
|
|
const retryable = realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"RECEIVE",
|
|
true,
|
|
);
|
|
const terminal = realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"RECEIVE",
|
|
false,
|
|
);
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
const closedWith = ++attempts === 1 ? retryable : terminal;
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => failed(closedWith),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
|
|
expect(attempts).toBe(2);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it("waits for an explicit online signal without a retry timer", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(false);
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
const attempt = vi.fn(() => failed(terminal));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
clock.advance(100);
|
|
await flush();
|
|
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
|
|
environment.emit(true);
|
|
await flush();
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
expect(attempt).toHaveBeenCalledTimes(1);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it("does not spend the elapsed budget while waiting offline", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(false);
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
const attempt = vi.fn(() => failed(terminal));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
// Offline for longer than maxElapsedMs before the first attempt.
|
|
clock.advance(6_000);
|
|
environment.emit(true);
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
expect(attempt).toHaveBeenCalledTimes(1);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it.each(["an offline blip", "a waitClosed throw"] as const)(
|
|
"reconnects after %s ends a healthy session older than the elapsed budget",
|
|
async (ending) => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const closing = deferred<RealtimeReconnectOutcome<void>>();
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context: RealtimeReconnectAttemptContext) => {
|
|
attempts += 1;
|
|
if (attempts > 1) return failed(terminal);
|
|
context.markValidHeartbeatOrEvent();
|
|
return opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: () =>
|
|
closing.resolve(
|
|
failed(
|
|
realtimeFailure("OFFLINE", "RECEIVE", true),
|
|
),
|
|
),
|
|
waitClosed: () =>
|
|
ending === "an offline blip"
|
|
? closing.promise
|
|
: closing.promise.then(() => {
|
|
throw new Error("transport failed");
|
|
}),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => attempts === 1);
|
|
await flush();
|
|
// Healthy for twice maxElapsedMs, then a one second outage.
|
|
clock.advance(10_000);
|
|
if (ending === "an offline blip") {
|
|
environment.emit(false);
|
|
await flush();
|
|
clock.advance(1_000);
|
|
environment.emit(true);
|
|
} else {
|
|
closing.resolve(closedSuccessfully());
|
|
}
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
|
|
clock.advance(50);
|
|
const result = await running;
|
|
expect(attempts).toBe(2);
|
|
expect(result).toBe(terminal);
|
|
},
|
|
);
|
|
|
|
it("observes attempts, retries, the session and why the run ended", async () => {
|
|
const clock = new ManualClock();
|
|
const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false);
|
|
const observations: unknown[] = [];
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
observe: (observation) => {
|
|
observations.push(observation);
|
|
throw new Error("sink failure must not change the run");
|
|
},
|
|
attempt: () =>
|
|
++attempts === 1
|
|
? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true))
|
|
: opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => failed(terminal),
|
|
}),
|
|
),
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(50);
|
|
await expect(running).resolves.toBe(terminal);
|
|
expect(observations).toEqual([
|
|
{ kind: "ATTEMPT_STARTED" },
|
|
{
|
|
kind: "RETRY_SCHEDULED",
|
|
failureKind: "CONNECT_TIMEOUT",
|
|
delayBucket: "UNDER_1S",
|
|
},
|
|
{ kind: "ATTEMPT_STARTED" },
|
|
{ kind: "SESSION_OPENED" },
|
|
{ kind: "SESSION_CLOSED" },
|
|
{ kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" },
|
|
]);
|
|
|
|
// A second run that spends its retries ends as EXHAUSTED, not TERMINAL.
|
|
const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true);
|
|
const ends: unknown[] = [];
|
|
const spending = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
observe: (observation) => {
|
|
if (observation.kind === "RUN_ENDED") ends.push(observation);
|
|
},
|
|
attempt: () => failed(exhausted),
|
|
});
|
|
const spent = spending.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
clock.advance(0);
|
|
await expect(spent).resolves.toBe(exhausted);
|
|
expect(ends).toEqual([
|
|
{
|
|
kind: "RUN_ENDED",
|
|
reason: "EXHAUSTED",
|
|
failureKind: "PROVIDER_UNAVAILABLE",
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("resets a spent retry only after a valid signal and a stable open", async () => {
|
|
const runScenario = async (markValid: boolean, openMs: number) => {
|
|
const clock = new ManualClock();
|
|
const firstFailure = realtimeFailure(
|
|
"CONNECT_TIMEOUT",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const sessionFailure = realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"RECEIVE",
|
|
true,
|
|
);
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"CONNECT",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const closing = deferred<RealtimeReconnectOutcome<void>>();
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (
|
|
context: RealtimeReconnectAttemptContext,
|
|
) => {
|
|
attempts += 1;
|
|
if (attempts === 1) return failed(firstFailure);
|
|
if (attempts === 2) {
|
|
if (markValid) {
|
|
context.markValidHeartbeatOrEvent();
|
|
}
|
|
return opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed: () => closing.promise,
|
|
}),
|
|
);
|
|
}
|
|
return failed(terminal);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
clock.advance(0);
|
|
await flushUntil(() => attempts === 2);
|
|
await flush();
|
|
clock.advance(openMs);
|
|
closing.resolve(failed(sessionFailure, 0));
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
return {
|
|
attempts,
|
|
closeSession,
|
|
result,
|
|
sessionFailure,
|
|
terminal,
|
|
};
|
|
};
|
|
|
|
// A heartbeat on a short-lived connection, or a quiet long one, keeps
|
|
// the spent budget; only both together reset it.
|
|
for (const [markValid, openMs] of [
|
|
[false, 0],
|
|
[true, 499],
|
|
[false, 500],
|
|
] as const) {
|
|
const spent = await runScenario(markValid, openMs);
|
|
expect(spent.attempts).toBe(2);
|
|
expect(spent.result).toBe(spent.sessionFailure);
|
|
expect(spent.closeSession).toHaveBeenCalledOnce();
|
|
}
|
|
|
|
const eligible = await runScenario(true, 500);
|
|
expect(eligible.attempts).toBe(3);
|
|
expect(eligible.result).toBe(eligible.terminal);
|
|
expect(eligible.closeSession).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("does not reconnect before the session's authoritative close result", async () => {
|
|
const clock = new ManualClock();
|
|
const closure =
|
|
deferred<RealtimeReconnectOutcome<void>>();
|
|
const terminal = realtimeFailure(
|
|
"FORBIDDEN",
|
|
"RECEIVE",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const attempt = vi.fn(() =>
|
|
opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed: () => closure.promise,
|
|
}),
|
|
),
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
let settled = false;
|
|
|
|
const running = coordinator.run().then((result) => {
|
|
settled = true;
|
|
return result;
|
|
});
|
|
await flush();
|
|
|
|
expect(attempt).toHaveBeenCalledOnce();
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
expect(settled).toBe(false);
|
|
expect(coordinator.getState()).toBe("RUNNING");
|
|
|
|
closure.resolve(failed(terminal));
|
|
const result = await running;
|
|
expect(result).toBe(terminal);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("carries an exact recovery token into the next attempt and opens only after confirmation", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const directive = recoveryDirective(recovery, terminal);
|
|
const completed = closedSuccessfully();
|
|
const confirmation = realtimeSuccess(undefined);
|
|
const confirmTransportBarrier = vi.fn(
|
|
(received: RealtimeCommittedRecovery) => {
|
|
expect(received).toBe(recovery);
|
|
return confirmation;
|
|
},
|
|
);
|
|
let attempts = 0;
|
|
let gateSettled = false;
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
type RecoveryCloseReceipt =
|
|
| RealtimeRecoveryReconnectDirective
|
|
| RealtimeReconnectOutcome<void>;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier,
|
|
attempt: (context) => {
|
|
attempts += 1;
|
|
if (attempts === 1) {
|
|
expect(context.pendingRecovery).toBeNull();
|
|
return opened<RecoveryCloseReceipt>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RecoveryCloseReceipt,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => directive,
|
|
}),
|
|
);
|
|
}
|
|
|
|
expect(context.pendingRecovery).toBe(recovery);
|
|
const gate =
|
|
context.waitForRecoveryBarrierConfirmation();
|
|
void gate.then((result) => {
|
|
gateSettled = true;
|
|
gateResult = result;
|
|
});
|
|
expect(gateSettled).toBe(false);
|
|
return opened<RecoveryCloseReceipt>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RecoveryCloseReceipt,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => completed,
|
|
}),
|
|
recovery,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
expect(clock.sleepCalls).toEqual([0]);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(attempts).toBe(2);
|
|
expect(confirmTransportBarrier).toHaveBeenCalledOnce();
|
|
expect(gateSettled).toBe(true);
|
|
expect(gateResult).toBe(confirmation);
|
|
expect(result).toBe(completed.result);
|
|
});
|
|
|
|
it.each([
|
|
["missing", undefined],
|
|
["cloned", recoveryToken()],
|
|
] as const)(
|
|
"rejects a %s recovery proof and closes the new session",
|
|
async (_case, proof) => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const waitClosed = vi.fn(closedSuccessfully);
|
|
const confirmTransportBarrier = vi.fn(
|
|
confirmationSucceeded,
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier,
|
|
attempt: (context) => {
|
|
expect(context.pendingRecovery).toBe(recovery);
|
|
const session = Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed,
|
|
});
|
|
return proof === undefined
|
|
? opened(session)
|
|
: opened(session, proof);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"RECOVER",
|
|
false,
|
|
),
|
|
);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
expect(waitClosed).not.toHaveBeenCalled();
|
|
expect(confirmTransportBarrier).not.toHaveBeenCalled();
|
|
},
|
|
);
|
|
|
|
it("settles the readiness gate and run with the exact confirmation failure", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const confirmationFailure = realtimeFailure(
|
|
"SCOPE_PROTOCOL_VIOLATION",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const waitClosed = vi.fn(closedSuccessfully);
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: () => confirmationFailure,
|
|
attempt: (context) => {
|
|
void context
|
|
.waitForRecoveryBarrierConfirmation()
|
|
.then((result) => {
|
|
gateResult = result;
|
|
});
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed,
|
|
}),
|
|
recovery,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(result).toBe(confirmationFailure);
|
|
expect(gateResult).toBe(confirmationFailure);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
expect(waitClosed).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("retains the exact recovery token across retryable attempt failures and returns its terminal result on exhaustion", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const attemptFailure = realtimeFailure(
|
|
"CONNECT_TIMEOUT",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const gateResults: RealtimeResult<void>[] = [];
|
|
const seenRecoveries: Array<
|
|
RealtimeCommittedRecovery | null
|
|
> = [];
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 2 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context) => {
|
|
seenRecoveries.push(context.pendingRecovery);
|
|
void context
|
|
.waitForRecoveryBarrierConfirmation()
|
|
.then((result) => gateResults.push(result));
|
|
return failed(attemptFailure);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
clock.advance(0);
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(seenRecoveries).toEqual([recovery, recovery]);
|
|
expect(gateResults).toEqual([
|
|
attemptFailure,
|
|
attemptFailure,
|
|
]);
|
|
expect(gateResults[0]).toBe(attemptFailure);
|
|
expect(gateResults[1]).toBe(attemptFailure);
|
|
expect(result).toBe(terminal);
|
|
});
|
|
|
|
it("fails a no-pending gate immediately and rejects any unsolicited proof", async () => {
|
|
const proof = recoveryToken();
|
|
const closeSession = vi.fn();
|
|
const confirmTransportBarrier = vi.fn(
|
|
confirmationSucceeded,
|
|
);
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier,
|
|
attempt: async (context) => {
|
|
gateResult =
|
|
await context.waitForRecoveryBarrierConfirmation();
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: closeSession,
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
proof,
|
|
);
|
|
},
|
|
});
|
|
|
|
const result = await coordinator.run();
|
|
|
|
expect(gateResult).toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"RECOVER",
|
|
false,
|
|
),
|
|
);
|
|
expect(result).toBe(gateResult);
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
expect(confirmTransportBarrier).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("applies retry ceilings to an initial recovery and preserves its exact terminal result", async () => {
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const attempt = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({
|
|
maxAttempts: 1,
|
|
maxElapsedMs: 500,
|
|
}),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const result = await coordinator.run({
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
500,
|
|
),
|
|
});
|
|
|
|
expect(result).toBe(terminal);
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("settles a pending readiness gate when the run is aborted", async () => {
|
|
const clock = new ManualClock();
|
|
const recovery = recoveryToken();
|
|
const terminal = realtimeFailure(
|
|
"CURSOR_EXPIRED",
|
|
"RECOVER",
|
|
false,
|
|
);
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
let gateResult: RealtimeResult<void> | undefined;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context) => {
|
|
void context
|
|
.waitForRecoveryBarrierConfirmation()
|
|
.then((result) => {
|
|
gateResult = result;
|
|
});
|
|
return attemptResult.promise;
|
|
},
|
|
});
|
|
const controller = new AbortController();
|
|
|
|
const running = coordinator.run({
|
|
signal: controller.signal,
|
|
initialRecovery: recoveryDirective(
|
|
recovery,
|
|
terminal,
|
|
),
|
|
});
|
|
await flushUntil(() => clock.sleepCalls.length === 1);
|
|
clock.advance(0);
|
|
await flush();
|
|
controller.abort();
|
|
const result = await running;
|
|
await flush();
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
expect(gateResult).toEqual(
|
|
realtimeFailure("ABORTED", "RECOVER", false),
|
|
);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
recovery,
|
|
),
|
|
);
|
|
await flush();
|
|
});
|
|
|
|
it("rejects accessors without invoking them and consumes proxy values only from one descriptor snapshot", async () => {
|
|
let getterReads = 0;
|
|
const accessorOutcome = Object.defineProperty({}, "result", {
|
|
enumerable: true,
|
|
get() {
|
|
getterReads += 1;
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
});
|
|
const accessorCoordinator =
|
|
createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
accessorOutcome as unknown as RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<unknown>
|
|
>,
|
|
});
|
|
|
|
await expect(accessorCoordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(getterReads).toBe(0);
|
|
|
|
const completed = closedSuccessfully();
|
|
const session = Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => completed,
|
|
});
|
|
const rawOutcome = opened(session);
|
|
const proxyReads: PropertyKey[] = [];
|
|
const proxyOutcome = new Proxy(rawOutcome, {
|
|
get(_target, key) {
|
|
proxyReads.push(key);
|
|
if (key === "then") return undefined;
|
|
throw new Error("raw property reread");
|
|
},
|
|
});
|
|
const proxyCoordinator =
|
|
createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => proxyOutcome,
|
|
});
|
|
|
|
await expect(proxyCoordinator.run()).resolves.toBe(
|
|
completed.result,
|
|
);
|
|
expect(proxyReads).toEqual(["then"]);
|
|
});
|
|
|
|
it("rechecks scope after attempt and waitClosed boundaries before accepting callbacks", async () => {
|
|
let attemptScopeCurrent = true;
|
|
const lateAttemptClose = vi.fn();
|
|
const lateAttemptWaitClosed = vi.fn(closedSuccessfully);
|
|
const lateAttemptCoordinator =
|
|
createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
isCurrent: () => attemptScopeCurrent,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attemptScopeCurrent = false;
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateAttemptClose,
|
|
waitClosed: lateAttemptWaitClosed,
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
await expect(lateAttemptCoordinator.run()).resolves.toEqual(
|
|
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
|
|
);
|
|
expect(lateAttemptClose).toHaveBeenCalledOnce();
|
|
expect(lateAttemptWaitClosed).not.toHaveBeenCalled();
|
|
|
|
let closeScopeCurrent = true;
|
|
const classifyClosed = vi.fn(
|
|
(receipt: RealtimeReconnectOutcome<void>) => receipt,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const closeCoordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
isCurrent: () => closeScopeCurrent,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed,
|
|
close: closeSession,
|
|
waitClosed: () => {
|
|
closeScopeCurrent = false;
|
|
return closedSuccessfully();
|
|
},
|
|
}),
|
|
),
|
|
});
|
|
|
|
await expect(closeCoordinator.run()).resolves.toEqual(
|
|
realtimeFailure("SCOPE_FENCED", "CONNECT", false),
|
|
);
|
|
expect(classifyClosed).not.toHaveBeenCalled();
|
|
expect(closeSession).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("rejects extra and missing outcome keys through the common result codec", async () => {
|
|
const validSession = Object.freeze({
|
|
classifyClosed: closedSuccessfully,
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
});
|
|
const invalidOutcomes = [
|
|
{
|
|
extra: true,
|
|
result: realtimeSuccess(validSession),
|
|
},
|
|
{
|
|
result: { ok: true },
|
|
},
|
|
{
|
|
result: {
|
|
ok: false,
|
|
error: {
|
|
extra: true,
|
|
kind: "FORBIDDEN",
|
|
operation: "CONNECT",
|
|
retryable: false,
|
|
},
|
|
},
|
|
},
|
|
];
|
|
|
|
for (const invalid of invalidOutcomes) {
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
invalid as unknown as RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<unknown>
|
|
>,
|
|
});
|
|
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
}
|
|
});
|
|
|
|
it("rechecks abort immediately after listener registration", async () => {
|
|
let abortReads = 0;
|
|
let listener: (() => void) | undefined;
|
|
const signal = {
|
|
get aborted() {
|
|
abortReads += 1;
|
|
return abortReads >= 2;
|
|
},
|
|
addEventListener(
|
|
type: string,
|
|
nextListener: () => void,
|
|
) {
|
|
if (type === "abort") listener = nextListener;
|
|
},
|
|
removeEventListener() {
|
|
listener = undefined;
|
|
},
|
|
} as unknown as AbortSignal;
|
|
const attempt = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt,
|
|
});
|
|
|
|
const result = await coordinator.run({ signal });
|
|
|
|
expect(listener).toBeUndefined();
|
|
expect(attempt).not.toHaveBeenCalled();
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
});
|
|
|
|
it("bounds a non-cooperative retry sleep after its phase is aborted", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const sleep = vi.fn(
|
|
() => new Promise<void>(() => undefined),
|
|
);
|
|
const clock: ClockPort = Object.freeze({
|
|
now: () => 0,
|
|
sleep,
|
|
});
|
|
const retryable = realtimeFailure(
|
|
"CONNECT_TIMEOUT",
|
|
"CONNECT",
|
|
true,
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => failed(retryable),
|
|
});
|
|
|
|
try {
|
|
const running = coordinator.run();
|
|
await flushUntil(() => sleep.mock.calls.length === 1);
|
|
environment.emit(false);
|
|
await flush();
|
|
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
|
|
);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("bounds a non-cooperative attempt and closes its late successful session before leaving DRAINING", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
const lateClose = vi.fn();
|
|
const lateWaitClosed = vi.fn(closedSuccessfully);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => attemptResult.promise,
|
|
});
|
|
|
|
try {
|
|
const running = coordinator.run();
|
|
await flush();
|
|
environment.emit(false);
|
|
await flush();
|
|
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
|
|
);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure(
|
|
"PROTOCOL_MISMATCH",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateClose,
|
|
waitClosed: lateWaitClosed,
|
|
}),
|
|
),
|
|
);
|
|
await flush();
|
|
|
|
expect(lateClose).toHaveBeenCalledOnce();
|
|
expect(lateWaitClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("cancels the drain timer on abort and keeps the late generation fenced", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
const lateClose = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => attemptResult.promise,
|
|
});
|
|
const controller = new AbortController();
|
|
|
|
try {
|
|
const running = coordinator.run({
|
|
signal: controller.signal,
|
|
});
|
|
await flush();
|
|
environment.emit(false);
|
|
await flush();
|
|
expect(vi.getTimerCount()).toBe(1);
|
|
|
|
controller.abort();
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateClose,
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
),
|
|
);
|
|
await flush();
|
|
|
|
expect(lateClose).toHaveBeenCalledOnce();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("keeps a healthy waitClosed unbounded but bounds its post-abort drain", async () => {
|
|
vi.useFakeTimers();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const closure =
|
|
deferred<RealtimeReconnectOutcome<void>>();
|
|
const classifyClosed = vi.fn(
|
|
(receipt: RealtimeReconnectOutcome<void>) => receipt,
|
|
);
|
|
const closeSession = vi.fn();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock: new ManualClock(),
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed,
|
|
close: closeSession,
|
|
waitClosed: () => closure.promise,
|
|
}),
|
|
),
|
|
});
|
|
|
|
try {
|
|
let settled = false;
|
|
const running = coordinator.run().then((result) => {
|
|
settled = true;
|
|
return result;
|
|
});
|
|
await flush();
|
|
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs * 2,
|
|
);
|
|
expect(settled).toBe(false);
|
|
expect(coordinator.getState()).toBe("RUNNING");
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
environment.emit(false);
|
|
await flush();
|
|
await vi.advanceTimersByTimeAsync(
|
|
REALTIME_RECONNECT_CEILINGS.drainTimeoutMs,
|
|
);
|
|
const result = await running;
|
|
|
|
expect(result).toEqual(
|
|
realtimeFailure(
|
|
"PROVIDER_UNAVAILABLE",
|
|
"CONNECT",
|
|
false,
|
|
),
|
|
);
|
|
expect(closeSession).toHaveBeenCalled();
|
|
expect(classifyClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
expect(environment.listeners.size).toBe(0);
|
|
expect(vi.getTimerCount()).toBe(0);
|
|
|
|
closure.resolve(closedSuccessfully());
|
|
await flush();
|
|
|
|
expect(classifyClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
} finally {
|
|
coordinator.close();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("uses waitClosed as authority and fences an aborted late generation", async () => {
|
|
const clock = new ManualClock();
|
|
const attemptResult =
|
|
deferred<
|
|
RealtimeReconnectOutcome<
|
|
RealtimeReconnectAttemptSuccess<
|
|
RealtimeReconnectOutcome<void>
|
|
>
|
|
>
|
|
>();
|
|
let captured:
|
|
| RealtimeReconnectAttemptContext
|
|
| undefined;
|
|
const lateClose = vi.fn();
|
|
const lateWaitClosed = vi.fn(closedSuccessfully);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: (context) => {
|
|
captured = context;
|
|
return attemptResult.promise;
|
|
},
|
|
});
|
|
const controller = new AbortController();
|
|
|
|
const running = coordinator.run({ signal: controller.signal });
|
|
await flush();
|
|
expect(captured?.isCurrent()).toBe(true);
|
|
|
|
controller.abort();
|
|
const result = await running;
|
|
expect(result).toEqual(
|
|
realtimeFailure("ABORTED", "CONNECT", false),
|
|
);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
expect(captured?.isCurrent()).toBe(false);
|
|
|
|
attemptResult.resolve(
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: lateClose,
|
|
waitClosed: lateWaitClosed,
|
|
}),
|
|
),
|
|
);
|
|
await flush();
|
|
|
|
expect(lateClose).toHaveBeenCalledOnce();
|
|
expect(lateWaitClosed).not.toHaveBeenCalled();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
});
|
|
|
|
class FakeWebSocket implements WebSocketFacade {
|
|
protocol: string = REALTIME_WEBSOCKET_PROTOCOL;
|
|
readyState = 0;
|
|
bufferedAmount = 0;
|
|
readonly listeners = new Map<
|
|
string,
|
|
Set<(event: unknown) => void>
|
|
>();
|
|
readonly sent: string[] = [];
|
|
|
|
send(data: string): void {
|
|
if (this.readyState !== 1) throw new Error("not open");
|
|
this.sent.push(data);
|
|
}
|
|
|
|
close(): void {
|
|
this.readyState = 3;
|
|
}
|
|
|
|
addEventListener(
|
|
type: string,
|
|
listener: (event: unknown) => void,
|
|
): void {
|
|
const listeners = this.listeners.get(type) ?? new Set();
|
|
listeners.add(listener);
|
|
this.listeners.set(type, listeners);
|
|
}
|
|
|
|
removeEventListener(
|
|
type: string,
|
|
listener: (event: unknown) => void,
|
|
): void {
|
|
this.listeners.get(type)?.delete(listener);
|
|
}
|
|
|
|
accept(): void {
|
|
this.readyState = 1;
|
|
this.emit("open", {});
|
|
this.emit("message", {
|
|
data: JSON.stringify({
|
|
type: "WELCOME",
|
|
protocol: REALTIME_WEBSOCKET_PROTOCOL,
|
|
connectionId: "connection.0001",
|
|
heartbeatMs: 5_000,
|
|
heartbeatAckTimeoutMs: 1_000,
|
|
limits: {
|
|
maxFrameBytes: 64 * 1_024,
|
|
maxSubscriptions: 32,
|
|
maxInboundQueueCount: 256,
|
|
maxInboundQueueBytes: 4 * 1_024 * 1_024,
|
|
maxOutboundQueueCount: 128,
|
|
maxOutboundQueueBytes: 256 * 1_024,
|
|
maxBufferedAmountBytes: 256 * 1_024,
|
|
maxEventsPerSecond: 256,
|
|
},
|
|
}),
|
|
});
|
|
}
|
|
|
|
nativeClose(code: number): void {
|
|
this.readyState = 3;
|
|
this.emit("close", { code, reason: "" });
|
|
}
|
|
|
|
ackHeartbeats(): void {
|
|
for (const raw of this.sent.splice(0)) {
|
|
const frame = JSON.parse(raw) as { type: string; nonce: string };
|
|
if (frame.type !== "HEARTBEAT") continue;
|
|
this.emit("message", {
|
|
data: JSON.stringify({
|
|
type: "HEARTBEAT_ACK",
|
|
protocol: REALTIME_WEBSOCKET_PROTOCOL,
|
|
nonce: frame.nonce,
|
|
}),
|
|
});
|
|
}
|
|
}
|
|
|
|
private emit(type: string, event: unknown): void {
|
|
for (const listener of [...(this.listeners.get(type) ?? [])]) {
|
|
listener(event);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The smallest SSE bridge design 6.3 leaves to composition: the attempt ends
|
|
* at `onOpen` and the rest of the read is the session.
|
|
*
|
|
* 1106/04-01. The coordinator retries a throwing attempt as
|
|
* PROVIDER_UNAVAILABLE, so the bridge checks its connection settings here,
|
|
* when it is built: a wrong endpoint throws before any run. Each attempt
|
|
* still makes its own connection, because the session's `close` ends that
|
|
* connection for good.
|
|
*/
|
|
function sseAttempt(
|
|
fetcher: typeof fetch,
|
|
clock: ManualClock,
|
|
endpoint = "https://app.example.test/events",
|
|
) {
|
|
const settings = Object.freeze({
|
|
endpoint,
|
|
applicationOrigin: "https://app.example.test",
|
|
recoveryMode: "CURSOR" as const,
|
|
fetcher,
|
|
clock,
|
|
});
|
|
// Creating a connection checks the settings and sends nothing.
|
|
createFetchSseConnection(settings).close();
|
|
return async (context: RealtimeReconnectAttemptContext) => {
|
|
const connection = createFetchSseConnection(settings);
|
|
const open = deferred<"OPEN">();
|
|
const reading = readSseForReconnect(connection, {
|
|
resumeCursor: null,
|
|
signal: context.signal,
|
|
onOpen: () => {
|
|
open.resolve("OPEN");
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
|
onComment: () => context.markValidHeartbeatOrEvent(),
|
|
});
|
|
const first = await Promise.race([open.promise, reading]);
|
|
if (first !== "OPEN") {
|
|
return first.result.ok
|
|
? failed(realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false))
|
|
: (first as RealtimeReconnectOutcome<never>);
|
|
}
|
|
return opened(
|
|
Object.freeze({
|
|
waitClosed: () => reading,
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<FetchSseClosedOutcome>,
|
|
) =>
|
|
receipt.result.ok
|
|
? failed(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECEIVE", false),
|
|
)
|
|
: (receipt as RealtimeReconnectOutcome<never>),
|
|
close: () => connection.close(),
|
|
}),
|
|
);
|
|
};
|
|
}
|
|
|
|
async function advanceUntil(
|
|
clock: ManualClock,
|
|
settled: () => boolean,
|
|
): Promise<void> {
|
|
// Ten simulated seconds outlast policy()'s whole elapsed budget.
|
|
for (let elapsed = 0; elapsed < 10_000 && !settled(); elapsed += 50) {
|
|
clock.advance(50);
|
|
await flush();
|
|
}
|
|
}
|
|
|
|
function observationLabel({
|
|
kind,
|
|
failureKind,
|
|
reason,
|
|
}: RealtimeReconnectObservation): string {
|
|
return [kind, failureKind, reason].filter(Boolean).join(":");
|
|
}
|
|
|
|
describe("transport failures through the reconnect coordinator", () => {
|
|
it.each([
|
|
["NETWORK_LOST", 1_006],
|
|
["RESTART", 1_012],
|
|
] as const)(
|
|
"reconnects a WebSocket %s close that carries no server hint",
|
|
async (_category, code) => {
|
|
const clock = new ManualClock();
|
|
const sockets: FakeWebSocket[] = [];
|
|
const coordinator = createRealtimeReconnectCoordinator<
|
|
WebSocketClosedReceipt
|
|
>({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const connection = createWebSocketConnection({
|
|
origin: "https://app.example.test",
|
|
endpointPath: "/_realtime/v1",
|
|
clock,
|
|
createNonce: () => "nonce.0001",
|
|
createSocket: () => {
|
|
const socket = new FakeWebSocket();
|
|
sockets.push(socket);
|
|
return socket;
|
|
},
|
|
onEvent: async () =>
|
|
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
|
|
onRecoveryRequired: () => undefined,
|
|
});
|
|
const connected = await connection.connect(context.signal);
|
|
if (!connected.ok) return failed(connected);
|
|
return opened(
|
|
Object.freeze({
|
|
waitClosed: () => connection.waitClosed(),
|
|
classifyClosed: (receipt: WebSocketClosedReceipt) =>
|
|
Object.freeze({
|
|
result: Object.freeze({
|
|
ok: false as const,
|
|
error: receipt.error,
|
|
}),
|
|
}),
|
|
close: () => connection.close(),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => sockets.length === 1);
|
|
sockets[0]!.accept();
|
|
await flush();
|
|
sockets[0]!.nativeClose(code);
|
|
await flushUntil(() => clock.sleepCalls.includes(50));
|
|
clock.advance(50);
|
|
await flushUntil(() => sockets.length === 2);
|
|
expect(sockets).toHaveLength(2);
|
|
|
|
// OVERLOADED carries no hint either and stays terminal.
|
|
sockets[1]!.accept();
|
|
await flush();
|
|
sockets[1]!.nativeClose(1_013);
|
|
await expect(running).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "RECEIVE",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(sockets).toHaveLength(2);
|
|
},
|
|
);
|
|
|
|
it("reconnects a quiet WebSocket whose heartbeats kept it healthy past the elapsed budget", async () => {
|
|
const clock = new ManualClock();
|
|
const sockets: FakeWebSocket[] = [];
|
|
let nonce = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator<
|
|
WebSocketClosedReceipt
|
|
>({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const connection = createWebSocketConnection({
|
|
origin: "https://app.example.test",
|
|
endpointPath: "/_realtime/v1",
|
|
clock,
|
|
createNonce: () => `nonce.${++nonce}`,
|
|
createSocket: () => {
|
|
const socket = new FakeWebSocket();
|
|
sockets.push(socket);
|
|
return socket;
|
|
},
|
|
onEvent: async () =>
|
|
realtimeSuccess(REALTIME_TRANSPORT_CONTINUE),
|
|
onRecoveryRequired: () => undefined,
|
|
observe: (observation) => {
|
|
if (observation.kind === "HEARTBEAT_ACK") {
|
|
context.markValidHeartbeatOrEvent();
|
|
}
|
|
},
|
|
});
|
|
const connected = await connection.connect(context.signal);
|
|
if (!connected.ok) return failed(connected);
|
|
return opened(
|
|
Object.freeze({
|
|
waitClosed: () => connection.waitClosed(),
|
|
classifyClosed: (receipt: WebSocketClosedReceipt) =>
|
|
Object.freeze({
|
|
result: Object.freeze({
|
|
ok: false as const,
|
|
error: receipt.error,
|
|
}),
|
|
}),
|
|
close: () => connection.close(),
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => sockets.length === 1);
|
|
sockets[0]!.accept();
|
|
await flush();
|
|
// No events, only acknowledged heartbeats, for longer than maxElapsedMs.
|
|
for (let beat = 0; beat < 2; beat += 1) {
|
|
clock.advance(5_000);
|
|
await flush();
|
|
sockets[0]!.ackHeartbeats();
|
|
clock.advance(1_000);
|
|
await flush();
|
|
}
|
|
sockets[0]!.nativeClose(1_012);
|
|
await flushUntil(() => clock.sleepCalls.includes(50));
|
|
clock.advance(50);
|
|
await flushUntil(() => sockets.length === 2);
|
|
expect(sockets).toHaveLength(2);
|
|
|
|
coordinator.close();
|
|
await expect(running).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "CLOSED" },
|
|
});
|
|
});
|
|
|
|
it.each([502, 504])(
|
|
"reconnects SSE HTTP %i that carries no server hint",
|
|
async (status) => {
|
|
const clock = new ManualClock();
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(null, { status }))
|
|
.mockResolvedValueOnce(new Response(null, { status: 403 }));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const read = await createFetchSseConnection({
|
|
endpoint: "https://app.example.test/events",
|
|
applicationOrigin: "https://app.example.test",
|
|
recoveryMode: "CURSOR",
|
|
fetcher,
|
|
clock,
|
|
}).read({
|
|
resumeCursor: null,
|
|
signal: context.signal,
|
|
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
|
});
|
|
// Only connect failures are exercised here.
|
|
return failed(
|
|
read.ok
|
|
? realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false)
|
|
: read,
|
|
);
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.includes(50));
|
|
clock.advance(50);
|
|
|
|
await expect(running).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "FORBIDDEN",
|
|
operation: "CONNECT",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(fetcher).toHaveBeenCalledTimes(2);
|
|
},
|
|
);
|
|
|
|
it.each([429, 503])(
|
|
"waits at least the SSE %i Retry-After through the reconnect bridge",
|
|
async (status) => {
|
|
const clock = new ManualClock();
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status,
|
|
headers: { "retry-after": "3" },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status,
|
|
headers: { "retry-after": "10" },
|
|
}),
|
|
);
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxDelayMs: 5_000, maxElapsedMs: 60_000 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: async (context) => {
|
|
const read = await readSseForReconnect(
|
|
createFetchSseConnection({
|
|
endpoint: "https://app.example.test/events",
|
|
applicationOrigin: "https://app.example.test",
|
|
recoveryMode: "CURSOR",
|
|
fetcher,
|
|
clock,
|
|
}),
|
|
{
|
|
resumeCursor: null,
|
|
signal: context.signal,
|
|
onEvent: () => realtimeSuccess(SSE_CONTINUE),
|
|
},
|
|
);
|
|
// Only connect failures are exercised here.
|
|
if (read.result.ok) {
|
|
return failed(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
}
|
|
return read as RealtimeReconnectOutcome<never>;
|
|
},
|
|
});
|
|
|
|
// 10_000 is each read's connect timeout; 3_000 is the reconnect wait.
|
|
const running = coordinator.run();
|
|
await flushUntil(() => clock.sleepCalls.length === 2);
|
|
expect(clock.sleepCalls).toEqual([10_000, 3_000]);
|
|
clock.advance(3_000);
|
|
|
|
// A hint above the policy's maxDelayMs still ends the run.
|
|
await expect(running).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { operation: "CONNECT", retryable: true },
|
|
});
|
|
expect(fetcher).toHaveBeenCalledTimes(2);
|
|
expect(clock.sleepCalls).toEqual([10_000, 3_000, 10_000]);
|
|
},
|
|
);
|
|
|
|
// navigator.onLine stays true when only the server is down, so no online
|
|
// event ever arrives (0929/04-01). The SSE adapter reports a rejected fetch
|
|
// or read as PROVIDER_UNAVAILABLE, as WebSocket does; OFFLINE comes only
|
|
// from the environment signal (1103/04-04).
|
|
it("retries an SSE fetch rejection with backoff while the browser stays online", async () => {
|
|
const clock = new ManualClock();
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockRejectedValue(new TypeError("Failed to fetch"));
|
|
const observed: string[] = [];
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: sseAttempt(fetcher, clock),
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(fetcher).toHaveBeenCalledTimes(3);
|
|
// Full-jitter backoff; 10_000 is each read's connect timeout.
|
|
expect(clock.sleepCalls.filter((ms) => ms !== 10_000)).toEqual([
|
|
50, 100,
|
|
]);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "CONNECT",
|
|
retryable: true,
|
|
},
|
|
});
|
|
expect(observed).toEqual([
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
|
|
"ATTEMPT_STARTED",
|
|
"RUN_ENDED:PROVIDER_UNAVAILABLE:EXHAUSTED",
|
|
]);
|
|
});
|
|
|
|
it("reconnects after an open SSE stream's reader rejects while the browser stays online", async () => {
|
|
const clock = new ManualClock();
|
|
let stream: ReadableStreamDefaultController<Uint8Array> | undefined;
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
stream = controller;
|
|
controller.enqueue(new TextEncoder().encode(": ping\n\n"));
|
|
},
|
|
}),
|
|
{ status: 200, headers: { "content-type": "text/event-stream" } },
|
|
),
|
|
)
|
|
.mockRejectedValue(new TypeError("Failed to fetch"));
|
|
const observed: string[] = [];
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: sseAttempt(fetcher, clock),
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await flushUntil(() => observed.includes("SESSION_OPENED"));
|
|
await flush();
|
|
// Healthy past stableOpenMs, then a proxy restart drops the stream.
|
|
clock.advance(1_000);
|
|
await flush();
|
|
stream!.error(new TypeError("network error"));
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(fetcher).toHaveBeenCalledTimes(3);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "CONNECT",
|
|
retryable: true,
|
|
},
|
|
});
|
|
expect(observed).toEqual([
|
|
"ATTEMPT_STARTED",
|
|
"SESSION_OPENED",
|
|
"SESSION_CLOSED",
|
|
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
|
|
"ATTEMPT_STARTED",
|
|
"RUN_ENDED:PROVIDER_UNAVAILABLE:EXHAUSTED",
|
|
]);
|
|
});
|
|
|
|
// 1105/04-01. No template transport reports OFFLINE any more (1103/04-04),
|
|
// so the two SSE tests above no longer reach the 0929/04-01 rule. A
|
|
// composer's transport (a STOMP one, say) still may. While the environment
|
|
// stays online, its OFFLINE is retried with backoff; waiting for an online
|
|
// event that never comes would park the run with no retry and no result.
|
|
it("retries a composer transport's OFFLINE attempt failure while the browser stays online", async () => {
|
|
const clock = new ManualClock();
|
|
const offline = realtimeFailure("OFFLINE", "CONNECT", true);
|
|
const observed: string[] = [];
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attempts += 1;
|
|
return failed(offline);
|
|
},
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(attempts).toBe(3);
|
|
expect(clock.sleepCalls).toEqual([50, 100]);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "OFFLINE", operation: "CONNECT", retryable: true },
|
|
});
|
|
expect(observed).toEqual([
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:OFFLINE",
|
|
"ATTEMPT_STARTED",
|
|
"RETRY_SCHEDULED:OFFLINE",
|
|
"ATTEMPT_STARTED",
|
|
"RUN_ENDED:OFFLINE:EXHAUSTED",
|
|
]);
|
|
});
|
|
|
|
it("reconnects after a composer session closes with OFFLINE while the browser stays online", async () => {
|
|
const clock = new ManualClock();
|
|
const offline = realtimeFailure("OFFLINE", "RECEIVE", true);
|
|
const observed: string[] = [];
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attempts += 1;
|
|
return opened(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => failed(offline),
|
|
}),
|
|
);
|
|
},
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(attempts).toBe(3);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "OFFLINE", operation: "RECEIVE", retryable: true },
|
|
});
|
|
expect(
|
|
observed.filter((label) => label === "RETRY_SCHEDULED:OFFLINE"),
|
|
).toHaveLength(2);
|
|
});
|
|
|
|
it.each(["before", "after"] as const)(
|
|
"waits for the online event when the browser goes offline %s the SSE fetch rejects",
|
|
async (order) => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableOnlineEnvironment(true);
|
|
const fetcher = vi
|
|
.fn<typeof fetch>()
|
|
.mockImplementationOnce(async () => {
|
|
if (order === "before") environment.emit(false);
|
|
throw new TypeError("Failed to fetch");
|
|
})
|
|
.mockResolvedValueOnce(new Response(null, { status: 403 }));
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: sseAttempt(fetcher, clock),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await flushUntil(() => fetcher.mock.calls.length === 1);
|
|
await flush();
|
|
if (order === "after") environment.emit(false);
|
|
await flush();
|
|
// Twenty offline minutes fire any retry timer that was armed.
|
|
clock.advance(20 * 60_000);
|
|
await flush();
|
|
expect(fetcher).toHaveBeenCalledTimes(1);
|
|
expect(result).toBeNull();
|
|
|
|
environment.emit(true);
|
|
await advanceUntil(clock, () => result !== null);
|
|
expect(fetcher).toHaveBeenCalledTimes(2);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "FORBIDDEN", operation: "CONNECT", retryable: false },
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("results that composition code hands the reconnect coordinator", () => {
|
|
// Composition code may import only the adapter barrel. Each case passes a
|
|
// value that type-checks as RealtimeReconnectOutcome.
|
|
it("retries a failure built with the barrel's realtimeFailure", async () => {
|
|
const clock = new ManualClock();
|
|
const last = publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT");
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy({ maxAttempts: 1 }),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: () => publicRealtimeSuccess(undefined),
|
|
attempt: () => ({
|
|
result:
|
|
++attempts === 1
|
|
? publicRealtimeFailure("CONNECT_TIMEOUT", "CONNECT")
|
|
: last,
|
|
}),
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(clock.sleepCalls).toEqual([50]);
|
|
clock.advance(50);
|
|
|
|
await expect(running).resolves.toBe(last);
|
|
expect(attempts).toBe(2);
|
|
});
|
|
|
|
const unfrozenFailure: RealtimeReconnectOutcome<never> = {
|
|
result: {
|
|
ok: false,
|
|
error: {
|
|
kind: "CONNECT_TIMEOUT",
|
|
operation: "CONNECT",
|
|
retryable: true,
|
|
},
|
|
},
|
|
};
|
|
const unfrozenError: RealtimeReconnectOutcome<never> = {
|
|
result: Object.freeze({
|
|
ok: false,
|
|
error: {
|
|
kind: "CONNECT_TIMEOUT",
|
|
operation: "CONNECT",
|
|
retryable: true,
|
|
},
|
|
} as const),
|
|
};
|
|
|
|
it.each([
|
|
["a plain failure object", unfrozenFailure],
|
|
["a frozen failure around a plain error", unfrozenError],
|
|
] as const)(
|
|
"ends the run without a retry when an attempt returns %s",
|
|
async (_label, outcome) => {
|
|
const clock = new ManualClock();
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attempts += 1;
|
|
return outcome;
|
|
},
|
|
});
|
|
|
|
const running = coordinator.run();
|
|
await flush();
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
await expect(running).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
expect(attempts).toBe(1);
|
|
},
|
|
);
|
|
|
|
it("ends the run when classifyClosed returns a plain success object", async () => {
|
|
const clock = new ManualClock();
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () =>
|
|
opened(
|
|
Object.freeze({
|
|
classifyClosed: (): RealtimeReconnectOutcome<void> => ({
|
|
result: { ok: true, value: undefined },
|
|
}),
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
),
|
|
});
|
|
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
});
|
|
|
|
// 1106/04-01. The rule the attempt and session types state. A throwing
|
|
// attempt or waitClosed is a transport failure and is retried; poll reads a
|
|
// throwing execute the other way (1105/04-02).
|
|
it.each([
|
|
["attempt", "CONNECT"],
|
|
["waitClosed", "RECEIVE"],
|
|
] as const)(
|
|
"retries a throwing %s as PROVIDER_UNAVAILABLE on %s",
|
|
async (thrower, operation) => {
|
|
const clock = new ManualClock();
|
|
const observed: string[] = [];
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
random: () => 0.5,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attempts += 1;
|
|
if (thrower === "attempt") throw new TypeError("composer defect");
|
|
return opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: (
|
|
receipt: RealtimeReconnectOutcome<void>,
|
|
) => receipt,
|
|
close: vi.fn(),
|
|
waitClosed: () => {
|
|
throw new TypeError("composer defect");
|
|
},
|
|
}),
|
|
);
|
|
},
|
|
observe: (observation) => observed.push(observationLabel(observation)),
|
|
});
|
|
|
|
let result: RealtimeResult<void> | null = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
await advanceUntil(clock, () => result !== null);
|
|
|
|
expect(attempts).toBe(3);
|
|
expect(clock.sleepCalls).toEqual([50, 100]);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: { kind: "PROVIDER_UNAVAILABLE", operation, retryable: true },
|
|
});
|
|
expect(
|
|
observed.filter((label) => label.startsWith("RETRY_SCHEDULED")),
|
|
).toEqual([
|
|
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
|
|
"RETRY_SCHEDULED:PROVIDER_UNAVAILABLE",
|
|
]);
|
|
expect(observed.at(-1)).toBe(
|
|
"RUN_ENDED:PROVIDER_UNAVAILABLE:EXHAUSTED",
|
|
);
|
|
},
|
|
);
|
|
|
|
it("ends the run without a retry when classifyClosed throws", async () => {
|
|
const clock = new ManualClock();
|
|
let attempts = 0;
|
|
const coordinator = createRealtimeReconnectCoordinator({
|
|
policy: policy(),
|
|
environment: new MutableOnlineEnvironment(true),
|
|
clock,
|
|
confirmTransportBarrier: confirmationSucceeded,
|
|
attempt: () => {
|
|
attempts += 1;
|
|
return opened<RealtimeReconnectOutcome<void>>(
|
|
Object.freeze({
|
|
classifyClosed: () => {
|
|
throw new TypeError("composer defect");
|
|
},
|
|
close: vi.fn(),
|
|
waitClosed: closedSuccessfully,
|
|
}),
|
|
);
|
|
},
|
|
});
|
|
|
|
await expect(coordinator.run()).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "CONNECT", false),
|
|
);
|
|
expect(attempts).toBe(1);
|
|
expect(clock.sleepCalls).toEqual([]);
|
|
});
|
|
|
|
// Review 04-R2 built this endpoint's connection inside the attempt: no
|
|
// request, three attempts, then PROVIDER_UNAVAILABLE (EXHAUSTED). The bridge
|
|
// now checks its settings when it is built, so the run never starts.
|
|
it("refuses a wrong SSE endpoint when the bridge is built", () => {
|
|
const fetcher = vi.fn<typeof fetch>();
|
|
const build = () =>
|
|
sseAttempt(fetcher, new ManualClock(), "http://app.example.test/events");
|
|
|
|
expect(build).toThrow(TypeError);
|
|
expect(build).toThrow("SSE endpoint must be fixed same-origin HTTPS.");
|
|
expect(fetcher).not.toHaveBeenCalled();
|
|
});
|
|
});
|