fix(realtime): add observe hooks to the reconnect and poll coordinators (04-07)

Design 14.1 connect and bounded_poll events had no place to come from, and
an exhausted budget looked the same as a terminal failure. Both coordinators
now emit a closed observation vocabulary through a best-effort sink.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-26 00:32:33 +09:00
co-authored by Claude Opus 5.5
parent 905189ef70
commit 03991aaafb
8 changed files with 274 additions and 0 deletions
@@ -1930,6 +1930,23 @@ push:
- `web_push_click_dispatched`
- `web_push_association_revoked`
reference runtime의 관측 지점은 다음과 같다. 조합 코드는 attempt나 session을
감싸지 않고 이 관측을 위 event로 옮긴다.
| runtime 관측 | 만들 수 있는 event |
| --- | --- |
| reconnect `ATTEMPT_STARTED` | `realtime_connect_attempted` |
| reconnect `SESSION_OPENED`, 또는 open 전의 `RETRY_SCHEDULED`·`RUN_ENDED` | `realtime_connect_finished` |
| reconnect `SESSION_CLOSED` | `realtime_connection_closed` |
| reconnect `RETRY_SCHEDULED`(실패 kind, 대기 bucket), `RUN_ENDED`(`EXHAUSTED`·`TERMINAL`·`ABORTED`·`COMPLETED`) | 14.2 재연결 시도·대기·소진 비율 |
| WebSocket `HEARTBEAT_ACK` | `realtime_heartbeat_observed` |
| polling `LEASE_STARTED`, `ATTEMPT_FINISHED`(`VALUE`·`UNCHANGED`·실패 kind), `LEASE_STOPPED` | `bounded_poll_*` |
근거는 `reconnect-coordinator.ts:128-182`,
`polling/bounded-poll-coordinator.ts:87-113`,
`websocket/websocket-connection.ts:376-378`이다. sink가 throw해도 제어 흐름은
바뀌지 않는다(`result.ts`의 `observeRealtime`).
모든 event는 semantic registry에 등록하고 arbitrary payload를 받지 않는다.
diagnostics sink failure는 connection, worker handler, poll lease와 cleanup을
실패시키지 않으며 재귀 telemetry를 만들지 않는다.
+1
View File
@@ -29,6 +29,7 @@ export {
type RealtimeReconnectCoordinator,
type RealtimeReconnectCoordinatorDependencies,
type RealtimeReconnectEnvironment,
type RealtimeReconnectObservation,
type RealtimeReconnectOutcome,
type RealtimeReconnectRunInput,
type RealtimeReconnectSession,
@@ -15,6 +15,7 @@ import { systemClock } from "../../platform/system-clock.ts";
import {
captureRealtimeDataSnapshot,
hasExactSnapshotKeys,
observeRealtime,
raceRealtimeTask,
realtimeFailure,
realtimeSuccess,
@@ -83,6 +84,16 @@ export type BoundedPollCoordinator<Value> = Readonly<{
close(): void;
}>;
/**
* Closed diagnostics for design 14.1 `bounded_poll_*`. `outcome` is VALUE,
* UNCHANGED or the failure kind for ATTEMPT_FINISHED, and TERMINAL or the
* failure kind for LEASE_STOPPED.
*/
export type BoundedPollObservation = Readonly<{
kind: "LEASE_STARTED" | "ATTEMPT_FINISHED" | "LEASE_STOPPED";
outcome?: "VALUE" | "UNCHANGED" | "TERMINAL" | RealtimeFailureKind;
}>;
export type BoundedPollCoordinatorDependencies<Value> = Readonly<{
policy: PollLeasePolicy;
operation: BoundedPollOperationContract;
@@ -99,6 +110,7 @@ export type BoundedPollCoordinatorDependencies<Value> = Readonly<{
isCurrent?: () => boolean;
clock?: ClockPort;
random?: () => number;
observe?(observation: BoundedPollObservation): void;
}>;
const SAFE_STATE = /^[A-Z][A-Z0-9_]{0,63}$/u;
@@ -128,8 +140,25 @@ export function createBoundedPollCoordinator<Value>(
let activeController: AbortController | null = null;
let generation = 0;
let pendingWork = 0;
const observe = (observation: BoundedPollObservation) =>
observeRealtime(dependencies.observe, observation);
async function run(
input?: BoundedPollRunInput<Value>,
): Promise<BoundedPollResult<Value>> {
const startedGeneration = generation;
const result = await runLease(input);
// A call rejected before starting leaves the generation unchanged.
if (generation !== startedGeneration) {
observe({
kind: "LEASE_STOPPED",
outcome: result.ok ? "TERMINAL" : result.error.kind,
});
}
return result;
}
async function runLease(
input: BoundedPollRunInput<Value> = {},
): Promise<BoundedPollResult<Value>> {
if (state === "CLOSED") return pollFailure("CLOSED");
@@ -149,6 +178,7 @@ export function createBoundedPollCoordinator<Value>(
state = "RUNNING";
const runGeneration = ++generation;
observe({ kind: "LEASE_STARTED" });
const controller = new AbortController();
activeController = controller;
let stopKind: RealtimeFailureKind | null = null;
@@ -344,6 +374,10 @@ export function createBoundedPollCoordinator<Value>(
return pollFailure("PROTOCOL_MISMATCH");
}
result = parsedResult;
observe({
kind: "ATTEMPT_FINISHED",
outcome: result.ok ? result.value.kind : result.error.kind,
});
if (!result.ok) {
// A missing server hint falls back to policy backoff; an executor
// that requires one reports `retryable: false` instead.
+1
View File
@@ -5,6 +5,7 @@ export {
type BoundedPollCoordinator,
type BoundedPollCoordinatorDependencies,
type BoundedPollEnvironment,
type BoundedPollObservation,
type BoundedPollResult,
type BoundedPollRunInput,
} from "./bounded-poll-coordinator.ts";
@@ -20,6 +20,7 @@ import {
hasExactSnapshotKeys,
isRealtimeResult,
isRealtimeTransportEventOutcome,
observeRealtime,
raceRealtimeTask,
realtimeFailure,
timerDeadline,
@@ -124,6 +125,32 @@ export type RealtimeReconnectCoordinator = Readonly<{
close(): void;
}>;
/**
* Closed, low-cardinality diagnostics for design 14.1/14.2: connect attempted
* (ATTEMPT_STARTED), connect finished (SESSION_OPENED, or the RETRY_SCHEDULED
* or RUN_ENDED that follows a failed attempt), connection closed
* (SESSION_CLOSED) and reconnect delay and exhaustion.
*/
export type RealtimeReconnectObservation = Readonly<{
kind:
| "ATTEMPT_STARTED"
| "SESSION_OPENED"
| "SESSION_CLOSED"
| "RETRY_SCHEDULED"
| "RUN_ENDED";
/**
* RETRY_SCHEDULED: the failure being retried. RUN_ENDED: the returned
* failure, absent on success.
*/
failureKind?: RealtimeFailureKind;
delayBucket?: "UNDER_1S" | "1S_TO_10S" | "10S_PLUS";
/**
* RUN_ENDED. EXHAUSTED means a retryable failure outlived the attempt or
* elapsed budget; TERMINAL means the failure itself was not retryable.
*/
reason?: "COMPLETED" | "EXHAUSTED" | "TERMINAL" | "ABORTED";
}>;
export type RealtimeReconnectCoordinatorDependencies<
ClosedReceipt = RealtimeReconnectOutcome<void>,
> = Readonly<{
@@ -152,6 +179,7 @@ export type RealtimeReconnectCoordinatorDependencies<
* event or server close.
*/
isCurrent?: () => boolean;
observe?(observation: RealtimeReconnectObservation): void;
}>;
const RETRYABLE_KINDS = Object.freeze([
@@ -277,6 +305,8 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
let phaseController: AbortController | null = null;
let session: RealtimeReconnectSession<ClosedReceipt> | null = null;
let activeRecoveryGate: BarrierGate | null = null;
const observe = (observation: RealtimeReconnectObservation) =>
observeRealtime(dependencies.observe, observation);
function retainTracked<Value>(
task: Promise<Settled<Value>>,
@@ -316,6 +346,18 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
}
async function run(
input?: RealtimeReconnectRunInput,
): Promise<RealtimeResult<void>> {
const startedGeneration = generation;
const result = await runLoop(input);
// A call rejected before starting leaves the generation unchanged.
if (generation !== startedGeneration) {
observe({ kind: "RUN_ENDED", ...runEnd(result) });
}
return result;
}
async function runLoop(
input: RealtimeReconnectRunInput = {},
): Promise<RealtimeResult<void>> {
if (state === "CLOSED") return failure("CLOSED");
@@ -524,6 +566,16 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
random,
);
if (delay === null) return retry.terminalResult;
observe({
kind: "RETRY_SCHEDULED",
failureKind: retry.result.error.kind,
delayBucket:
delay < 1_000
? "UNDER_1S"
: delay < 10_000
? "1S_TO_10S"
: "10S_PLUS",
});
const phase = beginPhase();
const sleeping = track(
@@ -640,6 +692,7 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
);
},
});
observe({ kind: "ATTEMPT_STARTED" });
const attempting = track(
() => attempt(context),
);
@@ -822,12 +875,14 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
retry = null;
session = active;
observe({ kind: "SESSION_OPENED" });
const waiting = track(() => active.waitClosed());
const closed = await race(
waiting,
phase.controller.signal,
);
const closeOffline = wentOffline(phase);
observe({ kind: "SESSION_CLOSED" });
// Every way an opened session ends, including offline and a throwing
// waitClosed, can earn a fresh budget, not only a classified close.
const endedAt = safeNow(clock);
@@ -1538,6 +1593,25 @@ function reconnectable(failureSnapshot: FailureSnapshot): boolean {
);
}
function runEnd(
result: RealtimeResult<void>,
): Pick<RealtimeReconnectObservation, "failureKind" | "reason"> {
if (result.ok) return { reason: "COMPLETED" };
const { kind, retryable } = result.error;
return {
failureKind: kind,
reason:
kind === "ABORTED" || kind === "CLOSED" || kind === "SCOPE_FENCED"
? "ABORTED"
: retryable &&
RETRYABLE_KINDS.includes(
kind as (typeof RETRYABLE_KINDS)[number],
)
? "EXHAUSTED"
: "TERMINAL",
};
}
function closeLateSession(
outcome: Settled<unknown>,
): void {
+15
View File
@@ -134,6 +134,21 @@ export function raceRealtimeTask<Value>(
);
}
/**
* Best-effort delivery to an optional observation sink. A throwing sink never
* changes realtime control flow.
*/
export function observeRealtime<Observation extends object>(
sink: ((observation: Observation) => void) | undefined,
observation: Observation,
): void {
try {
sink?.(Object.freeze(observation));
} catch {
// Diagnostics never control realtime behavior.
}
}
/**
* A `raceRealtimeTask` deadline on a host timer instead of the injected
* clock, for bounds that must hold even when the clock itself hangs.
@@ -139,6 +139,65 @@ async function flush(): Promise<void> {
}
describe("bounded poll coordinator", () => {
it("observes lease start, each attempt and the stop reason", async () => {
const clock = new ManualClock();
const results: BoundedPollAttemptResult<string>[] = [
{ ok: true, value: { kind: "UNCHANGED", responseBytes: 0 } },
{
ok: false,
error: {
kind: "RATE_LIMITED",
operation: "POLL",
retryable: true,
},
},
{
ok: true,
value: {
kind: "VALUE",
value: "done",
responseBytes: 8,
state: "COMPLETED",
},
},
];
const observations: unknown[] = [];
const coordinator = createBoundedPollCoordinator({
policy,
operation,
execute: async () => results.shift()!,
environment: new MutableEnvironment(),
clock,
random: () => 0.5,
observe: (observation) => {
observations.push(observation);
throw new Error("sink failure must not change the lease");
},
});
let settled = false;
const running = coordinator.run();
void running.then(() => {
settled = true;
});
for (let step = 0; step < 50 && !settled; step += 1) {
clock.advance(1_000);
await flush();
}
await expect(running).resolves.toMatchObject({
ok: true,
value: { kind: "TERMINAL", attempts: 3 },
});
expect(observations).toEqual([
{ kind: "LEASE_STARTED" },
{ kind: "ATTEMPT_FINISHED", outcome: "UNCHANGED" },
{ kind: "ATTEMPT_FINISHED", outcome: "RATE_LIMITED" },
{ kind: "ATTEMPT_FINISHED", outcome: "VALUE" },
{ kind: "LEASE_STOPPED", outcome: "TERMINAL" },
]);
});
it("chains completed attempts without overlap and stops at a terminal state", async () => {
const clock = new ManualClock();
const environment = new MutableEnvironment();
@@ -534,6 +534,79 @@ describe("realtime reconnect coordinator", () => {
},
);
it("observes attempts, retries, the session and why the run ended", async () => {
const clock = new ManualClock();
const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false);
const observations: unknown[] = [];
let attempts = 0;
const coordinator = createRealtimeReconnectCoordinator({
policy: policy(),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0.5,
confirmTransportBarrier: confirmationSucceeded,
observe: (observation) => {
observations.push(observation);
throw new Error("sink failure must not change the run");
},
attempt: () =>
++attempts === 1
? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true))
: opened(
Object.freeze({
classifyClosed: (
receipt: RealtimeReconnectOutcome<void>,
) => receipt,
close: vi.fn(),
waitClosed: () => failed(terminal),
}),
),
});
const running = coordinator.run();
await flushUntil(() => clock.sleepCalls.length === 1);
clock.advance(50);
await expect(running).resolves.toBe(terminal);
expect(observations).toEqual([
{ kind: "ATTEMPT_STARTED" },
{
kind: "RETRY_SCHEDULED",
failureKind: "CONNECT_TIMEOUT",
delayBucket: "UNDER_1S",
},
{ kind: "ATTEMPT_STARTED" },
{ kind: "SESSION_OPENED" },
{ kind: "SESSION_CLOSED" },
{ kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" },
]);
// A second run that spends its retries ends as EXHAUSTED, not TERMINAL.
const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true);
const ends: unknown[] = [];
const spending = createRealtimeReconnectCoordinator({
policy: policy({ maxAttempts: 1 }),
environment: new MutableOnlineEnvironment(true),
clock,
random: () => 0,
confirmTransportBarrier: confirmationSucceeded,
observe: (observation) => {
if (observation.kind === "RUN_ENDED") ends.push(observation);
},
attempt: () => failed(exhausted),
});
const spent = spending.run();
await flushUntil(() => clock.sleepCalls.length === 2);
clock.advance(0);
await expect(spent).resolves.toBe(exhausted);
expect(ends).toEqual([
{
kind: "RUN_ENDED",
reason: "EXHAUSTED",
failureKind: "PROVIDER_UNAVAILABLE",
},
]);
});
it("resets a spent retry only after a valid signal and a stable open", async () => {
const runScenario = async (markValid: boolean, openMs: number) => {
const clock = new ManualClock();