fix(realtime): add observe hooks to the reconnect and poll coordinators (04-07)
Design 14.1 connect and bounded_poll events had no place to come from, and an exhausted budget looked the same as a terminal failure. Both coordinators now emit a closed observation vocabulary through a best-effort sink. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
905189ef70
commit
03991aaafb
@@ -1930,6 +1930,23 @@ push:
|
||||
- `web_push_click_dispatched`
|
||||
- `web_push_association_revoked`
|
||||
|
||||
reference runtime의 관측 지점은 다음과 같다. 조합 코드는 attempt나 session을
|
||||
감싸지 않고 이 관측을 위 event로 옮긴다.
|
||||
|
||||
| runtime 관측 | 만들 수 있는 event |
|
||||
| --- | --- |
|
||||
| reconnect `ATTEMPT_STARTED` | `realtime_connect_attempted` |
|
||||
| reconnect `SESSION_OPENED`, 또는 open 전의 `RETRY_SCHEDULED`·`RUN_ENDED` | `realtime_connect_finished` |
|
||||
| reconnect `SESSION_CLOSED` | `realtime_connection_closed` |
|
||||
| reconnect `RETRY_SCHEDULED`(실패 kind, 대기 bucket), `RUN_ENDED`(`EXHAUSTED`·`TERMINAL`·`ABORTED`·`COMPLETED`) | 14.2 재연결 시도·대기·소진 비율 |
|
||||
| WebSocket `HEARTBEAT_ACK` | `realtime_heartbeat_observed` |
|
||||
| polling `LEASE_STARTED`, `ATTEMPT_FINISHED`(`VALUE`·`UNCHANGED`·실패 kind), `LEASE_STOPPED` | `bounded_poll_*` |
|
||||
|
||||
근거는 `reconnect-coordinator.ts:128-182`,
|
||||
`polling/bounded-poll-coordinator.ts:87-113`,
|
||||
`websocket/websocket-connection.ts:376-378`이다. sink가 throw해도 제어 흐름은
|
||||
바뀌지 않는다(`result.ts`의 `observeRealtime`).
|
||||
|
||||
모든 event는 semantic registry에 등록하고 arbitrary payload를 받지 않는다.
|
||||
diagnostics sink failure는 connection, worker handler, poll lease와 cleanup을
|
||||
실패시키지 않으며 재귀 telemetry를 만들지 않는다.
|
||||
|
||||
@@ -29,6 +29,7 @@ export {
|
||||
type RealtimeReconnectCoordinator,
|
||||
type RealtimeReconnectCoordinatorDependencies,
|
||||
type RealtimeReconnectEnvironment,
|
||||
type RealtimeReconnectObservation,
|
||||
type RealtimeReconnectOutcome,
|
||||
type RealtimeReconnectRunInput,
|
||||
type RealtimeReconnectSession,
|
||||
|
||||
@@ -15,6 +15,7 @@ import { systemClock } from "../../platform/system-clock.ts";
|
||||
import {
|
||||
captureRealtimeDataSnapshot,
|
||||
hasExactSnapshotKeys,
|
||||
observeRealtime,
|
||||
raceRealtimeTask,
|
||||
realtimeFailure,
|
||||
realtimeSuccess,
|
||||
@@ -83,6 +84,16 @@ export type BoundedPollCoordinator<Value> = Readonly<{
|
||||
close(): void;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* Closed diagnostics for design 14.1 `bounded_poll_*`. `outcome` is VALUE,
|
||||
* UNCHANGED or the failure kind for ATTEMPT_FINISHED, and TERMINAL or the
|
||||
* failure kind for LEASE_STOPPED.
|
||||
*/
|
||||
export type BoundedPollObservation = Readonly<{
|
||||
kind: "LEASE_STARTED" | "ATTEMPT_FINISHED" | "LEASE_STOPPED";
|
||||
outcome?: "VALUE" | "UNCHANGED" | "TERMINAL" | RealtimeFailureKind;
|
||||
}>;
|
||||
|
||||
export type BoundedPollCoordinatorDependencies<Value> = Readonly<{
|
||||
policy: PollLeasePolicy;
|
||||
operation: BoundedPollOperationContract;
|
||||
@@ -99,6 +110,7 @@ export type BoundedPollCoordinatorDependencies<Value> = Readonly<{
|
||||
isCurrent?: () => boolean;
|
||||
clock?: ClockPort;
|
||||
random?: () => number;
|
||||
observe?(observation: BoundedPollObservation): void;
|
||||
}>;
|
||||
|
||||
const SAFE_STATE = /^[A-Z][A-Z0-9_]{0,63}$/u;
|
||||
@@ -128,8 +140,25 @@ export function createBoundedPollCoordinator<Value>(
|
||||
let activeController: AbortController | null = null;
|
||||
let generation = 0;
|
||||
let pendingWork = 0;
|
||||
const observe = (observation: BoundedPollObservation) =>
|
||||
observeRealtime(dependencies.observe, observation);
|
||||
|
||||
async function run(
|
||||
input?: BoundedPollRunInput<Value>,
|
||||
): Promise<BoundedPollResult<Value>> {
|
||||
const startedGeneration = generation;
|
||||
const result = await runLease(input);
|
||||
// A call rejected before starting leaves the generation unchanged.
|
||||
if (generation !== startedGeneration) {
|
||||
observe({
|
||||
kind: "LEASE_STOPPED",
|
||||
outcome: result.ok ? "TERMINAL" : result.error.kind,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runLease(
|
||||
input: BoundedPollRunInput<Value> = {},
|
||||
): Promise<BoundedPollResult<Value>> {
|
||||
if (state === "CLOSED") return pollFailure("CLOSED");
|
||||
@@ -149,6 +178,7 @@ export function createBoundedPollCoordinator<Value>(
|
||||
|
||||
state = "RUNNING";
|
||||
const runGeneration = ++generation;
|
||||
observe({ kind: "LEASE_STARTED" });
|
||||
const controller = new AbortController();
|
||||
activeController = controller;
|
||||
let stopKind: RealtimeFailureKind | null = null;
|
||||
@@ -344,6 +374,10 @@ export function createBoundedPollCoordinator<Value>(
|
||||
return pollFailure("PROTOCOL_MISMATCH");
|
||||
}
|
||||
result = parsedResult;
|
||||
observe({
|
||||
kind: "ATTEMPT_FINISHED",
|
||||
outcome: result.ok ? result.value.kind : result.error.kind,
|
||||
});
|
||||
if (!result.ok) {
|
||||
// A missing server hint falls back to policy backoff; an executor
|
||||
// that requires one reports `retryable: false` instead.
|
||||
|
||||
@@ -5,6 +5,7 @@ export {
|
||||
type BoundedPollCoordinator,
|
||||
type BoundedPollCoordinatorDependencies,
|
||||
type BoundedPollEnvironment,
|
||||
type BoundedPollObservation,
|
||||
type BoundedPollResult,
|
||||
type BoundedPollRunInput,
|
||||
} from "./bounded-poll-coordinator.ts";
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
hasExactSnapshotKeys,
|
||||
isRealtimeResult,
|
||||
isRealtimeTransportEventOutcome,
|
||||
observeRealtime,
|
||||
raceRealtimeTask,
|
||||
realtimeFailure,
|
||||
timerDeadline,
|
||||
@@ -124,6 +125,32 @@ export type RealtimeReconnectCoordinator = Readonly<{
|
||||
close(): void;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* Closed, low-cardinality diagnostics for design 14.1/14.2: connect attempted
|
||||
* (ATTEMPT_STARTED), connect finished (SESSION_OPENED, or the RETRY_SCHEDULED
|
||||
* or RUN_ENDED that follows a failed attempt), connection closed
|
||||
* (SESSION_CLOSED) and reconnect delay and exhaustion.
|
||||
*/
|
||||
export type RealtimeReconnectObservation = Readonly<{
|
||||
kind:
|
||||
| "ATTEMPT_STARTED"
|
||||
| "SESSION_OPENED"
|
||||
| "SESSION_CLOSED"
|
||||
| "RETRY_SCHEDULED"
|
||||
| "RUN_ENDED";
|
||||
/**
|
||||
* RETRY_SCHEDULED: the failure being retried. RUN_ENDED: the returned
|
||||
* failure, absent on success.
|
||||
*/
|
||||
failureKind?: RealtimeFailureKind;
|
||||
delayBucket?: "UNDER_1S" | "1S_TO_10S" | "10S_PLUS";
|
||||
/**
|
||||
* RUN_ENDED. EXHAUSTED means a retryable failure outlived the attempt or
|
||||
* elapsed budget; TERMINAL means the failure itself was not retryable.
|
||||
*/
|
||||
reason?: "COMPLETED" | "EXHAUSTED" | "TERMINAL" | "ABORTED";
|
||||
}>;
|
||||
|
||||
export type RealtimeReconnectCoordinatorDependencies<
|
||||
ClosedReceipt = RealtimeReconnectOutcome<void>,
|
||||
> = Readonly<{
|
||||
@@ -152,6 +179,7 @@ export type RealtimeReconnectCoordinatorDependencies<
|
||||
* event or server close.
|
||||
*/
|
||||
isCurrent?: () => boolean;
|
||||
observe?(observation: RealtimeReconnectObservation): void;
|
||||
}>;
|
||||
|
||||
const RETRYABLE_KINDS = Object.freeze([
|
||||
@@ -277,6 +305,8 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
let phaseController: AbortController | null = null;
|
||||
let session: RealtimeReconnectSession<ClosedReceipt> | null = null;
|
||||
let activeRecoveryGate: BarrierGate | null = null;
|
||||
const observe = (observation: RealtimeReconnectObservation) =>
|
||||
observeRealtime(dependencies.observe, observation);
|
||||
|
||||
function retainTracked<Value>(
|
||||
task: Promise<Settled<Value>>,
|
||||
@@ -316,6 +346,18 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
}
|
||||
|
||||
async function run(
|
||||
input?: RealtimeReconnectRunInput,
|
||||
): Promise<RealtimeResult<void>> {
|
||||
const startedGeneration = generation;
|
||||
const result = await runLoop(input);
|
||||
// A call rejected before starting leaves the generation unchanged.
|
||||
if (generation !== startedGeneration) {
|
||||
observe({ kind: "RUN_ENDED", ...runEnd(result) });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runLoop(
|
||||
input: RealtimeReconnectRunInput = {},
|
||||
): Promise<RealtimeResult<void>> {
|
||||
if (state === "CLOSED") return failure("CLOSED");
|
||||
@@ -524,6 +566,16 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
random,
|
||||
);
|
||||
if (delay === null) return retry.terminalResult;
|
||||
observe({
|
||||
kind: "RETRY_SCHEDULED",
|
||||
failureKind: retry.result.error.kind,
|
||||
delayBucket:
|
||||
delay < 1_000
|
||||
? "UNDER_1S"
|
||||
: delay < 10_000
|
||||
? "1S_TO_10S"
|
||||
: "10S_PLUS",
|
||||
});
|
||||
|
||||
const phase = beginPhase();
|
||||
const sleeping = track(
|
||||
@@ -640,6 +692,7 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
);
|
||||
},
|
||||
});
|
||||
observe({ kind: "ATTEMPT_STARTED" });
|
||||
const attempting = track(
|
||||
() => attempt(context),
|
||||
);
|
||||
@@ -822,12 +875,14 @@ export function createRealtimeReconnectCoordinator<ClosedReceipt>(
|
||||
retry = null;
|
||||
|
||||
session = active;
|
||||
observe({ kind: "SESSION_OPENED" });
|
||||
const waiting = track(() => active.waitClosed());
|
||||
const closed = await race(
|
||||
waiting,
|
||||
phase.controller.signal,
|
||||
);
|
||||
const closeOffline = wentOffline(phase);
|
||||
observe({ kind: "SESSION_CLOSED" });
|
||||
// Every way an opened session ends, including offline and a throwing
|
||||
// waitClosed, can earn a fresh budget, not only a classified close.
|
||||
const endedAt = safeNow(clock);
|
||||
@@ -1538,6 +1593,25 @@ function reconnectable(failureSnapshot: FailureSnapshot): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function runEnd(
|
||||
result: RealtimeResult<void>,
|
||||
): Pick<RealtimeReconnectObservation, "failureKind" | "reason"> {
|
||||
if (result.ok) return { reason: "COMPLETED" };
|
||||
const { kind, retryable } = result.error;
|
||||
return {
|
||||
failureKind: kind,
|
||||
reason:
|
||||
kind === "ABORTED" || kind === "CLOSED" || kind === "SCOPE_FENCED"
|
||||
? "ABORTED"
|
||||
: retryable &&
|
||||
RETRYABLE_KINDS.includes(
|
||||
kind as (typeof RETRYABLE_KINDS)[number],
|
||||
)
|
||||
? "EXHAUSTED"
|
||||
: "TERMINAL",
|
||||
};
|
||||
}
|
||||
|
||||
function closeLateSession(
|
||||
outcome: Settled<unknown>,
|
||||
): void {
|
||||
|
||||
@@ -134,6 +134,21 @@ export function raceRealtimeTask<Value>(
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort delivery to an optional observation sink. A throwing sink never
|
||||
* changes realtime control flow.
|
||||
*/
|
||||
export function observeRealtime<Observation extends object>(
|
||||
sink: ((observation: Observation) => void) | undefined,
|
||||
observation: Observation,
|
||||
): void {
|
||||
try {
|
||||
sink?.(Object.freeze(observation));
|
||||
} catch {
|
||||
// Diagnostics never control realtime behavior.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A `raceRealtimeTask` deadline on a host timer instead of the injected
|
||||
* clock, for bounds that must hold even when the clock itself hangs.
|
||||
|
||||
@@ -139,6 +139,65 @@ async function flush(): Promise<void> {
|
||||
}
|
||||
|
||||
describe("bounded poll coordinator", () => {
|
||||
it("observes lease start, each attempt and the stop reason", async () => {
|
||||
const clock = new ManualClock();
|
||||
const results: BoundedPollAttemptResult<string>[] = [
|
||||
{ ok: true, value: { kind: "UNCHANGED", responseBytes: 0 } },
|
||||
{
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "RATE_LIMITED",
|
||||
operation: "POLL",
|
||||
retryable: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
ok: true,
|
||||
value: {
|
||||
kind: "VALUE",
|
||||
value: "done",
|
||||
responseBytes: 8,
|
||||
state: "COMPLETED",
|
||||
},
|
||||
},
|
||||
];
|
||||
const observations: unknown[] = [];
|
||||
const coordinator = createBoundedPollCoordinator({
|
||||
policy,
|
||||
operation,
|
||||
execute: async () => results.shift()!,
|
||||
environment: new MutableEnvironment(),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
observe: (observation) => {
|
||||
observations.push(observation);
|
||||
throw new Error("sink failure must not change the lease");
|
||||
},
|
||||
});
|
||||
|
||||
let settled = false;
|
||||
const running = coordinator.run();
|
||||
void running.then(() => {
|
||||
settled = true;
|
||||
});
|
||||
for (let step = 0; step < 50 && !settled; step += 1) {
|
||||
clock.advance(1_000);
|
||||
await flush();
|
||||
}
|
||||
|
||||
await expect(running).resolves.toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "TERMINAL", attempts: 3 },
|
||||
});
|
||||
expect(observations).toEqual([
|
||||
{ kind: "LEASE_STARTED" },
|
||||
{ kind: "ATTEMPT_FINISHED", outcome: "UNCHANGED" },
|
||||
{ kind: "ATTEMPT_FINISHED", outcome: "RATE_LIMITED" },
|
||||
{ kind: "ATTEMPT_FINISHED", outcome: "VALUE" },
|
||||
{ kind: "LEASE_STOPPED", outcome: "TERMINAL" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("chains completed attempts without overlap and stops at a terminal state", async () => {
|
||||
const clock = new ManualClock();
|
||||
const environment = new MutableEnvironment();
|
||||
|
||||
@@ -534,6 +534,79 @@ describe("realtime reconnect coordinator", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("observes attempts, retries, the session and why the run ended", async () => {
|
||||
const clock = new ManualClock();
|
||||
const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false);
|
||||
const observations: unknown[] = [];
|
||||
let attempts = 0;
|
||||
const coordinator = createRealtimeReconnectCoordinator({
|
||||
policy: policy(),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0.5,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
observe: (observation) => {
|
||||
observations.push(observation);
|
||||
throw new Error("sink failure must not change the run");
|
||||
},
|
||||
attempt: () =>
|
||||
++attempts === 1
|
||||
? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true))
|
||||
: opened(
|
||||
Object.freeze({
|
||||
classifyClosed: (
|
||||
receipt: RealtimeReconnectOutcome<void>,
|
||||
) => receipt,
|
||||
close: vi.fn(),
|
||||
waitClosed: () => failed(terminal),
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
const running = coordinator.run();
|
||||
await flushUntil(() => clock.sleepCalls.length === 1);
|
||||
clock.advance(50);
|
||||
await expect(running).resolves.toBe(terminal);
|
||||
expect(observations).toEqual([
|
||||
{ kind: "ATTEMPT_STARTED" },
|
||||
{
|
||||
kind: "RETRY_SCHEDULED",
|
||||
failureKind: "CONNECT_TIMEOUT",
|
||||
delayBucket: "UNDER_1S",
|
||||
},
|
||||
{ kind: "ATTEMPT_STARTED" },
|
||||
{ kind: "SESSION_OPENED" },
|
||||
{ kind: "SESSION_CLOSED" },
|
||||
{ kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" },
|
||||
]);
|
||||
|
||||
// A second run that spends its retries ends as EXHAUSTED, not TERMINAL.
|
||||
const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true);
|
||||
const ends: unknown[] = [];
|
||||
const spending = createRealtimeReconnectCoordinator({
|
||||
policy: policy({ maxAttempts: 1 }),
|
||||
environment: new MutableOnlineEnvironment(true),
|
||||
clock,
|
||||
random: () => 0,
|
||||
confirmTransportBarrier: confirmationSucceeded,
|
||||
observe: (observation) => {
|
||||
if (observation.kind === "RUN_ENDED") ends.push(observation);
|
||||
},
|
||||
attempt: () => failed(exhausted),
|
||||
});
|
||||
const spent = spending.run();
|
||||
await flushUntil(() => clock.sleepCalls.length === 2);
|
||||
clock.advance(0);
|
||||
await expect(spent).resolves.toBe(exhausted);
|
||||
expect(ends).toEqual([
|
||||
{
|
||||
kind: "RUN_ENDED",
|
||||
reason: "EXHAUSTED",
|
||||
failureKind: "PROVIDER_UNAVAILABLE",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("resets a spent retry only after a valid signal and a stable open", async () => {
|
||||
const runScenario = async (markValid: boolean, openMs: number) => {
|
||||
const clock = new ManualClock();
|
||||
|
||||
Reference in New Issue
Block a user