diff --git a/docs/architecture/realtime-events-web-push-and-bounded-polling.md b/docs/architecture/realtime-events-web-push-and-bounded-polling.md index d8701328..70010cc8 100644 --- a/docs/architecture/realtime-events-web-push-and-bounded-polling.md +++ b/docs/architecture/realtime-events-web-push-and-bounded-polling.md @@ -1930,6 +1930,23 @@ push: - `web_push_click_dispatched` - `web_push_association_revoked` +reference runtime의 관측 지점은 다음과 같다. 조합 코드는 attempt나 session을 +감싸지 않고 이 관측을 위 event로 옮긴다. + +| runtime 관측 | 만들 수 있는 event | +| --- | --- | +| reconnect `ATTEMPT_STARTED` | `realtime_connect_attempted` | +| reconnect `SESSION_OPENED`, 또는 open 전의 `RETRY_SCHEDULED`·`RUN_ENDED` | `realtime_connect_finished` | +| reconnect `SESSION_CLOSED` | `realtime_connection_closed` | +| reconnect `RETRY_SCHEDULED`(실패 kind, 대기 bucket), `RUN_ENDED`(`EXHAUSTED`·`TERMINAL`·`ABORTED`·`COMPLETED`) | 14.2 재연결 시도·대기·소진 비율 | +| WebSocket `HEARTBEAT_ACK` | `realtime_heartbeat_observed` | +| polling `LEASE_STARTED`, `ATTEMPT_FINISHED`(`VALUE`·`UNCHANGED`·실패 kind), `LEASE_STOPPED` | `bounded_poll_*` | + +근거는 `reconnect-coordinator.ts:128-182`, +`polling/bounded-poll-coordinator.ts:87-113`, +`websocket/websocket-connection.ts:376-378`이다. sink가 throw해도 제어 흐름은 +바뀌지 않는다(`result.ts`의 `observeRealtime`). + 모든 event는 semantic registry에 등록하고 arbitrary payload를 받지 않는다. diagnostics sink failure는 connection, worker handler, poll lease와 cleanup을 실패시키지 않으며 재귀 telemetry를 만들지 않는다. diff --git a/src/adapters/realtime/index.ts b/src/adapters/realtime/index.ts index 953b0181..a219ef49 100644 --- a/src/adapters/realtime/index.ts +++ b/src/adapters/realtime/index.ts @@ -29,6 +29,7 @@ export { type RealtimeReconnectCoordinator, type RealtimeReconnectCoordinatorDependencies, type RealtimeReconnectEnvironment, + type RealtimeReconnectObservation, type RealtimeReconnectOutcome, type RealtimeReconnectRunInput, type RealtimeReconnectSession, diff --git a/src/adapters/realtime/polling/bounded-poll-coordinator.ts b/src/adapters/realtime/polling/bounded-poll-coordinator.ts index 3ca803a1..dd463a8f 100644 --- a/src/adapters/realtime/polling/bounded-poll-coordinator.ts +++ b/src/adapters/realtime/polling/bounded-poll-coordinator.ts @@ -15,6 +15,7 @@ import { systemClock } from "../../platform/system-clock.ts"; import { captureRealtimeDataSnapshot, hasExactSnapshotKeys, + observeRealtime, raceRealtimeTask, realtimeFailure, realtimeSuccess, @@ -83,6 +84,16 @@ export type BoundedPollCoordinator = Readonly<{ close(): void; }>; +/** + * Closed diagnostics for design 14.1 `bounded_poll_*`. `outcome` is VALUE, + * UNCHANGED or the failure kind for ATTEMPT_FINISHED, and TERMINAL or the + * failure kind for LEASE_STOPPED. + */ +export type BoundedPollObservation = Readonly<{ + kind: "LEASE_STARTED" | "ATTEMPT_FINISHED" | "LEASE_STOPPED"; + outcome?: "VALUE" | "UNCHANGED" | "TERMINAL" | RealtimeFailureKind; +}>; + export type BoundedPollCoordinatorDependencies = Readonly<{ policy: PollLeasePolicy; operation: BoundedPollOperationContract; @@ -99,6 +110,7 @@ export type BoundedPollCoordinatorDependencies = Readonly<{ isCurrent?: () => boolean; clock?: ClockPort; random?: () => number; + observe?(observation: BoundedPollObservation): void; }>; const SAFE_STATE = /^[A-Z][A-Z0-9_]{0,63}$/u; @@ -128,8 +140,25 @@ export function createBoundedPollCoordinator( let activeController: AbortController | null = null; let generation = 0; let pendingWork = 0; + const observe = (observation: BoundedPollObservation) => + observeRealtime(dependencies.observe, observation); async function run( + input?: BoundedPollRunInput, + ): Promise> { + const startedGeneration = generation; + const result = await runLease(input); + // A call rejected before starting leaves the generation unchanged. + if (generation !== startedGeneration) { + observe({ + kind: "LEASE_STOPPED", + outcome: result.ok ? "TERMINAL" : result.error.kind, + }); + } + return result; + } + + async function runLease( input: BoundedPollRunInput = {}, ): Promise> { if (state === "CLOSED") return pollFailure("CLOSED"); @@ -149,6 +178,7 @@ export function createBoundedPollCoordinator( state = "RUNNING"; const runGeneration = ++generation; + observe({ kind: "LEASE_STARTED" }); const controller = new AbortController(); activeController = controller; let stopKind: RealtimeFailureKind | null = null; @@ -344,6 +374,10 @@ export function createBoundedPollCoordinator( return pollFailure("PROTOCOL_MISMATCH"); } result = parsedResult; + observe({ + kind: "ATTEMPT_FINISHED", + outcome: result.ok ? result.value.kind : result.error.kind, + }); if (!result.ok) { // A missing server hint falls back to policy backoff; an executor // that requires one reports `retryable: false` instead. diff --git a/src/adapters/realtime/polling/index.ts b/src/adapters/realtime/polling/index.ts index 076157f0..8855ffbf 100644 --- a/src/adapters/realtime/polling/index.ts +++ b/src/adapters/realtime/polling/index.ts @@ -5,6 +5,7 @@ export { type BoundedPollCoordinator, type BoundedPollCoordinatorDependencies, type BoundedPollEnvironment, + type BoundedPollObservation, type BoundedPollResult, type BoundedPollRunInput, } from "./bounded-poll-coordinator.ts"; diff --git a/src/adapters/realtime/reconnect-coordinator.ts b/src/adapters/realtime/reconnect-coordinator.ts index 664c1b43..1d67adc1 100644 --- a/src/adapters/realtime/reconnect-coordinator.ts +++ b/src/adapters/realtime/reconnect-coordinator.ts @@ -20,6 +20,7 @@ import { hasExactSnapshotKeys, isRealtimeResult, isRealtimeTransportEventOutcome, + observeRealtime, raceRealtimeTask, realtimeFailure, timerDeadline, @@ -124,6 +125,32 @@ export type RealtimeReconnectCoordinator = Readonly<{ close(): void; }>; +/** + * Closed, low-cardinality diagnostics for design 14.1/14.2: connect attempted + * (ATTEMPT_STARTED), connect finished (SESSION_OPENED, or the RETRY_SCHEDULED + * or RUN_ENDED that follows a failed attempt), connection closed + * (SESSION_CLOSED) and reconnect delay and exhaustion. + */ +export type RealtimeReconnectObservation = Readonly<{ + kind: + | "ATTEMPT_STARTED" + | "SESSION_OPENED" + | "SESSION_CLOSED" + | "RETRY_SCHEDULED" + | "RUN_ENDED"; + /** + * RETRY_SCHEDULED: the failure being retried. RUN_ENDED: the returned + * failure, absent on success. + */ + failureKind?: RealtimeFailureKind; + delayBucket?: "UNDER_1S" | "1S_TO_10S" | "10S_PLUS"; + /** + * RUN_ENDED. EXHAUSTED means a retryable failure outlived the attempt or + * elapsed budget; TERMINAL means the failure itself was not retryable. + */ + reason?: "COMPLETED" | "EXHAUSTED" | "TERMINAL" | "ABORTED"; +}>; + export type RealtimeReconnectCoordinatorDependencies< ClosedReceipt = RealtimeReconnectOutcome, > = Readonly<{ @@ -152,6 +179,7 @@ export type RealtimeReconnectCoordinatorDependencies< * event or server close. */ isCurrent?: () => boolean; + observe?(observation: RealtimeReconnectObservation): void; }>; const RETRYABLE_KINDS = Object.freeze([ @@ -277,6 +305,8 @@ export function createRealtimeReconnectCoordinator( let phaseController: AbortController | null = null; let session: RealtimeReconnectSession | null = null; let activeRecoveryGate: BarrierGate | null = null; + const observe = (observation: RealtimeReconnectObservation) => + observeRealtime(dependencies.observe, observation); function retainTracked( task: Promise>, @@ -316,6 +346,18 @@ export function createRealtimeReconnectCoordinator( } async function run( + input?: RealtimeReconnectRunInput, + ): Promise> { + const startedGeneration = generation; + const result = await runLoop(input); + // A call rejected before starting leaves the generation unchanged. + if (generation !== startedGeneration) { + observe({ kind: "RUN_ENDED", ...runEnd(result) }); + } + return result; + } + + async function runLoop( input: RealtimeReconnectRunInput = {}, ): Promise> { if (state === "CLOSED") return failure("CLOSED"); @@ -524,6 +566,16 @@ export function createRealtimeReconnectCoordinator( random, ); if (delay === null) return retry.terminalResult; + observe({ + kind: "RETRY_SCHEDULED", + failureKind: retry.result.error.kind, + delayBucket: + delay < 1_000 + ? "UNDER_1S" + : delay < 10_000 + ? "1S_TO_10S" + : "10S_PLUS", + }); const phase = beginPhase(); const sleeping = track( @@ -640,6 +692,7 @@ export function createRealtimeReconnectCoordinator( ); }, }); + observe({ kind: "ATTEMPT_STARTED" }); const attempting = track( () => attempt(context), ); @@ -822,12 +875,14 @@ export function createRealtimeReconnectCoordinator( retry = null; session = active; + observe({ kind: "SESSION_OPENED" }); const waiting = track(() => active.waitClosed()); const closed = await race( waiting, phase.controller.signal, ); const closeOffline = wentOffline(phase); + observe({ kind: "SESSION_CLOSED" }); // Every way an opened session ends, including offline and a throwing // waitClosed, can earn a fresh budget, not only a classified close. const endedAt = safeNow(clock); @@ -1538,6 +1593,25 @@ function reconnectable(failureSnapshot: FailureSnapshot): boolean { ); } +function runEnd( + result: RealtimeResult, +): Pick { + if (result.ok) return { reason: "COMPLETED" }; + const { kind, retryable } = result.error; + return { + failureKind: kind, + reason: + kind === "ABORTED" || kind === "CLOSED" || kind === "SCOPE_FENCED" + ? "ABORTED" + : retryable && + RETRYABLE_KINDS.includes( + kind as (typeof RETRYABLE_KINDS)[number], + ) + ? "EXHAUSTED" + : "TERMINAL", + }; +} + function closeLateSession( outcome: Settled, ): void { diff --git a/src/adapters/realtime/result.ts b/src/adapters/realtime/result.ts index 9325af0d..9bcb900e 100644 --- a/src/adapters/realtime/result.ts +++ b/src/adapters/realtime/result.ts @@ -134,6 +134,21 @@ export function raceRealtimeTask( ); } +/** + * Best-effort delivery to an optional observation sink. A throwing sink never + * changes realtime control flow. + */ +export function observeRealtime( + sink: ((observation: Observation) => void) | undefined, + observation: Observation, +): void { + try { + sink?.(Object.freeze(observation)); + } catch { + // Diagnostics never control realtime behavior. + } +} + /** * A `raceRealtimeTask` deadline on a host timer instead of the injected * clock, for bounds that must hold even when the clock itself hangs. diff --git a/tests/unit/realtime/bounded-poll-coordinator.test.ts b/tests/unit/realtime/bounded-poll-coordinator.test.ts index 9f08f28b..acbddaa4 100644 --- a/tests/unit/realtime/bounded-poll-coordinator.test.ts +++ b/tests/unit/realtime/bounded-poll-coordinator.test.ts @@ -139,6 +139,65 @@ async function flush(): Promise { } describe("bounded poll coordinator", () => { + it("observes lease start, each attempt and the stop reason", async () => { + const clock = new ManualClock(); + const results: BoundedPollAttemptResult[] = [ + { ok: true, value: { kind: "UNCHANGED", responseBytes: 0 } }, + { + ok: false, + error: { + kind: "RATE_LIMITED", + operation: "POLL", + retryable: true, + }, + }, + { + ok: true, + value: { + kind: "VALUE", + value: "done", + responseBytes: 8, + state: "COMPLETED", + }, + }, + ]; + const observations: unknown[] = []; + const coordinator = createBoundedPollCoordinator({ + policy, + operation, + execute: async () => results.shift()!, + environment: new MutableEnvironment(), + clock, + random: () => 0.5, + observe: (observation) => { + observations.push(observation); + throw new Error("sink failure must not change the lease"); + }, + }); + + let settled = false; + const running = coordinator.run(); + void running.then(() => { + settled = true; + }); + for (let step = 0; step < 50 && !settled; step += 1) { + clock.advance(1_000); + await flush(); + } + + await expect(running).resolves.toMatchObject({ + ok: true, + value: { kind: "TERMINAL", attempts: 3 }, + }); + expect(observations).toEqual([ + { kind: "LEASE_STARTED" }, + { kind: "ATTEMPT_FINISHED", outcome: "UNCHANGED" }, + { kind: "ATTEMPT_FINISHED", outcome: "RATE_LIMITED" }, + { kind: "ATTEMPT_FINISHED", outcome: "VALUE" }, + { kind: "LEASE_STOPPED", outcome: "TERMINAL" }, + ]); + }); + it("chains completed attempts without overlap and stops at a terminal state", async () => { const clock = new ManualClock(); const environment = new MutableEnvironment(); diff --git a/tests/unit/realtime/realtime-reconnect-coordinator.test.ts b/tests/unit/realtime/realtime-reconnect-coordinator.test.ts index 5d9a06d4..a3003fea 100644 --- a/tests/unit/realtime/realtime-reconnect-coordinator.test.ts +++ b/tests/unit/realtime/realtime-reconnect-coordinator.test.ts @@ -534,6 +534,79 @@ describe("realtime reconnect coordinator", () => { }, ); + it("observes attempts, retries, the session and why the run ended", async () => { + const clock = new ManualClock(); + const terminal = realtimeFailure("FORBIDDEN", "RECEIVE", false); + const observations: unknown[] = []; + let attempts = 0; + const coordinator = createRealtimeReconnectCoordinator({ + policy: policy(), + environment: new MutableOnlineEnvironment(true), + clock, + random: () => 0.5, + confirmTransportBarrier: confirmationSucceeded, + observe: (observation) => { + observations.push(observation); + throw new Error("sink failure must not change the run"); + }, + attempt: () => + ++attempts === 1 + ? failed(realtimeFailure("CONNECT_TIMEOUT", "CONNECT", true)) + : opened( + Object.freeze({ + classifyClosed: ( + receipt: RealtimeReconnectOutcome, + ) => receipt, + close: vi.fn(), + waitClosed: () => failed(terminal), + }), + ), + }); + + const running = coordinator.run(); + await flushUntil(() => clock.sleepCalls.length === 1); + clock.advance(50); + await expect(running).resolves.toBe(terminal); + expect(observations).toEqual([ + { kind: "ATTEMPT_STARTED" }, + { + kind: "RETRY_SCHEDULED", + failureKind: "CONNECT_TIMEOUT", + delayBucket: "UNDER_1S", + }, + { kind: "ATTEMPT_STARTED" }, + { kind: "SESSION_OPENED" }, + { kind: "SESSION_CLOSED" }, + { kind: "RUN_ENDED", reason: "TERMINAL", failureKind: "FORBIDDEN" }, + ]); + + // A second run that spends its retries ends as EXHAUSTED, not TERMINAL. + const exhausted = realtimeFailure("PROVIDER_UNAVAILABLE", "CONNECT", true); + const ends: unknown[] = []; + const spending = createRealtimeReconnectCoordinator({ + policy: policy({ maxAttempts: 1 }), + environment: new MutableOnlineEnvironment(true), + clock, + random: () => 0, + confirmTransportBarrier: confirmationSucceeded, + observe: (observation) => { + if (observation.kind === "RUN_ENDED") ends.push(observation); + }, + attempt: () => failed(exhausted), + }); + const spent = spending.run(); + await flushUntil(() => clock.sleepCalls.length === 2); + clock.advance(0); + await expect(spent).resolves.toBe(exhausted); + expect(ends).toEqual([ + { + kind: "RUN_ENDED", + reason: "EXHAUSTED", + failureKind: "PROVIDER_UNAVAILABLE", + }, + ]); + }); + it("resets a spent retry only after a valid signal and a stable open", async () => { const runScenario = async (markValid: boolean, openMs: number) => { const clock = new ManualClock();