donghyeon.kang/clean-architecture-frontend-template
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix: retain realtime work through draining

Browse Source 
R-02: add an OPEN/DRAINING/CLOSED lifecycle orthogonal to freshness. Effect and
recovery authorities are now awaited under a deadline: on expiry the commit
capability is revoked and the work aborted, the caller gets a bounded
non-retryable IDLE_TIMEOUT, and the underlying task is retained rather than
dropped. A draining stream refuses new events and recovery, and close() returns
a Promise that succeeds only once every retained task actually settled,
reporting IDLE_TIMEOUT otherwise.

R-03: a handoff fail-close moves active, probe, quiescing and transition leases
into a retired-writer set before clearing their references, and close() waits on
current and retired writers together, so an abandoned non-cooperative writer can
no longer make teardown report a false success.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-08-13 23:52:46 +09:00
co-authored by Claude Opus 5
parent c9e820aed5
commit 2f29ccbf1a
7 changed files with 409 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/adapters/realtime/live-poll-handoff-coordinator.ts
+31 -4
View File
@@ -168,6 +168,11 @@ export function createLivePollHandoffCoordinator<Value>(
let quiescing: InternalWriterLease<Value> | null = null;
let transitionCandidate: InternalWriterLease<Value> | null = null;
let closePromise: Promise<RealtimeResult<void>> | null = null;
/**
* R-03. Writers whose lease was fail-closed but whose tail may still be
* running. Membership keeps `close()` honest about quiescence.
*/
const retiredWriters = new Set<InternalWriterLease<Value>>();
active = createWriterLease(dependencies.initial.writer);
@@ -673,11 +678,15 @@ export function createLivePollHandoffCoordinator<Value>(
lifecycleGeneration += 1;
state = "CLOSED";
transitioning = true;
// R-03. Current and previously retired writers are waited on together and
// deduplicated, so a writer dropped by an overflow fail-close is still
// proved quiescent before close reports success.
const writers = uniqueLeases([
active,
probe?.lease ?? null,
quiescing,
transitionCandidate,
...retiredWriters,
]);
active = null;
const selectedProbe = probe;
@@ -691,6 +700,11 @@ export function createLivePollHandoffCoordinator<Value>(
quiescing = null;
transitionCandidate = null;
transitioning = false;
for (const [index, writer] of writers.entries()) {
// Only an actually settled writer leaves the retired set; the rest keep
// the coordinator DRAINING.
if (outcomes[index] === "QUIESCED") retiredWriters.delete(writer);
}
if (outcomes.includes("TIMED_OUT")) {
return handoffFailure("IDLE_TIMEOUT", "CLOSE");
}
@@ -771,20 +785,33 @@ export function createLivePollHandoffCoordinator<Value>(
);
}
/**
* R-03. A fail-close aborts every lease, but the underlying writers may be
* non-cooperative and still running. They move into the retired set before
* their references are cleared, so a later `close()` cannot report success
* while an abandoned writer is still executing.
*/
function failClosed(): void {
lifecycleGeneration += 1;
state = "CLOSED";
transitioning = false;
active?.controller.abort();
probe?.lease.controller.abort();
quiescing?.controller.abort();
transitionCandidate?.controller.abort();
for (const writer of uniqueLeases([
active,
probe?.lease ?? null,
quiescing,
transitionCandidate,
])) {
writer.controller.abort();
retiredWriters.add(writer);
}
active = null;
if (probe) {
probe.buffer.length = 0;
probe.bufferedBytes = 0;
}
probe = null;
quiescing = null;
transitionCandidate = null;
}
return Object.freeze({
src/adapters/realtime/stream-coordinator.ts
+214 -22
View File
@@ -41,6 +41,27 @@ import {
type RealtimeDataSnapshot,
} from "./result.ts";
/**
* R-02. Lifecycle is orthogonal to freshness. `DRAINING` means the coordinator
* has revoked commit capability and stopped admitting work, but a
* non-cooperative task it started is still running and is deliberately retained
* until it actually settles.
*/
export type RealtimeStreamLifecycle = "OPEN" | "DRAINING" | "CLOSED";
export type RealtimeStreamTaskLimits = Readonly<{
effectTimeoutMs: number;
recoveryTimeoutMs: number;
drainTimeoutMs: number;
}>;
export const DEFAULT_REALTIME_STREAM_TASK_LIMITS: RealtimeStreamTaskLimits =
Object.freeze({
effectTimeoutMs: 5_000,
recoveryTimeoutMs: 10_000,
drainTimeoutMs: 5_000,
});
export type RealtimeStreamCoordinatorDependencies = Readonly<{
registry: RealtimePolicyRegistry;
mappers: Readonly<Record<string, InstalledBoundaryMapper>>;
@@ -48,6 +69,10 @@ export type RealtimeStreamCoordinatorDependencies = Readonly<{
scope: RealtimeScopeSnapshot;
now?: () => number;
observe?: RealtimeEventObservationSink;
taskLimits?: Partial<RealtimeStreamTaskLimits>;
/** Test seam for the bounded task deadline. */
scheduleTimeout?: (callback: () => void, delayMs: number) => unknown;
clearScheduledTimeout?: (handle: unknown) => void;
}>;
export type RealtimeStreamCoordinator = Readonly<{
@@ -66,7 +91,13 @@ export type RealtimeStreamCoordinator = Readonly<{
): RealtimeResult<void>;
getResumeState(streamId: StreamRegistrationId): RealtimeResumeState | null;
inspect(streamId: StreamRegistrationId): RealtimeStreamInspection;
close(): void;
lifecycle(streamId: StreamRegistrationId): RealtimeStreamLifecycle;
/**
* R-02. Bounded quiescence. Success means every tracked task actually
* settled; `IDLE_TIMEOUT` means the coordinator is still `DRAINING` and the
* caller must not assume a clean teardown.
*/
close(): Promise<RealtimeResult<void>>;
}>;
type DedupeEntry = Readonly<{
@@ -96,6 +127,12 @@ type StreamState = {
awaitingTransportBarrier: boolean;
barrierCheckpoint: RealtimeRecoveryCheckpoint | null;
closed: boolean;
lifecycle: RealtimeStreamLifecycle;
/**
* Tasks whose public wait already ended but whose underlying promise has not
* settled. They keep the stream `DRAINING` and block new admission.
*/
retainedTasks: Set<Promise<unknown>>;
};
const SNAPSHOT_CHECKPOINT_KEYS = Object.freeze([
@@ -120,9 +157,59 @@ export function createRealtimeStreamCoordinator(
): RealtimeStreamCoordinator {
assertDependencies(dependencies);
const now = dependencies.now ?? Date.now;
const limits: RealtimeStreamTaskLimits = Object.freeze({
...DEFAULT_REALTIME_STREAM_TASK_LIMITS,
...dependencies.taskLimits,
});
const scheduleTimeout =
dependencies.scheduleTimeout ??
((callback: () => void, delayMs: number) => setTimeout(callback, delayMs));
const clearScheduledTimeout =
dependencies.clearScheduledTimeout ??
((handle: unknown) => {
clearTimeout(handle as ReturnType<typeof setTimeout>);
});
const states = new Map<StreamRegistrationId, StreamState>();
let closed = false;
const TASK_TIMED_OUT = Symbol("REALTIME_TASK_TIMED_OUT");
/**
* R-02. Bounds the public wait without discarding the task. A task that
* outlives its deadline is retained so `close()` can report honestly whether
* the stream is actually quiescent.
*/
async function awaitTaskWithinDeadline<Value>(
state: StreamState,
task: Promise<Value>,
timeoutMs: number,
revokeAndAbort: () => void,
): Promise<Value | typeof TASK_TIMED_OUT> {
task.catch(() => {});
let handle: unknown;
const timeout = new Promise<typeof TASK_TIMED_OUT>((resolve) => {
handle = scheduleTimeout(() => resolve(TASK_TIMED_OUT), timeoutMs);
});
const outcome = await Promise.race([task, timeout]);
clearScheduledTimeout(handle);
if (outcome !== TASK_TIMED_OUT) return outcome;
// The commit capability is revoked immediately; the work itself is not.
revokeAndAbort();
state.lifecycle = "DRAINING";
state.retainedTasks.add(task);
void task
.catch(() => {})
.finally(() => {
state.retainedTasks.delete(task);
if (state.retainedTasks.size === 0 && state.lifecycle === "DRAINING") {
state.lifecycle = state.closed ? "CLOSED" : "OPEN";
if (!state.closed) state.freshness = "STALE";
}
});
return TASK_TIMED_OUT;
}
for (const registration of dependencies.registry.listStreams()) {
states.set(registration.id, {
registration,
@@ -143,6 +230,8 @@ export function createRealtimeStreamCoordinator(
awaitingTransportBarrier: false,
barrierCheckpoint: null,
closed: false,
lifecycle: "OPEN",
retainedTasks: new Set(),
});
}
@@ -158,6 +247,11 @@ export function createRealtimeStreamCoordinator(
realtimeFailure("MALFORMED_EVENT", "RECEIVE"),
);
}
const draining = states.get(event.envelope.streamId);
if (draining && draining.lifecycle === "DRAINING") {
// R-02. New work is refused while a retained task is still running.
return Promise.resolve(realtimeFailure("CLOSED", "RECEIVE"));
}
const state = states.get(event.envelope.streamId);
if (!state) {
return Promise.resolve(
@@ -384,19 +478,37 @@ export function createRealtimeStreamCoordinator(
!effectAbort.signal.aborted &&
scopeIsCurrent(dependencies.scope);
let effect: unknown;
let effectTimedOut = false;
try {
effect = await dependencies.authority.effects.apply(
eventType.effectProfileId,
mapped.value,
Object.freeze({
streamId: state.registration.id,
eventType: eventType.id,
occurredAt: event.envelope.occurredAt,
scopeGeneration: dependencies.scope.generation,
isCurrent: effectIsCurrent,
}),
effectAbort.signal,
const applied = await awaitTaskWithinDeadline(
state,
Promise.resolve(
dependencies.authority.effects.apply(
eventType.effectProfileId,
mapped.value,
Object.freeze({
streamId: state.registration.id,
eventType: eventType.id,
occurredAt: event.envelope.occurredAt,
scopeGeneration: dependencies.scope.generation,
isCurrent: effectIsCurrent,
}),
effectAbort.signal,
),
),
limits.effectTimeoutMs,
() => {
// Commit capability is revoked permanently for this attempt.
effectLeaseActive = false;
effectAbort.abort();
},
);
if (applied === TASK_TIMED_OUT) {
effectTimedOut = true;
effect = null;
} else {
effect = applied;
}
} catch {
effect = null;
} finally {
@@ -406,6 +518,17 @@ export function createRealtimeStreamCoordinator(
state.activeEffectAbort = null;
}
}
if (effectTimedOut) {
// R-02. Bounded for the caller; the underlying task stays tracked.
observe({
operation: "APPLY",
outcome: "FAILED",
streamId: state.registration.id,
eventType: event.envelope.eventType,
reason: "IDLE_TIMEOUT",
});
return realtimeFailure("IDLE_TIMEOUT", "APPLY", false);
}
if (closed || state.closed) {
return dropped("CLOSED");
@@ -483,7 +606,7 @@ export function createRealtimeStreamCoordinator(
calledFromCurrentJob: boolean,
signal?: AbortSignal,
): Promise<RealtimeResult<RealtimeRecoveryCheckpoint>> {
if (closed || state.closed) {
if (closed || state.closed || state.lifecycle === "DRAINING") {
return Promise.resolve(realtimeFailure("CLOSED", "RECOVER"));
}
if (!scopeIsCurrent(dependencies.scope)) {
@@ -542,22 +665,50 @@ export function createRealtimeStreamCoordinator(
let recovered: unknown;
let recoveryThrew = false;
let recoveryTimedOut = false;
try {
recovered = await dependencies.authority.recovery.recover(
Object.freeze({
streamId: state.registration.id,
reason,
scopeGeneration: dependencies.scope.generation,
signal: recoveryAbort.signal,
isCurrent: recoveryIsCurrent,
}),
const outcome = await awaitTaskWithinDeadline(
state,
Promise.resolve(
dependencies.authority.recovery.recover(
Object.freeze({
streamId: state.registration.id,
reason,
scopeGeneration: dependencies.scope.generation,
signal: recoveryAbort.signal,
isCurrent: recoveryIsCurrent,
}),
),
),
limits.recoveryTimeoutMs,
() => {
recoveryLeaseActive = false;
recoveryAbort.abort();
},
);
if (outcome === TASK_TIMED_OUT) {
recoveryTimedOut = true;
recovered = null;
} else {
recovered = outcome;
}
} catch {
recovered = null;
recoveryThrew = true;
} finally {
recoveryLeaseActive = false;
}
if (recoveryTimedOut) {
// R-02. A late checkpoint from this attempt can never commit.
state.freshness = "UNKNOWN";
observe({
operation: "RECOVER",
outcome: "FAILED",
streamId: state.registration.id,
reason: "IDLE_TIMEOUT",
});
return realtimeFailure("IDLE_TIMEOUT", "RECOVER", false);
}
if (closed || state.closed) {
state.freshness = "UNKNOWN";
return realtimeFailure("CLOSED", "RECOVER");
@@ -806,7 +957,45 @@ export function createRealtimeStreamCoordinator(
});
}
function close(): void {
function lifecycleOf(streamId: StreamRegistrationId): RealtimeStreamLifecycle {
const state = states.get(streamId);
if (!state) return "CLOSED";
return state.lifecycle;
}
/**
* R-02. `close()` fences immediately but reports honestly: success only when
* every retained task actually settled within the drain bound.
*/
async function close(): Promise<RealtimeResult<void>> {
fenceAllStates();
const retained = [...states.values()].flatMap((state) => [
...state.retainedTasks,
]);
if (retained.length === 0) {
for (const state of states.values()) state.lifecycle = "CLOSED";
return realtimeSuccess(undefined);
}
let handle: unknown;
const drained = await Promise.race([
Promise.allSettled(retained).then(() => true),
new Promise<false>((resolve) => {
handle = scheduleTimeout(
() => resolve(false),
limits.drainTimeoutMs,
);
}),
]);
clearScheduledTimeout(handle);
if (!drained) {
// Still DRAINING: the caller must not treat this as quiescence.
return realtimeFailure("IDLE_TIMEOUT", "CLOSE", false);
}
for (const state of states.values()) state.lifecycle = "CLOSED";
return realtimeSuccess(undefined);
}
function fenceAllStates(): void {
if (closed) return;
closed = true;
for (const state of states.values()) {
@@ -826,6 +1015,8 @@ export function createRealtimeStreamCoordinator(
state.eventIds.clear();
state.sequences.clear();
state.dedupeBytes = 0;
state.lifecycle =
state.retainedTasks.size > 0 ? "DRAINING" : "CLOSED";
}
observe({
operation: "CLOSE",
@@ -911,6 +1102,7 @@ export function createRealtimeStreamCoordinator(
confirmTransportBarrier,
getResumeState,
inspect,
lifecycle: lifecycleOf,
close,
});
}