fix: track realtime physical work from invocation to settlement
RT-RR-01. An effect or recovery task was registered as retained only after its public wait expired, so a close() that arrived first saw an empty set and reported quiescence while the raw task was still running against the authority. Tasks are now registered when they are created and removed when they settle; DRAINING keeps its narrower meaning through a separate timed-out set. RT-RR-02. Admission happened when an event was queued; execution is a second decision. A queue entry admitted before the stream entered DRAINING no longer starts running inside it. And an abandoned task may have applied part of its effect, so the resume token it was based on is discarded and recovery is required explicitly — the next ordinary event can no longer skip authoritative recovery on the strength of state a timed-out effect may have invalidated. RT-RR-03. close() cached the first timeout forever, so a writer that later settled could never be proved quiescent and the retained registry could never be pruned. Only an in-flight close is shared now, every writer a close fences is retained until its tail actually settles, and the tail prunes itself. A second close therefore converges to success once the writer finishes. RT-RR-04. Checkpoint work joins writer tails in the physical-task registry from invocation to settlement and is drained on the same terms. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
a7390e3b3a
commit
c0f53d1855
@@ -173,6 +173,23 @@ export function createLivePollHandoffCoordinator<Value>(
|
||||
* running. Membership keeps `close()` honest about quiescence.
|
||||
*/
|
||||
const retiredWriters = new Set<InternalWriterLease<Value>>();
|
||||
/**
|
||||
* RT-RR-04. Checkpoint work is an external authority call like a writer tail,
|
||||
* so it belongs in a physical-task registry from invocation to settlement.
|
||||
* Racing it against a timeout bounded the public wait but left `close()` free
|
||||
* to report success while the checkpoint was still running.
|
||||
*/
|
||||
const checkpointTasks = new Set<Promise<unknown>>();
|
||||
|
||||
/** RT-RR-03. Prunes a settled writer without needing another `close()`. */
|
||||
function trackRetiredWriter(lease: InternalWriterLease<Value>): void {
|
||||
retiredWriters.add(lease);
|
||||
void lease.tail
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
retiredWriters.delete(lease);
|
||||
});
|
||||
}
|
||||
|
||||
active = createWriterLease(dependencies.initial.writer);
|
||||
|
||||
@@ -573,6 +590,12 @@ export function createLivePollHandoffCoordinator<Value>(
|
||||
(value) => ({ kind: "VALUE" as const, value }),
|
||||
() => ({ kind: "REJECTED" as const }),
|
||||
);
|
||||
// RT-RR-04. Registered at the moment the authority is called, and removed
|
||||
// only when it settles, so `close()` cannot report quiescence over it.
|
||||
checkpointTasks.add(operation);
|
||||
void operation.finally(() => {
|
||||
checkpointTasks.delete(operation);
|
||||
});
|
||||
const timeout = Promise.resolve()
|
||||
.then(async () => {
|
||||
await clock.sleep(
|
||||
@@ -643,6 +666,33 @@ export function createLivePollHandoffCoordinator<Value>(
|
||||
return realtimeSuccess(undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* RT-RR-04. The writer-tail quiescence bound applied to any retained physical
|
||||
* task, so checkpoint work is proved settled on the same terms.
|
||||
*/
|
||||
async function awaitTaskQuiescence(
|
||||
task: Promise<unknown>,
|
||||
): Promise<QuiescenceOutcome> {
|
||||
const timer = new AbortController();
|
||||
const settled = task.then(
|
||||
() => "QUIESCED" as const,
|
||||
() => "QUIESCED" as const,
|
||||
);
|
||||
const timeout = Promise.resolve()
|
||||
.then(async () => {
|
||||
await clock.sleep(limits.quiescenceTimeoutMs, timer.signal);
|
||||
return "TIMED_OUT" as const;
|
||||
})
|
||||
.catch(() =>
|
||||
timer.signal.aborted
|
||||
? ("QUIESCED" as const)
|
||||
: ("TIMER_FAILED" as const),
|
||||
);
|
||||
const outcome = await Promise.race([settled, timeout]);
|
||||
timer.abort();
|
||||
return outcome;
|
||||
}
|
||||
|
||||
async function awaitQuiescence(
|
||||
lease: InternalWriterLease<Value>,
|
||||
): Promise<QuiescenceOutcome> {
|
||||
@@ -670,7 +720,13 @@ export function createLivePollHandoffCoordinator<Value>(
|
||||
}
|
||||
|
||||
function close(): Promise<RealtimeResult<void>> {
|
||||
closePromise ??= performClose();
|
||||
// RT-RR-03. Only a close that is still running is shared. Caching the first
|
||||
// timeout forever meant a writer that later settled could never be proved
|
||||
// quiescent: every subsequent close replayed the stale failure and the
|
||||
// retained registry could never be pruned.
|
||||
closePromise ??= performClose().finally(() => {
|
||||
closePromise = null;
|
||||
});
|
||||
return closePromise;
|
||||
}
|
||||
|
||||
@@ -693,18 +749,23 @@ export function createLivePollHandoffCoordinator<Value>(
|
||||
probe = null;
|
||||
selectedProbe?.buffer.splice(0);
|
||||
if (selectedProbe) selectedProbe.bufferedBytes = 0;
|
||||
for (const writer of writers) writer.controller.abort();
|
||||
const outcomes = await Promise.all(
|
||||
writers.map(async (writer) => await awaitQuiescence(writer)),
|
||||
);
|
||||
for (const writer of writers) {
|
||||
// RT-RR-03. Every writer this close fences is retained until its tail
|
||||
// actually settles, so a later close still sees a writer that has not
|
||||
// finished — and stops seeing it the moment it does.
|
||||
trackRetiredWriter(writer);
|
||||
writer.controller.abort();
|
||||
}
|
||||
const outcomes = await Promise.all([
|
||||
...writers.map(async (writer) => await awaitQuiescence(writer)),
|
||||
// RT-RR-04. Checkpoint work is drained on the same terms as a writer tail.
|
||||
...[...checkpointTasks].map(
|
||||
async (task) => await awaitTaskQuiescence(task),
|
||||
),
|
||||
]);
|
||||
quiescing = null;
|
||||
transitionCandidate = null;
|
||||
transitioning = false;
|
||||
for (const [index, writer] of writers.entries()) {
|
||||
// Only an actually settled writer leaves the retired set; the rest keep
|
||||
// the coordinator DRAINING.
|
||||
if (outcomes[index] === "QUIESCED") retiredWriters.delete(writer);
|
||||
}
|
||||
if (outcomes.includes("TIMED_OUT")) {
|
||||
return handoffFailure("IDLE_TIMEOUT", "CLOSE");
|
||||
}
|
||||
@@ -802,7 +863,7 @@ export function createLivePollHandoffCoordinator<Value>(
|
||||
transitionCandidate,
|
||||
])) {
|
||||
writer.controller.abort();
|
||||
retiredWriters.add(writer);
|
||||
trackRetiredWriter(writer);
|
||||
}
|
||||
active = null;
|
||||
if (probe) {
|
||||
|
||||
@@ -129,10 +129,23 @@ type StreamState = {
|
||||
closed: boolean;
|
||||
lifecycle: RealtimeStreamLifecycle;
|
||||
/**
|
||||
* Tasks whose public wait already ended but whose underlying promise has not
|
||||
* settled. They keep the stream `DRAINING` and block new admission.
|
||||
* RT-RR-01. Every physical task this coordinator has handed to an external
|
||||
* authority, from the moment of the call until it settles. Registering only
|
||||
* after a timeout meant a `close()` that arrived first saw an empty set and
|
||||
* reported quiescence while the raw task was still running.
|
||||
*/
|
||||
retainedTasks: Set<Promise<unknown>>;
|
||||
/**
|
||||
* The subset of `retainedTasks` whose public wait already ended. These are
|
||||
* what keep the stream `DRAINING` and block new admission.
|
||||
*/
|
||||
timedOutTasks: Set<Promise<unknown>>;
|
||||
/**
|
||||
* RT-RR-02. Set when a task was abandoned at its deadline. The resume token
|
||||
* is discarded with it, so the next admitted event cannot skip authoritative
|
||||
* recovery on the strength of state a timed-out effect may have invalidated.
|
||||
*/
|
||||
recoveryRequired: boolean;
|
||||
};
|
||||
|
||||
const SNAPSHOT_CHECKPOINT_KEYS = Object.freeze([
|
||||
@@ -186,6 +199,19 @@ export function createRealtimeStreamCoordinator(
|
||||
revokeAndAbort: () => void,
|
||||
): Promise<Value | typeof TASK_TIMED_OUT> {
|
||||
task.catch(() => {});
|
||||
// RT-RR-01. The task is a physical effect the moment it is created, so it
|
||||
// is registered here rather than when its public wait happens to expire.
|
||||
state.retainedTasks.add(task);
|
||||
void task
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
state.retainedTasks.delete(task);
|
||||
state.timedOutTasks.delete(task);
|
||||
if (state.timedOutTasks.size === 0 && state.lifecycle === "DRAINING") {
|
||||
state.lifecycle = state.closed ? "CLOSED" : "OPEN";
|
||||
if (!state.closed) state.freshness = "STALE";
|
||||
}
|
||||
});
|
||||
let handle: unknown;
|
||||
const timeout = new Promise<typeof TASK_TIMED_OUT>((resolve) => {
|
||||
handle = scheduleTimeout(() => resolve(TASK_TIMED_OUT), timeoutMs);
|
||||
@@ -197,16 +223,12 @@ export function createRealtimeStreamCoordinator(
|
||||
// The commit capability is revoked immediately; the work itself is not.
|
||||
revokeAndAbort();
|
||||
state.lifecycle = "DRAINING";
|
||||
state.retainedTasks.add(task);
|
||||
void task
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
state.retainedTasks.delete(task);
|
||||
if (state.retainedTasks.size === 0 && state.lifecycle === "DRAINING") {
|
||||
state.lifecycle = state.closed ? "CLOSED" : "OPEN";
|
||||
if (!state.closed) state.freshness = "STALE";
|
||||
}
|
||||
});
|
||||
state.timedOutTasks.add(task);
|
||||
// RT-RR-02. An abandoned task may have applied part of its effect, so the
|
||||
// resume token it was based on is no longer authoritative evidence.
|
||||
state.recoveryRequired = true;
|
||||
state.resumeState = null;
|
||||
state.freshness = "UNKNOWN";
|
||||
return TASK_TIMED_OUT;
|
||||
}
|
||||
|
||||
@@ -232,6 +254,8 @@ export function createRealtimeStreamCoordinator(
|
||||
closed: false,
|
||||
lifecycle: "OPEN",
|
||||
retainedTasks: new Set(),
|
||||
timedOutTasks: new Set(),
|
||||
recoveryRequired: false,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -352,6 +376,10 @@ export function createRealtimeStreamCoordinator(
|
||||
return realtimeFailure("ABORTED", "RECEIVE");
|
||||
}
|
||||
if (closed || state.closed) return dropped("CLOSED");
|
||||
// RT-RR-02. Admission happened when this event was queued; execution is a
|
||||
// second decision. A queue entry admitted before the stream entered
|
||||
// DRAINING must not start running inside it.
|
||||
if (state.lifecycle === "DRAINING") return dropped("CLOSED");
|
||||
if (expectedGeneration !== state.processingGeneration) {
|
||||
return dropped("SCOPE_FENCED");
|
||||
}
|
||||
@@ -367,7 +395,11 @@ export function createRealtimeStreamCoordinator(
|
||||
signal,
|
||||
);
|
||||
}
|
||||
if (state.freshness === "UNKNOWN" || !state.resumeState) {
|
||||
if (
|
||||
state.recoveryRequired ||
|
||||
state.freshness === "UNKNOWN" ||
|
||||
!state.resumeState
|
||||
) {
|
||||
return recoverForAccept(state, "INITIALIZE", true, signal);
|
||||
}
|
||||
if (event.envelope.streamEpoch !== state.resumeState.streamEpoch) {
|
||||
@@ -782,6 +814,9 @@ export function createRealtimeStreamCoordinator(
|
||||
validatedResumeState as RealtimeRecoveryCheckpoint;
|
||||
|
||||
state.resumeState = resumeState;
|
||||
// RT-RR-02. Authoritative recovery is the only thing that clears the
|
||||
// requirement a timed-out task imposed.
|
||||
state.recoveryRequired = false;
|
||||
state.awaitingTransportBarrier =
|
||||
recoveryRequiresTransportBarrier(state.registration);
|
||||
state.barrierCheckpoint = state.awaitingTransportBarrier
|
||||
|
||||
Reference in New Issue
Block a user