fix: retain realtime work through draining

R-02: add an OPEN/DRAINING/CLOSED lifecycle orthogonal to freshness. Effect and
recovery authorities are now awaited under a deadline: on expiry the commit
capability is revoked and the work aborted, the caller gets a bounded
non-retryable IDLE_TIMEOUT, and the underlying task is retained rather than
dropped. A draining stream refuses new events and recovery, and close() returns
a Promise that succeeds only once every retained task actually settled,
reporting IDLE_TIMEOUT otherwise.

R-03: a handoff fail-close moves active, probe, quiescing and transition leases
into a retired-writer set before clearing their references, and close() waits on
current and retired writers together, so an abandoned non-cooperative writer can
no longer make teardown report a false success.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-08-13 23:52:46 +09:00
co-authored by Claude Opus 5
parent c9e820aed5
commit 2f29ccbf1a
7 changed files with 409 additions and 28 deletions
@@ -29,6 +29,91 @@ import {
} from "./fixture.ts";
describe("transport-independent realtime stream coordinator", () => {
it("keeps the stream DRAINING until a non-cooperative effect settles", async () => {
const wedged = deferred<RealtimeResult<void>>();
const harness = createHarness({
apply: async () => wedged.promise,
taskLimits: { effectTimeoutMs: 5, drainTimeoutMs: 5 },
});
await harness.initialize();
const applied = await harness.coordinator.accept(
harness.event({
eventId: "event-00000001",
sequence: "1",
resumeCursor: "cursor-00000001",
}),
);
// Bounded for the caller, and explicitly non-retryable.
expect(applied).toMatchObject({
ok: false,
error: { kind: "IDLE_TIMEOUT", operation: "APPLY", retryable: false },
});
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
// DRAINING refuses new admission rather than queueing behind the wedge.
await expect(
harness.coordinator.accept(
harness.event({
eventId: "event-00000002",
sequence: "2",
resumeCursor: "cursor-00000002",
}),
),
).resolves.toMatchObject({ ok: false, error: { kind: "CLOSED" } });
// close() cannot claim quiescence while the task is still running.
await expect(harness.coordinator.close()).resolves.toMatchObject({
ok: false,
error: { kind: "IDLE_TIMEOUT", operation: "CLOSE" },
});
// Only actual settlement ends DRAINING.
wedged.resolve(realtimeSuccess(undefined));
await expect(harness.coordinator.close()).resolves.toMatchObject({
ok: true,
});
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("CLOSED");
});
it("bounds non-cooperative recovery and rejects its late checkpoint", async () => {
const wedged = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
let recoveries = 0;
const harness = createHarness({
recover: async () => {
recoveries += 1;
return recoveries === 1
? realtimeSuccess(snapshotCommit("0"))
: wedged.promise;
},
taskLimits: { recoveryTimeoutMs: 5, drainTimeoutMs: 5 },
});
await harness.initialize();
const recovered = await harness.coordinator.recover(
STREAM_ID,
"SEQUENCE_GAP",
);
expect(recovered).toMatchObject({
ok: false,
error: { kind: "IDLE_TIMEOUT", operation: "RECOVER", retryable: false },
});
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
const beforeLateCommit = harness.coordinator.getResumeState(STREAM_ID);
// A late checkpoint from the abandoned attempt cannot commit.
wedged.resolve(realtimeSuccess(snapshotCommit("9")));
for (let flush = 0; flush < 10; flush += 1) await Promise.resolve();
expect(harness.coordinator.getResumeState(STREAM_ID)).toEqual(
beforeLateCommit,
);
// Settlement returns the stream to OPEN, marked STALE for an authoritative
// recovery rather than silently trusting the abandoned attempt.
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("OPEN");
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
});
it("applies one stream sequentially and commits each cursor after its effect", async () => {
const first = deferred<RealtimeResult<void>>();
const applied: string[] = [];
@@ -866,6 +951,11 @@ type HarnessOptions = Readonly<{
request: RealtimeRecoveryRequest,
) => Promise<RealtimeResult<RealtimeRecoveryCommit>>;
observe?: (observation: RealtimeObservation) => void;
taskLimits?: Readonly<{
effectTimeoutMs?: number;
recoveryTimeoutMs?: number;
drainTimeoutMs?: number;
}>;
}>;
function createHarness(options: HarnessOptions = {}) {
@@ -911,6 +1001,7 @@ function createHarness(options: HarnessOptions = {}) {
},
now: () => 10_000,
observe: options.observe,
...(options.taskLimits ? { taskLimits: options.taskLimits } : {}),
});
return {