fix: retain realtime work through draining
R-02: add an OPEN/DRAINING/CLOSED lifecycle orthogonal to freshness. Effect and recovery authorities are now awaited under a deadline: on expiry the commit capability is revoked and the work aborted, the caller gets a bounded non-retryable IDLE_TIMEOUT, and the underlying task is retained rather than dropped. A draining stream refuses new events and recovery, and close() returns a Promise that succeeds only once every retained task actually settled, reporting IDLE_TIMEOUT otherwise. R-03: a handoff fail-close moves active, probe, quiescing and transition leases into a retired-writer set before clearing their references, and close() waits on current and retired writers together, so an abandoned non-cooperative writer can no longer make teardown report a false success. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
c9e820aed5
commit
2f29ccbf1a
@@ -214,6 +214,47 @@ describe("live/poll authoritative writer handoff", () => {
|
||||
void second;
|
||||
});
|
||||
|
||||
it("tracks a retired active writer after handoff queue overflow", async () => {
|
||||
let release: ((result: RealtimeResult<void>) => void) | undefined;
|
||||
const harness = createHarness({
|
||||
limits: {
|
||||
...limits,
|
||||
maxActiveQueueCount: 2,
|
||||
maxActiveQueueBytes: 10,
|
||||
},
|
||||
apply: vi.fn(async ({ value }) => {
|
||||
if (value === "first") {
|
||||
return await new Promise<RealtimeResult<void>>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
}
|
||||
return realtimeSuccess(undefined);
|
||||
}),
|
||||
});
|
||||
const writer = harness.coordinator.currentWriter()!;
|
||||
const first = writer.write("first", 5);
|
||||
const second = writer.write("second", 5);
|
||||
await flush();
|
||||
await expect(writer.write("overflow", 1)).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "QUEUE_OVERFLOW" },
|
||||
});
|
||||
|
||||
// R-03. The fail-close dropped the active reference, but the writer is
|
||||
// still running, so close() must not claim quiescence.
|
||||
const closing = harness.coordinator.close();
|
||||
await flush();
|
||||
harness.clock.advance(100);
|
||||
await expect(closing).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "IDLE_TIMEOUT", operation: "CLOSE" },
|
||||
});
|
||||
|
||||
release?.(realtimeSuccess(undefined));
|
||||
void first;
|
||||
void second;
|
||||
});
|
||||
|
||||
it("fences and aborts live, waits for quiescence, then recovers before activating poll", async () => {
|
||||
const liveEffect = deferred<RealtimeResult<void>>();
|
||||
const checkpoint = deferred<RealtimeResult<void>>();
|
||||
|
||||
@@ -29,6 +29,91 @@ import {
|
||||
} from "./fixture.ts";
|
||||
|
||||
describe("transport-independent realtime stream coordinator", () => {
|
||||
it("keeps the stream DRAINING until a non-cooperative effect settles", async () => {
|
||||
const wedged = deferred<RealtimeResult<void>>();
|
||||
const harness = createHarness({
|
||||
apply: async () => wedged.promise,
|
||||
taskLimits: { effectTimeoutMs: 5, drainTimeoutMs: 5 },
|
||||
});
|
||||
await harness.initialize();
|
||||
|
||||
const applied = await harness.coordinator.accept(
|
||||
harness.event({
|
||||
eventId: "event-00000001",
|
||||
sequence: "1",
|
||||
resumeCursor: "cursor-00000001",
|
||||
}),
|
||||
);
|
||||
// Bounded for the caller, and explicitly non-retryable.
|
||||
expect(applied).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "IDLE_TIMEOUT", operation: "APPLY", retryable: false },
|
||||
});
|
||||
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
||||
|
||||
// DRAINING refuses new admission rather than queueing behind the wedge.
|
||||
await expect(
|
||||
harness.coordinator.accept(
|
||||
harness.event({
|
||||
eventId: "event-00000002",
|
||||
sequence: "2",
|
||||
resumeCursor: "cursor-00000002",
|
||||
}),
|
||||
),
|
||||
).resolves.toMatchObject({ ok: false, error: { kind: "CLOSED" } });
|
||||
|
||||
// close() cannot claim quiescence while the task is still running.
|
||||
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "IDLE_TIMEOUT", operation: "CLOSE" },
|
||||
});
|
||||
|
||||
// Only actual settlement ends DRAINING.
|
||||
wedged.resolve(realtimeSuccess(undefined));
|
||||
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("bounds non-cooperative recovery and rejects its late checkpoint", async () => {
|
||||
const wedged = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
||||
let recoveries = 0;
|
||||
const harness = createHarness({
|
||||
recover: async () => {
|
||||
recoveries += 1;
|
||||
return recoveries === 1
|
||||
? realtimeSuccess(snapshotCommit("0"))
|
||||
: wedged.promise;
|
||||
},
|
||||
taskLimits: { recoveryTimeoutMs: 5, drainTimeoutMs: 5 },
|
||||
});
|
||||
await harness.initialize();
|
||||
|
||||
const recovered = await harness.coordinator.recover(
|
||||
STREAM_ID,
|
||||
"SEQUENCE_GAP",
|
||||
);
|
||||
expect(recovered).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "IDLE_TIMEOUT", operation: "RECOVER", retryable: false },
|
||||
});
|
||||
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
||||
|
||||
const beforeLateCommit = harness.coordinator.getResumeState(STREAM_ID);
|
||||
|
||||
// A late checkpoint from the abandoned attempt cannot commit.
|
||||
wedged.resolve(realtimeSuccess(snapshotCommit("9")));
|
||||
for (let flush = 0; flush < 10; flush += 1) await Promise.resolve();
|
||||
expect(harness.coordinator.getResumeState(STREAM_ID)).toEqual(
|
||||
beforeLateCommit,
|
||||
);
|
||||
// Settlement returns the stream to OPEN, marked STALE for an authoritative
|
||||
// recovery rather than silently trusting the abandoned attempt.
|
||||
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("OPEN");
|
||||
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
||||
});
|
||||
|
||||
it("applies one stream sequentially and commits each cursor after its effect", async () => {
|
||||
const first = deferred<RealtimeResult<void>>();
|
||||
const applied: string[] = [];
|
||||
@@ -866,6 +951,11 @@ type HarnessOptions = Readonly<{
|
||||
request: RealtimeRecoveryRequest,
|
||||
) => Promise<RealtimeResult<RealtimeRecoveryCommit>>;
|
||||
observe?: (observation: RealtimeObservation) => void;
|
||||
taskLimits?: Readonly<{
|
||||
effectTimeoutMs?: number;
|
||||
recoveryTimeoutMs?: number;
|
||||
drainTimeoutMs?: number;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
function createHarness(options: HarnessOptions = {}) {
|
||||
@@ -911,6 +1001,7 @@ function createHarness(options: HarnessOptions = {}) {
|
||||
},
|
||||
now: () => 10_000,
|
||||
observe: options.observe,
|
||||
...(options.taskLimits ? { taskLimits: options.taskLimits } : {}),
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user