DongHyeonkaandClaude Opus 5
c0f53d1855
RT-RR-01. An effect or recovery task was registered as retained only after its public wait expired, so a close() that arrived first saw an empty set and reported quiescence while the raw task was still running against the authority. Tasks are now registered when they are created and removed when they settle; DRAINING keeps its narrower meaning through a separate timed-out set. RT-RR-02. Admission happened when an event was queued; execution is a second decision. A queue entry admitted before the stream entered DRAINING no longer starts running inside it. And an abandoned task may have applied part of its effect, so the resume token it was based on is discarded and recovery is required explicitly — the next ordinary event can no longer skip authoritative recovery on the strength of state a timed-out effect may have invalidated. RT-RR-03. close() cached the first timeout forever, so a writer that later settled could never be proved quiescent and the retained registry could never be pruned. Only an in-flight close is shared now, every writer a close fences is retained until its tail actually settles, and the tail prunes itself. A second close therefore converges to success once the writer finishes. RT-RR-04. Checkpoint work joins writer tails in the physical-task registry from invocation to settlement and is drained on the same terms. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1179 lines
36 KiB
TypeScript
1179 lines
36 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type {
|
|
ExternalRealtimeEventContext,
|
|
RealtimeEventAuthority,
|
|
RealtimeObservation,
|
|
RealtimeRecoveryCommit,
|
|
RealtimeRecoveryRequest,
|
|
} from "../../../src/application/ports/realtime/event-authority.ts";
|
|
import type { RealtimeResult } from "../../../src/application/ports/realtime/shared.ts";
|
|
import {
|
|
createRealtimeStreamCoordinator,
|
|
} from "../../../src/adapters/realtime/stream-coordinator.ts";
|
|
import {
|
|
realtimeFailure,
|
|
realtimeSuccess,
|
|
} from "../../../src/adapters/realtime/result.ts";
|
|
import type { ValidatedRealtimeEventDto } from "../../../src/adapters/realtime/event-codec.ts";
|
|
import type { RealtimePolicyRegistry } from "../../../src/contracts/realtime-streams.ts";
|
|
import {
|
|
STREAM_ID,
|
|
TEST_LIMITS,
|
|
TEST_MAPPER,
|
|
TEST_MAPPERS,
|
|
createTestRealtimeCodec,
|
|
createTestRealtimeRegistry,
|
|
realtimeEventJson,
|
|
type EventOverrides,
|
|
} from "./fixture.ts";
|
|
|
|
describe("transport-independent realtime stream coordinator", () => {
|
|
it("keeps the stream DRAINING until a non-cooperative effect settles", async () => {
|
|
const wedged = deferred<RealtimeResult<void>>();
|
|
const harness = createHarness({
|
|
apply: async () => wedged.promise,
|
|
taskLimits: { effectTimeoutMs: 5, drainTimeoutMs: 5 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applied = await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
resumeCursor: "cursor-00000001",
|
|
}),
|
|
);
|
|
// Bounded for the caller, and explicitly non-retryable.
|
|
expect(applied).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "APPLY", retryable: false },
|
|
});
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
// DRAINING refuses new admission rather than queueing behind the wedge.
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({ ok: false, error: { kind: "CLOSED" } });
|
|
|
|
// close() cannot claim quiescence while the task is still running.
|
|
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "CLOSE" },
|
|
});
|
|
|
|
// Only actual settlement ends DRAINING.
|
|
wedged.resolve(realtimeSuccess(undefined));
|
|
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
|
ok: true,
|
|
});
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("CLOSED");
|
|
});
|
|
|
|
it("bounds non-cooperative recovery and rejects its late checkpoint", async () => {
|
|
const wedged = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let recoveries = 0;
|
|
const harness = createHarness({
|
|
recover: async () => {
|
|
recoveries += 1;
|
|
return recoveries === 1
|
|
? realtimeSuccess(snapshotCommit("0"))
|
|
: wedged.promise;
|
|
},
|
|
taskLimits: { recoveryTimeoutMs: 5, drainTimeoutMs: 5 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const recovered = await harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"SEQUENCE_GAP",
|
|
);
|
|
expect(recovered).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "RECOVER", retryable: false },
|
|
});
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
const beforeLateCommit = harness.coordinator.getResumeState(STREAM_ID);
|
|
|
|
// A late checkpoint from the abandoned attempt cannot commit.
|
|
wedged.resolve(realtimeSuccess(snapshotCommit("9")));
|
|
for (let flush = 0; flush < 10; flush += 1) await Promise.resolve();
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toEqual(
|
|
beforeLateCommit,
|
|
);
|
|
// Settlement returns the stream to OPEN, marked STALE for an authoritative
|
|
// recovery rather than silently trusting the abandoned attempt.
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("OPEN");
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
|
|
it("applies one stream sequentially and commits each cursor after its effect", async () => {
|
|
const first = deferred<RealtimeResult<void>>();
|
|
const applied: string[] = [];
|
|
const harness = createHarness({
|
|
async apply(_profile, value) {
|
|
const selected = (value as Readonly<{ value: string }>).value;
|
|
applied.push(selected);
|
|
return selected === "first"
|
|
? first.promise
|
|
: realtimeSuccess(undefined);
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
const firstResult = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
resumeCursor: "cursor-00000001",
|
|
payload: { value: "first" },
|
|
}),
|
|
);
|
|
const secondResult = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
payload: { value: "second" },
|
|
}),
|
|
);
|
|
|
|
await vi.waitFor(() => expect(applied).toEqual(["first"]));
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: "cursor-snapshot-0",
|
|
});
|
|
|
|
first.resolve(realtimeSuccess(undefined));
|
|
await expect(firstResult).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
await expect(secondResult).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(applied).toEqual(["first", "second"]);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toEqual({
|
|
recoveryMode: "CURSOR",
|
|
streamEpoch: "stream-epoch-0001",
|
|
lastAppliedSequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
});
|
|
});
|
|
|
|
it("drops exact duplicates and stale events without replaying effects", async () => {
|
|
const harness = createHarness();
|
|
await harness.initialize();
|
|
const event = harness.event();
|
|
|
|
await harness.coordinator.accept(event);
|
|
await expect(harness.coordinator.accept(event)).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
outcome: "DROPPED",
|
|
reason: "DUPLICATE_EVENT",
|
|
},
|
|
});
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-stale-0001",
|
|
sequence: "0",
|
|
resumeCursor: "cursor-stale-0001",
|
|
}),
|
|
),
|
|
).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
outcome: "DROPPED",
|
|
reason: "STALE_EVENT",
|
|
},
|
|
});
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(1);
|
|
});
|
|
|
|
it("recovers without applying a conflicting ID, sequence gap or epoch", async () => {
|
|
for (const [overrides, expectedReason] of [
|
|
[
|
|
{
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
payload: { value: "conflict" },
|
|
},
|
|
"EVENT_CONFLICT",
|
|
],
|
|
[
|
|
{
|
|
eventId: "event-gap-000001",
|
|
sequence: "3",
|
|
resumeCursor: "cursor-gap-000001",
|
|
},
|
|
"SEQUENCE_GAP",
|
|
],
|
|
[
|
|
{
|
|
eventId: "event-epoch-0001",
|
|
sequence: "2",
|
|
streamEpoch: "stream-epoch-0002",
|
|
resumeCursor: "cursor-epoch-0001",
|
|
},
|
|
"STREAM_EPOCH_CHANGED",
|
|
],
|
|
] as const) {
|
|
const harness = createHarness();
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(harness.event());
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event(overrides)),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: expectedReason,
|
|
},
|
|
});
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
expectedReason,
|
|
]);
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
}
|
|
});
|
|
|
|
it("does not apply or advance when mapping fails", async () => {
|
|
const harness = createHarness({
|
|
mappers: {
|
|
ReferenceRealtimeMapper: {
|
|
...TEST_MAPPER,
|
|
map: () => ({
|
|
ok: false as const,
|
|
code: "MAPPING_INVARIANT_REJECTED" as const,
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "MAPPING_CONTRACT_VIOLATION",
|
|
},
|
|
});
|
|
expect(harness.effects).not.toHaveBeenCalled();
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
"MAPPING_CONTRACT_VIOLATION",
|
|
]);
|
|
});
|
|
|
|
it("leaves the prior checkpoint when an effect and its recovery fail", async () => {
|
|
let recoveryCount = 0;
|
|
const harness = createHarness({
|
|
apply: async () => realtimeFailure("APPLY_FAILED", "APPLY"),
|
|
recover: async () => {
|
|
recoveryCount += 1;
|
|
return recoveryCount === 1
|
|
? realtimeSuccess(snapshotCommit("0"))
|
|
: realtimeFailure("PROVIDER_UNAVAILABLE", "RECOVER");
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "RECOVER",
|
|
},
|
|
});
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: "cursor-snapshot-0",
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"UNKNOWN",
|
|
);
|
|
await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
);
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("rejects malformed authority results without advancing a checkpoint", async () => {
|
|
let recoveryCount = 0;
|
|
const harness = createHarness({
|
|
apply: async () =>
|
|
({
|
|
ok: true,
|
|
value: "not-void",
|
|
}) as unknown as RealtimeResult<void>,
|
|
recover: async () => {
|
|
recoveryCount += 1;
|
|
return recoveryCount === 1
|
|
? realtimeSuccess(snapshotCommit("0"))
|
|
: ({
|
|
ok: false,
|
|
error: {
|
|
kind: "FORBIDDEN",
|
|
operation: "APPLY",
|
|
retryable: false,
|
|
},
|
|
} as RealtimeResult<RealtimeRecoveryCommit>);
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECOVER"),
|
|
);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "0",
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"UNKNOWN",
|
|
);
|
|
});
|
|
|
|
it("rejects recovery accessors without invoking them", async () => {
|
|
let checkpointReads = 0;
|
|
const accessorCommit = Object.defineProperties(
|
|
{},
|
|
{
|
|
checkpoint: {
|
|
enumerable: true,
|
|
get() {
|
|
checkpointReads += 1;
|
|
return snapshotCommit("0");
|
|
},
|
|
},
|
|
kind: {
|
|
enumerable: true,
|
|
value: "SNAPSHOT_RESET",
|
|
},
|
|
},
|
|
) as RealtimeRecoveryCommit;
|
|
const harness = createHarness({
|
|
recover: async () => realtimeSuccess(accessorCommit),
|
|
});
|
|
|
|
await expect(
|
|
harness.coordinator.recover(STREAM_ID, "INITIALIZE"),
|
|
).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECOVER"),
|
|
);
|
|
expect(checkpointReads).toBe(0);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
});
|
|
|
|
it("commits only the one-shot descriptor snapshot of a recovery proxy", async () => {
|
|
const descriptorCommit = snapshotCommit("0");
|
|
const propertyReads: PropertyKey[] = [];
|
|
const proxyCommit = new Proxy(descriptorCommit, {
|
|
get(_target, key) {
|
|
propertyReads.push(key);
|
|
if (key === "kind") return "SESSION_REBUILD";
|
|
if (key === "checkpoint") return snapshotCommit("99");
|
|
return undefined;
|
|
},
|
|
});
|
|
const harness = createHarness({
|
|
recover: async () => realtimeSuccess(proxyCommit),
|
|
});
|
|
|
|
const recovered = await harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
|
|
expect(recovered).toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: "cursor-snapshot-0",
|
|
},
|
|
});
|
|
expect(propertyReads).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
Object.freeze({
|
|
...snapshotCommit("0"),
|
|
unexpected: true,
|
|
}),
|
|
Object.assign(
|
|
Object.create({ inherited: true }) as Record<string, unknown>,
|
|
snapshotCommit("0"),
|
|
),
|
|
])(
|
|
"rejects extra and inherited recovery commit shapes",
|
|
async (commit) => {
|
|
const harness = createHarness({
|
|
recover: async () =>
|
|
realtimeSuccess(commit as RealtimeRecoveryCommit),
|
|
});
|
|
|
|
await expect(
|
|
harness.coordinator.recover(STREAM_ID, "INITIALIZE"),
|
|
).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECOVER"),
|
|
);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
},
|
|
);
|
|
|
|
it("expires effect and recovery commit leases when callbacks settle", async () => {
|
|
let effectContext: ExternalRealtimeEventContext | undefined;
|
|
let recoveryRequest: RealtimeRecoveryRequest | undefined;
|
|
const harness = createHarness({
|
|
async apply(_profile, _value, context) {
|
|
effectContext = context;
|
|
expect(context.isCurrent()).toBe(true);
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
async recover(request) {
|
|
recoveryRequest = request;
|
|
expect(request.isCurrent()).toBe(true);
|
|
return realtimeSuccess(snapshotCommit("0"));
|
|
},
|
|
});
|
|
|
|
await harness.initialize();
|
|
expect(recoveryRequest?.isCurrent()).toBe(false);
|
|
await harness.coordinator.accept(harness.event());
|
|
expect(effectContext?.isCurrent()).toBe(false);
|
|
});
|
|
|
|
it("fences an in-flight successful effect before cursor commit", async () => {
|
|
const pending = deferred<RealtimeResult<void>>();
|
|
let effectContext: ExternalRealtimeEventContext | undefined;
|
|
const harness = createHarness({
|
|
apply: async (_profile, _value, context) => {
|
|
effectContext = context;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
const accepted = harness.coordinator.accept(harness.event());
|
|
await vi.waitFor(() =>
|
|
expect(harness.effects).toHaveBeenCalledOnce(),
|
|
);
|
|
harness.setCurrent(false);
|
|
expect(effectContext?.isCurrent()).toBe(false);
|
|
pending.resolve(realtimeSuccess(undefined));
|
|
|
|
await expect(accepted).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
outcome: "DROPPED",
|
|
reason: "SCOPE_FENCED",
|
|
},
|
|
});
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
expect(harness.coordinator.inspect(STREAM_ID)).toMatchObject({
|
|
freshness: "UNKNOWN",
|
|
dedupeEntries: 0,
|
|
queuedEvents: 0,
|
|
awaitingTransportBarrier: false,
|
|
});
|
|
});
|
|
|
|
it("aborts and quiesces the active effect before queue-overflow recovery", async () => {
|
|
const pending = deferred<RealtimeResult<void>>();
|
|
let capturedSignal: AbortSignal | undefined;
|
|
const registry = createTestRealtimeRegistry({
|
|
limits: {
|
|
...TEST_LIMITS,
|
|
maxQueueEvents: 1,
|
|
},
|
|
});
|
|
const harness = createHarness({
|
|
registry,
|
|
async apply(_profile, _value, _context, signal) {
|
|
capturedSignal = signal;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
const first = harness.coordinator.accept(harness.event());
|
|
await vi.waitFor(() =>
|
|
expect(harness.effects).toHaveBeenCalledOnce(),
|
|
);
|
|
const overflow = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
);
|
|
|
|
expect(capturedSignal?.aborted).toBe(true);
|
|
expect(harness.recoveryReasons).toEqual(["INITIALIZE"]);
|
|
pending.resolve(realtimeSuccess(undefined));
|
|
await expect(first).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "DROPPED" },
|
|
});
|
|
await expect(overflow).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "QUEUE_OVERFLOW",
|
|
},
|
|
});
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
"QUEUE_OVERFLOW",
|
|
]);
|
|
});
|
|
|
|
it("treats a current scope mismatch as a protocol violation, not a stale callback", async () => {
|
|
const harness = createHarness();
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({ scopeBinding: "other-scope-binding" }),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "SCOPE_PROTOCOL_VIOLATION",
|
|
},
|
|
});
|
|
expect(harness.effects).not.toHaveBeenCalled();
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
"SCOPE_PROTOCOL_VIOLATION",
|
|
]);
|
|
});
|
|
|
|
it("never claims CURRENT for a SNAPSHOT_ONLY stream without a barrier", async () => {
|
|
const registry = createTestRealtimeRegistry({
|
|
recovery: {
|
|
mode: "SNAPSHOT_ONLY",
|
|
snapshotOperationId: "GET_REFERENCE_REALTIME_SNAPSHOT",
|
|
checkpointCodecId: "ReferenceRealtimeCheckpoint",
|
|
barrier: "NONE",
|
|
},
|
|
delivery: "INVALIDATION_HINT",
|
|
stateBearing: false,
|
|
});
|
|
const harness = createHarness({
|
|
registry,
|
|
recover: async () =>
|
|
realtimeSuccess({
|
|
kind: "SNAPSHOT_RESET",
|
|
checkpoint: {
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
streamEpoch: "stream-epoch-0001",
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: null,
|
|
snapshotRevision: "snapshot-revision-0",
|
|
},
|
|
}),
|
|
});
|
|
await harness.initialize();
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
resumeCursor: null,
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
|
|
it("requires an exact transport barrier before promoting a recovered stream to CURRENT", async () => {
|
|
const harness = createHarness();
|
|
const recovered = await harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
expect(recovered.ok).toBe(true);
|
|
if (!recovered.ok) return;
|
|
await Promise.resolve();
|
|
|
|
expect(harness.coordinator.inspect(STREAM_ID)).toMatchObject({
|
|
freshness: "STALE",
|
|
awaitingTransportBarrier: true,
|
|
});
|
|
expect(
|
|
harness.coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
Object.freeze({
|
|
...recovered.value,
|
|
}) as typeof recovered.value,
|
|
),
|
|
).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
const publicResumeState =
|
|
harness.coordinator.getResumeState(STREAM_ID);
|
|
expect(publicResumeState).not.toBe(recovered.value);
|
|
expect(
|
|
harness.coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
publicResumeState as typeof recovered.value,
|
|
),
|
|
).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
expect(
|
|
harness.coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
recovered.value,
|
|
),
|
|
).toEqual(realtimeSuccess(undefined));
|
|
expect(harness.coordinator.inspect(STREAM_ID)).toMatchObject({
|
|
freshness: "CURRENT",
|
|
awaitingTransportBarrier: false,
|
|
});
|
|
});
|
|
|
|
it("marks a successful invalidation hint stale until authoritative refresh", async () => {
|
|
const registry = createTestRealtimeRegistry({
|
|
recovery: {
|
|
mode: "SNAPSHOT_ONLY",
|
|
snapshotOperationId: "GET_REFERENCE_REALTIME_SNAPSHOT",
|
|
checkpointCodecId: "ReferenceRealtimeCheckpoint",
|
|
barrier: "CONNECT_BUFFER",
|
|
},
|
|
delivery: "INVALIDATION_HINT",
|
|
stateBearing: false,
|
|
});
|
|
const harness = createHarness({
|
|
registry,
|
|
recover: async () =>
|
|
realtimeSuccess({
|
|
kind: "SNAPSHOT_RESET",
|
|
checkpoint: {
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
streamEpoch: "stream-epoch-0001",
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: null,
|
|
snapshotRevision: "snapshot-revision-0",
|
|
},
|
|
}),
|
|
});
|
|
await harness.initialize();
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"CURRENT",
|
|
);
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
resumeCursor: null,
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
|
|
it("recovers instead of evicting and continuing after dedupe capacity", async () => {
|
|
const harness = createHarness({
|
|
registry: createTestRealtimeRegistry({
|
|
limits: {
|
|
...TEST_LIMITS,
|
|
maxDedupeEntries: 1,
|
|
},
|
|
}),
|
|
});
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(harness.event());
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "DEDUPE_OVERFLOW",
|
|
},
|
|
});
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(0);
|
|
});
|
|
|
|
it("rejects a regressing same-epoch recovery checkpoint atomically", async () => {
|
|
let checkpoint = "5";
|
|
const harness = createHarness({
|
|
recover: async () => realtimeSuccess(snapshotCommit(checkpoint)),
|
|
});
|
|
await harness.initialize();
|
|
checkpoint = "4";
|
|
|
|
await expect(
|
|
harness.coordinator.recover(STREAM_ID, "CURSOR_EXPIRED"),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "5",
|
|
});
|
|
});
|
|
|
|
it("coalesces recovery, closes idempotently and rejects forged DTOs", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
const recover = vi.fn(async () => pending.promise);
|
|
const harness = createHarness({ recover });
|
|
|
|
const first = harness.coordinator.recover(STREAM_ID, "INITIALIZE");
|
|
const second = harness.coordinator.recover(STREAM_ID, "SEQUENCE_GAP");
|
|
await vi.waitFor(() => expect(recover).toHaveBeenCalledOnce());
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
await expect(first).resolves.toMatchObject({ ok: true });
|
|
await expect(second).resolves.toMatchObject({ ok: true });
|
|
|
|
harness.coordinator.close();
|
|
harness.coordinator.close();
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
Object.freeze({}) as ValidatedRealtimeEventDto,
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "MALFORMED_EVENT" },
|
|
});
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toEqual({
|
|
ok: true,
|
|
value: { outcome: "DROPPED", reason: "CLOSED" },
|
|
});
|
|
});
|
|
|
|
it("returns the exact shared checkpoint to an event waiting on active recovery", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
const recover = vi.fn(async () => pending.promise);
|
|
const harness = createHarness({ recover });
|
|
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await vi.waitFor(() => expect(recover).toHaveBeenCalledOnce());
|
|
const accepting = harness.coordinator.accept(harness.event());
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
|
|
const recovered = await recovering;
|
|
const accepted = await accepting;
|
|
expect(recovered.ok).toBe(true);
|
|
expect(accepted.ok).toBe(true);
|
|
if (!recovered.ok || !accepted.ok) return;
|
|
expect(accepted.value).toMatchObject({
|
|
outcome: "RECOVERED",
|
|
reason: "INITIALIZE",
|
|
});
|
|
if (accepted.value.outcome !== "RECOVERED") return;
|
|
expect(accepted.value.resumeState).toBe(recovered.value);
|
|
});
|
|
|
|
it("aborts only an event waiter without cancelling shared recovery", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let recoverySignal: AbortSignal | undefined;
|
|
const harness = createHarness({
|
|
recover: async (request) => {
|
|
recoverySignal = request.signal;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await vi.waitFor(() => expect(recoverySignal).toBeDefined());
|
|
const controller = new AbortController();
|
|
const accepting = harness.coordinator.accept(
|
|
harness.event(),
|
|
controller.signal,
|
|
);
|
|
|
|
controller.abort();
|
|
await expect(accepting).resolves.toEqual(
|
|
realtimeFailure("ABORTED", "RECEIVE"),
|
|
);
|
|
expect(recoverySignal?.aborted).toBe(false);
|
|
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
await expect(recovering).resolves.toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("aborts the active recovery authority when closed", async () => {
|
|
let recoverySignal: AbortSignal | undefined;
|
|
const harness = createHarness({
|
|
recover: async (request) => {
|
|
recoverySignal = request.signal;
|
|
return new Promise((resolve) => {
|
|
request.signal.addEventListener(
|
|
"abort",
|
|
() => resolve(realtimeFailure("ABORTED", "RECOVER")),
|
|
{ once: true },
|
|
);
|
|
});
|
|
},
|
|
});
|
|
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await vi.waitFor(() => expect(recoverySignal).toBeDefined());
|
|
harness.coordinator.close();
|
|
|
|
expect(recoverySignal?.aborted).toBe(true);
|
|
await expect(recovering).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "CLOSED", operation: "RECOVER" },
|
|
});
|
|
});
|
|
|
|
it("does not commit a non-cooperative recovery after caller cancellation", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let recoveryRequest: RealtimeRecoveryRequest | undefined;
|
|
const harness = createHarness({
|
|
recover: async (request) => {
|
|
recoveryRequest = request;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
const controller = new AbortController();
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
controller.signal,
|
|
);
|
|
await vi.waitFor(() => expect(recoveryRequest).toBeDefined());
|
|
|
|
controller.abort();
|
|
expect(recoveryRequest?.isCurrent()).toBe(false);
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
|
|
await expect(recovering).resolves.toEqual(
|
|
realtimeFailure("ABORTED", "RECOVER"),
|
|
);
|
|
expect(recoveryRequest?.signal.aborted).toBe(true);
|
|
expect(recoveryRequest?.isCurrent()).toBe(false);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"UNKNOWN",
|
|
);
|
|
});
|
|
|
|
it("emits only closed redacted observations and ignores sink failure", async () => {
|
|
const observations: RealtimeObservation[] = [];
|
|
const harness = createHarness({
|
|
observe(observation) {
|
|
observations.push(observation);
|
|
throw new Error("diagnostic sink unavailable");
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "sensitive-event-id",
|
|
resumeCursor: "sensitive-cursor",
|
|
payload: { value: "sensitive-payload" },
|
|
}),
|
|
);
|
|
|
|
const serialized = JSON.stringify(observations);
|
|
expect(serialized).not.toContain("sensitive-event-id");
|
|
expect(serialized).not.toContain("sensitive-cursor");
|
|
expect(serialized).not.toContain("sensitive-payload");
|
|
expect(serialized).not.toContain("scope-binding-0001");
|
|
expect(observations).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
operation: "RECOVER",
|
|
outcome: "RECOVERED",
|
|
reason: "INITIALIZE",
|
|
}),
|
|
expect.objectContaining({
|
|
operation: "APPLY",
|
|
outcome: "APPLIED",
|
|
}),
|
|
]),
|
|
);
|
|
});
|
|
});
|
|
|
|
/**
|
|
* RT-RR-01 / RT-RR-02. A physical effect exists from the moment the coordinator
|
|
* calls the authority, not from the moment its public wait expires. Registering
|
|
* only on timeout let a `close()` that arrived first see an empty retained set
|
|
* and report quiescence while the raw task was still running against the
|
|
* authority.
|
|
*/
|
|
describe("realtime physical task ownership", () => {
|
|
it("does not report quiescence while an apply is still running", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
const harness = createHarness({
|
|
apply: async () => await applyGate.promise,
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applying = harness.coordinator.accept(harness.event());
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
// close() arrives long before the effect deadline.
|
|
const closed = await harness.coordinator.close();
|
|
expect(closed.ok).toBe(false);
|
|
expect(closed.ok ? null : closed.error.kind).toBe("IDLE_TIMEOUT");
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
await applying;
|
|
});
|
|
|
|
it("reports quiescence once the raw task settles", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
const harness = createHarness({
|
|
apply: async () => await applyGate.promise,
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 200 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applying = harness.coordinator.accept(harness.event());
|
|
await Promise.resolve();
|
|
const closing = harness.coordinator.close();
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
await applying;
|
|
|
|
expect(await closing).toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("does not start a queued event once the stream is DRAINING", async () => {
|
|
const firstApply = deferred<RealtimeResult<void>>();
|
|
let applyCalls = 0;
|
|
const harness = createHarness({
|
|
apply: async () => {
|
|
applyCalls += 1;
|
|
if (applyCalls === 1) return await firstApply.promise;
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
taskLimits: { effectTimeoutMs: 15, drainTimeoutMs: 50 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const first = harness.coordinator.accept(
|
|
harness.event({ eventId: "event-0001", sequence: "1" }),
|
|
);
|
|
// Queued behind the first while it is still inside its deadline.
|
|
const second = harness.coordinator.accept(
|
|
harness.event({ eventId: "event-0002", sequence: "2" }),
|
|
);
|
|
|
|
await first;
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
// The queued event is dropped at execution time, not applied and not
|
|
// recovered: admission happened before DRAINING, execution happens inside
|
|
// it, and the second decision is the one that counts.
|
|
const secondResult = await second;
|
|
expect(secondResult).toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "DROPPED", reason: "CLOSED" },
|
|
});
|
|
expect(applyCalls).toBe(1);
|
|
|
|
firstApply.resolve(realtimeSuccess(undefined));
|
|
});
|
|
|
|
it("discards the resume token when an effect is abandoned", async () => {
|
|
const firstApply = deferred<RealtimeResult<void>>();
|
|
let applyCalls = 0;
|
|
const harness = createHarness({
|
|
apply: async () => {
|
|
applyCalls += 1;
|
|
if (applyCalls === 1) return await firstApply.promise;
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
taskLimits: { effectTimeoutMs: 15, drainTimeoutMs: 50 },
|
|
});
|
|
await harness.initialize();
|
|
expect(harness.coordinator.inspect(STREAM_ID).hasResumeState).toBe(true);
|
|
|
|
await harness.coordinator.accept(
|
|
harness.event({ eventId: "event-0001", sequence: "1" }),
|
|
);
|
|
|
|
// The abandoned effect may have applied part of its change, so the token it
|
|
// was based on is no longer authoritative evidence.
|
|
const inspection = harness.coordinator.inspect(STREAM_ID);
|
|
expect(inspection.hasResumeState).toBe(false);
|
|
expect(inspection.freshness).toBe("UNKNOWN");
|
|
|
|
firstApply.resolve(realtimeSuccess(undefined));
|
|
});
|
|
});
|
|
|
|
type HarnessOptions = Readonly<{
|
|
registry?: RealtimePolicyRegistry;
|
|
mappers?: typeof TEST_MAPPERS | Readonly<Record<string, typeof TEST_MAPPER>>;
|
|
apply?: RealtimeEventAuthority["effects"]["apply"];
|
|
recover?: (
|
|
request: RealtimeRecoveryRequest,
|
|
) => Promise<RealtimeResult<RealtimeRecoveryCommit>>;
|
|
observe?: (observation: RealtimeObservation) => void;
|
|
taskLimits?: Readonly<{
|
|
effectTimeoutMs?: number;
|
|
recoveryTimeoutMs?: number;
|
|
drainTimeoutMs?: number;
|
|
}>;
|
|
}>;
|
|
|
|
function createHarness(options: HarnessOptions = {}) {
|
|
const registry = options.registry ?? createTestRealtimeRegistry();
|
|
const codec = createTestRealtimeCodec(registry);
|
|
let current = true;
|
|
const recoveryReasons: string[] = [];
|
|
let defaultRecoveryCount = 0;
|
|
const effects = vi.fn(
|
|
options.apply ??
|
|
(async () => realtimeSuccess(undefined)),
|
|
);
|
|
const recover = vi.fn(
|
|
options.recover ??
|
|
(async (request: RealtimeRecoveryRequest) => {
|
|
recoveryReasons.push(request.reason);
|
|
defaultRecoveryCount += 1;
|
|
return realtimeSuccess(
|
|
snapshotCommit(
|
|
"0",
|
|
`stream-epoch-${String(defaultRecoveryCount).padStart(4, "0")}`,
|
|
),
|
|
);
|
|
}),
|
|
);
|
|
const authority: RealtimeEventAuthority = {
|
|
effects: { apply: effects },
|
|
recovery: {
|
|
async recover(request) {
|
|
if (options.recover) recoveryReasons.push(request.reason);
|
|
return recover(request);
|
|
},
|
|
},
|
|
};
|
|
const coordinator = createRealtimeStreamCoordinator({
|
|
registry,
|
|
mappers: options.mappers ?? TEST_MAPPERS,
|
|
authority,
|
|
scope: {
|
|
generation: 7,
|
|
scopeBinding: "scope-binding-0001",
|
|
isCurrent: () => current,
|
|
},
|
|
now: () => 10_000,
|
|
observe: options.observe,
|
|
...(options.taskLimits ? { taskLimits: options.taskLimits } : {}),
|
|
});
|
|
|
|
return {
|
|
coordinator,
|
|
effects,
|
|
recoveryReasons,
|
|
event(overrides: EventOverrides = {}) {
|
|
const decoded = codec.decode(realtimeEventJson(overrides));
|
|
if (!decoded.ok) {
|
|
throw new Error(`test event failed: ${decoded.error.kind}`);
|
|
}
|
|
return decoded.value;
|
|
},
|
|
async initialize() {
|
|
const result = await coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await Promise.resolve();
|
|
if (
|
|
result.ok &&
|
|
coordinator.inspect(STREAM_ID).awaitingTransportBarrier
|
|
) {
|
|
const confirmed = coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
result.value,
|
|
);
|
|
if (!confirmed.ok) {
|
|
throw new Error("test transport barrier failed");
|
|
}
|
|
}
|
|
return result;
|
|
},
|
|
setCurrent(value: boolean) {
|
|
current = value;
|
|
},
|
|
};
|
|
}
|
|
|
|
function snapshotCommit(
|
|
sequence: string,
|
|
streamEpoch = "stream-epoch-0001",
|
|
): RealtimeRecoveryCommit {
|
|
return Object.freeze({
|
|
kind: "SNAPSHOT_RESET",
|
|
checkpoint: Object.freeze({
|
|
recoveryMode: "CURSOR",
|
|
streamEpoch,
|
|
lastAppliedSequence: sequence,
|
|
resumeCursor: `cursor-snapshot-${sequence}`,
|
|
snapshotRevision: `snapshot-revision-${sequence}`,
|
|
}),
|
|
});
|
|
}
|
|
|
|
function deferred<Value>() {
|
|
let resolve!: (value: Value) => void;
|
|
const promise = new Promise<Value>((next) => {
|
|
resolve = next;
|
|
});
|
|
return { promise, resolve };
|
|
}
|