Carries 359 template commits, tag v0.2.0 plus one documentation commit: the generator-built feature index with its app, bounded-context and feature declarations, the src/platform kernel and scale tooling, the 2026-09-24 src review fixes, and the 2026-09-25 maturity review round with its CHANGELOG [보안]/[계약] rows. 691 of the 824 template paths are applied, 85 of them merged with product changes; 127 are not carried. Product identity is unchanged: name, version 0.1.0, routes, design and the TechLog domain. Three decisions worth naming. TechLog stays composed directly, as the last two syncs recorded; its directory carries a NOT_INSTALLED marker that says so, and every installed-* composer takes the template's projection with TechLog added on top. The reference sample is no longer installed: the template now installs a feature only through feature-module.ts and context-routes.ts, which load the screens this product deleted, so its three operations, six schemas and one topic leave the registries, recorded as registry change evidence. And AppRouter mounts the platform and TechLog routes only; the template's bounded-context route gateway waits for a generator-installed feature. Required fixes: readSubject on the BFF session owner (without it every HTTP Studio session would close as integration-failed), zod through the facade in nine TechLog files, noUnusedLocals, and public/config.json written out to match the local profile. Derived baselines were recomputed after reproducing both sides' values: CI contract 101 commands / 106 references / 141 artifacts, gate shape c9adcf85; registry approval 2dc6c3a3; the runtime config schema baseline equals the schema at8210d48. Verified here: types, lint, architecture, adapter inventory, CI contract, i18n, design system, diagnostics, browser security, dev release manifest, TechLog contract and build pass; 220 component, 155 integration, 15 contract, 418 TechLog, 43 reference-feature, 18 recipe and 46 release-coherence tests pass, and 2381 of 2383 unit tests. Reproduced at8210d48first and unchanged: check:registries (byte-identical), test:runtime-schema, verify:documentation. Brought by the template and escalated rather than suppressed: check:request-fanout on TechLog's cursor pagination, check:remediation-ledger (fails upstream too), the shared-chunk hash test, and the registry-dependent sample removal and second-feature tests. Playwright suites were not run. Full record, the five open questions and the follow-ups in docs/operations/template-sync-2026-09-29.md. (cherry picked from commit b2170cf265dea45dc9cc05d3c61e627e97f31cdb)
1439 lines
45 KiB
TypeScript
1439 lines
45 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type {
|
|
ExternalRealtimeEventContext,
|
|
RealtimeEventAuthority,
|
|
RealtimeObservation,
|
|
RealtimeRecoveryCommit,
|
|
RealtimeRecoveryRequest,
|
|
} from "../../../src/application/ports/realtime/event-authority.ts";
|
|
import type { RealtimeResult } from "../../../src/application/ports/realtime/shared.ts";
|
|
import {
|
|
createRealtimeStreamCoordinator,
|
|
} from "../../../src/adapters/realtime/stream-coordinator.ts";
|
|
import {
|
|
realtimeFailure,
|
|
realtimeSuccess,
|
|
} from "../../../src/adapters/realtime/result.ts";
|
|
import type { ValidatedRealtimeEventDto } from "../../../src/adapters/realtime/event-codec.ts";
|
|
import type { RealtimePolicyRegistry } from "../../../src/contracts/realtime-streams.ts";
|
|
import {
|
|
STREAM_ID,
|
|
TEST_LIMITS,
|
|
TEST_MAPPER,
|
|
TEST_MAPPERS,
|
|
createTestRealtimeCodec,
|
|
createTestRealtimeRegistry,
|
|
realtimeEventJson,
|
|
type EventOverrides,
|
|
} from "./fixture.ts";
|
|
|
|
describe("transport-independent realtime stream coordinator", () => {
|
|
it("keeps the stream DRAINING until a non-cooperative effect settles", async () => {
|
|
const wedged = deferred<RealtimeResult<void>>();
|
|
const harness = createHarness({
|
|
apply: async () => wedged.promise,
|
|
taskLimits: { effectTimeoutMs: 5, drainTimeoutMs: 5 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applied = await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
resumeCursor: "cursor-00000001",
|
|
}),
|
|
);
|
|
// Bounded for the caller, and explicitly non-retryable.
|
|
expect(applied).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "APPLY", retryable: false },
|
|
});
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
// DRAINING refuses new admission rather than queueing behind the wedge.
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({ ok: false, error: { kind: "CLOSED" } });
|
|
|
|
// close() cannot claim quiescence while the task is still running.
|
|
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "CLOSE" },
|
|
});
|
|
|
|
// Only actual settlement ends DRAINING.
|
|
wedged.resolve(realtimeSuccess(undefined));
|
|
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
|
ok: true,
|
|
});
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("CLOSED");
|
|
});
|
|
|
|
it("bounds non-cooperative recovery and rejects its late checkpoint", async () => {
|
|
const wedged = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let recoveries = 0;
|
|
const harness = createHarness({
|
|
recover: async () => {
|
|
recoveries += 1;
|
|
return recoveries === 1
|
|
? realtimeSuccess(snapshotCommit("0"))
|
|
: wedged.promise;
|
|
},
|
|
taskLimits: { recoveryTimeoutMs: 5, drainTimeoutMs: 5 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const recovered = await harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"SEQUENCE_GAP",
|
|
);
|
|
expect(recovered).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "RECOVER", retryable: false },
|
|
});
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
const beforeLateCommit = harness.coordinator.getResumeState(STREAM_ID);
|
|
|
|
// A late checkpoint from the abandoned attempt cannot commit.
|
|
wedged.resolve(realtimeSuccess(snapshotCommit("9")));
|
|
for (let flush = 0; flush < 10; flush += 1) await Promise.resolve();
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toEqual(
|
|
beforeLateCommit,
|
|
);
|
|
// Settlement returns the stream to OPEN, marked STALE for an authoritative
|
|
// recovery rather than silently trusting the abandoned attempt.
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("OPEN");
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
|
|
it("applies one stream sequentially and commits each cursor after its effect", async () => {
|
|
const first = deferred<RealtimeResult<void>>();
|
|
const applied: string[] = [];
|
|
const harness = createHarness({
|
|
async apply(_profile, value) {
|
|
const selected = (value as Readonly<{ value: string }>).value;
|
|
applied.push(selected);
|
|
return selected === "first"
|
|
? first.promise
|
|
: realtimeSuccess(undefined);
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
const firstResult = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
resumeCursor: "cursor-00000001",
|
|
payload: { value: "first" },
|
|
}),
|
|
);
|
|
const secondResult = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
payload: { value: "second" },
|
|
}),
|
|
);
|
|
|
|
await vi.waitFor(() => expect(applied).toEqual(["first"]));
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: "cursor-snapshot-0",
|
|
});
|
|
|
|
first.resolve(realtimeSuccess(undefined));
|
|
await expect(firstResult).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
await expect(secondResult).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(applied).toEqual(["first", "second"]);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toEqual({
|
|
recoveryMode: "CURSOR",
|
|
streamEpoch: "stream-epoch-0001",
|
|
lastAppliedSequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
});
|
|
});
|
|
|
|
it("drops exact duplicates and stale events without replaying effects", async () => {
|
|
const harness = createHarness();
|
|
await harness.initialize();
|
|
const event = harness.event();
|
|
|
|
await harness.coordinator.accept(event);
|
|
await expect(harness.coordinator.accept(event)).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
outcome: "DROPPED",
|
|
reason: "DUPLICATE_EVENT",
|
|
},
|
|
});
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-stale-0001",
|
|
sequence: "0",
|
|
resumeCursor: "cursor-stale-0001",
|
|
}),
|
|
),
|
|
).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
outcome: "DROPPED",
|
|
reason: "STALE_EVENT",
|
|
},
|
|
});
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(1);
|
|
});
|
|
|
|
it("recovers without applying a conflicting ID, sequence gap or epoch", async () => {
|
|
for (const [overrides, expectedReason] of [
|
|
[
|
|
{
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
payload: { value: "conflict" },
|
|
},
|
|
"EVENT_CONFLICT",
|
|
],
|
|
[
|
|
{
|
|
eventId: "event-gap-000001",
|
|
sequence: "3",
|
|
resumeCursor: "cursor-gap-000001",
|
|
},
|
|
"SEQUENCE_GAP",
|
|
],
|
|
[
|
|
{
|
|
eventId: "event-epoch-0001",
|
|
sequence: "2",
|
|
streamEpoch: "stream-epoch-0002",
|
|
resumeCursor: "cursor-epoch-0001",
|
|
},
|
|
"STREAM_EPOCH_CHANGED",
|
|
],
|
|
] as const) {
|
|
const harness = createHarness();
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(harness.event());
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event(overrides)),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: expectedReason,
|
|
},
|
|
});
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
expectedReason,
|
|
]);
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
}
|
|
});
|
|
|
|
it("does not apply or advance when mapping fails", async () => {
|
|
const harness = createHarness({
|
|
mappers: {
|
|
ReferenceRealtimeMapper: {
|
|
...TEST_MAPPER,
|
|
map: () => ({
|
|
ok: false as const,
|
|
code: "MAPPING_INVARIANT_REJECTED" as const,
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "MAPPING_CONTRACT_VIOLATION",
|
|
},
|
|
});
|
|
expect(harness.effects).not.toHaveBeenCalled();
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
"MAPPING_CONTRACT_VIOLATION",
|
|
]);
|
|
});
|
|
|
|
it("leaves the prior checkpoint when an effect and its recovery fail", async () => {
|
|
let recoveryCount = 0;
|
|
const harness = createHarness({
|
|
apply: async () => realtimeFailure("APPLY_FAILED", "APPLY"),
|
|
recover: async () => {
|
|
recoveryCount += 1;
|
|
return recoveryCount === 1
|
|
? realtimeSuccess(snapshotCommit("0"))
|
|
: realtimeFailure("PROVIDER_UNAVAILABLE", "RECOVER");
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "RECOVER",
|
|
},
|
|
});
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: "cursor-snapshot-0",
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"UNKNOWN",
|
|
);
|
|
await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
);
|
|
expect(harness.effects).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("rejects malformed authority results without advancing a checkpoint", async () => {
|
|
let recoveryCount = 0;
|
|
const harness = createHarness({
|
|
apply: async () =>
|
|
({
|
|
ok: true,
|
|
value: "not-void",
|
|
}) as unknown as RealtimeResult<void>,
|
|
recover: async () => {
|
|
recoveryCount += 1;
|
|
return recoveryCount === 1
|
|
? realtimeSuccess(snapshotCommit("0"))
|
|
: ({
|
|
ok: false,
|
|
error: {
|
|
kind: "FORBIDDEN",
|
|
operation: "APPLY",
|
|
retryable: false,
|
|
},
|
|
} as RealtimeResult<RealtimeRecoveryCommit>);
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECOVER"),
|
|
);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "0",
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"UNKNOWN",
|
|
);
|
|
});
|
|
|
|
it("rejects recovery accessors without invoking them", async () => {
|
|
let checkpointReads = 0;
|
|
const accessorCommit = Object.defineProperties(
|
|
{},
|
|
{
|
|
checkpoint: {
|
|
enumerable: true,
|
|
get() {
|
|
checkpointReads += 1;
|
|
return snapshotCommit("0");
|
|
},
|
|
},
|
|
kind: {
|
|
enumerable: true,
|
|
value: "SNAPSHOT_RESET",
|
|
},
|
|
},
|
|
) as RealtimeRecoveryCommit;
|
|
const harness = createHarness({
|
|
recover: async () => realtimeSuccess(accessorCommit),
|
|
});
|
|
|
|
await expect(
|
|
harness.coordinator.recover(STREAM_ID, "INITIALIZE"),
|
|
).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECOVER"),
|
|
);
|
|
expect(checkpointReads).toBe(0);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
});
|
|
|
|
it("commits only the one-shot descriptor snapshot of a recovery proxy", async () => {
|
|
const descriptorCommit = snapshotCommit("0");
|
|
const propertyReads: PropertyKey[] = [];
|
|
const proxyCommit = new Proxy(descriptorCommit, {
|
|
get(_target, key) {
|
|
propertyReads.push(key);
|
|
if (key === "kind") return "SESSION_REBUILD";
|
|
if (key === "checkpoint") return snapshotCommit("99");
|
|
return undefined;
|
|
},
|
|
});
|
|
const harness = createHarness({
|
|
recover: async () => realtimeSuccess(proxyCommit),
|
|
});
|
|
|
|
const recovered = await harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
|
|
expect(recovered).toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: "cursor-snapshot-0",
|
|
},
|
|
});
|
|
expect(propertyReads).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
Object.freeze({
|
|
...snapshotCommit("0"),
|
|
unexpected: true,
|
|
}),
|
|
Object.assign(
|
|
Object.create({ inherited: true }) as Record<string, unknown>,
|
|
snapshotCommit("0"),
|
|
),
|
|
])(
|
|
"rejects extra and inherited recovery commit shapes",
|
|
async (commit) => {
|
|
const harness = createHarness({
|
|
recover: async () =>
|
|
realtimeSuccess(commit as RealtimeRecoveryCommit),
|
|
});
|
|
|
|
await expect(
|
|
harness.coordinator.recover(STREAM_ID, "INITIALIZE"),
|
|
).resolves.toEqual(
|
|
realtimeFailure("PROTOCOL_MISMATCH", "RECOVER"),
|
|
);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
},
|
|
);
|
|
|
|
it("expires effect and recovery commit leases when callbacks settle", async () => {
|
|
let effectContext: ExternalRealtimeEventContext | undefined;
|
|
let recoveryRequest: RealtimeRecoveryRequest | undefined;
|
|
const harness = createHarness({
|
|
async apply(_profile, _value, context) {
|
|
effectContext = context;
|
|
expect(context.isCurrent()).toBe(true);
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
async recover(request) {
|
|
recoveryRequest = request;
|
|
expect(request.isCurrent()).toBe(true);
|
|
return realtimeSuccess(snapshotCommit("0"));
|
|
},
|
|
});
|
|
|
|
await harness.initialize();
|
|
expect(recoveryRequest?.isCurrent()).toBe(false);
|
|
await harness.coordinator.accept(harness.event());
|
|
expect(effectContext?.isCurrent()).toBe(false);
|
|
});
|
|
|
|
it("fences an in-flight successful effect before cursor commit", async () => {
|
|
const pending = deferred<RealtimeResult<void>>();
|
|
let effectContext: ExternalRealtimeEventContext | undefined;
|
|
const harness = createHarness({
|
|
apply: async (_profile, _value, context) => {
|
|
effectContext = context;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
const accepted = harness.coordinator.accept(harness.event());
|
|
await vi.waitFor(() =>
|
|
expect(harness.effects).toHaveBeenCalledOnce(),
|
|
);
|
|
harness.setCurrent(false);
|
|
expect(effectContext?.isCurrent()).toBe(false);
|
|
pending.resolve(realtimeSuccess(undefined));
|
|
|
|
await expect(accepted).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
outcome: "DROPPED",
|
|
reason: "SCOPE_FENCED",
|
|
},
|
|
});
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
expect(harness.coordinator.inspect(STREAM_ID)).toMatchObject({
|
|
freshness: "UNKNOWN",
|
|
dedupeEntries: 0,
|
|
queuedEvents: 0,
|
|
awaitingTransportBarrier: false,
|
|
});
|
|
});
|
|
|
|
it("aborts and quiesces the active effect before queue-overflow recovery", async () => {
|
|
const pending = deferred<RealtimeResult<void>>();
|
|
let capturedSignal: AbortSignal | undefined;
|
|
const registry = createTestRealtimeRegistry({
|
|
limits: {
|
|
...TEST_LIMITS,
|
|
maxQueueEvents: 1,
|
|
},
|
|
});
|
|
const harness = createHarness({
|
|
registry,
|
|
async apply(_profile, _value, _context, signal) {
|
|
capturedSignal = signal;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
const first = harness.coordinator.accept(harness.event());
|
|
await vi.waitFor(() =>
|
|
expect(harness.effects).toHaveBeenCalledOnce(),
|
|
);
|
|
const overflow = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
);
|
|
|
|
expect(capturedSignal?.aborted).toBe(true);
|
|
expect(harness.recoveryReasons).toEqual(["INITIALIZE"]);
|
|
pending.resolve(realtimeSuccess(undefined));
|
|
await expect(first).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "DROPPED" },
|
|
});
|
|
await expect(overflow).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "QUEUE_OVERFLOW",
|
|
},
|
|
});
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
"QUEUE_OVERFLOW",
|
|
]);
|
|
});
|
|
|
|
it("treats a current scope mismatch as a protocol violation, not a stale callback", async () => {
|
|
const harness = createHarness();
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({ scopeBinding: "other-scope-binding" }),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: {
|
|
outcome: "RECOVERED",
|
|
reason: "SCOPE_PROTOCOL_VIOLATION",
|
|
},
|
|
});
|
|
expect(harness.effects).not.toHaveBeenCalled();
|
|
expect(harness.recoveryReasons).toEqual([
|
|
"INITIALIZE",
|
|
"SCOPE_PROTOCOL_VIOLATION",
|
|
]);
|
|
});
|
|
|
|
it("never claims CURRENT for a SNAPSHOT_ONLY stream without a barrier", async () => {
|
|
const registry = createTestRealtimeRegistry({
|
|
recovery: {
|
|
mode: "SNAPSHOT_ONLY",
|
|
snapshotOperationId: "GET_REFERENCE_REALTIME_SNAPSHOT",
|
|
checkpointCodecId: "ReferenceRealtimeCheckpoint",
|
|
barrier: "NONE",
|
|
},
|
|
delivery: "INVALIDATION_HINT",
|
|
stateBearing: false,
|
|
});
|
|
const harness = createHarness({
|
|
registry,
|
|
recover: async () =>
|
|
realtimeSuccess({
|
|
kind: "SNAPSHOT_RESET",
|
|
checkpoint: {
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
streamEpoch: "stream-epoch-0001",
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: null,
|
|
snapshotRevision: "snapshot-revision-0",
|
|
},
|
|
}),
|
|
});
|
|
await harness.initialize();
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
resumeCursor: null,
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
|
|
it("requires an exact transport barrier before promoting a recovered stream to CURRENT", async () => {
|
|
const harness = createHarness();
|
|
const recovered = await harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
expect(recovered.ok).toBe(true);
|
|
if (!recovered.ok) return;
|
|
await Promise.resolve();
|
|
|
|
expect(harness.coordinator.inspect(STREAM_ID)).toMatchObject({
|
|
freshness: "STALE",
|
|
awaitingTransportBarrier: true,
|
|
});
|
|
expect(
|
|
harness.coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
Object.freeze({
|
|
...recovered.value,
|
|
}) as typeof recovered.value,
|
|
),
|
|
).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
const publicResumeState =
|
|
harness.coordinator.getResumeState(STREAM_ID);
|
|
expect(publicResumeState).not.toBe(recovered.value);
|
|
expect(
|
|
harness.coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
publicResumeState as typeof recovered.value,
|
|
),
|
|
).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
expect(
|
|
harness.coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
recovered.value,
|
|
),
|
|
).toEqual(realtimeSuccess(undefined));
|
|
expect(harness.coordinator.inspect(STREAM_ID)).toMatchObject({
|
|
freshness: "CURRENT",
|
|
awaitingTransportBarrier: false,
|
|
});
|
|
});
|
|
|
|
it("marks a successful invalidation hint stale until authoritative refresh", async () => {
|
|
const registry = createTestRealtimeRegistry({
|
|
recovery: {
|
|
mode: "SNAPSHOT_ONLY",
|
|
snapshotOperationId: "GET_REFERENCE_REALTIME_SNAPSHOT",
|
|
checkpointCodecId: "ReferenceRealtimeCheckpoint",
|
|
barrier: "CONNECT_BUFFER",
|
|
},
|
|
delivery: "INVALIDATION_HINT",
|
|
stateBearing: false,
|
|
});
|
|
const harness = createHarness({
|
|
registry,
|
|
recover: async () =>
|
|
realtimeSuccess({
|
|
kind: "SNAPSHOT_RESET",
|
|
checkpoint: {
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
streamEpoch: "stream-epoch-0001",
|
|
lastAppliedSequence: "0",
|
|
resumeCursor: null,
|
|
snapshotRevision: "snapshot-revision-0",
|
|
},
|
|
}),
|
|
});
|
|
await harness.initialize();
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"CURRENT",
|
|
);
|
|
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
harness.event({
|
|
recoveryMode: "SNAPSHOT_ONLY",
|
|
resumeCursor: null,
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
|
|
it("evicts the oldest dedupe entry and keeps applying at capacity", async () => {
|
|
const harness = createHarness({
|
|
registry: createTestRealtimeRegistry({
|
|
limits: {
|
|
...TEST_LIMITS,
|
|
maxDedupeEntries: 1,
|
|
},
|
|
}),
|
|
});
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(harness.event());
|
|
const second = {
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
};
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event(second)),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "APPLIED" },
|
|
});
|
|
expect(harness.effects).toHaveBeenCalledTimes(2);
|
|
expect(harness.recoveryReasons).toEqual(["INITIALIZE"]);
|
|
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(1);
|
|
// The newest entry still catches its duplicate; the evicted one is stale.
|
|
await expect(
|
|
harness.coordinator.accept(harness.event(second)),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "DROPPED", reason: "DUPLICATE_EVENT" },
|
|
});
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "DROPPED", reason: "STALE_EVENT" },
|
|
});
|
|
});
|
|
|
|
it("expires dedupe entries by TTL oldest first", async () => {
|
|
let now = 10_000;
|
|
const harness = createHarness({ now: () => now });
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(harness.event());
|
|
now += TEST_LIMITS.dedupeTtlMs / 2;
|
|
await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000002",
|
|
sequence: "2",
|
|
resumeCursor: "cursor-00000002",
|
|
}),
|
|
);
|
|
now += TEST_LIMITS.dedupeTtlMs / 2;
|
|
await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000003",
|
|
sequence: "3",
|
|
resumeCursor: "cursor-00000003",
|
|
}),
|
|
);
|
|
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(2);
|
|
});
|
|
|
|
it("rejects a regressing same-epoch recovery checkpoint atomically", async () => {
|
|
let checkpoint = "5";
|
|
const harness = createHarness({
|
|
recover: async () => realtimeSuccess(snapshotCommit(checkpoint)),
|
|
});
|
|
await harness.initialize();
|
|
checkpoint = "4";
|
|
|
|
await expect(
|
|
harness.coordinator.recover(STREAM_ID, "CURSOR_EXPIRED"),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toMatchObject({
|
|
lastAppliedSequence: "5",
|
|
});
|
|
});
|
|
|
|
it("coalesces recovery, closes idempotently and rejects forged DTOs", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
const recover = vi.fn(async () => pending.promise);
|
|
const harness = createHarness({ recover });
|
|
|
|
const first = harness.coordinator.recover(STREAM_ID, "INITIALIZE");
|
|
const second = harness.coordinator.recover(STREAM_ID, "SEQUENCE_GAP");
|
|
await vi.waitFor(() => expect(recover).toHaveBeenCalledOnce());
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
await expect(first).resolves.toMatchObject({ ok: true });
|
|
await expect(second).resolves.toMatchObject({ ok: true });
|
|
|
|
harness.coordinator.close();
|
|
harness.coordinator.close();
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
await expect(
|
|
harness.coordinator.accept(
|
|
Object.freeze({}) as ValidatedRealtimeEventDto,
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "MALFORMED_EVENT" },
|
|
});
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toEqual({
|
|
ok: true,
|
|
value: { outcome: "DROPPED", reason: "CLOSED" },
|
|
});
|
|
});
|
|
|
|
it("returns the exact shared checkpoint to an event waiting on active recovery", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
const recover = vi.fn(async () => pending.promise);
|
|
const harness = createHarness({ recover });
|
|
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await vi.waitFor(() => expect(recover).toHaveBeenCalledOnce());
|
|
const accepting = harness.coordinator.accept(harness.event());
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
|
|
const recovered = await recovering;
|
|
const accepted = await accepting;
|
|
expect(recovered.ok).toBe(true);
|
|
expect(accepted.ok).toBe(true);
|
|
if (!recovered.ok || !accepted.ok) return;
|
|
expect(accepted.value).toMatchObject({
|
|
outcome: "RECOVERED",
|
|
reason: "INITIALIZE",
|
|
});
|
|
if (accepted.value.outcome !== "RECOVERED") return;
|
|
expect(accepted.value.resumeState).toBe(recovered.value);
|
|
});
|
|
|
|
it("aborts only an event waiter without cancelling shared recovery", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let recoverySignal: AbortSignal | undefined;
|
|
const harness = createHarness({
|
|
recover: async (request) => {
|
|
recoverySignal = request.signal;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await vi.waitFor(() => expect(recoverySignal).toBeDefined());
|
|
const controller = new AbortController();
|
|
const accepting = harness.coordinator.accept(
|
|
harness.event(),
|
|
controller.signal,
|
|
);
|
|
|
|
controller.abort();
|
|
await expect(accepting).resolves.toEqual(
|
|
realtimeFailure("ABORTED", "RECEIVE"),
|
|
);
|
|
expect(recoverySignal?.aborted).toBe(false);
|
|
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
await expect(recovering).resolves.toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("aborts the active recovery authority when closed", async () => {
|
|
let recoverySignal: AbortSignal | undefined;
|
|
const harness = createHarness({
|
|
recover: async (request) => {
|
|
recoverySignal = request.signal;
|
|
return new Promise((resolve) => {
|
|
request.signal.addEventListener(
|
|
"abort",
|
|
() => resolve(realtimeFailure("ABORTED", "RECOVER")),
|
|
{ once: true },
|
|
);
|
|
});
|
|
},
|
|
});
|
|
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await vi.waitFor(() => expect(recoverySignal).toBeDefined());
|
|
harness.coordinator.close();
|
|
|
|
expect(recoverySignal?.aborted).toBe(true);
|
|
await expect(recovering).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "CLOSED", operation: "RECOVER" },
|
|
});
|
|
});
|
|
|
|
it("does not commit a non-cooperative recovery after caller cancellation", async () => {
|
|
const pending = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let recoveryRequest: RealtimeRecoveryRequest | undefined;
|
|
const harness = createHarness({
|
|
recover: async (request) => {
|
|
recoveryRequest = request;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
const controller = new AbortController();
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
controller.signal,
|
|
);
|
|
await vi.waitFor(() => expect(recoveryRequest).toBeDefined());
|
|
|
|
controller.abort();
|
|
expect(recoveryRequest?.isCurrent()).toBe(false);
|
|
pending.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
|
|
await expect(recovering).resolves.toEqual(
|
|
realtimeFailure("ABORTED", "RECOVER"),
|
|
);
|
|
expect(recoveryRequest?.signal.aborted).toBe(true);
|
|
expect(recoveryRequest?.isCurrent()).toBe(false);
|
|
expect(harness.coordinator.getResumeState(STREAM_ID)).toBeNull();
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe(
|
|
"UNKNOWN",
|
|
);
|
|
});
|
|
|
|
it("emits only closed redacted observations and ignores sink failure", async () => {
|
|
const observations: RealtimeObservation[] = [];
|
|
const harness = createHarness({
|
|
observe(observation) {
|
|
observations.push(observation);
|
|
throw new Error("diagnostic sink unavailable");
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
await harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "sensitive-event-id",
|
|
resumeCursor: "sensitive-cursor",
|
|
payload: { value: "sensitive-payload" },
|
|
}),
|
|
);
|
|
|
|
const serialized = JSON.stringify(observations);
|
|
expect(serialized).not.toContain("sensitive-event-id");
|
|
expect(serialized).not.toContain("sensitive-cursor");
|
|
expect(serialized).not.toContain("sensitive-payload");
|
|
expect(serialized).not.toContain("scope-binding-0001");
|
|
expect(observations).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
operation: "RECOVER",
|
|
outcome: "RECOVERED",
|
|
reason: "INITIALIZE",
|
|
}),
|
|
expect.objectContaining({
|
|
operation: "APPLY",
|
|
outcome: "APPLIED",
|
|
}),
|
|
]),
|
|
);
|
|
});
|
|
});
|
|
|
|
/**
|
|
* RT-RR-01 / RT-RR-02. A physical effect exists from the moment the coordinator
|
|
* calls the authority, not from the moment its public wait expires. Registering
|
|
* only on timeout let a `close()` that arrived first see an empty retained set
|
|
* and report quiescence while the raw task was still running against the
|
|
* authority.
|
|
*/
|
|
describe("realtime physical task ownership", () => {
|
|
it("does not report quiescence while an apply is still running", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
const harness = createHarness({
|
|
apply: async () => await applyGate.promise,
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applying = harness.coordinator.accept(harness.event());
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
|
|
// close() arrives long before the effect deadline.
|
|
const closed = await harness.coordinator.close();
|
|
expect(closed.ok).toBe(false);
|
|
expect(closed.ok ? null : closed.error.kind).toBe("IDLE_TIMEOUT");
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
await applying;
|
|
});
|
|
|
|
it("reports quiescence once the raw task settles", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
const harness = createHarness({
|
|
apply: async () => await applyGate.promise,
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 200 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applying = harness.coordinator.accept(harness.event());
|
|
await Promise.resolve();
|
|
const closing = harness.coordinator.close();
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
await applying;
|
|
|
|
expect(await closing).toMatchObject({ ok: true });
|
|
});
|
|
|
|
it("does not start a queued event once the stream is DRAINING", async () => {
|
|
const firstApply = deferred<RealtimeResult<void>>();
|
|
let applyCalls = 0;
|
|
const harness = createHarness({
|
|
apply: async () => {
|
|
applyCalls += 1;
|
|
if (applyCalls === 1) return await firstApply.promise;
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
taskLimits: { effectTimeoutMs: 15, drainTimeoutMs: 50 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const first = harness.coordinator.accept(
|
|
harness.event({ eventId: "event-0001", sequence: "1" }),
|
|
);
|
|
// Queued behind the first while it is still inside its deadline.
|
|
const second = harness.coordinator.accept(
|
|
harness.event({ eventId: "event-0002", sequence: "2" }),
|
|
);
|
|
|
|
await first;
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
// The queued event is dropped at execution time, not applied and not
|
|
// recovered: admission happened before DRAINING, execution happens inside
|
|
// it, and the second decision is the one that counts.
|
|
const secondResult = await second;
|
|
expect(secondResult).toMatchObject({
|
|
ok: true,
|
|
value: { outcome: "DROPPED", reason: "CLOSED" },
|
|
});
|
|
expect(applyCalls).toBe(1);
|
|
|
|
firstApply.resolve(realtimeSuccess(undefined));
|
|
});
|
|
|
|
it("discards the resume token when an effect is abandoned", async () => {
|
|
const firstApply = deferred<RealtimeResult<void>>();
|
|
let applyCalls = 0;
|
|
const harness = createHarness({
|
|
apply: async () => {
|
|
applyCalls += 1;
|
|
if (applyCalls === 1) return await firstApply.promise;
|
|
return realtimeSuccess(undefined);
|
|
},
|
|
taskLimits: { effectTimeoutMs: 15, drainTimeoutMs: 50 },
|
|
});
|
|
await harness.initialize();
|
|
expect(harness.coordinator.inspect(STREAM_ID).hasResumeState).toBe(true);
|
|
|
|
await harness.coordinator.accept(
|
|
harness.event({ eventId: "event-0001", sequence: "1" }),
|
|
);
|
|
|
|
// The abandoned effect may have applied part of its change, so the token it
|
|
// was based on is no longer authoritative evidence.
|
|
const inspection = harness.coordinator.inspect(STREAM_ID);
|
|
expect(inspection.hasResumeState).toBe(false);
|
|
expect(inspection.freshness).toBe("UNKNOWN");
|
|
|
|
firstApply.resolve(realtimeSuccess(undefined));
|
|
});
|
|
|
|
it("fails a recovery whose effect quiescence ends in DRAINING", async () => {
|
|
const wedged = deferred<RealtimeResult<void>>();
|
|
let recoveries = 0;
|
|
const harness = createHarness({
|
|
apply: async () => wedged.promise,
|
|
recover: async () => {
|
|
recoveries += 1;
|
|
return realtimeSuccess(snapshotCommit("0"));
|
|
},
|
|
taskLimits: { effectTimeoutMs: 5, drainTimeoutMs: 5 },
|
|
});
|
|
await harness.initialize();
|
|
recoveries = 0;
|
|
|
|
const applying = harness.coordinator.accept(
|
|
harness.event({
|
|
eventId: "event-00000001",
|
|
sequence: "1",
|
|
resumeCursor: "cursor-00000001",
|
|
}),
|
|
);
|
|
for (let flush = 0; flush < 10; flush += 1) await Promise.resolve();
|
|
expect(harness.effects).toHaveBeenCalledOnce();
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("OPEN");
|
|
// The effect ignores abort, so it outlives its deadline and the stream
|
|
// enters DRAINING while this recovery waits for quiescence.
|
|
const recovering = harness.coordinator.recover(
|
|
STREAM_ID,
|
|
"SEQUENCE_GAP",
|
|
);
|
|
|
|
await expect(applying).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "IDLE_TIMEOUT", operation: "APPLY" },
|
|
});
|
|
await expect(recovering).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "CLOSED", operation: "RECOVER" },
|
|
});
|
|
expect(recoveries).toBe(0);
|
|
expect(harness.coordinator.inspect(STREAM_ID).hasResumeState).toBe(
|
|
false,
|
|
);
|
|
|
|
// The late effect settles; the stream still needs a real recovery.
|
|
wedged.resolve(realtimeSuccess(undefined));
|
|
await vi.waitFor(() =>
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("OPEN"),
|
|
);
|
|
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
|
});
|
|
});
|
|
|
|
type HarnessOptions = Readonly<{
|
|
registry?: RealtimePolicyRegistry;
|
|
mappers?: typeof TEST_MAPPERS | Readonly<Record<string, typeof TEST_MAPPER>>;
|
|
apply?: RealtimeEventAuthority["effects"]["apply"];
|
|
recover?: (
|
|
request: RealtimeRecoveryRequest,
|
|
) => Promise<RealtimeResult<RealtimeRecoveryCommit>>;
|
|
observe?: (observation: RealtimeObservation) => void;
|
|
taskLimits?: Readonly<{
|
|
effectTimeoutMs?: number;
|
|
recoveryTimeoutMs?: number;
|
|
drainTimeoutMs?: number;
|
|
}>;
|
|
scheduleTimeout?: (callback: () => void, delayMs: number) => unknown;
|
|
clearScheduledTimeout?: (handle: unknown) => void;
|
|
now?: () => number;
|
|
}>;
|
|
|
|
function createHarness(options: HarnessOptions = {}) {
|
|
const registry = options.registry ?? createTestRealtimeRegistry();
|
|
const codec = createTestRealtimeCodec(registry);
|
|
let current = true;
|
|
const recoveryReasons: string[] = [];
|
|
let defaultRecoveryCount = 0;
|
|
const effects = vi.fn(
|
|
options.apply ??
|
|
(async () => realtimeSuccess(undefined)),
|
|
);
|
|
const recover = vi.fn(
|
|
options.recover ??
|
|
(async (request: RealtimeRecoveryRequest) => {
|
|
recoveryReasons.push(request.reason);
|
|
defaultRecoveryCount += 1;
|
|
return realtimeSuccess(
|
|
snapshotCommit(
|
|
"0",
|
|
`stream-epoch-${String(defaultRecoveryCount).padStart(4, "0")}`,
|
|
),
|
|
);
|
|
}),
|
|
);
|
|
const authority: RealtimeEventAuthority = {
|
|
effects: { apply: effects },
|
|
recovery: {
|
|
async recover(request) {
|
|
if (options.recover) recoveryReasons.push(request.reason);
|
|
return recover(request);
|
|
},
|
|
},
|
|
};
|
|
const coordinator = createRealtimeStreamCoordinator({
|
|
registry,
|
|
mappers: options.mappers ?? TEST_MAPPERS,
|
|
authority,
|
|
scope: {
|
|
generation: 7,
|
|
scopeBinding: "scope-binding-0001",
|
|
isCurrent: () => current,
|
|
},
|
|
now: options.now ?? (() => 10_000),
|
|
observe: options.observe,
|
|
...(options.taskLimits ? { taskLimits: options.taskLimits } : {}),
|
|
...(options.scheduleTimeout
|
|
? { scheduleTimeout: options.scheduleTimeout }
|
|
: {}),
|
|
...(options.clearScheduledTimeout
|
|
? { clearScheduledTimeout: options.clearScheduledTimeout }
|
|
: {}),
|
|
});
|
|
|
|
return {
|
|
coordinator,
|
|
effects,
|
|
recoveryReasons,
|
|
event(overrides: EventOverrides = {}) {
|
|
const decoded = codec.decode(realtimeEventJson(overrides));
|
|
if (!decoded.ok) {
|
|
throw new Error(`test event failed: ${decoded.error.kind}`);
|
|
}
|
|
return decoded.value;
|
|
},
|
|
async initialize() {
|
|
const result = await coordinator.recover(
|
|
STREAM_ID,
|
|
"INITIALIZE",
|
|
);
|
|
await Promise.resolve();
|
|
if (
|
|
result.ok &&
|
|
coordinator.inspect(STREAM_ID).awaitingTransportBarrier
|
|
) {
|
|
const confirmed = coordinator.confirmTransportBarrier(
|
|
STREAM_ID,
|
|
result.value,
|
|
);
|
|
if (!confirmed.ok) {
|
|
throw new Error("test transport barrier failed");
|
|
}
|
|
}
|
|
return result;
|
|
},
|
|
setCurrent(value: boolean) {
|
|
current = value;
|
|
},
|
|
};
|
|
}
|
|
|
|
function snapshotCommit(
|
|
sequence: string,
|
|
streamEpoch = "stream-epoch-0001",
|
|
): RealtimeRecoveryCommit {
|
|
return Object.freeze({
|
|
kind: "SNAPSHOT_RESET",
|
|
checkpoint: Object.freeze({
|
|
recoveryMode: "CURSOR",
|
|
streamEpoch,
|
|
lastAppliedSequence: sequence,
|
|
resumeCursor: `cursor-snapshot-${sequence}`,
|
|
snapshotRevision: `snapshot-revision-${sequence}`,
|
|
}),
|
|
});
|
|
}
|
|
|
|
function deferred<Value>() {
|
|
let resolve!: (value: Value) => void;
|
|
const promise = new Promise<Value>((next) => {
|
|
resolve = next;
|
|
});
|
|
return { promise, resolve };
|
|
}
|
|
|
|
/**
|
|
* RT-01. The registry entry has to exist before the collaborator is called.
|
|
* Registering after the invocation returned left a window in which an authority
|
|
* that re-entered `close()` from inside its own callback saw an empty set, so
|
|
* `close()` reported quiescence while its effect was still running.
|
|
*/
|
|
describe("RT-01 reentrant close cannot pass the registration window", () => {
|
|
it("refuses quiescence when apply re-enters close during its invocation", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
let closeResult: RealtimeResult<void> | undefined;
|
|
let harness: ReturnType<typeof createHarness> | undefined;
|
|
harness = createHarness({
|
|
apply: () => {
|
|
// Re-entered from inside the invocation itself.
|
|
void harness!.coordinator.close().then((result) => {
|
|
closeResult = result;
|
|
});
|
|
return applyGate.promise;
|
|
},
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const applying = harness.coordinator.accept(harness.event());
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
|
|
expect(closeResult?.ok).toBe(false);
|
|
expect(closeResult?.ok ? null : closeResult?.error.kind).toBe(
|
|
"IDLE_TIMEOUT",
|
|
);
|
|
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
await applying;
|
|
// Once the raw task settled, a second close does converge.
|
|
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
|
ok: true,
|
|
});
|
|
});
|
|
|
|
it("refuses quiescence when recovery re-enters close during its invocation", async () => {
|
|
const recoveryGate = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
|
let closeResult: RealtimeResult<void> | undefined;
|
|
let harness: ReturnType<typeof createHarness> | undefined;
|
|
let recoveries = 0;
|
|
harness = createHarness({
|
|
recover: () => {
|
|
recoveries += 1;
|
|
if (recoveries === 1) {
|
|
return Promise.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
}
|
|
void harness!.coordinator.close().then((result) => {
|
|
closeResult = result;
|
|
});
|
|
return recoveryGate.promise;
|
|
},
|
|
taskLimits: { recoveryTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
|
});
|
|
await harness.initialize();
|
|
|
|
const recovering = harness.coordinator.recover(STREAM_ID, "SEQUENCE_GAP");
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
|
|
expect(closeResult?.ok).toBe(false);
|
|
expect(closeResult?.ok ? null : closeResult?.error.kind).toBe(
|
|
"IDLE_TIMEOUT",
|
|
);
|
|
|
|
recoveryGate.resolve(realtimeSuccess(snapshotCommit("0")));
|
|
await recovering;
|
|
});
|
|
});
|
|
|
|
/**
|
|
* RT-02. A scheduler that cannot install a deadline leaves the wait unbounded.
|
|
* Letting the exception escape turned a typed realtime result into a native
|
|
* rejection and, through the caller's own catch, started a recovery that
|
|
* overlapped the effect still running.
|
|
*/
|
|
describe("RT-02 a throwing scheduler stays inside the result contract", () => {
|
|
/** Installs deadlines normally until the stream is initialized. */
|
|
function breakableScheduler() {
|
|
const state = { broken: false };
|
|
return {
|
|
state,
|
|
scheduleTimeout: (callback: () => void, delayMs: number) => {
|
|
if (state.broken) throw new TypeError("scheduleTimeout exploded");
|
|
return setTimeout(callback, delayMs);
|
|
},
|
|
clearScheduledTimeout: (handle: unknown) => {
|
|
clearTimeout(handle as ReturnType<typeof setTimeout>);
|
|
},
|
|
};
|
|
}
|
|
|
|
it("does not start a recovery that overlaps a pending apply", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
const scheduler = breakableScheduler();
|
|
let recoveriesAfterApply = 0;
|
|
const harness = createHarness({
|
|
apply: () => applyGate.promise,
|
|
recover: async () => {
|
|
recoveriesAfterApply += 1;
|
|
return realtimeSuccess(snapshotCommit("0"));
|
|
},
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
|
scheduleTimeout: scheduler.scheduleTimeout,
|
|
clearScheduledTimeout: scheduler.clearScheduledTimeout,
|
|
});
|
|
await harness.initialize();
|
|
recoveriesAfterApply = 0;
|
|
scheduler.state.broken = true;
|
|
|
|
const accepted = await harness.coordinator.accept(harness.event());
|
|
|
|
// The apply is still pending, so no recovery may have run beside it.
|
|
expect(recoveriesAfterApply).toBe(0);
|
|
expect(accepted.ok).toBe(false);
|
|
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
|
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
});
|
|
|
|
it("returns a typed close failure instead of rejecting natively", async () => {
|
|
const applyGate = deferred<RealtimeResult<void>>();
|
|
const scheduler = breakableScheduler();
|
|
const harness = createHarness({
|
|
apply: () => applyGate.promise,
|
|
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
|
scheduleTimeout: scheduler.scheduleTimeout,
|
|
clearScheduledTimeout: scheduler.clearScheduledTimeout,
|
|
});
|
|
await harness.initialize();
|
|
scheduler.state.broken = true;
|
|
|
|
const applying = harness.coordinator.accept(harness.event());
|
|
// Let the apply reach the authority before the fence arrives.
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
const closed = await harness.coordinator.close();
|
|
|
|
// A drain that could not be bounded is not proof of quiescence.
|
|
expect(closed.ok).toBe(false);
|
|
expect(closed.ok ? null : closed.error.kind).toBe("IDLE_TIMEOUT");
|
|
|
|
applyGate.resolve(realtimeSuccess(undefined));
|
|
await applying;
|
|
});
|
|
|
|
it("keeps the classified outcome when clearing a timer throws", async () => {
|
|
const harness = createHarness({
|
|
clearScheduledTimeout: () => {
|
|
throw new TypeError("clearScheduledTimeout exploded");
|
|
},
|
|
});
|
|
await harness.initialize();
|
|
|
|
await expect(
|
|
harness.coordinator.accept(harness.event()),
|
|
).resolves.toMatchObject({ ok: true });
|
|
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
|
ok: true,
|
|
});
|
|
});
|
|
});
|