fix(realtime): evict dedupe entries FIFO and store hashed fingerprints (Q-6.4-6)
A full dedupe window (2,048 IDs / 4 MiB) used to trigger a full DEDUPE_OVERFLOW recovery on every event past capacity, so a steady 3.4 events/s stream re-fetched snapshots and reconnected every window. The coordinator now drops the oldest entry (Map insertion order equals apply order) and keeps applying; a replay pushed out of the window still drops as STALE_EVENT through the sequence check. The design doc row and the maxDedupeBytes contract comment now say eviction instead of recovery. The semantic fingerprint is now a 16-char 64-bit FNV-1a hash of the canonical envelope instead of the canonical text itself (up to 64 KiB each). The canonical-size EVENT_TOO_LARGE check only bounded that stored text, so it goes too; the wire-size check stays. pruneDedupe stops at the first unexpired entry instead of scanning every entry. The old "recovers instead of evicting" coordinator test encoded the bug and is replaced by an eviction test. RED: the eviction test (got RECOVERED/DEDUPE_OVERFLOW) and a fixed-size fingerprint codec test (got 3326 vs 333 bytes) failed before the change. Added a TTL prune characterization test (passes before and after). Gates: architecture, types app/test, lint, adapter-inventory, test:unit, test:component pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
cc906f08b4
commit
bc17ed4bd7
@@ -618,7 +618,8 @@ stream + epoch별 `lastAppliedSequence`를 기준으로 처리한다.
|
||||
| `sequence == lastApplied + 1` | 순차 적용 |
|
||||
| `sequence > lastApplied + 1` | gap; delta 적용 중지, `RESYNC_REQUIRED` |
|
||||
| unknown `streamEpoch` | old/new epoch를 섞지 않고 registered snapshot 또는 session rebuild |
|
||||
| dedupe/queue ceiling 초과 | 임의 eviction 후 계속 적용하지 않고 registered recovery |
|
||||
| dedupe window 가득 참 | 가장 오래된 항목부터 제거(FIFO)하고 계속 적용. 창에서 밀려난 재전송은 `sequence <= lastApplied`로 drop |
|
||||
| queue ceiling 초과 | 임의 eviction 후 계속 적용하지 않고 registered recovery |
|
||||
| captured old runtime generation | late callback safe drop + aggregate diagnostic |
|
||||
| current connection의 `scopeBinding` mismatch | effect/sequence/cursor 0, freshness `UNKNOWN`, close + session revalidation + registered recovery |
|
||||
|
||||
|
||||
@@ -31,7 +31,9 @@ export type ValidatedRealtimeEventDto = Readonly<{
|
||||
wireBytes: number;
|
||||
/**
|
||||
* Adapter-private semantic identity used only by the bounded conflict
|
||||
* detector. It must never be logged or projected into diagnostics.
|
||||
* detector: a fixed-size hash of the canonical envelope, so dedupe memory
|
||||
* does not grow with payload size. It must never be logged or projected
|
||||
* into diagnostics.
|
||||
*/
|
||||
semanticFingerprint: string;
|
||||
fingerprintBytes: number;
|
||||
@@ -186,11 +188,8 @@ function decodeUnsafe(
|
||||
eventType,
|
||||
payloadSnapshot,
|
||||
);
|
||||
const semanticFingerprint = canonicalJson(envelope);
|
||||
const fingerprintBytes = encoder.encode(semanticFingerprint).byteLength;
|
||||
if (fingerprintBytes > stream.limits.maxEventBytes) {
|
||||
return realtimeFailure("EVENT_TOO_LARGE", "DECODE");
|
||||
}
|
||||
const semanticFingerprint = hash64(canonicalJson(envelope));
|
||||
const fingerprintBytes = semanticFingerprint.length;
|
||||
|
||||
const dto = Object.freeze({
|
||||
envelope,
|
||||
@@ -367,6 +366,26 @@ function snapshotJson(
|
||||
return visit(value, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Two differently seeded 32-bit FNV-1a hashes as 16 hex characters.
|
||||
* ponytail: not collision-proof against a hostile server; a collision only
|
||||
* lets a conflicting redelivery pass as a duplicate. Use SHA-256 if the
|
||||
* server is not trusted to keep event IDs unique.
|
||||
*/
|
||||
function hash64(text: string): string {
|
||||
let first = 0x811c9dc5;
|
||||
let second = 0x9e3779b9;
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const unit = text.charCodeAt(index);
|
||||
first = Math.imul(first ^ unit, 0x01000193);
|
||||
second = Math.imul(second ^ unit, 0x01000193);
|
||||
}
|
||||
return (
|
||||
(first >>> 0).toString(16).padStart(8, "0") +
|
||||
(second >>> 0).toString(16).padStart(8, "0")
|
||||
);
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
if (
|
||||
value === null ||
|
||||
|
||||
@@ -487,20 +487,6 @@ export function createRealtimeStreamCoordinator(
|
||||
signal,
|
||||
);
|
||||
}
|
||||
if (
|
||||
state.eventIds.size + 1 >
|
||||
state.registration.limits.maxDedupeEntries ||
|
||||
state.dedupeBytes + event.fingerprintBytes >
|
||||
state.registration.limits.maxDedupeBytes
|
||||
) {
|
||||
return recoverForAccept(
|
||||
state,
|
||||
"DEDUPE_OVERFLOW",
|
||||
true,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
const eventType = dependencies.registry.findStreamEventType(
|
||||
state.registration.id,
|
||||
event.envelope.eventType,
|
||||
@@ -626,6 +612,17 @@ export function createRealtimeStreamCoordinator(
|
||||
fingerprintBytes: event.fingerprintBytes,
|
||||
appliedAt: safeNow(state),
|
||||
});
|
||||
// The dedupe window is the newest entries: drop the oldest to make room.
|
||||
for (const [eventId, oldest] of state.eventIds) {
|
||||
if (
|
||||
state.eventIds.size < state.registration.limits.maxDedupeEntries &&
|
||||
state.dedupeBytes + entry.fingerprintBytes <=
|
||||
state.registration.limits.maxDedupeBytes
|
||||
) {
|
||||
break;
|
||||
}
|
||||
forgetDedupe(state, eventId, oldest);
|
||||
}
|
||||
state.eventIds.set(entry.eventId, entry);
|
||||
state.sequences.set(entry.sequence, entry);
|
||||
state.dedupeBytes += entry.fingerprintBytes;
|
||||
@@ -1113,6 +1110,7 @@ export function createRealtimeStreamCoordinator(
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/** Entries are kept in apply order and `appliedAt` never decreases. */
|
||||
function pruneDedupe(state: StreamState): void {
|
||||
const selectedNow = safeNow(state);
|
||||
for (const [eventId, entry] of state.eventIds) {
|
||||
@@ -1120,19 +1118,27 @@ export function createRealtimeStreamCoordinator(
|
||||
selectedNow - entry.appliedAt <
|
||||
state.registration.limits.dedupeTtlMs
|
||||
) {
|
||||
continue;
|
||||
break;
|
||||
}
|
||||
state.eventIds.delete(eventId);
|
||||
if (state.sequences.get(entry.sequence) === entry) {
|
||||
state.sequences.delete(entry.sequence);
|
||||
}
|
||||
state.dedupeBytes = Math.max(
|
||||
0,
|
||||
state.dedupeBytes - entry.fingerprintBytes,
|
||||
);
|
||||
forgetDedupe(state, eventId, entry);
|
||||
}
|
||||
}
|
||||
|
||||
function forgetDedupe(
|
||||
state: StreamState,
|
||||
eventId: string,
|
||||
entry: DedupeEntry,
|
||||
): void {
|
||||
state.eventIds.delete(eventId);
|
||||
if (state.sequences.get(entry.sequence) === entry) {
|
||||
state.sequences.delete(entry.sequence);
|
||||
}
|
||||
state.dedupeBytes = Math.max(
|
||||
0,
|
||||
state.dedupeBytes - entry.fingerprintBytes,
|
||||
);
|
||||
}
|
||||
|
||||
function fenceState(state: StreamState): void {
|
||||
state.processingGeneration += 1;
|
||||
state.activeEffectAbort?.abort();
|
||||
|
||||
@@ -46,8 +46,8 @@ export type RealtimeLimits = Readonly<{
|
||||
maxDedupeEntries: number;
|
||||
/**
|
||||
* Additional implementation memory ceiling for semantic conflict
|
||||
* fingerprints. Reaching it has the same recovery meaning as exhausting the
|
||||
* count/time dedupe window.
|
||||
* fingerprints. Reaching it evicts the oldest entries, the same as
|
||||
* exhausting the count/time dedupe window.
|
||||
*/
|
||||
maxDedupeBytes: number;
|
||||
dedupeTtlMs: number;
|
||||
|
||||
@@ -227,6 +227,24 @@ describe("REALTIME_EVENT_V1 codec", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a fixed-size semantic fingerprint however large the payload", () => {
|
||||
const codec = createTestRealtimeCodec();
|
||||
const small = codec.decode(realtimeEventJson());
|
||||
const large = codec.decode(
|
||||
realtimeEventJson({ payload: { value: "x".repeat(3_000) } }),
|
||||
);
|
||||
|
||||
expect(small.ok && large.ok).toBe(true);
|
||||
if (!small.ok || !large.ok) return;
|
||||
expect(large.value.fingerprintBytes).toBe(
|
||||
small.value.fingerprintBytes,
|
||||
);
|
||||
expect(large.value.fingerprintBytes).toBeLessThanOrEqual(32);
|
||||
expect(large.value.semanticFingerprint).not.toBe(
|
||||
small.value.semanticFingerprint,
|
||||
);
|
||||
});
|
||||
|
||||
it("uses bounded uint64 sequence and header-safe cursor helpers", () => {
|
||||
expect(isCanonicalRealtimeSequence("0")).toBe(true);
|
||||
expect(isCanonicalRealtimeSequence(REALTIME_MAX_SEQUENCE)).toBe(true);
|
||||
|
||||
@@ -707,7 +707,7 @@ describe("transport-independent realtime stream coordinator", () => {
|
||||
expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE");
|
||||
});
|
||||
|
||||
it("recovers instead of evicting and continuing after dedupe capacity", async () => {
|
||||
it("evicts the oldest dedupe entry and keeps applying at capacity", async () => {
|
||||
const harness = createHarness({
|
||||
registry: createTestRealtimeRegistry({
|
||||
limits: {
|
||||
@@ -718,24 +718,58 @@ describe("transport-independent realtime stream coordinator", () => {
|
||||
});
|
||||
await harness.initialize();
|
||||
await harness.coordinator.accept(harness.event());
|
||||
const second = {
|
||||
eventId: "event-00000002",
|
||||
sequence: "2",
|
||||
resumeCursor: "cursor-00000002",
|
||||
};
|
||||
|
||||
await expect(
|
||||
harness.coordinator.accept(
|
||||
harness.event({
|
||||
eventId: "event-00000002",
|
||||
sequence: "2",
|
||||
resumeCursor: "cursor-00000002",
|
||||
}),
|
||||
),
|
||||
harness.coordinator.accept(harness.event(second)),
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
outcome: "RECOVERED",
|
||||
reason: "DEDUPE_OVERFLOW",
|
||||
},
|
||||
value: { outcome: "APPLIED" },
|
||||
});
|
||||
expect(harness.effects).toHaveBeenCalledTimes(1);
|
||||
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(0);
|
||||
expect(harness.effects).toHaveBeenCalledTimes(2);
|
||||
expect(harness.recoveryReasons).toEqual(["INITIALIZE"]);
|
||||
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(1);
|
||||
// The newest entry still catches its duplicate; the evicted one is stale.
|
||||
await expect(
|
||||
harness.coordinator.accept(harness.event(second)),
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
value: { outcome: "DROPPED", reason: "DUPLICATE_EVENT" },
|
||||
});
|
||||
await expect(
|
||||
harness.coordinator.accept(harness.event()),
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
value: { outcome: "DROPPED", reason: "STALE_EVENT" },
|
||||
});
|
||||
});
|
||||
|
||||
it("expires dedupe entries by TTL oldest first", async () => {
|
||||
let now = 10_000;
|
||||
const harness = createHarness({ now: () => now });
|
||||
await harness.initialize();
|
||||
await harness.coordinator.accept(harness.event());
|
||||
now += TEST_LIMITS.dedupeTtlMs / 2;
|
||||
await harness.coordinator.accept(
|
||||
harness.event({
|
||||
eventId: "event-00000002",
|
||||
sequence: "2",
|
||||
resumeCursor: "cursor-00000002",
|
||||
}),
|
||||
);
|
||||
now += TEST_LIMITS.dedupeTtlMs / 2;
|
||||
await harness.coordinator.accept(
|
||||
harness.event({
|
||||
eventId: "event-00000003",
|
||||
sequence: "3",
|
||||
resumeCursor: "cursor-00000003",
|
||||
}),
|
||||
);
|
||||
expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(2);
|
||||
});
|
||||
|
||||
it("rejects a regressing same-epoch recovery checkpoint atomically", async () => {
|
||||
@@ -1122,6 +1156,7 @@ type HarnessOptions = Readonly<{
|
||||
}>;
|
||||
scheduleTimeout?: (callback: () => void, delayMs: number) => unknown;
|
||||
clearScheduledTimeout?: (handle: unknown) => void;
|
||||
now?: () => number;
|
||||
}>;
|
||||
|
||||
function createHarness(options: HarnessOptions = {}) {
|
||||
@@ -1165,7 +1200,7 @@ function createHarness(options: HarnessOptions = {}) {
|
||||
scopeBinding: "scope-binding-0001",
|
||||
isCurrent: () => current,
|
||||
},
|
||||
now: () => 10_000,
|
||||
now: options.now ?? (() => 10_000),
|
||||
observe: options.observe,
|
||||
...(options.taskLimits ? { taskLimits: options.taskLimits } : {}),
|
||||
...(options.scheduleTimeout
|
||||
|
||||
Reference in New Issue
Block a user