fix(realtime): evict dedupe entries FIFO and store hashed fingerprints (Q-6.4-6)

A full dedupe window (2,048 IDs / 4 MiB) used to trigger a full
DEDUPE_OVERFLOW recovery on every event past capacity, so a steady
3.4 events/s stream re-fetched snapshots and reconnected every window.
The coordinator now drops the oldest entry (Map insertion order equals
apply order) and keeps applying; a replay pushed out of the window still
drops as STALE_EVENT through the sequence check. The design doc row and
the maxDedupeBytes contract comment now say eviction instead of recovery.

The semantic fingerprint is now a 16-char 64-bit FNV-1a hash of the
canonical envelope instead of the canonical text itself (up to 64 KiB
each). The canonical-size EVENT_TOO_LARGE check only bounded that stored
text, so it goes too; the wire-size check stays. pruneDedupe stops at the
first unexpired entry instead of scanning every entry.

The old "recovers instead of evicting" coordinator test encoded the bug
and is replaced by an eviction test. RED: the eviction test (got
RECOVERED/DEDUPE_OVERFLOW) and a fixed-size fingerprint codec test (got
3326 vs 333 bytes) failed before the change. Added a TTL prune
characterization test (passes before and after). Gates: architecture,
types app/test, lint, adapter-inventory, test:unit, test:component pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-24 13:54:29 +09:00
co-authored by Claude Opus 5.5
parent cc906f08b4
commit bc17ed4bd7
6 changed files with 126 additions and 47 deletions
+25 -6
View File
@@ -31,7 +31,9 @@ export type ValidatedRealtimeEventDto = Readonly<{
wireBytes: number;
/**
* Adapter-private semantic identity used only by the bounded conflict
* detector. It must never be logged or projected into diagnostics.
* detector: a fixed-size hash of the canonical envelope, so dedupe memory
* does not grow with payload size. It must never be logged or projected
* into diagnostics.
*/
semanticFingerprint: string;
fingerprintBytes: number;
@@ -186,11 +188,8 @@ function decodeUnsafe(
eventType,
payloadSnapshot,
);
const semanticFingerprint = canonicalJson(envelope);
const fingerprintBytes = encoder.encode(semanticFingerprint).byteLength;
if (fingerprintBytes > stream.limits.maxEventBytes) {
return realtimeFailure("EVENT_TOO_LARGE", "DECODE");
}
const semanticFingerprint = hash64(canonicalJson(envelope));
const fingerprintBytes = semanticFingerprint.length;
const dto = Object.freeze({
envelope,
@@ -367,6 +366,26 @@ function snapshotJson(
return visit(value, 0);
}
/**
* Two differently seeded 32-bit FNV-1a hashes as 16 hex characters.
* ponytail: not collision-proof against a hostile server; a collision only
* lets a conflicting redelivery pass as a duplicate. Use SHA-256 if the
* server is not trusted to keep event IDs unique.
*/
function hash64(text: string): string {
let first = 0x811c9dc5;
let second = 0x9e3779b9;
for (let index = 0; index < text.length; index += 1) {
const unit = text.charCodeAt(index);
first = Math.imul(first ^ unit, 0x01000193);
second = Math.imul(second ^ unit, 0x01000193);
}
return (
(first >>> 0).toString(16).padStart(8, "0") +
(second >>> 0).toString(16).padStart(8, "0")
);
}
function canonicalJson(value: unknown): string {
if (
value === null ||
+29 -23
View File
@@ -487,20 +487,6 @@ export function createRealtimeStreamCoordinator(
signal,
);
}
if (
state.eventIds.size + 1 >
state.registration.limits.maxDedupeEntries ||
state.dedupeBytes + event.fingerprintBytes >
state.registration.limits.maxDedupeBytes
) {
return recoverForAccept(
state,
"DEDUPE_OVERFLOW",
true,
signal,
);
}
const eventType = dependencies.registry.findStreamEventType(
state.registration.id,
event.envelope.eventType,
@@ -626,6 +612,17 @@ export function createRealtimeStreamCoordinator(
fingerprintBytes: event.fingerprintBytes,
appliedAt: safeNow(state),
});
// The dedupe window is the newest entries: drop the oldest to make room.
for (const [eventId, oldest] of state.eventIds) {
if (
state.eventIds.size < state.registration.limits.maxDedupeEntries &&
state.dedupeBytes + entry.fingerprintBytes <=
state.registration.limits.maxDedupeBytes
) {
break;
}
forgetDedupe(state, eventId, oldest);
}
state.eventIds.set(entry.eventId, entry);
state.sequences.set(entry.sequence, entry);
state.dedupeBytes += entry.fingerprintBytes;
@@ -1113,6 +1110,7 @@ export function createRealtimeStreamCoordinator(
return candidate;
}
/** Entries are kept in apply order and `appliedAt` never decreases. */
function pruneDedupe(state: StreamState): void {
const selectedNow = safeNow(state);
for (const [eventId, entry] of state.eventIds) {
@@ -1120,19 +1118,27 @@ export function createRealtimeStreamCoordinator(
selectedNow - entry.appliedAt <
state.registration.limits.dedupeTtlMs
) {
continue;
break;
}
state.eventIds.delete(eventId);
if (state.sequences.get(entry.sequence) === entry) {
state.sequences.delete(entry.sequence);
}
state.dedupeBytes = Math.max(
0,
state.dedupeBytes - entry.fingerprintBytes,
);
forgetDedupe(state, eventId, entry);
}
}
function forgetDedupe(
state: StreamState,
eventId: string,
entry: DedupeEntry,
): void {
state.eventIds.delete(eventId);
if (state.sequences.get(entry.sequence) === entry) {
state.sequences.delete(entry.sequence);
}
state.dedupeBytes = Math.max(
0,
state.dedupeBytes - entry.fingerprintBytes,
);
}
function fenceState(state: StreamState): void {
state.processingGeneration += 1;
state.activeEffectAbort?.abort();
+2 -2
View File
@@ -46,8 +46,8 @@ export type RealtimeLimits = Readonly<{
maxDedupeEntries: number;
/**
* Additional implementation memory ceiling for semantic conflict
* fingerprints. Reaching it has the same recovery meaning as exhausting the
* count/time dedupe window.
* fingerprints. Reaching it evicts the oldest entries, the same as
* exhausting the count/time dedupe window.
*/
maxDedupeBytes: number;
dedupeTtlMs: number;