diff --git a/docs/architecture/realtime-events-web-push-and-bounded-polling.md b/docs/architecture/realtime-events-web-push-and-bounded-polling.md index 1628f6f2..3f65fbc5 100644 --- a/docs/architecture/realtime-events-web-push-and-bounded-polling.md +++ b/docs/architecture/realtime-events-web-push-and-bounded-polling.md @@ -618,7 +618,8 @@ stream + epoch별 `lastAppliedSequence`를 기준으로 처리한다. | `sequence == lastApplied + 1` | 순차 적용 | | `sequence > lastApplied + 1` | gap; delta 적용 중지, `RESYNC_REQUIRED` | | unknown `streamEpoch` | old/new epoch를 섞지 않고 registered snapshot 또는 session rebuild | -| dedupe/queue ceiling 초과 | 임의 eviction 후 계속 적용하지 않고 registered recovery | +| dedupe window 가득 참 | 가장 오래된 항목부터 제거(FIFO)하고 계속 적용. 창에서 밀려난 재전송은 `sequence <= lastApplied`로 drop | +| queue ceiling 초과 | 임의 eviction 후 계속 적용하지 않고 registered recovery | | captured old runtime generation | late callback safe drop + aggregate diagnostic | | current connection의 `scopeBinding` mismatch | effect/sequence/cursor 0, freshness `UNKNOWN`, close + session revalidation + registered recovery | diff --git a/src/adapters/realtime/event-codec.ts b/src/adapters/realtime/event-codec.ts index 90738014..e5d4d506 100644 --- a/src/adapters/realtime/event-codec.ts +++ b/src/adapters/realtime/event-codec.ts @@ -31,7 +31,9 @@ export type ValidatedRealtimeEventDto = Readonly<{ wireBytes: number; /** * Adapter-private semantic identity used only by the bounded conflict - * detector. It must never be logged or projected into diagnostics. + * detector: a fixed-size hash of the canonical envelope, so dedupe memory + * does not grow with payload size. It must never be logged or projected + * into diagnostics. */ semanticFingerprint: string; fingerprintBytes: number; @@ -186,11 +188,8 @@ function decodeUnsafe( eventType, payloadSnapshot, ); - const semanticFingerprint = canonicalJson(envelope); - const fingerprintBytes = encoder.encode(semanticFingerprint).byteLength; - if (fingerprintBytes > stream.limits.maxEventBytes) { - return realtimeFailure("EVENT_TOO_LARGE", "DECODE"); - } + const semanticFingerprint = hash64(canonicalJson(envelope)); + const fingerprintBytes = semanticFingerprint.length; const dto = Object.freeze({ envelope, @@ -367,6 +366,26 @@ function snapshotJson( return visit(value, 0); } +/** + * Two differently seeded 32-bit FNV-1a hashes as 16 hex characters. + * ponytail: not collision-proof against a hostile server; a collision only + * lets a conflicting redelivery pass as a duplicate. Use SHA-256 if the + * server is not trusted to keep event IDs unique. + */ +function hash64(text: string): string { + let first = 0x811c9dc5; + let second = 0x9e3779b9; + for (let index = 0; index < text.length; index += 1) { + const unit = text.charCodeAt(index); + first = Math.imul(first ^ unit, 0x01000193); + second = Math.imul(second ^ unit, 0x01000193); + } + return ( + (first >>> 0).toString(16).padStart(8, "0") + + (second >>> 0).toString(16).padStart(8, "0") + ); +} + function canonicalJson(value: unknown): string { if ( value === null || diff --git a/src/adapters/realtime/stream-coordinator.ts b/src/adapters/realtime/stream-coordinator.ts index c90268e7..654a02c7 100644 --- a/src/adapters/realtime/stream-coordinator.ts +++ b/src/adapters/realtime/stream-coordinator.ts @@ -487,20 +487,6 @@ export function createRealtimeStreamCoordinator( signal, ); } - if ( - state.eventIds.size + 1 > - state.registration.limits.maxDedupeEntries || - state.dedupeBytes + event.fingerprintBytes > - state.registration.limits.maxDedupeBytes - ) { - return recoverForAccept( - state, - "DEDUPE_OVERFLOW", - true, - signal, - ); - } - const eventType = dependencies.registry.findStreamEventType( state.registration.id, event.envelope.eventType, @@ -626,6 +612,17 @@ export function createRealtimeStreamCoordinator( fingerprintBytes: event.fingerprintBytes, appliedAt: safeNow(state), }); + // The dedupe window is the newest entries: drop the oldest to make room. + for (const [eventId, oldest] of state.eventIds) { + if ( + state.eventIds.size < state.registration.limits.maxDedupeEntries && + state.dedupeBytes + entry.fingerprintBytes <= + state.registration.limits.maxDedupeBytes + ) { + break; + } + forgetDedupe(state, eventId, oldest); + } state.eventIds.set(entry.eventId, entry); state.sequences.set(entry.sequence, entry); state.dedupeBytes += entry.fingerprintBytes; @@ -1113,6 +1110,7 @@ export function createRealtimeStreamCoordinator( return candidate; } + /** Entries are kept in apply order and `appliedAt` never decreases. */ function pruneDedupe(state: StreamState): void { const selectedNow = safeNow(state); for (const [eventId, entry] of state.eventIds) { @@ -1120,19 +1118,27 @@ export function createRealtimeStreamCoordinator( selectedNow - entry.appliedAt < state.registration.limits.dedupeTtlMs ) { - continue; + break; } - state.eventIds.delete(eventId); - if (state.sequences.get(entry.sequence) === entry) { - state.sequences.delete(entry.sequence); - } - state.dedupeBytes = Math.max( - 0, - state.dedupeBytes - entry.fingerprintBytes, - ); + forgetDedupe(state, eventId, entry); } } + function forgetDedupe( + state: StreamState, + eventId: string, + entry: DedupeEntry, + ): void { + state.eventIds.delete(eventId); + if (state.sequences.get(entry.sequence) === entry) { + state.sequences.delete(entry.sequence); + } + state.dedupeBytes = Math.max( + 0, + state.dedupeBytes - entry.fingerprintBytes, + ); + } + function fenceState(state: StreamState): void { state.processingGeneration += 1; state.activeEffectAbort?.abort(); diff --git a/src/contracts/realtime-streams.ts b/src/contracts/realtime-streams.ts index f1510ef6..42675281 100644 --- a/src/contracts/realtime-streams.ts +++ b/src/contracts/realtime-streams.ts @@ -46,8 +46,8 @@ export type RealtimeLimits = Readonly<{ maxDedupeEntries: number; /** * Additional implementation memory ceiling for semantic conflict - * fingerprints. Reaching it has the same recovery meaning as exhausting the - * count/time dedupe window. + * fingerprints. Reaching it evicts the oldest entries, the same as + * exhausting the count/time dedupe window. */ maxDedupeBytes: number; dedupeTtlMs: number; diff --git a/tests/unit/realtime/event-codec.test.ts b/tests/unit/realtime/event-codec.test.ts index a344cbe9..d1c7fdd1 100644 --- a/tests/unit/realtime/event-codec.test.ts +++ b/tests/unit/realtime/event-codec.test.ts @@ -227,6 +227,24 @@ describe("REALTIME_EVENT_V1 codec", () => { ); }); + it("keeps a fixed-size semantic fingerprint however large the payload", () => { + const codec = createTestRealtimeCodec(); + const small = codec.decode(realtimeEventJson()); + const large = codec.decode( + realtimeEventJson({ payload: { value: "x".repeat(3_000) } }), + ); + + expect(small.ok && large.ok).toBe(true); + if (!small.ok || !large.ok) return; + expect(large.value.fingerprintBytes).toBe( + small.value.fingerprintBytes, + ); + expect(large.value.fingerprintBytes).toBeLessThanOrEqual(32); + expect(large.value.semanticFingerprint).not.toBe( + small.value.semanticFingerprint, + ); + }); + it("uses bounded uint64 sequence and header-safe cursor helpers", () => { expect(isCanonicalRealtimeSequence("0")).toBe(true); expect(isCanonicalRealtimeSequence(REALTIME_MAX_SEQUENCE)).toBe(true); diff --git a/tests/unit/realtime/stream-coordinator.test.ts b/tests/unit/realtime/stream-coordinator.test.ts index f3d154e8..671a8f96 100644 --- a/tests/unit/realtime/stream-coordinator.test.ts +++ b/tests/unit/realtime/stream-coordinator.test.ts @@ -707,7 +707,7 @@ describe("transport-independent realtime stream coordinator", () => { expect(harness.coordinator.inspect(STREAM_ID).freshness).toBe("STALE"); }); - it("recovers instead of evicting and continuing after dedupe capacity", async () => { + it("evicts the oldest dedupe entry and keeps applying at capacity", async () => { const harness = createHarness({ registry: createTestRealtimeRegistry({ limits: { @@ -718,24 +718,58 @@ describe("transport-independent realtime stream coordinator", () => { }); await harness.initialize(); await harness.coordinator.accept(harness.event()); + const second = { + eventId: "event-00000002", + sequence: "2", + resumeCursor: "cursor-00000002", + }; await expect( - harness.coordinator.accept( - harness.event({ - eventId: "event-00000002", - sequence: "2", - resumeCursor: "cursor-00000002", - }), - ), + harness.coordinator.accept(harness.event(second)), ).resolves.toMatchObject({ ok: true, - value: { - outcome: "RECOVERED", - reason: "DEDUPE_OVERFLOW", - }, + value: { outcome: "APPLIED" }, }); - expect(harness.effects).toHaveBeenCalledTimes(1); - expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(0); + expect(harness.effects).toHaveBeenCalledTimes(2); + expect(harness.recoveryReasons).toEqual(["INITIALIZE"]); + expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(1); + // The newest entry still catches its duplicate; the evicted one is stale. + await expect( + harness.coordinator.accept(harness.event(second)), + ).resolves.toMatchObject({ + ok: true, + value: { outcome: "DROPPED", reason: "DUPLICATE_EVENT" }, + }); + await expect( + harness.coordinator.accept(harness.event()), + ).resolves.toMatchObject({ + ok: true, + value: { outcome: "DROPPED", reason: "STALE_EVENT" }, + }); + }); + + it("expires dedupe entries by TTL oldest first", async () => { + let now = 10_000; + const harness = createHarness({ now: () => now }); + await harness.initialize(); + await harness.coordinator.accept(harness.event()); + now += TEST_LIMITS.dedupeTtlMs / 2; + await harness.coordinator.accept( + harness.event({ + eventId: "event-00000002", + sequence: "2", + resumeCursor: "cursor-00000002", + }), + ); + now += TEST_LIMITS.dedupeTtlMs / 2; + await harness.coordinator.accept( + harness.event({ + eventId: "event-00000003", + sequence: "3", + resumeCursor: "cursor-00000003", + }), + ); + expect(harness.coordinator.inspect(STREAM_ID).dedupeEntries).toBe(2); }); it("rejects a regressing same-epoch recovery checkpoint atomically", async () => { @@ -1122,6 +1156,7 @@ type HarnessOptions = Readonly<{ }>; scheduleTimeout?: (callback: () => void, delayMs: number) => unknown; clearScheduledTimeout?: (handle: unknown) => void; + now?: () => number; }>; function createHarness(options: HarnessOptions = {}) { @@ -1165,7 +1200,7 @@ function createHarness(options: HarnessOptions = {}) { scopeBinding: "scope-binding-0001", isCurrent: () => current, }, - now: () => 10_000, + now: options.now ?? (() => 10_000), observe: options.observe, ...(options.taskLimits ? { taskLimits: options.taskLimits } : {}), ...(options.scheduleTimeout