From 6f2e97d4826fd0978fe01160773d747854a28c60 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 24 Sep 2026 12:09:26 +0900 Subject: [PATCH] fix(telemetry): deliver deduplicated occurrences to the collector (R-9.2) With the default queueMicrotask scheduler the first event of a burst was sent at once with occurrences=1, and every duplicate inside the 60s dedup window only bumped counters.deduplicated, so 1000 identical errors reached the collector as occurrences=1. An event dropped from the buffer also left its fingerprint in the dedup table, suppressing that event for 60s. Per D-6 the pipeline now holds duplicate counts per fingerprint. Held counts are folded into a buffered item of the same fingerprint or ride as summary items in the next batch; they get a batch of their own only on an explicit flush() or when the dedup window closes (one window timer), so a duplicate never triggers a send by itself. The held table is bounded by maxFingerprints. A rejected or evicted event (new buffer onEvict hook) and a sampled-out event forget their fingerprint (new TelemetryDeduplicator.forget), so the next occurrence is admitted again and a sampled-out event is never reported through a summary. Existing signatures are unchanged; onEvict and forget are additive. Test changes: - tests/features/reference-feature/reference-telemetry.test.ts expected exactly one send for a 500-failure storm, which encoded the lost count; it now expects 2 sends whose occurrences sum to 500. - telemetry-pipeline.test.ts:93 (auto flush off, 25 in one item) is kept as is and still passes. RED before fix: - "delivers every duplicate as occurrences with the default microtask scheduler" (sum was 1, expected 1000) - "reports held duplicates when the dedup window closes without another flush" (no window timer armed) - "does not let a rejected or evicted event suppress its fingerprint" (re-emitted event was deduplicated, not buffered) - "never reports a sampled-out event through a duplicate summary" (sampledOut 1, expected 3) Gates: check:architecture, check:types:app, check:types:test, lint, test:component, check:release-coherence, test:platform pass; also tests/scale/telemetry, tests/scale/session, tests/features/reference-feature pass. test:unit passes except tests/unit/removal-fixture.test.ts, which needs generated release evidence under artifacts/ that a fresh worktree does not have (unrelated to this change). Co-Authored-By: Claude Opus 5.5 --- src/platform/telemetry/telemetry-buffer.ts | 10 ++- .../telemetry/telemetry-deduplicator.ts | 5 ++ src/platform/telemetry/telemetry-pipeline.ts | 86 ++++++++++++++++--- .../reference-telemetry.test.ts | 14 ++- .../telemetry/telemetry-pipeline.test.ts | 82 ++++++++++++++++++ 5 files changed, 181 insertions(+), 16 deletions(-) diff --git a/src/platform/telemetry/telemetry-buffer.ts b/src/platform/telemetry/telemetry-buffer.ts index 5d3b9cda..f42dc97e 100644 --- a/src/platform/telemetry/telemetry-buffer.ts +++ b/src/platform/telemetry/telemetry-buffer.ts @@ -43,9 +43,14 @@ export function estimateTelemetryBytes(event: TelemetryEvent): number { } export function createTelemetryBuffer( - options: Readonly<{ maxEvents: number; maxBytes: number }>, + options: Readonly<{ + maxEvents: number; + maxBytes: number; + /** Called for each buffered event evicted to make room for a more important one. */ + onEvict?: (entry: BufferedTelemetryEvent) => void; + }>, ): TelemetryBuffer { - const { maxEvents, maxBytes } = options; + const { maxEvents, maxBytes, onEvict } = options; if (!Number.isSafeInteger(maxEvents) || maxEvents < 1) { throw new TypeError(`telemetry buffer maxEvents must be positive: ${maxEvents}`); } @@ -101,6 +106,7 @@ export function createTelemetryBuffer( entries.splice(victim, 1); bytes -= victimEntry.bytes; dropped += 1; + onEvict?.(victimEntry); } entries.push( Object.freeze( diff --git a/src/platform/telemetry/telemetry-deduplicator.ts b/src/platform/telemetry/telemetry-deduplicator.ts index a99acafe..f7bb87d2 100644 --- a/src/platform/telemetry/telemetry-deduplicator.ts +++ b/src/platform/telemetry/telemetry-deduplicator.ts @@ -57,6 +57,11 @@ export class TelemetryDeduplicator { return this.#entries.get(fingerprint)?.count ?? 0; } + /** Forgets a fingerprint whose event was never delivered, so it is admitted again. */ + forget(fingerprint: string): void { + this.#entries.delete(fingerprint); + } + #evict(now: number): void { if (this.#entries.size < this.maxEntries) return; for (const [fingerprint, entry] of this.#entries) { diff --git a/src/platform/telemetry/telemetry-pipeline.ts b/src/platform/telemetry/telemetry-pipeline.ts index d2733451..fe3c96cf 100644 --- a/src/platform/telemetry/telemetry-pipeline.ts +++ b/src/platform/telemetry/telemetry-pipeline.ts @@ -25,11 +25,15 @@ import { createDeterministicSampler } from "./telemetry-sampler.ts"; * * Every stage has an explicit ceiling, and no stage can throw or reject into * the business caller: internal failures become dropped-event counters. + * + * A duplicate inside the dedup window is not sent on its own. Its count is held + * per fingerprint and delivered as `occurrences` on the next flush, or on a + * flush when the dedup window closes, whichever comes first (D-6). */ export type TelemetryBatchItem = Readonly<{ event: TelemetryEvent; fingerprint: string; - /** Occurrences collapsed into this fingerprint in the current window. */ + /** Occurrences of this fingerprint not reported by an earlier item. */ occurrences: number; }>; @@ -125,13 +129,22 @@ export function createTelemetryPipeline( const buffer = createTelemetryBuffer({ maxEvents: budget.maxBufferedEvents, maxBytes: budget.maxBufferedBytes, + // An evicted event was never delivered; it must not keep suppressing. + onEvict: (entry) => { + if (entry.fingerprint !== undefined) dedup.forget(entry.fingerprint); + }, }); + /** Duplicates counted but not yet reported, per fingerprint. */ + const held = new Map; count: number }>(); /** Send timestamps inside the current rate window; never above the limit. */ const sendTimes: number[] = []; let disposed = false; let flushScheduled = false; let rateTimer: unknown = null; + let windowTimer: unknown = null; + /** An explicit flush or a closed dedup window asks for held counts. */ + let heldDue = false; let activeFlush: Promise | null = null; const counters = { accepted: 0, @@ -168,20 +181,58 @@ export function createTelemetryPipeline( }); } + function hold(event: TelemetryEvent, fingerprint: string): void { + const entry = held.get(fingerprint); + if (entry) { + entry.count += 1; + return; + } + // Bounded like the dedup table; past it a duplicate is only counted in stats. + if (held.size >= budget.maxFingerprints) return; + held.set(fingerprint, { event, count: 1 }); + if (windowTimer !== null || disposed) return; + windowTimer = setTimer(() => { + windowTimer = null; + void flushHeld(); + }, budget.dedupWindowMs); + } + + /** + * Held counts ride along with buffered events for free. They get a batch of + * their own only when due, so a duplicate never triggers a send by itself. + */ + function pending(): boolean { + return buffer.size > 0 || (heldDue && held.size > 0); + } + + function takeBatch(): TelemetryBatchItem[] { + const batch: TelemetryBatchItem[] = buffer.take(budget.maxBatchEvents).map((entry) => { + const fingerprint = entry.fingerprint ?? ""; + const duplicates = held.get(fingerprint)?.count ?? 0; + held.delete(fingerprint); + return Object.freeze({ event: entry.event, fingerprint, occurrences: 1 + duplicates }); + }); + for (const [fingerprint, entry] of held) { + if (batch.length >= budget.maxBatchEvents) break; + held.delete(fingerprint); + batch.push(Object.freeze({ event: entry.event, fingerprint, occurrences: entry.count })); + } + if (held.size === 0) { + heldDue = false; + if (windowTimer !== null) clearTimer(windowTimer); + windowTimer = null; + } + return batch; + } + async function drain(): Promise { - while (!disposed && buffer.size > 0) { + while (!disposed && pending()) { const at = now(); if (!rateAllows(at)) { armRateTimer(at); return; } - const batch = buffer.take(budget.maxBatchEvents).map((entry) => - Object.freeze({ - event: entry.event, - fingerprint: entry.fingerprint ?? "", - occurrences: Math.max(1, dedup.count(entry.fingerprint ?? "")), - }), - ); + const batch = takeBatch(); sendTimes.push(at); counters.sends += 1; try { @@ -195,7 +246,7 @@ export function createTelemetryPipeline( function flush(): Promise { if (activeFlush) return activeFlush; - if (disposed || buffer.size === 0) return Promise.resolve(); + if (disposed || !pending()) return Promise.resolve(); activeFlush = drain() .catch(() => undefined) .finally(() => { @@ -204,6 +255,11 @@ export function createTelemetryPipeline( return activeFlush; } + function flushHeld(): Promise { + heldDue = held.size > 0; + return flush(); + } + function admit(event: TelemetryEvent): void { if (disposed) return; if (!isTelemetryEnvelopeValid(event)) { @@ -228,6 +284,7 @@ export function createTelemetryPipeline( const fingerprint = fingerprintTelemetryEvent(redacted); if (!dedup.record(fingerprint, now())) { counters.deduplicated += 1; + hold(redacted, fingerprint); return; } // 4. sample (deterministic by interaction; severe events always kept) @@ -236,12 +293,16 @@ export function createTelemetryPipeline( !sampler.keep(redacted.interactionId ?? fingerprint) ) { counters.sampledOut += 1; + // Its duplicates must not be reported as held occurrences. + dedup.forget(fingerprint); return; } // 5. buffer if (buffer.push(redacted, estimateTelemetryBytes(redacted), fingerprint)) { counters.accepted += 1; counters.peakBuffered = Math.max(counters.peakBuffered, buffer.size); + } else { + dedup.forget(fingerprint); } // 6-8. rate limit, batch and transport happen on flush. scheduleFlush(); @@ -256,7 +317,7 @@ export function createTelemetryPipeline( } return Promise.resolve(); }, - flush, + flush: flushHeld, stats(): TelemetryPipelineStats { return Object.freeze({ accepted: counters.accepted, @@ -277,7 +338,10 @@ export function createTelemetryPipeline( disposed = true; if (rateTimer !== null) clearTimer(rateTimer); rateTimer = null; + if (windowTimer !== null) clearTimer(windowTimer); + windowTimer = null; buffer.clear(); + held.clear(); }, }); } diff --git a/tests/features/reference-feature/reference-telemetry.test.ts b/tests/features/reference-feature/reference-telemetry.test.ts index d238297d..7415c894 100644 --- a/tests/features/reference-feature/reference-telemetry.test.ts +++ b/tests/features/reference-feature/reference-telemetry.test.ts @@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest"; import { createReferenceFeatureInput } from "../../../src/features/reference-feature/application/reference-feature-api.ts"; import { createFailure } from "../../../src/contracts/errors.ts"; import type { TelemetryPort } from "../../../src/platform/telemetry/telemetry-port.ts"; -import { createTelemetryPipeline } from "../../../src/platform/telemetry/telemetry-pipeline.ts"; +import { + createTelemetryPipeline, + type TelemetryTransport, +} from "../../../src/platform/telemetry/telemetry-pipeline.ts"; import { REFERENCE_FEATURE_ADAPTER_CONTRIBUTION } from "../../../src/features/reference-feature/adapters/create-reference-feature-input.ts"; function failingGateway() { @@ -55,14 +58,19 @@ describe("reference feature telemetry through the platform port", () => { }); it("collapses a failure storm into bounded sends via the pipeline", async () => { - const send = vi.fn(); + const send = vi.fn(); const pipeline = createTelemetryPipeline({ transport: { send } }); const input = createReferenceFeatureInput(failingGateway(), { telemetry: pipeline }); for (let i = 0; i < 500; i += 1) await input.listResources({ limit: 20 }); await pipeline.flush(); - expect(send).toHaveBeenCalledTimes(1); + // The first failure goes out at once; the 499 duplicates ride on the flush. + expect(send).toHaveBeenCalledTimes(2); + const occurrences = send.mock.calls + .flatMap(([batch]) => batch) + .reduce((sum, item) => sum + item.occurrences, 0); + expect(occurrences).toBe(500); expect(pipeline.stats().deduplicated).toBe(499); pipeline.dispose(); }); diff --git a/tests/platform/telemetry/telemetry-pipeline.test.ts b/tests/platform/telemetry/telemetry-pipeline.test.ts index 7035071b..3c6551c8 100644 --- a/tests/platform/telemetry/telemetry-pipeline.test.ts +++ b/tests/platform/telemetry/telemetry-pipeline.test.ts @@ -101,6 +101,88 @@ describe("createTelemetryPipeline", () => { expect(batches[0]?.[0]?.occurrences).toBe(25); }); + it("delivers every duplicate as occurrences with the default microtask scheduler", async () => { + const batches: TelemetryBatchItem[][] = []; + const pipeline = createTelemetryPipeline({ + transport: { send: (batch) => void batches.push([...batch]) }, + budget: TEST_TELEMETRY_BUDGET, + }); + for (let i = 0; i < 1_000; i += 1) { + await pipeline.emit(REPEATED_ERROR); + } + await pipeline.flush(); + const items = batches.flat(); + expect(items.reduce((sum, item) => sum + item.occurrences, 0)).toBe(1_000); + expect(new Set(items.map((item) => item.fingerprint)).size).toBe(1); + expect(batches.length).toBeLessThanOrEqual(2); + pipeline.dispose(); + }); + + it("reports held duplicates when the dedup window closes without another flush", async () => { + const batches: TelemetryBatchItem[][] = []; + const timers: Array void; ms: number }>> = []; + const pipeline = createTelemetryPipeline({ + transport: { send: (batch) => void batches.push([...batch]) }, + budget: TEST_TELEMETRY_BUDGET, + setTimer: (callback, ms) => timers.push({ callback, ms }), + clearTimer: () => {}, + }); + for (let i = 0; i < 10; i += 1) { + await pipeline.emit(REPEATED_ERROR); + } + expect(timers.map((timer) => timer.ms)).toEqual([TEST_TELEMETRY_BUDGET.dedupWindowMs]); + for (const timer of timers.splice(0)) timer.callback(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(batches.flat().reduce((sum, item) => sum + item.occurrences, 0)).toBe(10); + pipeline.dispose(); + }); + + it("does not let a rejected or evicted event suppress its fingerprint", async () => { + const batches: TelemetryBatchItem[][] = []; + const pipeline = createTelemetryPipeline({ + transport: { send: (batch) => void batches.push([...batch]) }, + budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 1 }, + schedule: () => {}, + }); + const info: TelemetryEvent = { ...TEST_EVENT, kind: "catalog.load.slow", severity: "info" }; + // Rejected on arrival: the buffer holds a more important event. + await pipeline.emit(TEST_EVENT); + await pipeline.emit(info); + await pipeline.flush(); + await pipeline.emit(info); + expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 }); + await pipeline.flush(); + // Evicted after admission: a more important event takes its slot. + await pipeline.emit({ ...info, payload: { reason: "evicted" } }); + await pipeline.emit({ ...TEST_EVENT, payload: { reason: "other" } }); + await pipeline.flush(); + await pipeline.emit({ ...info, payload: { reason: "evicted" } }); + expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 }); + await pipeline.flush(); + const kinds = batches.flat().map((item) => `${item.event.kind}:${item.occurrences}`); + expect(kinds).toEqual([ + "catalog.load.failed:1", + "catalog.load.slow:1", + "catalog.load.failed:1", + "catalog.load.slow:1", + ]); + }); + + it("never reports a sampled-out event through a duplicate summary", async () => { + const send = vi.fn(); + const pipeline = createTelemetryPipeline({ + transport: { send }, + budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 }, + }); + for (let i = 0; i < 3; i += 1) { + await pipeline.emit({ kind: "a.info", severity: "info" }); + } + await pipeline.flush(); + expect(send).not.toHaveBeenCalled(); + expect(pipeline.stats().sampledOut).toBe(3); + pipeline.dispose(); + }); + it("rate-limits distinct events and resumes after the window reopens", async () => { let now = 0; const scheduling = manualScheduling();