fix(telemetry): deliver deduplicated occurrences to the collector (R-9.2)

With the default queueMicrotask scheduler the first event of a burst was
sent at once with occurrences=1, and every duplicate inside the 60s dedup
window only bumped counters.deduplicated, so 1000 identical errors reached
the collector as occurrences=1. An event dropped from the buffer also left
its fingerprint in the dedup table, suppressing that event for 60s.

Per D-6 the pipeline now holds duplicate counts per fingerprint. Held
counts are folded into a buffered item of the same fingerprint or ride as
summary items in the next batch; they get a batch of their own only on an
explicit flush() or when the dedup window closes (one window timer), so a
duplicate never triggers a send by itself. The held table is bounded by
maxFingerprints. A rejected or evicted event (new buffer onEvict hook) and
a sampled-out event forget their fingerprint (new
TelemetryDeduplicator.forget), so the next occurrence is admitted again
and a sampled-out event is never reported through a summary. Existing
signatures are unchanged; onEvict and forget are additive.

Test changes:
- tests/features/reference-feature/reference-telemetry.test.ts expected
  exactly one send for a 500-failure storm, which encoded the lost count;
  it now expects 2 sends whose occurrences sum to 500.
- telemetry-pipeline.test.ts:93 (auto flush off, 25 in one item) is kept
  as is and still passes.

RED before fix:
- "delivers every duplicate as occurrences with the default microtask
  scheduler" (sum was 1, expected 1000)
- "reports held duplicates when the dedup window closes without another
  flush" (no window timer armed)
- "does not let a rejected or evicted event suppress its fingerprint"
  (re-emitted event was deduplicated, not buffered)
- "never reports a sampled-out event through a duplicate summary"
  (sampledOut 1, expected 3)

Gates: check:architecture, check:types:app, check:types:test, lint,
test:component, check:release-coherence, test:platform pass; also
tests/scale/telemetry, tests/scale/session, tests/features/reference-feature
pass. test:unit passes except tests/unit/removal-fixture.test.ts, which
needs generated release evidence under artifacts/ that a fresh worktree
does not have (unrelated to this change).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-24 12:18:23 +09:00
co-authored by Claude Opus 5.5
parent 0983742b6b
commit 6f2e97d482
5 changed files with 181 additions and 16 deletions
+8 -2
View File
@@ -43,9 +43,14 @@ export function estimateTelemetryBytes(event: TelemetryEvent<unknown>): number {
}
export function createTelemetryBuffer(
options: Readonly<{ maxEvents: number; maxBytes: number }>,
options: Readonly<{
maxEvents: number;
maxBytes: number;
/** Called for each buffered event evicted to make room for a more important one. */
onEvict?: (entry: BufferedTelemetryEvent) => void;
}>,
): TelemetryBuffer {
const { maxEvents, maxBytes } = options;
const { maxEvents, maxBytes, onEvict } = options;
if (!Number.isSafeInteger(maxEvents) || maxEvents < 1) {
throw new TypeError(`telemetry buffer maxEvents must be positive: ${maxEvents}`);
}
@@ -101,6 +106,7 @@ export function createTelemetryBuffer(
entries.splice(victim, 1);
bytes -= victimEntry.bytes;
dropped += 1;
onEvict?.(victimEntry);
}
entries.push(
Object.freeze(
@@ -57,6 +57,11 @@ export class TelemetryDeduplicator {
return this.#entries.get(fingerprint)?.count ?? 0;
}
/** Forgets a fingerprint whose event was never delivered, so it is admitted again. */
forget(fingerprint: string): void {
this.#entries.delete(fingerprint);
}
#evict(now: number): void {
if (this.#entries.size < this.maxEntries) return;
for (const [fingerprint, entry] of this.#entries) {
+75 -11
View File
@@ -25,11 +25,15 @@ import { createDeterministicSampler } from "./telemetry-sampler.ts";
*
* Every stage has an explicit ceiling, and no stage can throw or reject into
* the business caller: internal failures become dropped-event counters.
*
* A duplicate inside the dedup window is not sent on its own. Its count is held
* per fingerprint and delivered as `occurrences` on the next flush, or on a
* flush when the dedup window closes, whichever comes first (D-6).
*/
export type TelemetryBatchItem = Readonly<{
event: TelemetryEvent<unknown>;
fingerprint: string;
/** Occurrences collapsed into this fingerprint in the current window. */
/** Occurrences of this fingerprint not reported by an earlier item. */
occurrences: number;
}>;
@@ -125,13 +129,22 @@ export function createTelemetryPipeline(
const buffer = createTelemetryBuffer({
maxEvents: budget.maxBufferedEvents,
maxBytes: budget.maxBufferedBytes,
// An evicted event was never delivered; it must not keep suppressing.
onEvict: (entry) => {
if (entry.fingerprint !== undefined) dedup.forget(entry.fingerprint);
},
});
/** Duplicates counted but not yet reported, per fingerprint. */
const held = new Map<string, { event: TelemetryEvent<unknown>; count: number }>();
/** Send timestamps inside the current rate window; never above the limit. */
const sendTimes: number[] = [];
let disposed = false;
let flushScheduled = false;
let rateTimer: unknown = null;
let windowTimer: unknown = null;
/** An explicit flush or a closed dedup window asks for held counts. */
let heldDue = false;
let activeFlush: Promise<void> | null = null;
const counters = {
accepted: 0,
@@ -168,20 +181,58 @@ export function createTelemetryPipeline(
});
}
function hold(event: TelemetryEvent<unknown>, fingerprint: string): void {
const entry = held.get(fingerprint);
if (entry) {
entry.count += 1;
return;
}
// Bounded like the dedup table; past it a duplicate is only counted in stats.
if (held.size >= budget.maxFingerprints) return;
held.set(fingerprint, { event, count: 1 });
if (windowTimer !== null || disposed) return;
windowTimer = setTimer(() => {
windowTimer = null;
void flushHeld();
}, budget.dedupWindowMs);
}
/**
* Held counts ride along with buffered events for free. They get a batch of
* their own only when due, so a duplicate never triggers a send by itself.
*/
function pending(): boolean {
return buffer.size > 0 || (heldDue && held.size > 0);
}
function takeBatch(): TelemetryBatchItem[] {
const batch: TelemetryBatchItem[] = buffer.take(budget.maxBatchEvents).map((entry) => {
const fingerprint = entry.fingerprint ?? "";
const duplicates = held.get(fingerprint)?.count ?? 0;
held.delete(fingerprint);
return Object.freeze({ event: entry.event, fingerprint, occurrences: 1 + duplicates });
});
for (const [fingerprint, entry] of held) {
if (batch.length >= budget.maxBatchEvents) break;
held.delete(fingerprint);
batch.push(Object.freeze({ event: entry.event, fingerprint, occurrences: entry.count }));
}
if (held.size === 0) {
heldDue = false;
if (windowTimer !== null) clearTimer(windowTimer);
windowTimer = null;
}
return batch;
}
async function drain(): Promise<void> {
while (!disposed && buffer.size > 0) {
while (!disposed && pending()) {
const at = now();
if (!rateAllows(at)) {
armRateTimer(at);
return;
}
const batch = buffer.take(budget.maxBatchEvents).map((entry) =>
Object.freeze({
event: entry.event,
fingerprint: entry.fingerprint ?? "",
occurrences: Math.max(1, dedup.count(entry.fingerprint ?? "")),
}),
);
const batch = takeBatch();
sendTimes.push(at);
counters.sends += 1;
try {
@@ -195,7 +246,7 @@ export function createTelemetryPipeline(
function flush(): Promise<void> {
if (activeFlush) return activeFlush;
if (disposed || buffer.size === 0) return Promise.resolve();
if (disposed || !pending()) return Promise.resolve();
activeFlush = drain()
.catch(() => undefined)
.finally(() => {
@@ -204,6 +255,11 @@ export function createTelemetryPipeline(
return activeFlush;
}
function flushHeld(): Promise<void> {
heldDue = held.size > 0;
return flush();
}
function admit(event: TelemetryEvent<unknown>): void {
if (disposed) return;
if (!isTelemetryEnvelopeValid(event)) {
@@ -228,6 +284,7 @@ export function createTelemetryPipeline(
const fingerprint = fingerprintTelemetryEvent(redacted);
if (!dedup.record(fingerprint, now())) {
counters.deduplicated += 1;
hold(redacted, fingerprint);
return;
}
// 4. sample (deterministic by interaction; severe events always kept)
@@ -236,12 +293,16 @@ export function createTelemetryPipeline(
!sampler.keep(redacted.interactionId ?? fingerprint)
) {
counters.sampledOut += 1;
// Its duplicates must not be reported as held occurrences.
dedup.forget(fingerprint);
return;
}
// 5. buffer
if (buffer.push(redacted, estimateTelemetryBytes(redacted), fingerprint)) {
counters.accepted += 1;
counters.peakBuffered = Math.max(counters.peakBuffered, buffer.size);
} else {
dedup.forget(fingerprint);
}
// 6-8. rate limit, batch and transport happen on flush.
scheduleFlush();
@@ -256,7 +317,7 @@ export function createTelemetryPipeline(
}
return Promise.resolve();
},
flush,
flush: flushHeld,
stats(): TelemetryPipelineStats {
return Object.freeze({
accepted: counters.accepted,
@@ -277,7 +338,10 @@ export function createTelemetryPipeline(
disposed = true;
if (rateTimer !== null) clearTimer(rateTimer);
rateTimer = null;
if (windowTimer !== null) clearTimer(windowTimer);
windowTimer = null;
buffer.clear();
held.clear();
},
});
}
@@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest";
import { createReferenceFeatureInput } from "../../../src/features/reference-feature/application/reference-feature-api.ts";
import { createFailure } from "../../../src/contracts/errors.ts";
import type { TelemetryPort } from "../../../src/platform/telemetry/telemetry-port.ts";
import { createTelemetryPipeline } from "../../../src/platform/telemetry/telemetry-pipeline.ts";
import {
createTelemetryPipeline,
type TelemetryTransport,
} from "../../../src/platform/telemetry/telemetry-pipeline.ts";
import { REFERENCE_FEATURE_ADAPTER_CONTRIBUTION } from "../../../src/features/reference-feature/adapters/create-reference-feature-input.ts";
function failingGateway() {
@@ -55,14 +58,19 @@ describe("reference feature telemetry through the platform port", () => {
});
it("collapses a failure storm into bounded sends via the pipeline", async () => {
const send = vi.fn();
const send = vi.fn<TelemetryTransport["send"]>();
const pipeline = createTelemetryPipeline({ transport: { send } });
const input = createReferenceFeatureInput(failingGateway(), { telemetry: pipeline });
for (let i = 0; i < 500; i += 1) await input.listResources({ limit: 20 });
await pipeline.flush();
expect(send).toHaveBeenCalledTimes(1);
// The first failure goes out at once; the 499 duplicates ride on the flush.
expect(send).toHaveBeenCalledTimes(2);
const occurrences = send.mock.calls
.flatMap(([batch]) => batch)
.reduce((sum, item) => sum + item.occurrences, 0);
expect(occurrences).toBe(500);
expect(pipeline.stats().deduplicated).toBe(499);
pipeline.dispose();
});
@@ -101,6 +101,88 @@ describe("createTelemetryPipeline", () => {
expect(batches[0]?.[0]?.occurrences).toBe(25);
});
it("delivers every duplicate as occurrences with the default microtask scheduler", async () => {
const batches: TelemetryBatchItem[][] = [];
const pipeline = createTelemetryPipeline({
transport: { send: (batch) => void batches.push([...batch]) },
budget: TEST_TELEMETRY_BUDGET,
});
for (let i = 0; i < 1_000; i += 1) {
await pipeline.emit(REPEATED_ERROR);
}
await pipeline.flush();
const items = batches.flat();
expect(items.reduce((sum, item) => sum + item.occurrences, 0)).toBe(1_000);
expect(new Set(items.map((item) => item.fingerprint)).size).toBe(1);
expect(batches.length).toBeLessThanOrEqual(2);
pipeline.dispose();
});
it("reports held duplicates when the dedup window closes without another flush", async () => {
const batches: TelemetryBatchItem[][] = [];
const timers: Array<Readonly<{ callback: () => void; ms: number }>> = [];
const pipeline = createTelemetryPipeline({
transport: { send: (batch) => void batches.push([...batch]) },
budget: TEST_TELEMETRY_BUDGET,
setTimer: (callback, ms) => timers.push({ callback, ms }),
clearTimer: () => {},
});
for (let i = 0; i < 10; i += 1) {
await pipeline.emit(REPEATED_ERROR);
}
expect(timers.map((timer) => timer.ms)).toEqual([TEST_TELEMETRY_BUDGET.dedupWindowMs]);
for (const timer of timers.splice(0)) timer.callback();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(batches.flat().reduce((sum, item) => sum + item.occurrences, 0)).toBe(10);
pipeline.dispose();
});
it("does not let a rejected or evicted event suppress its fingerprint", async () => {
const batches: TelemetryBatchItem[][] = [];
const pipeline = createTelemetryPipeline({
transport: { send: (batch) => void batches.push([...batch]) },
budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 1 },
schedule: () => {},
});
const info: TelemetryEvent<unknown> = { ...TEST_EVENT, kind: "catalog.load.slow", severity: "info" };
// Rejected on arrival: the buffer holds a more important event.
await pipeline.emit(TEST_EVENT);
await pipeline.emit(info);
await pipeline.flush();
await pipeline.emit(info);
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
await pipeline.flush();
// Evicted after admission: a more important event takes its slot.
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
await pipeline.emit({ ...TEST_EVENT, payload: { reason: "other" } });
await pipeline.flush();
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
await pipeline.flush();
const kinds = batches.flat().map((item) => `${item.event.kind}:${item.occurrences}`);
expect(kinds).toEqual([
"catalog.load.failed:1",
"catalog.load.slow:1",
"catalog.load.failed:1",
"catalog.load.slow:1",
]);
});
it("never reports a sampled-out event through a duplicate summary", async () => {
const send = vi.fn();
const pipeline = createTelemetryPipeline({
transport: { send },
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 },
});
for (let i = 0; i < 3; i += 1) {
await pipeline.emit({ kind: "a.info", severity: "info" });
}
await pipeline.flush();
expect(send).not.toHaveBeenCalled();
expect(pipeline.stats().sampledOut).toBe(3);
pipeline.dispose();
});
it("rate-limits distinct events and resumes after the window reopens", async () => {
let now = 0;
const scheduling = manualScheduling();