fix(telemetry): deliver deduplicated occurrences to the collector (R-9.2)
With the default queueMicrotask scheduler the first event of a burst was sent at once with occurrences=1, and every duplicate inside the 60s dedup window only bumped counters.deduplicated, so 1000 identical errors reached the collector as occurrences=1. An event dropped from the buffer also left its fingerprint in the dedup table, suppressing that event for 60s. Per D-6 the pipeline now holds duplicate counts per fingerprint. Held counts are folded into a buffered item of the same fingerprint or ride as summary items in the next batch; they get a batch of their own only on an explicit flush() or when the dedup window closes (one window timer), so a duplicate never triggers a send by itself. The held table is bounded by maxFingerprints. A rejected or evicted event (new buffer onEvict hook) and a sampled-out event forget their fingerprint (new TelemetryDeduplicator.forget), so the next occurrence is admitted again and a sampled-out event is never reported through a summary. Existing signatures are unchanged; onEvict and forget are additive. Test changes: - tests/features/reference-feature/reference-telemetry.test.ts expected exactly one send for a 500-failure storm, which encoded the lost count; it now expects 2 sends whose occurrences sum to 500. - telemetry-pipeline.test.ts:93 (auto flush off, 25 in one item) is kept as is and still passes. RED before fix: - "delivers every duplicate as occurrences with the default microtask scheduler" (sum was 1, expected 1000) - "reports held duplicates when the dedup window closes without another flush" (no window timer armed) - "does not let a rejected or evicted event suppress its fingerprint" (re-emitted event was deduplicated, not buffered) - "never reports a sampled-out event through a duplicate summary" (sampledOut 1, expected 3) Gates: check:architecture, check:types:app, check:types:test, lint, test:component, check:release-coherence, test:platform pass; also tests/scale/telemetry, tests/scale/session, tests/features/reference-feature pass. test:unit passes except tests/unit/removal-fixture.test.ts, which needs generated release evidence under artifacts/ that a fresh worktree does not have (unrelated to this change). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0983742b6b
commit
6f2e97d482
@@ -43,9 +43,14 @@ export function estimateTelemetryBytes(event: TelemetryEvent<unknown>): number {
|
||||
}
|
||||
|
||||
export function createTelemetryBuffer(
|
||||
options: Readonly<{ maxEvents: number; maxBytes: number }>,
|
||||
options: Readonly<{
|
||||
maxEvents: number;
|
||||
maxBytes: number;
|
||||
/** Called for each buffered event evicted to make room for a more important one. */
|
||||
onEvict?: (entry: BufferedTelemetryEvent) => void;
|
||||
}>,
|
||||
): TelemetryBuffer {
|
||||
const { maxEvents, maxBytes } = options;
|
||||
const { maxEvents, maxBytes, onEvict } = options;
|
||||
if (!Number.isSafeInteger(maxEvents) || maxEvents < 1) {
|
||||
throw new TypeError(`telemetry buffer maxEvents must be positive: ${maxEvents}`);
|
||||
}
|
||||
@@ -101,6 +106,7 @@ export function createTelemetryBuffer(
|
||||
entries.splice(victim, 1);
|
||||
bytes -= victimEntry.bytes;
|
||||
dropped += 1;
|
||||
onEvict?.(victimEntry);
|
||||
}
|
||||
entries.push(
|
||||
Object.freeze(
|
||||
|
||||
@@ -57,6 +57,11 @@ export class TelemetryDeduplicator {
|
||||
return this.#entries.get(fingerprint)?.count ?? 0;
|
||||
}
|
||||
|
||||
/** Forgets a fingerprint whose event was never delivered, so it is admitted again. */
|
||||
forget(fingerprint: string): void {
|
||||
this.#entries.delete(fingerprint);
|
||||
}
|
||||
|
||||
#evict(now: number): void {
|
||||
if (this.#entries.size < this.maxEntries) return;
|
||||
for (const [fingerprint, entry] of this.#entries) {
|
||||
|
||||
@@ -25,11 +25,15 @@ import { createDeterministicSampler } from "./telemetry-sampler.ts";
|
||||
*
|
||||
* Every stage has an explicit ceiling, and no stage can throw or reject into
|
||||
* the business caller: internal failures become dropped-event counters.
|
||||
*
|
||||
* A duplicate inside the dedup window is not sent on its own. Its count is held
|
||||
* per fingerprint and delivered as `occurrences` on the next flush, or on a
|
||||
* flush when the dedup window closes, whichever comes first (D-6).
|
||||
*/
|
||||
export type TelemetryBatchItem = Readonly<{
|
||||
event: TelemetryEvent<unknown>;
|
||||
fingerprint: string;
|
||||
/** Occurrences collapsed into this fingerprint in the current window. */
|
||||
/** Occurrences of this fingerprint not reported by an earlier item. */
|
||||
occurrences: number;
|
||||
}>;
|
||||
|
||||
@@ -125,13 +129,22 @@ export function createTelemetryPipeline(
|
||||
const buffer = createTelemetryBuffer({
|
||||
maxEvents: budget.maxBufferedEvents,
|
||||
maxBytes: budget.maxBufferedBytes,
|
||||
// An evicted event was never delivered; it must not keep suppressing.
|
||||
onEvict: (entry) => {
|
||||
if (entry.fingerprint !== undefined) dedup.forget(entry.fingerprint);
|
||||
},
|
||||
});
|
||||
/** Duplicates counted but not yet reported, per fingerprint. */
|
||||
const held = new Map<string, { event: TelemetryEvent<unknown>; count: number }>();
|
||||
/** Send timestamps inside the current rate window; never above the limit. */
|
||||
const sendTimes: number[] = [];
|
||||
|
||||
let disposed = false;
|
||||
let flushScheduled = false;
|
||||
let rateTimer: unknown = null;
|
||||
let windowTimer: unknown = null;
|
||||
/** An explicit flush or a closed dedup window asks for held counts. */
|
||||
let heldDue = false;
|
||||
let activeFlush: Promise<void> | null = null;
|
||||
const counters = {
|
||||
accepted: 0,
|
||||
@@ -168,20 +181,58 @@ export function createTelemetryPipeline(
|
||||
});
|
||||
}
|
||||
|
||||
function hold(event: TelemetryEvent<unknown>, fingerprint: string): void {
|
||||
const entry = held.get(fingerprint);
|
||||
if (entry) {
|
||||
entry.count += 1;
|
||||
return;
|
||||
}
|
||||
// Bounded like the dedup table; past it a duplicate is only counted in stats.
|
||||
if (held.size >= budget.maxFingerprints) return;
|
||||
held.set(fingerprint, { event, count: 1 });
|
||||
if (windowTimer !== null || disposed) return;
|
||||
windowTimer = setTimer(() => {
|
||||
windowTimer = null;
|
||||
void flushHeld();
|
||||
}, budget.dedupWindowMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Held counts ride along with buffered events for free. They get a batch of
|
||||
* their own only when due, so a duplicate never triggers a send by itself.
|
||||
*/
|
||||
function pending(): boolean {
|
||||
return buffer.size > 0 || (heldDue && held.size > 0);
|
||||
}
|
||||
|
||||
function takeBatch(): TelemetryBatchItem[] {
|
||||
const batch: TelemetryBatchItem[] = buffer.take(budget.maxBatchEvents).map((entry) => {
|
||||
const fingerprint = entry.fingerprint ?? "";
|
||||
const duplicates = held.get(fingerprint)?.count ?? 0;
|
||||
held.delete(fingerprint);
|
||||
return Object.freeze({ event: entry.event, fingerprint, occurrences: 1 + duplicates });
|
||||
});
|
||||
for (const [fingerprint, entry] of held) {
|
||||
if (batch.length >= budget.maxBatchEvents) break;
|
||||
held.delete(fingerprint);
|
||||
batch.push(Object.freeze({ event: entry.event, fingerprint, occurrences: entry.count }));
|
||||
}
|
||||
if (held.size === 0) {
|
||||
heldDue = false;
|
||||
if (windowTimer !== null) clearTimer(windowTimer);
|
||||
windowTimer = null;
|
||||
}
|
||||
return batch;
|
||||
}
|
||||
|
||||
async function drain(): Promise<void> {
|
||||
while (!disposed && buffer.size > 0) {
|
||||
while (!disposed && pending()) {
|
||||
const at = now();
|
||||
if (!rateAllows(at)) {
|
||||
armRateTimer(at);
|
||||
return;
|
||||
}
|
||||
const batch = buffer.take(budget.maxBatchEvents).map((entry) =>
|
||||
Object.freeze({
|
||||
event: entry.event,
|
||||
fingerprint: entry.fingerprint ?? "",
|
||||
occurrences: Math.max(1, dedup.count(entry.fingerprint ?? "")),
|
||||
}),
|
||||
);
|
||||
const batch = takeBatch();
|
||||
sendTimes.push(at);
|
||||
counters.sends += 1;
|
||||
try {
|
||||
@@ -195,7 +246,7 @@ export function createTelemetryPipeline(
|
||||
|
||||
function flush(): Promise<void> {
|
||||
if (activeFlush) return activeFlush;
|
||||
if (disposed || buffer.size === 0) return Promise.resolve();
|
||||
if (disposed || !pending()) return Promise.resolve();
|
||||
activeFlush = drain()
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
@@ -204,6 +255,11 @@ export function createTelemetryPipeline(
|
||||
return activeFlush;
|
||||
}
|
||||
|
||||
function flushHeld(): Promise<void> {
|
||||
heldDue = held.size > 0;
|
||||
return flush();
|
||||
}
|
||||
|
||||
function admit(event: TelemetryEvent<unknown>): void {
|
||||
if (disposed) return;
|
||||
if (!isTelemetryEnvelopeValid(event)) {
|
||||
@@ -228,6 +284,7 @@ export function createTelemetryPipeline(
|
||||
const fingerprint = fingerprintTelemetryEvent(redacted);
|
||||
if (!dedup.record(fingerprint, now())) {
|
||||
counters.deduplicated += 1;
|
||||
hold(redacted, fingerprint);
|
||||
return;
|
||||
}
|
||||
// 4. sample (deterministic by interaction; severe events always kept)
|
||||
@@ -236,12 +293,16 @@ export function createTelemetryPipeline(
|
||||
!sampler.keep(redacted.interactionId ?? fingerprint)
|
||||
) {
|
||||
counters.sampledOut += 1;
|
||||
// Its duplicates must not be reported as held occurrences.
|
||||
dedup.forget(fingerprint);
|
||||
return;
|
||||
}
|
||||
// 5. buffer
|
||||
if (buffer.push(redacted, estimateTelemetryBytes(redacted), fingerprint)) {
|
||||
counters.accepted += 1;
|
||||
counters.peakBuffered = Math.max(counters.peakBuffered, buffer.size);
|
||||
} else {
|
||||
dedup.forget(fingerprint);
|
||||
}
|
||||
// 6-8. rate limit, batch and transport happen on flush.
|
||||
scheduleFlush();
|
||||
@@ -256,7 +317,7 @@ export function createTelemetryPipeline(
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
flush,
|
||||
flush: flushHeld,
|
||||
stats(): TelemetryPipelineStats {
|
||||
return Object.freeze({
|
||||
accepted: counters.accepted,
|
||||
@@ -277,7 +338,10 @@ export function createTelemetryPipeline(
|
||||
disposed = true;
|
||||
if (rateTimer !== null) clearTimer(rateTimer);
|
||||
rateTimer = null;
|
||||
if (windowTimer !== null) clearTimer(windowTimer);
|
||||
windowTimer = null;
|
||||
buffer.clear();
|
||||
held.clear();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { createReferenceFeatureInput } from "../../../src/features/reference-feature/application/reference-feature-api.ts";
|
||||
import { createFailure } from "../../../src/contracts/errors.ts";
|
||||
import type { TelemetryPort } from "../../../src/platform/telemetry/telemetry-port.ts";
|
||||
import { createTelemetryPipeline } from "../../../src/platform/telemetry/telemetry-pipeline.ts";
|
||||
import {
|
||||
createTelemetryPipeline,
|
||||
type TelemetryTransport,
|
||||
} from "../../../src/platform/telemetry/telemetry-pipeline.ts";
|
||||
import { REFERENCE_FEATURE_ADAPTER_CONTRIBUTION } from "../../../src/features/reference-feature/adapters/create-reference-feature-input.ts";
|
||||
|
||||
function failingGateway() {
|
||||
@@ -55,14 +58,19 @@ describe("reference feature telemetry through the platform port", () => {
|
||||
});
|
||||
|
||||
it("collapses a failure storm into bounded sends via the pipeline", async () => {
|
||||
const send = vi.fn();
|
||||
const send = vi.fn<TelemetryTransport["send"]>();
|
||||
const pipeline = createTelemetryPipeline({ transport: { send } });
|
||||
const input = createReferenceFeatureInput(failingGateway(), { telemetry: pipeline });
|
||||
|
||||
for (let i = 0; i < 500; i += 1) await input.listResources({ limit: 20 });
|
||||
await pipeline.flush();
|
||||
|
||||
expect(send).toHaveBeenCalledTimes(1);
|
||||
// The first failure goes out at once; the 499 duplicates ride on the flush.
|
||||
expect(send).toHaveBeenCalledTimes(2);
|
||||
const occurrences = send.mock.calls
|
||||
.flatMap(([batch]) => batch)
|
||||
.reduce((sum, item) => sum + item.occurrences, 0);
|
||||
expect(occurrences).toBe(500);
|
||||
expect(pipeline.stats().deduplicated).toBe(499);
|
||||
pipeline.dispose();
|
||||
});
|
||||
|
||||
@@ -101,6 +101,88 @@ describe("createTelemetryPipeline", () => {
|
||||
expect(batches[0]?.[0]?.occurrences).toBe(25);
|
||||
});
|
||||
|
||||
it("delivers every duplicate as occurrences with the default microtask scheduler", async () => {
|
||||
const batches: TelemetryBatchItem[][] = [];
|
||||
const pipeline = createTelemetryPipeline({
|
||||
transport: { send: (batch) => void batches.push([...batch]) },
|
||||
budget: TEST_TELEMETRY_BUDGET,
|
||||
});
|
||||
for (let i = 0; i < 1_000; i += 1) {
|
||||
await pipeline.emit(REPEATED_ERROR);
|
||||
}
|
||||
await pipeline.flush();
|
||||
const items = batches.flat();
|
||||
expect(items.reduce((sum, item) => sum + item.occurrences, 0)).toBe(1_000);
|
||||
expect(new Set(items.map((item) => item.fingerprint)).size).toBe(1);
|
||||
expect(batches.length).toBeLessThanOrEqual(2);
|
||||
pipeline.dispose();
|
||||
});
|
||||
|
||||
it("reports held duplicates when the dedup window closes without another flush", async () => {
|
||||
const batches: TelemetryBatchItem[][] = [];
|
||||
const timers: Array<Readonly<{ callback: () => void; ms: number }>> = [];
|
||||
const pipeline = createTelemetryPipeline({
|
||||
transport: { send: (batch) => void batches.push([...batch]) },
|
||||
budget: TEST_TELEMETRY_BUDGET,
|
||||
setTimer: (callback, ms) => timers.push({ callback, ms }),
|
||||
clearTimer: () => {},
|
||||
});
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
await pipeline.emit(REPEATED_ERROR);
|
||||
}
|
||||
expect(timers.map((timer) => timer.ms)).toEqual([TEST_TELEMETRY_BUDGET.dedupWindowMs]);
|
||||
for (const timer of timers.splice(0)) timer.callback();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(batches.flat().reduce((sum, item) => sum + item.occurrences, 0)).toBe(10);
|
||||
pipeline.dispose();
|
||||
});
|
||||
|
||||
it("does not let a rejected or evicted event suppress its fingerprint", async () => {
|
||||
const batches: TelemetryBatchItem[][] = [];
|
||||
const pipeline = createTelemetryPipeline({
|
||||
transport: { send: (batch) => void batches.push([...batch]) },
|
||||
budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 1 },
|
||||
schedule: () => {},
|
||||
});
|
||||
const info: TelemetryEvent<unknown> = { ...TEST_EVENT, kind: "catalog.load.slow", severity: "info" };
|
||||
// Rejected on arrival: the buffer holds a more important event.
|
||||
await pipeline.emit(TEST_EVENT);
|
||||
await pipeline.emit(info);
|
||||
await pipeline.flush();
|
||||
await pipeline.emit(info);
|
||||
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
|
||||
await pipeline.flush();
|
||||
// Evicted after admission: a more important event takes its slot.
|
||||
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
|
||||
await pipeline.emit({ ...TEST_EVENT, payload: { reason: "other" } });
|
||||
await pipeline.flush();
|
||||
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
|
||||
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
|
||||
await pipeline.flush();
|
||||
const kinds = batches.flat().map((item) => `${item.event.kind}:${item.occurrences}`);
|
||||
expect(kinds).toEqual([
|
||||
"catalog.load.failed:1",
|
||||
"catalog.load.slow:1",
|
||||
"catalog.load.failed:1",
|
||||
"catalog.load.slow:1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("never reports a sampled-out event through a duplicate summary", async () => {
|
||||
const send = vi.fn();
|
||||
const pipeline = createTelemetryPipeline({
|
||||
transport: { send },
|
||||
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 },
|
||||
});
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
await pipeline.emit({ kind: "a.info", severity: "info" });
|
||||
}
|
||||
await pipeline.flush();
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(pipeline.stats().sampledOut).toBe(3);
|
||||
pipeline.dispose();
|
||||
});
|
||||
|
||||
it("rate-limits distinct events and resumes after the window reopens", async () => {
|
||||
let now = 0;
|
||||
const scheduling = manualScheduling();
|
||||
|
||||
Reference in New Issue
Block a user