With the default queueMicrotask scheduler the first event of a burst was sent at once with occurrences=1, and every duplicate inside the 60s dedup window only bumped counters.deduplicated, so 1000 identical errors reached the collector as occurrences=1. An event dropped from the buffer also left its fingerprint in the dedup table, suppressing that event for 60s. Per D-6 the pipeline now holds duplicate counts per fingerprint. Held counts are folded into a buffered item of the same fingerprint or ride as summary items in the next batch; they get a batch of their own only on an explicit flush() or when the dedup window closes (one window timer), so a duplicate never triggers a send by itself. The held table is bounded by maxFingerprints. A rejected or evicted event (new buffer onEvict hook) and a sampled-out event forget their fingerprint (new TelemetryDeduplicator.forget), so the next occurrence is admitted again and a sampled-out event is never reported through a summary. Existing signatures are unchanged; onEvict and forget are additive. Test changes: - tests/features/reference-feature/reference-telemetry.test.ts expected exactly one send for a 500-failure storm, which encoded the lost count; it now expects 2 sends whose occurrences sum to 500. - telemetry-pipeline.test.ts:93 (auto flush off, 25 in one item) is kept as is and still passes. RED before fix: - "delivers every duplicate as occurrences with the default microtask scheduler" (sum was 1, expected 1000) - "reports held duplicates when the dedup window closes without another flush" (no window timer armed) - "does not let a rejected or evicted event suppress its fingerprint" (re-emitted event was deduplicated, not buffered) - "never reports a sampled-out event through a duplicate summary" (sampledOut 1, expected 3) Gates: check:architecture, check:types:app, check:types:test, lint, test:component, check:release-coherence, test:platform pass; also tests/scale/telemetry, tests/scale/session, tests/features/reference-feature pass. test:unit passes except tests/unit/removal-fixture.test.ts, which needs generated release evidence under artifacts/ that a fresh worktree does not have (unrelated to this change). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
268 lines
9.5 KiB
TypeScript
268 lines
9.5 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import {
|
|
createTelemetryPipeline,
|
|
DEFAULT_TELEMETRY_PIPELINE_BUDGET,
|
|
type TelemetryBatchItem,
|
|
type TelemetryPipelineBudget,
|
|
} from "../../../src/platform/telemetry/telemetry-pipeline.ts";
|
|
import type { TelemetryEvent } from "../../../src/platform/telemetry/telemetry-event.ts";
|
|
|
|
const TEST_TELEMETRY_BUDGET: TelemetryPipelineBudget = {
|
|
...DEFAULT_TELEMETRY_PIPELINE_BUDGET,
|
|
maxBufferedEvents: 50,
|
|
maxSendsPerMinute: 10,
|
|
};
|
|
|
|
const TEST_EVENT: TelemetryEvent<unknown> = {
|
|
kind: "catalog.load.failed",
|
|
severity: "error",
|
|
featureId: "catalog",
|
|
payload: { reason: "REQUEST_TIMEOUT" },
|
|
};
|
|
const REPEATED_ERROR = TEST_EVENT;
|
|
|
|
/** Runs scheduled work synchronously-ish and keeps timers in the test's hand. */
|
|
function manualScheduling() {
|
|
const timers: Array<() => void> = [];
|
|
return {
|
|
schedule: (callback: () => void) => queueMicrotask(callback),
|
|
setTimer: (callback: () => void) => {
|
|
timers.push(callback);
|
|
return timers.length;
|
|
},
|
|
clearTimer: () => {},
|
|
runTimers() {
|
|
for (const callback of timers.splice(0)) callback();
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("createTelemetryPipeline", () => {
|
|
it("does not reject business flow when telemetry transport fails", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: {
|
|
async send() {
|
|
throw new Error("collector unavailable");
|
|
},
|
|
},
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
|
|
await expect(pipeline.emit(TEST_EVENT)).resolves.toBeUndefined();
|
|
await expect(pipeline.flush()).resolves.toBeUndefined();
|
|
expect(pipeline.stats().failedSends).toBe(1);
|
|
});
|
|
|
|
it("does not throw when the transport throws synchronously", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: {
|
|
send() {
|
|
throw new Error("sdk crashed");
|
|
},
|
|
},
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
await expect(pipeline.emit(TEST_EVENT)).resolves.toBeUndefined();
|
|
await expect(pipeline.flush()).resolves.toBeUndefined();
|
|
});
|
|
|
|
it("bounds network sends during a repeated error storm", async () => {
|
|
const send = vi.fn().mockResolvedValue(undefined);
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, maxSendsPerMinute: 10 },
|
|
});
|
|
|
|
for (let i = 0; i < 100_000; i += 1) {
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
|
|
await pipeline.flush();
|
|
expect(send.mock.calls.length).toBeLessThanOrEqual(10);
|
|
expect(pipeline.stats().deduplicated).toBe(99_999);
|
|
});
|
|
|
|
it("reports how many occurrences a delivered fingerprint collapsed", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
let now = 0;
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
now: () => now,
|
|
schedule: () => {},
|
|
});
|
|
for (let i = 0; i < 25; i += 1) {
|
|
now = i;
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
await pipeline.flush();
|
|
expect(batches.flat()).toHaveLength(1);
|
|
expect(batches[0]?.[0]?.occurrences).toBe(25);
|
|
});
|
|
|
|
it("delivers every duplicate as occurrences with the default microtask scheduler", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
for (let i = 0; i < 1_000; i += 1) {
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
await pipeline.flush();
|
|
const items = batches.flat();
|
|
expect(items.reduce((sum, item) => sum + item.occurrences, 0)).toBe(1_000);
|
|
expect(new Set(items.map((item) => item.fingerprint)).size).toBe(1);
|
|
expect(batches.length).toBeLessThanOrEqual(2);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("reports held duplicates when the dedup window closes without another flush", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const timers: Array<Readonly<{ callback: () => void; ms: number }>> = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
setTimer: (callback, ms) => timers.push({ callback, ms }),
|
|
clearTimer: () => {},
|
|
});
|
|
for (let i = 0; i < 10; i += 1) {
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
expect(timers.map((timer) => timer.ms)).toEqual([TEST_TELEMETRY_BUDGET.dedupWindowMs]);
|
|
for (const timer of timers.splice(0)) timer.callback();
|
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
expect(batches.flat().reduce((sum, item) => sum + item.occurrences, 0)).toBe(10);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("does not let a rejected or evicted event suppress its fingerprint", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 1 },
|
|
schedule: () => {},
|
|
});
|
|
const info: TelemetryEvent<unknown> = { ...TEST_EVENT, kind: "catalog.load.slow", severity: "info" };
|
|
// Rejected on arrival: the buffer holds a more important event.
|
|
await pipeline.emit(TEST_EVENT);
|
|
await pipeline.emit(info);
|
|
await pipeline.flush();
|
|
await pipeline.emit(info);
|
|
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
|
|
await pipeline.flush();
|
|
// Evicted after admission: a more important event takes its slot.
|
|
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
|
|
await pipeline.emit({ ...TEST_EVENT, payload: { reason: "other" } });
|
|
await pipeline.flush();
|
|
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
|
|
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
|
|
await pipeline.flush();
|
|
const kinds = batches.flat().map((item) => `${item.event.kind}:${item.occurrences}`);
|
|
expect(kinds).toEqual([
|
|
"catalog.load.failed:1",
|
|
"catalog.load.slow:1",
|
|
"catalog.load.failed:1",
|
|
"catalog.load.slow:1",
|
|
]);
|
|
});
|
|
|
|
it("never reports a sampled-out event through a duplicate summary", async () => {
|
|
const send = vi.fn();
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 },
|
|
});
|
|
for (let i = 0; i < 3; i += 1) {
|
|
await pipeline.emit({ kind: "a.info", severity: "info" });
|
|
}
|
|
await pipeline.flush();
|
|
expect(send).not.toHaveBeenCalled();
|
|
expect(pipeline.stats().sampledOut).toBe(3);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("rate-limits distinct events and resumes after the window reopens", async () => {
|
|
let now = 0;
|
|
const scheduling = manualScheduling();
|
|
const send = vi.fn().mockResolvedValue(undefined);
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: {
|
|
...TEST_TELEMETRY_BUDGET,
|
|
maxSendsPerMinute: 2,
|
|
maxBatchEvents: 1,
|
|
maxBufferedEvents: 100,
|
|
},
|
|
now: () => now,
|
|
...scheduling,
|
|
});
|
|
for (let i = 0; i < 5; i += 1) {
|
|
await pipeline.emit({ ...TEST_EVENT, payload: { reason: `r${i}` } });
|
|
}
|
|
await pipeline.flush();
|
|
expect(send).toHaveBeenCalledTimes(2);
|
|
expect(pipeline.stats().buffered).toBe(3);
|
|
|
|
now = 60_000;
|
|
scheduling.runTimers();
|
|
await pipeline.flush();
|
|
expect(send).toHaveBeenCalledTimes(4);
|
|
});
|
|
|
|
it("keeps memory bounded when events are all distinct and the collector is down", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: () => Promise.reject(new Error("down")) },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 20, maxFingerprints: 30 },
|
|
schedule: () => {},
|
|
});
|
|
for (let i = 0; i < 5_000; i += 1) {
|
|
await pipeline.emit({ ...TEST_EVENT, payload: { reason: `distinct-${i}` } });
|
|
}
|
|
const stats = pipeline.stats();
|
|
expect(stats.buffered).toBeLessThanOrEqual(20);
|
|
expect(stats.peakBuffered).toBeLessThanOrEqual(20);
|
|
expect(stats.fingerprints).toBeLessThanOrEqual(30);
|
|
});
|
|
|
|
it("redacts before anything is buffered or sent", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
await pipeline.emit({
|
|
...TEST_EVENT,
|
|
payload: { reason: "x", accessToken: "secret-token", headers: { cookie: "sid=1" } },
|
|
});
|
|
await pipeline.flush();
|
|
expect(JSON.stringify(batches)).not.toMatch(/secret-token|sid=1/);
|
|
});
|
|
|
|
it("samples low-severity events deterministically but always keeps errors", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: () => {} },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 },
|
|
schedule: () => {},
|
|
});
|
|
await pipeline.emit({ kind: "a.info", severity: "info", interactionId: "i-1" });
|
|
await pipeline.emit({ kind: "a.error", severity: "error", interactionId: "i-1" });
|
|
expect(pipeline.stats()).toMatchObject({ sampledOut: 1, buffered: 1 });
|
|
});
|
|
|
|
it("drops invalid envelopes and stops admitting after dispose", async () => {
|
|
const send = vi.fn();
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
await pipeline.emit({ kind: "NOT VALID", severity: "error" });
|
|
expect(pipeline.stats().invalid).toBe(1);
|
|
pipeline.dispose();
|
|
await pipeline.emit(TEST_EVENT);
|
|
await pipeline.flush();
|
|
expect(send).not.toHaveBeenCalled();
|
|
});
|
|
});
|