모듈 주석이 약속한 두 정책을 시험이 보지 않았다. buffer가 가득 차면 낮은 severity부터 내보낸다(telemetry-buffer.ts의 sort). 같은 interaction의 사건은 함께 남거나 함께 빠진다(pipeline이 interactionId로 표본을 뜬다). 내보내기 시험은 후보가 모두 같은 severity였고, 표본 시험은 sampleRate 0이라 key를 보기 전에 끝났다. 그래서 sort를 지우거나 표본 key를 fingerprint로 바꾸거나 표본 대상을 error 미만에서 warning 미만으로 줄여도 통과했다. buffer 시험: 가득 찬 [warning(오래됨), info]에 error가 오면 info만 내보내고 [warning, error]가 남는다. pipeline 시험: sampleRate 0.5에서 같은 sampler가 남기는 interaction 넷과 빼는 interaction 넷을 골라, 각 interaction의 info·warning 사건(payload가 달라 fingerprint가 다르다)을 보낸다. 보낸 묶음이 남는 interaction의 두 사건뿐이고 sampledOut이 8인지 본다. id를 고정 범위(interaction-0..15)로 두면 이 hash가 끝 글자만 다른 key를 한쪽으로 몰아 모두 빠져서, 남는 것과 빠지는 것을 골라 둔다. buffer 시험의 사건 helper는 두 시험이 같이 쓰게 파일 위로 올린다. 버린 안: sampler 단위 시험만 늘리기. key를 고르는 곳은 pipeline이라 그 변이를 못 잡는다. 위험: 시험만 바꾼다. 기본 표본률 1과 src/platform/telemetry는 그대로다. 검증(desktop): - 고치기 전 시험 + sort 삭제: buffer·pipeline·telemetry-storm 시험과 check:observability 통과. + keep(fingerprint) / `.error`→`.warning`: pipeline·storm 시험 통과. - 새 시험 + 같은 변이 셋: 각각 새 시험이 실패. 변이 없는 트리: 통과. Refs: REV-FE-0033
299 lines
11 KiB
TypeScript
299 lines
11 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import {
|
|
createTelemetryPipeline,
|
|
DEFAULT_TELEMETRY_PIPELINE_BUDGET,
|
|
type TelemetryBatchItem,
|
|
type TelemetryPipelineBudget,
|
|
} from "../../../src/platform/telemetry/telemetry-pipeline.ts";
|
|
import type { TelemetryEvent } from "../../../src/platform/telemetry/telemetry-event.ts";
|
|
import { createDeterministicSampler } from "../../../src/platform/telemetry/telemetry-sampler.ts";
|
|
|
|
const TEST_TELEMETRY_BUDGET: TelemetryPipelineBudget = {
|
|
...DEFAULT_TELEMETRY_PIPELINE_BUDGET,
|
|
maxBufferedEvents: 50,
|
|
maxSendsPerMinute: 10,
|
|
};
|
|
|
|
const TEST_EVENT: TelemetryEvent<unknown> = {
|
|
kind: "catalog.load.failed",
|
|
severity: "error",
|
|
featureId: "catalog",
|
|
payload: { reason: "REQUEST_TIMEOUT" },
|
|
};
|
|
const REPEATED_ERROR = TEST_EVENT;
|
|
|
|
/** Runs scheduled work synchronously-ish and keeps timers in the test's hand. */
|
|
function manualScheduling() {
|
|
const timers: Array<() => void> = [];
|
|
return {
|
|
schedule: (callback: () => void) => queueMicrotask(callback),
|
|
setTimer: (callback: () => void) => {
|
|
timers.push(callback);
|
|
return timers.length;
|
|
},
|
|
clearTimer: () => {},
|
|
runTimers() {
|
|
for (const callback of timers.splice(0)) callback();
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("createTelemetryPipeline", () => {
|
|
it("does not reject business flow when telemetry transport fails", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: {
|
|
async send() {
|
|
throw new Error("collector unavailable");
|
|
},
|
|
},
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
|
|
await expect(pipeline.emit(TEST_EVENT)).resolves.toBeUndefined();
|
|
await expect(pipeline.flush()).resolves.toBeUndefined();
|
|
expect(pipeline.stats().failedSends).toBe(1);
|
|
});
|
|
|
|
it("does not throw when the transport throws synchronously", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: {
|
|
send() {
|
|
throw new Error("sdk crashed");
|
|
},
|
|
},
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
await expect(pipeline.emit(TEST_EVENT)).resolves.toBeUndefined();
|
|
await expect(pipeline.flush()).resolves.toBeUndefined();
|
|
});
|
|
|
|
it("bounds network sends during a repeated error storm", async () => {
|
|
const send = vi.fn().mockResolvedValue(undefined);
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, maxSendsPerMinute: 10 },
|
|
});
|
|
|
|
for (let i = 0; i < 100_000; i += 1) {
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
|
|
await pipeline.flush();
|
|
expect(send.mock.calls.length).toBeLessThanOrEqual(10);
|
|
expect(pipeline.stats().deduplicated).toBe(99_999);
|
|
});
|
|
|
|
it("reports how many occurrences a delivered fingerprint collapsed", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
let now = 0;
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
now: () => now,
|
|
schedule: () => {},
|
|
});
|
|
for (let i = 0; i < 25; i += 1) {
|
|
now = i;
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
await pipeline.flush();
|
|
expect(batches.flat()).toHaveLength(1);
|
|
expect(batches[0]?.[0]?.occurrences).toBe(25);
|
|
});
|
|
|
|
it("delivers every duplicate as occurrences with the default microtask scheduler", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
for (let i = 0; i < 1_000; i += 1) {
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
await pipeline.flush();
|
|
const items = batches.flat();
|
|
expect(items.reduce((sum, item) => sum + item.occurrences, 0)).toBe(1_000);
|
|
expect(new Set(items.map((item) => item.fingerprint)).size).toBe(1);
|
|
expect(batches.length).toBeLessThanOrEqual(2);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("reports held duplicates when the dedup window closes without another flush", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const timers: Array<Readonly<{ callback: () => void; ms: number }>> = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
setTimer: (callback, ms) => timers.push({ callback, ms }),
|
|
clearTimer: () => {},
|
|
});
|
|
for (let i = 0; i < 10; i += 1) {
|
|
await pipeline.emit(REPEATED_ERROR);
|
|
}
|
|
expect(timers.map((timer) => timer.ms)).toEqual([TEST_TELEMETRY_BUDGET.dedupWindowMs]);
|
|
for (const timer of timers.splice(0)) timer.callback();
|
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
expect(batches.flat().reduce((sum, item) => sum + item.occurrences, 0)).toBe(10);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("does not let a rejected or evicted event suppress its fingerprint", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 1 },
|
|
schedule: () => {},
|
|
});
|
|
const info: TelemetryEvent<unknown> = { ...TEST_EVENT, kind: "catalog.load.slow", severity: "info" };
|
|
// Rejected on arrival: the buffer holds a more important event.
|
|
await pipeline.emit(TEST_EVENT);
|
|
await pipeline.emit(info);
|
|
await pipeline.flush();
|
|
await pipeline.emit(info);
|
|
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
|
|
await pipeline.flush();
|
|
// Evicted after admission: a more important event takes its slot.
|
|
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
|
|
await pipeline.emit({ ...TEST_EVENT, payload: { reason: "other" } });
|
|
await pipeline.flush();
|
|
await pipeline.emit({ ...info, payload: { reason: "evicted" } });
|
|
expect(pipeline.stats()).toMatchObject({ buffered: 1, deduplicated: 0 });
|
|
await pipeline.flush();
|
|
const kinds = batches.flat().map((item) => `${item.event.kind}:${item.occurrences}`);
|
|
expect(kinds).toEqual([
|
|
"catalog.load.failed:1",
|
|
"catalog.load.slow:1",
|
|
"catalog.load.failed:1",
|
|
"catalog.load.slow:1",
|
|
]);
|
|
});
|
|
|
|
it("never reports a sampled-out event through a duplicate summary", async () => {
|
|
const send = vi.fn();
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 },
|
|
});
|
|
for (let i = 0; i < 3; i += 1) {
|
|
await pipeline.emit({ kind: "a.info", severity: "info" });
|
|
}
|
|
await pipeline.flush();
|
|
expect(send).not.toHaveBeenCalled();
|
|
expect(pipeline.stats().sampledOut).toBe(3);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("rate-limits distinct events and resumes after the window reopens", async () => {
|
|
let now = 0;
|
|
const scheduling = manualScheduling();
|
|
const send = vi.fn().mockResolvedValue(undefined);
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: {
|
|
...TEST_TELEMETRY_BUDGET,
|
|
maxSendsPerMinute: 2,
|
|
maxBatchEvents: 1,
|
|
maxBufferedEvents: 100,
|
|
},
|
|
now: () => now,
|
|
...scheduling,
|
|
});
|
|
for (let i = 0; i < 5; i += 1) {
|
|
await pipeline.emit({ ...TEST_EVENT, payload: { reason: `r${i}` } });
|
|
}
|
|
await pipeline.flush();
|
|
expect(send).toHaveBeenCalledTimes(2);
|
|
expect(pipeline.stats().buffered).toBe(3);
|
|
|
|
now = 60_000;
|
|
scheduling.runTimers();
|
|
await pipeline.flush();
|
|
expect(send).toHaveBeenCalledTimes(4);
|
|
});
|
|
|
|
it("keeps memory bounded when events are all distinct and the collector is down", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: () => Promise.reject(new Error("down")) },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, maxBufferedEvents: 20, maxFingerprints: 30 },
|
|
schedule: () => {},
|
|
});
|
|
for (let i = 0; i < 5_000; i += 1) {
|
|
await pipeline.emit({ ...TEST_EVENT, payload: { reason: `distinct-${i}` } });
|
|
}
|
|
const stats = pipeline.stats();
|
|
expect(stats.buffered).toBeLessThanOrEqual(20);
|
|
expect(stats.peakBuffered).toBeLessThanOrEqual(20);
|
|
expect(stats.fingerprints).toBeLessThanOrEqual(30);
|
|
});
|
|
|
|
it("redacts before anything is buffered or sent", async () => {
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
await pipeline.emit({
|
|
...TEST_EVENT,
|
|
payload: { reason: "x", accessToken: "secret-token", headers: { cookie: "sid=1" } },
|
|
});
|
|
await pipeline.flush();
|
|
expect(JSON.stringify(batches)).not.toMatch(/secret-token|sid=1/);
|
|
});
|
|
|
|
it("samples low-severity events deterministically but always keeps errors", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: () => {} },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate: 0 },
|
|
schedule: () => {},
|
|
});
|
|
await pipeline.emit({ kind: "a.info", severity: "info", interactionId: "i-1" });
|
|
await pipeline.emit({ kind: "a.error", severity: "error", interactionId: "i-1" });
|
|
expect(pipeline.stats()).toMatchObject({ sampledOut: 1, buffered: 1 });
|
|
});
|
|
|
|
it("keeps or drops an interaction's info and warning events together", async () => {
|
|
const sampleRate = 0.5;
|
|
const sampler = createDeterministicSampler({ rate: sampleRate });
|
|
// Four interactions the rate keeps and four it drops, so both paths run.
|
|
const candidates = Array.from({ length: 1_000 }, (_, i) => `interaction-${i}`);
|
|
const kept = candidates.filter((interactionId) => sampler.keep(interactionId)).slice(0, 4);
|
|
const dropped = candidates.filter((interactionId) => !sampler.keep(interactionId)).slice(0, 4);
|
|
expect([kept.length, dropped.length]).toEqual([4, 4]);
|
|
const interactions = [...dropped, ...kept];
|
|
const batches: TelemetryBatchItem[][] = [];
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: (batch) => void batches.push([...batch]) },
|
|
budget: { ...TEST_TELEMETRY_BUDGET, sampleRate },
|
|
schedule: () => {},
|
|
});
|
|
// Each event has its own payload, so its fingerprint (which leaves the
|
|
// interaction id out) differs from every other event's.
|
|
for (const [step, interactionId] of interactions.entries()) {
|
|
await pipeline.emit({ kind: "a.info", severity: "info", interactionId, payload: { step } });
|
|
await pipeline.emit({ kind: "a.warning", severity: "warning", interactionId, payload: { step } });
|
|
}
|
|
await pipeline.flush();
|
|
|
|
expect(
|
|
batches.flat().map((item) => `${item.event.interactionId}:${item.event.severity}`),
|
|
).toEqual(kept.flatMap((interactionId) => [`${interactionId}:info`, `${interactionId}:warning`]));
|
|
expect(pipeline.stats().sampledOut).toBe(2 * dropped.length);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("drops invalid envelopes and stops admitting after dispose", async () => {
|
|
const send = vi.fn();
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send },
|
|
budget: TEST_TELEMETRY_BUDGET,
|
|
});
|
|
await pipeline.emit({ kind: "NOT VALID", severity: "error" });
|
|
expect(pipeline.stats().invalid).toBe(1);
|
|
pipeline.dispose();
|
|
await pipeline.emit(TEST_EVENT);
|
|
await pipeline.flush();
|
|
expect(send).not.toHaveBeenCalled();
|
|
});
|
|
});
|