import { describe, expect, it, vi } from "vitest"; import { createTelemetryPipeline, DEFAULT_TELEMETRY_PIPELINE_BUDGET, } from "../../../src/platform/telemetry/telemetry-pipeline.ts"; import type { TelemetryEvent } from "../../../src/platform/telemetry/telemetry-event.ts"; import { telemetryAmplification } from "../traffic/amplification-metrics.ts"; import { recordScaleMetrics } from "../scale-artifact.ts"; const IDENTICAL: TelemetryEvent = { kind: "catalog.load.failed", severity: "error", featureId: "catalog", payload: { operation: "list", reason: "NETWORK_UNREACHABLE" }, }; /** * Spec ยง43. Many logical failures -> fingerprint/dedup/sample -> a bounded * number of buffered and sent telemetry events. */ async function runTelemetryStorm( options: Readonly<{ identicalEvents: number; distinctKinds?: number; maxSendsPerMinute: number; maxBufferedEvents: number; collector?: "up" | "down"; }>, ) { let networkSends = 0; let now = 0; const pipeline = createTelemetryPipeline({ transport: { async send() { networkSends += 1; if (options.collector === "down") throw new Error("collector unavailable"); }, }, budget: { ...DEFAULT_TELEMETRY_PIPELINE_BUDGET, maxSendsPerMinute: options.maxSendsPerMinute, maxBufferedEvents: options.maxBufferedEvents, }, now: () => now, setTimer: () => 0, clearTimer: () => {}, }); let peakBufferedEvents = 0; const distinct = options.distinctKinds ?? 1; // The whole storm lands inside one rate/dedup window (0.5ms apart). for (let i = 0; i < options.identicalEvents; i += 1) { now += 0.5; await pipeline.emit({ ...IDENTICAL, payload: { operation: "list", reason: `R${i % distinct}` }, interactionId: `interaction-${i}`, }); if (i % 1_000 === 0) { await pipeline.flush(); peakBufferedEvents = Math.max(peakBufferedEvents, pipeline.stats().buffered); } } await pipeline.flush(); const stats = pipeline.stats(); peakBufferedEvents = Math.max(peakBufferedEvents, stats.peakBuffered); pipeline.dispose(); return { networkSends, peakBufferedEvents, stats }; } describe("telemetry storm", () => { it("does not turn 100000 identical failures into 100000 sends", async () => { const result = await runTelemetryStorm({ identicalEvents: 100_000, maxSendsPerMinute: 20, maxBufferedEvents: 200, }); expect(result.networkSends).toBeLessThanOrEqual(20); expect(result.peakBufferedEvents).toBeLessThanOrEqual(200); expect(result.stats.deduplicated).toBe(99_999); expect( telemetryAmplification({ logicalEvents: 100_000, telemetrySends: result.networkSends }), ).toBeLessThanOrEqual(0.0002); await recordScaleMetrics("telemetry-storm-identical", { emitted: 100_000, deduplicated: result.stats.deduplicated, sampledOut: result.stats.sampledOut, dropped: result.stats.dropped, transmitted: result.networkSends, peakBuffered: result.peakBufferedEvents, }); }); it("bounds sends and buffer when a storm has many distinct fingerprints", async () => { const result = await runTelemetryStorm({ identicalEvents: 100_000, distinctKinds: 5_000, maxSendsPerMinute: 20, maxBufferedEvents: 200, }); expect(result.networkSends).toBeLessThanOrEqual(20); expect(result.peakBufferedEvents).toBeLessThanOrEqual(200); expect(result.stats.fingerprints).toBeLessThanOrEqual( DEFAULT_TELEMETRY_PIPELINE_BUDGET.maxFingerprints, ); await recordScaleMetrics("telemetry-storm-distinct", { emitted: 100_000, deduplicated: result.stats.deduplicated, dropped: result.stats.dropped, transmitted: result.networkSends, peakBuffered: result.peakBufferedEvents, }); }); }); /** * A feature-shaped business operation: it reports its failure through the * telemetry port fire-and-forget and returns its own result regardless. */ function createCatalogLoader( telemetry: Readonly<{ emit(event: TelemetryEvent): unknown }>, upstream: () => Promise, ) { return async (): Promise> => { const ok = await upstream(); if (!ok) { try { void Promise.resolve( telemetry.emit({ ...IDENTICAL, severity: "warning" }), ).catch(() => undefined); } catch { // Contained at the call site. } } return { ok }; }; } describe("collector outage", () => { it("keeps business promises fulfilled and buffers bounded while every send fails", async () => { const pipeline = createTelemetryPipeline({ transport: { send: () => Promise.reject(new Error("collector unavailable")) }, budget: { ...DEFAULT_TELEMETRY_PIPELINE_BUDGET, maxBufferedEvents: 50 }, }); let call = 0; const load = createCatalogLoader(pipeline, async () => call++ % 2 === 1); const results = await Promise.all(Array.from({ length: 5_000 }, () => load())); await pipeline.flush(); expect(results).toHaveLength(5_000); expect(results.filter((result) => result.ok)).toHaveLength(2_500); expect(pipeline.stats().buffered).toBeLessThanOrEqual(50); expect(pipeline.stats().peakBuffered).toBeLessThanOrEqual(50); pipeline.dispose(); }); it("keeps the business result when the port itself throws", async () => { const throwing = { emit: vi.fn(() => { throw new Error("port broken"); }) }; const load = createCatalogLoader(throwing, async () => false); await expect(load()).resolves.toEqual({ ok: false }); expect(throwing.emit).toHaveBeenCalledOnce(); }); });