Files
clean-architecture-frontend…/tests/scale/telemetry/telemetry-storm.test.ts
T

166 lines
5.6 KiB
TypeScript

import { describe, expect, it, vi } from "vitest";
import {
createTelemetryPipeline,
DEFAULT_TELEMETRY_PIPELINE_BUDGET,
} from "../../../src/platform/telemetry/telemetry-pipeline.ts";
import type { TelemetryEvent } from "../../../src/platform/telemetry/telemetry-event.ts";
import { telemetryAmplification } from "../traffic/amplification-metrics.ts";
import { recordScaleMetrics } from "../scale-artifact.ts";
const IDENTICAL: TelemetryEvent<unknown> = {
kind: "catalog.load.failed",
severity: "error",
featureId: "catalog",
payload: { operation: "list", reason: "NETWORK_UNREACHABLE" },
};
/**
* Spec §43. Many logical failures -> fingerprint/dedup/sample -> a bounded
* number of buffered and sent telemetry events.
*/
async function runTelemetryStorm(
options: Readonly<{
identicalEvents: number;
distinctKinds?: number;
maxSendsPerMinute: number;
maxBufferedEvents: number;
collector?: "up" | "down";
}>,
) {
let networkSends = 0;
let now = 0;
const pipeline = createTelemetryPipeline({
transport: {
async send() {
networkSends += 1;
if (options.collector === "down") throw new Error("collector unavailable");
},
},
budget: {
...DEFAULT_TELEMETRY_PIPELINE_BUDGET,
maxSendsPerMinute: options.maxSendsPerMinute,
maxBufferedEvents: options.maxBufferedEvents,
},
now: () => now,
setTimer: () => 0,
clearTimer: () => {},
});
let peakBufferedEvents = 0;
const distinct = options.distinctKinds ?? 1;
// The whole storm lands inside one rate/dedup window (0.5ms apart).
for (let i = 0; i < options.identicalEvents; i += 1) {
now += 0.5;
await pipeline.emit({
...IDENTICAL,
payload: { operation: "list", reason: `R${i % distinct}` },
interactionId: `interaction-${i}`,
});
if (i % 1_000 === 0) {
await pipeline.flush();
peakBufferedEvents = Math.max(peakBufferedEvents, pipeline.stats().buffered);
}
}
await pipeline.flush();
const stats = pipeline.stats();
peakBufferedEvents = Math.max(peakBufferedEvents, stats.peakBuffered);
pipeline.dispose();
return { networkSends, peakBufferedEvents, stats };
}
describe("telemetry storm", () => {
it("does not turn 100000 identical failures into 100000 sends", async () => {
const result = await runTelemetryStorm({
identicalEvents: 100_000,
maxSendsPerMinute: 20,
maxBufferedEvents: 200,
});
expect(result.networkSends).toBeLessThanOrEqual(20);
expect(result.peakBufferedEvents).toBeLessThanOrEqual(200);
expect(result.stats.deduplicated).toBe(99_999);
expect(
telemetryAmplification({ logicalEvents: 100_000, telemetrySends: result.networkSends }),
).toBeLessThanOrEqual(0.0002);
await recordScaleMetrics("telemetry-storm-identical", {
emitted: 100_000,
deduplicated: result.stats.deduplicated,
sampledOut: result.stats.sampledOut,
dropped: result.stats.dropped,
transmitted: result.networkSends,
peakBuffered: result.peakBufferedEvents,
});
});
it("bounds sends and buffer when a storm has many distinct fingerprints", async () => {
const result = await runTelemetryStorm({
identicalEvents: 100_000,
distinctKinds: 5_000,
maxSendsPerMinute: 20,
maxBufferedEvents: 200,
});
expect(result.networkSends).toBeLessThanOrEqual(20);
expect(result.peakBufferedEvents).toBeLessThanOrEqual(200);
expect(result.stats.fingerprints).toBeLessThanOrEqual(
DEFAULT_TELEMETRY_PIPELINE_BUDGET.maxFingerprints,
);
await recordScaleMetrics("telemetry-storm-distinct", {
emitted: 100_000,
deduplicated: result.stats.deduplicated,
dropped: result.stats.dropped,
transmitted: result.networkSends,
peakBuffered: result.peakBufferedEvents,
});
});
});
/**
* A feature-shaped business operation: it reports its failure through the
* telemetry port fire-and-forget and returns its own result regardless.
*/
function createCatalogLoader(
telemetry: Readonly<{ emit(event: TelemetryEvent<unknown>): unknown }>,
upstream: () => Promise<boolean>,
) {
return async (): Promise<Readonly<{ ok: boolean }>> => {
const ok = await upstream();
if (!ok) {
try {
void Promise.resolve(
telemetry.emit({ ...IDENTICAL, severity: "warning" }),
).catch(() => undefined);
} catch {
// Contained at the call site.
}
}
return { ok };
};
}
describe("collector outage", () => {
it("keeps business promises fulfilled and buffers bounded while every send fails", async () => {
const pipeline = createTelemetryPipeline({
transport: { send: () => Promise.reject(new Error("collector unavailable")) },
budget: { ...DEFAULT_TELEMETRY_PIPELINE_BUDGET, maxBufferedEvents: 50 },
});
let call = 0;
const load = createCatalogLoader(pipeline, async () => call++ % 2 === 1);
const results = await Promise.all(Array.from({ length: 5_000 }, () => load()));
await pipeline.flush();
expect(results).toHaveLength(5_000);
expect(results.filter((result) => result.ok)).toHaveLength(2_500);
expect(pipeline.stats().buffered).toBeLessThanOrEqual(50);
expect(pipeline.stats().peakBuffered).toBeLessThanOrEqual(50);
pipeline.dispose();
});
it("keeps the business result when the port itself throws", async () => {
const throwing = { emit: vi.fn(() => { throw new Error("port broken"); }) };
const load = createCatalogLoader(throwing, async () => false);
await expect(load()).resolves.toEqual({ ok: false });
expect(throwing.emit).toHaveBeenCalledOnce();
});
});