166 lines
5.6 KiB
TypeScript
166 lines
5.6 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import {
|
|
createTelemetryPipeline,
|
|
DEFAULT_TELEMETRY_PIPELINE_BUDGET,
|
|
} from "../../../src/platform/telemetry/telemetry-pipeline.ts";
|
|
import type { TelemetryEvent } from "../../../src/platform/telemetry/telemetry-event.ts";
|
|
import { telemetryAmplification } from "../traffic/amplification-metrics.ts";
|
|
import { recordScaleMetrics } from "../scale-artifact.ts";
|
|
|
|
const IDENTICAL: TelemetryEvent<unknown> = {
|
|
kind: "catalog.load.failed",
|
|
severity: "error",
|
|
featureId: "catalog",
|
|
payload: { operation: "list", reason: "NETWORK_UNREACHABLE" },
|
|
};
|
|
|
|
/**
|
|
* Spec §43. Many logical failures -> fingerprint/dedup/sample -> a bounded
|
|
* number of buffered and sent telemetry events.
|
|
*/
|
|
async function runTelemetryStorm(
|
|
options: Readonly<{
|
|
identicalEvents: number;
|
|
distinctKinds?: number;
|
|
maxSendsPerMinute: number;
|
|
maxBufferedEvents: number;
|
|
collector?: "up" | "down";
|
|
}>,
|
|
) {
|
|
let networkSends = 0;
|
|
let now = 0;
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: {
|
|
async send() {
|
|
networkSends += 1;
|
|
if (options.collector === "down") throw new Error("collector unavailable");
|
|
},
|
|
},
|
|
budget: {
|
|
...DEFAULT_TELEMETRY_PIPELINE_BUDGET,
|
|
maxSendsPerMinute: options.maxSendsPerMinute,
|
|
maxBufferedEvents: options.maxBufferedEvents,
|
|
},
|
|
now: () => now,
|
|
setTimer: () => 0,
|
|
clearTimer: () => {},
|
|
});
|
|
let peakBufferedEvents = 0;
|
|
const distinct = options.distinctKinds ?? 1;
|
|
// The whole storm lands inside one rate/dedup window (0.5ms apart).
|
|
for (let i = 0; i < options.identicalEvents; i += 1) {
|
|
now += 0.5;
|
|
await pipeline.emit({
|
|
...IDENTICAL,
|
|
payload: { operation: "list", reason: `R${i % distinct}` },
|
|
interactionId: `interaction-${i}`,
|
|
});
|
|
if (i % 1_000 === 0) {
|
|
await pipeline.flush();
|
|
peakBufferedEvents = Math.max(peakBufferedEvents, pipeline.stats().buffered);
|
|
}
|
|
}
|
|
await pipeline.flush();
|
|
const stats = pipeline.stats();
|
|
peakBufferedEvents = Math.max(peakBufferedEvents, stats.peakBuffered);
|
|
pipeline.dispose();
|
|
return { networkSends, peakBufferedEvents, stats };
|
|
}
|
|
|
|
describe("telemetry storm", () => {
|
|
it("does not turn 100000 identical failures into 100000 sends", async () => {
|
|
const result = await runTelemetryStorm({
|
|
identicalEvents: 100_000,
|
|
maxSendsPerMinute: 20,
|
|
maxBufferedEvents: 200,
|
|
});
|
|
|
|
expect(result.networkSends).toBeLessThanOrEqual(20);
|
|
expect(result.peakBufferedEvents).toBeLessThanOrEqual(200);
|
|
expect(result.stats.deduplicated).toBe(99_999);
|
|
expect(
|
|
telemetryAmplification({ logicalEvents: 100_000, telemetrySends: result.networkSends }),
|
|
).toBeLessThanOrEqual(0.0002);
|
|
await recordScaleMetrics("telemetry-storm-identical", {
|
|
emitted: 100_000,
|
|
deduplicated: result.stats.deduplicated,
|
|
sampledOut: result.stats.sampledOut,
|
|
dropped: result.stats.dropped,
|
|
transmitted: result.networkSends,
|
|
peakBuffered: result.peakBufferedEvents,
|
|
});
|
|
});
|
|
|
|
it("bounds sends and buffer when a storm has many distinct fingerprints", async () => {
|
|
const result = await runTelemetryStorm({
|
|
identicalEvents: 100_000,
|
|
distinctKinds: 5_000,
|
|
maxSendsPerMinute: 20,
|
|
maxBufferedEvents: 200,
|
|
});
|
|
expect(result.networkSends).toBeLessThanOrEqual(20);
|
|
expect(result.peakBufferedEvents).toBeLessThanOrEqual(200);
|
|
expect(result.stats.fingerprints).toBeLessThanOrEqual(
|
|
DEFAULT_TELEMETRY_PIPELINE_BUDGET.maxFingerprints,
|
|
);
|
|
await recordScaleMetrics("telemetry-storm-distinct", {
|
|
emitted: 100_000,
|
|
deduplicated: result.stats.deduplicated,
|
|
dropped: result.stats.dropped,
|
|
transmitted: result.networkSends,
|
|
peakBuffered: result.peakBufferedEvents,
|
|
});
|
|
});
|
|
});
|
|
|
|
/**
|
|
* A feature-shaped business operation: it reports its failure through the
|
|
* telemetry port fire-and-forget and returns its own result regardless.
|
|
*/
|
|
function createCatalogLoader(
|
|
telemetry: Readonly<{ emit(event: TelemetryEvent<unknown>): unknown }>,
|
|
upstream: () => Promise<boolean>,
|
|
) {
|
|
return async (): Promise<Readonly<{ ok: boolean }>> => {
|
|
const ok = await upstream();
|
|
if (!ok) {
|
|
try {
|
|
void Promise.resolve(
|
|
telemetry.emit({ ...IDENTICAL, severity: "warning" }),
|
|
).catch(() => undefined);
|
|
} catch {
|
|
// Contained at the call site.
|
|
}
|
|
}
|
|
return { ok };
|
|
};
|
|
}
|
|
|
|
describe("collector outage", () => {
|
|
it("keeps business promises fulfilled and buffers bounded while every send fails", async () => {
|
|
const pipeline = createTelemetryPipeline({
|
|
transport: { send: () => Promise.reject(new Error("collector unavailable")) },
|
|
budget: { ...DEFAULT_TELEMETRY_PIPELINE_BUDGET, maxBufferedEvents: 50 },
|
|
});
|
|
let call = 0;
|
|
const load = createCatalogLoader(pipeline, async () => call++ % 2 === 1);
|
|
|
|
const results = await Promise.all(Array.from({ length: 5_000 }, () => load()));
|
|
await pipeline.flush();
|
|
|
|
expect(results).toHaveLength(5_000);
|
|
expect(results.filter((result) => result.ok)).toHaveLength(2_500);
|
|
expect(pipeline.stats().buffered).toBeLessThanOrEqual(50);
|
|
expect(pipeline.stats().peakBuffered).toBeLessThanOrEqual(50);
|
|
pipeline.dispose();
|
|
});
|
|
|
|
it("keeps the business result when the port itself throws", async () => {
|
|
const throwing = { emit: vi.fn(() => { throw new Error("port broken"); }) };
|
|
const load = createCatalogLoader(throwing, async () => false);
|
|
await expect(load()).resolves.toEqual({ ok: false });
|
|
expect(throwing.emit).toHaveBeenCalledOnce();
|
|
});
|
|
});
|