The application port (best-effort-telemetry) and the feature port (feature-telemetry pipeline) each posted to TELEMETRY_ENDPOINT on their own: 20+20 sends per minute, cookies on the application side, no pagehide flush or dispose abort on the feature side, and no deadline on any fetch, so a hung collector held activeFlush forever (6.3-10). New src/adapters/telemetry/telemetry-collector.ts is the one transport: shared send-rate window (createTelemetrySendRate, now in the platform pipeline and injectable via `sendRate`), credentials "omit", cache "no-store", a per-request deadline through the kernel createAbortableOperation, and one pagehide fan-out. runtime-adapters builds one collector and passes it to both ports. Body projections are unchanged and pinned by characterization tests first. Verified: new tests in tests/unit/telemetry.test.ts (5 RED before wiring: credentials/cache, hung collector timeout, shared rate, feature pagehide flush, feature dispose abort); all gates pass except the known tests/unit/removal-fixture.test.ts (artifacts/ absent); check:adapter-inventory 137/137; FE-RB-004 drill PASS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
380 lines
12 KiB
TypeScript
380 lines
12 KiB
TypeScript
import {
|
|
createTelemetryBuffer,
|
|
estimateTelemetryBytes,
|
|
} from "./telemetry-buffer.ts";
|
|
import { TelemetryDeduplicator } from "./telemetry-deduplicator.ts";
|
|
import {
|
|
isTelemetryEnvelopeValid,
|
|
TELEMETRY_SEVERITY_RANK,
|
|
type TelemetryEvent,
|
|
} from "./telemetry-event.ts";
|
|
import { fingerprintTelemetryEvent } from "./telemetry-fingerprint.ts";
|
|
import type { TelemetryPort } from "./telemetry-port.ts";
|
|
import {
|
|
DEFAULT_TELEMETRY_PAYLOAD_BUDGET,
|
|
redactTelemetryPayload,
|
|
type TelemetryPayloadBudget,
|
|
} from "./redact-telemetry.ts";
|
|
import { createDeterministicSampler } from "./telemetry-sampler.ts";
|
|
|
|
/**
|
|
* Spec §25–§28. The bounded, fail-open telemetry pipeline:
|
|
*
|
|
* redact -> fingerprint -> deduplicate -> sample -> buffer
|
|
* -> rate limit -> batch -> transport
|
|
*
|
|
* Every stage has an explicit ceiling, and no stage can throw or reject into
|
|
* the business caller: internal failures become dropped-event counters.
|
|
*
|
|
* A duplicate inside the dedup window is not sent on its own. Its count is held
|
|
* per fingerprint and delivered as `occurrences` on the next flush, or on a
|
|
* flush when the dedup window closes, whichever comes first (D-6).
|
|
*/
|
|
export type TelemetryBatchItem = Readonly<{
|
|
event: TelemetryEvent<unknown>;
|
|
fingerprint: string;
|
|
/** Occurrences of this fingerprint not reported by an earlier item. */
|
|
occurrences: number;
|
|
}>;
|
|
|
|
export interface TelemetryTransport {
|
|
send(batch: readonly TelemetryBatchItem[]): void | Promise<void>;
|
|
}
|
|
|
|
export interface TelemetryPipelineBudget {
|
|
readonly maxBufferedEvents: number;
|
|
readonly maxBufferedBytes: number;
|
|
readonly maxBatchEvents: number;
|
|
readonly maxSendsPerMinute: number;
|
|
readonly dedupWindowMs: number;
|
|
readonly maxFingerprints: number;
|
|
/** Sampling rate for `info`/`warning`; `error`/`critical` are never sampled out. */
|
|
readonly sampleRate: number;
|
|
readonly payload?: TelemetryPayloadBudget;
|
|
}
|
|
|
|
export const DEFAULT_TELEMETRY_PIPELINE_BUDGET: TelemetryPipelineBudget =
|
|
Object.freeze({
|
|
maxBufferedEvents: 200,
|
|
maxBufferedBytes: 256 * 1024,
|
|
maxBatchEvents: 50,
|
|
maxSendsPerMinute: 20,
|
|
dedupWindowMs: 60_000,
|
|
maxFingerprints: 500,
|
|
sampleRate: 1,
|
|
});
|
|
|
|
export type TelemetryPipelineStats = Readonly<{
|
|
accepted: number;
|
|
invalid: number;
|
|
deduplicated: number;
|
|
sampledOut: number;
|
|
dropped: number;
|
|
sends: number;
|
|
failedSends: number;
|
|
buffered: number;
|
|
bufferedBytes: number;
|
|
peakBuffered: number;
|
|
fingerprints: number;
|
|
}>;
|
|
|
|
export interface TelemetryPipeline extends TelemetryPort {
|
|
emit(event: TelemetryEvent<unknown>): Promise<void>;
|
|
flush(): Promise<void>;
|
|
stats(): TelemetryPipelineStats;
|
|
dispose(): void;
|
|
}
|
|
|
|
const RATE_WINDOW_MS = 60_000;
|
|
|
|
function assertPositive(name: string, value: number): void {
|
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
throw new TypeError(`telemetry budget ${name} must be a positive safe integer: ${value}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Spec §27. A rolling one-minute send budget. Several queues that post to the
|
|
* same collector share one instance, so the limit belongs to the collector
|
|
* rather than to each queue (Q-6.9-4).
|
|
*/
|
|
export interface TelemetrySendRate {
|
|
/**
|
|
* Takes a send slot and returns 0, or returns the milliseconds until a slot
|
|
* reopens. `force` always takes one: page exit may exceed the window once.
|
|
*/
|
|
take(force?: boolean): number;
|
|
}
|
|
|
|
export function createTelemetrySendRate(
|
|
maxSendsPerMinute: number,
|
|
now: () => number = () => performance.now(),
|
|
): TelemetrySendRate {
|
|
assertPositive("maxSendsPerMinute", maxSendsPerMinute);
|
|
/** Send instants inside the current window; never longer than the limit. */
|
|
const sendTimes: number[] = [];
|
|
return Object.freeze({
|
|
take(force = false): number {
|
|
const at = now();
|
|
while (sendTimes.length > 0 && at - (sendTimes[0] as number) >= RATE_WINDOW_MS) {
|
|
sendTimes.shift();
|
|
}
|
|
if (!force && sendTimes.length >= maxSendsPerMinute) {
|
|
return Math.max(1, (sendTimes[0] as number) + RATE_WINDOW_MS - at);
|
|
}
|
|
sendTimes.push(at);
|
|
if (sendTimes.length > maxSendsPerMinute) sendTimes.shift();
|
|
return 0;
|
|
},
|
|
});
|
|
}
|
|
|
|
export function createTelemetryPipeline(
|
|
options: Readonly<{
|
|
transport: TelemetryTransport;
|
|
budget?: TelemetryPipelineBudget;
|
|
/** Monotonic clock. */
|
|
now?: () => number;
|
|
/**
|
|
* A send-rate window shared with other queues posting to the same
|
|
* collector. Default: a private one from `budget.maxSendsPerMinute`.
|
|
*/
|
|
sendRate?: TelemetrySendRate;
|
|
/** Schedules a flush soon (default: microtask). */
|
|
schedule?: (callback: () => void) => void;
|
|
/** Schedules a flush after a rate window reopens (default: setTimeout). */
|
|
setTimer?: (callback: () => void, ms: number) => unknown;
|
|
clearTimer?: (handle: unknown) => void;
|
|
}>,
|
|
): TelemetryPipeline {
|
|
const budget = options.budget ?? DEFAULT_TELEMETRY_PIPELINE_BUDGET;
|
|
assertPositive("maxBufferedEvents", budget.maxBufferedEvents);
|
|
assertPositive("maxBufferedBytes", budget.maxBufferedBytes);
|
|
assertPositive("maxBatchEvents", budget.maxBatchEvents);
|
|
assertPositive("maxSendsPerMinute", budget.maxSendsPerMinute);
|
|
const payloadBudget = budget.payload ?? DEFAULT_TELEMETRY_PAYLOAD_BUDGET;
|
|
const now = options.now ?? (() => performance.now());
|
|
const schedule = options.schedule ?? queueMicrotask;
|
|
const setTimer =
|
|
options.setTimer ??
|
|
((callback: () => void, ms: number) => setTimeout(callback, ms));
|
|
const clearTimer =
|
|
options.clearTimer ??
|
|
((handle: unknown) => clearTimeout(handle as ReturnType<typeof setTimeout>));
|
|
|
|
const dedup = new TelemetryDeduplicator({
|
|
maxEntries: budget.maxFingerprints,
|
|
windowMs: budget.dedupWindowMs,
|
|
});
|
|
const sampler = createDeterministicSampler({ rate: budget.sampleRate });
|
|
const buffer = createTelemetryBuffer({
|
|
maxEvents: budget.maxBufferedEvents,
|
|
maxBytes: budget.maxBufferedBytes,
|
|
// An evicted event was never delivered; it must not keep suppressing.
|
|
onEvict: (entry) => {
|
|
if (entry.fingerprint !== undefined) dedup.forget(entry.fingerprint);
|
|
},
|
|
});
|
|
/** Duplicates counted but not yet reported, per fingerprint. */
|
|
const held = new Map<string, { event: TelemetryEvent<unknown>; count: number }>();
|
|
const sendRate =
|
|
options.sendRate ?? createTelemetrySendRate(budget.maxSendsPerMinute, now);
|
|
|
|
let disposed = false;
|
|
let flushScheduled = false;
|
|
let rateTimer: unknown = null;
|
|
let windowTimer: unknown = null;
|
|
/** An explicit flush or a closed dedup window asks for held counts. */
|
|
let heldDue = false;
|
|
let activeFlush: Promise<void> | null = null;
|
|
const counters = {
|
|
accepted: 0,
|
|
invalid: 0,
|
|
deduplicated: 0,
|
|
sampledOut: 0,
|
|
sends: 0,
|
|
failedSends: 0,
|
|
peakBuffered: 0,
|
|
};
|
|
|
|
function armRateTimer(reopenIn: number): void {
|
|
if (rateTimer !== null || disposed) return;
|
|
rateTimer = setTimer(() => {
|
|
rateTimer = null;
|
|
void flush();
|
|
}, reopenIn);
|
|
}
|
|
|
|
function scheduleFlush(): void {
|
|
if (flushScheduled || disposed) return;
|
|
flushScheduled = true;
|
|
schedule(() => {
|
|
flushScheduled = false;
|
|
void flush();
|
|
});
|
|
}
|
|
|
|
function hold(event: TelemetryEvent<unknown>, fingerprint: string): void {
|
|
const entry = held.get(fingerprint);
|
|
if (entry) {
|
|
entry.count += 1;
|
|
return;
|
|
}
|
|
// Bounded like the dedup table; past it a duplicate is only counted in stats.
|
|
if (held.size >= budget.maxFingerprints) return;
|
|
held.set(fingerprint, { event, count: 1 });
|
|
if (windowTimer !== null || disposed) return;
|
|
windowTimer = setTimer(() => {
|
|
windowTimer = null;
|
|
void flushHeld();
|
|
}, budget.dedupWindowMs);
|
|
}
|
|
|
|
/**
|
|
* Held counts ride along with buffered events for free. They get a batch of
|
|
* their own only when due, so a duplicate never triggers a send by itself.
|
|
*/
|
|
function pending(): boolean {
|
|
return buffer.size > 0 || (heldDue && held.size > 0);
|
|
}
|
|
|
|
function takeBatch(): TelemetryBatchItem[] {
|
|
const batch: TelemetryBatchItem[] = buffer.take(budget.maxBatchEvents).map((entry) => {
|
|
const fingerprint = entry.fingerprint ?? "";
|
|
const duplicates = held.get(fingerprint)?.count ?? 0;
|
|
held.delete(fingerprint);
|
|
return Object.freeze({ event: entry.event, fingerprint, occurrences: 1 + duplicates });
|
|
});
|
|
for (const [fingerprint, entry] of held) {
|
|
if (batch.length >= budget.maxBatchEvents) break;
|
|
held.delete(fingerprint);
|
|
batch.push(Object.freeze({ event: entry.event, fingerprint, occurrences: entry.count }));
|
|
}
|
|
if (held.size === 0) {
|
|
heldDue = false;
|
|
if (windowTimer !== null) clearTimer(windowTimer);
|
|
windowTimer = null;
|
|
}
|
|
return batch;
|
|
}
|
|
|
|
async function drain(): Promise<void> {
|
|
while (!disposed && pending()) {
|
|
const reopenIn = sendRate.take();
|
|
if (reopenIn > 0) {
|
|
armRateTimer(reopenIn);
|
|
return;
|
|
}
|
|
const batch = takeBatch();
|
|
counters.sends += 1;
|
|
try {
|
|
await options.transport.send(batch);
|
|
} catch {
|
|
// Best effort: a failed batch is counted, never retried unboundedly.
|
|
counters.failedSends += batch.length;
|
|
}
|
|
}
|
|
}
|
|
|
|
function flush(): Promise<void> {
|
|
if (activeFlush) return activeFlush;
|
|
if (disposed || !pending()) return Promise.resolve();
|
|
activeFlush = drain()
|
|
.catch(() => undefined)
|
|
.finally(() => {
|
|
activeFlush = null;
|
|
});
|
|
return activeFlush;
|
|
}
|
|
|
|
function flushHeld(): Promise<void> {
|
|
heldDue = held.size > 0;
|
|
return flush();
|
|
}
|
|
|
|
function admit(event: TelemetryEvent<unknown>): void {
|
|
if (disposed) return;
|
|
if (!isTelemetryEnvelopeValid(event)) {
|
|
counters.invalid += 1;
|
|
return;
|
|
}
|
|
// 1. redact (before anything is retained)
|
|
const redacted: TelemetryEvent<unknown> = Object.freeze({
|
|
kind: event.kind,
|
|
severity: event.severity,
|
|
...(event.contextId === undefined ? {} : { contextId: event.contextId }),
|
|
...(event.featureId === undefined ? {} : { featureId: event.featureId }),
|
|
...(event.interactionId === undefined
|
|
? {}
|
|
: { interactionId: event.interactionId }),
|
|
...(event.traceId === undefined ? {} : { traceId: event.traceId }),
|
|
...(event.payload === undefined
|
|
? {}
|
|
: { payload: redactTelemetryPayload(event.payload, payloadBudget) }),
|
|
});
|
|
// 2. fingerprint 3. deduplicate
|
|
const fingerprint = fingerprintTelemetryEvent(redacted);
|
|
if (!dedup.record(fingerprint, now())) {
|
|
counters.deduplicated += 1;
|
|
hold(redacted, fingerprint);
|
|
return;
|
|
}
|
|
// 4. sample (deterministic by interaction; severe events always kept)
|
|
if (
|
|
TELEMETRY_SEVERITY_RANK[redacted.severity] < TELEMETRY_SEVERITY_RANK.error &&
|
|
!sampler.keep(redacted.interactionId ?? fingerprint)
|
|
) {
|
|
counters.sampledOut += 1;
|
|
// Its duplicates must not be reported as held occurrences.
|
|
dedup.forget(fingerprint);
|
|
return;
|
|
}
|
|
// 5. buffer
|
|
if (buffer.push(redacted, estimateTelemetryBytes(redacted), fingerprint)) {
|
|
counters.accepted += 1;
|
|
counters.peakBuffered = Math.max(counters.peakBuffered, buffer.size);
|
|
} else {
|
|
dedup.forget(fingerprint);
|
|
}
|
|
// 6-8. rate limit, batch and transport happen on flush.
|
|
scheduleFlush();
|
|
}
|
|
|
|
return Object.freeze({
|
|
emit(event: TelemetryEvent<unknown>): Promise<void> {
|
|
try {
|
|
admit(event);
|
|
} catch {
|
|
counters.invalid += 1;
|
|
}
|
|
return Promise.resolve();
|
|
},
|
|
flush: flushHeld,
|
|
stats(): TelemetryPipelineStats {
|
|
return Object.freeze({
|
|
accepted: counters.accepted,
|
|
invalid: counters.invalid,
|
|
deduplicated: counters.deduplicated,
|
|
sampledOut: counters.sampledOut,
|
|
dropped: buffer.dropped + counters.failedSends + counters.invalid,
|
|
sends: counters.sends,
|
|
failedSends: counters.failedSends,
|
|
buffered: buffer.size,
|
|
bufferedBytes: buffer.bytes,
|
|
peakBuffered: counters.peakBuffered,
|
|
fingerprints: dedup.size,
|
|
});
|
|
},
|
|
dispose(): void {
|
|
if (disposed) return;
|
|
disposed = true;
|
|
if (rateTimer !== null) clearTimer(rateTimer);
|
|
rateTimer = null;
|
|
if (windowTimer !== null) clearTimer(windowTimer);
|
|
windowTimer = null;
|
|
buffer.clear();
|
|
held.clear();
|
|
},
|
|
});
|
|
}
|