Files
clean-architecture-frontend…/tests/scale/traffic/failure-scenarios.test.ts
T

166 lines
5.4 KiB
TypeScript

import { afterEach, describe, expect, it, vi } from "vitest";
import {
manualClock,
scopeSnapshot,
testExecutionContext,
testExecutor,
type Transport,
} from "../../helpers/traffic-test-kit.ts";
import {
requestAmplification,
retryAmplification,
} from "./amplification-metrics.ts";
import { runSyntheticTraffic, TEST_QUERY, upstream } from "./synthetic-traffic.ts";
afterEach(() => {
vi.useRealTimers();
});
describe("total outage", () => {
it("bounds physical requests when all upstream calls fail", async () => {
const result = await runSyntheticTraffic({
interactions: 10_000,
upstream: "always-fail",
maxPhysicalRequestsPerInteraction: 2,
maxRetriesPerInteraction: 1,
});
expect(result.physicalRequests).toBeLessThanOrEqual(20_000);
expect(result.succeeded).toBe(0);
expect(requestAmplification(result)).toBeLessThanOrEqual(2);
expect(
retryAmplification({
initialRequests: result.initialRequests,
totalRequests: result.physicalRequests,
}),
).toBeLessThanOrEqual(2);
}, 60_000);
it("keeps amplification bounded when one interaction fans out into many operations", async () => {
const result = await runSyntheticTraffic({
interactions: 1_000,
upstream: "always-fail",
maxPhysicalRequestsPerInteraction: 3,
maxRetriesPerInteraction: 1,
operationsPerInteraction: 6,
});
// Unbudgeted: 6 operations x 3 attempts = 18 per interaction.
expect(requestAmplification(result)).toBeLessThanOrEqual(3);
}, 60_000);
it("stays at 1x amplification when upstream is healthy", async () => {
const result = await runSyntheticTraffic({
interactions: 2_000,
upstream: "healthy",
maxPhysicalRequestsPerInteraction: 2,
maxRetriesPerInteraction: 1,
});
expect(result.succeeded).toBe(2_000);
expect(requestAmplification(result)).toBe(1);
}, 60_000);
});
describe("partial outage", () => {
it("lets healthy contexts complete while one context fails completely", async () => {
const transports = {
payment: upstream("always-fail"),
order: upstream("healthy"),
catalog: upstream("healthy"),
};
const executors = Object.fromEntries(
Object.entries(transports).map(([contextId, transport]) => [
contextId,
testExecutor(transport, { maxRetryAttempts: 2 }),
]),
) as Record<keyof typeof transports, ReturnType<typeof testExecutor>>;
const successes: Record<string, number> = { payment: 0, order: 0, catalog: 0 };
await Promise.all(
Array.from({ length: 300 }, async (_, index) => {
const contextId = (["payment", "order", "catalog"] as const)[index % 3] as
| "payment"
| "order"
| "catalog";
const execution = testExecutionContext({ maxPhysicalRequests: 2, maxRetries: 1 });
const outcome = await executors[contextId].execute(TEST_QUERY, { limit: 1 }, {
routeId: "TEST_ROUTE",
scope: scopeSnapshot(),
execution,
});
if (outcome.kind === "SUCCESS") successes[contextId] = (successes[contextId] ?? 0) + 1;
}),
);
expect(successes).toEqual({ payment: 0, order: 100, catalog: 100 });
expect(transports.payment.calls).toBeLessThanOrEqual(200);
expect(transports.order.calls).toBe(100);
expect(transports.catalog.calls).toBe(100);
});
});
describe("slow upstream", () => {
it("bounds active and queued work, drains waiters at the deadline, and never retries after it", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const clock = manualClock(0);
let active = 0;
let peakActive = 0;
let calls = 0;
const transport: Transport = (_input, init) => {
calls += 1;
active += 1;
peakActive = Math.max(peakActive, active);
return new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener(
"abort",
() => {
active -= 1;
reject(new DOMException("aborted", "AbortError"));
},
{ once: true },
);
});
};
const execution = testExecutionContext({
deadlineMs: 1_000,
maxPhysicalRequests: 20,
maxRetries: 5,
maxConcurrentRequests: 2,
maxQueuedRequests: 3,
clock: clock.now,
});
const executor = testExecutor(transport, { monotonicNow: clock.now });
const context = { routeId: "TEST_ROUTE", scope: scopeSnapshot(), execution };
const pending = Array.from({ length: 10 }, () =>
executor.execute(TEST_QUERY, { limit: 1 }, context),
);
await vi.advanceTimersByTimeAsync(0);
expect(execution.traffic.concurrency.active).toBe(2);
expect(execution.traffic.concurrency.queued).toBe(3);
clock.advance(1_000);
await vi.advanceTimersByTimeAsync(1_000);
const outcomes = await Promise.all(pending);
expect(peakActive).toBeLessThanOrEqual(2);
expect(execution.traffic.concurrency.active).toBe(0);
expect(execution.traffic.concurrency.queued).toBe(0);
expect(calls).toBe(2);
expect(execution.traffic.retries.consumed).toBe(0);
expect(
outcomes.filter(
(outcome) =>
outcome.kind === "TRANSPORT_FAILURE" &&
outcome.failure.kind === "OVERLOADED_BEFORE_SEND",
),
).toHaveLength(5);
expect(
outcomes.filter(
(outcome) =>
outcome.kind === "TRANSPORT_FAILURE" && outcome.failure.kind === "TIMEOUT",
),
).toHaveLength(5);
});
});