Files
clean-architecture-frontend…/scripts/scale/browser-soak.ts

555 lines
20 KiB
TypeScript

import { createReadStream } from "node:fs";
import { access, mkdir, readFile, stat, writeFile } from "node:fs/promises";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import path from "node:path";
import {
classifyCounterSeries,
classifyMemorySlope,
type MemorySlopeClass,
} from "./memory-slope.ts";
/**
* Spec §33 / §44. Browser soak on the existing Playwright runner.
*
* Serves the built `dist/` on an OS-assigned 127.0.0.1 port and keeps one
* long-lived page cycling through the app's routes by client-side navigation.
* The served runtime config turns telemetry on (pointed at a local collector)
* and points the API at a stub that always answers 503, so every cycle also
* exercises bounded retries and telemetry under a sustained backend failure.
*
* Collected per sample: JS heap, DOM nodes, event listeners, live timers,
* in-flight fetches, Web Storage size. Collected overall: cycles, navigation
* failures, API requests, telemetry POSTs/events. Without a build or browser
* the result is `skipped` (never a pass); too little evidence fails.
*/
export type SoakSample = Readonly<{
elapsedMs: number;
cycles: number;
heapBytes: number;
domNodes: number;
listeners: number;
activeTimers: number;
inFlightRequests: number;
storageKeys: number;
storageBytes: number;
}>;
export type SoakSummary = Readonly<{
cycles: number;
navigationFailures: number;
maxHeapBytes: number;
heapClassification: MemorySlopeClass;
listenerDelta: number;
listenerClassification: MemorySlopeClass;
timerDelta: number;
maxActiveTimers: number;
domNodeDelta: number;
maxInFlightRequests: number;
finalInFlightRequests: number;
storageKeyDelta: number;
storageByteDelta: number;
apiRequests: number;
referenceVisits: number;
settledReferenceVisits: number;
apiRequestsPerReferenceVisit: number;
telemetryPosts: number;
telemetryEvents: number;
maxTelemetryPostsPerMinute: number;
}>;
export type SoakCheck = Readonly<{ name: string; passed: boolean; detail: string }>;
export type SoakArtifact = Readonly<{
schemaVersion: 2;
status: "passed" | "failed" | "skipped";
reason?: string;
durationMs: number;
classification?: MemorySlopeClass;
summary?: SoakSummary;
checks?: readonly SoakCheck[];
/** Identity of the soaked build (from dist/release-manifest.json). */
build?: Readonly<{ buildId?: string; releaseId?: string; builtAt?: string }>;
browser?: string;
samples: readonly SoakSample[];
}>;
/** Thresholds a soak must meet; each is checked and recorded by name. */
export const SOAK_LIMITS = Object.freeze({
minCycles: 200,
minPostWarmupSamples: 3,
maxListenerDelta: 50,
maxTimerDelta: 10,
maxStorageKeyDelta: 2,
/** One logical execution: 1 attempt + at most 2 retries. */
maxApiRequestsPerReferenceVisit: 3,
/** Two bounded pipelines (platform events + feature events) x 20/min. */
maxTelemetryPostsPerMinute: 40,
});
export function summarizeSoak(
samples: readonly Pick<SoakSample, "elapsedMs" | "heapBytes">[],
warmupMs: number,
): MemorySlopeClass {
return classifyMemorySlope(
samples.map((sample) => ({
minute: sample.elapsedMs / 60_000,
heapMb: sample.heapBytes / (1024 * 1024),
})),
{ warmupMinutes: warmupMs / 60_000, maxSlopeMbPerMinute: 1, maxRelativeGrowth: 0.2 },
);
}
export type SoakTotals = Readonly<{
cycles: number;
navigationFailures: number;
referenceVisits: number;
settledReferenceVisits: number;
finalInFlightRequests: number;
apiRequests: number;
telemetryPosts: number;
telemetryEvents: number;
telemetryPostTimes: readonly number[];
}>;
export function evaluateSoak(
samples: readonly SoakSample[],
totals: SoakTotals,
warmupMs: number,
): Readonly<{ summary: SoakSummary; checks: readonly SoakCheck[] }> {
const measured = samples.filter((sample) => sample.elapsedMs >= warmupMs);
const first = measured[0];
const last = measured[measured.length - 1];
const delta = (pick: (sample: SoakSample) => number) =>
first && last ? pick(last) - pick(first) : 0;
let maxTelemetryPostsPerMinute = 0;
for (const time of totals.telemetryPostTimes) {
const inWindow = totals.telemetryPostTimes.filter(
(other) => other >= time && other < time + 60_000,
).length;
maxTelemetryPostsPerMinute = Math.max(maxTelemetryPostsPerMinute, inWindow);
}
const summary: SoakSummary = {
cycles: totals.cycles,
navigationFailures: totals.navigationFailures,
maxHeapBytes: Math.max(0, ...samples.map((sample) => sample.heapBytes)),
heapClassification: summarizeSoak(samples, warmupMs),
listenerDelta: delta((sample) => sample.listeners),
listenerClassification: classifyCounterSeries(
measured.map((sample) => sample.listeners),
{ maxSlopePerSample: 1 },
),
timerDelta: delta((sample) => sample.activeTimers),
maxActiveTimers: Math.max(0, ...samples.map((sample) => sample.activeTimers)),
domNodeDelta: delta((sample) => sample.domNodes),
maxInFlightRequests: Math.max(0, ...samples.map((sample) => sample.inFlightRequests)),
finalInFlightRequests: totals.finalInFlightRequests,
storageKeyDelta: delta((sample) => sample.storageKeys),
storageByteDelta: delta((sample) => sample.storageBytes),
apiRequests: totals.apiRequests,
referenceVisits: totals.referenceVisits,
settledReferenceVisits: totals.settledReferenceVisits,
apiRequestsPerReferenceVisit:
totals.referenceVisits === 0
? 0
: Math.round((totals.apiRequests / totals.referenceVisits) * 1_000) / 1_000,
telemetryPosts: totals.telemetryPosts,
telemetryEvents: totals.telemetryEvents,
maxTelemetryPostsPerMinute,
};
const checks: SoakCheck[] = [
{ name: "cycles", passed: summary.cycles >= SOAK_LIMITS.minCycles, detail: `${summary.cycles} >= ${SOAK_LIMITS.minCycles}` },
{
name: "post-warmup samples",
passed: measured.length >= SOAK_LIMITS.minPostWarmupSamples,
detail: `${measured.length} >= ${SOAK_LIMITS.minPostWarmupSamples}`,
},
{ name: "navigation", passed: summary.navigationFailures === 0, detail: `${summary.navigationFailures} failures` },
{ name: "heap slope", passed: summary.heapClassification === "stable", detail: summary.heapClassification },
{
name: "listeners",
passed:
summary.listenerClassification === "stable" &&
summary.listenerDelta <= SOAK_LIMITS.maxListenerDelta,
detail: `${summary.listenerClassification}, delta ${summary.listenerDelta}`,
},
{
name: "timers",
passed: summary.timerDelta <= SOAK_LIMITS.maxTimerDelta,
detail: `delta ${summary.timerDelta}, max ${summary.maxActiveTimers}`,
},
{
name: "in-flight settles",
passed: summary.finalInFlightRequests === 0,
detail: `final ${summary.finalInFlightRequests}, max ${summary.maxInFlightRequests}`,
},
{
name: "storage growth",
passed: summary.storageKeyDelta <= SOAK_LIMITS.maxStorageKeyDelta,
detail: `keys ${summary.storageKeyDelta}, bytes ${summary.storageByteDelta}`,
},
{
name: "backend failure exercised",
passed:
summary.settledReferenceVisits > 0 && summary.apiRequests > 0 && summary.telemetryEvents > 0,
detail: `${summary.settledReferenceVisits} settled visits, ${summary.apiRequests} API requests, ${summary.telemetryEvents} telemetry events`,
},
{
name: "request amplification",
passed:
summary.referenceVisits > 0 &&
summary.apiRequestsPerReferenceVisit <= SOAK_LIMITS.maxApiRequestsPerReferenceVisit,
detail: `${summary.apiRequests} requests / ${summary.referenceVisits} visits`,
},
{
name: "telemetry rate",
passed: summary.maxTelemetryPostsPerMinute <= SOAK_LIMITS.maxTelemetryPostsPerMinute,
detail: `max ${summary.maxTelemetryPostsPerMinute}/min, ${summary.telemetryPosts} posts, ${summary.telemetryEvents} events`,
},
];
return { summary, checks };
}
const CONTENT_TYPES: Readonly<Record<string, string>> = {
".css": "text/css; charset=utf-8",
".html": "text/html; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".json": "application/json; charset=utf-8",
".svg": "image/svg+xml",
".png": "image/png",
".webmanifest": "application/manifest+json",
};
type Counters = {
apiRequests: number;
telemetryPosts: number;
telemetryEvents: number;
telemetryPostTimes: number[];
};
async function serveSoakTarget(
root: string,
counters: Counters,
startedAt: () => number,
): Promise<Readonly<{ server: Server; baseUrl: string }>> {
let baseUrl = "";
const baseConfig = JSON.parse(
await readFile(path.join(root, "config.json"), "utf8"),
) as Record<string, unknown>;
const server = createServer(async (request, response) => {
try {
const url = new URL(request.url ?? "/", "http://127.0.0.1");
if (url.pathname === "/__telemetry") {
let body = "";
request.on("data", (chunk: Buffer) => {
body += chunk.toString("utf8");
});
request.on("end", () => {
counters.telemetryPosts += 1;
counters.telemetryPostTimes.push(Date.now() - startedAt());
try {
const parsed = JSON.parse(body) as { events?: unknown[] };
counters.telemetryEvents += Array.isArray(parsed.events) ? parsed.events.length : 0;
} catch {
// Counted as a post regardless of its body.
}
response.writeHead(204).end();
});
return;
}
if (url.pathname.startsWith("/api/")) {
counters.apiRequests += 1;
response.writeHead(503, { "Content-Type": "application/json" }).end("{}");
return;
}
if (url.pathname === "/config.json") {
response
.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" })
.end(
JSON.stringify({
...baseConfig,
API_BASE_URL: `${baseUrl}/api/`,
TELEMETRY_ENABLED: true,
TELEMETRY_ENDPOINT: `${baseUrl}/__telemetry`,
}),
);
return;
}
const requested = path.resolve(root, `.${decodeURIComponent(url.pathname)}`);
if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) {
response.writeHead(403).end();
return;
}
const details = await stat(requested).catch(() => null);
const file = details?.isFile() ? requested : path.join(root, "index.html");
response.writeHead(200, {
"Content-Type": CONTENT_TYPES[path.extname(file)] ?? "application/octet-stream",
"Cache-Control": "no-store",
});
createReadStream(file).pipe(response);
} catch {
response.writeHead(404).end();
}
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const { port } = server.address() as AddressInfo;
baseUrl = `http://127.0.0.1:${port}`;
return { server, baseUrl };
}
const PAGE_PROBES = `
(() => {
const live = new Set();
const setT = window.setTimeout, clearT = window.clearTimeout;
const setI = window.setInterval, clearI = window.clearInterval;
window.setTimeout = (fn, ms, ...rest) => {
const id = setT(() => { live.delete(id); typeof fn === "function" && fn(...rest); }, ms);
live.add(id); return id;
};
window.clearTimeout = (id) => { live.delete(id); clearT(id); };
window.setInterval = (fn, ms, ...rest) => { const id = setI(fn, ms, ...rest); live.add(id); return id; };
window.clearInterval = (id) => { live.delete(id); clearI(id); };
let inFlight = 0;
const nativeFetch = window.fetch.bind(window);
window.fetch = (...args) => {
inFlight += 1;
return nativeFetch(...args).finally(() => { inFlight -= 1; });
};
const storageSize = (storage) => {
let bytes = 0;
for (let i = 0; i < storage.length; i += 1) {
const key = storage.key(i) ?? "";
bytes += key.length + (storage.getItem(key) ?? "").length;
}
return { keys: storage.length, bytes };
};
Object.defineProperty(window, "__soakProbe", {
value: () => {
const local = storageSize(window.localStorage);
const session = storageSize(window.sessionStorage);
return {
activeTimers: live.size,
inFlight,
storageKeys: local.keys + session.keys,
storageBytes: local.bytes + session.bytes,
};
},
});
})();
`;
type Probe = { activeTimers: number; inFlight: number; storageKeys: number; storageBytes: number };
export async function runBrowserSoak(
options: Readonly<{
distRoot: string;
durationMs: number;
intervalMs: number;
warmupMs: number;
routes: readonly string[];
}>,
): Promise<SoakArtifact> {
try {
await access(path.join(options.distRoot, "index.html"));
await access(path.join(options.distRoot, "config.json"));
} catch {
return {
schemaVersion: 2,
status: "skipped",
reason: "no built dist/index.html + config.json",
durationMs: 0,
samples: [],
};
}
let chromium: (typeof import("@playwright/test"))["chromium"];
try {
({ chromium } = await import("@playwright/test"));
} catch {
return { schemaVersion: 2, status: "skipped", reason: "playwright unavailable", durationMs: 0, samples: [] };
}
const counters: Counters = {
apiRequests: 0,
telemetryPosts: 0,
telemetryEvents: 0,
telemetryPostTimes: [],
};
let started = Date.now();
const { server, baseUrl } = await serveSoakTarget(
path.resolve(options.distRoot),
counters,
() => started,
);
let browser: Awaited<ReturnType<typeof chromium.launch>> | undefined;
try {
try {
browser = await chromium.launch();
} catch (error) {
return {
schemaVersion: 2,
status: "skipped",
reason: `browser unavailable: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`,
durationMs: 0,
samples: [],
};
}
const page = await browser.newPage();
await page.addInitScript(PAGE_PROBES);
const cdp = await page.context().newCDPSession(page);
await cdp.send("Performance.enable");
await page.goto(`${baseUrl}/examples/reference-resources`, { waitUntil: "load" });
await page.waitForSelector("#root > *", { timeout: 15_000 });
// Protected routes need a session: start the local demo session once, so
// the reference feature really dispatches its (failing) API requests.
const login = page.getByRole("button", { name: /로그인 시작|Sign in|Start sign-in/u });
if (await login.count()) {
await login.first().click();
await page.waitForTimeout(500);
}
const samples: SoakSample[] = [];
started = Date.now();
let nextSample = 0;
let cycles = 0;
let navigationFailures = 0;
let referenceVisits = 0;
let settledReferenceVisits = 0;
const probe = async () =>
(await page.evaluate(() =>
(window as unknown as { __soakProbe: () => Probe }).__soakProbe(),
)) as Probe;
while (Date.now() - started < options.durationMs) {
const route = options.routes[cycles % options.routes.length] ?? "/";
cycles += 1;
if (route === "/examples/reference-resources") referenceVisits += 1;
// Client-side navigation keeps one long-lived document, like a real session.
await page.evaluate((target) => {
window.history.pushState({}, "", target);
window.dispatchEvent(new PopStateEvent("popstate"));
}, route);
await page.waitForTimeout(50);
const rendered = await page.evaluate(
(target) =>
window.location.pathname === target &&
(document.getElementById("root")?.childElementCount ?? 0) > 0,
route,
);
if (!rendered) navigationFailures += 1;
// Most visits leave after 50ms, which cancels the query (a cancelled
// request is intentionally not reported). Every 5th reference visit
// dwells until its interaction settles, so the failing backend is really
// observed end to end: bounded retries, then feature/platform telemetry.
if (route === "/examples/reference-resources" && referenceVisits % 5 === 1) {
await page.waitForTimeout(200);
await page
.waitForFunction(
() => (window as unknown as { __soakProbe: () => Probe }).__soakProbe().inFlight === 0,
undefined,
{ timeout: 5_000 },
)
.catch(() => undefined);
settledReferenceVisits += 1;
}
const elapsedMs = Date.now() - started;
if (elapsedMs >= nextSample) {
nextSample = elapsedMs + options.intervalMs;
await cdp.send("HeapProfiler.collectGarbage").catch(() => undefined);
const { metrics } = (await cdp.send("Performance.getMetrics")) as {
metrics: { name: string; value: number }[];
};
const metric = (name: string) => metrics.find((entry) => entry.name === name)?.value ?? 0;
const probed = await probe();
samples.push({
elapsedMs,
cycles,
heapBytes: metric("JSHeapUsedSize"),
domNodes: metric("Nodes"),
listeners: metric("JSEventListeners"),
activeTimers: probed.activeTimers,
inFlightRequests: probed.inFlight,
storageKeys: probed.storageKeys,
storageBytes: probed.storageBytes,
});
}
}
// Let the last interaction's bounded retries and deadlines settle.
let finalInFlightRequests = (await probe()).inFlight;
for (let wait = 0; wait < 30 && finalInFlightRequests > 0; wait += 1) {
await page.waitForTimeout(500);
finalInFlightRequests = (await probe()).inFlight;
}
const evaluation = evaluateSoak(
samples,
{
cycles,
navigationFailures,
referenceVisits,
settledReferenceVisits,
finalInFlightRequests,
apiRequests: counters.apiRequests,
telemetryPosts: counters.telemetryPosts,
telemetryEvents: counters.telemetryEvents,
telemetryPostTimes: counters.telemetryPostTimes,
},
options.warmupMs,
);
const build = await readFile(path.join(options.distRoot, "release-manifest.json"), "utf8")
.then((raw) => {
const manifest = JSON.parse(raw) as Record<string, unknown>;
const pick = (key: string) =>
typeof manifest[key] === "string" ? (manifest[key] as string) : undefined;
return {
...(pick("buildId") ? { buildId: pick("buildId") } : {}),
...(pick("releaseId") ? { releaseId: pick("releaseId") } : {}),
...(pick("builtAt") ? { builtAt: pick("builtAt") } : {}),
};
})
.catch(() => ({}));
return {
schemaVersion: 2,
status: evaluation.checks.every((check) => check.passed) ? "passed" : "failed",
durationMs: options.durationMs,
classification: evaluation.summary.heapClassification,
summary: evaluation.summary,
checks: evaluation.checks,
build,
browser: `chromium ${browser.version()}`,
samples,
};
} finally {
await browser?.close().catch(() => undefined);
await new Promise<void>((resolve) => server.close(() => resolve()));
}
}
async function main(): Promise<void> {
const argument = (name: string, fallback: string) =>
process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;
const artifact = path.resolve(argument("artifact", "artifacts/scale/browser-soak.json"));
const result = await runBrowserSoak({
distRoot: argument("dist", "dist"),
durationMs: Number(argument("duration-ms", "60000")),
intervalMs: Number(argument("interval-ms", "5000")),
warmupMs: Number(argument("warmup-ms", "15000")),
routes: argument(
"routes",
"/,/examples/reference-resources,/examples/platform,/examples/ui,/examples/states,/examples/reference-resources/status",
).split(","),
});
await mkdir(path.dirname(artifact), { recursive: true });
await writeFile(artifact, `${JSON.stringify(result, null, 2)}\n`);
process.stdout.write(
`browser soak: ${result.status}${result.reason ? ` (${result.reason})` : ""}` +
`${result.classification ? ` classification=${result.classification}` : ""} samples=${result.samples.length}` +
`${result.summary ? ` cycles=${result.summary.cycles}` : ""}\n`,
);
for (const check of result.checks ?? []) {
process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`);
}
if (result.status === "failed") process.exitCode = 1;
}
if (import.meta.url === `file://${process.argv[1]}`) {
await main();
}