import { createReadStream } from "node:fs"; import { access, mkdir, readFile, stat, writeFile } from "node:fs/promises"; import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import path from "node:path"; import { classifyCounterSeries, classifyMemorySlope, type MemorySlopeClass, } from "./memory-slope.ts"; /** * scale-architecture.md §33 / §44. Browser soak on the existing Playwright runner. * * Serves the built `dist/` on an OS-assigned 127.0.0.1 port and keeps one * long-lived page cycling through the app's routes by client-side navigation. * The served runtime config turns telemetry on (pointed at a local collector) * and points the API at a stub that always answers 503, so every cycle also * exercises bounded retries and telemetry under a sustained backend failure. * * Collected per sample: JS heap, DOM nodes, event listeners, live timers, * in-flight fetches, Web Storage size. Collected overall: cycles, navigation * failures, API requests, telemetry POSTs/events. Without a build or browser * the result is `skipped` (never a pass); too little evidence fails. */ export type SoakSample = Readonly<{ elapsedMs: number; cycles: number; heapBytes: number; domNodes: number; listeners: number; activeTimers: number; inFlightRequests: number; storageKeys: number; storageBytes: number; }>; export type SoakSummary = Readonly<{ cycles: number; navigationFailures: number; maxHeapBytes: number; heapClassification: MemorySlopeClass; listenerDelta: number; listenerClassification: MemorySlopeClass; timerDelta: number; maxActiveTimers: number; domNodeDelta: number; maxInFlightRequests: number; finalInFlightRequests: number; storageKeyDelta: number; storageByteDelta: number; apiRequests: number; referenceVisits: number; settledReferenceVisits: number; apiRequestsPerReferenceVisit: number; telemetryPosts: number; telemetryEvents: number; maxTelemetryPostsPerMinute: number; }>; export type SoakCheck = Readonly<{ name: string; passed: boolean; detail: string }>; export type SoakArtifact = Readonly<{ schemaVersion: 2; status: "passed" | "failed" | "skipped"; reason?: string; durationMs: number; classification?: MemorySlopeClass; summary?: SoakSummary; checks?: readonly SoakCheck[]; /** Identity of the soaked build (from dist/release-manifest.json). */ build?: Readonly<{ buildId?: string; releaseId?: string; builtAt?: string }>; browser?: string; samples: readonly SoakSample[]; }>; /** Thresholds a soak must meet; each is checked and recorded by name. */ export const SOAK_LIMITS = Object.freeze({ minCycles: 200, minPostWarmupSamples: 3, maxListenerDelta: 50, maxTimerDelta: 10, maxStorageKeyDelta: 2, /** One logical execution: 1 attempt + at most 2 retries. */ maxApiRequestsPerReferenceVisit: 3, /** Two bounded pipelines (platform events + feature events) x 20/min. */ maxTelemetryPostsPerMinute: 40, }); export function summarizeSoak( samples: readonly Pick[], warmupMs: number, ): MemorySlopeClass { return classifyMemorySlope( samples.map((sample) => ({ minute: sample.elapsedMs / 60_000, heapMb: sample.heapBytes / (1024 * 1024), })), { warmupMinutes: warmupMs / 60_000, maxSlopeMbPerMinute: 1, maxRelativeGrowth: 0.2 }, ); } export type SoakTotals = Readonly<{ cycles: number; navigationFailures: number; referenceVisits: number; settledReferenceVisits: number; finalInFlightRequests: number; apiRequests: number; telemetryPosts: number; telemetryEvents: number; telemetryPostTimes: readonly number[]; }>; export function evaluateSoak( samples: readonly SoakSample[], totals: SoakTotals, warmupMs: number, ): Readonly<{ summary: SoakSummary; checks: readonly SoakCheck[] }> { const measured = samples.filter((sample) => sample.elapsedMs >= warmupMs); const first = measured[0]; const last = measured[measured.length - 1]; const delta = (pick: (sample: SoakSample) => number) => first && last ? pick(last) - pick(first) : 0; let maxTelemetryPostsPerMinute = 0; for (const time of totals.telemetryPostTimes) { const inWindow = totals.telemetryPostTimes.filter( (other) => other >= time && other < time + 60_000, ).length; maxTelemetryPostsPerMinute = Math.max(maxTelemetryPostsPerMinute, inWindow); } const summary: SoakSummary = { cycles: totals.cycles, navigationFailures: totals.navigationFailures, maxHeapBytes: Math.max(0, ...samples.map((sample) => sample.heapBytes)), heapClassification: summarizeSoak(samples, warmupMs), listenerDelta: delta((sample) => sample.listeners), listenerClassification: classifyCounterSeries( measured.map((sample) => sample.listeners), { maxSlopePerSample: 1 }, ), timerDelta: delta((sample) => sample.activeTimers), maxActiveTimers: Math.max(0, ...samples.map((sample) => sample.activeTimers)), domNodeDelta: delta((sample) => sample.domNodes), maxInFlightRequests: Math.max(0, ...samples.map((sample) => sample.inFlightRequests)), finalInFlightRequests: totals.finalInFlightRequests, storageKeyDelta: delta((sample) => sample.storageKeys), storageByteDelta: delta((sample) => sample.storageBytes), apiRequests: totals.apiRequests, referenceVisits: totals.referenceVisits, settledReferenceVisits: totals.settledReferenceVisits, apiRequestsPerReferenceVisit: totals.referenceVisits === 0 ? 0 : Math.round((totals.apiRequests / totals.referenceVisits) * 1_000) / 1_000, telemetryPosts: totals.telemetryPosts, telemetryEvents: totals.telemetryEvents, maxTelemetryPostsPerMinute, }; const checks: SoakCheck[] = [ { name: "cycles", passed: summary.cycles >= SOAK_LIMITS.minCycles, detail: `${summary.cycles} >= ${SOAK_LIMITS.minCycles}` }, { name: "post-warmup samples", passed: measured.length >= SOAK_LIMITS.minPostWarmupSamples, detail: `${measured.length} >= ${SOAK_LIMITS.minPostWarmupSamples}`, }, { name: "navigation", passed: summary.navigationFailures === 0, detail: `${summary.navigationFailures} failures` }, { name: "heap slope", passed: summary.heapClassification === "stable", detail: summary.heapClassification }, { name: "listeners", passed: summary.listenerClassification === "stable" && summary.listenerDelta <= SOAK_LIMITS.maxListenerDelta, detail: `${summary.listenerClassification}, delta ${summary.listenerDelta}`, }, { name: "timers", passed: summary.timerDelta <= SOAK_LIMITS.maxTimerDelta, detail: `delta ${summary.timerDelta}, max ${summary.maxActiveTimers}`, }, { name: "in-flight settles", passed: summary.finalInFlightRequests === 0, detail: `final ${summary.finalInFlightRequests}, max ${summary.maxInFlightRequests}`, }, { name: "storage growth", passed: summary.storageKeyDelta <= SOAK_LIMITS.maxStorageKeyDelta, detail: `keys ${summary.storageKeyDelta}, bytes ${summary.storageByteDelta}`, }, { name: "backend failure exercised", passed: summary.settledReferenceVisits > 0 && summary.apiRequests > 0 && summary.telemetryEvents > 0, detail: `${summary.settledReferenceVisits} settled visits, ${summary.apiRequests} API requests, ${summary.telemetryEvents} telemetry events`, }, { name: "request amplification", passed: summary.referenceVisits > 0 && summary.apiRequestsPerReferenceVisit <= SOAK_LIMITS.maxApiRequestsPerReferenceVisit, detail: `${summary.apiRequests} requests / ${summary.referenceVisits} visits`, }, { name: "telemetry rate", passed: summary.maxTelemetryPostsPerMinute <= SOAK_LIMITS.maxTelemetryPostsPerMinute, detail: `max ${summary.maxTelemetryPostsPerMinute}/min, ${summary.telemetryPosts} posts, ${summary.telemetryEvents} events`, }, ]; return { summary, checks }; } const CONTENT_TYPES: Readonly> = { ".css": "text/css; charset=utf-8", ".html": "text/html; charset=utf-8", ".js": "text/javascript; charset=utf-8", ".json": "application/json; charset=utf-8", ".svg": "image/svg+xml", ".png": "image/png", ".webmanifest": "application/manifest+json", }; type Counters = { apiRequests: number; telemetryPosts: number; telemetryEvents: number; telemetryPostTimes: number[]; }; async function serveSoakTarget( root: string, counters: Counters, startedAt: () => number, ): Promise> { let baseUrl = ""; const baseConfig = JSON.parse( await readFile(path.join(root, "config.json"), "utf8"), ) as Record; const server = createServer(async (request, response) => { try { const url = new URL(request.url ?? "/", "http://127.0.0.1"); if (url.pathname === "/__telemetry") { let body = ""; request.on("data", (chunk: Buffer) => { body += chunk.toString("utf8"); }); request.on("end", () => { counters.telemetryPosts += 1; counters.telemetryPostTimes.push(Date.now() - startedAt()); try { const parsed = JSON.parse(body) as { events?: unknown[] }; counters.telemetryEvents += Array.isArray(parsed.events) ? parsed.events.length : 0; } catch { // Counted as a post regardless of its body. } response.writeHead(204).end(); }); return; } if (url.pathname.startsWith("/api/")) { counters.apiRequests += 1; response.writeHead(503, { "Content-Type": "application/json" }).end("{}"); return; } if (url.pathname === "/config.json") { response .writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" }) .end( JSON.stringify({ ...baseConfig, API_BASE_URL: `${baseUrl}/api/`, TELEMETRY_ENABLED: true, TELEMETRY_ENDPOINT: `${baseUrl}/__telemetry`, }), ); return; } const requested = path.resolve(root, `.${decodeURIComponent(url.pathname)}`); if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) { response.writeHead(403).end(); return; } const details = await stat(requested).catch(() => null); const file = details?.isFile() ? requested : path.join(root, "index.html"); response.writeHead(200, { "Content-Type": CONTENT_TYPES[path.extname(file)] ?? "application/octet-stream", "Cache-Control": "no-store", }); createReadStream(file).pipe(response); } catch { response.writeHead(404).end(); } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const { port } = server.address() as AddressInfo; baseUrl = `http://127.0.0.1:${port}`; return { server, baseUrl }; } const PAGE_PROBES = ` (() => { const live = new Set(); const setT = window.setTimeout, clearT = window.clearTimeout; const setI = window.setInterval, clearI = window.clearInterval; window.setTimeout = (fn, ms, ...rest) => { const id = setT(() => { live.delete(id); typeof fn === "function" && fn(...rest); }, ms); live.add(id); return id; }; window.clearTimeout = (id) => { live.delete(id); clearT(id); }; window.setInterval = (fn, ms, ...rest) => { const id = setI(fn, ms, ...rest); live.add(id); return id; }; window.clearInterval = (id) => { live.delete(id); clearI(id); }; let inFlight = 0; const nativeFetch = window.fetch.bind(window); window.fetch = (...args) => { inFlight += 1; return nativeFetch(...args).finally(() => { inFlight -= 1; }); }; const storageSize = (storage) => { let bytes = 0; for (let i = 0; i < storage.length; i += 1) { const key = storage.key(i) ?? ""; bytes += key.length + (storage.getItem(key) ?? "").length; } return { keys: storage.length, bytes }; }; Object.defineProperty(window, "__soakProbe", { value: () => { const local = storageSize(window.localStorage); const session = storageSize(window.sessionStorage); return { activeTimers: live.size, inFlight, storageKeys: local.keys + session.keys, storageBytes: local.bytes + session.bytes, }; }, }); })(); `; type Probe = { activeTimers: number; inFlight: number; storageKeys: number; storageBytes: number }; export async function runBrowserSoak( options: Readonly<{ distRoot: string; durationMs: number; intervalMs: number; warmupMs: number; routes: readonly string[]; }>, ): Promise { try { await access(path.join(options.distRoot, "index.html")); await access(path.join(options.distRoot, "config.json")); } catch { return { schemaVersion: 2, status: "skipped", reason: "no built dist/index.html + config.json", durationMs: 0, samples: [], }; } let chromium: (typeof import("@playwright/test"))["chromium"]; try { ({ chromium } = await import("@playwright/test")); } catch { return { schemaVersion: 2, status: "skipped", reason: "playwright unavailable", durationMs: 0, samples: [] }; } const counters: Counters = { apiRequests: 0, telemetryPosts: 0, telemetryEvents: 0, telemetryPostTimes: [], }; let started = Date.now(); const { server, baseUrl } = await serveSoakTarget( path.resolve(options.distRoot), counters, () => started, ); let browser: Awaited> | undefined; try { try { browser = await chromium.launch(); } catch (error) { return { schemaVersion: 2, status: "skipped", reason: `browser unavailable: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`, durationMs: 0, samples: [], }; } const page = await browser.newPage(); await page.addInitScript(PAGE_PROBES); const cdp = await page.context().newCDPSession(page); await cdp.send("Performance.enable"); await page.goto(`${baseUrl}/examples/reference-resources`, { waitUntil: "load" }); await page.waitForSelector("#root > *", { timeout: 15_000 }); // Protected routes need a session: start the local demo session once, so // the reference feature really dispatches its (failing) API requests. const login = page.getByRole("button", { name: /로그인 시작|Sign in|Start sign-in/u }); if (await login.count()) { await login.first().click(); await page.waitForTimeout(500); } const samples: SoakSample[] = []; started = Date.now(); let nextSample = 0; let cycles = 0; let navigationFailures = 0; let referenceVisits = 0; let settledReferenceVisits = 0; const probe = async () => (await page.evaluate(() => (window as unknown as { __soakProbe: () => Probe }).__soakProbe(), )) as Probe; while (Date.now() - started < options.durationMs) { const route = options.routes[cycles % options.routes.length] ?? "/"; cycles += 1; if (route === "/examples/reference-resources") referenceVisits += 1; // Client-side navigation keeps one long-lived document, like a real session. await page.evaluate((target) => { window.history.pushState({}, "", target); window.dispatchEvent(new PopStateEvent("popstate")); }, route); await page.waitForTimeout(50); const rendered = await page.evaluate( (target) => window.location.pathname === target && (document.getElementById("root")?.childElementCount ?? 0) > 0, route, ); if (!rendered) navigationFailures += 1; // Most visits leave after 50ms, which cancels the query (a cancelled // request is intentionally not reported). Every 5th reference visit // dwells until its interaction settles, so the failing backend is really // observed end to end: bounded retries, then feature/platform telemetry. if (route === "/examples/reference-resources" && referenceVisits % 5 === 1) { await page.waitForTimeout(200); await page .waitForFunction( () => (window as unknown as { __soakProbe: () => Probe }).__soakProbe().inFlight === 0, undefined, { timeout: 5_000 }, ) .catch(() => undefined); settledReferenceVisits += 1; } const elapsedMs = Date.now() - started; if (elapsedMs >= nextSample) { nextSample = elapsedMs + options.intervalMs; await cdp.send("HeapProfiler.collectGarbage").catch(() => undefined); const { metrics } = (await cdp.send("Performance.getMetrics")) as { metrics: { name: string; value: number }[]; }; const metric = (name: string) => metrics.find((entry) => entry.name === name)?.value ?? 0; const probed = await probe(); samples.push({ elapsedMs, cycles, heapBytes: metric("JSHeapUsedSize"), domNodes: metric("Nodes"), listeners: metric("JSEventListeners"), activeTimers: probed.activeTimers, inFlightRequests: probed.inFlight, storageKeys: probed.storageKeys, storageBytes: probed.storageBytes, }); } } // Let the last interaction's bounded retries and deadlines settle. let finalInFlightRequests = (await probe()).inFlight; for (let wait = 0; wait < 30 && finalInFlightRequests > 0; wait += 1) { await page.waitForTimeout(500); finalInFlightRequests = (await probe()).inFlight; } const evaluation = evaluateSoak( samples, { cycles, navigationFailures, referenceVisits, settledReferenceVisits, finalInFlightRequests, apiRequests: counters.apiRequests, telemetryPosts: counters.telemetryPosts, telemetryEvents: counters.telemetryEvents, telemetryPostTimes: counters.telemetryPostTimes, }, options.warmupMs, ); const build = await readFile(path.join(options.distRoot, "release-manifest.json"), "utf8") .then((raw) => { const manifest = JSON.parse(raw) as Record; const pick = (key: string) => typeof manifest[key] === "string" ? (manifest[key] as string) : undefined; return { ...(pick("buildId") ? { buildId: pick("buildId") } : {}), ...(pick("releaseId") ? { releaseId: pick("releaseId") } : {}), ...(pick("builtAt") ? { builtAt: pick("builtAt") } : {}), }; }) .catch(() => ({})); return { schemaVersion: 2, status: evaluation.checks.every((check) => check.passed) ? "passed" : "failed", durationMs: options.durationMs, classification: evaluation.summary.heapClassification, summary: evaluation.summary, checks: evaluation.checks, build, browser: `chromium ${browser.version()}`, samples, }; } finally { await browser?.close().catch(() => undefined); await new Promise((resolve) => server.close(() => resolve())); } } async function main(): Promise { const argument = (name: string, fallback: string) => process.argv.find((entry) => entry.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback; const artifact = path.resolve(argument("artifact", "artifacts/scale/browser-soak.json")); const result = await runBrowserSoak({ distRoot: argument("dist", "dist"), durationMs: Number(argument("duration-ms", "60000")), intervalMs: Number(argument("interval-ms", "5000")), warmupMs: Number(argument("warmup-ms", "15000")), routes: argument( "routes", "/,/examples/reference-resources,/examples/platform,/examples/ui,/examples/states,/examples/reference-resources/status", ).split(","), }); await mkdir(path.dirname(artifact), { recursive: true }); await writeFile(artifact, `${JSON.stringify(result, null, 2)}\n`); process.stdout.write( `browser soak: ${result.status}${result.reason ? ` (${result.reason})` : ""}` + `${result.classification ? ` classification=${result.classification}` : ""} samples=${result.samples.length}` + `${result.summary ? ` cycles=${result.summary.cycles}` : ""}\n`, ); for (const check of result.checks ?? []) { process.stdout.write(`${check.passed ? "PASS" : "FAIL"} ${check.name} (${check.detail})\n`); } if (result.status === "failed") process.exitCode = 1; } if (import.meta.url === `file://${process.argv[1]}`) { await main(); }