diff --git a/docs/testing/frontend-platform-testing-strategy.md b/docs/testing/frontend-platform-testing-strategy.md index e262a2ae..d4fa1549 100644 --- a/docs/testing/frontend-platform-testing-strategy.md +++ b/docs/testing/frontend-platform-testing-strategy.md @@ -145,7 +145,11 @@ operation별 success/empty/slow/network/timeout/content/envelope/schema/auth/ unhandled-request server(`tests/mocks/server.ts`의 `onUnhandledRequest: "error"`)를 공유한다. browser test는 공용 fixture (`tests/support/browser/strict-browser-test.ts`)가 console error/warning, -page error, 실패한 request를 test 실패로 만든다. 시간은 각 test가 +page error, 실패한 request, 끝까지 실행되지 않은 preload를 test 실패로 만든다. +WebKit의 "preload했지만 몇 초 안에 쓰지 않았다" 경고는 load 3초 뒤 한 번 하는 +추정이라 그 문구만 무시하고, 대신 preload된 build script가 실제로 실행됐는지를 +세 엔진에서 같은 방법으로 검사한다(`tests/support/browser/preload-use.ts`, +1103/11-09). 시간은 각 test가 `vi.useFakeTimers()`로 통제하고, 전역 `afterEach`가 실제 timer로 되돌린다 (`tests/setup.ts`). 공용 clock/random/scheduler helper는 두지 않는다. diff --git a/tests/e2e/boot-module-failure.spec.ts b/tests/e2e/boot-module-failure.spec.ts index 5e1f5718..546dc2f3 100644 --- a/tests/e2e/boot-module-failure.spec.ts +++ b/tests/e2e/boot-module-failure.spec.ts @@ -1,3 +1,4 @@ +import { readFile } from "node:fs/promises"; import path from "node:path"; import { expect, test } from "../support/browser/strict-browser-test.ts"; @@ -38,6 +39,30 @@ test.afterAll(async () => { await removeFixtureBuilds(builds); }); +// 1103/11-09. Vite preloads the boot chunk with everything it imports. A +// module of that graph throws, so the graph stops evaluating and its +// remaining chunks stay preloaded and unrun: the provoked failure's own +// effect. Only the boot graph's chunks are allowed. +test.use({ + allowedBrowserFailure: async ({ browserName: _engine }, provide) => { + if (!builds) throw new Error("Fixture build was not prepared."); + const manifest = JSON.parse( + await readFile(path.join(builds.dist.broken, ".vite/manifest.json"), "utf8"), + ) as Record; + const files = new Set(); + const pending = ["src/bootstrap/boot.tsx"]; + for (let key = pending.pop(); key !== undefined; key = pending.pop()) { + const entry = manifest[key]; + if (!entry || files.has(entry.file)) continue; + files.add(entry.file); + pending.push(...(entry.imports ?? [])); + } + if (files.size === 0) throw new Error("The broken build has no boot chunk."); + const alternatives = [...files].map((file) => file.replaceAll(".", String.raw`\.`)); + await provide(new RegExp(String.raw`^preloadunused:/(?:${alternatives.join("|")})$`, "u")); + }, +}); + test("shows the boot error shell when a module throws while it evaluates", async ({ page, }) => { diff --git a/tests/e2e/old-tab-chunk-recovery.spec.ts b/tests/e2e/old-tab-chunk-recovery.spec.ts index 72635b16..60e87ba5 100644 --- a/tests/e2e/old-tab-chunk-recovery.spec.ts +++ b/tests/e2e/old-tab-chunk-recovery.spec.ts @@ -141,33 +141,36 @@ async function preloadedWith(dist: string, page: string): Promise { return [...files]; } -// WebKit only. Three seconds after a document's load event, WebKit warns once -// for each link-preloaded resource the document has not used yet. Vite -// preloads a chunk together with the chunks it imports, so a missing chunk -// leaves the chunks preloaded with it unused: in the A tab those of A's -// platform page (the tab is still on A while recovery reads the release -// manifest), in the B tab those of B's context routes module for the UI page. Whether that check runs before -// the reload, or before the test ends, is timing, so the warnings come and go. -// They are the provoked failure's own effect; only those chunks are allowed. +// 1103/11-09. Vite preloads a chunk together with the chunks it imports, so a +// missing chunk leaves itself and the chunks preloaded with it unused: in the +// A tab those of A's platform page (the tab is still on A while recovery +// reads the release manifest), in the B tab those of B's context routes +// module for the UI page. The fixture's preload check reports them in every +// engine (WebKit's own three-second warning only did when its timer ran +// before the test ended). They are the provoked failure's own effect; only +// those chunks are allowed. test.use({ - allowedBrowserFailure: async ({ browserName }, provide) => { - const patterns = [...MISSING_PAGE_FAILURES]; - if (browserName === "webkit") { - if (!builds) throw new Error("Fixture builds were not prepared."); - const aManifest = await viteManifest(builds.dist.a); - const bManifest = await viteManifest(builds.dist.b); - const files = [ - ...(await preloadedWith(builds.dist.a, pageSource(aManifest, PLATFORM))), - ...(await preloadedWith( - builds.dist.b, - routesSource(bManifest, pageSource(bManifest, UI)), - )), - ].map((file) => file.replaceAll(".", String.raw`\.`)); - patterns.push( - String.raw`^console:warning:The resource http://127\.0\.0\.1:\d+/(?:${files.join("|")}) was preloaded using link preload but not used within a few seconds from the window's load event\. Please make sure it wasn't preloaded for nothing\.$`, - ); - } - await provide(new RegExp(patterns.join("|"), "u")); + allowedBrowserFailure: async ({ browserName: _engine }, provide) => { + if (!builds) throw new Error("Fixture builds were not prepared."); + const aManifest = await viteManifest(builds.dist.a); + const bManifest = await viteManifest(builds.dist.b); + const files = [ + ...(await preloadedWith(builds.dist.a, pageSource(aManifest, PLATFORM))), + ...(await preloadedWith( + builds.dist.b, + routesSource(bManifest, pageSource(bManifest, UI)), + )), + ].map((file) => file.replaceAll(".", String.raw`\.`)); + await provide( + new RegExp( + [ + ...MISSING_PAGE_FAILURES, + String.raw`^preloadunused:${MISSING_PAGE}$`, + String.raw`^preloadunused:/(?:${files.join("|")})$`, + ].join("|"), + "u", + ), + ); }, }); diff --git a/tests/e2e/preload-use.spec.ts b/tests/e2e/preload-use.spec.ts new file mode 100644 index 00000000..a891cfa0 --- /dev/null +++ b/tests/e2e/preload-use.spec.ts @@ -0,0 +1,144 @@ +import { readFile } from "node:fs/promises"; + +import { expect, test } from "@playwright/test"; + +import { trackPreloadUse } from "../support/browser/preload-use.ts"; +import { test as strictTest } from "../support/browser/strict-browser-test.ts"; + +/** + * 1103/11-09. The strict browser fixture ignores WebKit's "preloaded using + * link preload but not used within a few seconds" warning. WebKit checks once, + * three seconds after the load event, so a preload whose module the page + * reaches a moment later was reported as wasted, and one the page never runs + * went unreported when the test ended first. The fixture checks instead, in + * every engine, that each preloaded script was run by the end of the test. + * These tests prove that check on a real build. + */ + +type ManifestEntry = Readonly<{ + file: string; + imports?: readonly string[]; + isEntry?: boolean; +}>; + +async function buildManifest(): Promise>> { + return JSON.parse(await readFile("dist/.vite/manifest.json", "utf8")) as Record< + string, + ManifestEntry + >; +} + +function staticClosure( + manifest: Readonly>, + roots: readonly string[], +): Set { + const reached = new Set(); + const pending = [...roots]; + for (let key = pending.pop(); key !== undefined; key = pending.pop()) { + if (reached.has(key) || !manifest[key]) continue; + reached.add(key); + pending.push(...(manifest[key].imports ?? [])); + } + return reached; +} + +test("reports a modulepreload the page never runs", async ({ page }) => { + const preloads = await trackPreloadUse(page); + await page.goto("/"); + await expect(page.getByRole("heading", { level: 1 })).toBeVisible(); + + const scripts = Object.values(await buildManifest()) + .map((entry) => `/${entry.file}`) + .filter((file) => file.endsWith(".js")); + // Mutation: one chunk the page has not fetched, preloaded for nothing. + const unused = await page.evaluate(async (files) => { + const fetched = new Set( + performance + .getEntriesByType("resource") + .map((entry) => new URL(entry.name).pathname), + ); + const file = files.find((candidate) => !fetched.has(candidate)); + if (file === undefined) throw new Error("every build chunk was fetched"); + const link = document.createElement("link"); + link.rel = "modulepreload"; + link.href = file; + document.head.append(link); + return file; + }, scripts); + await page.waitForFunction( + (file) => + performance + .getEntriesByType("resource") + .some((entry) => new URL(entry.name).pathname === file), + unused, + ); + + expect((await preloads.check({ timeoutMs: 1_000 })).failures).toEqual([ + `preloadunused:${unused}`, + ]); +}); + +test("counts the chunks a route preloads and then runs as used", async ({ + page, +}) => { + const preloads = await trackPreloadUse(page); + await page.goto("/"); + await page + .getByRole("navigation", { name: "주요 탐색" }) + .getByRole("link", { name: "플랫폼 구성" }) + .click(); + await expect( + page.getByRole("table", { name: "설치된 HTTP 오퍼레이션" }), + ).toBeVisible(); + + const result = await preloads.check(); + expect(result.preloaded.length).toBeGreaterThan(0); + expect(result.failures).toEqual([]); +}); + +/** + * The 1103/11-09 failure, made deterministic: the route's runtime chunks are + * preloaded right after load, and the chunk that imports the deepest one + * arrives 3.5 s later. WebKit's check runs in between and warns about a chunk + * the page then runs. The strict fixture must pass. + */ +strictTest( + "passes a route whose deepest preloaded chunk is reached after WebKit's check", + async ({ page }) => { + const manifest = await buildManifest(); + const initial = staticClosure(manifest, [ + ...Object.keys(manifest).filter((key) => manifest[key]?.isEntry), + "src/bootstrap/boot.tsx", + ]); + // A feature runtime's static import that itself imports a chunk the + // boot has not loaded: the module graph reaches that chunk only once + // this one has arrived. + const parent = Object.entries(manifest) + .filter(([key]) => /^src\/features\/[^/]+\/feature-runtime\.ts$/u.test(key)) + .flatMap(([, runtime]) => runtime.imports ?? []) + .find( + (key) => + !initial.has(key) && + (manifest[key]?.imports ?? []).some((child) => !initial.has(child)), + ); + + await page.goto("/"); + if (parent !== undefined) { + const file = `/${manifest[parent]?.file}`; + await page.route( + (url) => url.pathname === file, + async (route) => { + await new Promise((resolve) => setTimeout(resolve, 3_500)); + await route.fallback(); + }, + ); + } + await page + .getByRole("navigation", { name: "주요 탐색" }) + .getByRole("link", { name: "플랫폼 구성" }) + .click(); + await expect( + page.getByRole("table", { name: "설치된 HTTP 오퍼레이션" }), + ).toBeVisible({ timeout: 15_000 }); + }, +); diff --git a/tests/e2e/production-csp.spec.ts b/tests/e2e/production-csp.spec.ts index 954eafd4..c4065fd6 100644 --- a/tests/e2e/production-csp.spec.ts +++ b/tests/e2e/production-csp.spec.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; +import { fulfillBuildScript } from "../support/browser/preload-use.ts"; import { expect, test, type Page } from "../support/browser/strict-browser-test.ts"; // 13-08. The Content-Security-Policy in config/hosting/security-headers.json was @@ -30,9 +31,11 @@ test.beforeEach(async ({ page, baseURL }) => { (url) => url.origin === origin, async (route) => { const response = await route.fetch(); - await route.fulfill({ - response, - headers: { ...response.headers(), ...securityHeaders }, + // 1103/11-09. Keeps the build scripts' evaluation mark for the + // fixture's preload check. + await fulfillBuildScript(route, response, { + ...response.headers(), + ...securityHeaders, }); }, ); diff --git a/tests/e2e/service-worker-update.spec.ts b/tests/e2e/service-worker-update.spec.ts index 361c281a..585df28c 100644 --- a/tests/e2e/service-worker-update.spec.ts +++ b/tests/e2e/service-worker-update.spec.ts @@ -42,6 +42,15 @@ type Probe = typeof globalThis & { let builds: ServiceWorkerBuilds | undefined; +// 1103/11-09. Once a worker controls the page it answers the page's script +// requests, so the fixture cannot mark them and cannot tell whether a +// preload made then was run. It names those preloads instead of calling them +// unused; here they are expected. Preloads made before the worker took +// control are still checked. +test.use({ + allowedBrowserFailure: /^preloadunverifiable:service-worker:\/assets\/[\w-]+\.js$/u, +}); + // B changes one route chunk on purpose. Route chunks used to differ only // because they embedded the build ID; since 09-02 they do not, so two builds // of the same source emit the same route chunks. A dynamically imported diff --git a/tests/support/browser/preload-use.ts b/tests/support/browser/preload-use.ts new file mode 100644 index 00000000..af942415 --- /dev/null +++ b/tests/support/browser/preload-use.ts @@ -0,0 +1,233 @@ +import type { APIResponse, Frame, Page, Route } from "@playwright/test"; + +/** + * 1103/11-09. Which link-preloaded scripts a page actually ran. + * + * WebKit warns about a preload no request has used three seconds after the + * load event, once. That is a timing guess: a route entered just before the + * check preloads chunks the page runs a moment later, and a preload the page + * never runs goes unreported when the test ends first. This check does not + * depend on when it runs, and it is the same in every engine: + * + * - Every `` and `` a document gets + * is recorded (a MutationObserver from before the document's first byte, + * plus the links still in the document at the end). + * - Every build script the page receives (`/assets/*.js`) starts with one + * statement that records its own URL when it is evaluated. A preload only + * fetches (WebKit, Firefox) or fetches and parses (Chromium); none of them + * evaluates, so a preloaded chunk counts as used only once the module graph + * ran it. + * - At the end, a preloaded script that was not run is `preloadunused:`. The + * check waits for a dynamic import still in flight (up to `timeoutMs`). + * + * What it cannot see fails closed, under its own name: + * - `preloadunverifiable::` a preload of anything but a script. + * - `preloadunverifiable:service-worker:` a script preloaded while a Service + * Worker controlled the document. The worker answers the page's requests, + * so this route never sees them; marking what the worker fetches would + * change bytes it checks against the release manifest. + * - `preloadunverifiable:unmarked:` a script this route could not fetch to + * mark (the browser got it unmarked). + * A script outside `/assets/`, or one a test's own handler fulfilled without + * `fulfillBuildScript`, never records itself and is `preloadunused:`. + * + * Only the documents open at the end are checked: a document the test + * navigated away from takes its records with it. + */ + +const BUILD_SCRIPT = /\/assets\/[^/?#]+\.js$/u; +const STATE = "__e2ePreloadUse"; +const FETCH_ATTEMPTS = 3; + +type Preload = Readonly<{ href: string; kind: string }>; + +type FrameRecord = Readonly<{ + preloaded: readonly Preload[]; + evaluated: readonly string[]; +}>; + +export type PreloadUseCheck = Readonly<{ + /** Every preloaded URL path the open documents recorded. */ + preloaded: readonly string[]; + /** `preloadunused:` and `preloadunverifiable::` lines. */ + failures: readonly string[]; +}>; + +export type PreloadUseTracker = Readonly<{ + check(options?: Readonly<{ timeoutMs?: number }>): Promise; +}>; + +/** Runs in every document before its first byte. */ +function recordPreloads(state: string): void { + const record = { + preloaded: new Map(), + evaluated: new Set(), + }; + Object.defineProperty(globalThis, state, { value: record }); + const remember = (node: Node) => { + if (!(node instanceof HTMLLinkElement) || node.href === "") return; + const script = + node.relList.contains("modulepreload") || + (node.relList.contains("preload") && node.as === "script"); + if (script) { + const controlled = navigator.serviceWorker?.controller != null; + record.preloaded.set(node.href, controlled ? "service-worker" : "script"); + } else if (node.relList.contains("preload")) { + record.preloaded.set(node.href, node.as || "unknown"); + } + }; + new MutationObserver((mutations) => { + for (const mutation of mutations) { + for (const node of mutation.addedNodes) { + remember(node); + if (node instanceof Element) node.querySelectorAll("link").forEach(remember); + } + } + }).observe(document, { childList: true, subtree: true }); +} + +function readRecord(state: string): FrameRecord { + const record = (globalThis as Record)[state] as + | { preloaded: Map; evaluated: Set } + | undefined; + const preloaded = new Map(record?.preloaded); + const controlled = navigator.serviceWorker?.controller != null; + for (const link of document.querySelectorAll("link")) { + if (link.href === "" || preloaded.has(link.href)) continue; + if ( + link.relList.contains("modulepreload") || + (link.relList.contains("preload") && link.as === "script") + ) { + preloaded.set(link.href, controlled ? "service-worker" : "script"); + } else if (link.relList.contains("preload")) { + preloaded.set(link.href, link.as || "unknown"); + } + } + return { + preloaded: [...preloaded].map(([href, kind]) => ({ href, kind })), + evaluated: [...(record?.evaluated ?? [])], + }; +} + +/** + * Fulfills a request with `response`, the evaluation mark added when it is a + * build script. A test that fulfills build scripts itself (for example to add + * headers) uses this, or every preload of those scripts reads as unused. + */ +export async function fulfillBuildScript( + route: Route, + response: APIResponse, + headers: Readonly> = response.headers(), +): Promise { + const url = route.request().url(); + const type = response.headers()["content-type"] ?? ""; + if ( + !BUILD_SCRIPT.test(new URL(url).pathname) || + response.status() !== 200 || + !/javascript/u.test(type) + ) { + return route.fulfill({ response, headers: { ...headers } }); + } + const marked = { ...headers }; + delete marked["content-length"]; + delete marked["content-encoding"]; + await route.fulfill({ + status: response.status(), + headers: marked, + // Valid in a module and in a classic script; a no-op where the page's + // record does not exist (a worker). + body: `globalThis.${STATE}?.evaluated.add(${JSON.stringify(url)});\n${await response.text()}`, + }); +} + +function pathOf(href: string, origin: string): string { + const url = new URL(href); + return url.origin === origin ? `${url.pathname}${url.search}` : href; +} + +export async function trackPreloadUse(page: Page): Promise { + /** URL → why it reached the browser without the mark. */ + const unmarked = new Map(); + + async function markEvaluation(route: Route): Promise { + if (route.request().method() !== "GET") return route.fallback(); + let lastError: unknown; + for (let attempt = 0; attempt < FETCH_ATTEMPTS; attempt += 1) { + let response: APIResponse; + try { + response = await route.fetch(); + } catch (error) { + lastError = error; + continue; + } + return fulfillBuildScript(route, response); + } + // The browser gets the request as the server answers it, unmarked. + unmarked.set( + route.request().url(), + String(lastError).split("\n")[0] ?? "fetch failed", + ); + return route.continue(); + } + + async function frameResult(frame: Frame): Promise< + Readonly<{ preloaded: string[]; failures: string[] }> | undefined + > { + if (frame.isDetached()) return { preloaded: [], failures: [] }; + let record: FrameRecord; + let origin: string; + try { + [record, origin] = await Promise.all([ + frame.evaluate(readRecord, STATE), + frame.evaluate(() => location.origin), + ]); + } catch { + // The document is being replaced; read the next one. + return undefined; + } + const evaluated = new Set(record.evaluated); + const failures: string[] = []; + for (const { href, kind } of record.preloaded) { + const where = pathOf(href, origin); + if (kind !== "script" && kind !== "service-worker") { + failures.push(`preloadunverifiable:${kind}:${where}`); + } else if (evaluated.has(href)) { + continue; + } else if (unmarked.has(href)) { + failures.push(`preloadunverifiable:unmarked:${where}: ${unmarked.get(href)}`); + } else if (kind === "service-worker") { + failures.push(`preloadunverifiable:service-worker:${where}`); + } else { + failures.push(`preloadunused:${where}`); + } + } + return { + preloaded: record.preloaded.map(({ href }) => pathOf(href, origin)), + failures, + }; + } + + await page.addInitScript(recordPreloads, STATE); + await page.route((url) => BUILD_SCRIPT.test(url.pathname), markEvaluation); + return Object.freeze({ + async check({ timeoutMs = 5_000 } = {}) { + const deadline = Date.now() + timeoutMs; + for (;;) { + if (page.isClosed()) return { preloaded: [], failures: [] }; + const frames = await Promise.all(page.frames().map(frameResult)); + const settled = frames.every((frame) => frame !== undefined); + const preloaded = frames.flatMap((frame) => frame?.preloaded ?? []).sort(); + const failures = frames.flatMap((frame) => frame?.failures ?? []).sort(); + if ((settled && failures.length === 0) || Date.now() >= deadline) { + return { + preloaded, + failures: settled + ? failures + : [...failures, "preloadcheck:unreadable-document"], + }; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }, + }); +} diff --git a/tests/support/browser/strict-browser-test.ts b/tests/support/browser/strict-browser-test.ts index 23aaad7b..0ba19808 100644 --- a/tests/support/browser/strict-browser-test.ts +++ b/tests/support/browser/strict-browser-test.ts @@ -5,18 +5,31 @@ import { type Page, } from "@playwright/test"; +import { trackPreloadUse } from "./preload-use.ts"; + +/** + * 1103/11-09. WebKit's guess, three seconds after the load event, that a + * preload went unused. It fired for chunks the page ran a moment later and + * missed preloads the page never ran once the test had ended. The fixture + * checks preload use itself, in every engine (`preloadunused:` below). + */ +const WEBKIT_UNUSED_PRELOAD_GUESS = + /^The resource \S+ was preloaded using link preload but not used within a few seconds from the window's load event\. Please make sure it wasn't preloaded for nothing\.$/u; + function ignoredConsole(message: ConsoleMessage) { return ( message.type() === "warning" && - message.text().includes("NO_COLOR") + (message.text().includes("NO_COLOR") || + WEBKIT_UNUSED_PRELOAD_GUESS.test(message.text())) ); } export const test = base.extend<{ /** * Browser failures a test provokes on purpose, matched against the recorded - * `console:…`, `pageerror:…` or `requestfailed:…` line (for example the - * browser's log line for a mocked 4xx). Everything else still fails. One + * `console:…`, `pageerror:…`, `requestfailed:…` or `preloadunused:…` line + * (for example the browser's log line for a mocked 4xx, or the chunks + * preloaded with a chunk the test removed). Everything else still fails. One * pattern, not an array: `test.use` reads an array as a `[value, options]` * fixture tuple. */ @@ -24,6 +37,7 @@ export const test = base.extend<{ }>({ allowedBrowserFailure: [undefined, { option: true }], page: async ({ page, allowedBrowserFailure }, use) => { + const preloads = await trackPreloadUse(page); const failures: string[] = []; const onConsole = (message: ConsoleMessage) => { if ( @@ -45,6 +59,7 @@ export const test = base.extend<{ page.on("pageerror", onPageError); page.on("requestfailed", onRequestFailed); await use(page); + failures.push(...(await preloads.check()).failures); page.off("console", onConsole); page.off("pageerror", onPageError); page.off("requestfailed", onRequestFailed);