diff --git a/scripts/test-browser-file-storage-runtime-removal.ts b/scripts/test-browser-file-storage-runtime-removal.ts index 2dd386de..0607e916 100644 --- a/scripts/test-browser-file-storage-runtime-removal.ts +++ b/scripts/test-browser-file-storage-runtime-removal.ts @@ -131,9 +131,16 @@ try { } await writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`); - await rm(path.join(fixtureRoot, "playwright.capabilities.config.ts"), { - force: true, - }); + // 1106/10-01. The capabilities config goes with this runtime, and so do its + // run-isolation checks, by name: the template's test:unit runs them, this + // copy has nothing for them to check. Without `force`, so a renamed file + // stops the drill here instead of failing its typecheck later. + for (const capabilityConfigFile of [ + "playwright.capabilities.config.ts", + "tests/unit/playwright-capabilities-run-isolation.test.ts", + ]) { + await rm(path.join(fixtureRoot, capabilityConfigFile)); + } await rm( path.join(fixtureRoot, "scripts/check-browser-file-storage-boundaries.ts"), { force: true }, diff --git a/tests/helpers/playwright-run-isolation.ts b/tests/helpers/playwright-run-isolation.ts new file mode 100644 index 00000000..4e39f623 --- /dev/null +++ b/tests/helpers/playwright-run-isolation.ts @@ -0,0 +1,197 @@ +import { createServer, type Server } from "node:net"; +import path from "node:path"; + +import type { PlaywrightTestConfig } from "@playwright/test"; +import { describe, expect, it, vi } from "vitest"; + +/** + * 1105/12-04. The run-isolation checks every Playwright config shares, for + * tests/unit/playwright-run-isolation.test.ts (the four configs every project + * has) and tests/unit/playwright-capabilities-run-isolation.test.ts (the + * capabilities config, which goes with the browser file/storage runtime). + * + * 1106/10-01. Each file imports its configs by name and passes them here, so + * no check depends on which config files a checkout happens to have. + */ +export type ConfigName = "e2e" | "visual" | "capabilities" | "storybook" | "dev"; +export type ConfigLoaders = Readonly< + Partial Promise<{ default: PlaywrightTestConfig }>>> +>; + +const PORT_VARIABLES: Readonly> = { + e2e: "TEST_E2E_PORT", + visual: "TEST_VISUAL_PORT", + capabilities: "TEST_CAPABILITIES_PORT", + storybook: "TEST_STORYBOOK_PORT", + dev: "TEST_DEV_PORT", +}; +const RUN_VARIABLES = ["TEST_RUN_ID", "RUN_OUTPUT_ROOT", ...Object.values(PORT_VARIABLES)]; +const held: Server[] = []; + +/** For afterEach: forgets the loaded configs and frees the ports `hold` took. */ +export async function releaseRuns(): Promise { + vi.resetModules(); + await Promise.all( + held.splice(0).map((server) => new Promise((resolve) => server.close(() => resolve()))), + ); +} + +type ConfigProcess = Readonly<{ + config: PlaywrightTestConfig; + after: Readonly>; +}>; + +export type ConfigRuns = Readonly<{ + /** The config as one process sees it, and what it left in that environment. */ + loadProcess( + name: ConfigName, + environment?: Readonly>, + ): Promise; + load(name: ConfigName, environment?: Readonly>): Promise; +}>; + +/** Each load evaluates the config again with the environment of one process. */ +export function configRuns(loaders: ConfigLoaders): ConfigRuns { + async function loadProcess( + name: ConfigName, + environment: Readonly> = {}, + ): Promise { + const loader = loaders[name]; + if (loader === undefined) throw new TypeError(`no loader for the ${name} config`); + const saved = RUN_VARIABLES.map((variable) => [variable, process.env[variable]] as const); + for (const variable of RUN_VARIABLES) delete process.env[variable]; + Object.assign(process.env, environment); + vi.resetModules(); + try { + const config = (await loader()).default; + return { + config, + after: Object.fromEntries(RUN_VARIABLES.map((variable) => [variable, process.env[variable]])), + }; + } finally { + for (const [variable, value] of saved) { + if (value === undefined) delete process.env[variable]; + else process.env[variable] = value; + } + } + } + return { + loadProcess, + load: async (name, environment = {}) => (await loadProcess(name, environment)).config, + }; +} + +export function portOf(config: PlaywrightTestConfig): number { + return Number(new URL(String(config.use?.baseURL)).port); +} + +/** Listens on `port` the way the first run's server does. */ +export async function hold(port: number): Promise { + const server = createServer(); + held.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve()); + }); +} + +function reporterOption(config: PlaywrightTestConfig, reporter: string, option: string): string { + const reporters: readonly unknown[] = Array.isArray(config.reporter) ? config.reporter : []; + const entry = reporters.find( + (candidate) => Array.isArray(candidate) && candidate[0] === reporter, + ) as readonly [string, Record] | undefined; + return String(entry?.[1][option]); +} + +function outputs(config: PlaywrightTestConfig) { + return { + results: path.resolve(String(config.outputDir)), + report: path.resolve(reporterOption(config, "html", "outputFolder")), + junit: path.resolve(reporterOption(config, "junit", "outputFile")), + }; +} + +export function webServer(config: PlaywrightTestConfig) { + const server = config.webServer; + if (!server || Array.isArray(server)) throw new TypeError("expected one web server"); + return server; +} + +/** Each config's results, report and JUnit file, by its folder under artifacts/tests. */ +export function describeResultFolders( + runs: ConfigRuns, + folders: readonly (readonly [ConfigName, string])[], +): void { + describe.each(folders)("the %s config", (name, folder) => { + const expected = (root: string) => ({ + results: path.resolve(root, `artifacts/tests/${folder}/results`), + report: path.resolve(root, `artifacts/tests/${folder}/report`), + junit: path.resolve(root, `artifacts/tests/${folder}/results.xml`), + }); + + it("writes where the gates read without a run ID", async () => { + expect(outputs(await runs.load(name as ConfigName))).toEqual(expected(".")); + }); + + it("gives each run ID its own results, report and JUnit file", async () => { + const a = outputs(await runs.load(name as ConfigName, { TEST_RUN_ID: "a" })); + const b = outputs(await runs.load(name as ConfigName, { TEST_RUN_ID: "b" })); + + expect(a).toEqual(expected(".tmp/runs/a")); + expect(b).toEqual(expected(".tmp/runs/b")); + }); + }); +} + +/** Inside "the build a run serves": the dev servers' Vite cache. */ +export function itKeepsTheViteCacheInTheRun(runs: ConfigRuns, names: readonly ConfigName[]): void { + it.each(names)("keeps the dev server's Vite cache in the run's folder (%s)", async (name) => { + expect(webServer(await runs.load(name)).env?.["RUN_OUTPUT_ROOT"]).toBe("."); + expect(webServer(await runs.load(name, { TEST_RUN_ID: "a" })).env?.["RUN_OUTPUT_ROOT"]).toBe( + ".tmp/runs/a", + ); + }); +} + +export function describeServerPorts(runs: ConfigRuns, names: readonly ConfigName[]): void { + describe.each(names)("the %s server port", (name) => { + it("is a free port the runner hands to its workers", async () => { + const runner = await runs.loadProcess(name); + const port = portOf(runner.config); + const server = webServer(runner.config); + + // Playwright evaluates the config again in each worker, which inherits + // the runner's environment: the port is there, so baseURL agrees. + expect(runner.after[PORT_VARIABLES[name]]).toBe(String(port)); + expect(portOf(await runs.load(name, { [PORT_VARIABLES[name]]: String(port) }))).toBe(port); + expect(server.url).toBe(`http://127.0.0.1:${port}`); + expect(server.command).toMatch(new RegExp(` (?:--port )?${port}(?: |$)`, "u")); + expect(server.reuseExistingServer).toBe(false); + }); + + it("moves for a second run while the first run's server holds its port", async () => { + const first = portOf(await runs.load(name)); + await hold(first); + + expect(portOf(await runs.load(name))).not.toBe(first); + }); + + it("is the one its variable pins", async () => { + const pinned = await runs.load(name, { [PORT_VARIABLES[name]]: "40123" }); + expect(portOf(pinned)).toBe(40123); + expect(webServer(pinned).url).toBe("http://127.0.0.1:40123"); + }); + }); +} + +/** Inside "vite servers": the configs whose server is vite. */ +export function itExitsWhenItsPortIsTaken(runs: ConfigRuns, names: readonly ConfigName[]): void { + it.each(names)( + "exit instead of moving to the next port when theirs is taken (%s)", + async (name) => { + // Without --strictPort vite moved on, and Playwright, waiting on the + // configured URL, tested whatever already listened there. + expect(webServer(await runs.load(name)).command).toMatch(/ --port \d+ --strictPort(?: |$)/u); + }, + ); +} diff --git a/tests/unit/playwright-capabilities-run-isolation.test.ts b/tests/unit/playwright-capabilities-run-isolation.test.ts new file mode 100644 index 00000000..98d4f3f4 --- /dev/null +++ b/tests/unit/playwright-capabilities-run-isolation.test.ts @@ -0,0 +1,51 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + configRuns, + describeResultFolders, + describeServerPorts, + hold, + itExitsWhenItsPortIsTaken, + itKeepsTheViteCacheInTheRun, + portOf, + releaseRuns, +} from "../helpers/playwright-run-isolation.ts"; + +/** + * 1105/12-04 for the capabilities config; why each check exists is in + * tests/unit/playwright-run-isolation.test.ts. + * + * 1106/10-01. This config goes with the browser file/storage runtime: + * test:browser-file-storage-removal deletes it, and deletes this file by + * name with it. Its eight checks used to sit with the other configs and were + * defined only while `existsSync` found the config. test:unit fails on a + * skipped test (1101/10-04), but a test that is never defined is not + * reported: the checks left that drill's copy unnoticed, and a renamed config + * would have taken them out of the template the same way. Imported by name, a + * renamed config fails check:types and this file instead. + */ +const runs = configRuns({ + capabilities: () => import("../../playwright.capabilities.config.ts"), + visual: () => import("../../playwright.visual.config.ts"), +}); + +afterEach(releaseRuns); + +describeResultFolders(runs, [["capabilities", "browser-capabilities"]]); + +describe("the build a run serves", () => { + itKeepsTheViteCacheInTheRun(runs, ["capabilities"]); +}); + +describeServerPorts(runs, ["capabilities"]); + +describe("vite servers", () => { + itExitsWhenItsPortIsTaken(runs, ["capabilities"]); + + it("no longer share one port between the visual preview and the capability dev server", async () => { + const visual = portOf(await runs.load("visual")); + await hold(visual); + + expect(portOf(await runs.load("capabilities"))).not.toBe(visual); + }); +}); diff --git a/tests/unit/playwright-run-isolation.test.ts b/tests/unit/playwright-run-isolation.test.ts index 2f70cb22..ec91aafb 100644 --- a/tests/unit/playwright-run-isolation.test.ts +++ b/tests/unit/playwright-run-isolation.test.ts @@ -1,9 +1,16 @@ -import { existsSync, readFileSync } from "node:fs"; -import { createServer, type Server } from "node:net"; -import path from "node:path"; +import { readFileSync } from "node:fs"; -import type { PlaywrightTestConfig } from "@playwright/test"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + configRuns, + describeResultFolders, + describeServerPorts, + itExitsWhenItsPortIsTaken, + itKeepsTheViteCacheInTheRun, + releaseRuns, + webServer, +} from "../helpers/playwright-run-isolation.ts"; /** * 1105/12-04. Every Playwright config wrote its results, report and JUnit @@ -20,151 +27,36 @@ import { afterEach, describe, expect, it, vi } from "vitest"; * * Each load evaluates the config again with the environment of one process. * - * test:browser-file-storage-removal deletes the capabilities config with the - * capability it runs, so a project without it checks the other four. Its path - * is a variable so that typecheck does not need the file. + * 1106/10-01. The capabilities config is checked in + * tests/unit/playwright-capabilities-run-isolation.test.ts: it goes with the + * browser file/storage runtime, and test:browser-file-storage-removal deletes + * the config and that file by name. This file has the four configs every + * project has. */ -const CAPABILITIES_CONFIG = "playwright.capabilities.config.ts"; -const HAS_CAPABILITIES = existsSync(CAPABILITIES_CONFIG); -const LOADERS = { +const runs = configRuns({ e2e: () => import("../../playwright.config.ts"), visual: () => import("../../playwright.visual.config.ts"), - capabilities: () => - import(/* @vite-ignore */ `../../${CAPABILITIES_CONFIG}`) as Promise<{ - default: PlaywrightTestConfig; - }>, storybook: () => import("../../playwright.storybook.config.ts"), dev: () => import("../../playwright.dev.config.ts"), -} as const; -type ConfigName = keyof typeof LOADERS; -/** The names among `names` whose config file this checkout has. */ -function installed(names: readonly T[]): T[] { - return names.filter((name) => name !== "capabilities" || HAS_CAPABILITIES); -} -const CONFIG_NAMES = installed(Object.keys(LOADERS) as ConfigName[]); -const PORT_VARIABLES: Readonly> = { - e2e: "TEST_E2E_PORT", - visual: "TEST_VISUAL_PORT", - capabilities: "TEST_CAPABILITIES_PORT", - storybook: "TEST_STORYBOOK_PORT", - dev: "TEST_DEV_PORT", -}; -const RUN_VARIABLES = ["TEST_RUN_ID", "RUN_OUTPUT_ROOT", ...Object.values(PORT_VARIABLES)]; -const held: Server[] = []; - -afterEach(async () => { - vi.resetModules(); - await Promise.all( - held.splice(0).map((server) => new Promise((resolve) => server.close(() => resolve()))), - ); }); -/** The config as one process sees it, and what it left in that environment. */ -async function loadProcess( - name: ConfigName, - environment: Readonly> = {}, -): Promise<{ config: PlaywrightTestConfig; after: Readonly> }> { - const saved = RUN_VARIABLES.map((variable) => [variable, process.env[variable]] as const); - for (const variable of RUN_VARIABLES) delete process.env[variable]; - Object.assign(process.env, environment); - vi.resetModules(); - try { - const config = (await LOADERS[name]()).default; - return { - config, - after: Object.fromEntries(RUN_VARIABLES.map((variable) => [variable, process.env[variable]])), - }; - } finally { - for (const [variable, value] of saved) { - if (value === undefined) delete process.env[variable]; - else process.env[variable] = value; - } - } -} +afterEach(releaseRuns); -async function load( - name: ConfigName, - environment: Readonly> = {}, -): Promise { - return (await loadProcess(name, environment)).config; -} - -function portOf(config: PlaywrightTestConfig): number { - return Number(new URL(String(config.use?.baseURL)).port); -} - -/** Listens on `port` the way the first run's server does. */ -async function hold(port: number): Promise { - const server = createServer(); - held.push(server); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve()); - }); -} - -function reporterOption(config: PlaywrightTestConfig, reporter: string, option: string): string { - const reporters: readonly unknown[] = Array.isArray(config.reporter) ? config.reporter : []; - const entry = reporters.find( - (candidate) => Array.isArray(candidate) && candidate[0] === reporter, - ) as readonly [string, Record] | undefined; - return String(entry?.[1][option]); -} - -function outputs(config: PlaywrightTestConfig) { - return { - results: path.resolve(String(config.outputDir)), - report: path.resolve(reporterOption(config, "html", "outputFolder")), - junit: path.resolve(reporterOption(config, "junit", "outputFile")), - }; -} - -function webServer(config: PlaywrightTestConfig) { - const server = config.webServer; - if (!server || Array.isArray(server)) throw new TypeError("expected one web server"); - return server; -} - -const RESULT_FOLDERS: Readonly, string>> = { - e2e: "e2e", - visual: "visual", - capabilities: "browser-capabilities", - storybook: "storybook", -}; - -const INSTALLED_RESULT_FOLDERS = installed( - Object.keys(RESULT_FOLDERS) as Exclude[], -).map((name) => [name, RESULT_FOLDERS[name]] as const); - -describe.each(INSTALLED_RESULT_FOLDERS)("the %s config", (name, folder) => { - const expected = (root: string) => ({ - results: path.resolve(root, `artifacts/tests/${folder}/results`), - report: path.resolve(root, `artifacts/tests/${folder}/report`), - junit: path.resolve(root, `artifacts/tests/${folder}/results.xml`), - }); - - it("writes where the gates read without a run ID", async () => { - expect(outputs(await load(name as ConfigName))).toEqual(expected(".")); - }); - - it("gives each run ID its own results, report and JUnit file", async () => { - const a = outputs(await load(name as ConfigName, { TEST_RUN_ID: "a" })); - const b = outputs(await load(name as ConfigName, { TEST_RUN_ID: "b" })); - - expect(a).toEqual(expected(".tmp/runs/a")); - expect(b).toEqual(expected(".tmp/runs/b")); - }); -}); +describeResultFolders(runs, [ + ["e2e", "e2e"], + ["visual", "visual"], + ["storybook", "storybook"], +]); describe("the build a run serves", () => { it.each(["e2e", "visual"] as const)( "is the checkout's dist without a run ID, and the run's own with one (%s)", async (name) => { - const checkout = webServer(await load(name)); + const checkout = webServer(await runs.load(name)); expect(checkout.env?.["RUN_OUTPUT_ROOT"]).toBe("."); expect(checkout.command).toMatch(/ --outDir dist(?: |$)/u); - const run = webServer(await load(name, { TEST_RUN_ID: "a" })); + const run = webServer(await runs.load(name, { TEST_RUN_ID: "a" })); // The build writes under RUN_OUTPUT_ROOT and the preview serves it. expect(run.env?.["RUN_OUTPUT_ROOT"]).toBe(".tmp/runs/a"); expect(run.command).toMatch(/^corepack pnpm build && corepack pnpm preview /u); @@ -172,23 +64,15 @@ describe("the build a run serves", () => { }, ); - it.each(installed(["capabilities", "dev"] as const))( - "keeps the dev server's Vite cache in the run's folder (%s)", - async (name) => { - expect(webServer(await load(name)).env?.["RUN_OUTPUT_ROOT"]).toBe("."); - expect(webServer(await load(name, { TEST_RUN_ID: "a" })).env?.["RUN_OUTPUT_ROOT"]).toBe( - ".tmp/runs/a", - ); - }, - ); + itKeepsTheViteCacheInTheRun(runs, ["dev"]); it("builds the workshop into the run's folder the way build:storybook does", async () => { - const checkout = webServer(await load("storybook")).command; + const checkout = webServer(await runs.load("storybook")).command; expect(checkout).toMatch( /^corepack pnpm build:storybook && node scripts\/serve-static\.ts artifacts\/storybook\/static /u, ); - const run = webServer(await load("storybook", { TEST_RUN_ID: "a" })).command; + const run = webServer(await runs.load("storybook", { TEST_RUN_ID: "a" })).command; const script = ( JSON.parse(readFileSync("package.json", "utf8")) as { scripts: Record } ).scripts["build:storybook"]!; @@ -201,53 +85,8 @@ describe("the build a run serves", () => { }); }); -describe.each(CONFIG_NAMES)("the %s server port", (name) => { - it("is a free port the runner hands to its workers", async () => { - const runner = await loadProcess(name); - const port = portOf(runner.config); - const server = webServer(runner.config); - - // Playwright evaluates the config again in each worker, which inherits - // the runner's environment: the port is there, so baseURL agrees. - expect(runner.after[PORT_VARIABLES[name]]).toBe(String(port)); - expect(portOf(await load(name, { [PORT_VARIABLES[name]]: String(port) }))).toBe(port); - expect(server.url).toBe(`http://127.0.0.1:${port}`); - expect(server.command).toMatch(new RegExp(` (?:--port )?${port}(?: |$)`, "u")); - expect(server.reuseExistingServer).toBe(false); - }); - - it("moves for a second run while the first run's server holds its port", async () => { - const first = portOf(await load(name)); - await hold(first); - - expect(portOf(await load(name))).not.toBe(first); - }); - - it("is the one its variable pins", async () => { - const pinned = await load(name, { [PORT_VARIABLES[name]]: "40123" }); - expect(portOf(pinned)).toBe(40123); - expect(webServer(pinned).url).toBe("http://127.0.0.1:40123"); - }); -}); +describeServerPorts(runs, ["e2e", "visual", "storybook", "dev"]); describe("vite servers", () => { - it.each(installed(["e2e", "visual", "capabilities", "dev"] as const))( - "exit instead of moving to the next port when theirs is taken (%s)", - async (name) => { - // Without --strictPort vite moved on, and Playwright, waiting on the - // configured URL, tested whatever already listened there. - expect(webServer(await load(name)).command).toMatch(/ --port \d+ --strictPort(?: |$)/u); - }, - ); - - // Defined only where the capabilities config exists, not skipped: - // test:unit fails on a skipped test (0929/10-04). - if (HAS_CAPABILITIES) { - it("no longer share one port between the visual preview and the capability dev server", async () => { - const visual = portOf(await load("visual")); - await hold(visual); - - expect(portOf(await load("capabilities"))).not.toBe(visual); - }); - } + itExitsWhenItsPortIsTaken(runs, ["e2e", "visual", "dev"]); }); diff --git a/tests/unit/vite-build-inputs.test.ts b/tests/unit/vite-build-inputs.test.ts index 228272c1..c53cde92 100644 --- a/tests/unit/vite-build-inputs.test.ts +++ b/tests/unit/vite-build-inputs.test.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { REMOVABLE_TEMPLATE_FEATURES } from "../../scripts/lib/removable-template-features.ts"; + /** * 1105/15-03. Two copies of the list of files `vite build` reads from the * root (the initial JavaScript measurement and the @@ -12,14 +14,25 @@ import { describe, expect, it } from "vitest"; * now, and it has to cover what vite.config.ts imports. */ const LIST_MODULE = "scripts/lib/vite-build-inputs.ts"; -// The measurement goes with the reference feature it copies -// (test:sample-removal), so a project without that feature checks the build -// test alone. In the template, tests/unit/feature-shape-growth.test.ts -// imports the measurement and fails if the file is gone. +const MEASUREMENT = "scripts/scale/measure-feature-shape-growth.ts"; +// 1106/10-01. The measurement copies the reference sample and is one of its +// files (scripts/lib/removable-template-features.ts), so test:sample-removal +// deletes it with the sample. It is left out only where its owner is not +// installed: the owner's feature-module.ts is gone, the rule +// verify:documentation uses. The build test is a template file and is always +// read. Filtering both on "the file exists" let a renamed consumer drop out of +// the check unnoticed. The owner is found by its files, not written here: the +// sample drill fails on any test that still names the sample. +const MEASUREMENT_OWNER = REMOVABLE_TEMPLATE_FEATURES.find((feature) => + feature.extraPaths.includes(MEASUREMENT), +); +const OWNER_INSTALLED = + MEASUREMENT_OWNER !== undefined && + existsSync(`src/features/${MEASUREMENT_OWNER.featureId}/feature-module.ts`); const CONSUMERS = [ - "scripts/scale/measure-feature-shape-growth.ts", "tests/unit/product-feature-selection-build.test.ts", -].filter((file) => existsSync(file)); + ...(OWNER_INSTALLED ? [MEASUREMENT] : []), +]; const LOCAL_IMPORT = /\bfrom\s+["'](\.{1,2}\/[^"']+)["']|\bimport\s+["'](\.{1,2}\/[^"']+)["']/gu; @@ -47,6 +60,8 @@ function localImportGraph(entry: string): string[] { describe("vite build inputs (1105/15-03)", () => { it("defines the list in one module that both builds import", () => { + // The measurement may be missing only because a removable feature owns it. + expect(MEASUREMENT_OWNER, `no removable template feature owns ${MEASUREMENT}`).toBeDefined(); for (const consumer of CONSUMERS) { const source = readFileSync(consumer, "utf8"); expect(source, consumer).not.toMatch(/\bconst BUILD_INPUTS\s*=/u);