diff --git a/playwright.capabilities.config.ts b/playwright.capabilities.config.ts index 4fded730..50061e55 100644 --- a/playwright.capabilities.config.ts +++ b/playwright.capabilities.config.ts @@ -2,13 +2,20 @@ import { defineConfig, devices } from "@playwright/test"; import { LOCAL_BUILD_IDENTITY } from "./scripts/lib/build-environment.ts"; import { + reservePort, RUN_OUTPUT_ROOT_VARIABLE, + TEST_PORT_VARIABLES, testRunPath, testRunRoot, } from "./scripts/lib/run-isolation.ts"; // 1105/12-04. Under TEST_RUN_ID the results and report are the run's own -// (.tmp/runs//), and so is the dev server's Vite cache. +// (.tmp/runs//), and so is the dev server's Vite cache. The dev server +// listens on a free port (TEST_CAPABILITIES_PORT pins one); it shared 4174 +// with the visual preview. Specs read the origin from `use.baseURL`. +const port = await reservePort(TEST_PORT_VARIABLES.capabilities); +const origin = `http://127.0.0.1:${port}`; + export default defineConfig({ testDir: "./tests/browser-capabilities", forbidOnly: true, @@ -30,7 +37,7 @@ export default defineConfig({ ], ], use: { - baseURL: "http://127.0.0.1:4174", + baseURL: origin, trace: "retain-on-failure", screenshot: "only-on-failure", }, @@ -40,9 +47,8 @@ export default defineConfig({ // built into the dev bundle it makes every page a rolling-release mismatch // that reloads once and then fails closed (CONFIG_BUILD_MISMATCH). env: { ...LOCAL_BUILD_IDENTITY, [RUN_OUTPUT_ROOT_VARIABLE]: testRunRoot() }, - command: - "corepack pnpm dev --host 127.0.0.1 --port 4174", - url: "http://127.0.0.1:4174", + command: `corepack pnpm dev --host 127.0.0.1 --port ${port} --strictPort`, + url: origin, reuseExistingServer: false, }, projects: [ diff --git a/playwright.config.ts b/playwright.config.ts index 6c17f4a7..1ea46419 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,7 +1,9 @@ import { defineConfig, devices } from "@playwright/test"; import { + reservePort, RUN_OUTPUT_ROOT_VARIABLE, + TEST_PORT_VARIABLES, testRunPath, testRunRoot, } from "./scripts/lib/run-isolation.ts"; @@ -10,8 +12,12 @@ import { // its preview serves are under .tmp/runs// (scripts/lib/run-isolation.ts), // so a second run in the same checkout neither empties this run's dist nor // deletes its results. Unset, the paths are artifacts/tests/e2e/* and dist, -// which FE-GATE-008 and the CI upload read. +// which FE-GATE-008 and the CI upload read. The preview listens on a free +// port (TEST_E2E_PORT pins one); a fixed 4173 made the second run fail with +// "is already used". const dist = testRunPath("dist"); +const port = await reservePort(TEST_PORT_VARIABLES.e2e); +const origin = `http://127.0.0.1:${port}`; export default defineConfig({ testDir: "./tests/e2e", @@ -23,15 +29,15 @@ export default defineConfig({ ["junit", { outputFile: testRunPath("artifacts/tests/e2e/results.xml") }], ], use: { - baseURL: "http://127.0.0.1:4173", + baseURL: origin, trace: "retain-on-failure", screenshot: "only-on-failure", }, webServer: { env: { [RUN_OUTPUT_ROOT_VARIABLE]: testRunRoot() }, command: - `corepack pnpm build && corepack pnpm preview --host 127.0.0.1 --port 4173 --outDir ${dist}`, - url: "http://127.0.0.1:4173", + `corepack pnpm build && corepack pnpm preview --host 127.0.0.1 --port ${port} --strictPort --outDir ${dist}`, + url: origin, reuseExistingServer: false, }, projects: [ diff --git a/playwright.dev.config.ts b/playwright.dev.config.ts index bc63a678..0c772faf 100644 --- a/playwright.dev.config.ts +++ b/playwright.dev.config.ts @@ -2,13 +2,25 @@ import { defineConfig } from "@playwright/test"; import releaseConfig from "./playwright.config.ts"; import { LOCAL_BUILD_IDENTITY } from "./scripts/lib/build-environment.ts"; -import { RUN_OUTPUT_ROOT_VARIABLE, testRunRoot } from "./scripts/lib/run-isolation.ts"; +import { + reservePort, + RUN_OUTPUT_ROOT_VARIABLE, + TEST_PORT_VARIABLES, + testRunRoot, +} from "./scripts/lib/run-isolation.ts"; + +// 1105/12-04. The dev server used to reuse whatever already answered on 5173, +// so the specs could run against another checkout's or another branch's dev +// server without saying so. It now starts its own on a free port +// (TEST_DEV_PORT pins one) and never reuses a server. +const port = await reservePort(TEST_PORT_VARIABLES.dev); +const origin = `http://127.0.0.1:${port}`; export default defineConfig({ ...releaseConfig, use: { ...releaseConfig.use, - baseURL: "http://127.0.0.1:5173", + baseURL: origin, }, webServer: { // The dev server serves public/config.json, the local profile, whose @@ -17,8 +29,8 @@ export default defineConfig({ // that reloads once and then fails closed (CONFIG_BUILD_MISMATCH). // 1105/12-04. Under TEST_RUN_ID, Vite's cache is the run's own. env: { ...LOCAL_BUILD_IDENTITY, [RUN_OUTPUT_ROOT_VARIABLE]: testRunRoot() }, - command: "corepack pnpm dev --host 127.0.0.1", - url: "http://127.0.0.1:5173", - reuseExistingServer: true, + command: `corepack pnpm dev --host 127.0.0.1 --port ${port} --strictPort`, + url: origin, + reuseExistingServer: false, }, }); diff --git a/playwright.storybook.config.ts b/playwright.storybook.config.ts index de5dbc76..a36bfb5f 100644 --- a/playwright.storybook.config.ts +++ b/playwright.storybook.config.ts @@ -1,12 +1,20 @@ import { defineConfig, devices } from "@playwright/test"; -import { testRunPath, testRunRoot } from "./scripts/lib/run-isolation.ts"; +import { + reservePort, + TEST_PORT_VARIABLES, + testRunPath, + testRunRoot, +} from "./scripts/lib/run-isolation.ts"; // 1105/12-04. Under TEST_RUN_ID the workshop is built into the run's own // folder (.tmp/runs//artifacts/storybook/static) with build:storybook's // two steps and another output folder, and the results and report are the // run's own. Unset, the gate's build:storybook writes artifacts/storybook/static. +// The static server listens on a free port (TEST_STORYBOOK_PORT pins one). const staticDirectory = testRunPath("artifacts/storybook/static"); +const port = await reservePort(TEST_PORT_VARIABLES.storybook); +const origin = `http://127.0.0.1:${port}`; const buildWorkshop = testRunRoot() === "." ? "corepack pnpm build:storybook" @@ -32,14 +40,14 @@ export default defineConfig({ ], ], use: { - baseURL: "http://127.0.0.1:6006", + baseURL: origin, trace: "retain-on-failure", screenshot: "only-on-failure", }, webServer: { command: - `${buildWorkshop} && node scripts/serve-static.ts ${staticDirectory} 6006`, - url: "http://127.0.0.1:6006", + `${buildWorkshop} && node scripts/serve-static.ts ${staticDirectory} ${port}`, + url: origin, reuseExistingServer: false, }, projects: [ diff --git a/playwright.visual.config.ts b/playwright.visual.config.ts index e7ff7d45..977fd864 100644 --- a/playwright.visual.config.ts +++ b/playwright.visual.config.ts @@ -4,14 +4,20 @@ import { defineConfig, devices } from "@playwright/test"; import { LOCAL_BUILD_IDENTITY } from "./scripts/lib/build-environment.ts"; import { + reservePort, RUN_OUTPUT_ROOT_VARIABLE, + TEST_PORT_VARIABLES, testRunPath, testRunRoot, } from "./scripts/lib/run-isolation.ts"; // 1105/12-04. Under TEST_RUN_ID the results, report and build are the run's // own (.tmp/runs//); the goldens in snapshotDir are shared and only read. +// The preview listens on a free port (TEST_VISUAL_PORT pins one); it shared +// 4174 with the capability dev server. const dist = testRunPath("dist"); +const port = await reservePort(TEST_PORT_VARIABLES.visual); +const origin = `http://127.0.0.1:${port}`; export default defineConfig({ testDir: "./tests/visual", @@ -40,7 +46,7 @@ export default defineConfig({ }, use: { ...devices["Desktop Chrome"], - baseURL: "http://127.0.0.1:4174", + baseURL: origin, colorScheme: "light", locale: "en-US", trace: "retain-on-failure", @@ -65,8 +71,8 @@ export default defineConfig({ // 1103/11-08: CI too, or the build is checked as a release build. env: { ...LOCAL_BUILD_IDENTITY, [RUN_OUTPUT_ROOT_VARIABLE]: testRunRoot() }, command: - `corepack pnpm build && corepack pnpm preview --host 127.0.0.1 --port 4174 --outDir ${dist}`, - url: "http://127.0.0.1:4174", + `corepack pnpm build && corepack pnpm preview --host 127.0.0.1 --port ${port} --strictPort --outDir ${dist}`, + url: origin, reuseExistingServer: false, }, projects: [ diff --git a/scripts/lib/run-isolation.ts b/scripts/lib/run-isolation.ts index 37068f7f..6475532d 100644 --- a/scripts/lib/run-isolation.ts +++ b/scripts/lib/run-isolation.ts @@ -1,3 +1,4 @@ +import { createServer } from "node:net"; import path from "node:path"; /** @@ -24,6 +25,10 @@ import path from "node:path"; * Tailwind's source scan (a Playwright report there would add classes to the * CSS), Vite's file watcher, Vitest and the e2e helpers that copy the * repository. + * + * Ports are separate from the run ID: every config serves on a free port + * unless its `TEST_*_PORT` variable pins one (`TEST_PORT_VARIABLES`), so two + * checkouts on one machine do not collide either. */ export const TEST_RUN_ID_VARIABLE = "TEST_RUN_ID"; export const RUN_OUTPUT_ROOT_VARIABLE = "RUN_OUTPUT_ROOT"; @@ -77,6 +82,68 @@ export function runOutputPath( return path.posix.join(runOutputRoot(environment), relativePath); } +/** + * The port variable of each Playwright config's web server. Unset, the config + * picks a free port; set, it uses that one (to pin a port while debugging). + */ +export const TEST_PORT_VARIABLES = Object.freeze({ + e2e: "TEST_E2E_PORT", + visual: "TEST_VISUAL_PORT", + capabilities: "TEST_CAPABILITIES_PORT", + storybook: "TEST_STORYBOOK_PORT", + dev: "TEST_DEV_PORT", +} as const); + +/** + * A port nothing listens on now: the one the kernel gives a socket bound to + * port 0, which is closed again. Another process can still take it before the + * web server binds it. The configs start their servers with `--strictPort` + * (vite) or a fixed `listen` (serve-static) and `reuseExistingServer: false`, + * so that case is a start failure, never a run against someone else's server. + */ +export async function findFreePort(host = "127.0.0.1"): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.unref(); + server.once("error", reject); + server.listen(0, host, () => { + const address = server.address(); + server.close(() => { + if (address !== null && typeof address === "object") resolve(address.port); + else reject(new Error(`no port was bound on ${host}`)); + }); + }); + }); +} + +/** + * The port one config's web server listens on, kept in `variable`. + * + * Playwright evaluates the config again in each worker process, and the + * workers inherit the runner's environment. The runner's evaluation writes + * the port it picked into `variable`; the workers' evaluations read it, so + * `baseURL` is the same everywhere. A value already set is used as given. + */ +export async function reservePort( + variable: string, + environment: Record = process.env, + find: () => Promise = findFreePort, +): Promise { + const pinned = environment[variable]; + if (pinned !== undefined && pinned !== "") { + const port = Number(pinned); + if (!/^\d+$/u.test(pinned) || port < 1 || port > 65_535) { + throw new TypeError( + `${variable} must be a port from 1 to 65535; received ${JSON.stringify(pinned)}`, + ); + } + return port; + } + const port = await find(); + environment[variable] = String(port); + return port; +} + /** * The import specifier a generated module at `fromFile` uses for `toFile`, * both relative to the checkout. A generated module under an output root sits diff --git a/tests/unit/playwright-run-isolation.test.ts b/tests/unit/playwright-run-isolation.test.ts index 22abc97e..2f70cb22 100644 --- a/tests/unit/playwright-run-isolation.test.ts +++ b/tests/unit/playwright-run-isolation.test.ts @@ -1,4 +1,5 @@ import { existsSync, readFileSync } from "node:fs"; +import { createServer, type Server } from "node:net"; import path from "node:path"; import type { PlaywrightTestConfig } from "@playwright/test"; @@ -12,7 +13,12 @@ import { afterEach, describe, expect, it, vi } from "vitest"; * TEST_RUN_ID each run gets its own paths; without it the paths stay where * FE-GATE-008 and the CI upload read them. * - * Each load evaluates the config again with the environment of one run. + * The servers listened on fixed ports (e2e 4173, visual and capabilities both + * 4174, storybook 6006, dev 5173), so the second run failed with "is already + * used", and the dev config reused whatever answered on 5173. Each now picks + * a free port, which the runner's environment hands to the workers. + * + * Each load evaluates the config again with the environment of one process. * * test:browser-file-storage-removal deletes the capabilities config with the * capability it runs, so a project without it checks the other four. Its path @@ -35,22 +41,39 @@ type ConfigName = keyof typeof LOADERS; function installed(names: readonly T[]): T[] { return names.filter((name) => name !== "capabilities" || HAS_CAPABILITIES); } -const RUN_VARIABLES = ["TEST_RUN_ID", "RUN_OUTPUT_ROOT"] as const; +const CONFIG_NAMES = installed(Object.keys(LOADERS) as ConfigName[]); +const PORT_VARIABLES: Readonly> = { + e2e: "TEST_E2E_PORT", + visual: "TEST_VISUAL_PORT", + capabilities: "TEST_CAPABILITIES_PORT", + storybook: "TEST_STORYBOOK_PORT", + dev: "TEST_DEV_PORT", +}; +const RUN_VARIABLES = ["TEST_RUN_ID", "RUN_OUTPUT_ROOT", ...Object.values(PORT_VARIABLES)]; +const held: Server[] = []; -afterEach(() => { +afterEach(async () => { vi.resetModules(); + await Promise.all( + held.splice(0).map((server) => new Promise((resolve) => server.close(() => resolve()))), + ); }); -async function load( +/** The config as one process sees it, and what it left in that environment. */ +async function loadProcess( name: ConfigName, environment: Readonly> = {}, -): Promise { +): Promise<{ config: PlaywrightTestConfig; after: Readonly> }> { const saved = RUN_VARIABLES.map((variable) => [variable, process.env[variable]] as const); for (const variable of RUN_VARIABLES) delete process.env[variable]; Object.assign(process.env, environment); vi.resetModules(); try { - return (await LOADERS[name]()).default; + const config = (await LOADERS[name]()).default; + return { + config, + after: Object.fromEntries(RUN_VARIABLES.map((variable) => [variable, process.env[variable]])), + }; } finally { for (const [variable, value] of saved) { if (value === undefined) delete process.env[variable]; @@ -59,6 +82,27 @@ async function load( } } +async function load( + name: ConfigName, + environment: Readonly> = {}, +): Promise { + return (await loadProcess(name, environment)).config; +} + +function portOf(config: PlaywrightTestConfig): number { + return Number(new URL(String(config.use?.baseURL)).port); +} + +/** Listens on `port` the way the first run's server does. */ +async function hold(port: number): Promise { + const server = createServer(); + held.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve()); + }); +} + function reporterOption(config: PlaywrightTestConfig, reporter: string, option: string): string { const reporters: readonly unknown[] = Array.isArray(config.reporter) ? config.reporter : []; const entry = reporters.find( @@ -156,3 +200,54 @@ describe("the build a run serves", () => { expect(run).toMatch(/ node scripts\/serve-static\.ts \.tmp\/runs\/a\/artifacts\/storybook\/static /u); }); }); + +describe.each(CONFIG_NAMES)("the %s server port", (name) => { + it("is a free port the runner hands to its workers", async () => { + const runner = await loadProcess(name); + const port = portOf(runner.config); + const server = webServer(runner.config); + + // Playwright evaluates the config again in each worker, which inherits + // the runner's environment: the port is there, so baseURL agrees. + expect(runner.after[PORT_VARIABLES[name]]).toBe(String(port)); + expect(portOf(await load(name, { [PORT_VARIABLES[name]]: String(port) }))).toBe(port); + expect(server.url).toBe(`http://127.0.0.1:${port}`); + expect(server.command).toMatch(new RegExp(` (?:--port )?${port}(?: |$)`, "u")); + expect(server.reuseExistingServer).toBe(false); + }); + + it("moves for a second run while the first run's server holds its port", async () => { + const first = portOf(await load(name)); + await hold(first); + + expect(portOf(await load(name))).not.toBe(first); + }); + + it("is the one its variable pins", async () => { + const pinned = await load(name, { [PORT_VARIABLES[name]]: "40123" }); + expect(portOf(pinned)).toBe(40123); + expect(webServer(pinned).url).toBe("http://127.0.0.1:40123"); + }); +}); + +describe("vite servers", () => { + it.each(installed(["e2e", "visual", "capabilities", "dev"] as const))( + "exit instead of moving to the next port when theirs is taken (%s)", + async (name) => { + // Without --strictPort vite moved on, and Playwright, waiting on the + // configured URL, tested whatever already listened there. + expect(webServer(await load(name)).command).toMatch(/ --port \d+ --strictPort(?: |$)/u); + }, + ); + + // Defined only where the capabilities config exists, not skipped: + // test:unit fails on a skipped test (0929/10-04). + if (HAS_CAPABILITIES) { + it("no longer share one port between the visual preview and the capability dev server", async () => { + const visual = portOf(await load("visual")); + await hold(visual); + + expect(portOf(await load("capabilities"))).not.toBe(visual); + }); + } +}); diff --git a/tests/unit/run-isolation.test.ts b/tests/unit/run-isolation.test.ts index 4bd739f3..ddc33a5f 100644 --- a/tests/unit/run-isolation.test.ts +++ b/tests/unit/run-isolation.test.ts @@ -1,13 +1,36 @@ -import { describe, expect, it } from "vitest"; +import { createServer, type Server } from "node:net"; + +import { afterEach, describe, expect, it, vi } from "vitest"; import { + findFreePort, moduleSpecifier, + reservePort, runOutputPath, runOutputRoot, testRunPath, testRunRoot, } from "../../scripts/lib/run-isolation.ts"; +const servers: Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map((server) => new Promise((resolve) => server.close(() => resolve()))), + ); +}); + +async function listen(port: number): Promise { + const server = createServer(); + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + return typeof address === "object" && address !== null ? address.port : -1; +} + /** * 1105/12-04. Builds and Playwright runs wrote to fixed paths (`dist`, * `artifacts/tests/*`), so a second run in one checkout emptied the first @@ -70,6 +93,45 @@ describe("build output root (RUN_OUTPUT_ROOT)", () => { }); }); +describe("server ports", () => { + it("finds a port a server can listen on right away", async () => { + const port = await findFreePort(); + + expect(port).toBeGreaterThan(0); + expect(await listen(port)).toBe(port); + }); + + it("never offers a port another run's server holds", async () => { + const taken = await listen(0); + + for (let attempt = 0; attempt < 20; attempt += 1) { + expect(await findFreePort()).not.toBe(taken); + } + }); + + it("keeps the port it picked in the variable, so the next evaluation reuses it", async () => { + const environment: Record = {}; + const find = vi.fn(async () => 41_234); + + expect(await reservePort("TEST_E2E_PORT", environment, find)).toBe(41_234); + expect(environment["TEST_E2E_PORT"]).toBe("41234"); + // A Playwright worker: same config, the runner's environment. + expect(await reservePort("TEST_E2E_PORT", environment, find)).toBe(41_234); + expect(find).toHaveBeenCalledTimes(1); + }); + + it("uses a port the caller pinned and refuses one that is not a port", async () => { + const find = vi.fn(async () => 41_234); + + expect(await reservePort("TEST_DEV_PORT", { TEST_DEV_PORT: "5173" }, find)).toBe(5173); + expect(find).not.toHaveBeenCalled(); + for (const value of ["0", "65536", "-1", "4173x", "abc", "41.5"]) { + await expect(reservePort("TEST_DEV_PORT", { TEST_DEV_PORT: value }, find), value) + .rejects.toThrow(/TEST_DEV_PORT/u); + } + }); +}); + describe("generated module specifiers", () => { it("keep today's specifier in .generated and reach src from a deeper root", () => { expect(