import { createServer, type Server } from "node:net"; import path from "node:path"; import type { PlaywrightTestConfig } from "@playwright/test"; import { describe, expect, it, vi } from "vitest"; /** * 1105/12-04. The run-isolation checks every Playwright config shares, for * tests/unit/playwright-run-isolation.test.ts (the four configs every project * has) and tests/unit/playwright-capabilities-run-isolation.test.ts (the * capabilities config, which goes with the browser file/storage runtime). * * 1106/10-01. Each file imports its configs by name and passes them here, so * no check depends on which config files a checkout happens to have. */ export type ConfigName = "e2e" | "visual" | "capabilities" | "storybook" | "dev"; export type ConfigLoaders = Readonly< Partial Promise<{ default: PlaywrightTestConfig }>>> >; const PORT_VARIABLES: Readonly> = { e2e: "TEST_E2E_PORT", visual: "TEST_VISUAL_PORT", capabilities: "TEST_CAPABILITIES_PORT", storybook: "TEST_STORYBOOK_PORT", dev: "TEST_DEV_PORT", }; /** The port variables of the configs' second servers. */ const SECOND_SERVER_PORT_VARIABLES = ["TEST_VISUAL_WORKSHOP_PORT"]; const RUN_VARIABLES = [ "TEST_RUN_ID", "RUN_OUTPUT_ROOT", ...Object.values(PORT_VARIABLES), ...SECOND_SERVER_PORT_VARIABLES, ]; const held: Server[] = []; /** For afterEach: forgets the loaded configs and frees the ports `hold` took. */ export async function releaseRuns(): Promise { vi.resetModules(); await Promise.all( held.splice(0).map((server) => new Promise((resolve) => server.close(() => resolve()))), ); } type ConfigProcess = Readonly<{ config: PlaywrightTestConfig; after: Readonly>; }>; export type ConfigRuns = Readonly<{ /** The config as one process sees it, and what it left in that environment. */ loadProcess( name: ConfigName, environment?: Readonly>, ): Promise; load(name: ConfigName, environment?: Readonly>): Promise; }>; /** Each load evaluates the config again with the environment of one process. */ export function configRuns(loaders: ConfigLoaders): ConfigRuns { async function loadProcess( name: ConfigName, environment: Readonly> = {}, ): Promise { const loader = loaders[name]; if (loader === undefined) throw new TypeError(`no loader for the ${name} config`); const saved = RUN_VARIABLES.map((variable) => [variable, process.env[variable]] as const); for (const variable of RUN_VARIABLES) delete process.env[variable]; Object.assign(process.env, environment); vi.resetModules(); try { const config = (await loader()).default; return { config, after: Object.fromEntries(RUN_VARIABLES.map((variable) => [variable, process.env[variable]])), }; } finally { for (const [variable, value] of saved) { if (value === undefined) delete process.env[variable]; else process.env[variable] = value; } } } return { loadProcess, load: async (name, environment = {}) => (await loadProcess(name, environment)).config, }; } export function portOf(config: PlaywrightTestConfig): number { return Number(new URL(String(config.use?.baseURL)).port); } /** Listens on `port` the way the first run's server does. */ export async function hold(port: number): Promise { const server = createServer(); held.push(server); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(port, "127.0.0.1", () => resolve()); }); } function reporterOption(config: PlaywrightTestConfig, reporter: string, option: string): string { const reporters: readonly unknown[] = Array.isArray(config.reporter) ? config.reporter : []; const entry = reporters.find( (candidate) => Array.isArray(candidate) && candidate[0] === reporter, ) as readonly [string, Record] | undefined; return String(entry?.[1][option]); } function outputs(config: PlaywrightTestConfig) { return { results: path.resolve(String(config.outputDir)), report: path.resolve(reporterOption(config, "html", "outputFolder")), junit: path.resolve(reporterOption(config, "junit", "outputFile")), }; } /** The config's own server: the first one when it starts more than one. */ export function webServer(config: PlaywrightTestConfig) { const [server] = webServers(config); if (!server) throw new TypeError("expected a web server"); return server; } export function webServers(config: PlaywrightTestConfig) { const servers = config.webServer; return servers === undefined ? [] : Array.isArray(servers) ? servers : [servers]; } /** Each config's results, report and JUnit file, by its folder under artifacts/tests. */ export function describeResultFolders( runs: ConfigRuns, folders: readonly (readonly [ConfigName, string])[], ): void { describe.each(folders)("the %s config", (name, folder) => { const expected = (root: string) => ({ results: path.resolve(root, `artifacts/tests/${folder}/results`), report: path.resolve(root, `artifacts/tests/${folder}/report`), junit: path.resolve(root, `artifacts/tests/${folder}/results.xml`), }); it("writes where the gates read without a run ID", async () => { expect(outputs(await runs.load(name as ConfigName))).toEqual(expected(".")); }); it("gives each run ID its own results, report and JUnit file", async () => { const a = outputs(await runs.load(name as ConfigName, { TEST_RUN_ID: "a" })); const b = outputs(await runs.load(name as ConfigName, { TEST_RUN_ID: "b" })); expect(a).toEqual(expected(".tmp/runs/a")); expect(b).toEqual(expected(".tmp/runs/b")); }); }); } /** Inside "the build a run serves": the dev servers' Vite cache. */ export function itKeepsTheViteCacheInTheRun(runs: ConfigRuns, names: readonly ConfigName[]): void { it.each(names)("keeps the dev server's Vite cache in the run's folder (%s)", async (name) => { expect(webServer(await runs.load(name)).env?.["RUN_OUTPUT_ROOT"]).toBe("."); expect(webServer(await runs.load(name, { TEST_RUN_ID: "a" })).env?.["RUN_OUTPUT_ROOT"]).toBe( ".tmp/runs/a", ); }); } export function describeServerPorts(runs: ConfigRuns, names: readonly ConfigName[]): void { describe.each(names)("the %s server port", (name) => { it("is a free port the runner hands to its workers", async () => { const runner = await runs.loadProcess(name); const port = portOf(runner.config); const server = webServer(runner.config); // Playwright evaluates the config again in each worker, which inherits // the runner's environment: the port is there, so baseURL agrees. expect(runner.after[PORT_VARIABLES[name]]).toBe(String(port)); expect(portOf(await runs.load(name, { [PORT_VARIABLES[name]]: String(port) }))).toBe(port); expect(server.url).toBe(`http://127.0.0.1:${port}`); expect(server.command).toMatch(new RegExp(` (?:--port )?${port}(?: |$)`, "u")); expect(server.reuseExistingServer).toBe(false); }); it("moves for a second run while the first run's server holds its port", async () => { const first = portOf(await runs.load(name)); await hold(first); expect(portOf(await runs.load(name))).not.toBe(first); }); it("is the one its variable pins", async () => { const pinned = await runs.load(name, { [PORT_VARIABLES[name]]: "40123" }); expect(portOf(pinned)).toBe(40123); expect(webServer(pinned).url).toBe("http://127.0.0.1:40123"); }); }); } /** Inside "vite servers": the configs whose server is vite. */ export function itExitsWhenItsPortIsTaken(runs: ConfigRuns, names: readonly ConfigName[]): void { it.each(names)( "exit instead of moving to the next port when theirs is taken (%s)", async (name) => { // Without --strictPort vite moved on, and Playwright, waiting on the // configured URL, tested whatever already listened there. expect(webServer(await runs.load(name)).command).toMatch(/ --port \d+ --strictPort(?: |$)/u); }, ); }