import { existsSync, readFileSync } from "node:fs"; import { createServer, type Server } from "node:net"; import path from "node:path"; import type { PlaywrightTestConfig } from "@playwright/test"; import { afterEach, describe, expect, it, vi } from "vitest"; /** * 1105/12-04. Every Playwright config wrote its results, report and JUnit * file, and the e2e and visual builds wrote `dist`, at fixed paths. A second * run in one checkout deleted the first one's results (Playwright empties * `outputDir` at start) and emptied the `dist` its preview was serving. Under * TEST_RUN_ID each run gets its own paths; without it the paths stay where * FE-GATE-008 and the CI upload read them. * * The servers listened on fixed ports (e2e 4173, visual and capabilities both * 4174, storybook 6006, dev 5173), so the second run failed with "is already * used", and the dev config reused whatever answered on 5173. Each now picks * a free port, which the runner's environment hands to the workers. * * Each load evaluates the config again with the environment of one process. * * test:browser-file-storage-removal deletes the capabilities config with the * capability it runs, so a project without it checks the other four. Its path * is a variable so that typecheck does not need the file. */ const CAPABILITIES_CONFIG = "playwright.capabilities.config.ts"; const HAS_CAPABILITIES = existsSync(CAPABILITIES_CONFIG); const LOADERS = { e2e: () => import("../../playwright.config.ts"), visual: () => import("../../playwright.visual.config.ts"), capabilities: () => import(/* @vite-ignore */ `../../${CAPABILITIES_CONFIG}`) as Promise<{ default: PlaywrightTestConfig; }>, storybook: () => import("../../playwright.storybook.config.ts"), dev: () => import("../../playwright.dev.config.ts"), } as const; type ConfigName = keyof typeof LOADERS; /** The names among `names` whose config file this checkout has. */ function installed(names: readonly T[]): T[] { return names.filter((name) => name !== "capabilities" || HAS_CAPABILITIES); } const CONFIG_NAMES = installed(Object.keys(LOADERS) as ConfigName[]); const PORT_VARIABLES: Readonly> = { e2e: "TEST_E2E_PORT", visual: "TEST_VISUAL_PORT", capabilities: "TEST_CAPABILITIES_PORT", storybook: "TEST_STORYBOOK_PORT", dev: "TEST_DEV_PORT", }; const RUN_VARIABLES = ["TEST_RUN_ID", "RUN_OUTPUT_ROOT", ...Object.values(PORT_VARIABLES)]; const held: Server[] = []; afterEach(async () => { vi.resetModules(); await Promise.all( held.splice(0).map((server) => new Promise((resolve) => server.close(() => resolve()))), ); }); /** The config as one process sees it, and what it left in that environment. */ async function loadProcess( name: ConfigName, environment: Readonly> = {}, ): Promise<{ config: PlaywrightTestConfig; after: Readonly> }> { const saved = RUN_VARIABLES.map((variable) => [variable, process.env[variable]] as const); for (const variable of RUN_VARIABLES) delete process.env[variable]; Object.assign(process.env, environment); vi.resetModules(); try { const config = (await LOADERS[name]()).default; return { config, after: Object.fromEntries(RUN_VARIABLES.map((variable) => [variable, process.env[variable]])), }; } finally { for (const [variable, value] of saved) { if (value === undefined) delete process.env[variable]; else process.env[variable] = value; } } } async function load( name: ConfigName, environment: Readonly> = {}, ): Promise { return (await loadProcess(name, environment)).config; } function portOf(config: PlaywrightTestConfig): number { return Number(new URL(String(config.use?.baseURL)).port); } /** Listens on `port` the way the first run's server does. */ async function hold(port: number): Promise { const server = createServer(); held.push(server); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(port, "127.0.0.1", () => resolve()); }); } function reporterOption(config: PlaywrightTestConfig, reporter: string, option: string): string { const reporters: readonly unknown[] = Array.isArray(config.reporter) ? config.reporter : []; const entry = reporters.find( (candidate) => Array.isArray(candidate) && candidate[0] === reporter, ) as readonly [string, Record] | undefined; return String(entry?.[1][option]); } function outputs(config: PlaywrightTestConfig) { return { results: path.resolve(String(config.outputDir)), report: path.resolve(reporterOption(config, "html", "outputFolder")), junit: path.resolve(reporterOption(config, "junit", "outputFile")), }; } function webServer(config: PlaywrightTestConfig) { const server = config.webServer; if (!server || Array.isArray(server)) throw new TypeError("expected one web server"); return server; } const RESULT_FOLDERS: Readonly, string>> = { e2e: "e2e", visual: "visual", capabilities: "browser-capabilities", storybook: "storybook", }; const INSTALLED_RESULT_FOLDERS = installed( Object.keys(RESULT_FOLDERS) as Exclude[], ).map((name) => [name, RESULT_FOLDERS[name]] as const); describe.each(INSTALLED_RESULT_FOLDERS)("the %s config", (name, folder) => { const expected = (root: string) => ({ results: path.resolve(root, `artifacts/tests/${folder}/results`), report: path.resolve(root, `artifacts/tests/${folder}/report`), junit: path.resolve(root, `artifacts/tests/${folder}/results.xml`), }); it("writes where the gates read without a run ID", async () => { expect(outputs(await load(name as ConfigName))).toEqual(expected(".")); }); it("gives each run ID its own results, report and JUnit file", async () => { const a = outputs(await load(name as ConfigName, { TEST_RUN_ID: "a" })); const b = outputs(await load(name as ConfigName, { TEST_RUN_ID: "b" })); expect(a).toEqual(expected(".tmp/runs/a")); expect(b).toEqual(expected(".tmp/runs/b")); }); }); describe("the build a run serves", () => { it.each(["e2e", "visual"] as const)( "is the checkout's dist without a run ID, and the run's own with one (%s)", async (name) => { const checkout = webServer(await load(name)); expect(checkout.env?.["RUN_OUTPUT_ROOT"]).toBe("."); expect(checkout.command).toMatch(/ --outDir dist(?: |$)/u); const run = webServer(await load(name, { TEST_RUN_ID: "a" })); // The build writes under RUN_OUTPUT_ROOT and the preview serves it. expect(run.env?.["RUN_OUTPUT_ROOT"]).toBe(".tmp/runs/a"); expect(run.command).toMatch(/^corepack pnpm build && corepack pnpm preview /u); expect(run.command).toMatch(/ --outDir \.tmp\/runs\/a\/dist(?: |$)/u); }, ); it.each(installed(["capabilities", "dev"] as const))( "keeps the dev server's Vite cache in the run's folder (%s)", async (name) => { expect(webServer(await load(name)).env?.["RUN_OUTPUT_ROOT"]).toBe("."); expect(webServer(await load(name, { TEST_RUN_ID: "a" })).env?.["RUN_OUTPUT_ROOT"]).toBe( ".tmp/runs/a", ); }, ); it("builds the workshop into the run's folder the way build:storybook does", async () => { const checkout = webServer(await load("storybook")).command; expect(checkout).toMatch( /^corepack pnpm build:storybook && node scripts\/serve-static\.ts artifacts\/storybook\/static /u, ); const run = webServer(await load("storybook", { TEST_RUN_ID: "a" })).command; const script = ( JSON.parse(readFileSync("package.json", "utf8")) as { scripts: Record } ).scripts["build:storybook"]!; const sameStepsOtherFolder = script .replace("&& storybook build", "&& corepack pnpm exec storybook build") .replace("-o artifacts/storybook/static", "-o .tmp/runs/a/artifacts/storybook/static"); expect(sameStepsOtherFolder).not.toBe(script); expect(run.startsWith(`${sameStepsOtherFolder} && `)).toBe(true); expect(run).toMatch(/ node scripts\/serve-static\.ts \.tmp\/runs\/a\/artifacts\/storybook\/static /u); }); }); describe.each(CONFIG_NAMES)("the %s server port", (name) => { it("is a free port the runner hands to its workers", async () => { const runner = await loadProcess(name); const port = portOf(runner.config); const server = webServer(runner.config); // Playwright evaluates the config again in each worker, which inherits // the runner's environment: the port is there, so baseURL agrees. expect(runner.after[PORT_VARIABLES[name]]).toBe(String(port)); expect(portOf(await load(name, { [PORT_VARIABLES[name]]: String(port) }))).toBe(port); expect(server.url).toBe(`http://127.0.0.1:${port}`); expect(server.command).toMatch(new RegExp(` (?:--port )?${port}(?: |$)`, "u")); expect(server.reuseExistingServer).toBe(false); }); it("moves for a second run while the first run's server holds its port", async () => { const first = portOf(await load(name)); await hold(first); expect(portOf(await load(name))).not.toBe(first); }); it("is the one its variable pins", async () => { const pinned = await load(name, { [PORT_VARIABLES[name]]: "40123" }); expect(portOf(pinned)).toBe(40123); expect(webServer(pinned).url).toBe("http://127.0.0.1:40123"); }); }); describe("vite servers", () => { it.each(installed(["e2e", "visual", "capabilities", "dev"] as const))( "exit instead of moving to the next port when theirs is taken (%s)", async (name) => { // Without --strictPort vite moved on, and Playwright, waiting on the // configured URL, tested whatever already listened there. expect(webServer(await load(name)).command).toMatch(/ --port \d+ --strictPort(?: |$)/u); }, ); // Defined only where the capabilities config exists, not skipped: // test:unit fails on a skipped test (0929/10-04). if (HAS_CAPABILITIES) { it("no longer share one port between the visual preview and the capability dev server", async () => { const visual = portOf(await load("visual")); await hold(visual); expect(portOf(await load("capabilities"))).not.toBe(visual); }); } });