문제: 다섯 Playwright 설정의 서버 포트가 고정이다(e2e 4173, visual과 capability가 둘 다 4174, storybook 6006, dev 5173). 한 checkout이든 같은 기계의 두 worktree든 두 번째 실행은 `http://127.0.0.1:4173 is already used`로 시작하지 못한다(00-R1). dev 설정만 `reuseExistingServer: true`라 5173에 다른 checkout·브랜치의 dev 서버가 떠 있으면 말없이 그 서버를 검사한다. vite는 `--strictPort` 없이 떠서, 포트가 그 사이 잡히면 다음 포트로 옮기고 Playwright는 원래 URL의 남의 서버를 검사할 수 있다 (1105/12-04). 고른 방법: 설정마다 `reservePort(TEST_<이름>_PORT)`로 빈 포트를 고른다 (`scripts/lib/run-isolation.ts`). 커널이 port 0에 준 포트를 받고 바로 닫는다. 고른 포트를 그 변수에 써 둔다. Playwright는 worker마다 설정을 다시 평가하고 worker는 runner 환경을 물려받으므로, worker의 평가가 같은 포트를 읽어 `baseURL`이 어디서나 같다. 변수를 미리 주면 그 포트를 쓴다 (디버깅용 고정). vite 서버 넷 모두 `--strictPort`로 띄우고, dev 설정은 `reuseExistingServer: false`로 바꿨다. run ID가 없어도 빈 포트를 쓴다. 두 worktree는 출력 경로가 이미 다르고 포트만 부딪히기 때문이다. 고정 번호를 읽는 곳은 없다. spec은 `baseURL`에서 origin을 얻는다 (`resumable-upload.spec.ts`, `production-csp.spec.ts`). 버린 대안: 포트 범위를 run ID에서 계산하기. 두 checkout이 같은 run ID를 쓰거나 다른 프로그램이 그 포트를 쓰면 다시 부딪힌다. Playwright `webServer.wait`로 서버가 고른 포트를 stdout에서 읽기: `use.baseURL`은 설정을 읽을 때 정해져서 쓸 수 없다. 위험·한계: 포트를 고른 뒤 서버가 잡기 전에 다른 프로세스가 같은 포트를 잡을 수 있다. 그때는 `--strictPort`/serve-static의 listen 실패나 Playwright의 "already used"로 시작이 실패한다. 남의 서버로 검사가 새지 않는다. 기본 실행의 포트 번호가 매번 달라진다. 고정이 필요하면 `TEST_E2E_PORT=4173`처럼 준다. `test:e2e:dev`는 이제 떠 있는 dev 서버를 쓰지 않고 자기 서버를 띄운다. capability 설정이 없는 사본 (test:browser-file-storage-removal)에서는 그 설정의 포트 테스트를 돌리지 않는다. 검증: desktop batch에서 확인. node scripts/run-vitest.ts run tests/unit/run-isolation.test.ts tests/unit/playwright-run-isolation.test.ts corepack pnpm test:browser-file-storage-removal Refs: 1105/12-04
254 lines
10 KiB
TypeScript
254 lines
10 KiB
TypeScript
import { existsSync, readFileSync } from "node:fs";
|
|
import { createServer, type Server } from "node:net";
|
|
import path from "node:path";
|
|
|
|
import type { PlaywrightTestConfig } from "@playwright/test";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
|
|
/**
|
|
* 1105/12-04. Every Playwright config wrote its results, report and JUnit
|
|
* file, and the e2e and visual builds wrote `dist`, at fixed paths. A second
|
|
* run in one checkout deleted the first one's results (Playwright empties
|
|
* `outputDir` at start) and emptied the `dist` its preview was serving. Under
|
|
* TEST_RUN_ID each run gets its own paths; without it the paths stay where
|
|
* FE-GATE-008 and the CI upload read them.
|
|
*
|
|
* The servers listened on fixed ports (e2e 4173, visual and capabilities both
|
|
* 4174, storybook 6006, dev 5173), so the second run failed with "is already
|
|
* used", and the dev config reused whatever answered on 5173. Each now picks
|
|
* a free port, which the runner's environment hands to the workers.
|
|
*
|
|
* Each load evaluates the config again with the environment of one process.
|
|
*
|
|
* test:browser-file-storage-removal deletes the capabilities config with the
|
|
* capability it runs, so a project without it checks the other four. Its path
|
|
* is a variable so that typecheck does not need the file.
|
|
*/
|
|
const CAPABILITIES_CONFIG = "playwright.capabilities.config.ts";
|
|
const HAS_CAPABILITIES = existsSync(CAPABILITIES_CONFIG);
|
|
const LOADERS = {
|
|
e2e: () => import("../../playwright.config.ts"),
|
|
visual: () => import("../../playwright.visual.config.ts"),
|
|
capabilities: () =>
|
|
import(/* @vite-ignore */ `../../${CAPABILITIES_CONFIG}`) as Promise<{
|
|
default: PlaywrightTestConfig;
|
|
}>,
|
|
storybook: () => import("../../playwright.storybook.config.ts"),
|
|
dev: () => import("../../playwright.dev.config.ts"),
|
|
} as const;
|
|
type ConfigName = keyof typeof LOADERS;
|
|
/** The names among `names` whose config file this checkout has. */
|
|
function installed<T extends ConfigName>(names: readonly T[]): T[] {
|
|
return names.filter((name) => name !== "capabilities" || HAS_CAPABILITIES);
|
|
}
|
|
const CONFIG_NAMES = installed(Object.keys(LOADERS) as ConfigName[]);
|
|
const PORT_VARIABLES: Readonly<Record<ConfigName, string>> = {
|
|
e2e: "TEST_E2E_PORT",
|
|
visual: "TEST_VISUAL_PORT",
|
|
capabilities: "TEST_CAPABILITIES_PORT",
|
|
storybook: "TEST_STORYBOOK_PORT",
|
|
dev: "TEST_DEV_PORT",
|
|
};
|
|
const RUN_VARIABLES = ["TEST_RUN_ID", "RUN_OUTPUT_ROOT", ...Object.values(PORT_VARIABLES)];
|
|
const held: Server[] = [];
|
|
|
|
afterEach(async () => {
|
|
vi.resetModules();
|
|
await Promise.all(
|
|
held.splice(0).map((server) => new Promise<void>((resolve) => server.close(() => resolve()))),
|
|
);
|
|
});
|
|
|
|
/** The config as one process sees it, and what it left in that environment. */
|
|
async function loadProcess(
|
|
name: ConfigName,
|
|
environment: Readonly<Record<string, string>> = {},
|
|
): Promise<{ config: PlaywrightTestConfig; after: Readonly<Record<string, string | undefined>> }> {
|
|
const saved = RUN_VARIABLES.map((variable) => [variable, process.env[variable]] as const);
|
|
for (const variable of RUN_VARIABLES) delete process.env[variable];
|
|
Object.assign(process.env, environment);
|
|
vi.resetModules();
|
|
try {
|
|
const config = (await LOADERS[name]()).default;
|
|
return {
|
|
config,
|
|
after: Object.fromEntries(RUN_VARIABLES.map((variable) => [variable, process.env[variable]])),
|
|
};
|
|
} finally {
|
|
for (const [variable, value] of saved) {
|
|
if (value === undefined) delete process.env[variable];
|
|
else process.env[variable] = value;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function load(
|
|
name: ConfigName,
|
|
environment: Readonly<Record<string, string>> = {},
|
|
): Promise<PlaywrightTestConfig> {
|
|
return (await loadProcess(name, environment)).config;
|
|
}
|
|
|
|
function portOf(config: PlaywrightTestConfig): number {
|
|
return Number(new URL(String(config.use?.baseURL)).port);
|
|
}
|
|
|
|
/** Listens on `port` the way the first run's server does. */
|
|
async function hold(port: number): Promise<void> {
|
|
const server = createServer();
|
|
held.push(server);
|
|
await new Promise<void>((resolve, reject) => {
|
|
server.once("error", reject);
|
|
server.listen(port, "127.0.0.1", () => resolve());
|
|
});
|
|
}
|
|
|
|
function reporterOption(config: PlaywrightTestConfig, reporter: string, option: string): string {
|
|
const reporters: readonly unknown[] = Array.isArray(config.reporter) ? config.reporter : [];
|
|
const entry = reporters.find(
|
|
(candidate) => Array.isArray(candidate) && candidate[0] === reporter,
|
|
) as readonly [string, Record<string, unknown>] | undefined;
|
|
return String(entry?.[1][option]);
|
|
}
|
|
|
|
function outputs(config: PlaywrightTestConfig) {
|
|
return {
|
|
results: path.resolve(String(config.outputDir)),
|
|
report: path.resolve(reporterOption(config, "html", "outputFolder")),
|
|
junit: path.resolve(reporterOption(config, "junit", "outputFile")),
|
|
};
|
|
}
|
|
|
|
function webServer(config: PlaywrightTestConfig) {
|
|
const server = config.webServer;
|
|
if (!server || Array.isArray(server)) throw new TypeError("expected one web server");
|
|
return server;
|
|
}
|
|
|
|
const RESULT_FOLDERS: Readonly<Record<Exclude<ConfigName, "dev">, string>> = {
|
|
e2e: "e2e",
|
|
visual: "visual",
|
|
capabilities: "browser-capabilities",
|
|
storybook: "storybook",
|
|
};
|
|
|
|
const INSTALLED_RESULT_FOLDERS = installed(
|
|
Object.keys(RESULT_FOLDERS) as Exclude<ConfigName, "dev">[],
|
|
).map((name) => [name, RESULT_FOLDERS[name]] as const);
|
|
|
|
describe.each(INSTALLED_RESULT_FOLDERS)("the %s config", (name, folder) => {
|
|
const expected = (root: string) => ({
|
|
results: path.resolve(root, `artifacts/tests/${folder}/results`),
|
|
report: path.resolve(root, `artifacts/tests/${folder}/report`),
|
|
junit: path.resolve(root, `artifacts/tests/${folder}/results.xml`),
|
|
});
|
|
|
|
it("writes where the gates read without a run ID", async () => {
|
|
expect(outputs(await load(name as ConfigName))).toEqual(expected("."));
|
|
});
|
|
|
|
it("gives each run ID its own results, report and JUnit file", async () => {
|
|
const a = outputs(await load(name as ConfigName, { TEST_RUN_ID: "a" }));
|
|
const b = outputs(await load(name as ConfigName, { TEST_RUN_ID: "b" }));
|
|
|
|
expect(a).toEqual(expected(".tmp/runs/a"));
|
|
expect(b).toEqual(expected(".tmp/runs/b"));
|
|
});
|
|
});
|
|
|
|
describe("the build a run serves", () => {
|
|
it.each(["e2e", "visual"] as const)(
|
|
"is the checkout's dist without a run ID, and the run's own with one (%s)",
|
|
async (name) => {
|
|
const checkout = webServer(await load(name));
|
|
expect(checkout.env?.["RUN_OUTPUT_ROOT"]).toBe(".");
|
|
expect(checkout.command).toMatch(/ --outDir dist(?: |$)/u);
|
|
|
|
const run = webServer(await load(name, { TEST_RUN_ID: "a" }));
|
|
// The build writes under RUN_OUTPUT_ROOT and the preview serves it.
|
|
expect(run.env?.["RUN_OUTPUT_ROOT"]).toBe(".tmp/runs/a");
|
|
expect(run.command).toMatch(/^corepack pnpm build && corepack pnpm preview /u);
|
|
expect(run.command).toMatch(/ --outDir \.tmp\/runs\/a\/dist(?: |$)/u);
|
|
},
|
|
);
|
|
|
|
it.each(installed(["capabilities", "dev"] as const))(
|
|
"keeps the dev server's Vite cache in the run's folder (%s)",
|
|
async (name) => {
|
|
expect(webServer(await load(name)).env?.["RUN_OUTPUT_ROOT"]).toBe(".");
|
|
expect(webServer(await load(name, { TEST_RUN_ID: "a" })).env?.["RUN_OUTPUT_ROOT"]).toBe(
|
|
".tmp/runs/a",
|
|
);
|
|
},
|
|
);
|
|
|
|
it("builds the workshop into the run's folder the way build:storybook does", async () => {
|
|
const checkout = webServer(await load("storybook")).command;
|
|
expect(checkout).toMatch(
|
|
/^corepack pnpm build:storybook && node scripts\/serve-static\.ts artifacts\/storybook\/static /u,
|
|
);
|
|
|
|
const run = webServer(await load("storybook", { TEST_RUN_ID: "a" })).command;
|
|
const script = (
|
|
JSON.parse(readFileSync("package.json", "utf8")) as { scripts: Record<string, string> }
|
|
).scripts["build:storybook"]!;
|
|
const sameStepsOtherFolder = script
|
|
.replace("&& storybook build", "&& corepack pnpm exec storybook build")
|
|
.replace("-o artifacts/storybook/static", "-o .tmp/runs/a/artifacts/storybook/static");
|
|
expect(sameStepsOtherFolder).not.toBe(script);
|
|
expect(run.startsWith(`${sameStepsOtherFolder} && `)).toBe(true);
|
|
expect(run).toMatch(/ node scripts\/serve-static\.ts \.tmp\/runs\/a\/artifacts\/storybook\/static /u);
|
|
});
|
|
});
|
|
|
|
describe.each(CONFIG_NAMES)("the %s server port", (name) => {
|
|
it("is a free port the runner hands to its workers", async () => {
|
|
const runner = await loadProcess(name);
|
|
const port = portOf(runner.config);
|
|
const server = webServer(runner.config);
|
|
|
|
// Playwright evaluates the config again in each worker, which inherits
|
|
// the runner's environment: the port is there, so baseURL agrees.
|
|
expect(runner.after[PORT_VARIABLES[name]]).toBe(String(port));
|
|
expect(portOf(await load(name, { [PORT_VARIABLES[name]]: String(port) }))).toBe(port);
|
|
expect(server.url).toBe(`http://127.0.0.1:${port}`);
|
|
expect(server.command).toMatch(new RegExp(` (?:--port )?${port}(?: |$)`, "u"));
|
|
expect(server.reuseExistingServer).toBe(false);
|
|
});
|
|
|
|
it("moves for a second run while the first run's server holds its port", async () => {
|
|
const first = portOf(await load(name));
|
|
await hold(first);
|
|
|
|
expect(portOf(await load(name))).not.toBe(first);
|
|
});
|
|
|
|
it("is the one its variable pins", async () => {
|
|
const pinned = await load(name, { [PORT_VARIABLES[name]]: "40123" });
|
|
expect(portOf(pinned)).toBe(40123);
|
|
expect(webServer(pinned).url).toBe("http://127.0.0.1:40123");
|
|
});
|
|
});
|
|
|
|
describe("vite servers", () => {
|
|
it.each(installed(["e2e", "visual", "capabilities", "dev"] as const))(
|
|
"exit instead of moving to the next port when theirs is taken (%s)",
|
|
async (name) => {
|
|
// Without --strictPort vite moved on, and Playwright, waiting on the
|
|
// configured URL, tested whatever already listened there.
|
|
expect(webServer(await load(name)).command).toMatch(/ --port \d+ --strictPort(?: |$)/u);
|
|
},
|
|
);
|
|
|
|
// Defined only where the capabilities config exists, not skipped:
|
|
// test:unit fails on a skipped test (0929/10-04).
|
|
if (HAS_CAPABILITIES) {
|
|
it("no longer share one port between the visual preview and the capability dev server", async () => {
|
|
const visual = portOf(await load("visual"));
|
|
await hold(visual);
|
|
|
|
expect(portOf(await load("capabilities"))).not.toBe(visual);
|
|
});
|
|
}
|
|
});
|