import { spawnSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { constants } from "node:fs"; import { access, readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { formatProviderPrerequisiteUnitName, MINIMUM_PROVIDER_SYSTEMD_VERSION, systemdRunPrerequisiteProbeArguments, USER_MANAGER_VERSION_ARGUMENTS, } from "./lib/provider-cgroup.ts"; import { PROVIDER_SANDBOX_NAMESPACE_ARGUMENTS } from "./lib/provider-sandbox-namespaces.ts"; /** What the check reads from the host; a test passes a fake. */ export type SystemTestHost = Readonly<{ /** Resolves when `target` is executable, rejects otherwise. */ access(target: string): Promise; readFile(target: string): Promise; spawn( command: string, args: readonly string[], timeoutMs: number, ): Readonly<{ status: number | null; stdout?: string; stderr: string; error?: Error | undefined }>; /** Names the provider-shaped scope the probe starts; a test passes a fixed one. */ probeUnitName?: () => string; }>; export const nodeSystemTestHost: SystemTestHost = Object.freeze({ access: async (target: string) => access(target, constants.X_OK), readFile: async (target: string) => readFile(target, "utf8"), spawn: (command: string, args: readonly string[], timeoutMs: number) => { const result = spawnSync(command, [...args], { encoding: "utf8", timeout: timeoutMs }); return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? "", error: result.error }; }, probeUnitName: defaultProbeUnitName, }); function defaultProbeUnitName(): string { return formatProviderPrerequisiteUnitName(process.pid, randomBytes(12).toString("hex")); } const BWRAP = "/usr/bin/bwrap"; const SYSTEMCTL = "/usr/bin/systemctl"; const SYSTEMD_RUN = "/usr/bin/systemd-run"; /** * 1105/12-01. The provider sandbox's own namespace options on /bin/true. The * check used to see only that /usr/bin/bwrap was executable, so a host that * refuses unprivileged user namespaces passed it and then failed 18 provider * tests with `bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted`. */ export const NAMESPACE_PROBE_ARGUMENTS: readonly string[] = Object.freeze([ ...PROVIDER_SANDBOX_NAMESPACE_ARGUMENTS, "--ro-bind", "/", "/", "--", "/bin/true", ]); /** Host settings that decide whether an unprivileged process may create user namespaces. */ const USER_NAMESPACE_SETTINGS = Object.freeze([ ["kernel.apparmor_restrict_unprivileged_userns", "/proc/sys/kernel/apparmor_restrict_unprivileged_userns"], ["kernel.unprivileged_userns_clone", "/proc/sys/kernel/unprivileged_userns_clone"], ["user.max_user_namespaces", "/proc/sys/user/max_user_namespaces"], ] as const); const PROBE_TIMEOUT_MS = 5_000; const SCOPE_PROBE_TIMEOUT_MS = 15_000; const MAX_REPORTED_STDERR_LINES = 4; export async function findSystemTestPrerequisiteFailures( host: SystemTestHost = nodeSystemTestHost, ): Promise { const failures: string[] = []; const executable = new Map(); for (const [target, label] of [ [BWRAP, "bubblewrap"], ["/usr/bin/prlimit", "prlimit"], [SYSTEMD_RUN, "systemd-run"], [SYSTEMCTL, "systemctl"], ["/usr/bin/tar", "tar"], ] as const) { try { await host.access(target); executable.set(target, true); } catch { executable.set(target, false); failures.push(`${label} is required at ${target}`); } } try { const controllers = await host.readFile("/sys/fs/cgroup/cgroup.controllers"); if (controllers.trim().length === 0) { failures.push("cgroup v2 controllers are unavailable"); } } catch { failures.push("cgroup v2 is required at /sys/fs/cgroup/cgroup.controllers"); } // 1112/08-03. The provider asks the user manager (not the system manager) // for its version and then starts its sandbox in a `systemd-run --user // --scope` (scripts/run-and-validate-provider.ts). The check makes the same // calls, so a host without a user bus, with an old manager, or without // delegated controllers fails here by name instead of in each provider test. if (executable.get(SYSTEMCTL)) { const userManager = findUserManagerFailure(host); if (userManager !== undefined) { failures.push(userManager); } else if (executable.get(SYSTEMD_RUN)) { const scope = findProviderScopeFailure(host); if (scope !== undefined) failures.push(scope); } } if (executable.get(BWRAP)) { const failure = await findProviderSandboxNamespaceFailure(host); if (failure !== undefined) failures.push(failure); } return failures; } function findUserManagerFailure(host: SystemTestHost): string | undefined { const probe = host.spawn(SYSTEMCTL, USER_MANAGER_VERSION_ARGUMENTS, 3_000); const command = `${SYSTEMCTL} ${USER_MANAGER_VERSION_ARGUMENTS.join(" ")}`; if (probe.error || probe.status !== 0) { return [ `a reachable systemd user manager is required (${command} ${outcomeOf(probe)}).`, ...stderrLines(probe.stderr, "systemctl"), " The provider runs in a `systemd-run --user --scope`; give the runner user a user", " manager (for example `loginctl enable-linger `) and a user bus in XDG_RUNTIME_DIR.", ].join("\n"); } const reported = (probe.stdout ?? "").trim(); const match = /^([0-9]+)/u.exec(reported); const version = match ? Number(match[1]) : Number.NaN; if (!Number.isSafeInteger(version) || version < MINIMUM_PROVIDER_SYSTEMD_VERSION) { return `systemd user manager ${MINIMUM_PROVIDER_SYSTEMD_VERSION} or newer is required (${command} reported ${JSON.stringify(reported)})`; } return undefined; } function findProviderScopeFailure(host: SystemTestHost): string | undefined { const args = systemdRunPrerequisiteProbeArguments((host.probeUnitName ?? defaultProbeUnitName)()); const probe = host.spawn(SYSTEMD_RUN, args, SCOPE_PROBE_TIMEOUT_MS); if (!probe.error && probe.status === 0) return undefined; return [ "systemd-run could not start the provider's cgroup scope around /bin/true " + `(${SYSTEMD_RUN} --user --scope ... ${outcomeOf(probe)}).`, ...stderrLines(probe.stderr, "systemd-run"), " The scope needs the memory, pids and cpu controllers delegated to the user manager", " (Delegate= in user@.service) and cgroup v2. Runner requirements: docs/operations/ci-quality-gates.md.", ].join("\n"); } function outcomeOf(probe: ReturnType): string { return probe.error ? `did not finish: ${probe.error.message}` : `exited ${String(probe.status)}`; } function stderrLines(stderr: string, label: string): string[] { const lines = stderr .split("\n") .map((line) => line.trim()) .filter((line) => line.length > 0) .slice(0, MAX_REPORTED_STDERR_LINES); return lines.length > 0 ? lines.map((line) => ` ${label} stderr: ${line}`) : [` ${label} stderr: (empty)`]; } /** * 1101/13-03. Whether this host lets the provider sandbox create its * namespaces, and if not, why and what to do. The release provider jobs run * it before the provider (scripts/run-and-validate-provider.ts). */ export async function findProviderSandboxNamespaceFailure( host: SystemTestHost = nodeSystemTestHost, ): Promise { const probe = host.spawn(BWRAP, NAMESPACE_PROBE_ARGUMENTS, PROBE_TIMEOUT_MS); return probe.error || probe.status !== 0 ? namespaceFailure(host, probe) : undefined; } async function namespaceFailure( host: SystemTestHost, probe: ReturnType, ): Promise { const settings: string[] = []; for (const [name, file] of USER_NAMESPACE_SETTINGS) { try { settings.push(`${name} = ${(await host.readFile(file)).trim()}`); } catch { // Absent on this kernel; the others still say what the host allows. } } return [ "this host does not allow unprivileged user and network namespaces, which the provider " + `sandbox needs (bwrap ${PROVIDER_SANDBOX_NAMESPACE_ARGUMENTS.join(" ")}).`, ` probe: ${BWRAP} ${NAMESPACE_PROBE_ARGUMENTS.join(" ")} ${outcomeOf(probe)}`, ...stderrLines(probe.stderr, "bwrap"), ...(settings.length > 0 ? [` host settings: ${settings.join(", ")}`] : []), " To fix, do one of these:", " - give /usr/bin/bwrap an AppArmor profile that allows `userns` (Ubuntu 23.10 and later),", " - on a host you own, set sysctl kernel.apparmor_restrict_unprivileged_userns=0", " (or kernel.unprivileged_userns_clone=1 where that setting exists),", " - run FE-GATE-013 on another runner that allows them.", " Runner requirements: docs/operations/ci-quality-gates.md.", ].join("\n"); } export function formatSystemTestPrerequisiteReport(failures: readonly string[]): string { return [ "System/CI-runner test prerequisites are unavailable:", ...failures.map((failure) => ` - ${failure}`), "", "Run test:unit/test:contract/test:component/test:integration for the", "developer loop. test:system is intentionally reserved for a compatible", "Linux CI-runner host.", "", ].join("\n"); } const isCli = process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); if (isCli) { const failures = await findSystemTestPrerequisiteFailures(); if (failures.length > 0) { process.stderr.write(formatSystemTestPrerequisiteReport(failures)); process.exit(1); } process.stdout.write("System test prerequisites: PASS\n"); }