import { spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { constants } from "node:fs"; import { access, appendFile, lstat } from "node:fs/promises"; import path from "node:path"; import { provenanceProviderAttestationSchema, vulnerabilityProviderReportSchema, } from "./lib/provider-evidence.ts"; import { readBoundedRegularFile } from "./lib/ci-artifact-validator.ts"; import { readProviderTrust } from "./lib/provider-trust.ts"; import { superviseProviderEvidence } from "./lib/provider-supervisor.ts"; import { serializeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts"; import { assertSafePublishLeaf, ensureSafePublishDirectory, } from "./lib/ci-gate-log.ts"; import { encodeProviderBwrapInput, encodeProviderScopeFrame, formatProviderCgroupUnitName, systemctlKillProviderArguments, systemdRunProviderArguments, } from "./lib/provider-cgroup.ts"; import { assertProviderGuardianLeasePaths, createProviderScopeGuardianLatch, startProviderGuardian, type ProviderGuardianLease, } from "./lib/provider-guardian-client.ts"; import { createProviderOutputLimiter } from "./lib/provider-output-limiter.ts"; import { PROVIDER_SANDBOX_ISOLATION_ARGUMENTS } from "./lib/provider-sandbox-probe.ts"; const kind = process.argv[process.argv.indexOf("--kind") + 1]; const PROVIDER_TMP_BYTES = 16_777_216; const PROVIDER_MASK_BYTES = 1_048_576; const PROVIDER_MAX_OUTPUT_BYTES = 1_048_576; const DEFAULT_PROVIDER_CPU_SECONDS = 1_200; const DEFAULT_PROVIDER_TIMEOUT_MS = 30 * 60 * 1_000; const PROVIDER_POSTPROCESS_TIMEOUT_MS = 10 * 60 * 1_000; const PROVIDER_REAP_TIMEOUT_MS = 5_000; const PROVIDER_CONTROL_TIMEOUT_MS = 5_000; const PROVIDER_CONTROL_MAX_OUTPUT_BYTES = 65_536; const PROVIDER_CONTROL_POLL_MS = 25; const MINIMUM_PROVIDER_SYSTEMD_VERSION = 254; if (kind !== "vulnerability" && kind !== "provenance") { process.stderr.write("Usage: run-and-validate-provider --kind vulnerability|provenance\n"); process.exit(2); } const command = kind === "vulnerability" ? process.env.VULNERABILITY_PROVIDER_COMMAND : process.env.PROVENANCE_PROVIDER_COMMAND; const reportPath = kind === "vulnerability" ? process.env.VULNERABILITY_REPORT_PATH : process.env.PROVENANCE_ATTESTATION_PATH; const sealedPath = process.env.VALIDATED_PROVIDER_REPORT_PATH; const archivePath = process.env.CANDIDATE_ARCHIVE_PATH; const archiveSha256 = process.env.CANDIDATE_ARCHIVE_SHA256; const publicKeyPath = kind === "vulnerability" ? process.env.VULNERABILITY_PUBLIC_KEY_PATH : process.env.PROVENANCE_PUBLIC_KEY_PATH; const keyId = kind === "vulnerability" ? process.env.VULNERABILITY_KEY_ID : process.env.PROVENANCE_KEY_ID; const runId = process.env.GITEA_RUN_ID ?? process.env.GITHUB_RUN_ID ?? process.env.CI_RUN_ID; const runAttemptSource = process.env.GITEA_RUN_ATTEMPT ?? process.env.GITHUB_RUN_ATTEMPT ?? process.env.CI_RUN_ATTEMPT; const sourceRevision = process.env.EXPECTED_SOURCE_REVISION ?? process.env.VITE_COMMIT_SHA; if ( !command || !reportPath || !sealedPath || !archivePath || !archiveSha256 || !publicKeyPath || !keyId || !runId || !runAttemptSource || !sourceRevision ) { process.stderr.write("Provider supervisor environment is incomplete\n"); process.exit(2); } const runAttempt = Number(runAttemptSource); if (!Number.isInteger(runAttempt) || runAttempt < 1 || runAttempt > 1_000) { throw new TypeError("provider supervisor run attempt is invalid"); } const workspaceRoot = process.cwd(); const reportAbsolute = path.resolve(reportPath); const rawDirectory = path.dirname(reportAbsolute); const sealedAbsolute = path.resolve(sealedPath); const expectedRawLeaf = kind === "vulnerability" ? "vulnerability-report.json" : "provenance-attestation.json"; const expectedRawDirectory = path.resolve(workspaceRoot, "provider-evidence/untrusted"); if ( reportAbsolute !== path.join(expectedRawDirectory, expectedRawLeaf) || path.basename(rawDirectory) !== "untrusted" || path.dirname(rawDirectory) !== path.dirname(sealedAbsolute) || reportAbsolute === sealedAbsolute ) { throw new TypeError("provider raw and sealed evidence paths are not isolated"); } await prepareMissingProviderOutput(workspaceRoot, reportAbsolute, reportPath, "raw provider report"); await prepareMissingProviderOutput(workspaceRoot, sealedAbsolute, sealedPath, "sealed provider report"); await access("/usr/bin/bwrap", constants.X_OK).catch(() => { throw new Error("provider sandbox unavailable: /usr/bin/bwrap is required"); }); await access("/usr/bin/prlimit", constants.X_OK).catch(() => { throw new Error("provider sandbox unavailable: /usr/bin/prlimit is required"); }); await access("/usr/bin/systemd-run", constants.X_OK).catch(() => { throw new Error("provider cgroup unavailable: /usr/bin/systemd-run is required"); }); await access("/usr/bin/systemctl", constants.X_OK).catch(() => { throw new Error("provider cgroup unavailable: /usr/bin/systemctl is required"); }); await assertProviderCgroupManagerAvailable(); const trust = await readProviderTrust(workspaceRoot, publicKeyPath, keyId); if (!trust) throw new TypeError("provider supervisor trust key is invalid"); let guardianLease: ProviderGuardianLease | undefined; let guardianTerminalStarted = false; try { const providerWallTimeoutMs = providerTimeoutMs(); const supervised = await superviseProviderEvidence({ kind, archivePath, expectedArchiveSha256: archiveSha256, expectedRun: { id: runId, attempt: runAttempt, sourceRevision }, trust, executeProvider: async ({ candidateRoot, environment }) => { guardianLease = await startProviderGuardian({ kind, workspaceRoot, leaseMs: providerWallTimeoutMs + PROVIDER_POSTPROCESS_TIMEOUT_MS, guardianScript: path.join(workspaceRoot, "scripts/lib/provider-raw-guardian.ts"), }); const activeGuardian = guardianLease; assertProviderGuardianLeasePaths(activeGuardian, { rawPath: reportAbsolute, sealedPath: sealedAbsolute, }); const childEnvironment = createProviderEnvironment(kind, reportPath, environment); await runProviderInSandbox( kind, command, childEnvironment, activeGuardian.rawPath, activeGuardian.rawIdentity, workspaceRoot, candidateRoot, providerWallTimeoutMs, activeGuardian.prematureExit, ); await assertOwnedProviderOutput(activeGuardian.rawPath, activeGuardian.rawIdentity); }, captureReport: async () => { if (!guardianLease) throw new Error("provider raw guardian lease was not established"); await assertOwnedProviderOutput(guardianLease.rawPath, guardianLease.rawIdentity); return readBoundedRegularFile({ root: workspaceRoot, relativePath: path.relative(workspaceRoot, guardianLease.rawPath).replaceAll(path.sep, "/"), maxBytes: 8_388_608, }); }, }); if (!guardianLease) throw new Error("provider raw guardian lease was not established"); await assertSafePublishLeaf(sealedAbsolute, sealedPath); await guardianLease.publish(serializeValidatedJsonArtifact({ path: sealedPath, schema: kind === "vulnerability" ? vulnerabilityProviderReportSchema : provenanceProviderAttestationSchema, value: supervised.evidence, })); if (process.env.GITHUB_OUTPUT) { await appendFile( process.env.GITHUB_OUTPUT, `invocation_nonce=${supervised.invocationNonce}\n`, "utf8", ); } guardianTerminalStarted = true; await guardianLease.commit(); } catch (providerError) { const failures = [toError(providerError)]; if (guardianLease && !guardianTerminalStarted) { try { await guardianLease.abort(); } catch (guardianError) { failures.push(toError(guardianError)); } } if (failures.length > 1) { throw new AggregateError(failures, "provider lifecycle and owned output cleanup failed", { cause: providerError, }); } throw providerError; } process.stdout.write(`${kind} provider supervised validation: PASS\n`); function createProviderEnvironment( providerKind: "vulnerability" | "provenance", rawReportPath: string, bindings: Readonly>, ): NodeJS.ProcessEnv { const environment: NodeJS.ProcessEnv = { PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin", HOME: "/tmp/provider-home", TMPDIR: "/tmp", CI: "true", GITHUB_ENV: "/tmp/github-env", GITHUB_PATH: "/tmp/github-path", ...bindings, ...(providerKind === "vulnerability" ? { VULNERABILITY_REPORT_PATH: rawReportPath } : { PROVENANCE_ATTESTATION_PATH: rawReportPath }), }; for (const name of ["LANG", "LC_ALL"] as const) { if (process.env[name]) environment[name] = process.env[name]; } const credentialPrefix = `${providerKind.toUpperCase()}_PROVIDER_`; for (const [name, value] of Object.entries(process.env)) { if (name.startsWith(credentialPrefix) && !name.endsWith("_COMMAND") && value) { environment[name] = value; } } return environment; } async function runProviderInSandbox( providerKind: "vulnerability" | "provenance", command: string, environment: NodeJS.ProcessEnv, reportAbsolute: string, reportIdentity: Readonly<{ dev: number; ino: number }>, workspaceRoot: string, candidateRoot: string, timeoutMs: number, guardianExit: Promise, ): Promise { const cpuSeconds = providerCpuSeconds(); const bwrapArguments = [ ...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS, "--size", String(PROVIDER_TMP_BYTES), "--tmpfs", "/tmp", "--dir", "/tmp/provider-home", "--size", String(PROVIDER_MASK_BYTES), "--tmpfs", "/etc", ]; for (const source of ["/usr", "/bin", "/lib", "/lib64"]) { if (await exists(source)) bwrapArguments.push("--ro-bind", source, source); } bwrapArguments.push("--ro-bind", process.execPath, "/tmp/node"); for (const source of [ "/etc/ca-certificates", "/etc/ssl", "/etc/nsswitch.conf", "/etc/passwd", "/etc/group", ]) { if (await exists(source)) bwrapArguments.push("--ro-bind", source, source); } bwrapArguments.push("--remount-ro", "/etc"); for (const directory of missingDestinationAncestors(workspaceRoot)) { bwrapArguments.push("--dir", directory); } bwrapArguments.push("--ro-bind", workspaceRoot, workspaceRoot); if (await exists(path.join(workspaceRoot, ".git"))) { bwrapArguments.push( "--size", String(PROVIDER_MASK_BYTES), "--tmpfs", path.join(workspaceRoot, ".git"), "--remount-ro", path.join(workspaceRoot, ".git"), ); } bwrapArguments.push( "--ro-bind", candidateRoot, "/candidate", "--remount-ro", "/tmp", "--remount-ro", "/", "--bind", reportAbsolute, reportAbsolute, "--chdir", workspaceRoot, ); /** * Everything above is a bubblewrap *option* and travels in the args file, so * host paths never reach `/proc//cmdline`. The command below cannot: an * args file's option stream ends at the first non-option and bubblewrap drops * the remainder, so a command written there is never executed. It stays on * real argv, and it is safe there because the provider command and its * credentials are passed as `--setenv PROVIDER_COMMAND` inside the args file * and only expanded by the innermost shell. */ const bwrapCommand = [ "/usr/bin/prlimit", "--core=0:0", "--fsize=8388607:8388607", "--nofile=64:64", `--cpu=${cpuSeconds}:${cpuSeconds}`, "--", "/bin/sh", "-eu", "-c", 'exec /bin/sh -eu -c "$PROVIDER_COMMAND"', ]; const unitName = formatProviderCgroupUnitName( providerKind, process.pid, randomBytes(12).toString("hex"), ); const bwrapInput = encodeProviderBwrapInput(bwrapArguments, { ...environment, PATH: `/tmp:${environment.PATH ?? ""}`, PROVIDER_COMMAND: command, }); const scopeFrame = encodeProviderScopeFrame({ bwrapInput, bwrapCommand, reportPath: reportAbsolute, reportDev: reportIdentity.dev, reportIno: reportIdentity.ino, }); await waitForProvider( spawn("/usr/bin/systemd-run", systemdRunProviderArguments( unitName, timeoutMs, cpuSeconds, process.execPath, path.join(workspaceRoot, "scripts/lib/provider-scope-wrapper.ts"), reportAbsolute, reportIdentity.dev, reportIdentity.ino, ), { env: providerCgroupClientEnvironment(), stdio: ["pipe", "pipe", "pipe"], }), timeoutMs, unitName, scopeFrame, guardianExit, ); } async function waitForProvider( child: ReturnType, timeoutMs: number, unitName: string, scopeFrame: Buffer, guardianExit: Promise, ): Promise { let termination: "guardian" | "timeout" | "output" | undefined; let guardianError: Error | undefined; let signalTermination!: () => void; let kill: Promise | undefined; const terminationStarted = new Promise((resolve) => { signalTermination = resolve; }); const close = new Promise>((resolve) => { child.once("error", (error) => resolve({ code: null, error, signal: null })); child.once("close", (code, signal) => resolve({ code, signal })); }); let inputError: Error | undefined; child.stdin?.once("error", (error) => { inputError = error; }); if (child.stdin) child.stdin.write(scopeFrame); else inputError = new Error("systemd-run provider argument pipe is unavailable"); const terminate = (reason: "guardian" | "timeout" | "output"): void => { if (termination) return; termination = reason; kill = killProviderUnit(unitName); signalTermination(); }; const guardianLatch = createProviderScopeGuardianLatch(guardianExit); void guardianLatch.activeFailure.then((error) => { guardianError = error; terminate("guardian"); }); const outputLimiter = createProviderOutputLimiter( PROVIDER_MAX_OUTPUT_BYTES, () => terminate("output"), ); /** * Lines the sandbox tooling itself emits, kept so a launch failure can say * why. Everything else the child writes is provider output and may carry * credentials, so it is counted and discarded as before. * * Without this a sandbox that never started reported only `exit=1`, and the * actual cause — `bwrap: loopback: Failed RTM_NEWADDR: Operation not * permitted` on a host with `kernel.apparmor_restrict_unprivileged_userns=1` * — was invisible. That turned a host restriction into an unexplained * product failure. */ const SANDBOX_DIAGNOSTIC = /^(?:bwrap|prlimit|systemd-run|systemctl):\s.*$/gmu; const sandboxDiagnostics: string[] = []; const capture = (chunk: Buffer | string): void => { for (const line of String(chunk).matchAll(SANDBOX_DIAGNOSTIC)) { if (sandboxDiagnostics.length < 8 && !sandboxDiagnostics.includes(line[0])) { sandboxDiagnostics.push(line[0]); } } if (termination) return; outputLimiter.consume(chunk); }; child.stdout?.on("data", capture); child.stderr?.on("data", capture); const timeout = setTimeout(() => terminate("timeout"), timeoutMs); try { const first = await Promise.race([ close.then((result) => ({ type: "close" as const, result })), terminationStarted.then(() => ({ type: "termination" as const })), ]); let collection: Promise | undefined; if (first.type === "close" && !termination) { collection = waitForProviderUnitCollected(unitName); const scopeOutcome = await Promise.race([ collection.then(() => "collected" as const), terminationStarted.then(() => "termination" as const), ]); if (scopeOutcome === "collected") { await guardianLatch.close(); const boundaryFailure = guardianLatch.failure(); if (boundaryFailure && !termination) { guardianError = boundaryFailure; terminate("guardian"); } } } if (termination) { let killError: Error | undefined; try { await kill; } catch (error) { killError = toError(error); child.kill("SIGKILL"); } const closed = await waitForProviderClose(close); if (!closed) child.kill("SIGKILL"); let collectionError: Error | undefined; try { await (collection ?? waitForProviderUnitCollected(unitName)); } catch (error) { collectionError = toError(error); } await guardianLatch.close(); const reason = termination === "timeout" ? "sandboxed external provider command timed out" : termination === "output" ? `sandboxed external provider output exceeded the ${PROVIDER_MAX_OUTPUT_BYTES}-byte aggregate limit` : `provider raw guardian failed${guardianError ? `: ${guardianError.message}` : ""}`; if (!closed) throw new Error(`${reason} and systemd-run did not close within the reap bound`); if (collectionError) throw new Error(`${reason}; provider cgroup collection failed: ${collectionError.message}`); if (killError) throw new Error(`${reason}; provider cgroup kill failed: ${killError.message}`); throw new Error(reason); } const result = first.type === "close" ? first.result : await close; await collection; if (result.error) throw result.error; if (result.code !== 0 || result.signal !== null) { throw new Error( `sandboxed external provider failed: exit=${result.code ?? "none"}, ` + `signal=${result.signal ?? "none"}` + (sandboxDiagnostics.length > 0 ? `; sandbox reported: ${sandboxDiagnostics.join("; ")}` : ""), ); } if (inputError) throw inputError; } finally { clearTimeout(timeout); await guardianLatch.close(); } } async function waitForProviderClose( close: Promise>, ): Promise { let timer: NodeJS.Timeout | undefined; const closed = await Promise.race([ close.then(() => true), new Promise((resolve) => { timer = setTimeout(() => resolve(false), PROVIDER_REAP_TIMEOUT_MS); }), ]); if (timer) clearTimeout(timer); return closed; } async function assertProviderCgroupManagerAvailable(): Promise { const output = await runBoundedSystemctl( ["--user", "show", "--property=Version", "--value"], "provider cgroup user manager probe", ); const match = /^([0-9]+)/u.exec(output.trim()); const version = match ? Number(match[1]) : Number.NaN; if (!Number.isSafeInteger(version) || version < MINIMUM_PROVIDER_SYSTEMD_VERSION) { throw new Error(`provider cgroup unavailable: systemd ${MINIMUM_PROVIDER_SYSTEMD_VERSION} or newer is required`); } } async function killProviderUnit(unitName: string): Promise { await runBoundedSystemctl(systemctlKillProviderArguments(unitName), "provider cgroup unit kill"); } async function waitForProviderUnitCollected(unitName: string): Promise { const deadline = Date.now() + PROVIDER_CONTROL_TIMEOUT_MS; let loadState = "unknown"; while (Date.now() <= deadline) { loadState = (await runBoundedSystemctl( ["--user", "show", unitName, "--property=LoadState", "--value"], "provider cgroup collection probe", )).trim(); if (loadState === "not-found") return; await new Promise((resolve) => setTimeout(resolve, PROVIDER_CONTROL_POLL_MS)); } throw new Error(`provider unit remained loaded with state ${loadState || "unknown"}`); } async function runBoundedSystemctl(arguments_: readonly string[], label: string): Promise { return await new Promise((resolve, reject) => { const child = spawn("/usr/bin/systemctl", arguments_, { env: providerCgroupClientEnvironment(), stdio: ["ignore", "pipe", "pipe"], }); let bytes = 0; const chunks: Buffer[] = []; let settled = false; let termination: "output" | "timeout" | undefined; let reap: NodeJS.Timeout | undefined; const finish = (error?: Error): void => { if (settled) return; settled = true; clearTimeout(timeout); if (reap) clearTimeout(reap); error ? reject(error) : resolve(Buffer.concat(chunks).toString("utf8")); }; const terminate = (reason: "output" | "timeout"): void => { if (termination) return; termination = reason; child.kill("SIGKILL"); reap = setTimeout(() => finish(new Error(`${label} ${reason} bound was exceeded and systemctl did not close`)), PROVIDER_REAP_TIMEOUT_MS); }; const capture = (chunk: Buffer | string): void => { if (termination) return; const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); const remaining = PROVIDER_CONTROL_MAX_OUTPUT_BYTES - bytes; if (remaining > 0) { chunks.push(value.subarray(0, remaining)); bytes += Math.min(value.length, remaining); } if (value.length > remaining) terminate("output"); }; child.stdout?.on("data", capture); child.stderr?.on("data", capture); const timeout = setTimeout(() => terminate("timeout"), PROVIDER_CONTROL_TIMEOUT_MS); child.once("error", (error) => finish(error)); child.once("close", (code, signal) => { if (termination) return finish(new Error(`${label} ${termination} bound was exceeded`)); if (code === 0 && signal === null) return finish(); const detail = Buffer.concat(chunks).toString("utf8").trim(); finish(new Error(`${label} failed: exit=${code ?? "none"}, signal=${signal ?? "none"}${detail ? `, output=${detail}` : ""}`)); }); }); } function providerCgroupClientEnvironment(): NodeJS.ProcessEnv { const environment: NodeJS.ProcessEnv = { PATH: "/usr/bin:/bin" }; for (const name of ["DBUS_SESSION_BUS_ADDRESS", "HOME", "LANG", "LC_ALL", "LOGNAME", "USER", "XDG_RUNTIME_DIR"] as const) { if (process.env[name]) environment[name] = process.env[name]; } return environment; } function providerCpuSeconds(): number { const value = process.env.PROVIDER_SUPERVISOR_CPU_SECONDS; if (!value) return DEFAULT_PROVIDER_CPU_SECONDS; const parsed = Number(value); if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > DEFAULT_PROVIDER_CPU_SECONDS) { throw new TypeError("PROVIDER_SUPERVISOR_CPU_SECONDS must be a positive integer no greater than 1200"); } return parsed; } function providerTimeoutMs(): number { const value = process.env.PROVIDER_SUPERVISOR_TIMEOUT_MS; if (!value) return DEFAULT_PROVIDER_TIMEOUT_MS; const parsed = Number(value); if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > DEFAULT_PROVIDER_TIMEOUT_MS) { throw new TypeError("PROVIDER_SUPERVISOR_TIMEOUT_MS must be a positive integer no greater than 1800000"); } return parsed; } function toError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } function missingDestinationAncestors(target: string): string[] { const ancestors: string[] = []; let current = path.dirname(path.resolve(target)); while (current !== path.parse(current).root && !["/usr", "/bin", "/lib", "/lib64", "/tmp"].includes(current)) { ancestors.push(current); current = path.dirname(current); } return ancestors.reverse(); } async function exists(target: string): Promise { try { await lstat(target); return true; } catch (error) { if (hasErrorCode(error, "ENOENT")) return false; throw error; } } async function prepareMissingProviderOutput( root: string, absolutePath: string, configuredPath: string, label: string, ): Promise { await ensureSafePublishDirectory(root, path.dirname(absolutePath)); await assertSafePublishLeaf(absolutePath, configuredPath); try { await lstat(absolutePath); throw new Error(`${label} already exists: ${configuredPath}`); } catch (error) { if (!hasErrorCode(error, "ENOENT")) throw error; } } async function assertOwnedProviderOutput( absolutePath: string, identity: Readonly<{ dev: number; ino: number }>, ): Promise { const metadata = await lstat(absolutePath); if ( metadata.isSymbolicLink() || !metadata.isFile() || metadata.dev !== identity.dev || metadata.ino !== identity.ino ) { throw new Error("provider raw output identity changed"); } } function hasErrorCode(error: unknown, code: string): boolean { return Boolean(error && typeof error === "object" && "code" in error && error.code === code); }