Files
clean-architecture-frontend…/scripts/run-and-validate-provider.ts
T

599 lines
22 KiB
TypeScript

import { spawn } from "node:child_process";
import { randomBytes } from "node:crypto";
import { constants } from "node:fs";
import { access, appendFile, lstat } from "node:fs/promises";
import path from "node:path";
import {
provenanceProviderAttestationSchema,
vulnerabilityProviderReportSchema,
} from "./lib/provider-evidence.ts";
import { readBoundedRegularFile } from "./lib/ci-artifact-validator.ts";
import { readProviderTrust } from "./lib/provider-trust.ts";
import { superviseProviderEvidence } from "./lib/provider-supervisor.ts";
import { serializeValidatedJsonArtifact } from "./lib/validated-json-artifact.ts";
import {
assertSafePublishLeaf,
ensureSafePublishDirectory,
} from "./lib/ci-gate-log.ts";
import {
encodeProviderBwrapInput,
encodeProviderScopeFrame,
formatProviderCgroupUnitName,
systemctlKillProviderArguments,
systemdRunProviderArguments,
} from "./lib/provider-cgroup.ts";
import {
assertProviderGuardianLeasePaths,
createProviderScopeGuardianLatch,
startProviderGuardian,
type ProviderGuardianLease,
} from "./lib/provider-guardian-client.ts";
import { createProviderOutputLimiter } from "./lib/provider-output-limiter.ts";
const kind = process.argv[process.argv.indexOf("--kind") + 1];
const PROVIDER_TMP_BYTES = 16_777_216;
const PROVIDER_MASK_BYTES = 1_048_576;
const PROVIDER_MAX_OUTPUT_BYTES = 1_048_576;
const DEFAULT_PROVIDER_CPU_SECONDS = 1_200;
const DEFAULT_PROVIDER_TIMEOUT_MS = 30 * 60 * 1_000;
const PROVIDER_POSTPROCESS_TIMEOUT_MS = 10 * 60 * 1_000;
const PROVIDER_REAP_TIMEOUT_MS = 5_000;
const PROVIDER_CONTROL_TIMEOUT_MS = 5_000;
const PROVIDER_CONTROL_MAX_OUTPUT_BYTES = 65_536;
const PROVIDER_CONTROL_POLL_MS = 25;
const MINIMUM_PROVIDER_SYSTEMD_VERSION = 254;
if (kind !== "vulnerability" && kind !== "provenance") {
process.stderr.write("Usage: run-and-validate-provider --kind vulnerability|provenance\n");
process.exit(2);
}
const command =
kind === "vulnerability"
? process.env.VULNERABILITY_PROVIDER_COMMAND
: process.env.PROVENANCE_PROVIDER_COMMAND;
const reportPath =
kind === "vulnerability"
? process.env.VULNERABILITY_REPORT_PATH
: process.env.PROVENANCE_ATTESTATION_PATH;
const sealedPath = process.env.VALIDATED_PROVIDER_REPORT_PATH;
const archivePath = process.env.CANDIDATE_ARCHIVE_PATH;
const archiveSha256 = process.env.CANDIDATE_ARCHIVE_SHA256;
const publicKeyPath = kind === "vulnerability"
? process.env.VULNERABILITY_PUBLIC_KEY_PATH
: process.env.PROVENANCE_PUBLIC_KEY_PATH;
const keyId = kind === "vulnerability"
? process.env.VULNERABILITY_KEY_ID
: process.env.PROVENANCE_KEY_ID;
const runId = process.env.GITEA_RUN_ID ?? process.env.GITHUB_RUN_ID ?? process.env.CI_RUN_ID;
const runAttemptSource = process.env.GITEA_RUN_ATTEMPT ??
process.env.GITHUB_RUN_ATTEMPT ?? process.env.CI_RUN_ATTEMPT;
const sourceRevision = process.env.EXPECTED_SOURCE_REVISION ?? process.env.VITE_COMMIT_SHA;
if (
!command ||
!reportPath ||
!sealedPath ||
!archivePath ||
!archiveSha256 ||
!publicKeyPath ||
!keyId ||
!runId ||
!runAttemptSource ||
!sourceRevision
) {
process.stderr.write("Provider supervisor environment is incomplete\n");
process.exit(2);
}
const runAttempt = Number(runAttemptSource);
if (!Number.isInteger(runAttempt) || runAttempt < 1 || runAttempt > 1_000) {
throw new TypeError("provider supervisor run attempt is invalid");
}
const workspaceRoot = process.cwd();
const reportAbsolute = path.resolve(reportPath);
const rawDirectory = path.dirname(reportAbsolute);
const sealedAbsolute = path.resolve(sealedPath);
const expectedRawLeaf = kind === "vulnerability"
? "vulnerability-report.json"
: "provenance-attestation.json";
const expectedRawDirectory = path.resolve(workspaceRoot, "provider-evidence/untrusted");
if (
reportAbsolute !== path.join(expectedRawDirectory, expectedRawLeaf) ||
path.basename(rawDirectory) !== "untrusted" ||
path.dirname(rawDirectory) !== path.dirname(sealedAbsolute) ||
reportAbsolute === sealedAbsolute
) {
throw new TypeError("provider raw and sealed evidence paths are not isolated");
}
await prepareMissingProviderOutput(workspaceRoot, reportAbsolute, reportPath, "raw provider report");
await prepareMissingProviderOutput(workspaceRoot, sealedAbsolute, sealedPath, "sealed provider report");
await access("/usr/bin/bwrap", constants.X_OK).catch(() => {
throw new Error("provider sandbox unavailable: /usr/bin/bwrap is required");
});
await access("/usr/bin/prlimit", constants.X_OK).catch(() => {
throw new Error("provider sandbox unavailable: /usr/bin/prlimit is required");
});
await access("/usr/bin/systemd-run", constants.X_OK).catch(() => {
throw new Error("provider cgroup unavailable: /usr/bin/systemd-run is required");
});
await access("/usr/bin/systemctl", constants.X_OK).catch(() => {
throw new Error("provider cgroup unavailable: /usr/bin/systemctl is required");
});
await assertProviderCgroupManagerAvailable();
const trust = await readProviderTrust(workspaceRoot, publicKeyPath, keyId);
if (!trust) throw new TypeError("provider supervisor trust key is invalid");
let guardianLease: ProviderGuardianLease | undefined;
let guardianTerminalStarted = false;
try {
const providerWallTimeoutMs = providerTimeoutMs();
const supervised = await superviseProviderEvidence({
kind,
archivePath,
expectedArchiveSha256: archiveSha256,
expectedRun: { id: runId, attempt: runAttempt, sourceRevision },
trust,
executeProvider: async ({ candidateRoot, environment }) => {
guardianLease = await startProviderGuardian({
kind,
workspaceRoot,
leaseMs: providerWallTimeoutMs + PROVIDER_POSTPROCESS_TIMEOUT_MS,
guardianScript: path.join(workspaceRoot, "scripts/lib/provider-raw-guardian.ts"),
});
const activeGuardian = guardianLease;
assertProviderGuardianLeasePaths(activeGuardian, {
rawPath: reportAbsolute,
sealedPath: sealedAbsolute,
});
const childEnvironment = createProviderEnvironment(kind, reportPath, environment);
await runProviderInSandbox(
kind,
command,
childEnvironment,
activeGuardian.rawPath,
activeGuardian.rawIdentity,
workspaceRoot,
candidateRoot,
providerWallTimeoutMs,
activeGuardian.prematureExit,
);
await assertOwnedProviderOutput(activeGuardian.rawPath, activeGuardian.rawIdentity);
},
captureReport: async () => {
if (!guardianLease) throw new Error("provider raw guardian lease was not established");
await assertOwnedProviderOutput(guardianLease.rawPath, guardianLease.rawIdentity);
return readBoundedRegularFile({
root: workspaceRoot,
relativePath: path.relative(workspaceRoot, guardianLease.rawPath).replaceAll(path.sep, "/"),
maxBytes: 8_388_608,
});
},
});
if (!guardianLease) throw new Error("provider raw guardian lease was not established");
await assertSafePublishLeaf(sealedAbsolute, sealedPath);
await guardianLease.publish(serializeValidatedJsonArtifact({
path: sealedPath,
schema:
kind === "vulnerability"
? vulnerabilityProviderReportSchema
: provenanceProviderAttestationSchema,
value: supervised.evidence,
}));
if (process.env.GITHUB_OUTPUT) {
await appendFile(
process.env.GITHUB_OUTPUT,
`invocation_nonce=${supervised.invocationNonce}\n`,
"utf8",
);
}
guardianTerminalStarted = true;
await guardianLease.commit();
} catch (providerError) {
const failures = [toError(providerError)];
if (guardianLease && !guardianTerminalStarted) {
try {
await guardianLease.abort();
} catch (guardianError) {
failures.push(toError(guardianError));
}
}
if (failures.length > 1) {
throw new AggregateError(failures, "provider lifecycle and owned output cleanup failed", {
cause: providerError,
});
}
throw providerError;
}
process.stdout.write(`${kind} provider supervised validation: PASS\n`);
function createProviderEnvironment(
providerKind: "vulnerability" | "provenance",
rawReportPath: string,
bindings: Readonly<Record<string, string>>,
): NodeJS.ProcessEnv {
const environment: NodeJS.ProcessEnv = {
PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin",
HOME: "/tmp/provider-home",
TMPDIR: "/tmp",
CI: "true",
GITHUB_ENV: "/tmp/github-env",
GITHUB_PATH: "/tmp/github-path",
...bindings,
...(providerKind === "vulnerability"
? { VULNERABILITY_REPORT_PATH: rawReportPath }
: { PROVENANCE_ATTESTATION_PATH: rawReportPath }),
};
for (const name of ["LANG", "LC_ALL"] as const) {
if (process.env[name]) environment[name] = process.env[name];
}
const credentialPrefix = `${providerKind.toUpperCase()}_PROVIDER_`;
for (const [name, value] of Object.entries(process.env)) {
if (name.startsWith(credentialPrefix) && !name.endsWith("_COMMAND") && value) {
environment[name] = value;
}
}
return environment;
}
async function runProviderInSandbox(
providerKind: "vulnerability" | "provenance",
command: string,
environment: NodeJS.ProcessEnv,
reportAbsolute: string,
reportIdentity: Readonly<{ dev: number; ino: number }>,
workspaceRoot: string,
candidateRoot: string,
timeoutMs: number,
guardianExit: Promise<Error>,
): Promise<void> {
const cpuSeconds = providerCpuSeconds();
const bwrapArguments = [
"--die-with-parent", "--new-session", "--as-pid-1",
"--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net",
"--dev", "/dev", "--remount-ro", "/dev",
"--proc", "/proc", "--remount-ro", "/proc",
"--size", String(PROVIDER_TMP_BYTES), "--tmpfs", "/tmp",
"--dir", "/tmp/provider-home",
"--size", String(PROVIDER_MASK_BYTES), "--tmpfs", "/etc",
];
for (const source of ["/usr", "/bin", "/lib", "/lib64"]) {
if (await exists(source)) bwrapArguments.push("--ro-bind", source, source);
}
bwrapArguments.push("--ro-bind", process.execPath, "/tmp/node");
for (const source of [
"/etc/ca-certificates", "/etc/ssl", "/etc/nsswitch.conf", "/etc/passwd", "/etc/group",
]) {
if (await exists(source)) bwrapArguments.push("--ro-bind", source, source);
}
bwrapArguments.push("--remount-ro", "/etc");
for (const directory of missingDestinationAncestors(workspaceRoot)) {
bwrapArguments.push("--dir", directory);
}
bwrapArguments.push("--ro-bind", workspaceRoot, workspaceRoot);
if (await exists(path.join(workspaceRoot, ".git"))) {
bwrapArguments.push(
"--size", String(PROVIDER_MASK_BYTES),
"--tmpfs", path.join(workspaceRoot, ".git"),
"--remount-ro", path.join(workspaceRoot, ".git"),
);
}
bwrapArguments.push(
"--ro-bind", candidateRoot, "/candidate",
"--remount-ro", "/tmp",
"--remount-ro", "/",
"--bind", reportAbsolute, reportAbsolute,
"--chdir", workspaceRoot,
"--", "/usr/bin/prlimit",
"--core=0:0",
"--fsize=8388607:8388607",
"--nofile=64:64",
`--cpu=${cpuSeconds}:${cpuSeconds}`,
"--", "/bin/sh", "-eu", "-c",
'exec /bin/sh -eu -c "$PROVIDER_COMMAND"',
);
const unitName = formatProviderCgroupUnitName(
providerKind,
process.pid,
randomBytes(12).toString("hex"),
);
const bwrapInput = encodeProviderBwrapInput(bwrapArguments, {
...environment,
PATH: `/tmp:${environment.PATH ?? ""}`,
PROVIDER_COMMAND: command,
});
const scopeFrame = encodeProviderScopeFrame({
bwrapInput,
reportPath: reportAbsolute,
reportDev: reportIdentity.dev,
reportIno: reportIdentity.ino,
});
await waitForProvider(
spawn("/usr/bin/systemd-run", systemdRunProviderArguments(
unitName,
timeoutMs,
cpuSeconds,
process.execPath,
path.join(workspaceRoot, "scripts/lib/provider-scope-wrapper.ts"),
reportAbsolute,
reportIdentity.dev,
reportIdentity.ino,
), {
env: providerCgroupClientEnvironment(),
stdio: ["pipe", "pipe", "pipe"],
}),
timeoutMs,
unitName,
scopeFrame,
guardianExit,
);
}
async function waitForProvider(
child: ReturnType<typeof spawn>,
timeoutMs: number,
unitName: string,
scopeFrame: Buffer,
guardianExit: Promise<Error>,
): Promise<void> {
let termination: "guardian" | "timeout" | "output" | undefined;
let guardianError: Error | undefined;
let signalTermination!: () => void;
let kill: Promise<void> | undefined;
const terminationStarted = new Promise<void>((resolve) => { signalTermination = resolve; });
const close = new Promise<Readonly<{ code: number | null; error?: Error; signal: NodeJS.Signals | null }>>((resolve) => {
child.once("error", (error) => resolve({ code: null, error, signal: null }));
child.once("close", (code, signal) => resolve({ code, signal }));
});
let inputError: Error | undefined;
child.stdin?.once("error", (error) => { inputError = error; });
if (child.stdin) child.stdin.write(scopeFrame);
else inputError = new Error("systemd-run provider argument pipe is unavailable");
const terminate = (reason: "guardian" | "timeout" | "output"): void => {
if (termination) return;
termination = reason;
kill = killProviderUnit(unitName);
signalTermination();
};
const guardianLatch = createProviderScopeGuardianLatch(guardianExit);
void guardianLatch.activeFailure.then((error) => {
guardianError = error;
terminate("guardian");
});
const outputLimiter = createProviderOutputLimiter(
PROVIDER_MAX_OUTPUT_BYTES,
() => terminate("output"),
);
const capture = (chunk: Buffer | string): void => {
if (termination) return;
outputLimiter.consume(chunk);
};
child.stdout?.on("data", capture);
child.stderr?.on("data", capture);
const timeout = setTimeout(() => terminate("timeout"), timeoutMs);
try {
const first = await Promise.race([
close.then((result) => ({ type: "close" as const, result })),
terminationStarted.then(() => ({ type: "termination" as const })),
]);
let collection: Promise<void> | undefined;
if (first.type === "close" && !termination) {
collection = waitForProviderUnitCollected(unitName);
const scopeOutcome = await Promise.race([
collection.then(() => "collected" as const),
terminationStarted.then(() => "termination" as const),
]);
if (scopeOutcome === "collected") {
await guardianLatch.close();
const boundaryFailure = guardianLatch.failure();
if (boundaryFailure && !termination) {
guardianError = boundaryFailure;
terminate("guardian");
}
}
}
if (termination) {
let killError: Error | undefined;
try { await kill; } catch (error) { killError = toError(error); child.kill("SIGKILL"); }
const closed = await waitForProviderClose(close);
if (!closed) child.kill("SIGKILL");
let collectionError: Error | undefined;
try {
await (collection ?? waitForProviderUnitCollected(unitName));
} catch (error) {
collectionError = toError(error);
}
await guardianLatch.close();
const reason = termination === "timeout"
? "sandboxed external provider command timed out"
: termination === "output"
? `sandboxed external provider output exceeded the ${PROVIDER_MAX_OUTPUT_BYTES}-byte aggregate limit`
: `provider raw guardian failed${guardianError ? `: ${guardianError.message}` : ""}`;
if (!closed) throw new Error(`${reason} and systemd-run did not close within the reap bound`);
if (collectionError) throw new Error(`${reason}; provider cgroup collection failed: ${collectionError.message}`);
if (killError) throw new Error(`${reason}; provider cgroup kill failed: ${killError.message}`);
throw new Error(reason);
}
const result = first.type === "close" ? first.result : await close;
await collection;
if (result.error) throw result.error;
if (result.code !== 0 || result.signal !== null) {
throw new Error(`sandboxed external provider failed: exit=${result.code ?? "none"}, signal=${result.signal ?? "none"}`);
}
if (inputError) throw inputError;
} finally {
clearTimeout(timeout);
await guardianLatch.close();
}
}
async function waitForProviderClose(
close: Promise<Readonly<{ code: number | null; error?: Error; signal: NodeJS.Signals | null }>>,
): Promise<boolean> {
let timer: NodeJS.Timeout | undefined;
const closed = await Promise.race([
close.then(() => true),
new Promise<false>((resolve) => { timer = setTimeout(() => resolve(false), PROVIDER_REAP_TIMEOUT_MS); }),
]);
if (timer) clearTimeout(timer);
return closed;
}
async function assertProviderCgroupManagerAvailable(): Promise<void> {
const output = await runBoundedSystemctl(
["--user", "show", "--property=Version", "--value"],
"provider cgroup user manager probe",
);
const match = /^([0-9]+)/u.exec(output.trim());
const version = match ? Number(match[1]) : Number.NaN;
if (!Number.isSafeInteger(version) || version < MINIMUM_PROVIDER_SYSTEMD_VERSION) {
throw new Error(`provider cgroup unavailable: systemd ${MINIMUM_PROVIDER_SYSTEMD_VERSION} or newer is required`);
}
}
async function killProviderUnit(unitName: string): Promise<void> {
await runBoundedSystemctl(systemctlKillProviderArguments(unitName), "provider cgroup unit kill");
}
async function waitForProviderUnitCollected(unitName: string): Promise<void> {
const deadline = Date.now() + PROVIDER_CONTROL_TIMEOUT_MS;
let loadState = "unknown";
while (Date.now() <= deadline) {
loadState = (await runBoundedSystemctl(
["--user", "show", unitName, "--property=LoadState", "--value"],
"provider cgroup collection probe",
)).trim();
if (loadState === "not-found") return;
await new Promise<void>((resolve) => setTimeout(resolve, PROVIDER_CONTROL_POLL_MS));
}
throw new Error(`provider unit remained loaded with state ${loadState || "unknown"}`);
}
async function runBoundedSystemctl(arguments_: readonly string[], label: string): Promise<string> {
return await new Promise<string>((resolve, reject) => {
const child = spawn("/usr/bin/systemctl", arguments_, {
env: providerCgroupClientEnvironment(),
stdio: ["ignore", "pipe", "pipe"],
});
let bytes = 0;
const chunks: Buffer[] = [];
let settled = false;
let termination: "output" | "timeout" | undefined;
let reap: NodeJS.Timeout | undefined;
const finish = (error?: Error): void => {
if (settled) return;
settled = true;
clearTimeout(timeout);
if (reap) clearTimeout(reap);
error ? reject(error) : resolve(Buffer.concat(chunks).toString("utf8"));
};
const terminate = (reason: "output" | "timeout"): void => {
if (termination) return;
termination = reason;
child.kill("SIGKILL");
reap = setTimeout(() => finish(new Error(`${label} ${reason} bound was exceeded and systemctl did not close`)), PROVIDER_REAP_TIMEOUT_MS);
};
const capture = (chunk: Buffer | string): void => {
if (termination) return;
const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
const remaining = PROVIDER_CONTROL_MAX_OUTPUT_BYTES - bytes;
if (remaining > 0) { chunks.push(value.subarray(0, remaining)); bytes += Math.min(value.length, remaining); }
if (value.length > remaining) terminate("output");
};
child.stdout?.on("data", capture);
child.stderr?.on("data", capture);
const timeout = setTimeout(() => terminate("timeout"), PROVIDER_CONTROL_TIMEOUT_MS);
child.once("error", (error) => finish(error));
child.once("close", (code, signal) => {
if (termination) return finish(new Error(`${label} ${termination} bound was exceeded`));
if (code === 0 && signal === null) return finish();
const detail = Buffer.concat(chunks).toString("utf8").trim();
finish(new Error(`${label} failed: exit=${code ?? "none"}, signal=${signal ?? "none"}${detail ? `, output=${detail}` : ""}`));
});
});
}
function providerCgroupClientEnvironment(): NodeJS.ProcessEnv {
const environment: NodeJS.ProcessEnv = { PATH: "/usr/bin:/bin" };
for (const name of ["DBUS_SESSION_BUS_ADDRESS", "HOME", "LANG", "LC_ALL", "LOGNAME", "USER", "XDG_RUNTIME_DIR"] as const) {
if (process.env[name]) environment[name] = process.env[name];
}
return environment;
}
function providerCpuSeconds(): number {
const value = process.env.PROVIDER_SUPERVISOR_CPU_SECONDS;
if (!value) return DEFAULT_PROVIDER_CPU_SECONDS;
const parsed = Number(value);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > DEFAULT_PROVIDER_CPU_SECONDS) {
throw new TypeError("PROVIDER_SUPERVISOR_CPU_SECONDS must be a positive integer no greater than 1200");
}
return parsed;
}
function providerTimeoutMs(): number {
const value = process.env.PROVIDER_SUPERVISOR_TIMEOUT_MS;
if (!value) return DEFAULT_PROVIDER_TIMEOUT_MS;
const parsed = Number(value);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > DEFAULT_PROVIDER_TIMEOUT_MS) {
throw new TypeError("PROVIDER_SUPERVISOR_TIMEOUT_MS must be a positive integer no greater than 1800000");
}
return parsed;
}
function toError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
function missingDestinationAncestors(target: string): string[] {
const ancestors: string[] = [];
let current = path.dirname(path.resolve(target));
while (current !== path.parse(current).root && !["/usr", "/bin", "/lib", "/lib64", "/tmp"].includes(current)) {
ancestors.push(current);
current = path.dirname(current);
}
return ancestors.reverse();
}
async function exists(target: string): Promise<boolean> {
try {
await lstat(target);
return true;
} catch (error) {
if (hasErrorCode(error, "ENOENT")) return false;
throw error;
}
}
async function prepareMissingProviderOutput(
root: string,
absolutePath: string,
configuredPath: string,
label: string,
): Promise<void> {
await ensureSafePublishDirectory(root, path.dirname(absolutePath));
await assertSafePublishLeaf(absolutePath, configuredPath);
try {
await lstat(absolutePath);
throw new Error(`${label} already exists: ${configuredPath}`);
} catch (error) {
if (!hasErrorCode(error, "ENOENT")) throw error;
}
}
async function assertOwnedProviderOutput(
absolutePath: string,
identity: Readonly<{ dev: number; ino: number }>,
): Promise<void> {
const metadata = await lstat(absolutePath);
if (
metadata.isSymbolicLink() ||
!metadata.isFile() ||
metadata.dev !== identity.dev ||
metadata.ino !== identity.ino
) {
throw new Error("provider raw output identity changed");
}
}
function hasErrorCode(error: unknown, code: string): boolean {
return Boolean(error && typeof error === "object" && "code" in error && error.code === code);
}