import { spawn } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; import { constants } from "node:fs"; import { lstat, open, type FileHandle } from "node:fs/promises"; import path from "node:path"; import { decodeProviderGuardianPublished, decodeProviderGuardianReady, encodeProviderGuardianCommit, encodeProviderGuardianGuard, encodeProviderGuardianPublish, MAX_PROVIDER_GUARDIAN_FRAME_PAYLOAD_BYTES, MAX_PROVIDER_GUARDIAN_LEASE_MS, MAX_PROVIDER_SEALED_BYTES, providerGuardianRawStagingLeaf, providerGuardianSealedTempLeaf, type ProviderGuardianKind, } from "./provider-guardian-protocol.ts"; import { cleanupOwnedProviderReport } from "./provider-raw-cleanup.ts"; const RESPONSE_TIMEOUT_MS = 5_000; const CLOSE_TIMEOUT_MS = 5_000; const MAX_CONTROL_OUTPUT_BYTES = 4_096; type OwnedIdentity = Readonly<{ dev: number; ino: number }>; type RecoveryAuthority = Readonly<{ rawDirectoryHandle: FileHandle; evidenceDirectoryHandle: FileHandle; rawStagingHandle: FileHandle; sealedTempHandle: FileHandle; rawIdentity: OwnedIdentity; sealedIdentity: OwnedIdentity; rawStagingPinnedPath: string; rawPinnedPath: string; sealedTempPinnedPath: string; sealedPinnedPath: string; }>; export type ProviderGuardianLease = Readonly<{ pid: number; rawPath: string; rawIdentity: OwnedIdentity; sealedPath: string; sealedTempPath: string; sealedIdentity: OwnedIdentity; prematureExit: Promise; publish(bytes: Buffer): Promise; commit(): Promise; abort(): Promise; }>; export type StartProviderGuardianInput = Readonly<{ kind: ProviderGuardianKind; workspaceRoot: string; leaseMs: number; guardianScript: string; }>; export type ProviderScopeGuardianLatch = Readonly<{ activeFailure: Promise; close(): Promise; failure(): Error | undefined; }>; export function createProviderScopeGuardianLatch( guardianExit: Promise, ): ProviderScopeGuardianLatch { let active = true; let closing: Promise | undefined; let observedFailure: Error | undefined; let signalActiveFailure!: (error: Error) => void; const activeFailure = new Promise((resolve) => { signalActiveFailure = resolve; }); void guardianExit.then((error) => { observedFailure = error; if (active) signalActiveFailure(error); }); return Object.freeze({ activeFailure, close: () => { closing ??= Promise.resolve().then(() => { active = false; }); return closing; }, failure: () => observedFailure, }); } export function assertProviderGuardianLeasePaths( lease: Readonly<{ rawPath: string; sealedPath: string }>, expected: Readonly<{ rawPath: string; sealedPath: string }>, ): void { if (lease.rawPath !== expected.rawPath) { throw new Error("provider guardian returned a noncanonical raw path"); } if (lease.sealedPath !== expected.sealedPath) { throw new Error("provider guardian returned a noncanonical sealed path"); } } type GuardianResult = Readonly<{ code: number | null; error?: Error; signal: NodeJS.Signals | null; }>; export async function startProviderGuardian( input: StartProviderGuardianInput, ): Promise { if ( (input.kind !== "vulnerability" && input.kind !== "provenance") || !path.isAbsolute(input.workspaceRoot) || !path.isAbsolute(input.guardianScript) || !Number.isSafeInteger(input.leaseMs) || input.leaseMs <= 0 || input.leaseMs > MAX_PROVIDER_GUARDIAN_LEASE_MS ) { throw new TypeError("provider guardian client input is invalid"); } const rawLeaf = input.kind === "vulnerability" ? "vulnerability-report.json" : "provenance-attestation.json"; const evidenceRoot = path.resolve(input.workspaceRoot, "provider-evidence"); const rawDirectory = path.join(evidenceRoot, "untrusted"); const rawPath = path.join(evidenceRoot, "untrusted", rawLeaf); const sealedPath = path.join(evidenceRoot, rawLeaf); const nonce = randomBytes(32); const rawStagingLeaf = providerGuardianRawStagingLeaf(input.kind, nonce); const sealedTempLeaf = providerGuardianSealedTempLeaf(input.kind, nonce); const sealedTempPath = path.join(evidenceRoot, sealedTempLeaf); const recovery = await openRecoveryAuthority({ rawDirectory, evidenceRoot, rawLeaf, rawStagingLeaf, sealedLeaf: rawLeaf, sealedTempLeaf, }); let child: ReturnType; try { await assertRecoveryLeavesMissing(recovery); child = spawn(process.execPath, [input.guardianScript], { cwd: input.workspaceRoot, env: {}, stdio: [ "pipe", "pipe", "pipe", recovery.rawDirectoryHandle.fd, recovery.evidenceDirectoryHandle.fd, recovery.rawStagingHandle.fd, recovery.sealedTempHandle.fd, ], }); } catch (error) { return await closeRecoveryAndThrow(recovery, error); } if (!child.pid || !child.stdin || !child.stdout || !child.stderr) { child.kill("SIGKILL"); return await closeRecoveryAndThrow( recovery, new Error("provider guardian process pipes are unavailable"), ); } let state: "starting" | "guarding" | "publishing" | "published" | "committing" | "aborting" | "terminated" = "starting"; let stderr = Buffer.alloc(0); let inputError: Error | undefined; child.stdin.once("error", (error) => { inputError = error; }); child.stderr.on("data", (chunk: Buffer | string) => { const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); if (stderr.byteLength < MAX_CONTROL_OUTPUT_BYTES) { stderr = Buffer.concat([stderr, bytes.subarray(0, MAX_CONTROL_OUTPUT_BYTES - stderr.byteLength)]); } }); const completion = guardianCompletion(child); let signalPrematureExit!: (error: Error) => void; const prematureExit = new Promise((resolve) => { signalPrematureExit = resolve; }); void completion.then((result) => { if (state === "guarding" || state === "publishing" || state === "published") { signalPrematureExit(guardianCloseError(result, stderr)); } }); let ready: ReturnType; try { const readyResponse = waitForFrame(child.stdout, completion, "READY"); child.stdin.write(encodeProviderGuardianGuard({ kind: input.kind, nonce, deadlineEpochMs: Date.now() + input.leaseMs, })); ready = decodeProviderGuardianReady(await readyResponse, nonce); if ( ready.sealedTempLeaf !== sealedTempLeaf || ready.rawDev !== recovery.rawIdentity.dev || ready.rawIno !== recovery.rawIdentity.ino || ready.sealedDev !== recovery.sealedIdentity.dev || ready.sealedIno !== recovery.sealedIdentity.ino ) { throw new TypeError("provider guardian READY identity is invalid for its allocation"); } await assertPinnedLeafIdentity(recovery.rawPinnedPath, { dev: ready.rawDev, ino: ready.rawIno, }, 0o600); await assertPinnedLeafIdentity(recovery.sealedTempPinnedPath, { dev: ready.sealedDev, ino: ready.sealedIno, }, 0o600); await assertPinnedLeafMissing(recovery.rawStagingPinnedPath); if (child.exitCode !== null || child.signalCode !== null) { throw guardianCloseError(await completion, stderr); } state = "guarding"; } catch (error) { state = "aborting"; child.stdin.end(); const failures = [toError(error)]; try { await waitForClose(completion, child); } catch (closeError) { failures.push(toError(closeError)); } try { await cleanupStartupRecovery(recovery); } catch (cleanupError) { failures.push(toError(cleanupError)); } try { await closeRecoveryAuthority(recovery); } catch (closeError) { failures.push(toError(closeError)); } state = "terminated"; if (failures.length > 1) { throw new AggregateError(failures, "provider guardian startup failed", { cause: error }); } throw failures[0]!; } const rawIdentity = Object.freeze({ dev: ready.rawDev, ino: ready.rawIno }); const sealedIdentity = Object.freeze({ dev: ready.sealedDev, ino: ready.sealedIno }); const fallback = Object.freeze({ rawStagingPath: recovery.rawStagingPinnedPath, rawPath: recovery.rawPinnedPath, rawIdentity, sealedPath: recovery.sealedPinnedPath, sealedTempPath: recovery.sealedTempPinnedPath, sealedIdentity, }); const publish = async (bytes: Buffer): Promise => { if (state !== "guarding") throw new Error("provider guardian lease is not ready to publish"); if (!Buffer.isBuffer(bytes) || bytes.byteLength <= 0 || bytes.byteLength > MAX_PROVIDER_SEALED_BYTES) { throw new TypeError("provider guardian sealed bytes are invalid"); } state = "publishing"; try { await writePinnedSealedBytes(recovery.sealedTempHandle, sealedIdentity, bytes); const publishedResponse = waitForFrame(child.stdout!, completion, "PUBLISHED"); child.stdin!.write(encodeProviderGuardianPublish({ nonce, sealedDev: sealedIdentity.dev, sealedIno: sealedIdentity.ino, size: bytes.byteLength, sha256: createHash("sha256").update(bytes).digest("hex"), })); decodeProviderGuardianPublished( await publishedResponse, nonce, sealedIdentity, ); if (inputError) throw inputError; state = "published"; } catch (error) { state = "guarding"; throw error; } }; const commit = async (): Promise => { if (state !== "published") throw new Error("provider guardian lease is not ready to commit"); state = "committing"; child.stdin!.write(encodeProviderGuardianCommit(nonce)); child.stdin!.end(); let result: GuardianResult; try { result = await waitForClose(completion, child); } catch (error) { state = "terminated"; return await cleanupFallbackCloseAndThrow(fallback, recovery, error); } state = "terminated"; if (inputError) return await cleanupFallbackCloseAndThrow(fallback, recovery, inputError); if (result.error || result.code !== 0 || result.signal !== null) { return await cleanupFallbackCloseAndThrow( fallback, recovery, guardianCloseError(result, stderr), ); } await closeRecoveryAuthority(recovery); }; const abort = async (): Promise => { if (state !== "guarding" && state !== "published") { throw new Error("provider guardian lease already terminated"); } state = "aborting"; child.stdin!.end(); let closeError: unknown; try { await waitForClose(completion, child); } catch (error) { closeError = error; } state = "terminated"; if (closeError) return await cleanupFallbackCloseAndThrow(fallback, recovery, closeError); await cleanupFallbackAndClose(fallback, recovery); }; return Object.freeze({ pid: child.pid, rawPath, rawIdentity, sealedPath, sealedTempPath, sealedIdentity, prematureExit, publish, commit, abort, }); } async function writePinnedSealedBytes( handle: FileHandle, identity: OwnedIdentity, bytes: Buffer, ): Promise { assertPinnedMetadata(await handle.stat(), identity, 0o600, 0); await handle.truncate(0); await handle.writeFile(bytes); await handle.chmod(0o400); await handle.sync(); assertPinnedMetadata(await handle.stat(), identity, 0o400, bytes.byteLength); } function assertPinnedMetadata( metadata: Awaited>["stat"]>>, identity: OwnedIdentity, mode: number, size: number, ): void { if ( !metadata.isFile() || Number(metadata.dev) !== identity.dev || Number(metadata.ino) !== identity.ino || Number(metadata.nlink) !== 1 || (Number(metadata.mode) & 0o777) !== mode || Number(metadata.size) !== size ) { throw new TypeError("provider guardian sealed temp identity changed"); } } function guardianCompletion(child: ReturnType): Promise { return new Promise((resolve) => { child.once("error", (error) => resolve({ code: null, error, signal: null })); child.once("close", (code, signal) => resolve({ code, signal })); }); } async function waitForFrame( stdout: NodeJS.ReadableStream, completion: Promise, label: string, ): Promise { let timer: NodeJS.Timeout | undefined; let pending = Buffer.alloc(0); const response = new Promise((resolve, reject) => { const onData = (chunk: Buffer | string): void => { pending = Buffer.concat([pending, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]); if (pending.byteLength > MAX_CONTROL_OUTPUT_BYTES) { reject(new Error(`provider guardian ${label} output exceeded its bound`)); return; } if (pending.byteLength < 4) return; const payloadBytes = pending.readUInt32BE(0); if (payloadBytes <= 0 || payloadBytes > MAX_PROVIDER_GUARDIAN_FRAME_PAYLOAD_BYTES) { reject(new Error(`provider guardian ${label} frame length is invalid`)); return; } if (pending.byteLength < payloadBytes + 4) return; if (pending.byteLength !== payloadBytes + 4) { reject(new Error(`provider guardian ${label} output has trailing bytes`)); return; } resolve(pending.subarray(4)); }; stdout.on("data", onData); }); try { return await Promise.race([ response, completion.then((result) => { throw guardianCloseError(result, Buffer.alloc(0)); }), new Promise((_resolve, reject) => { timer = setTimeout( () => reject(new Error(`provider guardian ${label} timed out`)), RESPONSE_TIMEOUT_MS, ); }), ]); } finally { if (timer) clearTimeout(timer); stdout.removeAllListeners("data"); } } async function waitForClose( completion: Promise, child: ReturnType, ): Promise { let timer: NodeJS.Timeout | undefined; try { return await Promise.race([ completion, new Promise((_resolve, reject) => { timer = setTimeout(() => { child.kill("SIGKILL"); reject(new Error("provider guardian did not close within its bound")); }, CLOSE_TIMEOUT_MS); }), ]); } finally { if (timer) clearTimeout(timer); } } type FallbackIdentity = Readonly<{ rawStagingPath: string; rawPath: string; rawIdentity: OwnedIdentity; sealedPath: string; sealedTempPath: string; sealedIdentity: OwnedIdentity; }>; async function cleanupFallback(input: FallbackIdentity): Promise { const failures: Error[] = []; for (const target of [ { path: input.rawStagingPath, identity: input.rawIdentity }, { path: input.rawPath, identity: input.rawIdentity }, { path: input.sealedTempPath, identity: input.sealedIdentity }, { path: input.sealedPath, identity: input.sealedIdentity }, ]) { try { await cleanupOwnedProviderReport({ reportPath: target.path, reportDev: target.identity.dev, reportIno: target.identity.ino, }); } catch (error) { failures.push(toError(error)); } } if (failures.length > 0) { throw new AggregateError(failures, "provider guardian fallback cleanup failed", { cause: failures[0], }); } } async function cleanupFallbackCloseAndThrow( fallback: FallbackIdentity, recovery: RecoveryAuthority, primaryError: unknown, ): Promise { const failures = [toError(primaryError)]; try { await cleanupFallback(fallback); } catch (cleanupError) { failures.push(toError(cleanupError)); } try { await closeRecoveryAuthority(recovery); } catch (closeError) { failures.push(toError(closeError)); } if (failures.length > 1) { throw new AggregateError(failures, "provider guardian failure and recovery failed", { cause: failures[0], }); } throw failures[0]!; } async function cleanupFallbackAndClose( fallback: FallbackIdentity, recovery: RecoveryAuthority, ): Promise { const failures: Error[] = []; try { await cleanupFallback(fallback); } catch (cleanupError) { failures.push(toError(cleanupError)); } try { await closeRecoveryAuthority(recovery); } catch (closeError) { failures.push(toError(closeError)); } if (failures.length > 0) { throw new AggregateError(failures, "provider guardian abort recovery failed", { cause: failures[0], }); } } async function openRecoveryAuthority(input: Readonly<{ rawDirectory: string; evidenceRoot: string; rawLeaf: string; rawStagingLeaf: string; sealedLeaf: string; sealedTempLeaf: string; }>): Promise { let rawDirectoryHandle: FileHandle | undefined; let evidenceDirectoryHandle: FileHandle | undefined; let rawStagingHandle: FileHandle | undefined; let sealedTempHandle: FileHandle | undefined; let rawIdentity: OwnedIdentity | undefined; let sealedIdentity: OwnedIdentity | undefined; let rawStagingPinnedPath: string | undefined; let rawPinnedPath: string | undefined; let sealedTempPinnedPath: string | undefined; let sealedPinnedPath: string | undefined; try { rawDirectoryHandle = await open( input.rawDirectory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, ); await assertPinnedDirectory(rawDirectoryHandle, input.rawDirectory, "raw"); evidenceDirectoryHandle = await open( input.evidenceRoot, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, ); await assertPinnedDirectory(evidenceDirectoryHandle, input.evidenceRoot, "evidence"); rawStagingPinnedPath = `/proc/self/fd/${rawDirectoryHandle.fd}/${input.rawStagingLeaf}`; rawPinnedPath = `/proc/self/fd/${rawDirectoryHandle.fd}/${input.rawLeaf}`; sealedTempPinnedPath = `/proc/self/fd/${evidenceDirectoryHandle.fd}/${input.sealedTempLeaf}`; sealedPinnedPath = `/proc/self/fd/${evidenceDirectoryHandle.fd}/${input.sealedLeaf}`; rawStagingHandle = await open( rawStagingPinnedPath, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600, ); const rawMetadata = await rawStagingHandle.stat(); rawIdentity = Object.freeze({ dev: rawMetadata.dev, ino: rawMetadata.ino }); assertAllocatedPrivateMetadata(rawMetadata, rawIdentity, "raw staging"); await assertPinnedLeafIdentity(rawStagingPinnedPath, rawIdentity, 0o600); sealedTempHandle = await open( sealedTempPinnedPath, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600, ); const sealedMetadata = await sealedTempHandle.stat(); sealedIdentity = Object.freeze({ dev: sealedMetadata.dev, ino: sealedMetadata.ino }); assertAllocatedPrivateMetadata(sealedMetadata, sealedIdentity, "sealed temp"); await assertPinnedLeafIdentity(sealedTempPinnedPath, sealedIdentity, 0o600); return Object.freeze({ rawDirectoryHandle, evidenceDirectoryHandle, rawStagingHandle, sealedTempHandle, rawIdentity, sealedIdentity, rawStagingPinnedPath, rawPinnedPath, sealedTempPinnedPath, sealedPinnedPath, }); } catch (error) { const failures = [toError(error)]; for (const target of [ { path: rawStagingPinnedPath, identity: rawIdentity }, { path: rawPinnedPath, identity: rawIdentity }, { path: sealedTempPinnedPath, identity: sealedIdentity }, { path: sealedPinnedPath, identity: sealedIdentity }, ]) { if (!target.path || !target.identity) continue; try { await cleanupOwnedProviderReport({ reportPath: target.path, reportDev: target.identity.dev, reportIno: target.identity.ino, }); } catch (cleanupError) { failures.push(toError(cleanupError)); } } for (const handle of [ sealedTempHandle, rawStagingHandle, evidenceDirectoryHandle, rawDirectoryHandle, ]) { if (!handle) continue; try { await handle.close(); } catch (closeError) { failures.push(toError(closeError)); } } if (failures.length > 1) { throw new AggregateError(failures, "provider guardian recovery setup failed", { cause: error, }); } throw failures[0]!; } } function assertAllocatedPrivateMetadata( metadata: Awaited>, identity: OwnedIdentity, label: string, ): void { if ( !metadata.isFile() || Number(metadata.dev) !== identity.dev || Number(metadata.ino) !== identity.ino || Number(metadata.nlink) !== 1 || (Number(metadata.mode) & 0o777) !== 0o600 || Number(metadata.size) !== 0 ) { throw new TypeError(`provider guardian ${label} allocation is invalid`); } } async function assertPinnedDirectory( handle: FileHandle, canonicalPath: string, label: string, ): Promise { const [descriptorMetadata, pathMetadata] = await Promise.all([ handle.stat(), lstat(canonicalPath), ]); if ( !descriptorMetadata.isDirectory() || !pathMetadata.isDirectory() || pathMetadata.isSymbolicLink() || descriptorMetadata.dev !== pathMetadata.dev || descriptorMetadata.ino !== pathMetadata.ino ) { throw new TypeError(`provider guardian ${label} recovery directory identity changed`); } } async function assertRecoveryLeavesMissing(recovery: RecoveryAuthority): Promise { await assertPinnedLeafMissing(recovery.rawPinnedPath); await assertPinnedLeafMissing(recovery.sealedPinnedPath); assertAllocatedPrivateMetadata( await recovery.rawStagingHandle.stat(), recovery.rawIdentity, "raw staging", ); assertAllocatedPrivateMetadata( await recovery.sealedTempHandle.stat(), recovery.sealedIdentity, "sealed temp", ); await assertPinnedLeafIdentity( recovery.rawStagingPinnedPath, recovery.rawIdentity, 0o600, ); await assertPinnedLeafIdentity( recovery.sealedTempPinnedPath, recovery.sealedIdentity, 0o600, ); } async function assertPinnedLeafMissing(target: string): Promise { try { await lstat(target); throw new Error("provider guardian transaction leaf already exists"); } catch (error) { if (!hasErrorCode(error, "ENOENT")) throw error; } } async function assertPinnedLeafIdentity( target: string, identity: OwnedIdentity, mode: number, ): Promise { const metadata = await lstat(target); if ( !metadata.isFile() || metadata.isSymbolicLink() || metadata.dev !== identity.dev || metadata.ino !== identity.ino || metadata.nlink !== 1 || (metadata.mode & 0o777) !== mode || metadata.size !== 0 ) { throw new TypeError("provider guardian READY identity changed"); } } async function cleanupStartupRecovery(recovery: RecoveryAuthority): Promise { await cleanupFallback({ rawStagingPath: recovery.rawStagingPinnedPath, rawPath: recovery.rawPinnedPath, rawIdentity: recovery.rawIdentity, sealedTempPath: recovery.sealedTempPinnedPath, sealedPath: recovery.sealedPinnedPath, sealedIdentity: recovery.sealedIdentity, }); } async function closeRecoveryAuthority(recovery: RecoveryAuthority): Promise { const failures: Error[] = []; for (const handle of [ recovery.rawStagingHandle, recovery.sealedTempHandle, recovery.rawDirectoryHandle, recovery.evidenceDirectoryHandle, ]) { try { await handle.close(); } catch (error) { failures.push(toError(error)); } } if (failures.length > 0) { throw new AggregateError(failures, "provider guardian recovery directory close failed", { cause: failures[0], }); } } async function closeRecoveryAndThrow( recovery: RecoveryAuthority, primaryError: unknown, ): Promise { const failures = [toError(primaryError)]; try { await cleanupStartupRecovery(recovery); } catch (cleanupError) { failures.push(toError(cleanupError)); } try { await closeRecoveryAuthority(recovery); } catch (closeError) { failures.push(toError(closeError)); } if (failures.length > 1) { throw new AggregateError(failures, "provider guardian failure and recovery close failed", { cause: failures[0] }); } throw failures[0]!; } function hasErrorCode(error: unknown, code: string): boolean { return Boolean(error && typeof error === "object" && "code" in error && error.code === code); } function guardianCloseError(result: GuardianResult, stderr: Buffer): Error { if (result.error) return result.error; const detail = stderr.toString("utf8").trim(); return new Error( `provider guardian failed: exit=${result.code ?? "none"}, signal=${result.signal ?? "none"}${detail ? `, output=${detail}` : ""}`, ); } function toError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); }