import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import { lstat, mkdir, open, readFile, readdir, rename, rm, unlink, } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { RELEASE_CANDIDATE_EVIDENCE_PATHS, RELEASE_CANDIDATE_MANIFEST_PATH, distSha256, releaseCandidateManifestSchema, type ReleaseCandidateManifest, } from "./release-candidate.ts"; import { supplyChainDigest } from "./supply-chain.ts"; import { assertSafePublishLeaf, ensureSafePublishDirectory, } from "./ci-gate-log.ts"; import { makePrivateTemporaryDirectory, withPrivateUmask, } from "./private-filesystem.ts"; const MAX_ARCHIVE_BYTES = 268_435_456; const MAX_CANDIDATE_FILES = 4_096; const MAX_ARCHIVE_MEMBERS = 8_192; const MAX_MEMBER_PATH_BYTES = 1_024; const TAR_EXECUTABLE = "/usr/bin/tar"; const TAR_ENVIRONMENT = Object.freeze({ PATH: "/usr/bin:/bin", LC_ALL: "C", LANG: "C" }); export type CapturedCandidateArchive = Readonly<{ bytes: Buffer; archiveSha256: string; }>; export async function captureCiCandidateArchive(input: Readonly<{ archivePath: string; expectedSha256: string; }>): Promise { if (!/^[a-f0-9]{64}$/u.test(input.expectedSha256)) { throw new TypeError("expected candidate archive SHA-256 is invalid"); } const absolute = path.resolve(input.archivePath); const before = await lstat(absolute); if (!before.isFile() || before.isSymbolicLink()) { throw new TypeError("candidate archive must be a regular non-symlink file"); } if (before.size <= 0 || before.size > MAX_ARCHIVE_BYTES) { throw new RangeError(`candidate archive size is outside 1..${MAX_ARCHIVE_BYTES}`); } const handle = await open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW); let bytes: Buffer; try { assertSameIdentity(before, await handle.stat()); bytes = await readCapturedArchive(handle, before.size); assertSameIdentity(before, await handle.stat()); } finally { await handle.close(); } const archiveSha256 = createHash("sha256").update(bytes).digest("hex"); if (archiveSha256 !== input.expectedSha256) { throw new Error("candidate archive SHA-256 mismatch"); } return Object.freeze({ bytes, archiveSha256 }); } export async function withVerifiedCapturedCandidate(input: Readonly<{ captured: CapturedCandidateArchive; verify: (view: Readonly<{ extractionRoot: string; manifest: ReleaseCandidateManifest; }>) => Promise; }>): Promise { let result: T | undefined; await verifyCapturedCiCandidateArchive( input.captured.bytes, input.captured.archiveSha256, { verifyExtracted: async (extractionRoot, manifest) => { result = await input.verify({ extractionRoot, manifest }); }, }, ); return result as T; } export async function verifyCiCandidateArchive( input: Readonly<{ archivePath: string; expectedSha256?: string; extractTo?: string; repositoryRoot?: string; }>, dependencies: Readonly<{ afterArchiveRead?: () => Promise }> = {}, ): Promise> { if (input.expectedSha256 && !/^[a-f0-9]{64}$/u.test(input.expectedSha256)) { throw new TypeError("expected candidate archive SHA-256 is invalid"); } const absolute = path.resolve(input.archivePath); const before = await lstat(absolute); if (!before.isFile() || before.isSymbolicLink()) { throw new TypeError("candidate archive must be a regular non-symlink file"); } if (before.size <= 0 || before.size > MAX_ARCHIVE_BYTES) { throw new RangeError(`candidate archive size is outside 1..${MAX_ARCHIVE_BYTES}`); } const handle = await open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW); let archive: Buffer; try { assertSameIdentity(before, await handle.stat()); archive = await readCapturedArchive(handle, before.size); assertSameIdentity(before, await handle.stat()); } finally { await handle.close(); } if (archive.byteLength !== before.size) { throw new Error("candidate archive changed size during capture"); } await dependencies.afterArchiveRead?.(); const archiveSha256 = createHash("sha256").update(archive).digest("hex"); if (input.expectedSha256 && archiveSha256 !== input.expectedSha256) { throw new Error("candidate archive SHA-256 mismatch"); } const extractionTarget = input.extractTo ? path.resolve(input.extractTo) : undefined; let extractionRoot: string; let extractionParentIdentity: Awaited> | undefined; if (extractionTarget) { if (!input.repositoryRoot) { throw new TypeError("repositoryRoot is required when publishing an extracted candidate"); } const repositoryRoot = path.resolve(input.repositoryRoot); extractionParentIdentity = await ensureSafePublishDirectory( repositoryRoot, path.dirname(extractionTarget), ); await assertSafePublishLeaf(extractionTarget, input.extractTo); extractionRoot = makePrivateTemporaryDirectory( path.join(path.dirname(extractionTarget), `.${path.basename(extractionTarget)}.verified-`), ); } else { extractionRoot = makePrivateTemporaryDirectory(path.join(tmpdir(), "ci-candidate-archive-")); } let published = false; try { const captured = await materializeCapturedArchive(archive); try { const preflightManifest = preflightArchiveHandle(captured.handle); extractArchiveHandle(captured.handle, extractionRoot); const verified = await verifyExtractedTree(extractionRoot, preflightManifest); if (extractionTarget) { const repositoryRoot = path.resolve(input.repositoryRoot!); const currentParentIdentity = await ensureSafePublishDirectory( repositoryRoot, path.dirname(extractionTarget), ); if ( !extractionParentIdentity || extractionParentIdentity.dev <= 0 || extractionParentIdentity.ino <= 0 || currentParentIdentity.dev !== extractionParentIdentity.dev || currentParentIdentity.ino !== extractionParentIdentity.ino ) { throw new Error("verified extraction parent identity changed"); } await assertSafePublishLeaf(extractionTarget, input.extractTo); if (await pathExists(extractionTarget)) { throw new Error(`verified extraction target already exists: ${input.extractTo}`); } await rename(extractionRoot, extractionTarget); published = true; } return Object.freeze({ archiveSha256, memberCount: verified.memberCount, manifest: verified.manifest, }); } finally { await captured.handle.close(); await rm(captured.root, { recursive: true, force: true }); } } finally { if (!published) await rm(extractionRoot, { recursive: true, force: true }); } } export async function verifyCapturedCiCandidateArchive( archive: Buffer, expectedSha256: string, dependencies: Readonly<{ verifyExtracted?: ( extractionRoot: string, manifest: ReleaseCandidateManifest, ) => Promise; }> = {}, ): Promise> { if (archive.byteLength <= 0 || archive.byteLength > MAX_ARCHIVE_BYTES) { throw new RangeError(`candidate archive size is outside 1..${MAX_ARCHIVE_BYTES}`); } if (!/^[a-f0-9]{64}$/u.test(expectedSha256)) { throw new TypeError("expected candidate archive SHA-256 is invalid"); } const archiveSha256 = createHash("sha256").update(archive).digest("hex"); if (archiveSha256 !== expectedSha256) { throw new Error("candidate archive SHA-256 mismatch"); } const captured = await materializeCapturedArchive(archive); const extractionRoot = makePrivateTemporaryDirectory(path.join(tmpdir(), "ci-captured-candidate-")); try { const manifest = preflightArchiveHandle(captured.handle); extractArchiveHandle(captured.handle, extractionRoot); const verified = await verifyExtractedTree(extractionRoot, manifest); await dependencies.verifyExtracted?.(extractionRoot, verified.manifest); return Object.freeze({ archiveSha256, memberCount: verified.memberCount, manifest: verified.manifest, }); } finally { await rm(extractionRoot, { recursive: true, force: true }); await captured.handle.close(); await rm(captured.root, { recursive: true, force: true }); } } function preflightArchiveHandle(archiveHandle: FileHandle): ReleaseCandidateManifest { const listed = spawnSync( TAR_EXECUTABLE, ["--list", "--verbose", "--numeric-owner", "--full-time", "--gzip", "--file", "/proc/self/fd/3"], { encoding: "utf8", maxBuffer: 16_777_216, timeout: 10_000, env: TAR_ENVIRONMENT, stdio: ["ignore", "pipe", "pipe", archiveHandle.fd], }, ); if (listed.status !== 0 || listed.signal || listed.error) { throw new Error( `candidate archive listing failed: ${listed.stderr || listed.error?.message || listed.signal}`, ); } const seen = new Set(); const regularMembers = new Set(); const directoryMembers = new Set(); let totalBytes = 0; const lines = listed.stdout.split(/\r?\n/u).filter(Boolean); if (lines.length === 0 || lines.length > MAX_ARCHIVE_MEMBERS) { throw new RangeError(`candidate archive member count is outside 1..${MAX_ARCHIVE_MEMBERS}`); } for (const line of lines) { const match = /^(?.{10})\s+\d+\/\d+\s+(?\d+)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:\s+[+-]\d{4})?\s+(?.+)$/u.exec(line); if (!match?.groups) throw new Error(`candidate archive listing is unparseable: ${line}`); const member = match.groups.path!.endsWith("/") ? match.groups.path!.slice(0, -1) : match.groups.path!; assertSafeMemberPath(member); if (seen.has(member)) throw new Error(`candidate archive duplicate member: ${member}`); seen.add(member); const mode = match.groups.mode!; if (!mode.startsWith("-") && !mode.startsWith("d")) { throw new Error(`candidate archive contains non-regular member: ${member}`); } if (mode.startsWith("-")) { const memberBytes = Number(match.groups.bytes); if ( member === RELEASE_CANDIDATE_MANIFEST_PATH && memberBytes > 8_388_608 ) { throw new RangeError("candidate manifest exceeds 8388608 bytes"); } totalBytes += memberBytes; if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_ARCHIVE_BYTES) { throw new RangeError("candidate archive expanded bytes exceed the bound"); } regularMembers.add(member); } else { directoryMembers.add(member); } } const manifest = readManifestFromArchive(archiveHandle); validateManifestSemantics(manifest); const expectedFiles = new Set([ ...manifest.files.map(({ path: member }) => member), RELEASE_CANDIDATE_MANIFEST_PATH, ]); for (const member of expectedFiles) assertSafeMemberPath(member); const expectedDirectories = new Set( directoryAncestors([...expectedFiles]).filter( (member) => member === "dist" || member.startsWith("dist/"), ), ); if ( JSON.stringify([...regularMembers].sort(asciiCompare)) !== JSON.stringify([...expectedFiles].sort(asciiCompare)) || JSON.stringify([...directoryMembers].sort(asciiCompare)) !== JSON.stringify([...expectedDirectories].sort(asciiCompare)) ) { throw new Error("candidate archive exact member set drift before extraction"); } return manifest; } function extractArchiveHandle(archiveHandle: FileHandle, extractionRoot: string): void { // `--no-same-permissions` is what keeps an untrusted archive from choosing // its own modes, but it hands the decision to the inherited umask instead. // Under a hardened `umask 077x` tar then creates directories it cannot // descend into and extraction fails part-way. Pinning the umask for the // duration makes the extracted tree exactly private, whatever the caller's // ambient state is. `spawnSync` keeps this window free of interleaved work. const extracted = withPrivateUmask(() => spawnSync( TAR_EXECUTABLE, [ "--extract", "--gzip", "--file", "/proc/self/fd/3", "--directory", extractionRoot, "--no-same-owner", "--no-same-permissions", ], { encoding: "utf8", maxBuffer: 1_048_576, timeout: 30_000, env: TAR_ENVIRONMENT, stdio: ["ignore", "pipe", "pipe", archiveHandle.fd], }, ), ); if (extracted.status !== 0 || extracted.signal || extracted.error) { throw new Error( `candidate archive isolated extraction failed: ${extracted.stderr || extracted.error?.message || extracted.signal}`, ); } } function validateManifestSemantics(manifest: ReleaseCandidateManifest): void { if (manifest.files.length === 0 || manifest.files.length > MAX_CANDIDATE_FILES) { throw new RangeError(`candidate manifest exceeds ${MAX_CANDIDATE_FILES} files`); } const canonicalFiles = [...manifest.files].sort((left, right) => asciiCompare(left.path, right.path), ); if (JSON.stringify(manifest.files) !== JSON.stringify(canonicalFiles)) { throw new Error("candidate manifest files are not in canonical ASCII order"); } const expectedFiles = new Map>(); let declaredBytes = 0; for (const file of manifest.files) { assertSafeMemberPath(file.path); if (expectedFiles.has(file.path)) { throw new Error(`candidate manifest duplicate file: ${file.path}`); } declaredBytes += file.bytes; if (!Number.isSafeInteger(declaredBytes) || declaredBytes > MAX_ARCHIVE_BYTES) { throw new RangeError("candidate manifest declared bytes exceed the archive bound"); } expectedFiles.set(file.path, { bytes: file.bytes, sha256: file.sha256 }); } const evidencePaths = [...expectedFiles.keys()] .filter((member) => !member.startsWith("dist/")) .sort(asciiCompare); if ( JSON.stringify(evidencePaths) !== JSON.stringify([...RELEASE_CANDIDATE_EVIDENCE_PATHS].sort(asciiCompare)) ) { throw new Error("candidate manifest evidence member set drift"); } const distFiles = manifest.files.filter(({ path: member }) => member.startsWith("dist/")); if (distFiles.length === 0) throw new Error("candidate manifest has no dist files"); const lockfile = expectedFiles.get("pnpm-lock.yaml"); if (!lockfile || lockfile.sha256 !== manifest.lockfileSha256) { throw new Error("candidate manifest lockfile digest summary mismatch"); } if ( distSha256(distFiles.map((file) => ({ ...file, gzipBytes: 0 }))) !== manifest.distSha256 ) { throw new Error("candidate manifest dist digest summary mismatch"); } if (supplyChainDigest(manifest.files) !== manifest.bundleSha256) { throw new Error("candidate manifest bundle digest summary mismatch"); } } async function verifyExtractedTree( extractionRoot: string, preflightManifest: ReleaseCandidateManifest, ): Promise> { const entries = await walkExtractedTree(extractionRoot); if (entries.length === 0 || entries.length > MAX_ARCHIVE_MEMBERS) { throw new RangeError(`candidate archive member count is outside 1..${MAX_ARCHIVE_MEMBERS}`); } const manifest = releaseCandidateManifestSchema.parse( JSON.parse( await readFile(path.join(extractionRoot, RELEASE_CANDIDATE_MANIFEST_PATH), "utf8"), ) as unknown, ); if (JSON.stringify(manifest) !== JSON.stringify(preflightManifest)) { throw new Error("candidate manifest changed between preflight and extraction"); } if (manifest.files.length === 0 || manifest.files.length > MAX_CANDIDATE_FILES) { throw new RangeError(`candidate manifest exceeds ${MAX_CANDIDATE_FILES} files`); } const canonicalFiles = [...manifest.files].sort((left, right) => asciiCompare(left.path, right.path), ); if (JSON.stringify(manifest.files) !== JSON.stringify(canonicalFiles)) { throw new Error("candidate manifest files are not in canonical ASCII order"); } const expectedFiles = new Map>(); let declaredBytes = 0; for (const file of manifest.files) { assertSafeMemberPath(file.path); if (expectedFiles.has(file.path)) throw new Error(`candidate manifest duplicate file: ${file.path}`); declaredBytes += file.bytes; if (!Number.isSafeInteger(declaredBytes) || declaredBytes > MAX_ARCHIVE_BYTES) { throw new RangeError("candidate manifest declared bytes exceed the archive bound"); } expectedFiles.set(file.path, { bytes: file.bytes, sha256: file.sha256 }); } const evidencePaths = [...expectedFiles.keys()] .filter((member) => !member.startsWith("dist/")) .sort(asciiCompare); if ( JSON.stringify(evidencePaths) !== JSON.stringify([...RELEASE_CANDIDATE_EVIDENCE_PATHS].sort(asciiCompare)) ) { throw new Error("candidate manifest evidence member set drift"); } const distFiles = manifest.files.filter(({ path: member }) => member.startsWith("dist/")); if (distFiles.length === 0) throw new Error("candidate manifest has no dist files"); const lockfile = expectedFiles.get("pnpm-lock.yaml"); if (!lockfile || lockfile.sha256 !== manifest.lockfileSha256) { throw new Error("candidate manifest lockfile digest summary mismatch"); } if ( distSha256(distFiles.map((file) => ({ ...file, gzipBytes: 0 }))) !== manifest.distSha256 ) { throw new Error("candidate manifest dist digest summary mismatch"); } if (supplyChainDigest(manifest.files) !== manifest.bundleSha256) { throw new Error("candidate manifest bundle digest summary mismatch"); } const expectedFilePaths = new Set([ ...expectedFiles.keys(), RELEASE_CANDIDATE_MANIFEST_PATH, ]); const expectedDirectories = new Set(directoryAncestors([...expectedFilePaths])); for (const entry of entries) { assertSafeMemberPath(entry.path); if (entry.type === "directory") { if (!expectedDirectories.has(entry.path)) { throw new Error(`candidate archive contains unexpected directory: ${entry.path}`); } } else if (!expectedFilePaths.has(entry.path)) { throw new Error(`candidate archive contains unexpected file: ${entry.path}`); } } const actualFiles = new Set( entries.filter(({ type }) => type === "file").map(({ path: member }) => member), ); for (const expected of expectedFilePaths) { if (!actualFiles.has(expected)) throw new Error(`candidate archive is missing file: ${expected}`); } for (const [member, expected] of expectedFiles) { const bytes = await readFile(path.join(extractionRoot, member)); if (bytes.byteLength !== expected.bytes) { throw new Error(`candidate archive member size mismatch: ${member}`); } if (createHash("sha256").update(bytes).digest("hex") !== expected.sha256) { throw new Error(`candidate archive member digest mismatch: ${member}`); } } return Object.freeze({ memberCount: entries.length, manifest }); } async function walkExtractedTree( root: string, relativeDirectory = "", ): Promise>> { const children = await readdir(path.join(root, relativeDirectory), { withFileTypes: true, }); const entries: Array> = []; for (const child of children.sort((left, right) => asciiCompare(left.name, right.name))) { const relative = relativeDirectory ? `${relativeDirectory}/${child.name}` : child.name; assertSafeMemberPath(relative); const metadata = await lstat(path.join(root, relative)); if (metadata.isSymbolicLink()) { throw new Error(`candidate archive contains non-regular member: ${relative}`); } if (metadata.isDirectory() && child.isDirectory()) { entries.push(Object.freeze({ path: relative, type: "directory" })); entries.push(...(await walkExtractedTree(root, relative))); } else if (metadata.isFile() && child.isFile()) { if (metadata.nlink !== 1) { throw new Error(`candidate archive contains hard-linked member: ${relative}`); } entries.push(Object.freeze({ path: relative, type: "file" })); } else { throw new Error(`candidate archive contains non-regular member: ${relative}`); } if (entries.length > MAX_ARCHIVE_MEMBERS) { throw new RangeError(`candidate archive exceeds ${MAX_ARCHIVE_MEMBERS} members`); } } return entries; } function assertSameIdentity( before: Awaited>, after: Awaited>, ): void { if ( !after.isFile() || before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size ) { throw new Error("candidate archive file identity changed"); } } async function readCapturedArchive( handle: FileHandle, expectedSize: number, ): Promise { const captured = Buffer.allocUnsafe(expectedSize + 1); let offset = 0; while (offset < captured.byteLength) { const { bytesRead } = await handle.read( captured, offset, captured.byteLength - offset, offset, ); if (bytesRead === 0) break; offset += bytesRead; } if (offset !== expectedSize) { throw new Error("candidate archive changed size during bounded capture"); } return captured.subarray(0, offset); } function readManifestFromArchive(archiveHandle: FileHandle): ReleaseCandidateManifest { const extracted = spawnSync( TAR_EXECUTABLE, [ "--extract", "--gzip", "--to-stdout", "--file", "/proc/self/fd/3", "--", RELEASE_CANDIDATE_MANIFEST_PATH, ], { maxBuffer: 8_388_609, timeout: 10_000, env: TAR_ENVIRONMENT, stdio: ["ignore", "pipe", "pipe", archiveHandle.fd], }, ); if (extracted.status !== 0 || extracted.signal || extracted.error) { throw new Error( `candidate manifest preflight failed: ${String(extracted.stderr) || extracted.error?.message || extracted.signal}`, ); } const bytes = Buffer.from(extracted.stdout); if (bytes.byteLength === 0 || bytes.byteLength > 8_388_608) { throw new RangeError("candidate manifest preflight size is outside 1..8388608"); } const source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); return releaseCandidateManifestSchema.parse(JSON.parse(source) as unknown); } async function materializeCapturedArchive( archive: Buffer, ): Promise> { const root = makePrivateTemporaryDirectory(path.join(tmpdir(), "ci-captured-archive-")); const file = path.join(root, "candidate.tar.gz"); let handle: FileHandle | undefined; try { handle = await open( file, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600, ); // `open` subtracts the umask too. The extractor re-opens this file by // `/proc/self/fd/N` from a child process, and that re-open is a real // permission check, so a umask-zeroed mode makes `tar` fail to read the // candidate it was just handed. await handle.chmod(0o600); await handle.writeFile(archive); await handle.sync(); await unlink(file); return Object.freeze({ root, handle }); } catch (error) { if (handle) await handle.close().catch(() => undefined); await rm(root, { recursive: true, force: true }); throw error; } } function assertSafeMemberPath(member: string): void { if ( !member || member.startsWith("-") || Buffer.byteLength(member, "utf8") > MAX_MEMBER_PATH_BYTES || member.includes("\\") || [...member].some((character) => { const codePoint = character.codePointAt(0)!; return codePoint <= 0x1f || codePoint === 0x7f; }) || path.posix.isAbsolute(member) || path.posix.normalize(member) !== member || member === ".." || member.startsWith("../") || member.includes("/../") ) { throw new TypeError(`candidate archive contains unsafe member path: ${member}`); } } function directoryAncestors(files: readonly string[]): string[] { const directories = new Set(); for (const file of files) { let directory = path.posix.dirname(file); while (directory !== ".") { directories.add(directory); directory = path.posix.dirname(directory); } } return [...directories]; } function asciiCompare(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } function hasErrorCode(error: unknown, code: string): boolean { return Boolean(error && typeof error === "object" && "code" in error && error.code === code); } async function pathExists(target: string): Promise { try { await lstat(target); return true; } catch (error) { if (hasErrorCode(error, "ENOENT")) return false; throw error; } }