diff --git a/scripts/lib/build-environment.ts b/scripts/lib/build-environment.ts index 9db5193..bb624b8 100644 --- a/scripts/lib/build-environment.ts +++ b/scripts/lib/build-environment.ts @@ -6,10 +6,22 @@ export const CI_BUILD_ENVIRONMENT_VARIABLES = Object.freeze([ "SOURCE_DATE_EPOCH", ]); +/** + * The repository's single definition of "this is a CI run". The workflow sets + * `CI: "true"` at the top-level `env:` block, so it holds in every job; the + * `CI_RUN_ID` family is declared only by the release-tier provider jobs and is + * absent from the merge gates. + */ +export function isCiRun( + environment: Readonly>, +) { + return environment.CI === "true"; +} + export function ciBuildEnvironmentFailures( environment: Readonly>, ) { - if (environment.CI !== "true") return []; + if (!isCiRun(environment)) return []; const failures = CI_BUILD_ENVIRONMENT_VARIABLES.filter( (name) => !environment[name]?.trim(), diff --git a/scripts/lib/provider-sandbox-probe.ts b/scripts/lib/provider-sandbox-probe.ts new file mode 100644 index 0000000..90c8afd --- /dev/null +++ b/scripts/lib/provider-sandbox-probe.ts @@ -0,0 +1,109 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; + +import { isCiRun } from "./build-environment.ts"; + +export const PROVIDER_SANDBOX_BINARY = "/usr/bin/bwrap"; + +/** + * The isolation options `runProviderInSandbox` opens its bubblewrap argument + * vector with. That function spreads this exact list, so the probe below can + * never clear a weaker sandbox than the one the provider actually runs in — and + * `tests/unit/provider-sandbox-probe.test.ts` fails if an isolation flag is + * added to the run without being added here. + */ +export const PROVIDER_SANDBOX_ISOLATION_ARGUMENTS: readonly string[] = Object.freeze([ + "--die-with-parent", "--new-session", "--as-pid-1", + "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net", + "--dev", "/dev", "--remount-ro", "/dev", + "--proc", "/proc", "--remount-ro", "/proc", +]); + +/** + * The read-only roots the real run binds before it can execute anything. The + * probe binds the same set so `/bin/true` resolves the way a provider command + * would. + */ +const PROVIDER_SANDBOX_PROBE_BINDINGS: readonly string[] = Object.freeze([ + "/usr", "/bin", "/lib", "/lib64", +]); + +export const PROVIDER_SANDBOX_UNAVAILABLE = "the provider sandbox is unavailable"; + +export type ProviderSandboxProbe = + | Readonly<{ usable: true }> + | Readonly<{ usable: false; reason: string }>; + +export type ProviderSandboxDecision = + | Readonly<{ outcome: "run" }> + | Readonly<{ outcome: "skip"; reason: string }> + | Readonly<{ outcome: "fail"; reason: string }>; + +export function providerSandboxProbeArguments(): string[] { + return [ + ...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS, + ...PROVIDER_SANDBOX_PROBE_BINDINGS.flatMap((source) => + existsSync(source) ? ["--ro-bind", source, source] : []), + "--", "/bin/true", + ]; +} + +/** + * Runs a trivial command under the real isolation options and reports what + * happened. Capability is never inferred from the binary being installed or + * from a kernel/version string: on a host with + * `kernel.apparmor_restrict_unprivileged_userns=1` the binary exists, reports a + * version, and still cannot create the user namespace it needs. + */ +export function probeProviderSandbox( + binary: string = PROVIDER_SANDBOX_BINARY, +): ProviderSandboxProbe { + const result = spawnSync(binary, providerSandboxProbeArguments(), { + encoding: "utf8", + timeout: 20_000, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.error) { + const code = (result.error as NodeJS.ErrnoException).code; + return { + usable: false, + reason: code === "ENOENT" + ? `${binary} is not installed` + : `${binary} could not be executed: ${result.error.message}`, + }; + } + if (result.status === 0) return { usable: true }; + return { usable: false, reason: sandboxFailureReason(binary, result) }; +} + +function sandboxFailureReason( + binary: string, + result: Readonly<{ status: number | null; signal: NodeJS.Signals | null; stderr: string }>, +): string { + const diagnostics = [...String(result.stderr ?? "") + .matchAll(/^(?:bwrap|prlimit):\s.*$/gmu)].map(([line]) => line); + const detail = diagnostics.length > 0 + ? diagnostics.join("; ") + : String(result.stderr ?? "").trim() || "no diagnostic output"; + const exit = result.signal ? `signal=${result.signal}` : `exit=${result.status}`; + return `${binary} could not start an isolated sandbox (${exit}): ${detail}`; +} + +/** + * Locally an unusable sandbox is an environment fact and the sandboxed suites + * are skipped with the reason attached. In CI it is a regression — a security + * gate that silently stopped running is exactly what those suites exist to + * catch — so the same probe result fails the run instead. + */ +export function providerSandboxDecision( + probe: ProviderSandboxProbe, + environment: Readonly>, +): ProviderSandboxDecision { + if (probe.usable) return { outcome: "run" }; + const reason = `${PROVIDER_SANDBOX_UNAVAILABLE}: ${probe.reason}`; + return isCiRun(environment) ? { outcome: "fail", reason } : { outcome: "skip", reason }; +} + +export function assertProviderSandboxUsable(decision: ProviderSandboxDecision): void { + if (decision.outcome === "fail") throw new Error(decision.reason); +} diff --git a/scripts/run-and-validate-provider.ts b/scripts/run-and-validate-provider.ts index 2e69b7c..b2022c3 100644 --- a/scripts/run-and-validate-provider.ts +++ b/scripts/run-and-validate-provider.ts @@ -30,6 +30,7 @@ import { type ProviderGuardianLease, } from "./lib/provider-guardian-client.ts"; import { createProviderOutputLimiter } from "./lib/provider-output-limiter.ts"; +import { PROVIDER_SANDBOX_ISOLATION_ARGUMENTS } from "./lib/provider-sandbox-probe.ts"; const kind = process.argv[process.argv.indexOf("--kind") + 1]; const PROVIDER_TMP_BYTES = 16_777_216; @@ -246,10 +247,7 @@ async function runProviderInSandbox( ): Promise { const cpuSeconds = providerCpuSeconds(); const bwrapArguments = [ - "--die-with-parent", "--new-session", "--as-pid-1", - "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net", - "--dev", "/dev", "--remount-ro", "/dev", - "--proc", "/proc", "--remount-ro", "/proc", + ...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS, "--size", String(PROVIDER_TMP_BYTES), "--tmpfs", "/tmp", "--dir", "/tmp/provider-home", "--size", String(PROVIDER_MASK_BYTES), "--tmpfs", "/etc", diff --git a/tests/unit/ci-artifact-contract.test.ts b/tests/unit/ci-artifact-contract.test.ts index e6587f8..a4c2e71 100644 --- a/tests/unit/ci-artifact-contract.test.ts +++ b/tests/unit/ci-artifact-contract.test.ts @@ -6,7 +6,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; -import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it, type TestContext } from "vitest"; import type { CiGateArtifact, @@ -40,6 +40,11 @@ import { } from "../../scripts/lib/provider-evidence.ts"; import { localEvidenceAssessmentArtifactSchema } from "../../scripts/contracts/release-artifacts.ts"; import { readProviderTrust } from "../../scripts/lib/provider-trust.ts"; +import { + assertProviderSandboxUsable, + probeProviderSandbox, + providerSandboxDecision, +} from "../../scripts/lib/provider-sandbox-probe.ts"; import { createReleaseCandidateManifest, LOCAL_EVIDENCE_ASSESSMENT_PATH, @@ -55,6 +60,26 @@ import { */ const PROCESS_HEAVY_TIMEOUT_MS = 30_000; +/** + * Every test that reaches `runProviderSupervisor` needs bubblewrap to actually + * create the namespaces `PROVIDER_SANDBOX_ISOLATION_ARGUMENTS` asks for. Where + * unprivileged user namespaces are denied — a host with + * `kernel.apparmor_restrict_unprivileged_userns=1`, a container without + * `CAP_SYS_ADMIN` — bwrap is installed and answers `--version` but still cannot + * run, and those tests fail on the environment rather than on the product. + * + * The probe runs once per file and decides between two very different + * responses: locally the affected tests are skipped with the bwrap diagnostic + * attached, in CI the same result fails the run through the guard test below. + * Skipping in CI too would quietly disarm the sandbox gate, which is exactly + * the regression these tests exist to catch. + */ +const providerSandbox = providerSandboxDecision(probeProviderSandbox(), process.env); + +function requireProviderSandbox(ctx: TestContext): void { + if (providerSandbox.outcome !== "run") ctx.skip(providerSandbox.reason); +} + const temporaryRoots: string[] = []; let providerBaseRoot: string | undefined; const sha256 = (value: Buffer | string) => @@ -441,6 +466,10 @@ describe("CI artifact validator", () => { }); describe("candidate archive and provider upload boundaries", () => { + it("fails the run in CI when the provider sandbox is unavailable", () => { + expect(() => assertProviderSandboxUsable(providerSandbox)).not.toThrow(); + }); + it("accepts only the manifest-bound candidate member set and bytes", async () => { const fixture = await createCandidateArchiveFixture(); await expect( @@ -652,9 +681,11 @@ describe("candidate archive and provider upload boundaries", () => { await expect(readFile(markerPath)).rejects.toMatchObject({ code: "ENOENT" }); }); - it.each(["invalid trust", "invalid archive", "provider failure"] as const)( + it.for(["invalid trust", "invalid archive", "provider failure"] as const)( "leaves no stale raw report after %s and permits a clean retry", - async (failure) => { + { timeout: 20_000 }, + async (failure, ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); const archiveBytes = await readFile(fixture.archivePath); const publicKeyBytes = await readFile(fixture.publicKeyPath); @@ -678,10 +709,10 @@ describe("candidate archive and provider upload boundaries", () => { }); expect(retried.status, retried.stderr).toBe(0); }, - 20_000, ); - it("drains and kills provider background processes before sealing evidence", async () => { + it("drains and kills provider background processes before sealing evidence", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const sealedPath = path.join( @@ -722,7 +753,8 @@ describe("candidate archive and provider upload boundaries", () => { ); }, 10_000); - it("does not expose or mutate a host path outside the sandboxed workspace", async () => { + it("does not expose or mutate a host path outside the sandboxed workspace", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const outside = await temporaryRoot("provider-host-canary-"); @@ -743,7 +775,8 @@ describe("candidate archive and provider upload boundaries", () => { await expect(readFile(canary, "utf8")).resolves.toBe("host-secret\n"); }, 10_000); - it("applies effective aggregate cgroup limits without exposing command or credentials", async () => { + it("applies effective aggregate cgroup limits without exposing command or credentials", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const credential = "live-provider-credential-sentinel"; @@ -830,7 +863,8 @@ describe("candidate archive and provider upload boundaries", () => { await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]); }, PROCESS_HEAVY_TIMEOUT_MS); - it("kills and collects an active provider when its guardian dies", async () => { + it("kills and collects an active provider when its guardian dies", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const sealedPath = path.join(fixture.root, "provider-evidence/vulnerability-report.json"); @@ -873,7 +907,8 @@ describe("candidate archive and provider upload boundaries", () => { expect(retried.status, retried.stderr).toBe(0); }, 20_000); - it("collects the whole provider scope when its supervisor dies", async () => { + it("collects the whole provider scope when its supervisor dies", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const providerScript = path.join(fixture.root, "provider-parent-death.mjs"); @@ -913,7 +948,8 @@ describe("candidate archive and provider upload boundaries", () => { await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]); }, 10_000); - it("fails closed when the guardian dies after scope collection and before commit", async () => { + it("fails closed when the guardian dies after scope collection and before commit", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); await writeFile(fixture.providerWriter, providerV2WriterSource()); @@ -958,7 +994,8 @@ describe("candidate archive and provider upload boundaries", () => { } }, 10_000); - it("cleans published evidence when the supervisor dies and permits same-workspace retry", async () => { + it("cleans published evidence when the supervisor dies and permits same-workspace retry", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); await writeFile(fixture.providerWriter, providerV2WriterSource()); @@ -1004,7 +1041,8 @@ describe("candidate archive and provider upload boundaries", () => { } }, 15_000); - it("observes EMFILE at the provider FD limit and still emits valid evidence", async () => { + it("observes EMFILE at the provider FD limit and still emits valid evidence", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const adapter = path.join(fixture.root, "provider-fd-limit.mjs"); @@ -1025,7 +1063,8 @@ describe("candidate archive and provider upload boundaries", () => { await expect(lstat(fixture.reportPath)).rejects.toMatchObject({ code: "ENOENT" }); }, 10_000); - it("enforces CPU RLIMIT before the independent wall-clock timeout", async () => { + it("enforces CPU RLIMIT before the independent wall-clock timeout", async (ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const started = Date.now(); @@ -1042,9 +1081,11 @@ describe("candidate archive and provider upload boundaries", () => { await expect(lstat(fixture.reportPath)).rejects.toMatchObject({ code: "ENOENT" }); }, 10_000); - it.each(["output", "timeout"] as const)( + it.for(["output", "timeout"] as const)( "kills descendants and collects the scope on provider %s termination", - async (reason) => { + { timeout: 10_000 }, + async (reason, ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); await rm(fixture.reportPath); const providerScript = path.join(fixture.root, `provider-${reason}-descendant.mjs`); @@ -1084,12 +1125,13 @@ describe("candidate archive and provider upload boundaries", () => { expect(cgroupPids.every((pid) => !processExists(pid))).toBe(true); await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]); }, - 10_000, ); - it.each(["vulnerability", "provenance"] as const)( + it.for(["vulnerability", "provenance"] as const)( "runs the %s adapter offline and collects its unit", - async (kind) => { + { timeout: 10_000 }, + async (kind, ctx) => { + requireProviderSandbox(ctx); const fixture = await createProviderFixture(); const canary = await startLoopbackCanary(fixture.root); try { @@ -1127,7 +1169,6 @@ describe("candidate archive and provider upload boundaries", () => { await waitForChildClose(canary.child); } }, - 10_000, ); }); diff --git a/tests/unit/provider-sandbox-probe.test.ts b/tests/unit/provider-sandbox-probe.test.ts new file mode 100644 index 0000000..f8b9e56 --- /dev/null +++ b/tests/unit/provider-sandbox-probe.test.ts @@ -0,0 +1,95 @@ +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + PROVIDER_SANDBOX_ISOLATION_ARGUMENTS, + PROVIDER_SANDBOX_UNAVAILABLE, + assertProviderSandboxUsable, + probeProviderSandbox, + providerSandboxDecision, + providerSandboxProbeArguments, +} from "../../scripts/lib/provider-sandbox-probe.ts"; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function stubSandboxBinary(body: string): Promise { + const root = await mkdtemp(path.join(tmpdir(), "sandbox-probe-")); + roots.push(root); + const binary = path.join(root, "bwrap"); + await writeFile(binary, `#!/bin/sh\n${body}\n`); + await chmod(binary, 0o755); + return binary; +} + +describe("provider sandbox probe", () => { + it("probes execution rather than the existence of the binary", async () => { + const refuses = await stubSandboxBinary( + 'echo "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted" >&2\nexit 1', + ); + const probe = probeProviderSandbox(refuses); + expect(probe.usable).toBe(false); + expect(probe.usable ? "" : probe.reason).toContain( + "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted", + ); + expect(probeProviderSandbox(await stubSandboxBinary("exit 0"))).toEqual({ usable: true }); + }); + + it("reports a missing sandbox binary as its own reason", async () => { + const missing = path.join(await mkdtemp(path.join(tmpdir(), "sandbox-absent-")), "bwrap"); + roots.push(path.dirname(missing)); + const probe = probeProviderSandbox(missing); + expect(probe.usable).toBe(false); + expect(probe.usable ? "" : probe.reason).toMatch(/is not installed/u); + }); + + it("probes with the same isolation options the real provider run applies", async () => { + const source = await readFile( + new URL("../../scripts/run-and-validate-provider.ts", import.meta.url), + "utf8", + ); + // The run builds its bubblewrap vector from the shared constant and + // declares no isolation option of its own, so an option cannot be added to + // the run without the probe having to clear it as well. + expect(source).toContain("...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS"); + expect(source.match(/"(?:--unshare-[a-z]+|--as-pid-1|--new-session|--die-with-parent)"/gu)) + .toBeNull(); + expect(PROVIDER_SANDBOX_ISOLATION_ARGUMENTS).toEqual( + expect.arrayContaining([ + "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net", "--as-pid-1", + ]), + ); + const probeArguments = providerSandboxProbeArguments(); + expect(probeArguments.slice(0, PROVIDER_SANDBOX_ISOLATION_ARGUMENTS.length)).toEqual([ + ...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS, + ]); + expect(probeArguments.at(-1)).toBe("/bin/true"); + }); + + it("skips locally and fails in CI on the same unusable sandbox", () => { + const probe = { usable: false, reason: "bwrap: setting up uid map: Permission denied" } as const; + expect(providerSandboxDecision(probe, {})).toEqual({ + outcome: "skip", + reason: `${PROVIDER_SANDBOX_UNAVAILABLE}: bwrap: setting up uid map: Permission denied`, + }); + expect(providerSandboxDecision(probe, { CI: "true" })).toEqual({ + outcome: "fail", + reason: `${PROVIDER_SANDBOX_UNAVAILABLE}: bwrap: setting up uid map: Permission denied`, + }); + expect(providerSandboxDecision({ usable: true }, { CI: "true" })).toEqual({ outcome: "run" }); + }); + + it("asserts only the CI failure outcome", () => { + const probe = { usable: false, reason: "denied" } as const; + expect(() => assertProviderSandboxUsable(providerSandboxDecision(probe, { CI: "true" }))) + .toThrow(new RegExp(PROVIDER_SANDBOX_UNAVAILABLE, "u")); + expect(() => assertProviderSandboxUsable(providerSandboxDecision(probe, {}))).not.toThrow(); + expect(() => assertProviderSandboxUsable(providerSandboxDecision({ usable: true }, { CI: "true" }))) + .not.toThrow(); + }); +});