test: probe the provider sandbox instead of failing on it

The 16 sandboxed provider tests in `tests/unit/ci-artifact-contract.test.ts`
fail wherever unprivileged user namespaces are denied. bubblewrap is installed
and answers `--version`, but `bwrap --unshare-net … -- /bin/true` exits 1 with
`bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted`, and it fails the
same way with no netns flag at all (`setting up uid map: Permission denied`), so
this is the whole nested-userns capability and not one option. Sixteen
assertion errors on every local run buried whatever else the file had to say.

`scripts/lib/provider-sandbox-probe.ts` now runs a trivial command under the
real isolation options — `runProviderInSandbox` spreads the same
`PROVIDER_SANDBOX_ISOLATION_ARGUMENTS`, and a test fails if an isolation option
is added to the run without the probe having to clear it. Capability is
measured, never inferred from the binary existing or from a version string;
both would pass here.

An unusable sandbox means two different things in two places, so the decision
is explicit. Locally it is an environment fact: the affected tests skip and
carry the bwrap diagnostic as their skip note, visible as `↓ … [reason]`. In CI
it is a regression — a security gate that silently stopped running is exactly
what these tests exist to catch — so the same probe result fails the run
through one guard test that says "the provider sandbox is unavailable" instead
of sixteen assertion errors.

CI is detected with `CI === "true"` via a new `isCiRun`, sharing the predicate
that already gates `ciBuildEnvironmentFailures`. The workflow sets it at the
top-level `env:` block, so it holds in every job; `CI_RUN_ID` and its
`GITEA_`/`GITHUB_` fallbacks are declared only by the release-tier provider
jobs and are absent from the merge gates that run this file, so keying on them
would have left the CI branch permanently dead.

The three `it.each` groups become `it.for` because only `.for` passes the test
context, which is what carries the skip note.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/unit/ci-artifact-contract.test.ts

@@ -6,7 +6,7 @@ import { tmpdir } from "node:os";
 import path from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 
-import { afterAll, afterEach, describe, expect, it } from "vitest";
+import { afterAll, afterEach, describe, expect, it, type TestContext } from "vitest";
 
 import type {
   CiGateArtifact,
@@ -40,6 +40,11 @@ import {
 } from "../../scripts/lib/provider-evidence.ts";
 import { localEvidenceAssessmentArtifactSchema } from "../../scripts/contracts/release-artifacts.ts";
 import { readProviderTrust } from "../../scripts/lib/provider-trust.ts";
+import {
+  assertProviderSandboxUsable,
+  probeProviderSandbox,
+  providerSandboxDecision,
+} from "../../scripts/lib/provider-sandbox-probe.ts";
 import {
   createReleaseCandidateManifest,
   LOCAL_EVIDENCE_ASSESSMENT_PATH,
@@ -55,6 +60,26 @@ import {
  */
 const PROCESS_HEAVY_TIMEOUT_MS = 30_000;
 
+/**
+ * Every test that reaches `runProviderSupervisor` needs bubblewrap to actually
+ * create the namespaces `PROVIDER_SANDBOX_ISOLATION_ARGUMENTS` asks for. Where
+ * unprivileged user namespaces are denied — a host with
+ * `kernel.apparmor_restrict_unprivileged_userns=1`, a container without
+ * `CAP_SYS_ADMIN` — bwrap is installed and answers `--version` but still cannot
+ * run, and those tests fail on the environment rather than on the product.
+ *
+ * The probe runs once per file and decides between two very different
+ * responses: locally the affected tests are skipped with the bwrap diagnostic
+ * attached, in CI the same result fails the run through the guard test below.
+ * Skipping in CI too would quietly disarm the sandbox gate, which is exactly
+ * the regression these tests exist to catch.
+ */
+const providerSandbox = providerSandboxDecision(probeProviderSandbox(), process.env);
+
+function requireProviderSandbox(ctx: TestContext): void {
+  if (providerSandbox.outcome !== "run") ctx.skip(providerSandbox.reason);
+}
+
 const temporaryRoots: string[] = [];
 let providerBaseRoot: string | undefined;
 const sha256 = (value: Buffer | string) =>
@@ -441,6 +466,10 @@ describe("CI artifact validator", () => {
 });
 
 describe("candidate archive and provider upload boundaries", () => {
+  it("fails the run in CI when the provider sandbox is unavailable", () => {
+    expect(() => assertProviderSandboxUsable(providerSandbox)).not.toThrow();
+  });
+
   it("accepts only the manifest-bound candidate member set and bytes", async () => {
     const fixture = await createCandidateArchiveFixture();
     await expect(
@@ -652,9 +681,11 @@ describe("candidate archive and provider upload boundaries", () => {
     await expect(readFile(markerPath)).rejects.toMatchObject({ code: "ENOENT" });
   });
 
-  it.each(["invalid trust", "invalid archive", "provider failure"] as const)(
+  it.for(["invalid trust", "invalid archive", "provider failure"] as const)(
     "leaves no stale raw report after %s and permits a clean retry",
-    async (failure) => {
+    { timeout: 20_000 },
+    async (failure, ctx) => {
+      requireProviderSandbox(ctx);
       const fixture = await createProviderFixture();
       const archiveBytes = await readFile(fixture.archivePath);
       const publicKeyBytes = await readFile(fixture.publicKeyPath);
@@ -678,10 +709,10 @@ describe("candidate archive and provider upload boundaries", () => {
       });
       expect(retried.status, retried.stderr).toBe(0);
     },
-    20_000,
   );
 
-  it("drains and kills provider background processes before sealing evidence", async () => {
+  it("drains and kills provider background processes before sealing evidence", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const sealedPath = path.join(
@@ -722,7 +753,8 @@ describe("candidate archive and provider upload boundaries", () => {
     );
   }, 10_000);
 
-  it("does not expose or mutate a host path outside the sandboxed workspace", async () => {
+  it("does not expose or mutate a host path outside the sandboxed workspace", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const outside = await temporaryRoot("provider-host-canary-");
@@ -743,7 +775,8 @@ describe("candidate archive and provider upload boundaries", () => {
     await expect(readFile(canary, "utf8")).resolves.toBe("host-secret\n");
   }, 10_000);
 
-  it("applies effective aggregate cgroup limits without exposing command or credentials", async () => {
+  it("applies effective aggregate cgroup limits without exposing command or credentials", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const credential = "live-provider-credential-sentinel";
@@ -830,7 +863,8 @@ describe("candidate archive and provider upload boundaries", () => {
     await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]);
   }, PROCESS_HEAVY_TIMEOUT_MS);
 
-  it("kills and collects an active provider when its guardian dies", async () => {
+  it("kills and collects an active provider when its guardian dies", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const sealedPath = path.join(fixture.root, "provider-evidence/vulnerability-report.json");
@@ -873,7 +907,8 @@ describe("candidate archive and provider upload boundaries", () => {
     expect(retried.status, retried.stderr).toBe(0);
   }, 20_000);
 
-  it("collects the whole provider scope when its supervisor dies", async () => {
+  it("collects the whole provider scope when its supervisor dies", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const providerScript = path.join(fixture.root, "provider-parent-death.mjs");
@@ -913,7 +948,8 @@ describe("candidate archive and provider upload boundaries", () => {
     await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]);
   }, 10_000);
 
-  it("fails closed when the guardian dies after scope collection and before commit", async () => {
+  it("fails closed when the guardian dies after scope collection and before commit", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     await writeFile(fixture.providerWriter, providerV2WriterSource());
@@ -958,7 +994,8 @@ describe("candidate archive and provider upload boundaries", () => {
     }
   }, 10_000);
 
-  it("cleans published evidence when the supervisor dies and permits same-workspace retry", async () => {
+  it("cleans published evidence when the supervisor dies and permits same-workspace retry", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     await writeFile(fixture.providerWriter, providerV2WriterSource());
@@ -1004,7 +1041,8 @@ describe("candidate archive and provider upload boundaries", () => {
     }
   }, 15_000);
 
-  it("observes EMFILE at the provider FD limit and still emits valid evidence", async () => {
+  it("observes EMFILE at the provider FD limit and still emits valid evidence", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const adapter = path.join(fixture.root, "provider-fd-limit.mjs");
@@ -1025,7 +1063,8 @@ describe("candidate archive and provider upload boundaries", () => {
     await expect(lstat(fixture.reportPath)).rejects.toMatchObject({ code: "ENOENT" });
   }, 10_000);
 
-  it("enforces CPU RLIMIT before the independent wall-clock timeout", async () => {
+  it("enforces CPU RLIMIT before the independent wall-clock timeout", async (ctx) => {
+    requireProviderSandbox(ctx);
     const fixture = await createProviderFixture();
     await rm(fixture.reportPath);
     const started = Date.now();
@@ -1042,9 +1081,11 @@ describe("candidate archive and provider upload boundaries", () => {
     await expect(lstat(fixture.reportPath)).rejects.toMatchObject({ code: "ENOENT" });
   }, 10_000);
 
-  it.each(["output", "timeout"] as const)(
+  it.for(["output", "timeout"] as const)(
     "kills descendants and collects the scope on provider %s termination",
-    async (reason) => {
+    { timeout: 10_000 },
+    async (reason, ctx) => {
+      requireProviderSandbox(ctx);
       const fixture = await createProviderFixture();
       await rm(fixture.reportPath);
       const providerScript = path.join(fixture.root, `provider-${reason}-descendant.mjs`);
@@ -1084,12 +1125,13 @@ describe("candidate archive and provider upload boundaries", () => {
       expect(cgroupPids.every((pid) => !processExists(pid))).toBe(true);
       await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]);
     },
-    10_000,
   );
 
-  it.each(["vulnerability", "provenance"] as const)(
+  it.for(["vulnerability", "provenance"] as const)(
     "runs the %s adapter offline and collects its unit",
-    async (kind) => {
+    { timeout: 10_000 },
+    async (kind, ctx) => {
+      requireProviderSandbox(ctx);
       const fixture = await createProviderFixture();
       const canary = await startLoopbackCanary(fixture.root);
       try {
@@ -1127,7 +1169,6 @@ describe("candidate archive and provider upload boundaries", () => {
         await waitForChildClose(canary.child);
       }
     },
-    10_000,
   );
 });
 

tests/unit/provider-sandbox-probe.test.ts

@@ -0,0 +1,95 @@
+import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  PROVIDER_SANDBOX_ISOLATION_ARGUMENTS,
+  PROVIDER_SANDBOX_UNAVAILABLE,
+  assertProviderSandboxUsable,
+  probeProviderSandbox,
+  providerSandboxDecision,
+  providerSandboxProbeArguments,
+} from "../../scripts/lib/provider-sandbox-probe.ts";
+
+const roots: string[] = [];
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
+});
+
+async function stubSandboxBinary(body: string): Promise<string> {
+  const root = await mkdtemp(path.join(tmpdir(), "sandbox-probe-"));
+  roots.push(root);
+  const binary = path.join(root, "bwrap");
+  await writeFile(binary, `#!/bin/sh\n${body}\n`);
+  await chmod(binary, 0o755);
+  return binary;
+}
+
+describe("provider sandbox probe", () => {
+  it("probes execution rather than the existence of the binary", async () => {
+    const refuses = await stubSandboxBinary(
+      'echo "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted" >&2\nexit 1',
+    );
+    const probe = probeProviderSandbox(refuses);
+    expect(probe.usable).toBe(false);
+    expect(probe.usable ? "" : probe.reason).toContain(
+      "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted",
+    );
+    expect(probeProviderSandbox(await stubSandboxBinary("exit 0"))).toEqual({ usable: true });
+  });
+
+  it("reports a missing sandbox binary as its own reason", async () => {
+    const missing = path.join(await mkdtemp(path.join(tmpdir(), "sandbox-absent-")), "bwrap");
+    roots.push(path.dirname(missing));
+    const probe = probeProviderSandbox(missing);
+    expect(probe.usable).toBe(false);
+    expect(probe.usable ? "" : probe.reason).toMatch(/is not installed/u);
+  });
+
+  it("probes with the same isolation options the real provider run applies", async () => {
+    const source = await readFile(
+      new URL("../../scripts/run-and-validate-provider.ts", import.meta.url),
+      "utf8",
+    );
+    // The run builds its bubblewrap vector from the shared constant and
+    // declares no isolation option of its own, so an option cannot be added to
+    // the run without the probe having to clear it as well.
+    expect(source).toContain("...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS");
+    expect(source.match(/"(?:--unshare-[a-z]+|--as-pid-1|--new-session|--die-with-parent)"/gu))
+      .toBeNull();
+    expect(PROVIDER_SANDBOX_ISOLATION_ARGUMENTS).toEqual(
+      expect.arrayContaining([
+        "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net", "--as-pid-1",
+      ]),
+    );
+    const probeArguments = providerSandboxProbeArguments();
+    expect(probeArguments.slice(0, PROVIDER_SANDBOX_ISOLATION_ARGUMENTS.length)).toEqual([
+      ...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS,
+    ]);
+    expect(probeArguments.at(-1)).toBe("/bin/true");
+  });
+
+  it("skips locally and fails in CI on the same unusable sandbox", () => {
+    const probe = { usable: false, reason: "bwrap: setting up uid map: Permission denied" } as const;
+    expect(providerSandboxDecision(probe, {})).toEqual({
+      outcome: "skip",
+      reason: `${PROVIDER_SANDBOX_UNAVAILABLE}: bwrap: setting up uid map: Permission denied`,
+    });
+    expect(providerSandboxDecision(probe, { CI: "true" })).toEqual({
+      outcome: "fail",
+      reason: `${PROVIDER_SANDBOX_UNAVAILABLE}: bwrap: setting up uid map: Permission denied`,
+    });
+    expect(providerSandboxDecision({ usable: true }, { CI: "true" })).toEqual({ outcome: "run" });
+  });
+
+  it("asserts only the CI failure outcome", () => {
+    const probe = { usable: false, reason: "denied" } as const;
+    expect(() => assertProviderSandboxUsable(providerSandboxDecision(probe, { CI: "true" })))
+      .toThrow(new RegExp(PROVIDER_SANDBOX_UNAVAILABLE, "u"));
+    expect(() => assertProviderSandboxUsable(providerSandboxDecision(probe, {}))).not.toThrow();
+    expect(() => assertProviderSandboxUsable(providerSandboxDecision({ usable: true }, { CI: "true" })))
+      .not.toThrow();
+  });
+});