test: probe the provider sandbox instead of failing on it

The 16 sandboxed provider tests in `tests/unit/ci-artifact-contract.test.ts` fail wherever unprivileged user namespaces are denied. bubblewrap is installed and answers `--version`, but `bwrap --unshare-net … -- /bin/true` exits 1 with `bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted`, and it fails the same way with no netns flag at all (`setting up uid map: Permission denied`), so this is the whole nested-userns capability and not one option. Sixteen assertion errors on every local run buried whatever else the file had to say. `scripts/lib/provider-sandbox-probe.ts` now runs a trivial command under the real isolation options — `runProviderInSandbox` spreads the same `PROVIDER_SANDBOX_ISOLATION_ARGUMENTS`, and a test fails if an isolation option is added to the run without the probe having to clear it. Capability is measured, never inferred from the binary existing or from a version string; both would pass here. An unusable sandbox means two different things in two places, so the decision is explicit. Locally it is an environment fact: the affected tests skip and carry the bwrap diagnostic as their skip note, visible as `↓ … [reason]`. In CI it is a regression — a security gate that silently stopped running is exactly what these tests exist to catch — so the same probe result fails the run through one guard test that says "the provider sandbox is unavailable" instead of sixteen assertion errors. CI is detected with `CI === "true"` via a new `isCiRun`, sharing the predicate that already gates `ciBuildEnvironmentFailures`. The workflow sets it at the top-level `env:` block, so it holds in every job; `CI_RUN_ID` and its `GITEA_`/`GITHUB_` fallbacks are declared only by the release-tier provider jobs and are absent from the merge gates that run this file, so keying on them would have left the CI branch permanently dead. The three `it.each` groups become `it.for` because only `.for` passes the test context, which is what carries the skip note. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-18 12:55:56 +09:00
parent 1a40522ff8
commit 74281b0277
5 changed files with 279 additions and 24 deletions
@@ -0,0 +1,95 @@
+import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  PROVIDER_SANDBOX_ISOLATION_ARGUMENTS,
+  PROVIDER_SANDBOX_UNAVAILABLE,
+  assertProviderSandboxUsable,
+  probeProviderSandbox,
+  providerSandboxDecision,
+  providerSandboxProbeArguments,
+} from "../../scripts/lib/provider-sandbox-probe.ts";
+
+const roots: string[] = [];
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
+});
+
+async function stubSandboxBinary(body: string): Promise<string> {
+  const root = await mkdtemp(path.join(tmpdir(), "sandbox-probe-"));
+  roots.push(root);
+  const binary = path.join(root, "bwrap");
+  await writeFile(binary, `#!/bin/sh\n${body}\n`);
+  await chmod(binary, 0o755);
+  return binary;
+}
+
+describe("provider sandbox probe", () => {
+  it("probes execution rather than the existence of the binary", async () => {
+    const refuses = await stubSandboxBinary(
+      'echo "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted" >&2\nexit 1',
+    );
+    const probe = probeProviderSandbox(refuses);
+    expect(probe.usable).toBe(false);
+    expect(probe.usable ? "" : probe.reason).toContain(
+      "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted",
+    );
+    expect(probeProviderSandbox(await stubSandboxBinary("exit 0"))).toEqual({ usable: true });
+  });
+
+  it("reports a missing sandbox binary as its own reason", async () => {
+    const missing = path.join(await mkdtemp(path.join(tmpdir(), "sandbox-absent-")), "bwrap");
+    roots.push(path.dirname(missing));
+    const probe = probeProviderSandbox(missing);
+    expect(probe.usable).toBe(false);
+    expect(probe.usable ? "" : probe.reason).toMatch(/is not installed/u);
+  });
+
+  it("probes with the same isolation options the real provider run applies", async () => {
+    const source = await readFile(
+      new URL("../../scripts/run-and-validate-provider.ts", import.meta.url),
+      "utf8",
+    );
+    // The run builds its bubblewrap vector from the shared constant and
+    // declares no isolation option of its own, so an option cannot be added to
+    // the run without the probe having to clear it as well.
+    expect(source).toContain("...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS");
+    expect(source.match(/"(?:--unshare-[a-z]+|--as-pid-1|--new-session|--die-with-parent)"/gu))
+      .toBeNull();
+    expect(PROVIDER_SANDBOX_ISOLATION_ARGUMENTS).toEqual(
+      expect.arrayContaining([
+        "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net", "--as-pid-1",
+      ]),
+    );
+    const probeArguments = providerSandboxProbeArguments();
+    expect(probeArguments.slice(0, PROVIDER_SANDBOX_ISOLATION_ARGUMENTS.length)).toEqual([
+      ...PROVIDER_SANDBOX_ISOLATION_ARGUMENTS,
+    ]);
+    expect(probeArguments.at(-1)).toBe("/bin/true");
+  });
+
+  it("skips locally and fails in CI on the same unusable sandbox", () => {
+    const probe = { usable: false, reason: "bwrap: setting up uid map: Permission denied" } as const;
+    expect(providerSandboxDecision(probe, {})).toEqual({
+      outcome: "skip",
+      reason: `${PROVIDER_SANDBOX_UNAVAILABLE}: bwrap: setting up uid map: Permission denied`,
+    });
+    expect(providerSandboxDecision(probe, { CI: "true" })).toEqual({
+      outcome: "fail",
+      reason: `${PROVIDER_SANDBOX_UNAVAILABLE}: bwrap: setting up uid map: Permission denied`,
+    });
+    expect(providerSandboxDecision({ usable: true }, { CI: "true" })).toEqual({ outcome: "run" });
+  });
+
+  it("asserts only the CI failure outcome", () => {
+    const probe = { usable: false, reason: "denied" } as const;
+    expect(() => assertProviderSandboxUsable(providerSandboxDecision(probe, { CI: "true" })))
+      .toThrow(new RegExp(PROVIDER_SANDBOX_UNAVAILABLE, "u"));
+    expect(() => assertProviderSandboxUsable(providerSandboxDecision(probe, {}))).not.toThrow();
+    expect(() => assertProviderSandboxUsable(providerSandboxDecision({ usable: true }, { CI: "true" })))
+      .not.toThrow();
+  });
+});