donghyeon.kang/tech-log-frontend
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

test: probe the provider sandbox instead of failing on it

Browse Source 
The 16 sandboxed provider tests in `tests/unit/ci-artifact-contract.test.ts`
fail wherever unprivileged user namespaces are denied. bubblewrap is installed
and answers `--version`, but `bwrap --unshare-net … -- /bin/true` exits 1 with
`bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted`, and it fails the
same way with no netns flag at all (`setting up uid map: Permission denied`), so
this is the whole nested-userns capability and not one option. Sixteen
assertion errors on every local run buried whatever else the file had to say.

`scripts/lib/provider-sandbox-probe.ts` now runs a trivial command under the
real isolation options — `runProviderInSandbox` spreads the same
`PROVIDER_SANDBOX_ISOLATION_ARGUMENTS`, and a test fails if an isolation option
is added to the run without the probe having to clear it. Capability is
measured, never inferred from the binary existing or from a version string;
both would pass here.

An unusable sandbox means two different things in two places, so the decision
is explicit. Locally it is an environment fact: the affected tests skip and
carry the bwrap diagnostic as their skip note, visible as `↓ … [reason]`. In CI
it is a regression — a security gate that silently stopped running is exactly
what these tests exist to catch — so the same probe result fails the run
through one guard test that says "the provider sandbox is unavailable" instead
of sixteen assertion errors.

CI is detected with `CI === "true"` via a new `isCiRun`, sharing the predicate
that already gates `ciBuildEnvironmentFailures`. The workflow sets it at the
top-level `env:` block, so it holds in every job; `CI_RUN_ID` and its
`GITEA_`/`GITHUB_` fallbacks are declared only by the release-tier provider
jobs and are absent from the merge gates that run this file, so keying on them
would have left the CI branch permanently dead.

The three `it.each` groups become `it.for` because only `.for` passes the test
context, which is what carries the skip note.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-08-18 12:55:56 +09:00
co-authored by Claude Opus 5
parent 1a40522ff8
commit 74281b0277
5 changed files with 279 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/ci-artifact-contract.test.ts
+60 -19
View File
@@ -6,7 +6,7 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { afterAll, afterEach, describe, expect, it } from "vitest";
import { afterAll, afterEach, describe, expect, it, type TestContext } from "vitest";
import type {
CiGateArtifact,
@@ -40,6 +40,11 @@ import {
} from "../../scripts/lib/provider-evidence.ts";
import { localEvidenceAssessmentArtifactSchema } from "../../scripts/contracts/release-artifacts.ts";
import { readProviderTrust } from "../../scripts/lib/provider-trust.ts";
import {
assertProviderSandboxUsable,
probeProviderSandbox,
providerSandboxDecision,
} from "../../scripts/lib/provider-sandbox-probe.ts";
import {
createReleaseCandidateManifest,
LOCAL_EVIDENCE_ASSESSMENT_PATH,
@@ -55,6 +60,26 @@ import {
*/
const PROCESS_HEAVY_TIMEOUT_MS = 30_000;
/**
* Every test that reaches `runProviderSupervisor` needs bubblewrap to actually
* create the namespaces `PROVIDER_SANDBOX_ISOLATION_ARGUMENTS` asks for. Where
* unprivileged user namespaces are denied — a host with
* `kernel.apparmor_restrict_unprivileged_userns=1`, a container without
* `CAP_SYS_ADMIN` — bwrap is installed and answers `--version` but still cannot
* run, and those tests fail on the environment rather than on the product.
*
* The probe runs once per file and decides between two very different
* responses: locally the affected tests are skipped with the bwrap diagnostic
* attached, in CI the same result fails the run through the guard test below.
* Skipping in CI too would quietly disarm the sandbox gate, which is exactly
* the regression these tests exist to catch.
*/
const providerSandbox = providerSandboxDecision(probeProviderSandbox(), process.env);
function requireProviderSandbox(ctx: TestContext): void {
if (providerSandbox.outcome !== "run") ctx.skip(providerSandbox.reason);
}
const temporaryRoots: string[] = [];
let providerBaseRoot: string | undefined;
const sha256 = (value: Buffer | string) =>
@@ -441,6 +466,10 @@ describe("CI artifact validator", () => {
});
describe("candidate archive and provider upload boundaries", () => {
it("fails the run in CI when the provider sandbox is unavailable", () => {
expect(() => assertProviderSandboxUsable(providerSandbox)).not.toThrow();
});
it("accepts only the manifest-bound candidate member set and bytes", async () => {
const fixture = await createCandidateArchiveFixture();
await expect(
@@ -652,9 +681,11 @@ describe("candidate archive and provider upload boundaries", () => {
await expect(readFile(markerPath)).rejects.toMatchObject({ code: "ENOENT" });
});
it.each(["invalid trust", "invalid archive", "provider failure"] as const)(
it.for(["invalid trust", "invalid archive", "provider failure"] as const)(
"leaves no stale raw report after %s and permits a clean retry",
async (failure) => {
{ timeout: 20_000 },
async (failure, ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
const archiveBytes = await readFile(fixture.archivePath);
const publicKeyBytes = await readFile(fixture.publicKeyPath);
@@ -678,10 +709,10 @@ describe("candidate archive and provider upload boundaries", () => {
});
expect(retried.status, retried.stderr).toBe(0);
},
20_000,
);
it("drains and kills provider background processes before sealing evidence", async () => {
it("drains and kills provider background processes before sealing evidence", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const sealedPath = path.join(
@@ -722,7 +753,8 @@ describe("candidate archive and provider upload boundaries", () => {
);
}, 10_000);
it("does not expose or mutate a host path outside the sandboxed workspace", async () => {
it("does not expose or mutate a host path outside the sandboxed workspace", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const outside = await temporaryRoot("provider-host-canary-");
@@ -743,7 +775,8 @@ describe("candidate archive and provider upload boundaries", () => {
await expect(readFile(canary, "utf8")).resolves.toBe("host-secret\n");
}, 10_000);
it("applies effective aggregate cgroup limits without exposing command or credentials", async () => {
it("applies effective aggregate cgroup limits without exposing command or credentials", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const credential = "live-provider-credential-sentinel";
@@ -830,7 +863,8 @@ describe("candidate archive and provider upload boundaries", () => {
await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]);
}, PROCESS_HEAVY_TIMEOUT_MS);
it("kills and collects an active provider when its guardian dies", async () => {
it("kills and collects an active provider when its guardian dies", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const sealedPath = path.join(fixture.root, "provider-evidence/vulnerability-report.json");
@@ -873,7 +907,8 @@ describe("candidate archive and provider upload boundaries", () => {
expect(retried.status, retried.stderr).toBe(0);
}, 20_000);
it("collects the whole provider scope when its supervisor dies", async () => {
it("collects the whole provider scope when its supervisor dies", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const providerScript = path.join(fixture.root, "provider-parent-death.mjs");
@@ -913,7 +948,8 @@ describe("candidate archive and provider upload boundaries", () => {
await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]);
}, 10_000);
it("fails closed when the guardian dies after scope collection and before commit", async () => {
it("fails closed when the guardian dies after scope collection and before commit", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
await writeFile(fixture.providerWriter, providerV2WriterSource());
@@ -958,7 +994,8 @@ describe("candidate archive and provider upload boundaries", () => {
}
}, 10_000);
it("cleans published evidence when the supervisor dies and permits same-workspace retry", async () => {
it("cleans published evidence when the supervisor dies and permits same-workspace retry", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
await writeFile(fixture.providerWriter, providerV2WriterSource());
@@ -1004,7 +1041,8 @@ describe("candidate archive and provider upload boundaries", () => {
}
}, 15_000);
it("observes EMFILE at the provider FD limit and still emits valid evidence", async () => {
it("observes EMFILE at the provider FD limit and still emits valid evidence", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const adapter = path.join(fixture.root, "provider-fd-limit.mjs");
@@ -1025,7 +1063,8 @@ describe("candidate archive and provider upload boundaries", () => {
await expect(lstat(fixture.reportPath)).rejects.toMatchObject({ code: "ENOENT" });
}, 10_000);
it("enforces CPU RLIMIT before the independent wall-clock timeout", async () => {
it("enforces CPU RLIMIT before the independent wall-clock timeout", async (ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const started = Date.now();
@@ -1042,9 +1081,11 @@ describe("candidate archive and provider upload boundaries", () => {
await expect(lstat(fixture.reportPath)).rejects.toMatchObject({ code: "ENOENT" });
}, 10_000);
it.each(["output", "timeout"] as const)(
it.for(["output", "timeout"] as const)(
"kills descendants and collects the scope on provider %s termination",
async (reason) => {
{ timeout: 10_000 },
async (reason, ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
await rm(fixture.reportPath);
const providerScript = path.join(fixture.root, `provider-${reason}-descendant.mjs`);
@@ -1084,12 +1125,13 @@ describe("candidate archive and provider upload boundaries", () => {
expect(cgroupPids.every((pid) => !processExists(pid))).toBe(true);
await expect(findProviderRawReferences(fixture.reportPath, reportIdentity)).resolves.toEqual([]);
},
10_000,
);
it.each(["vulnerability", "provenance"] as const)(
it.for(["vulnerability", "provenance"] as const)(
"runs the %s adapter offline and collects its unit",
async (kind) => {
{ timeout: 10_000 },
async (kind, ctx) => {
requireProviderSandbox(ctx);
const fixture = await createProviderFixture();
const canary = await startLoopbackCanary(fixture.root);
try {
@@ -1127,7 +1169,6 @@ describe("candidate archive and provider upload boundaries", () => {
await waitForChildClose(canary.child);
}
},
10_000,
);
});