diff --git a/tests/integration/provider-guardian-transaction.test.ts b/tests/integration/provider-guardian-transaction.test.ts index aad089f5..d7fb0f42 100644 --- a/tests/integration/provider-guardian-transaction.test.ts +++ b/tests/integration/provider-guardian-transaction.test.ts @@ -35,6 +35,26 @@ const PROCESS_HEAVY_TIMEOUT_MS = 30_000; const PROCESS_HANDSHAKE_WATCHDOG_MS = 5_000; /** The extra pipe on which spawnGuardian's wrapper reports the guardian loaded. */ const GUARDIAN_LOADED_FD = 7; +/** + * The lease of a test whose subject is not the lease: as long as the test's + * timeout, so the guardian's deadline cannot end a test first. A guard carries + * an absolute deadline. Under load a 2 to 3 s lease ran out before commit and + * the guardian killed itself (FE-GATE-007 and FE-GATE-020 drills on desktop, + * 2026-11-12, load average 28): startProviderGuardian stamps the deadline when + * it spawns the guardian, so its lease also paid for node start-up. + */ +const LEASE_NOT_UNDER_TEST_MS = PROCESS_HEAVY_TIMEOUT_MS; +/* + * The leases a test measures come from desktop runs of the direct-protocol + * publish test (2026-11-12, eight runs each, with and without a 25% CPU + * quota): guard to READY took at most 17 ms, 334 ms under the quota; publish + * and commit at most 34 ms, 104 ms under the quota. + */ +/** Has to outlast guard to READY before it expires: six times the 334 ms. */ +const EXPIRING_LEASE_MS = 2_000; +/** Publish and commit go out in the lease's last NEAR_DEADLINE_MARGIN_MS, twenty times the 104 ms. */ +const NEAR_DEADLINE_LEASE_MS = 4_000; +const NEAR_DEADLINE_MARGIN_MS = 2_000; const roots: string[] = []; @@ -218,7 +238,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 2_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian READY"), @@ -243,7 +263,7 @@ describe("provider guardian transaction protocol", () => { await expect(lstat(rawPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(lstat(sealedTempPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(readdir(path.join(workspace, "provider-evidence/untrusted"))).resolves.toEqual([]); - }, childProcessTimeout(253)); + }, childProcessTimeout(854)); it("creates no evidence for empty or truncated guard input", async () => { const { startProviderGuardian } = await import( @@ -271,7 +291,7 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); await retry.abort(); @@ -280,7 +300,7 @@ describe("provider guardian transaction protocol", () => { await expect(readdir(path.join(workspace, "provider-evidence"))) .resolves.toEqual(["untrusted"]); } - }, childProcessTimeout(1_226)); + }, childProcessTimeout(1_516)); it.each(["raw", "sealed"] as const)( "cleans the independently bound %s private alias when peer bootstrap validation fails", @@ -317,7 +337,7 @@ describe("provider guardian transaction protocol", () => { const nonce = Buffer.alloc(32, 0x2b); const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); - await sendGuard(child, nonce, 2_000, { closeInput: true }); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS, { closeInput: true }); await expect(within(completion, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian parent startup death")) .resolves.toEqual({ code: 125, signal: null }); @@ -334,7 +354,7 @@ describe("provider guardian transaction protocol", () => { const completion = waitForChild(child); try { // A different nonce than the inherited private aliases were named for. - await sendGuard(child, Buffer.alloc(32, 0x3e), 2_000); + await sendGuard(child, Buffer.alloc(32, 0x3e), LEASE_NOT_UNDER_TEST_MS); await expect(within(completion, PROCESS_HANDSHAKE_WATCHDOG_MS, "rejected-guard guardian exit")) .resolves.toEqual({ code: 126, signal: null }); } finally { @@ -359,7 +379,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 3_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian READY"), nonce, @@ -438,7 +458,7 @@ describe("provider guardian transaction protocol", () => { const lease = await start({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript, }); expect((await readFile(`/proc/${lease.pid}/cmdline`)).toString("utf8") @@ -479,13 +499,13 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 2_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); await retry.abort(); await expect(readdir(path.join(workspace, "provider-evidence/untrusted"))).resolves.toEqual([]); await expect(readdir(path.join(workspace, "provider-evidence"))).resolves.toEqual(["untrusted"]); - }, childProcessTimeout(429)); + }, childProcessTimeout(2_455)); it("fails closed when a separate trailing frame arrives after commit", async () => { const { encodeProviderGuardianCommit } = await import( @@ -513,7 +533,7 @@ describe("provider guardian transaction protocol", () => { const lease = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); @@ -539,7 +559,7 @@ describe("provider guardian transaction protocol", () => { const starting = startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript, }); const guardianPid = await waitForNewDirectChild(existingChildren); @@ -561,7 +581,7 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); expect(path.basename(retry.sealedTempPath)).toMatch( @@ -585,7 +605,7 @@ describe("provider guardian transaction protocol", () => { const starting = startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript, }); const guardianPid = Number(await waitForFile(markerPath)); @@ -612,7 +632,7 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); await retry.abort(); @@ -630,7 +650,7 @@ describe("provider guardian transaction protocol", () => { const input = { kind: "vulnerability" as const, workspaceRoot: workspace, - leaseMs: 10_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }; @@ -670,7 +690,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 3_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "closed-stderr guardian READY"), nonce, @@ -685,7 +705,7 @@ describe("provider guardian transaction protocol", () => { .rejects.toMatchObject({ code: "ENOENT" }); await expect(lstat(path.join(evidenceRoot, ready.sealedTempLeaf))) .rejects.toMatchObject({ code: "ENOENT" }); - }, childProcessTimeout(469)); + }, childProcessTimeout(576)); it("expires an uncommitted lease only after cleaning every owned object", async () => { const { decodeProviderGuardianReady } = await import( @@ -698,7 +718,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 350); + await sendGuard(child, nonce, EXPIRING_LEASE_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "deadline guardian READY"), nonce, @@ -710,32 +730,70 @@ describe("provider guardian transaction protocol", () => { await expect(lstat(rawPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(lstat(sealedTempPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(readdir(evidenceRoot)).resolves.toEqual(["untrusted"]); - }, PROCESS_HEAVY_TIMEOUT_MS); + }, childProcessTimeout(2_114)); it("still publishes near the lease deadline when post-processing completes in time", async () => { - const { startProviderGuardian } = await import( - "../../scripts/lib/provider-guardian-client.ts" - ); + const { + decodeProviderGuardianPublished, + decodeProviderGuardianReady, + encodeProviderGuardianCommit, + encodeProviderGuardianPublish, + } = await import("../../scripts/lib/provider-guardian-protocol.ts"); const workspace = await createWorkspace("provider-guardian-near-timeout-"); - // The client stamps the deadline after it spawns the guardian, so the - // deadline is at least 2s from here. Waking 1s from here, not 1s after - // start-up returns, keeps publish and commit 1s on a slow host too. - const startedAt = Date.now(); - const lease = await startProviderGuardian({ - kind: "vulnerability", - workspaceRoot: workspace, - leaseMs: 2_000, - guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), - }); - await new Promise((resolve) => - setTimeout(resolve, Math.max(0, startedAt + 1_000 - Date.now()))); + const evidenceRoot = path.join(workspace, "provider-evidence"); + const rawPath = path.join(evidenceRoot, "untrusted/vulnerability-report.json"); + const sealedPath = path.join(evidenceRoot, "vulnerability-report.json"); + const nonce = Buffer.alloc(32, 0x5d); + const child = spawnGuardian(workspace, nonce); + const completion = waitForChild(child); + const readyPayload = readFrame(child.stdout!); + // The direct protocol starts the deadline once the guardian has loaded, + // so start-up on a loaded host does not use up the lease. Only publish + // and commit run on the real clock, inside NEAR_DEADLINE_MARGIN_MS. + const deadlineEpochMs = await sendGuard(child, nonce, NEAR_DEADLINE_LEASE_MS); + const ready = decodeProviderGuardianReady( + await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "near-deadline guardian READY"), + nonce, + ); const bytes = Buffer.from('{"validated":"near-timeout"}\n'); + const handle = await open( + path.join(evidenceRoot, ready.sealedTempLeaf), + constants.O_WRONLY | constants.O_NOFOLLOW, + ); + try { + await handle.writeFile(bytes); + await handle.chmod(0o400); + await handle.sync(); + } finally { + await handle.close(); + } + // A timer can fire a millisecond before Date.now() reaches its target, so + // wait again until it has. + const nearDeadlineEpochMs = deadlineEpochMs - NEAR_DEADLINE_MARGIN_MS; + while (Date.now() < nearDeadlineEpochMs) { + await new Promise((resolve) => setTimeout(resolve, nearDeadlineEpochMs - Date.now())); + } + expect(deadlineEpochMs - Date.now()).toBeLessThanOrEqual(NEAR_DEADLINE_MARGIN_MS); - await lease.publish(bytes); - await lease.commit(); - await expect(lstat(lease.rawPath)).rejects.toMatchObject({ code: "ENOENT" }); - await expect(readFile(lease.sealedPath)).resolves.toEqual(bytes); - }, childProcessTimeout(1_126)); + const publishedPayload = readFrame(child.stdout!); + child.stdin!.write(encodeProviderGuardianPublish({ + nonce, + sealedDev: ready.sealedDev, + sealedIno: ready.sealedIno, + size: bytes.byteLength, + sha256: createHash("sha256").update(bytes).digest("hex"), + })); + decodeProviderGuardianPublished( + await within(publishedPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "near-deadline guardian PUBLISHED"), + nonce, + { dev: ready.sealedDev, ino: ready.sealedIno }, + ); + child.stdin!.end(encodeProviderGuardianCommit(nonce)); + await expect(within(completion, PROCESS_HANDSHAKE_WATCHDOG_MS, "near-deadline guardian commit EOF")) + .resolves.toEqual({ code: 0, signal: null }); + await expect(lstat(rawPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readFile(sealedPath)).resolves.toEqual(bytes); + }, childProcessTimeout(3_234)); it("signals guardian exit only while the provider scope latch is active", async () => { const lifecycle = await import("../../scripts/lib/provider-guardian-client.ts") as @@ -875,7 +933,7 @@ async function establishPublishedGuardian( const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 3_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian READY"), nonce, @@ -1007,7 +1065,7 @@ function spawnGuardian( /** * A guard carries an absolute deadline, so it goes out only once the guardian - * reads stdin. Sent at spawn, the lease also pays for process start-up, which + * reads stdin. Returns that deadline. Sent at spawn, the lease also pays for process start-up, which * has no bound on a loaded host: a 350ms lease then expires before the * guardian decodes the guard. */ @@ -1016,7 +1074,7 @@ async function sendGuard( nonce: Buffer, leaseMs: number, options: Readonly<{ closeInput?: boolean }> = {}, -): Promise { +): Promise { const { encodeProviderGuardianGuard } = await import( "../../scripts/lib/provider-guardian-protocol.ts" ); @@ -1025,13 +1083,11 @@ async function sendGuard( loaded.once("data", () => resolve()); loaded.once("end", () => reject(new Error("guardian exited before it loaded"))); }), PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian start-up"); - const guard = encodeProviderGuardianGuard({ - kind: "vulnerability", - nonce, - deadlineEpochMs: Date.now() + leaseMs, - }); + const deadlineEpochMs = Date.now() + leaseMs; + const guard = encodeProviderGuardianGuard({ kind: "vulnerability", nonce, deadlineEpochMs }); if (options.closeInput) child.stdin!.end(guard); else child.stdin!.write(guard); + return deadlineEpochMs; } function spawnGuardianWithPrivateModes(