From 77c01b03c1dd1623624af894fcacf73b7a15a899 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 12 Nov 2026 10:52:54 +0900 Subject: [PATCH] =?UTF-8?q?test(guardian):=20lease=EA=B0=80=20=EC=8B=9C?= =?UTF-8?q?=ED=97=98=20=EB=8C=80=EC=83=81=EC=9D=B4=20=EC=95=84=EB=8B=88?= =?UTF-8?q?=EB=A9=B4=20=EC=8B=9C=ED=97=98=20timeout=EB=A7=8C=ED=81=BC=20?= =?UTF-8?q?=EC=A4=80=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/integration/provider-guardian-transaction.test.ts가 부하 때 lease 만료로 실패한다. guardian이 스스로 SIGKILL한 것이다. desktop 기록 둘: - FE-GATE-007: 「still publishes near the lease deadline」 - FE-GATE-020 drill: 「owns raw and sealed publication through the authenticated client lease」 FE-B 실행에서도 load 28에 같은 실패가 2건 났다. 다시 돌리면 통과한다. 원인은 두 가지다. - startProviderGuardian은 guardian을 띄우자마자 절대 deadline (Date.now() + leaseMs)을 찍는다. 그래서 client 경로 시험의 2~3초 lease가 node 시작 시간까지 낸다. 부하 때 시작에는 상한이 없다. - 직접 프로토콜 시험은 guardian이 load를 마친 뒤 가드를 보낸다 (c8620042). 그래도 가드 → READY 처리는 실제 시계로 돈다. desktop에서 CPU 25%로 제한하면 이 구간이 17ms에서 334ms로 늘어나, 350ms lease로 만료를 보는 시험의 여유가 거의 없다. lease가 시험 대상인지로 나눠 고쳤다. - lease가 대상이 아닌 15곳(client 경로 10, 직접 경로 5)은 LEASE_NOT_UNDER_TEST_MS다. 값은 시험 timeout과 같아서, lease가 시험보다 먼저 끝날 수 없다. guardian이 멈추면 시험 timeout이 잡는다. - lease 만료를 보는 시험은 350ms → 2초다. 가드 → READY의 CPU 25% 최대 334ms의 여섯 배다. - near-deadline 시험은 client 경로에서 직접 프로토콜로 옮겼다. - deadline이 guardian load 뒤에 시작해서 시작 시간이 빠진다. - sendGuard가 보낸 deadline을 돌려준다. - 시험은 lease 4초의 마지막 2초 안에 publish·commit을 보내고, 그 구간에 있었는지 단언한다. - 2초는 publish·commit의 CPU 25% 최대 104ms의 약 스무 배다. - client의 publish·commit은 「owns raw and sealed publication」이 그대로 본다. 버린 것: - guardian에 가짜 시계 주입: 운영 스크립트에 시험용 훅을 넣어야 한다 (c8620042와 같은 이유). - client가 guardian load 뒤에 deadline을 찍게 바꾸기: 운영 프로토콜에 준비 신호를 더해야 한다. 이 PR 범위를 넘는다. - 모든 lease를 늘리기: 만료와 near-deadline 시험은 lease가 대상이라 늘리면 보는 것이 흐려진다. 위험: - 시험이 실패하면 guardian이 lease 끝까지(최대 30초) 남을 수 있다. 전에는 2~3초였다. - near-deadline 시험에는 실제 시계 구간이 남는다. publish·commit이 2초를 넘으면 실패한다. 잰 최대의 약 스무 배다. - 운영 client가 lease에 node 시작 시간을 포함하는 것은 그대로다. 운영 lease는 분 단위라 시험만큼 빠듯하지 않다. 검증(desktop): - 이 파일을 10회 돌려 22개 모두 통과했다(load 3~4). - CPU 20% 제한에서 main d2239c67은 5회 모두 1~6개 실패했다. 원인은 가드 처리 중 deadline 지남(126), READY·PUBLISHED 대기 시간 초과, EPIPE였다. 같은 조건에서 이 커밋은 5회 모두 통과했다. - 시험 timeout은 잰 최대로 고쳤다. near-deadline 3.2초, 만료 2.1초. - 첫 최종 실행에서 FE-GATE-007이 near-deadline 시험의 「마지막 2초 안」 단언으로 실패했다(남은 2,001ms). 타이머가 Date.now()보다 1ms 일찍 깼다. 목표 시각에 닿을 때까지 다시 기다리게 고쳤다. 고친 뒤 이 파일을 30회(load 3.6~21), CPU 20% 제한으로 5회 돌려 모두 통과했다. --- .../provider-guardian-transaction.test.ts | 152 ++++++++++++------ 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/tests/integration/provider-guardian-transaction.test.ts b/tests/integration/provider-guardian-transaction.test.ts index aad089f5..d7fb0f42 100644 --- a/tests/integration/provider-guardian-transaction.test.ts +++ b/tests/integration/provider-guardian-transaction.test.ts @@ -35,6 +35,26 @@ const PROCESS_HEAVY_TIMEOUT_MS = 30_000; const PROCESS_HANDSHAKE_WATCHDOG_MS = 5_000; /** The extra pipe on which spawnGuardian's wrapper reports the guardian loaded. */ const GUARDIAN_LOADED_FD = 7; +/** + * The lease of a test whose subject is not the lease: as long as the test's + * timeout, so the guardian's deadline cannot end a test first. A guard carries + * an absolute deadline. Under load a 2 to 3 s lease ran out before commit and + * the guardian killed itself (FE-GATE-007 and FE-GATE-020 drills on desktop, + * 2026-11-12, load average 28): startProviderGuardian stamps the deadline when + * it spawns the guardian, so its lease also paid for node start-up. + */ +const LEASE_NOT_UNDER_TEST_MS = PROCESS_HEAVY_TIMEOUT_MS; +/* + * The leases a test measures come from desktop runs of the direct-protocol + * publish test (2026-11-12, eight runs each, with and without a 25% CPU + * quota): guard to READY took at most 17 ms, 334 ms under the quota; publish + * and commit at most 34 ms, 104 ms under the quota. + */ +/** Has to outlast guard to READY before it expires: six times the 334 ms. */ +const EXPIRING_LEASE_MS = 2_000; +/** Publish and commit go out in the lease's last NEAR_DEADLINE_MARGIN_MS, twenty times the 104 ms. */ +const NEAR_DEADLINE_LEASE_MS = 4_000; +const NEAR_DEADLINE_MARGIN_MS = 2_000; const roots: string[] = []; @@ -218,7 +238,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 2_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian READY"), @@ -243,7 +263,7 @@ describe("provider guardian transaction protocol", () => { await expect(lstat(rawPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(lstat(sealedTempPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(readdir(path.join(workspace, "provider-evidence/untrusted"))).resolves.toEqual([]); - }, childProcessTimeout(253)); + }, childProcessTimeout(854)); it("creates no evidence for empty or truncated guard input", async () => { const { startProviderGuardian } = await import( @@ -271,7 +291,7 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); await retry.abort(); @@ -280,7 +300,7 @@ describe("provider guardian transaction protocol", () => { await expect(readdir(path.join(workspace, "provider-evidence"))) .resolves.toEqual(["untrusted"]); } - }, childProcessTimeout(1_226)); + }, childProcessTimeout(1_516)); it.each(["raw", "sealed"] as const)( "cleans the independently bound %s private alias when peer bootstrap validation fails", @@ -317,7 +337,7 @@ describe("provider guardian transaction protocol", () => { const nonce = Buffer.alloc(32, 0x2b); const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); - await sendGuard(child, nonce, 2_000, { closeInput: true }); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS, { closeInput: true }); await expect(within(completion, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian parent startup death")) .resolves.toEqual({ code: 125, signal: null }); @@ -334,7 +354,7 @@ describe("provider guardian transaction protocol", () => { const completion = waitForChild(child); try { // A different nonce than the inherited private aliases were named for. - await sendGuard(child, Buffer.alloc(32, 0x3e), 2_000); + await sendGuard(child, Buffer.alloc(32, 0x3e), LEASE_NOT_UNDER_TEST_MS); await expect(within(completion, PROCESS_HANDSHAKE_WATCHDOG_MS, "rejected-guard guardian exit")) .resolves.toEqual({ code: 126, signal: null }); } finally { @@ -359,7 +379,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 3_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian READY"), nonce, @@ -438,7 +458,7 @@ describe("provider guardian transaction protocol", () => { const lease = await start({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript, }); expect((await readFile(`/proc/${lease.pid}/cmdline`)).toString("utf8") @@ -479,13 +499,13 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 2_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); await retry.abort(); await expect(readdir(path.join(workspace, "provider-evidence/untrusted"))).resolves.toEqual([]); await expect(readdir(path.join(workspace, "provider-evidence"))).resolves.toEqual(["untrusted"]); - }, childProcessTimeout(429)); + }, childProcessTimeout(2_455)); it("fails closed when a separate trailing frame arrives after commit", async () => { const { encodeProviderGuardianCommit } = await import( @@ -513,7 +533,7 @@ describe("provider guardian transaction protocol", () => { const lease = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); @@ -539,7 +559,7 @@ describe("provider guardian transaction protocol", () => { const starting = startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript, }); const guardianPid = await waitForNewDirectChild(existingChildren); @@ -561,7 +581,7 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); expect(path.basename(retry.sealedTempPath)).toMatch( @@ -585,7 +605,7 @@ describe("provider guardian transaction protocol", () => { const starting = startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript, }); const guardianPid = Number(await waitForFile(markerPath)); @@ -612,7 +632,7 @@ describe("provider guardian transaction protocol", () => { const retry = await startProviderGuardian({ kind: "vulnerability", workspaceRoot: workspace, - leaseMs: 3_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }); await retry.abort(); @@ -630,7 +650,7 @@ describe("provider guardian transaction protocol", () => { const input = { kind: "vulnerability" as const, workspaceRoot: workspace, - leaseMs: 10_000, + leaseMs: LEASE_NOT_UNDER_TEST_MS, guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), }; @@ -670,7 +690,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 3_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "closed-stderr guardian READY"), nonce, @@ -685,7 +705,7 @@ describe("provider guardian transaction protocol", () => { .rejects.toMatchObject({ code: "ENOENT" }); await expect(lstat(path.join(evidenceRoot, ready.sealedTempLeaf))) .rejects.toMatchObject({ code: "ENOENT" }); - }, childProcessTimeout(469)); + }, childProcessTimeout(576)); it("expires an uncommitted lease only after cleaning every owned object", async () => { const { decodeProviderGuardianReady } = await import( @@ -698,7 +718,7 @@ describe("provider guardian transaction protocol", () => { const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 350); + await sendGuard(child, nonce, EXPIRING_LEASE_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "deadline guardian READY"), nonce, @@ -710,32 +730,70 @@ describe("provider guardian transaction protocol", () => { await expect(lstat(rawPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(lstat(sealedTempPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(readdir(evidenceRoot)).resolves.toEqual(["untrusted"]); - }, PROCESS_HEAVY_TIMEOUT_MS); + }, childProcessTimeout(2_114)); it("still publishes near the lease deadline when post-processing completes in time", async () => { - const { startProviderGuardian } = await import( - "../../scripts/lib/provider-guardian-client.ts" - ); + const { + decodeProviderGuardianPublished, + decodeProviderGuardianReady, + encodeProviderGuardianCommit, + encodeProviderGuardianPublish, + } = await import("../../scripts/lib/provider-guardian-protocol.ts"); const workspace = await createWorkspace("provider-guardian-near-timeout-"); - // The client stamps the deadline after it spawns the guardian, so the - // deadline is at least 2s from here. Waking 1s from here, not 1s after - // start-up returns, keeps publish and commit 1s on a slow host too. - const startedAt = Date.now(); - const lease = await startProviderGuardian({ - kind: "vulnerability", - workspaceRoot: workspace, - leaseMs: 2_000, - guardianScript: path.resolve("scripts/lib/provider-raw-guardian.ts"), - }); - await new Promise((resolve) => - setTimeout(resolve, Math.max(0, startedAt + 1_000 - Date.now()))); + const evidenceRoot = path.join(workspace, "provider-evidence"); + const rawPath = path.join(evidenceRoot, "untrusted/vulnerability-report.json"); + const sealedPath = path.join(evidenceRoot, "vulnerability-report.json"); + const nonce = Buffer.alloc(32, 0x5d); + const child = spawnGuardian(workspace, nonce); + const completion = waitForChild(child); + const readyPayload = readFrame(child.stdout!); + // The direct protocol starts the deadline once the guardian has loaded, + // so start-up on a loaded host does not use up the lease. Only publish + // and commit run on the real clock, inside NEAR_DEADLINE_MARGIN_MS. + const deadlineEpochMs = await sendGuard(child, nonce, NEAR_DEADLINE_LEASE_MS); + const ready = decodeProviderGuardianReady( + await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "near-deadline guardian READY"), + nonce, + ); const bytes = Buffer.from('{"validated":"near-timeout"}\n'); + const handle = await open( + path.join(evidenceRoot, ready.sealedTempLeaf), + constants.O_WRONLY | constants.O_NOFOLLOW, + ); + try { + await handle.writeFile(bytes); + await handle.chmod(0o400); + await handle.sync(); + } finally { + await handle.close(); + } + // A timer can fire a millisecond before Date.now() reaches its target, so + // wait again until it has. + const nearDeadlineEpochMs = deadlineEpochMs - NEAR_DEADLINE_MARGIN_MS; + while (Date.now() < nearDeadlineEpochMs) { + await new Promise((resolve) => setTimeout(resolve, nearDeadlineEpochMs - Date.now())); + } + expect(deadlineEpochMs - Date.now()).toBeLessThanOrEqual(NEAR_DEADLINE_MARGIN_MS); - await lease.publish(bytes); - await lease.commit(); - await expect(lstat(lease.rawPath)).rejects.toMatchObject({ code: "ENOENT" }); - await expect(readFile(lease.sealedPath)).resolves.toEqual(bytes); - }, childProcessTimeout(1_126)); + const publishedPayload = readFrame(child.stdout!); + child.stdin!.write(encodeProviderGuardianPublish({ + nonce, + sealedDev: ready.sealedDev, + sealedIno: ready.sealedIno, + size: bytes.byteLength, + sha256: createHash("sha256").update(bytes).digest("hex"), + })); + decodeProviderGuardianPublished( + await within(publishedPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "near-deadline guardian PUBLISHED"), + nonce, + { dev: ready.sealedDev, ino: ready.sealedIno }, + ); + child.stdin!.end(encodeProviderGuardianCommit(nonce)); + await expect(within(completion, PROCESS_HANDSHAKE_WATCHDOG_MS, "near-deadline guardian commit EOF")) + .resolves.toEqual({ code: 0, signal: null }); + await expect(lstat(rawPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readFile(sealedPath)).resolves.toEqual(bytes); + }, childProcessTimeout(3_234)); it("signals guardian exit only while the provider scope latch is active", async () => { const lifecycle = await import("../../scripts/lib/provider-guardian-client.ts") as @@ -875,7 +933,7 @@ async function establishPublishedGuardian( const child = spawnGuardian(workspace, nonce); const completion = waitForChild(child); const readyPayload = readFrame(child.stdout!); - await sendGuard(child, nonce, 3_000); + await sendGuard(child, nonce, LEASE_NOT_UNDER_TEST_MS); const ready = decodeProviderGuardianReady( await within(readyPayload, PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian READY"), nonce, @@ -1007,7 +1065,7 @@ function spawnGuardian( /** * A guard carries an absolute deadline, so it goes out only once the guardian - * reads stdin. Sent at spawn, the lease also pays for process start-up, which + * reads stdin. Returns that deadline. Sent at spawn, the lease also pays for process start-up, which * has no bound on a loaded host: a 350ms lease then expires before the * guardian decodes the guard. */ @@ -1016,7 +1074,7 @@ async function sendGuard( nonce: Buffer, leaseMs: number, options: Readonly<{ closeInput?: boolean }> = {}, -): Promise { +): Promise { const { encodeProviderGuardianGuard } = await import( "../../scripts/lib/provider-guardian-protocol.ts" ); @@ -1025,13 +1083,11 @@ async function sendGuard( loaded.once("data", () => resolve()); loaded.once("end", () => reject(new Error("guardian exited before it loaded"))); }), PROCESS_HANDSHAKE_WATCHDOG_MS, "guardian start-up"); - const guard = encodeProviderGuardianGuard({ - kind: "vulnerability", - nonce, - deadlineEpochMs: Date.now() + leaseMs, - }); + const deadlineEpochMs = Date.now() + leaseMs; + const guard = encodeProviderGuardianGuard({ kind: "vulnerability", nonce, deadlineEpochMs }); if (options.closeInput) child.stdin!.end(guard); else child.stdin!.write(guard); + return deadlineEpochMs; } function spawnGuardianWithPrivateModes(