From 2b24f6d8f0722087fe452d53bb2474d59a250a98 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Fri, 25 Sep 2026 22:41:31 +0900 Subject: [PATCH] fix(storage): fail a put whose source stalls so the OPFS origin lock is freed (05-07) A put holds the origin-wide mutation lock from BEGIN to FINISH, and the page waited on its source with no bound, so one stalled source blocked every tab's writes until their lock wait failed. Each wait for source bytes is now bounded by putIdleTimeoutMs; the put fails NOT_READABLE (retryable) and its compensation releases the lock. The port now says writes are one at a time. Co-Authored-By: Claude Opus 5.5 --- .../browser-file-storage-recovery.md | 5 +++ src/adapters/storage/opfs/opfs-policy.ts | 11 +++++ .../storage/opfs/opfs-worker-client.ts | 42 ++++++++++++----- .../ports/browser-file-storage/opfs-ports.ts | 6 +++ tests/unit/opfs-worker-runtime.test.ts | 45 +++++++++++++++++++ 5 files changed, 99 insertions(+), 10 deletions(-) diff --git a/docs/operations/browser-file-storage-recovery.md b/docs/operations/browser-file-storage-recovery.md index bb35dda3..36367cff 100644 --- a/docs/operations/browser-file-storage-recovery.md +++ b/docs/operations/browser-file-storage-recovery.md @@ -366,6 +366,11 @@ VD-15 목표 runtime을 먼저 구현한다. - committed logical row와 physical manifest/file 불일치 - digest/length mismatch - worker crash, handle lock timeout, `NoModificationAllowedError` +- put의 retryable `NOT_READABLE`: source가 `putIdleTimeoutMs`(기본 2분) 동안 다음 + bytes를 주지 않아 put을 끝내고 origin mutation lock을 풀었다(05-07, + `opfs-worker-client.ts` `preparePut`, `opfs-policy.ts`). OPFS 쓰기는 origin + 전체에서 한 번에 하나라서, 이 bound가 없으면 멈춘 source 하나가 모든 tab의 쓰기를 + `mutationLockWaitTimeoutMs` 뒤 `BLOCKED`로 만든다 - sensitive policy maintenance의 `POLICY_REJECTED`, proof replay/expiry 또는 authority provider/consumer unavailable diff --git a/src/adapters/storage/opfs/opfs-policy.ts b/src/adapters/storage/opfs/opfs-policy.ts index 5f0a9954..d5dd8a95 100644 --- a/src/adapters/storage/opfs/opfs-policy.ts +++ b/src/adapters/storage/opfs/opfs-policy.ts @@ -20,6 +20,13 @@ export type OpfsRuntimePolicy = Readonly<{ * is bounded on its own and never counted against `rpcTimeoutMs`. */ mutationLockWaitTimeoutMs: number; + /** + * 05-07. How long a put may wait for its source's next bytes. The put holds + * the origin mutation lock from BEGIN to FINISH, so a stalled source would + * otherwise stop every tab's writes until their lock wait ran out. Keep it + * below `mutationLockWaitTimeoutMs` so a queued writer outlasts it. + */ + putIdleTimeoutMs: number; reconciliationBudgetMs: number; reconciliationBatchSize: number; orphanGracePeriodMs: number; @@ -56,6 +63,7 @@ export const DEFAULT_OPFS_RUNTIME_POLICY: OpfsRuntimePolicy = Object.freeze({ maxChunkCount: 512, rpcTimeoutMs: 60_000, mutationLockWaitTimeoutMs: 10 * 60_000, + putIdleTimeoutMs: 2 * 60_000, reconciliationBudgetMs: 5_000, reconciliationBatchSize: 100, orphanGracePeriodMs: 24 * 60 * 60 * 1_000, @@ -94,6 +102,9 @@ export function assertOpfsRuntimePolicy(policy: OpfsRuntimePolicy): void { !Number.isSafeInteger(policy.mutationLockWaitTimeoutMs) || policy.mutationLockWaitTimeoutMs < 1_000 || policy.mutationLockWaitTimeoutMs > 60 * 60_000 || + !Number.isSafeInteger(policy.putIdleTimeoutMs) || + policy.putIdleTimeoutMs < 1_000 || + policy.putIdleTimeoutMs > 60 * 60_000 || !Number.isSafeInteger(policy.reconciliationBudgetMs) || policy.reconciliationBudgetMs < 1 || policy.reconciliationBudgetMs > 60_000 || diff --git a/src/adapters/storage/opfs/opfs-worker-client.ts b/src/adapters/storage/opfs/opfs-worker-client.ts index 9707ce79..2dffd10a 100644 --- a/src/adapters/storage/opfs/opfs-worker-client.ts +++ b/src/adapters/storage/opfs/opfs-worker-client.ts @@ -14,6 +14,7 @@ import { browserDataFailure, browserDataSuccess, } from "../../browser-file-storage/result.ts"; +import { createAbortableOperation } from "../../platform/abortable-operation.ts"; import type { OpfsRuntimePolicy } from "./opfs-policy.ts"; import { isPreparedObject } from "./opfs-prepared-object.ts"; import { @@ -272,13 +273,36 @@ export function createOpfsWorkerGateway( let sequence = 0; let transferredBytes = 0; notifyProgress(request, "TRANSFERRING", 0); + // 05-07. BEGIN took the origin mutation lock, so every wait for the + // source is bounded by `putIdleTimeoutMs`. A source that stops fails the + // put, and the coordinator's compensation releases the lock. + const sourceStop = new AbortController(); + const stopSource = () => sourceStop.abort(); + request.signal?.addEventListener("abort", stopSource, { once: true }); + if (request.signal?.aborted) stopSource(); + const chunks = rechunk( + request.source.stream(sourceStop.signal), + dependencies.policy.chunkSizeBytes, + dependencies.policy.maxObjectBytes, + request.signal, + ); try { - for await (const chunk of rechunk( - request.source.stream(requiredSignal(request.signal)), - dependencies.policy.chunkSizeBytes, - dependencies.policy.maxObjectBytes, - request.signal, - )) { + for (;;) { + const idle = createAbortableOperation({ + signal: request.signal, + timeoutMs: dependencies.policy.putIdleTimeoutMs, + }); + const next = await idle.race(chunks.next()); + idle.close(); + if (next.kind === "REJECTED") throw next.reason; + if (next.kind === "TERMINAL") { + stopSource(); + throw new OpfsRpcError( + next.terminal === "DEADLINE" ? "NOT_READABLE" : "ABORTED", + ); + } + if (next.value.done) break; + const chunk = next.value.value; const chunkByteLength = chunk.byteLength; const append = await invoke( "OBJECT_WRITE", @@ -320,6 +344,8 @@ export function createOpfsWorkerGateway( error instanceof OpfsRpcError ? error.code : "NOT_READABLE", "OBJECT_WRITE", ); + } finally { + request.signal?.removeEventListener("abort", stopSource); } }, @@ -585,10 +611,6 @@ async function* rechunk( } } -function requiredSignal(signal: AbortSignal | undefined): AbortSignal { - return signal ?? new AbortController().signal; -} - function notifyProgress( request: PreparePhysicalObjectRequest, phase: "TRANSFERRING" | "VERIFYING", diff --git a/src/application/ports/browser-file-storage/opfs-ports.ts b/src/application/ports/browser-file-storage/opfs-ports.ts index fdc2d3d1..227b314d 100644 --- a/src/application/ports/browser-file-storage/opfs-ports.ts +++ b/src/application/ports/browser-file-storage/opfs-ports.ts @@ -76,6 +76,12 @@ export type OpfsCapabilities = Readonly<{ export interface DurableObjectStorePort { capabilities(): Promise>; + /** + * 05-07. OPFS writes are one at a time across the whole origin: a put holds + * the origin mutation lock while it streams, and every put, remove and GC in + * every tab waits for it. A source that stops for the runtime's put idle + * bound fails the put with a retryable `NOT_READABLE` and frees the lock. + */ put( request: PutDurableObjectRequest, ): Promise>; diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 3c3a6a52..dbac0184 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -2398,3 +2398,48 @@ describe("05-04 a LOGOUT or ACCOUNT_DELETION purge leaves no partition bytes beh }); }); }); + +describe("05-07 a put whose source stops lets go of the origin lock", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("fails the stalled put at the idle bound so another tab's put runs", async () => { + const policy = resolveOpfsRuntimePolicy({ + ...runtimePolicy, + putIdleTimeoutMs: 30_000, + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const { tab, leases } = sharedRootTabs(policy); + const tabA = tab("a"); + const tabB = tab("b"); + const stalled = stalledSource(); + const putA = tabA.objects.put({ + objectId: "object_stalled_01", + expectedGeneration: null, + mediaType: "application/octet-stream", + source: stalled.source, + }); + await stalled.streaming; + expect(leases.held).toBe(true); + const putB = tabB.objects.put({ + objectId: "object_waiting_01", + expectedGeneration: null, + mediaType: "application/octet-stream", + source: oneByteSource(), + }); + + await vi.advanceTimersByTimeAsync(policy.putIdleTimeoutMs + 1); + vi.useRealTimers(); + const settledA = await Promise.race([ + putA, + new Promise((resolve) => setTimeout(() => resolve("STILL_RUNNING"), 2_000)), + ]); + expect(settledA).toMatchObject({ + ok: false, + error: { code: "NOT_READABLE", retryable: true }, + }); + expect(await putB).toMatchObject({ ok: true, value: { generation: 1 } }); + stalled.release(); + }); +});