From 68f4a93a192f965ef1933ea138d4f41e4745e203 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 24 Sep 2026 12:10:58 +0900 Subject: [PATCH] fix(storage): keep mutation lock waits out of the RPC timeout (R-5.3) A put holds the origin-wide OPFS mutation lock from BEGIN to FINISH, and every other BEGIN_PUT, REMOVE_OBJECT, FINALIZE_PUT, cleanup and GC request queues on it. The page started each request's 60s RPC timer at post time, so a second put or a remove queued behind an upload longer than 60s failed UNAVAILABLE while the worker kept waiting and later ran it anyway. Per D-3 the lock scope is unchanged. The worker now acquires the lock through one wrapper built on the platform abortable-operation kernel that bounds the wait by a new policy value, mutationLockWaitTimeoutMs (default 10 min, validated 1s..1h); a request still queued at the bound leaves the queue and fails BLOCKED, retryable, and a lease granted after the wait ended is released at once. The page gives requests that can queue on the lock (listed once in the protocol as OPFS_LOCK_WAITING_REQUEST_KINDS) a timer of rpcTimeoutMs + mutationLockWaitTimeoutMs, so queue time never eats the request's own 60s. Other requests keep rpcTimeoutMs. Verified: new tests "R-5.3 lock waiting is not counted against the RPC timeout" use an exclusive FIFO lease manager and the real client and runtime. Before the fix a put and a remove queued behind a stalled put settled as failures at 60s, and at the policy bound they reported UNAVAILABLE instead of BLOCKED. Now they wait, then complete once the lock frees, or fail BLOCKED at the bound with the queue emptied. Gates: check:architecture, check:types:app/test/web-worker, lint, test:component pass; the hand-rolled abort ratchet in check:adapter-inventory stays at 21; test:unit passes except removal-fixture.test.ts, which needs gitignored release artifacts absent from a fresh worktree. Co-Authored-By: Claude Opus 5.5 --- src/adapters/storage/opfs/opfs-policy.ts | 10 + .../storage/opfs/opfs-worker-client.ts | 9 +- .../storage/opfs/opfs-worker-protocol.ts | 17 ++ .../storage/opfs/opfs-worker-runtime.ts | 42 +++- tests/unit/opfs-worker-runtime.test.ts | 201 +++++++++++++++++- 5 files changed, 269 insertions(+), 10 deletions(-) diff --git a/src/adapters/storage/opfs/opfs-policy.ts b/src/adapters/storage/opfs/opfs-policy.ts index 67876611..5f0a9954 100644 --- a/src/adapters/storage/opfs/opfs-policy.ts +++ b/src/adapters/storage/opfs/opfs-policy.ts @@ -14,6 +14,12 @@ export type OpfsRuntimePolicy = Readonly<{ maxObjectBytes: number; maxChunkCount: number; rpcTimeoutMs: number; + /** + * R-5.3 (D-3). How long a request may queue for the origin mutation lock. + * Another context's put holds that lock for its whole upload, so this wait + * is bounded on its own and never counted against `rpcTimeoutMs`. + */ + mutationLockWaitTimeoutMs: number; reconciliationBudgetMs: number; reconciliationBatchSize: number; orphanGracePeriodMs: number; @@ -49,6 +55,7 @@ export const DEFAULT_OPFS_RUNTIME_POLICY: OpfsRuntimePolicy = Object.freeze({ maxObjectBytes: 2 * 1024 * 1024 * 1024, maxChunkCount: 512, rpcTimeoutMs: 60_000, + mutationLockWaitTimeoutMs: 10 * 60_000, reconciliationBudgetMs: 5_000, reconciliationBatchSize: 100, orphanGracePeriodMs: 24 * 60 * 60 * 1_000, @@ -84,6 +91,9 @@ export function assertOpfsRuntimePolicy(policy: OpfsRuntimePolicy): void { policy.maxObjectBytes > policy.chunkSizeBytes * policy.maxChunkCount || !Number.isSafeInteger(policy.rpcTimeoutMs) || policy.rpcTimeoutMs < 1_000 || + !Number.isSafeInteger(policy.mutationLockWaitTimeoutMs) || + policy.mutationLockWaitTimeoutMs < 1_000 || + policy.mutationLockWaitTimeoutMs > 60 * 60_000 || !Number.isSafeInteger(policy.reconciliationBudgetMs) || policy.reconciliationBudgetMs < 1 || policy.reconciliationBudgetMs > 60_000 || diff --git a/src/adapters/storage/opfs/opfs-worker-client.ts b/src/adapters/storage/opfs/opfs-worker-client.ts index 43db8f25..03869bae 100644 --- a/src/adapters/storage/opfs/opfs-worker-client.ts +++ b/src/adapters/storage/opfs/opfs-worker-client.ts @@ -16,6 +16,7 @@ import { } from "../../browser-file-storage/result.ts"; import type { OpfsRuntimePolicy } from "./opfs-policy.ts"; import { + OPFS_LOCK_WAITING_REQUEST_KINDS, OPFS_WORKER_PROTOCOL_VERSION, } from "./opfs-worker-protocol.ts"; import type { @@ -167,13 +168,19 @@ export function createOpfsWorkerGateway( reject(new OpfsRpcError("ABORTED")); }; signal?.addEventListener("abort", abort, { once: true }); + // R-5.3. Time queued behind another context's lock holder is the + // worker's to bound; it must not eat this request's own RPC budget. + const timeoutMs = OPFS_LOCK_WAITING_REQUEST_KINDS.has(request.kind) + ? dependencies.policy.rpcTimeoutMs + + dependencies.policy.mutationLockWaitTimeoutMs + : dependencies.policy.rpcTimeoutMs; const timeout = setTimeout(() => { const item = pending.get(requestId); if (!item) return; pending.delete(requestId); item.removeAbortListener(); reject(new OpfsRpcError("UNAVAILABLE")); - }, dependencies.policy.rpcTimeoutMs); + }, timeoutMs); pending.set(requestId, { // STO-07. The expected kind is stored so a reply for a different // operation can never satisfy this request. diff --git a/src/adapters/storage/opfs/opfs-worker-protocol.ts b/src/adapters/storage/opfs/opfs-worker-protocol.ts index 70062c06..67e68f38 100644 --- a/src/adapters/storage/opfs/opfs-worker-protocol.ts +++ b/src/adapters/storage/opfs/opfs-worker-protocol.ts @@ -136,6 +136,23 @@ export type OpfsWorkerRequest = olderThanEpochMs: number; }>; +/** + * R-5.3 (D-3). Requests the worker may park on the origin mutation lock. The + * worker bounds that wait by `mutationLockWaitTimeoutMs`; the page adds the + * same bound to these requests' RPC timer. + */ +export const OPFS_LOCK_WAITING_REQUEST_KINDS: ReadonlySet< + OpfsWorkerRequest["kind"] +> = new Set([ + "BEGIN_PUT", + "ABORT_PUT", + "REMOVE_OBJECT", + "CLEANUP_TRANSACTION", + "FINALIZE_PUT", + "LIST_ORPHAN_CANDIDATES", + "DELETE_ORPHAN_CHUNK", +]); + export type OpfsWorkerRequestBody = OpfsWorkerRequest extends infer Request ? Request extends OpfsWorkerRequest diff --git a/src/adapters/storage/opfs/opfs-worker-runtime.ts b/src/adapters/storage/opfs/opfs-worker-runtime.ts index 6a5e9a9f..ff31ba39 100644 --- a/src/adapters/storage/opfs/opfs-worker-runtime.ts +++ b/src/adapters/storage/opfs/opfs-worker-runtime.ts @@ -10,6 +10,7 @@ import { assertValidStoragePolicy, type BrowserDataFailureCode, } from "../../../application/ports/browser-file-storage/shared.ts"; +import { createAbortableOperation } from "../../platform/abortable-operation.ts"; import { resolveOpfsRuntimePolicy, isValidOpfsStorageScope, @@ -360,7 +361,7 @@ export function createOpfsWorkerRuntime( pendingBegins.set(transactionKey, beginAbort); let lease: OpfsMutationLease; try { - lease = await dependencies.leaseManager!.acquire(beginAbort.signal); + lease = await acquireLease(beginAbort.signal); } finally { pendingBegins.delete(transactionKey); } @@ -777,7 +778,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const prefixDirectory = await getDirectory( dependencies.root, @@ -827,7 +828,7 @@ export function createOpfsWorkerRuntime( throw error; } // Every destructive step below runs while the lease is held. - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { return await cleanupTransactionLocked( scope, @@ -911,7 +912,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const descriptor = preparedObject.descriptor; const objectDirectory = await getDirectory( @@ -968,7 +969,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const staged = await stagedChunkDigests(scope); if (!staged.safeToSweep) { @@ -1051,7 +1052,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const staged = await stagedChunkDigests(scope); if (!staged.safeToSweep) { @@ -1312,6 +1313,35 @@ export function createOpfsWorkerRuntime( throw new OpfsRuntimeFailure("UNSUPPORTED"); } } + + /** + * R-5.3 (D-3). The lock stays origin-wide so GC and dedup keep seeing one + * writer; only the wait is bounded. A request still queued after + * `mutationLockWaitTimeoutMs` leaves the queue and fails BLOCKED (retryable). + */ + async function acquireLease( + signal?: AbortSignal, + ): Promise { + const wait = createAbortableOperation({ + signal, + timeoutMs: dependencies.policy.mutationLockWaitTimeoutMs, + }); + // A lease granted after the wait already ended is handed straight back. + const outcome = await wait.race( + dependencies.leaseManager!.acquire(wait.signal), + (lease) => lease.release(), + ); + wait.close(); + if (outcome.kind === "VALUE") return outcome.value; + if (outcome.kind === "REJECTED") throw outcome.reason; + if (outcome.terminal === "DEADLINE") { + throw new OpfsRuntimeFailure("BLOCKED", true); + } + throw new OpfsRuntimeFailure( + outcome.terminal === "CALLER_ABORT" ? "ABORTED" : "UNAVAILABLE", + outcome.terminal !== "CALLER_ABORT", + ); + } } export function installOpfsWorkerMessageHandler( diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 0cc7cedd..af91d1f3 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpfsJournalPort, @@ -8,6 +8,7 @@ import type { } from "../../src/application/ports/browser-file-storage/opfs-ports.ts"; import type { BrowserStoragePolicy, + ByteSource, } from "../../src/application/ports/browser-file-storage/shared.ts"; import { browserDataSuccess } from "../../src/adapters/browser-file-storage/result.ts"; import { createOpfsByteStoreAdapter } from "../../src/adapters/storage/opfs/opfs-byte-store-adapter.ts"; @@ -604,7 +605,10 @@ async function seedChunk( } /** Wires the real client gateway to the real runtime through the protocol. */ -function loopbackGateway(runtime: OpfsWorkerRuntime): OpfsWorkerGateway { +function loopbackGateway( + runtime: OpfsWorkerRuntime, + policy = runtimePolicy, +): OpfsWorkerGateway { let listener: ((event: MessageEvent) => void) | undefined; let sequence = 0; return createOpfsWorkerGateway({ @@ -621,7 +625,7 @@ function loopbackGateway(runtime: OpfsWorkerRuntime): OpfsWorkerGateway { listener = undefined; }, }, - policy: runtimePolicy, + policy, createRequestId: () => `request_loopback_${++sequence}`, }); } @@ -819,6 +823,197 @@ describe("R-5.2 a dedup-reused chunk is protected by the grace period again", () }); }); +/** + * One holder at a time, FIFO, and a queued request leaves the queue when its + * signal aborts, the way a Web Lock request does. + */ +function exclusiveLeases(): OpfsMutationLeaseManager & { + readonly waiting: number; + readonly held: boolean; +} { + let held = false; + const queue: Array<() => void> = []; + const lease = () => { + let released = false; + return { + release() { + if (released) return; + released = true; + const next = queue.shift(); + if (next) next(); + else held = false; + }, + }; + }; + return { + get waiting() { + return queue.length; + }, + get held() { + return held; + }, + acquire(signal) { + if (signal?.aborted) { + return Promise.reject(new DOMException("Aborted", "AbortError")); + } + if (!held) { + held = true; + return Promise.resolve(lease()); + } + return new Promise((resolve, reject) => { + const grant = () => { + signal?.removeEventListener("abort", onAbort); + resolve(lease()); + }; + const onAbort = () => { + queue.splice(queue.indexOf(grant), 1); + reject(new DOMException("Aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + queue.push(grant); + }); + }, + }; +} + +/** A one-byte source that stalls until released, holding its put's lock. */ +function stalledSource(): Readonly<{ + source: ByteSource; + streaming: Promise; + release: () => void; +}> { + let markStreaming!: () => void; + const streaming = new Promise((resolve) => { + markStreaming = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + return { + source: { + byteLength: 1, + async *stream() { + markStreaming(); + await gate; + yield browserDataSuccess(new Uint8Array([5])); + }, + }, + streaming, + release, + }; +} + +function oneByteSource(): ByteSource { + return { + byteLength: 1, + async *stream() { + yield browserDataSuccess(new Uint8Array([6])); + }, + }; +} + +function putRequest( + suffix: string, + source: ByteSource, +): Parameters[0] { + return { + transactionId: `transaction_lock_${suffix}`, + physicalGenerationId: suffix.padStart(32, "0") as OpfsPhysicalGenerationId, + descriptor: { + objectId: `object_lock_${suffix}`, + scope: scopeA, + generation: 1, + byteLength: source.byteLength!, + mediaType: "application/octet-stream", + createdAtEpochMs: 100, + storagePolicy, + }, + source, + }; +} + +describe("R-5.3 lock waiting is not counted against the RPC timeout", () => { + const lockPolicy = resolveOpfsRuntimePolicy({ + ...runtimePolicy, + rpcTimeoutMs: 60_000, + mutationLockWaitTimeoutMs: 300_000, + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + /** + * Put A holds the origin mutation lock while its source stalls. Put B's + * BEGIN and a remove queue behind it, then the fake clock moves past + * `advanceMs`. + */ + async function contendedLock(advanceMs: number) { + const leases = exclusiveLeases(); + const runtime = createOpfsWorkerRuntime({ + root: new MemoryDirectory() as unknown as FileSystemDirectoryHandle, + crypto: globalThis.crypto, + policy: lockPolicy, + leaseManager: leases, + dedicatedWorker: true, + supportsSynchronousAccessHandles: false, + }); + const gateway = loopbackGateway(runtime, lockPolicy); + const stalled = stalledSource(); + const putA = gateway.preparePut(putRequest("a", stalled.source)); + await stalled.streaming; + expect(leases.held).toBe(true); + + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const settled = new Set(); + const putB = gateway.preparePut(putRequest("b", oneByteSource())); + const removal = gateway.removeObject(scopeA, "object_lock_remove", 1); + void putB.then(() => settled.add("putB")); + void removal.then(() => settled.add("remove")); + await vi.advanceTimersByTimeAsync(0); + expect(leases.waiting).toBe(2); + await vi.advanceTimersByTimeAsync(advanceMs); + vi.useRealTimers(); + return { leases, stalled, putA, putB, removal, settled }; + } + + it("lets a put and a remove queued behind a long put run once the lock frees", async () => { + const { leases, stalled, putA, putB, removal, settled } = + await contendedLock(lockPolicy.rpcTimeoutMs + 1); + expect(settled).toEqual(new Set()); + expect(leases.waiting).toBe(2); + + stalled.release(); + expect(await putA).toMatchObject({ ok: true }); + expect(await putB).toMatchObject({ + ok: true, + value: { descriptor: { objectId: "object_lock_b" } }, + }); + expect(await removal).toMatchObject({ ok: true }); + expect(leases.held).toBe(false); + }); + + it("ends a lock wait at the policy bound as a retryable BLOCKED", async () => { + const { leases, stalled, putA, putB, removal } = await contendedLock( + lockPolicy.mutationLockWaitTimeoutMs + 1, + ); + expect(await putB).toMatchObject({ + ok: false, + error: { code: "BLOCKED", retryable: true }, + }); + expect(await removal).toMatchObject({ + ok: false, + error: { code: "BLOCKED", retryable: true }, + }); + expect(leases.waiting).toBe(0); + + stalled.release(); + expect(await putA).toMatchObject({ ok: true }); + expect(leases.held).toBe(false); + }); +}); + describe("OPFS worker client lifecycle", () => { it("rejects a colliding concurrent request id without replacing the first RPC", async () => { let listener: ((event: MessageEvent) => void) | undefined;