diff --git a/src/adapters/storage/opfs/opfs-policy.ts b/src/adapters/storage/opfs/opfs-policy.ts index 67876611..5f0a9954 100644 --- a/src/adapters/storage/opfs/opfs-policy.ts +++ b/src/adapters/storage/opfs/opfs-policy.ts @@ -14,6 +14,12 @@ export type OpfsRuntimePolicy = Readonly<{ maxObjectBytes: number; maxChunkCount: number; rpcTimeoutMs: number; + /** + * R-5.3 (D-3). How long a request may queue for the origin mutation lock. + * Another context's put holds that lock for its whole upload, so this wait + * is bounded on its own and never counted against `rpcTimeoutMs`. + */ + mutationLockWaitTimeoutMs: number; reconciliationBudgetMs: number; reconciliationBatchSize: number; orphanGracePeriodMs: number; @@ -49,6 +55,7 @@ export const DEFAULT_OPFS_RUNTIME_POLICY: OpfsRuntimePolicy = Object.freeze({ maxObjectBytes: 2 * 1024 * 1024 * 1024, maxChunkCount: 512, rpcTimeoutMs: 60_000, + mutationLockWaitTimeoutMs: 10 * 60_000, reconciliationBudgetMs: 5_000, reconciliationBatchSize: 100, orphanGracePeriodMs: 24 * 60 * 60 * 1_000, @@ -84,6 +91,9 @@ export function assertOpfsRuntimePolicy(policy: OpfsRuntimePolicy): void { policy.maxObjectBytes > policy.chunkSizeBytes * policy.maxChunkCount || !Number.isSafeInteger(policy.rpcTimeoutMs) || policy.rpcTimeoutMs < 1_000 || + !Number.isSafeInteger(policy.mutationLockWaitTimeoutMs) || + policy.mutationLockWaitTimeoutMs < 1_000 || + policy.mutationLockWaitTimeoutMs > 60 * 60_000 || !Number.isSafeInteger(policy.reconciliationBudgetMs) || policy.reconciliationBudgetMs < 1 || policy.reconciliationBudgetMs > 60_000 || diff --git a/src/adapters/storage/opfs/opfs-worker-client.ts b/src/adapters/storage/opfs/opfs-worker-client.ts index 43db8f25..03869bae 100644 --- a/src/adapters/storage/opfs/opfs-worker-client.ts +++ b/src/adapters/storage/opfs/opfs-worker-client.ts @@ -16,6 +16,7 @@ import { } from "../../browser-file-storage/result.ts"; import type { OpfsRuntimePolicy } from "./opfs-policy.ts"; import { + OPFS_LOCK_WAITING_REQUEST_KINDS, OPFS_WORKER_PROTOCOL_VERSION, } from "./opfs-worker-protocol.ts"; import type { @@ -167,13 +168,19 @@ export function createOpfsWorkerGateway( reject(new OpfsRpcError("ABORTED")); }; signal?.addEventListener("abort", abort, { once: true }); + // R-5.3. Time queued behind another context's lock holder is the + // worker's to bound; it must not eat this request's own RPC budget. + const timeoutMs = OPFS_LOCK_WAITING_REQUEST_KINDS.has(request.kind) + ? dependencies.policy.rpcTimeoutMs + + dependencies.policy.mutationLockWaitTimeoutMs + : dependencies.policy.rpcTimeoutMs; const timeout = setTimeout(() => { const item = pending.get(requestId); if (!item) return; pending.delete(requestId); item.removeAbortListener(); reject(new OpfsRpcError("UNAVAILABLE")); - }, dependencies.policy.rpcTimeoutMs); + }, timeoutMs); pending.set(requestId, { // STO-07. The expected kind is stored so a reply for a different // operation can never satisfy this request. diff --git a/src/adapters/storage/opfs/opfs-worker-protocol.ts b/src/adapters/storage/opfs/opfs-worker-protocol.ts index 70062c06..67e68f38 100644 --- a/src/adapters/storage/opfs/opfs-worker-protocol.ts +++ b/src/adapters/storage/opfs/opfs-worker-protocol.ts @@ -136,6 +136,23 @@ export type OpfsWorkerRequest = olderThanEpochMs: number; }>; +/** + * R-5.3 (D-3). Requests the worker may park on the origin mutation lock. The + * worker bounds that wait by `mutationLockWaitTimeoutMs`; the page adds the + * same bound to these requests' RPC timer. + */ +export const OPFS_LOCK_WAITING_REQUEST_KINDS: ReadonlySet< + OpfsWorkerRequest["kind"] +> = new Set([ + "BEGIN_PUT", + "ABORT_PUT", + "REMOVE_OBJECT", + "CLEANUP_TRANSACTION", + "FINALIZE_PUT", + "LIST_ORPHAN_CANDIDATES", + "DELETE_ORPHAN_CHUNK", +]); + export type OpfsWorkerRequestBody = OpfsWorkerRequest extends infer Request ? Request extends OpfsWorkerRequest diff --git a/src/adapters/storage/opfs/opfs-worker-runtime.ts b/src/adapters/storage/opfs/opfs-worker-runtime.ts index 6a5e9a9f..ff31ba39 100644 --- a/src/adapters/storage/opfs/opfs-worker-runtime.ts +++ b/src/adapters/storage/opfs/opfs-worker-runtime.ts @@ -10,6 +10,7 @@ import { assertValidStoragePolicy, type BrowserDataFailureCode, } from "../../../application/ports/browser-file-storage/shared.ts"; +import { createAbortableOperation } from "../../platform/abortable-operation.ts"; import { resolveOpfsRuntimePolicy, isValidOpfsStorageScope, @@ -360,7 +361,7 @@ export function createOpfsWorkerRuntime( pendingBegins.set(transactionKey, beginAbort); let lease: OpfsMutationLease; try { - lease = await dependencies.leaseManager!.acquire(beginAbort.signal); + lease = await acquireLease(beginAbort.signal); } finally { pendingBegins.delete(transactionKey); } @@ -777,7 +778,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const prefixDirectory = await getDirectory( dependencies.root, @@ -827,7 +828,7 @@ export function createOpfsWorkerRuntime( throw error; } // Every destructive step below runs while the lease is held. - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { return await cleanupTransactionLocked( scope, @@ -911,7 +912,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const descriptor = preparedObject.descriptor; const objectDirectory = await getDirectory( @@ -968,7 +969,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const staged = await stagedChunkDigests(scope); if (!staged.safeToSweep) { @@ -1051,7 +1052,7 @@ export function createOpfsWorkerRuntime( ) { throw new OpfsRuntimeFailure("INVALID_INPUT"); } - const lease = await dependencies.leaseManager!.acquire(); + const lease = await acquireLease(); try { const staged = await stagedChunkDigests(scope); if (!staged.safeToSweep) { @@ -1312,6 +1313,35 @@ export function createOpfsWorkerRuntime( throw new OpfsRuntimeFailure("UNSUPPORTED"); } } + + /** + * R-5.3 (D-3). The lock stays origin-wide so GC and dedup keep seeing one + * writer; only the wait is bounded. A request still queued after + * `mutationLockWaitTimeoutMs` leaves the queue and fails BLOCKED (retryable). + */ + async function acquireLease( + signal?: AbortSignal, + ): Promise { + const wait = createAbortableOperation({ + signal, + timeoutMs: dependencies.policy.mutationLockWaitTimeoutMs, + }); + // A lease granted after the wait already ended is handed straight back. + const outcome = await wait.race( + dependencies.leaseManager!.acquire(wait.signal), + (lease) => lease.release(), + ); + wait.close(); + if (outcome.kind === "VALUE") return outcome.value; + if (outcome.kind === "REJECTED") throw outcome.reason; + if (outcome.terminal === "DEADLINE") { + throw new OpfsRuntimeFailure("BLOCKED", true); + } + throw new OpfsRuntimeFailure( + outcome.terminal === "CALLER_ABORT" ? "ABORTED" : "UNAVAILABLE", + outcome.terminal !== "CALLER_ABORT", + ); + } } export function installOpfsWorkerMessageHandler( diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 0cc7cedd..af91d1f3 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpfsJournalPort, @@ -8,6 +8,7 @@ import type { } from "../../src/application/ports/browser-file-storage/opfs-ports.ts"; import type { BrowserStoragePolicy, + ByteSource, } from "../../src/application/ports/browser-file-storage/shared.ts"; import { browserDataSuccess } from "../../src/adapters/browser-file-storage/result.ts"; import { createOpfsByteStoreAdapter } from "../../src/adapters/storage/opfs/opfs-byte-store-adapter.ts"; @@ -604,7 +605,10 @@ async function seedChunk( } /** Wires the real client gateway to the real runtime through the protocol. */ -function loopbackGateway(runtime: OpfsWorkerRuntime): OpfsWorkerGateway { +function loopbackGateway( + runtime: OpfsWorkerRuntime, + policy = runtimePolicy, +): OpfsWorkerGateway { let listener: ((event: MessageEvent) => void) | undefined; let sequence = 0; return createOpfsWorkerGateway({ @@ -621,7 +625,7 @@ function loopbackGateway(runtime: OpfsWorkerRuntime): OpfsWorkerGateway { listener = undefined; }, }, - policy: runtimePolicy, + policy, createRequestId: () => `request_loopback_${++sequence}`, }); } @@ -819,6 +823,197 @@ describe("R-5.2 a dedup-reused chunk is protected by the grace period again", () }); }); +/** + * One holder at a time, FIFO, and a queued request leaves the queue when its + * signal aborts, the way a Web Lock request does. + */ +function exclusiveLeases(): OpfsMutationLeaseManager & { + readonly waiting: number; + readonly held: boolean; +} { + let held = false; + const queue: Array<() => void> = []; + const lease = () => { + let released = false; + return { + release() { + if (released) return; + released = true; + const next = queue.shift(); + if (next) next(); + else held = false; + }, + }; + }; + return { + get waiting() { + return queue.length; + }, + get held() { + return held; + }, + acquire(signal) { + if (signal?.aborted) { + return Promise.reject(new DOMException("Aborted", "AbortError")); + } + if (!held) { + held = true; + return Promise.resolve(lease()); + } + return new Promise((resolve, reject) => { + const grant = () => { + signal?.removeEventListener("abort", onAbort); + resolve(lease()); + }; + const onAbort = () => { + queue.splice(queue.indexOf(grant), 1); + reject(new DOMException("Aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + queue.push(grant); + }); + }, + }; +} + +/** A one-byte source that stalls until released, holding its put's lock. */ +function stalledSource(): Readonly<{ + source: ByteSource; + streaming: Promise; + release: () => void; +}> { + let markStreaming!: () => void; + const streaming = new Promise((resolve) => { + markStreaming = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + return { + source: { + byteLength: 1, + async *stream() { + markStreaming(); + await gate; + yield browserDataSuccess(new Uint8Array([5])); + }, + }, + streaming, + release, + }; +} + +function oneByteSource(): ByteSource { + return { + byteLength: 1, + async *stream() { + yield browserDataSuccess(new Uint8Array([6])); + }, + }; +} + +function putRequest( + suffix: string, + source: ByteSource, +): Parameters[0] { + return { + transactionId: `transaction_lock_${suffix}`, + physicalGenerationId: suffix.padStart(32, "0") as OpfsPhysicalGenerationId, + descriptor: { + objectId: `object_lock_${suffix}`, + scope: scopeA, + generation: 1, + byteLength: source.byteLength!, + mediaType: "application/octet-stream", + createdAtEpochMs: 100, + storagePolicy, + }, + source, + }; +} + +describe("R-5.3 lock waiting is not counted against the RPC timeout", () => { + const lockPolicy = resolveOpfsRuntimePolicy({ + ...runtimePolicy, + rpcTimeoutMs: 60_000, + mutationLockWaitTimeoutMs: 300_000, + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + /** + * Put A holds the origin mutation lock while its source stalls. Put B's + * BEGIN and a remove queue behind it, then the fake clock moves past + * `advanceMs`. + */ + async function contendedLock(advanceMs: number) { + const leases = exclusiveLeases(); + const runtime = createOpfsWorkerRuntime({ + root: new MemoryDirectory() as unknown as FileSystemDirectoryHandle, + crypto: globalThis.crypto, + policy: lockPolicy, + leaseManager: leases, + dedicatedWorker: true, + supportsSynchronousAccessHandles: false, + }); + const gateway = loopbackGateway(runtime, lockPolicy); + const stalled = stalledSource(); + const putA = gateway.preparePut(putRequest("a", stalled.source)); + await stalled.streaming; + expect(leases.held).toBe(true); + + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const settled = new Set(); + const putB = gateway.preparePut(putRequest("b", oneByteSource())); + const removal = gateway.removeObject(scopeA, "object_lock_remove", 1); + void putB.then(() => settled.add("putB")); + void removal.then(() => settled.add("remove")); + await vi.advanceTimersByTimeAsync(0); + expect(leases.waiting).toBe(2); + await vi.advanceTimersByTimeAsync(advanceMs); + vi.useRealTimers(); + return { leases, stalled, putA, putB, removal, settled }; + } + + it("lets a put and a remove queued behind a long put run once the lock frees", async () => { + const { leases, stalled, putA, putB, removal, settled } = + await contendedLock(lockPolicy.rpcTimeoutMs + 1); + expect(settled).toEqual(new Set()); + expect(leases.waiting).toBe(2); + + stalled.release(); + expect(await putA).toMatchObject({ ok: true }); + expect(await putB).toMatchObject({ + ok: true, + value: { descriptor: { objectId: "object_lock_b" } }, + }); + expect(await removal).toMatchObject({ ok: true }); + expect(leases.held).toBe(false); + }); + + it("ends a lock wait at the policy bound as a retryable BLOCKED", async () => { + const { leases, stalled, putA, putB, removal } = await contendedLock( + lockPolicy.mutationLockWaitTimeoutMs + 1, + ); + expect(await putB).toMatchObject({ + ok: false, + error: { code: "BLOCKED", retryable: true }, + }); + expect(await removal).toMatchObject({ + ok: false, + error: { code: "BLOCKED", retryable: true }, + }); + expect(leases.waiting).toBe(0); + + stalled.release(); + expect(await putA).toMatchObject({ ok: true }); + expect(leases.held).toBe(false); + }); +}); + describe("OPFS worker client lifecycle", () => { it("rejects a colliding concurrent request id without replacing the first RPC", async () => { let listener: ((event: MessageEvent) => void) | undefined;