fix(storage): keep mutation lock waits out of the RPC timeout (R-5.3)

A put holds the origin-wide OPFS mutation lock from BEGIN to FINISH, and
every other BEGIN_PUT, REMOVE_OBJECT, FINALIZE_PUT, cleanup and GC request
queues on it. The page started each request's 60s RPC timer at post time,
so a second put or a remove queued behind an upload longer than 60s failed
UNAVAILABLE while the worker kept waiting and later ran it anyway.

Per D-3 the lock scope is unchanged. The worker now acquires the lock
through one wrapper built on the platform abortable-operation kernel that
bounds the wait by a new policy value, mutationLockWaitTimeoutMs (default
10 min, validated 1s..1h); a request still queued at the bound leaves the
queue and fails BLOCKED, retryable, and a lease granted after the wait
ended is released at once. The page gives requests that can queue on the
lock (listed once in the protocol as OPFS_LOCK_WAITING_REQUEST_KINDS) a
timer of rpcTimeoutMs + mutationLockWaitTimeoutMs, so queue time never
eats the request's own 60s. Other requests keep rpcTimeoutMs.

Verified: new tests "R-5.3 lock waiting is not counted against the RPC
timeout" use an exclusive FIFO lease manager and the real client and
runtime. Before the fix a put and a remove queued behind a stalled put
settled as failures at 60s, and at the policy bound they reported
UNAVAILABLE instead of BLOCKED. Now they wait, then complete once the
lock frees, or fail BLOCKED at the bound with the queue emptied.
Gates: check:architecture, check:types:app/test/web-worker, lint,
test:component pass; the hand-rolled abort ratchet in
check:adapter-inventory stays at 21; test:unit passes except
removal-fixture.test.ts, which needs gitignored release artifacts absent
from a fresh worktree.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-24 12:27:48 +09:00
co-authored by Claude Opus 5.5
parent e28e9f75fb
commit 68f4a93a19
5 changed files with 269 additions and 10 deletions
+10
View File
@@ -14,6 +14,12 @@ export type OpfsRuntimePolicy = Readonly<{
maxObjectBytes: number;
maxChunkCount: number;
rpcTimeoutMs: number;
/**
* R-5.3 (D-3). How long a request may queue for the origin mutation lock.
* Another context's put holds that lock for its whole upload, so this wait
* is bounded on its own and never counted against `rpcTimeoutMs`.
*/
mutationLockWaitTimeoutMs: number;
reconciliationBudgetMs: number;
reconciliationBatchSize: number;
orphanGracePeriodMs: number;
@@ -49,6 +55,7 @@ export const DEFAULT_OPFS_RUNTIME_POLICY: OpfsRuntimePolicy = Object.freeze({
maxObjectBytes: 2 * 1024 * 1024 * 1024,
maxChunkCount: 512,
rpcTimeoutMs: 60_000,
mutationLockWaitTimeoutMs: 10 * 60_000,
reconciliationBudgetMs: 5_000,
reconciliationBatchSize: 100,
orphanGracePeriodMs: 24 * 60 * 60 * 1_000,
@@ -84,6 +91,9 @@ export function assertOpfsRuntimePolicy(policy: OpfsRuntimePolicy): void {
policy.maxObjectBytes > policy.chunkSizeBytes * policy.maxChunkCount ||
!Number.isSafeInteger(policy.rpcTimeoutMs) ||
policy.rpcTimeoutMs < 1_000 ||
!Number.isSafeInteger(policy.mutationLockWaitTimeoutMs) ||
policy.mutationLockWaitTimeoutMs < 1_000 ||
policy.mutationLockWaitTimeoutMs > 60 * 60_000 ||
!Number.isSafeInteger(policy.reconciliationBudgetMs) ||
policy.reconciliationBudgetMs < 1 ||
policy.reconciliationBudgetMs > 60_000 ||
@@ -16,6 +16,7 @@ import {
} from "../../browser-file-storage/result.ts";
import type { OpfsRuntimePolicy } from "./opfs-policy.ts";
import {
OPFS_LOCK_WAITING_REQUEST_KINDS,
OPFS_WORKER_PROTOCOL_VERSION,
} from "./opfs-worker-protocol.ts";
import type {
@@ -167,13 +168,19 @@ export function createOpfsWorkerGateway(
reject(new OpfsRpcError("ABORTED"));
};
signal?.addEventListener("abort", abort, { once: true });
// R-5.3. Time queued behind another context's lock holder is the
// worker's to bound; it must not eat this request's own RPC budget.
const timeoutMs = OPFS_LOCK_WAITING_REQUEST_KINDS.has(request.kind)
? dependencies.policy.rpcTimeoutMs +
dependencies.policy.mutationLockWaitTimeoutMs
: dependencies.policy.rpcTimeoutMs;
const timeout = setTimeout(() => {
const item = pending.get(requestId);
if (!item) return;
pending.delete(requestId);
item.removeAbortListener();
reject(new OpfsRpcError("UNAVAILABLE"));
}, dependencies.policy.rpcTimeoutMs);
}, timeoutMs);
pending.set(requestId, {
// STO-07. The expected kind is stored so a reply for a different
// operation can never satisfy this request.
@@ -136,6 +136,23 @@ export type OpfsWorkerRequest =
olderThanEpochMs: number;
}>;
/**
* R-5.3 (D-3). Requests the worker may park on the origin mutation lock. The
* worker bounds that wait by `mutationLockWaitTimeoutMs`; the page adds the
* same bound to these requests' RPC timer.
*/
export const OPFS_LOCK_WAITING_REQUEST_KINDS: ReadonlySet<
OpfsWorkerRequest["kind"]
> = new Set([
"BEGIN_PUT",
"ABORT_PUT",
"REMOVE_OBJECT",
"CLEANUP_TRANSACTION",
"FINALIZE_PUT",
"LIST_ORPHAN_CANDIDATES",
"DELETE_ORPHAN_CHUNK",
]);
export type OpfsWorkerRequestBody =
OpfsWorkerRequest extends infer Request
? Request extends OpfsWorkerRequest
@@ -10,6 +10,7 @@ import {
assertValidStoragePolicy,
type BrowserDataFailureCode,
} from "../../../application/ports/browser-file-storage/shared.ts";
import { createAbortableOperation } from "../../platform/abortable-operation.ts";
import {
resolveOpfsRuntimePolicy,
isValidOpfsStorageScope,
@@ -360,7 +361,7 @@ export function createOpfsWorkerRuntime(
pendingBegins.set(transactionKey, beginAbort);
let lease: OpfsMutationLease;
try {
lease = await dependencies.leaseManager!.acquire(beginAbort.signal);
lease = await acquireLease(beginAbort.signal);
} finally {
pendingBegins.delete(transactionKey);
}
@@ -777,7 +778,7 @@ export function createOpfsWorkerRuntime(
) {
throw new OpfsRuntimeFailure("INVALID_INPUT");
}
const lease = await dependencies.leaseManager!.acquire();
const lease = await acquireLease();
try {
const prefixDirectory = await getDirectory(
dependencies.root,
@@ -827,7 +828,7 @@ export function createOpfsWorkerRuntime(
throw error;
}
// Every destructive step below runs while the lease is held.
const lease = await dependencies.leaseManager!.acquire();
const lease = await acquireLease();
try {
return await cleanupTransactionLocked(
scope,
@@ -911,7 +912,7 @@ export function createOpfsWorkerRuntime(
) {
throw new OpfsRuntimeFailure("INVALID_INPUT");
}
const lease = await dependencies.leaseManager!.acquire();
const lease = await acquireLease();
try {
const descriptor = preparedObject.descriptor;
const objectDirectory = await getDirectory(
@@ -968,7 +969,7 @@ export function createOpfsWorkerRuntime(
) {
throw new OpfsRuntimeFailure("INVALID_INPUT");
}
const lease = await dependencies.leaseManager!.acquire();
const lease = await acquireLease();
try {
const staged = await stagedChunkDigests(scope);
if (!staged.safeToSweep) {
@@ -1051,7 +1052,7 @@ export function createOpfsWorkerRuntime(
) {
throw new OpfsRuntimeFailure("INVALID_INPUT");
}
const lease = await dependencies.leaseManager!.acquire();
const lease = await acquireLease();
try {
const staged = await stagedChunkDigests(scope);
if (!staged.safeToSweep) {
@@ -1312,6 +1313,35 @@ export function createOpfsWorkerRuntime(
throw new OpfsRuntimeFailure("UNSUPPORTED");
}
}
/**
* R-5.3 (D-3). The lock stays origin-wide so GC and dedup keep seeing one
* writer; only the wait is bounded. A request still queued after
* `mutationLockWaitTimeoutMs` leaves the queue and fails BLOCKED (retryable).
*/
async function acquireLease(
signal?: AbortSignal,
): Promise<OpfsMutationLease> {
const wait = createAbortableOperation({
signal,
timeoutMs: dependencies.policy.mutationLockWaitTimeoutMs,
});
// A lease granted after the wait already ended is handed straight back.
const outcome = await wait.race(
dependencies.leaseManager!.acquire(wait.signal),
(lease) => lease.release(),
);
wait.close();
if (outcome.kind === "VALUE") return outcome.value;
if (outcome.kind === "REJECTED") throw outcome.reason;
if (outcome.terminal === "DEADLINE") {
throw new OpfsRuntimeFailure("BLOCKED", true);
}
throw new OpfsRuntimeFailure(
outcome.terminal === "CALLER_ABORT" ? "ABORTED" : "UNAVAILABLE",
outcome.terminal !== "CALLER_ABORT",
);
}
}
export function installOpfsWorkerMessageHandler(
+198 -3
View File
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import type {
OpfsJournalPort,
@@ -8,6 +8,7 @@ import type {
} from "../../src/application/ports/browser-file-storage/opfs-ports.ts";
import type {
BrowserStoragePolicy,
ByteSource,
} from "../../src/application/ports/browser-file-storage/shared.ts";
import { browserDataSuccess } from "../../src/adapters/browser-file-storage/result.ts";
import { createOpfsByteStoreAdapter } from "../../src/adapters/storage/opfs/opfs-byte-store-adapter.ts";
@@ -604,7 +605,10 @@ async function seedChunk(
}
/** Wires the real client gateway to the real runtime through the protocol. */
function loopbackGateway(runtime: OpfsWorkerRuntime): OpfsWorkerGateway {
function loopbackGateway(
runtime: OpfsWorkerRuntime,
policy = runtimePolicy,
): OpfsWorkerGateway {
let listener: ((event: MessageEvent<unknown>) => void) | undefined;
let sequence = 0;
return createOpfsWorkerGateway({
@@ -621,7 +625,7 @@ function loopbackGateway(runtime: OpfsWorkerRuntime): OpfsWorkerGateway {
listener = undefined;
},
},
policy: runtimePolicy,
policy,
createRequestId: () => `request_loopback_${++sequence}`,
});
}
@@ -819,6 +823,197 @@ describe("R-5.2 a dedup-reused chunk is protected by the grace period again", ()
});
});
/**
* One holder at a time, FIFO, and a queued request leaves the queue when its
* signal aborts, the way a Web Lock request does.
*/
function exclusiveLeases(): OpfsMutationLeaseManager & {
readonly waiting: number;
readonly held: boolean;
} {
let held = false;
const queue: Array<() => void> = [];
const lease = () => {
let released = false;
return {
release() {
if (released) return;
released = true;
const next = queue.shift();
if (next) next();
else held = false;
},
};
};
return {
get waiting() {
return queue.length;
},
get held() {
return held;
},
acquire(signal) {
if (signal?.aborted) {
return Promise.reject(new DOMException("Aborted", "AbortError"));
}
if (!held) {
held = true;
return Promise.resolve(lease());
}
return new Promise((resolve, reject) => {
const grant = () => {
signal?.removeEventListener("abort", onAbort);
resolve(lease());
};
const onAbort = () => {
queue.splice(queue.indexOf(grant), 1);
reject(new DOMException("Aborted", "AbortError"));
};
signal?.addEventListener("abort", onAbort, { once: true });
queue.push(grant);
});
},
};
}
/** A one-byte source that stalls until released, holding its put's lock. */
function stalledSource(): Readonly<{
source: ByteSource;
streaming: Promise<void>;
release: () => void;
}> {
let markStreaming!: () => void;
const streaming = new Promise<void>((resolve) => {
markStreaming = resolve;
});
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
return {
source: {
byteLength: 1,
async *stream() {
markStreaming();
await gate;
yield browserDataSuccess(new Uint8Array([5]));
},
},
streaming,
release,
};
}
function oneByteSource(): ByteSource {
return {
byteLength: 1,
async *stream() {
yield browserDataSuccess(new Uint8Array([6]));
},
};
}
function putRequest(
suffix: string,
source: ByteSource,
): Parameters<OpfsWorkerGateway["preparePut"]>[0] {
return {
transactionId: `transaction_lock_${suffix}`,
physicalGenerationId: suffix.padStart(32, "0") as OpfsPhysicalGenerationId,
descriptor: {
objectId: `object_lock_${suffix}`,
scope: scopeA,
generation: 1,
byteLength: source.byteLength!,
mediaType: "application/octet-stream",
createdAtEpochMs: 100,
storagePolicy,
},
source,
};
}
describe("R-5.3 lock waiting is not counted against the RPC timeout", () => {
const lockPolicy = resolveOpfsRuntimePolicy({
...runtimePolicy,
rpcTimeoutMs: 60_000,
mutationLockWaitTimeoutMs: 300_000,
});
afterEach(() => {
vi.useRealTimers();
});
/**
* Put A holds the origin mutation lock while its source stalls. Put B's
* BEGIN and a remove queue behind it, then the fake clock moves past
* `advanceMs`.
*/
async function contendedLock(advanceMs: number) {
const leases = exclusiveLeases();
const runtime = createOpfsWorkerRuntime({
root: new MemoryDirectory() as unknown as FileSystemDirectoryHandle,
crypto: globalThis.crypto,
policy: lockPolicy,
leaseManager: leases,
dedicatedWorker: true,
supportsSynchronousAccessHandles: false,
});
const gateway = loopbackGateway(runtime, lockPolicy);
const stalled = stalledSource();
const putA = gateway.preparePut(putRequest("a", stalled.source));
await stalled.streaming;
expect(leases.held).toBe(true);
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const settled = new Set<string>();
const putB = gateway.preparePut(putRequest("b", oneByteSource()));
const removal = gateway.removeObject(scopeA, "object_lock_remove", 1);
void putB.then(() => settled.add("putB"));
void removal.then(() => settled.add("remove"));
await vi.advanceTimersByTimeAsync(0);
expect(leases.waiting).toBe(2);
await vi.advanceTimersByTimeAsync(advanceMs);
vi.useRealTimers();
return { leases, stalled, putA, putB, removal, settled };
}
it("lets a put and a remove queued behind a long put run once the lock frees", async () => {
const { leases, stalled, putA, putB, removal, settled } =
await contendedLock(lockPolicy.rpcTimeoutMs + 1);
expect(settled).toEqual(new Set());
expect(leases.waiting).toBe(2);
stalled.release();
expect(await putA).toMatchObject({ ok: true });
expect(await putB).toMatchObject({
ok: true,
value: { descriptor: { objectId: "object_lock_b" } },
});
expect(await removal).toMatchObject({ ok: true });
expect(leases.held).toBe(false);
});
it("ends a lock wait at the policy bound as a retryable BLOCKED", async () => {
const { leases, stalled, putA, putB, removal } = await contendedLock(
lockPolicy.mutationLockWaitTimeoutMs + 1,
);
expect(await putB).toMatchObject({
ok: false,
error: { code: "BLOCKED", retryable: true },
});
expect(await removal).toMatchObject({
ok: false,
error: { code: "BLOCKED", retryable: true },
});
expect(leases.waiting).toBe(0);
stalled.release();
expect(await putA).toMatchObject({ ok: true });
expect(leases.held).toBe(false);
});
});
describe("OPFS worker client lifecycle", () => {
it("rejects a colliding concurrent request id without replacing the first RPC", async () => {
let listener: ((event: MessageEvent<unknown>) => void) | undefined;