diff --git a/docs/operations/browser-file-storage-recovery.md b/docs/operations/browser-file-storage-recovery.md index bb35dda3..36367cff 100644 --- a/docs/operations/browser-file-storage-recovery.md +++ b/docs/operations/browser-file-storage-recovery.md @@ -366,6 +366,11 @@ VD-15 목표 runtime을 먼저 구현한다. - committed logical row와 physical manifest/file 불일치 - digest/length mismatch - worker crash, handle lock timeout, `NoModificationAllowedError` +- put의 retryable `NOT_READABLE`: source가 `putIdleTimeoutMs`(기본 2분) 동안 다음 + bytes를 주지 않아 put을 끝내고 origin mutation lock을 풀었다(05-07, + `opfs-worker-client.ts` `preparePut`, `opfs-policy.ts`). OPFS 쓰기는 origin + 전체에서 한 번에 하나라서, 이 bound가 없으면 멈춘 source 하나가 모든 tab의 쓰기를 + `mutationLockWaitTimeoutMs` 뒤 `BLOCKED`로 만든다 - sensitive policy maintenance의 `POLICY_REJECTED`, proof replay/expiry 또는 authority provider/consumer unavailable diff --git a/src/adapters/storage/opfs/opfs-policy.ts b/src/adapters/storage/opfs/opfs-policy.ts index 5f0a9954..d5dd8a95 100644 --- a/src/adapters/storage/opfs/opfs-policy.ts +++ b/src/adapters/storage/opfs/opfs-policy.ts @@ -20,6 +20,13 @@ export type OpfsRuntimePolicy = Readonly<{ * is bounded on its own and never counted against `rpcTimeoutMs`. */ mutationLockWaitTimeoutMs: number; + /** + * 05-07. How long a put may wait for its source's next bytes. The put holds + * the origin mutation lock from BEGIN to FINISH, so a stalled source would + * otherwise stop every tab's writes until their lock wait ran out. Keep it + * below `mutationLockWaitTimeoutMs` so a queued writer outlasts it. + */ + putIdleTimeoutMs: number; reconciliationBudgetMs: number; reconciliationBatchSize: number; orphanGracePeriodMs: number; @@ -56,6 +63,7 @@ export const DEFAULT_OPFS_RUNTIME_POLICY: OpfsRuntimePolicy = Object.freeze({ maxChunkCount: 512, rpcTimeoutMs: 60_000, mutationLockWaitTimeoutMs: 10 * 60_000, + putIdleTimeoutMs: 2 * 60_000, reconciliationBudgetMs: 5_000, reconciliationBatchSize: 100, orphanGracePeriodMs: 24 * 60 * 60 * 1_000, @@ -94,6 +102,9 @@ export function assertOpfsRuntimePolicy(policy: OpfsRuntimePolicy): void { !Number.isSafeInteger(policy.mutationLockWaitTimeoutMs) || policy.mutationLockWaitTimeoutMs < 1_000 || policy.mutationLockWaitTimeoutMs > 60 * 60_000 || + !Number.isSafeInteger(policy.putIdleTimeoutMs) || + policy.putIdleTimeoutMs < 1_000 || + policy.putIdleTimeoutMs > 60 * 60_000 || !Number.isSafeInteger(policy.reconciliationBudgetMs) || policy.reconciliationBudgetMs < 1 || policy.reconciliationBudgetMs > 60_000 || diff --git a/src/adapters/storage/opfs/opfs-worker-client.ts b/src/adapters/storage/opfs/opfs-worker-client.ts index 9707ce79..2dffd10a 100644 --- a/src/adapters/storage/opfs/opfs-worker-client.ts +++ b/src/adapters/storage/opfs/opfs-worker-client.ts @@ -14,6 +14,7 @@ import { browserDataFailure, browserDataSuccess, } from "../../browser-file-storage/result.ts"; +import { createAbortableOperation } from "../../platform/abortable-operation.ts"; import type { OpfsRuntimePolicy } from "./opfs-policy.ts"; import { isPreparedObject } from "./opfs-prepared-object.ts"; import { @@ -272,13 +273,36 @@ export function createOpfsWorkerGateway( let sequence = 0; let transferredBytes = 0; notifyProgress(request, "TRANSFERRING", 0); + // 05-07. BEGIN took the origin mutation lock, so every wait for the + // source is bounded by `putIdleTimeoutMs`. A source that stops fails the + // put, and the coordinator's compensation releases the lock. + const sourceStop = new AbortController(); + const stopSource = () => sourceStop.abort(); + request.signal?.addEventListener("abort", stopSource, { once: true }); + if (request.signal?.aborted) stopSource(); + const chunks = rechunk( + request.source.stream(sourceStop.signal), + dependencies.policy.chunkSizeBytes, + dependencies.policy.maxObjectBytes, + request.signal, + ); try { - for await (const chunk of rechunk( - request.source.stream(requiredSignal(request.signal)), - dependencies.policy.chunkSizeBytes, - dependencies.policy.maxObjectBytes, - request.signal, - )) { + for (;;) { + const idle = createAbortableOperation({ + signal: request.signal, + timeoutMs: dependencies.policy.putIdleTimeoutMs, + }); + const next = await idle.race(chunks.next()); + idle.close(); + if (next.kind === "REJECTED") throw next.reason; + if (next.kind === "TERMINAL") { + stopSource(); + throw new OpfsRpcError( + next.terminal === "DEADLINE" ? "NOT_READABLE" : "ABORTED", + ); + } + if (next.value.done) break; + const chunk = next.value.value; const chunkByteLength = chunk.byteLength; const append = await invoke( "OBJECT_WRITE", @@ -320,6 +344,8 @@ export function createOpfsWorkerGateway( error instanceof OpfsRpcError ? error.code : "NOT_READABLE", "OBJECT_WRITE", ); + } finally { + request.signal?.removeEventListener("abort", stopSource); } }, @@ -585,10 +611,6 @@ async function* rechunk( } } -function requiredSignal(signal: AbortSignal | undefined): AbortSignal { - return signal ?? new AbortController().signal; -} - function notifyProgress( request: PreparePhysicalObjectRequest, phase: "TRANSFERRING" | "VERIFYING", diff --git a/src/application/ports/browser-file-storage/opfs-ports.ts b/src/application/ports/browser-file-storage/opfs-ports.ts index fdc2d3d1..227b314d 100644 --- a/src/application/ports/browser-file-storage/opfs-ports.ts +++ b/src/application/ports/browser-file-storage/opfs-ports.ts @@ -76,6 +76,12 @@ export type OpfsCapabilities = Readonly<{ export interface DurableObjectStorePort { capabilities(): Promise>; + /** + * 05-07. OPFS writes are one at a time across the whole origin: a put holds + * the origin mutation lock while it streams, and every put, remove and GC in + * every tab waits for it. A source that stops for the runtime's put idle + * bound fails the put with a retryable `NOT_READABLE` and frees the lock. + */ put( request: PutDurableObjectRequest, ): Promise>; diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 3c3a6a52..dbac0184 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -2398,3 +2398,48 @@ describe("05-04 a LOGOUT or ACCOUNT_DELETION purge leaves no partition bytes beh }); }); }); + +describe("05-07 a put whose source stops lets go of the origin lock", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("fails the stalled put at the idle bound so another tab's put runs", async () => { + const policy = resolveOpfsRuntimePolicy({ + ...runtimePolicy, + putIdleTimeoutMs: 30_000, + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const { tab, leases } = sharedRootTabs(policy); + const tabA = tab("a"); + const tabB = tab("b"); + const stalled = stalledSource(); + const putA = tabA.objects.put({ + objectId: "object_stalled_01", + expectedGeneration: null, + mediaType: "application/octet-stream", + source: stalled.source, + }); + await stalled.streaming; + expect(leases.held).toBe(true); + const putB = tabB.objects.put({ + objectId: "object_waiting_01", + expectedGeneration: null, + mediaType: "application/octet-stream", + source: oneByteSource(), + }); + + await vi.advanceTimersByTimeAsync(policy.putIdleTimeoutMs + 1); + vi.useRealTimers(); + const settledA = await Promise.race([ + putA, + new Promise((resolve) => setTimeout(() => resolve("STILL_RUNNING"), 2_000)), + ]); + expect(settledA).toMatchObject({ + ok: false, + error: { code: "NOT_READABLE", retryable: true }, + }); + expect(await putB).toMatchObject({ ok: true, value: { generation: 1 } }); + stalled.release(); + }); +});