diff --git a/docs/operations/browser-file-storage-recovery.md b/docs/operations/browser-file-storage-recovery.md index 9a99dff5..66ae77e5 100644 --- a/docs/operations/browser-file-storage-recovery.md +++ b/docs/operations/browser-file-storage-recovery.md @@ -462,12 +462,26 @@ VD-15 목표 runtime을 먼저 구현한다. `corruptObjects`. 이 report면 그 partition의 bytes는 모두 지워진 것이다. `chunkSweep`은 없다. 디렉터리는 지웠는데 journal 정리가 실패하면 실패를 반환한다. 같은 purge를 다시 호출하면 이어서 끝난다. - - 한계: lease는 put의 streaming 구간(BEGIN~FINISH)만 막는다. 그 밖에 있던 다른 - tab의 같은 partition put은 journal row가 지워져 다음 journal 단계에서 - fencing 검사에 걸려 실패할 것으로 본다. 이 경합은 실행으로 확인하지 않았다. - purge가 시작된 뒤 새로 시작한 put의 데이터는 다른 purge처럼 남는다 - (point-in-time). purge 전에 그 partition의 쓰기를 멈추는 것이 composition의 - 몫이다. + - 다른 tab의 put·delete와의 순서: 모든 put·delete는 journal `begin`부터 끝날 + 때까지 partition lock(Web Lock + `:partition:::`)을 + shared로 쥐고, partition 삭제는 디렉터리·journal 두 단계를 그 lock을 + exclusive로 쥔 채 한다. 그래서 진행 중이던 put은 삭제 전에 끝나고 삭제가 + 그 결과까지 지운다. 삭제가 기다리는 동안 새로 온 put은 삭제 뒤에 돈다 + ([0929] 05-01, `opfs-byte-store-adapter.ts` `withPartitionLock`). + origin lease는 put의 BEGIN~FINISH만 막으므로 이 lock이 없을 때는, 삭제의 + 디렉터리 단계가 다른 tab put의 BEGIN보다 먼저 끝나면 그 put이 새 디렉터리에 + 파일을 쓰고 journal 단계가 그 row를 지워 파일을 치울 주체가 없었다(commit + 전이면 manifest·staging receipt·chunk, commit 뒤면 manifest·chunk가 남음). + 삭제 전체가 put의 FINISH~commit 사이에 들어가는 경우는 그때도 put이 + `CONFLICT`로 실패하고 bytes가 남지 않았다. 막는 테스트: + `tests/unit/opfs-worker-runtime.test.ts` "leaves no partition bytes when an + approved purge meets another tab's put parked before …"(5가지 순서). + - 한계: 삭제는 진행 중인 put·delete를 기다린다. source가 멈춘 put은 put idle + bound(05-07)까지 삭제를 붙잡는다. purge의 `signal`을 abort하면 기다림을 + 멈추고 `ABORTED`를 반환한다. 삭제가 lock을 요청한 뒤 온 put의 데이터는 다른 + purge처럼 남는다(point-in-time). purge 전에 그 partition의 쓰기를 멈추는 + 것이 composition의 몫이다. - lock/worker 문제면 OPFS write off → approved capped fallback 또는 online-only - scope binding mismatch면 affected scope를 read-only로 격리하고 registry/ composition을 forward fix; 다른 token path로 bytes를 이동하거나 추측 복구 금지 diff --git a/src/adapters/storage/opfs/opfs-byte-store-adapter.ts b/src/adapters/storage/opfs/opfs-byte-store-adapter.ts index 14afe5ab..0f68c3f6 100644 --- a/src/adapters/storage/opfs/opfs-byte-store-adapter.ts +++ b/src/adapters/storage/opfs/opfs-byte-store-adapter.ts @@ -101,6 +101,8 @@ export type OpfsByteStoreDependencies = Readonly<{ * holds an exclusive per-transaction lock from journal `begin` until it * returns; reconcile only touches rows whose lock it can take at once, so a * live transaction in any tab is never cleaned up underneath its owner. + * [0929] 05-01. It also holds the partition lock shared over the same span; + * a partition delete takes that lock exclusive (`withPartitionLock`). */ lockManager?: Pick; now?: () => number; @@ -914,23 +916,30 @@ export function createOpfsByteStoreAdapter( // under the origin lease and then the partition's journal rows. Both // are keyed by this partition's tokens alone; no other partition is // touched. Like that delete, this step is not bounded by the budget. + // Both steps run under the exclusive partition lock, so they first + // wait for every put and delete of the partition in any tab. const partitionDeleted = sweeping && (corruptObjects > 0 || damaged) && !request.signal?.aborted; if (partitionDeleted) { - const removedFiles = await dependencies.worker.removePartition( - dependencies.scope, + const removed = await withPartitionLock( + "exclusive", + "OBJECT_RECONCILE", request.signal, + async () => { + const removedFiles = await dependencies.worker.removePartition( + dependencies.scope, + request.signal, + ); + if (!removedFiles.ok) return removedFiles; + return await dependencies.journal.removePartition( + dependencies.scope, + ); + }, ); - if (!removedFiles.ok) { - return rebaseFailure(removedFiles.error, "OBJECT_RECONCILE"); - } - const removedRows = await dependencies.journal.removePartition( - dependencies.scope, - ); - if (!removedRows.ok) { - return rebaseFailure(removedRows.error, "OBJECT_RECONCILE"); + if (!removed.ok) { + return rebaseFailure(removed.error, "OBJECT_RECONCILE"); } chunkSweep = undefined; moreObjectsAvailable = false; @@ -1106,12 +1115,49 @@ export function createOpfsByteStoreAdapter( // A fresh transaction id is never held; a holder means a duplicate id. if (!release) return browserDataFailure("CONFLICT", operation); try { - return await run(); + return await withPartitionLock("shared", operation, undefined, run); } finally { release(); } } + /** + * [0929] 05-01. Every put and delete of this partition holds this lock + * shared from journal `begin` until it returns, and the partition delete + * holds it exclusive over both of its steps, so a mutation lies wholly + * before or wholly after that delete. The origin lease covers a put only + * from BEGIN to FINISH: without this, a put begun in another tab wrote its + * files after the directory went, the second step took its journal row, and + * nothing was left to clean those files. Tested by "leaves no partition + * bytes when an approved purge meets another tab's put parked before …". + */ + async function withPartitionLock( + mode: LockMode, + operation: BrowserDataOperation, + signal: AbortSignal | undefined, + run: () => Promise>, + ): Promise> { + let granted = false; + try { + return await requestLock( + `${policy.mutationLockName}:partition:${scope.authorityToken}:${scope.namespaceToken}:${scope.partitionToken}`, + { mode, ...(signal ? { signal } : {}) }, + async () => { + granted = true; + return await run(); + }, + ); + } catch (error) { + if (granted) throw error; + return signal?.aborted + ? browserDataFailure("ABORTED", operation) + : browserDataFailure("UNAVAILABLE", operation, { + retryable: true, + recovery: "RETRY", + }); + } + } + /** * STO-01. The compensating half of the put saga. * diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 07059c1a..e3514111 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -2651,8 +2651,169 @@ describe("05-04 a LOGOUT or ACCOUNT_DELETION purge leaves no partition bytes beh expect(root.has(partitionPath(scopeP2))).toBe(true); expect(leases.held).toBe(false); }); + + // [0929] 05-01. The origin lease covers a put only from BEGIN to FINISH, and + // the partition delete runs in two steps: the directory under that lease, + // then the journal rows. Another tab's put that had begun before the purge + // was not checked once it left the lease. Each case parks that put at a + // journal step past FINISH, and optionally lets the purge remove the + // directory before the put reaches BEGIN, so the rows go while it is parked. + it.each([ + ["markFilesReady", false], + ["commitPut", false], + ["markFilesReady", true], + ["commitPut", true], + ["complete", true], + ] as const)( + "leaves no partition bytes when an approved purge meets another tab's put parked before %s (directory removed before its BEGIN: %s)", + async (parkedStep, directoryFirst) => { + const { root, leases, idb, tab } = sharedRootTabs(); + const runtime = () => + createOpfsWorkerRuntime({ + root: root as unknown as FileSystemDirectoryHandle, + crypto: globalThis.crypto, + policy: runtimePolicy, + leaseManager: leases, + dedicatedWorker: true, + supportsSynchronousAccessHandles: false, + }); + const journal = createIndexedDbOpfsJournal({ + authorityToken: scopeA.authorityToken, + factory: idb.factory, + keyRange: idb.keyRange, + }); + const beforeBegin = checkpoint(); + const parked = checkpoint(); + const rowsPending = checkpoint(); + const writerWorker = loopbackGateway(runtime()); + const writer = tab("w", { + journal: { + ...journal, + [parkedStep]: async ( + ...input: Parameters + ) => { + await parked.pass(); + return await ( + journal[parkedStep] as (...args: typeof input) => unknown + )(...input); + }, + } as OpfsJournalPort, + worker: { + ...writerWorker, + async preparePut(input) { + if (directoryFirst) await beforeBegin.pass(); + return await writerWorker.preparePut(input); + }, + }, + }); + const purgerWorker = loopbackGateway(runtime()); + const purger = tab("p", { + ...authority, + worker: { + ...purgerWorker, + async removePartition(scope, signal) { + const removed = await purgerWorker.removePartition(scope, signal); + await rowsPending.pass(); + return removed; + }, + }, + }); + expect(await putBytes(purger, "object_secret_1", [1, 2, 3])).toMatchObject({ ok: true }); + idb.seed("opfs-objects", corruptObjectRow(scopeA, "object_secret_2")); + const settle = (step: Promise) => + Promise.race([ + step, + new Promise((resolve) => setTimeout(resolve, 100)), + ]); + + const put = putBytes(writer, "object_inflight1", [8, 8]); + let purge: ReturnType; + if (directoryFirst) { + // The put's journal row is PREPARING; it has written nothing yet. + await beforeBegin.reached; + purge = purger.maintenance.enforcePolicies({ reason: "ACCOUNT_DELETION" }); + await settle(rowsPending.reached); + beforeBegin.open(); + await parked.reached; + } else { + await parked.reached; + purge = purger.maintenance.enforcePolicies({ reason: "ACCOUNT_DELETION" }); + } + rowsPending.open(); + await settle(purge); + parked.open(); + const [report, written] = await Promise.all([purge, put]); + + // The put lies wholly before the partition delete, which then takes it. + expect(written).toMatchObject({ ok: true, value: { generation: 1 } }); + expect(report).toMatchObject({ + ok: true, + value: { partitionDeleted: true, moreObjectsAvailable: false }, + }); + expect(filesUnder(root, scopeA)).toEqual([]); + expect(idb.readRaw("opfs-journal", "transaction_w_1")).toBeUndefined(); + expect(await openedBytes(writer, "object_inflight1")).toBe("NOT_FOUND"); + }, + ); + + it("stops a partition delete that waits on another tab's put when the purge is aborted", async () => { + const { root, idb, tab } = sharedRootTabs(); + const purger = tab("p", authority); + expect(await putBytes(purger, "object_secret_1", [1, 2, 3])).toMatchObject({ ok: true }); + idb.seed("opfs-objects", corruptObjectRow(scopeA, "object_secret_2")); + const lockName = `${runtimePolicy.mutationLockName}:partition:${scopeA.authorityToken}:${scopeA.namespaceToken}:${scopeA.partitionToken}`; + const putHolding = checkpoint(); + // Stands in for a put of this partition in another tab. + const holder = navigator.locks.request(lockName, { mode: "shared" }, () => + putHolding.pass(), + ); + await putHolding.reached; + const controller = new AbortController(); + + const purge = purger.maintenance.enforcePolicies({ + reason: "ACCOUNT_DELETION", + signal: controller.signal, + }); + while ( + !(await navigator.locks.query()).pending?.some( + (lock) => lock.name === lockName && lock.mode === "exclusive", + ) + ) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + controller.abort(); + + expect(await purge).toMatchObject({ ok: false, error: { code: "ABORTED" } }); + expect(filesUnder(root, scopeA)).not.toEqual([]); + putHolding.open(); + await holder; + }); }); +/** A step that parks until opened and says when something reached it. */ +function checkpoint(): Readonly<{ + reached: Promise; + pass: () => Promise; + open: () => void; +}> { + let reach!: () => void; + const reached = new Promise((resolve) => { + reach = resolve; + }); + let open!: () => void; + const opened = new Promise((resolve) => { + open = resolve; + }); + return { + reached, + open, + async pass() { + reach(); + await opened; + }, + }; +} + describe("05-07 a put whose source stops lets go of the origin lock", () => { afterEach(() => { vi.useRealTimers();