diff --git a/docs/operations/browser-file-storage-recovery.md b/docs/operations/browser-file-storage-recovery.md index 8a3191d2..9a99dff5 100644 --- a/docs/operations/browser-file-storage-recovery.md +++ b/docs/operations/browser-file-storage-recovery.md @@ -34,11 +34,18 @@ cleanup 결과를 확인하지 못하면 성공 값이 아니라 실패로 답 않는다(05-01, `reconcile`의 `holdTransactionLock`). 결함이 아니다. - `failedTransactions`: 이번에 정리하지 못해 journal에 남긴 row 수다. 나머지 row와 orphan GC는 계속 돈다(05-02). 같은 row가 계속 실패하면 §7을 본다. + - `corruptTransactions`: 읽을 수 없어 건너뛴 journal row 수다. 0이 아니면 + 저장소 손상이다. 예전에는 이런 row 하나가 목록 전체를 실패시켜 authority의 + 모든 partition에서 reconcile과 GC가 멈췄다. 이제 나머지 row와 orphan GC는 + 계속 돌고, 관측은 `OBJECT_RECONCILE` `DEGRADED` / `CORRUPT_DATA`로 남는다 + ([0929] 05-01). reconcile은 그 row를 지우지 않는다. 그 row가 속한 + partition이 승인된 purge로 통째로 지워질 때만 함께 지워진다(§7). 3. journal row를 수동으로 삭제하지 않는다. row가 사라지면 stale staging을 추적할 근거가 사라지고 quota만 누수된다. 4. OPFS root나 journal database를 통째로 삭제하거나 schema를 downgrade하지 않는다. rollback은 새 v2 write admission을 닫고 v1+v2 reader를 유지하는 것으로 - 수행한다. + 수행한다. runtime이 스스로 지우는 것은 승인된 purge가 손상 때문에 끝나지 못할 + 때의 그 partition 디렉터리 하나와 그 partition의 journal row뿐이다(§7). ## 1. 공통 원칙 @@ -50,6 +57,9 @@ incident 중에도 다음 작업은 금지한다. 하나만 지운다(§5 "database 전체 삭제" 참고). - origin의 모든 `caches.keys()` 삭제 - user-authored/unsynced IndexedDB 또는 OPFS data 자동 purge + - OPFS의 예외도 하나다. authority가 승인한 LOGOUT/ACCOUNT_DELETION이 손상된 + journal row나 staging 때문에 끝나지 못할 때, runtime이 그 partition + 디렉터리 하나와 그 partition의 journal row만 지운다(§7 recovery). - schema version downgrade - 무한 reload/update loop - filename, path, record/object key, URL/query, digest, body를 incident log에 복사 @@ -436,6 +446,28 @@ VD-15 목표 runtime을 먼저 구현한다. 처리한다. 다른 탭이 바꾸고 있던 객체를 지우지 못하면 report는 `moreObjectsAvailable: true`이고 `chunkSweep`이 없다. 이때도 같은 purge를 다시 호출한다([0929] 05-02) +- 손상 때문에 끝나지 못하는 `LOGOUT`/`ACCOUNT_DELETION` purge는 partition을 + 통째로 지운다([0929] 05-01, `opfs-byte-store-adapter.ts` `enforcePolicies`). + - 조건: 정책이 그 reason을 허용하고, authority가 이번 purge의 proof를 내주고 + consumer가 소비했을 때(IndexedDB의 database 전체 삭제와 같은 승인). 그리고 + 다음 중 하나를 만났을 때: 목록에서 건너뛴 손상 object row, 손상 row 때문에 + `CORRUPT_DATA`로 끝난 remove나 chunk sweep, `STAGING_STATE_UNREADABLE`. + TTL·SESSION_END·UNTIL_SYNCED·PRESSURE는 손상 row를 건너뛰고 세기만 한다. + - 범위: worker가 origin mutation lease를 쥔 채 + `authorities///` 디렉터리를 + 지우고, 이어서 journal에서 그 partition 키로 시작하는 object·journal·chunk + reference row와 budget row를 지운다. 값을 읽지 않고 키로 찾으므로 손상 row도 + 지워진다. scope binding row는 남긴다. 다른 partition은 건드리지 않는다. + - 결과: `partitionDeleted: true`, `moreObjectsAvailable: false`, 건너뛴 row 수 + `corruptObjects`. 이 report면 그 partition의 bytes는 모두 지워진 것이다. + `chunkSweep`은 없다. 디렉터리는 지웠는데 journal 정리가 실패하면 실패를 + 반환한다. 같은 purge를 다시 호출하면 이어서 끝난다. + - 한계: lease는 put의 streaming 구간(BEGIN~FINISH)만 막는다. 그 밖에 있던 다른 + tab의 같은 partition put은 journal row가 지워져 다음 journal 단계에서 + fencing 검사에 걸려 실패할 것으로 본다. 이 경합은 실행으로 확인하지 않았다. + purge가 시작된 뒤 새로 시작한 put의 데이터는 다른 purge처럼 남는다 + (point-in-time). purge 전에 그 partition의 쓰기를 멈추는 것이 composition의 + 몫이다. - lock/worker 문제면 OPFS write off → approved capped fallback 또는 online-only - scope binding mismatch면 affected scope를 read-only로 격리하고 registry/ composition을 forward fix; 다른 token path로 bytes를 이동하거나 추측 복구 금지 diff --git a/src/adapters/storage/opfs/indexeddb-opfs-journal.ts b/src/adapters/storage/opfs/indexeddb-opfs-journal.ts index dd8fb8ce..e215e086 100644 --- a/src/adapters/storage/opfs/indexeddb-opfs-journal.ts +++ b/src/adapters/storage/opfs/indexeddb-opfs-journal.ts @@ -698,6 +698,7 @@ export function createIndexedDbOpfsJournal( "INDEXEDDB_READ", (transaction, context) => { const rows: OpfsJournalTransaction[] = []; + let corruptRows = 0; walkIndexedDbCursor({ request: transaction .objectStore(JOURNAL_STORE) @@ -708,21 +709,18 @@ export function createIndexedDbOpfsJournal( // OP-4. Neither a signal nor a budget is passed: the journal // offers no cancellation and bounds the page on `limit` alone. visit: ({ cursor }) => { - // The row is validated before the page limit is checked, so a - // corrupt row one past the page still fails the read rather - // than being hidden behind `moreAvailable`. - if ( - !isStoredJournalRow(cursor.value) || - !isBoundScope(cursor.value.scope) - ) { - context.fail(corrupt("INDEXEDDB_READ")); - // `fail` aborts, so the pump has nowhere to go. Suspending - // without ever resuming says that without claiming a summary - // the scan never reached. - return { kind: "SUSPEND" }; - } if (rows.length === limit) return { kind: "STOP" }; - rows.push(cursor.value); + // [0929] 05-01. One unreadable row used to fail the read, which + // stopped reconcile and GC for the whole authority. It is + // skipped and counted; nothing here deletes it. + if ( + isStoredJournalRow(cursor.value) && + isBoundScope(cursor.value.scope) + ) { + rows.push(cursor.value); + } else { + corruptRows += 1; + } return { kind: "CONTINUE" }; }, done: (summary) => { @@ -731,6 +729,7 @@ export function createIndexedDbOpfsJournal( transactions: Object.freeze(rows), moreAvailable: summary.reason === "STOPPED" && rows.length === limit, + corruptRows, }), ); }, @@ -774,6 +773,7 @@ export function createIndexedDbOpfsJournal( false, ); const objects: OpfsPreparedObject[] = []; + let corruptRows = 0; walkIndexedDbCursor({ request: transaction .objectStore(OBJECT_STORE) @@ -782,20 +782,21 @@ export function createIndexedDbOpfsJournal( sink: context, translate: translateFor("INDEXEDDB_READ"), visit: ({ cursor }) => { + if (objects.length === request.limit) { + return { kind: "STOP" }; + } + // [0929] 05-01. Skipped and counted, as in `listIncomplete`. if ( - !isStoredObjectRow(cursor.value) || - !sameScope( + isStoredObjectRow(cursor.value) && + sameScope( cursor.value.preparedObject.descriptor.scope, request.scope, ) ) { - context.fail(corrupt("INDEXEDDB_READ")); - return { kind: "SUSPEND" }; + objects.push(cursor.value.preparedObject); + } else { + corruptRows += 1; } - if (objects.length === request.limit) { - return { kind: "STOP" }; - } - objects.push(cursor.value.preparedObject); return { kind: "CONTINUE" }; }, done: (summary) => { @@ -811,6 +812,7 @@ export function createIndexedDbOpfsJournal( ? objects.at(-1)?.descriptor.objectId ?? null : null, moreAvailable: more, + corruptRows, }), ); }, @@ -862,6 +864,62 @@ export function createIndexedDbOpfsJournal( ); }, + async removePartition(scope) { + if ( + !keyRange || + !isBoundScope(scope) || + !isValidOpfsStorageScope(scope) + ) { + return browserDataFailure( + keyRange ? "INVALID_INPUT" : "UNSUPPORTED", + "INDEXEDDB_WRITE", + ); + } + return await withDatabase("INDEXEDDB_WRITE", (db) => + runTransaction( + db, + [JOURNAL_STORE, OBJECT_STORE, BUDGET_STORE, CHUNK_REFERENCE_STORE], + "readwrite", + "INDEXEDDB_WRITE", + (transaction, context) => { + // [0929] 05-01. Rows are found by key, never by value, so a row + // nothing can read still goes. Tokens cannot hold `|`, so this + // prefix matches this one partition only. + const scopeKey = storageScopeKey(scope); + const range = keyRange.bound( + `${scopeKey}|`, + `${scopeKey}|￿`, + ); + // The transaction's commit publishes this; a failed delete aborts + // it and replaces it. + context.succeed(undefined); + watchWrite( + transaction.objectStore(BUDGET_STORE).delete(scopeKey), + context, + ); + for (const [storeName, indexName] of [ + [JOURNAL_STORE, LOGICAL_KEY_INDEX], + [OBJECT_STORE], + [CHUNK_REFERENCE_STORE], + ] as const) { + const store = transaction.objectStore(storeName); + walkIndexedDbCursor({ + request: (indexName ? store.index(indexName) : store) + .openCursor(range), + sink: context, + translate: translateFor("INDEXEDDB_WRITE"), + visit: ({ cursor }) => { + watchWrite(store.delete(cursor.primaryKey), context); + return { kind: "CONTINUE" }; + }, + done: () => {}, + }); + } + }, + ), + ); + }, + close() { connection.close(); }, diff --git a/src/adapters/storage/opfs/opfs-byte-store-adapter.ts b/src/adapters/storage/opfs/opfs-byte-store-adapter.ts index 1aab66bd..14afe5ab 100644 --- a/src/adapters/storage/opfs/opfs-byte-store-adapter.ts +++ b/src/adapters/storage/opfs/opfs-byte-store-adapter.ts @@ -614,6 +614,7 @@ export function createOpfsByteStoreAdapter( deadlineReached: boolean; skippedLiveTransactions: number; failedTransactions: number; + corruptTransactions: number; } = { inspectedTransactions: 0, committedTransactions: 0, @@ -626,6 +627,7 @@ export function createOpfsByteStoreAdapter( deadlineReached: false, skippedLiveTransactions: 0, failedTransactions: 0, + corruptTransactions: 0, }; // 05-01 (D-3). A row is reconciled only while this call holds its // per-transaction lock, so a put or delete still running in any tab is @@ -654,6 +656,8 @@ export function createOpfsByteStoreAdapter( return rebaseFailure(page.error, "OBJECT_RECONCILE"); } report.moreTransactionsAvailable = page.value.moreAvailable; + // [0929] 05-01. The journal skipped these rows; they stop nothing now. + report.corruptTransactions = page.value.corruptRows; for (const transaction of page.value.transactions) { if (!held.has(transaction.transactionId)) continue; if (request.signal?.aborted) { @@ -698,8 +702,11 @@ export function createOpfsByteStoreAdapter( } observeOpfsSafely(dependencies.observer, { operation: "OBJECT_RECONCILE", - outcome: "SUCCEEDED", transactionBucket: transactionBucket(report.inspectedTransactions), + // A skipped corrupt row is observed, not hidden behind a success. + ...(report.corruptTransactions + ? { outcome: "DEGRADED", failureCode: "CORRUPT_DATA" } + : { outcome: "SUCCEEDED" }), }); return browserDataSuccess( Object.freeze({ ...report }), @@ -781,6 +788,13 @@ export function createOpfsByteStoreAdapter( let releasedBytes = 0; let moreObjectsAvailable = false; let deadlineReached = false; + const sweeping = + request.reason === "LOGOUT" || request.reason === "ACCOUNT_DELETION"; + // [0929] 05-01. Rows the journal could not read and skipped, and damage + // met past the listing: a remove or the sweep failing on an unreadable + // row, or staging the sweep cannot read. + let corruptObjects = 0; + let damaged = false; for (;;) { // 1101/13.6. Only stopping before a page is read leaves objects @@ -801,6 +815,7 @@ export function createOpfsByteStoreAdapter( if (!page.ok) { return rebaseFailure(page.error, "OBJECT_RECONCILE"); } + corruptObjects += page.value.corruptRows; const candidates = [...page.value.objects].sort( policyMaintenanceOrder, ); @@ -825,11 +840,16 @@ export function createOpfsByteStoreAdapter( if (removed.error.code === "CONFLICT" || removed.error.code === "NOT_FOUND") { // [0929] 05-02. Another context changed this object, so it may // still be here: report more work and skip the chunk sweep. No - // later assignment clears this — the last-page one ORs, and the - // PRESSURE one only runs for a purge that never sweeps. + // later assignment clears this — the last-page one ORs, the + // PRESSURE one only runs for a purge that never sweeps, and a + // partition delete leaves no object behind. moreObjectsAvailable = true; continue; } + if (sweeping && removed.error.code === "CORRUPT_DATA") { + damaged = true; + continue; + } return rebaseFailure(removed.error, "OBJECT_RECONCILE"); } removedObjects += 1; @@ -861,8 +881,9 @@ export function createOpfsByteStoreAdapter( // staged put references. let chunkSweep: OpfsPolicyMaintenanceReport["chunkSweep"]; if ( - (request.reason === "LOGOUT" || - request.reason === "ACCOUNT_DELETION") && + sweeping && + !corruptObjects && + !damaged && !moreObjectsAvailable && !deadlineReached && !request.signal?.aborted @@ -872,13 +893,48 @@ export function createOpfsByteStoreAdapter( request.signal, now(), ); - if (!swept.ok) return rebaseFailure(swept.error, "OBJECT_RECONCILE"); - chunkSweep = swept.value.status; - if (chunkSweep === "DEADLINE_REACHED") { - deadlineReached = true; - moreObjectsAvailable = true; + if (!swept.ok) { + if (swept.error.code !== "CORRUPT_DATA") { + return rebaseFailure(swept.error, "OBJECT_RECONCILE"); + } + damaged = true; + } else { + chunkSweep = swept.value.status; + damaged = chunkSweep === "STAGING_STATE_UNREADABLE"; + if (chunkSweep === "DEADLINE_REACHED") { + deadlineReached = true; + moreObjectsAvailable = true; + } } } + // [0929] 05-01. A row nothing can read never lets this purge finish, so + // the user's bytes would stay on the device for good. The authority + // approved this exact LOGOUT or ACCOUNT_DELETION (the same approval the + // IndexedDB dataset delete runs on), so the partition directory goes + // under the origin lease and then the partition's journal rows. Both + // are keyed by this partition's tokens alone; no other partition is + // touched. Like that delete, this step is not bounded by the budget. + const partitionDeleted = + sweeping && + (corruptObjects > 0 || damaged) && + !request.signal?.aborted; + if (partitionDeleted) { + const removedFiles = await dependencies.worker.removePartition( + dependencies.scope, + request.signal, + ); + if (!removedFiles.ok) { + return rebaseFailure(removedFiles.error, "OBJECT_RECONCILE"); + } + const removedRows = await dependencies.journal.removePartition( + dependencies.scope, + ); + if (!removedRows.ok) { + return rebaseFailure(removedRows.error, "OBJECT_RECONCILE"); + } + chunkSweep = undefined; + moreObjectsAvailable = false; + } const report: OpfsPolicyMaintenanceReport = Object.freeze({ inspectedObjects, removedObjects, @@ -886,6 +942,8 @@ export function createOpfsByteStoreAdapter( moreObjectsAvailable, deadlineReached, ...(chunkSweep ? { chunkSweep } : {}), + ...(corruptObjects ? { corruptObjects } : {}), + ...(partitionDeleted ? { partitionDeleted } : {}), }); return browserDataSuccess(report); }, @@ -1506,6 +1564,7 @@ function snapshotOpfsJournal( listIncomplete, listCommittedObjects, isChunkReferenced, + removePartition, } = source; if ( [ @@ -1519,6 +1578,7 @@ function snapshotOpfsJournal( listIncomplete, listCommittedObjects, isChunkReferenced, + removePartition, ].some((method) => typeof method !== "function") ) { throw new TypeError("OPFS journal dependency is invalid."); @@ -1534,6 +1594,7 @@ function snapshotOpfsJournal( listIncomplete: listIncomplete.bind(source), listCommittedObjects: listCommittedObjects.bind(source), isChunkReferenced: isChunkReferenced.bind(source), + removePartition: removePartition.bind(source), }); } @@ -1554,6 +1615,7 @@ function snapshotOpfsWorker( finalizePut, listOrphanCandidates, deleteOrphanChunk, + removePartition, close, } = source; if ( @@ -1568,6 +1630,7 @@ function snapshotOpfsWorker( finalizePut, listOrphanCandidates, deleteOrphanChunk, + removePartition, close, ].some((method) => typeof method !== "function") ) { @@ -1584,6 +1647,7 @@ function snapshotOpfsWorker( finalizePut: finalizePut.bind(source), listOrphanCandidates: listOrphanCandidates.bind(source), deleteOrphanChunk: deleteOrphanChunk.bind(source), + removePartition: removePartition.bind(source), close: close.bind(source), }); } diff --git a/src/adapters/storage/opfs/opfs-worker-client.ts b/src/adapters/storage/opfs/opfs-worker-client.ts index 2dffd10a..ee48df9d 100644 --- a/src/adapters/storage/opfs/opfs-worker-client.ts +++ b/src/adapters/storage/opfs/opfs-worker-client.ts @@ -511,6 +511,17 @@ export function createOpfsWorkerGateway( ); }, + async removePartition( + scope: OpfsPreparedObject["descriptor"]["scope"], + signal?: AbortSignal, + ) { + return await invoke( + "OBJECT_RECONCILE", + { kind: "REMOVE_PARTITION", scope }, + signal, + ); + }, + close() { if (disposed) return; disposed = true; diff --git a/src/adapters/storage/opfs/opfs-worker-protocol.ts b/src/adapters/storage/opfs/opfs-worker-protocol.ts index 69a7892a..fc3fee36 100644 --- a/src/adapters/storage/opfs/opfs-worker-protocol.ts +++ b/src/adapters/storage/opfs/opfs-worker-protocol.ts @@ -140,6 +140,13 @@ export type OpfsWorkerRequest = scope: OpfsStorageScope; digestHex: string; olderThanEpochMs: number; + }> + | Readonly<{ + requestId: string; + protocolVersion: typeof OPFS_WORKER_PROTOCOL_VERSION; + /** [0929] 05-01. Removes `authorities///

` under the lease. */ + kind: "REMOVE_PARTITION"; + scope: OpfsStorageScope; }>; /** @@ -157,6 +164,7 @@ export const OPFS_LOCK_WAITING_REQUEST_KINDS: ReadonlySet< "FINALIZE_PUT", "LIST_ORPHAN_CANDIDATES", "DELETE_ORPHAN_CHUNK", + "REMOVE_PARTITION", ]); export type OpfsWorkerRequestBody = @@ -276,5 +284,14 @@ export interface OpfsWorkerGateway { olderThanEpochMs: number, signal?: AbortSignal, ): Promise>; + /** + * [0929] 05-01. Deletes the whole partition directory while holding the + * origin mutation lease. Only an approved LOGOUT or ACCOUNT_DELETION purge + * that damage kept from finishing calls it. + */ + removePartition( + scope: OpfsStorageScope, + signal?: AbortSignal, + ): Promise>; close(): void; } diff --git a/src/adapters/storage/opfs/opfs-worker-runtime.ts b/src/adapters/storage/opfs/opfs-worker-runtime.ts index 44231cd4..48622533 100644 --- a/src/adapters/storage/opfs/opfs-worker-runtime.ts +++ b/src/adapters/storage/opfs/opfs-worker-runtime.ts @@ -308,6 +308,9 @@ export function createOpfsWorkerRuntime( request.olderThanEpochMs, ), ); + case "REMOVE_PARTITION": + await removePartition(request.scope); + return success(request.requestId, request.kind); } } catch (error) { // STO-RR-02. The kind travels with the failure so the client's @@ -816,6 +819,36 @@ export function createOpfsWorkerRuntime( } } + /** + * [0929] 05-01. The exit an approved purge takes when damage keeps it from + * finishing. Under the origin lease no put is streaming, so nothing is + * half-written into the directory while it goes. + */ + async function removePartition( + scope: OpfsPreparedObject["descriptor"]["scope"], + ): Promise { + assertWorkerAvailable(); + if (!isValidOpfsStorageScope(scope)) { + throw new OpfsRuntimeFailure("INVALID_INPUT"); + } + const lease = await acquireLease(); + try { + await removeEntryIfPresent( + await getDirectory( + dependencies.root, + scopeRootPath(scope).slice(0, -1), + false, + ), + scope.partitionToken, + true, + ); + } catch (error) { + if (!isNotFound(error)) throw error; + } finally { + lease.release(); + } + } + /** * STO-01. Cleanup deletes exactly one transaction's physical generation while * holding the origin mutation lease, and reports whether the effect actually @@ -1892,6 +1925,7 @@ const WORKER_REQUEST_KINDS = new Set([ "FINALIZE_PUT", "LIST_ORPHAN_CANDIDATES", "DELETE_ORPHAN_CHUNK", + "REMOVE_PARTITION", ]); function extractReceiptDigests( diff --git a/src/application/ports/browser-file-storage/opfs-ports.ts b/src/application/ports/browser-file-storage/opfs-ports.ts index 5893d8a0..1d21467e 100644 --- a/src/application/ports/browser-file-storage/opfs-ports.ts +++ b/src/application/ports/browser-file-storage/opfs-ports.ts @@ -125,6 +125,12 @@ export type OpfsReconciliationReport = Readonly<{ * stops the rest of the page or orphan GC. */ failedTransactions: number; + /** + * [0929] 05-01. Journal rows this run could not read. They are skipped, not + * deleted, and no longer stop the other rows or orphan GC. Anything above + * zero is storage damage for an operator to look at. + */ + corruptTransactions: number; }>; export type OpfsPolicyMaintenanceReport = Readonly<{ @@ -141,9 +147,20 @@ export type OpfsPolicyMaintenanceReport = Readonly<{ /** * 05-04. LOGOUT and ACCOUNT_DELETION only, once every object is removed: * how the sweep of the partition's now unreferenced chunks ended. Chunk - * bytes are gone only when it is `COMPLETED`. + * bytes are gone only when it is `COMPLETED`, or when `partitionDeleted`. */ chunkSweep?: "COMPLETED" | "DEADLINE_REACHED" | "STAGING_STATE_UNREADABLE"; + /** + * [0929] 05-01. Committed object rows this run could not read and skipped. + * Present only when there were some. + */ + corruptObjects?: number; + /** + * [0929] 05-01. An approved LOGOUT or ACCOUNT_DELETION that damage kept + * from finishing removed the whole partition directory and the partition's + * journal rows instead. Every byte of the partition is gone. + */ + partitionDeleted?: true; }>; export type OpfsSensitiveMaintenanceReason = @@ -291,15 +308,23 @@ export type BeginOpfsJournalTransaction = Readonly<{ startedAtEpochMs: number; }>; +/** + * [0929] 05-01. A listing skips a row it cannot read and counts it in + * `corruptRows`; it never deletes it. The rows of one page do not include + * skipped ones, so a full page stops before the next row and a row is counted + * once, by the page that reaches it. + */ export type OpfsJournalPage = Readonly<{ transactions: readonly OpfsJournalTransaction[]; moreAvailable: boolean; + corruptRows: number; }>; export type OpfsCommittedObjectPage = Readonly<{ objects: readonly OpfsPreparedObject[]; nextObjectId: string | null; moreAvailable: boolean; + corruptRows: number; }>; export interface OpfsJournalPort { @@ -351,4 +376,13 @@ export interface OpfsJournalPort { scope: OpfsStorageScope, digestHex: string, ): Promise>; + /** + * [0929] 05-01. Deletes every row keyed to this one partition, readable or + * not: committed objects, journal rows, chunk references and the budget. + * Scope bindings stay. Only an approved LOGOUT or ACCOUNT_DELETION purge + * calls it, after the partition's directory is gone. + */ + removePartition( + scope: OpfsStorageScope, + ): Promise>; } diff --git a/tests/unit/indexeddb-opfs-journal.test.ts b/tests/unit/indexeddb-opfs-journal.test.ts index e5d79ee5..daca31de 100644 --- a/tests/unit/indexeddb-opfs-journal.test.ts +++ b/tests/unit/indexeddb-opfs-journal.test.ts @@ -170,8 +170,9 @@ async function commitObject( transactionId: string, objectId: string, byteLength: number, + targetScope = scope, ): Promise { - const input = beginInput(transactionId, objectId, byteLength); + const input = beginInput(transactionId, objectId, byteLength, targetScope); const begun = await journal.begin(input); if (!begun.ok) throw new Error(`begin failed for ${objectId}`); const token = begun.value.fencingToken; @@ -488,7 +489,7 @@ describe("IndexedDB OPFS journal", () => { }); }); - it("reopens after versionchange and rejects corrupt persisted journal rows", async () => { + it("reopens after versionchange and skips a corrupt persisted journal row", async () => { const { factory, journal } = createHarness(); expect( await journal.getCommittedObject(scope, "object_12345678"), @@ -504,9 +505,13 @@ describe("IndexedDB OPFS journal", () => { startedAtEpochMs: 1, corrupt: true, }); - expect(await journal.listIncomplete(10)).toMatchObject({ - ok: false, - error: { code: "CORRUPT_DATA" }, + // [0929] 05-01. The row is counted, not returned and not deleted. + expect(await journal.listIncomplete(10)).toEqual({ + ok: true, + value: { transactions: [], moreAvailable: false, corruptRows: 1 }, + }); + expect(factory.readRaw("opfs-journal", "transaction_88888888")).toMatchObject({ + corrupt: true, }); }); @@ -642,7 +647,7 @@ describe("IndexedDB OPFS journal", () => { expect(tracker.aborts).toEqual([]); }); - it("pages incomplete transactions and validates a row before the page limit", async () => { + it("pages incomplete transactions and counts a corrupt row on the page that reaches it", async () => { const { factory, journal } = createHarness(); for (const index of [1, 2, 3]) { const id = `transaction_page000${index}`; @@ -666,18 +671,27 @@ describe("IndexedDB OPFS journal", () => { expect(whole.value.transactions).toHaveLength(3); expect(whole.value.moreAvailable).toBe(false); - // A corrupt row one past the requested page still fails the read. The - // journal validates a row before it checks the limit, so a page that is - // already full does not hide the next row's damage behind `moreAvailable`. + // [0929] 05-01. A corrupt row no longer fails the read: one bad row used to + // stop reconcile and GC for the whole authority. It is skipped and counted. + // A full page stops before the next row, so a row one past the page is + // counted once, by the page that reaches it, and `moreAvailable` says so. factory.seed("opfs-journal", { transactionId: "transaction_page9999", startedAtEpochMs: 9_000, corrupt: true, }); - expect(await journal.listIncomplete(3)).toMatchObject({ - ok: false, - error: { code: "CORRUPT_DATA", recovery: "READ_ONLY" }, + const full = await journal.listIncomplete(3); + expect(full).toMatchObject({ + ok: true, + value: { moreAvailable: true, corruptRows: 0 }, }); + expect(full.ok && full.value.transactions).toHaveLength(3); + const past = await journal.listIncomplete(10); + expect(past).toMatchObject({ + ok: true, + value: { moreAvailable: false, corruptRows: 1 }, + }); + expect(past.ok && past.value.transactions).toHaveLength(3); }); it("pages committed objects and reports where the next page resumes", async () => { @@ -820,3 +834,90 @@ describe("05-06 a journal from a newer release", () => { }); }); }); + +describe("[0929] 05-01 a corrupt row is skipped and counted; a partition's rows go only together", () => { + const scopeKeyOf = (target: OpfsStorageScope) => + `${target.authorityToken}|${target.namespaceToken}|${target.partitionToken}`; + const corruptObjectRow = (target: OpfsStorageScope, objectId: string) => ({ + logicalKey: `${scopeKeyOf(target)}|${objectId}`, + scopeObjectKey: `${scopeKeyOf(target)}|${objectId}`, + scopeKey: scopeKeyOf(target), + objectId, + preparedObject: { corrupt: true }, + }); + + it("lists the committed objects past a corrupt row, counts it and keeps it", async () => { + const { factory, journal } = createHarness(); + await commitObject(journal, "transaction_obj00001", "object_aaaa0001", 1); + await commitObject(journal, "transaction_obj00002", "object_bbbb0002", 1); + factory.seed("opfs-objects", corruptObjectRow(scope, "object_aaaa0001")); + + const page = await journal.listCommittedObjects({ scope, limit: 10 }); + expect(page).toMatchObject({ + ok: true, + value: { corruptRows: 1, moreAvailable: false, nextObjectId: null }, + }); + expect( + page.ok && page.value.objects.map((o) => o.descriptor.objectId), + ).toEqual(["object_bbbb0002"]); + expect( + factory.readRaw("opfs-objects", `${scopeKeyOf(scope)}|object_aaaa0001`), + ).toMatchObject({ preparedObject: { corrupt: true } }); + }); + + it("removes every row keyed to one partition, the corrupt one included, and none of another", async () => { + const { factory, journal } = createHarness(); + await commitObject(journal, "transaction_obj00001", "object_aaaa0001", 1); + await commitObject(journal, "transaction_obj00002", "object_bbbb0002", 1); + await commitObject( + journal, + "transaction_obj00003", + "object_kept_0003", + 1, + otherPartition, + ); + expect( + await journal.begin( + beginInput("transaction_live0004", "object_cccc0004", 1), + ), + ).toMatchObject({ ok: true }); + factory.seed("opfs-objects", corruptObjectRow(scope, "object_aaaa0001")); + + expect(await journal.removePartition(scope)).toEqual({ + ok: true, + value: undefined, + }); + + expect(await journal.listCommittedObjects({ scope, limit: 10 })).toEqual({ + ok: true, + value: { + objects: [], + nextObjectId: null, + moreAvailable: false, + corruptRows: 0, + }, + }); + expect( + factory.readRaw("opfs-journal", "transaction_live0004"), + ).toBeUndefined(); + expect(factory.readRaw("opfs-budgets", scopeKeyOf(scope))).toBeUndefined(); + expect(await journal.isChunkReferenced(scope, "b".repeat(64))).toEqual({ + ok: true, + value: false, + }); + + const kept = await journal.listCommittedObjects({ + scope: otherPartition, + limit: 10, + }); + expect( + kept.ok && kept.value.objects.map((o) => o.descriptor.objectId), + ).toEqual(["object_kept_0003"]); + expect( + await journal.isChunkReferenced(otherPartition, "b".repeat(64)), + ).toEqual({ ok: true, value: true }); + expect( + factory.readRaw("opfs-budgets", scopeKeyOf(otherPartition)), + ).toMatchObject({ committedBytes: 1 }); + }); +}); diff --git a/tests/unit/opfs-byte-store.test.ts b/tests/unit/opfs-byte-store.test.ts index 68653d46..2467ad41 100644 --- a/tests/unit/opfs-byte-store.test.ts +++ b/tests/unit/opfs-byte-store.test.ts @@ -217,6 +217,7 @@ function createJournal(): OpfsJournalPort & { const page: OpfsJournalPage = { transactions: all.slice(0, limit), moreAvailable: all.length > limit, + corruptRows: 0, }; return browserDataSuccess(page); }, @@ -232,6 +233,7 @@ function createJournal(): OpfsJournalPort & { objects: values.slice(0, request.limit), nextObjectId: null, moreAvailable: values.length > request.limit, + corruptRows: 0, }); }, async isChunkReferenced(_scope, digestHex) { @@ -241,6 +243,11 @@ function createJournal(): OpfsJournalPort & { ), ); }, + async removePartition() { + objects.clear(); + transactions.clear(); + return browserDataSuccess(undefined); + }, }; } @@ -310,6 +317,10 @@ function createWorker( skippedUnsafe: false, }); }, + async removePartition() { + physical.clear(); + return browserDataSuccess(undefined); + }, close() {}, }; return Object.assign(worker, overrides); @@ -695,7 +706,11 @@ describe("OPFS byte-store coordinator", () => { preparedObject: committedObject("object_12345678", storagePolicy, 100), }); vi.spyOn(journal, "listIncomplete").mockResolvedValueOnce( - browserDataSuccess({ transactions: [row], moreAvailable: false }), + browserDataSuccess({ + transactions: [row], + moreAvailable: false, + corruptRows: 0, + }), ); const worker = createWorker(); const cleanupTransaction = vi.spyOn(worker, "cleanupTransaction"); diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 8aa17744..07059c1a 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -649,8 +649,13 @@ function referenceJournal(referenced: ReadonlySet): OpfsJournalPort { complete: unused, rollback: unused, listCommittedObjects: unused, + removePartition: unused, async listIncomplete() { - return browserDataSuccess({ transactions: [], moreAvailable: false }); + return browserDataSuccess({ + transactions: [], + moreAvailable: false, + corruptRows: 0, + }); }, async isChunkReferenced(_scope, digestHex) { return browserDataSuccess(referenced.has(digestHex)); @@ -1105,6 +1110,7 @@ describe("R-5.5 stored objects are checked against their own chunk size", () => return browserDataSuccess({ transactions: [transaction], moreAvailable: false, + corruptRows: 0, }); }, async commitPut() { @@ -2473,6 +2479,178 @@ describe("05-04 a LOGOUT or ACCOUNT_DELETION purge leaves no partition bytes beh }); expect(filesUnder(root, scopeA)).toEqual([]); }); + + // [0929] 05-01. A journal row the purge cannot read used to fail the whole + // purge, so not even the readable objects went and the partition's bytes + // stayed for good. The row is now skipped and counted, and an approved + // LOGOUT or ACCOUNT_DELETION that cannot finish because of it removes the + // partition directory and that partition's journal rows instead. + const corruptObjectRow = (scope: OpfsStorageScope, objectId: string) => { + const scopeKey = [ + scope.authorityToken, + scope.namespaceToken, + scope.partitionToken, + ].join("|"); + return { + logicalKey: `${scopeKey}|${objectId}`, + scopeObjectKey: `${scopeKey}|${objectId}`, + scopeKey, + objectId, + preparedObject: { corrupt: true }, + }; + }; + + it.each(["LOGOUT", "ACCOUNT_DELETION"] as const)( + "removes the partition when a corrupt object row stops an approved %s purge", + async (reason) => { + const { root, idb, tab } = sharedRootTabs(); + const purged = tab("p", authority); + const kept = tab("k", { ...authority, scope: scopeP2 }); + expect(await putBytes(purged, "object_secret_1", [1, 2, 3])).toMatchObject({ ok: true }); + expect(await putBytes(purged, "object_secret_2", [4, 5])).toMatchObject({ ok: true }); + expect(await putBytes(kept, "object_kept_0001", [7, 7])).toMatchObject({ ok: true }); + idb.seed("opfs-objects", corruptObjectRow(scopeA, "object_secret_2")); + + const report = await purged.maintenance.enforcePolicies({ reason }); + expect(report).toEqual({ + ok: true, + value: { + inspectedObjects: 1, + removedObjects: 1, + releasedBytes: 3, + moreObjectsAvailable: false, + deadlineReached: false, + corruptObjects: 1, + partitionDeleted: true, + }, + }); + expect(filesUnder(root, scopeA)).toEqual([]); + expect(await openedBytes(kept, "object_kept_0001")).toEqual({ + generation: 1, + bytes: [7, 7], + }); + // The partition starts over: its journal rows went with its files. + expect(await putBytes(purged, "object_secret_2", [9])).toMatchObject({ + ok: true, + value: { generation: 1 }, + }); + expect(await openedBytes(purged, "object_secret_2")).toEqual({ + generation: 1, + bytes: [9], + }); + }, + ); + + it("never removes a corrupt partition for a purge the authority refused", async () => { + const { root, idb, tab } = sharedRootTabs(); + const writer = tab("w", authority); + expect(await putBytes(writer, "object_secret_1", [1, 2, 3])).toMatchObject({ ok: true }); + expect(await putBytes(writer, "object_secret_2", [4, 5])).toMatchObject({ ok: true }); + idb.seed("opfs-objects", corruptObjectRow(scopeA, "object_secret_2")); + const refused = tab("r", { + requestMaintenanceAuthority: () => ({ authorized: false as const }), + consumeMaintenanceAuthority: () => true, + }); + + expect( + await refused.maintenance.enforcePolicies({ reason: "ACCOUNT_DELETION" }), + ).toMatchObject({ ok: false, error: { code: "POLICY_REJECTED" } }); + expect(filesUnder(root, scopeA)).not.toEqual([]); + expect(await openedBytes(writer, "object_secret_1")).toEqual({ + generation: 1, + bytes: [1, 2, 3], + }); + }); + + it("reconciles the other rows and sweeps orphans past a corrupt journal row", async () => { + const { root, idb, tab } = sharedRootTabs(); + const store = tab("r"); + expect(await putBytes(store, "object_secret_1", [1, 2, 3])).toMatchObject({ ok: true }); + const journal = createIndexedDbOpfsJournal({ + authorityToken: scopeA.authorityToken, + factory: idb.factory, + keyRange: idb.keyRange, + }); + // A put that died before it wrote anything leaves a PREPARING row. + expect( + await journal.begin({ + transactionId: "transaction_left_0001", + mutation: "PUT", + scope: scopeA, + objectId: "object_left_0001", + expectedGeneration: null, + targetGeneration: 1, + targetByteLength: 1, + targetStoragePolicy: storagePolicy, + startedAtEpochMs: 2, + }), + ).toMatchObject({ ok: true }); + idb.seed("opfs-journal", { + transactionId: "transaction_corrupt1", + startedAtEpochMs: 1, + corrupt: true, + }); + const orphan = "0".repeat(64); + await seedChunk(root, scopeA, orphan); + + expect(await store.maintenance.reconcile()).toMatchObject({ + ok: true, + value: { + inspectedTransactions: 1, + failedTransactions: 0, + corruptTransactions: 1, + deletedOrphanChunks: 1, + orphanGcStatus: "COMPLETED", + }, + }); + expect(root.has(chunkFilePath(scopeA, orphan))).toBe(false); + expect(idb.readRaw("opfs-journal", "transaction_left_0001")).toBeUndefined(); + expect(idb.readRaw("opfs-journal", "transaction_corrupt1")).toMatchObject({ + corrupt: true, + }); + }); + + it("removes a partition directory only under the origin mutation lease", async () => { + const root = new MemoryDirectory(); + const leases = exclusiveLeases(); + const runtime = createOpfsWorkerRuntime({ + root: root as unknown as FileSystemDirectoryHandle, + crypto: globalThis.crypto, + policy: runtimePolicy, + leaseManager: leases, + dedicatedWorker: true, + supportsSynchronousAccessHandles: false, + }); + await seedChunk(root, scopeA, "0".repeat(64)); + await seedChunk(root, scopeP2, "0".repeat(64)); + const partitionPath = (scope: OpfsStorageScope) => [ + "authorities", + scope.authorityToken, + scope.namespaceToken, + scope.partitionToken, + ]; + const held = await leases.acquire(); + let settled = false; + const removal = runtime + .handleRequest({ + requestId: "request_remove_partition", + protocolVersion: OPFS_WORKER_PROTOCOL_VERSION, + kind: "REMOVE_PARTITION", + scope: scopeA, + }) + .finally(() => { + settled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(settled).toBe(false); + expect(root.has(partitionPath(scopeA))).toBe(true); + + held.release(); + expect(await removal).toMatchObject({ ok: true, kind: "REMOVE_PARTITION" }); + expect(root.has(partitionPath(scopeA))).toBe(false); + expect(root.has(partitionPath(scopeP2))).toBe(true); + expect(leases.held).toBe(false); + }); }); describe("05-07 a put whose source stops lets go of the origin lock", () => {