diff --git a/docs/operations/browser-file-storage-recovery.md b/docs/operations/browser-file-storage-recovery.md index ddf1006d..07f5f29e 100644 --- a/docs/operations/browser-file-storage-recovery.md +++ b/docs/operations/browser-file-storage-recovery.md @@ -38,8 +38,18 @@ cleanup 결과를 확인하지 못하면 성공 값이 아니라 실패로 답 저장소 손상이다. 예전에는 이런 row 하나가 목록 전체를 실패시켜 authority의 모든 partition에서 reconcile과 GC가 멈췄다. 이제 나머지 row와 orphan GC는 계속 돌고, 관측은 `OBJECT_RECONCILE` `DEGRADED` / `CORRUPT_DATA`로 남는다 - ([0929] 05-01). reconcile은 그 row를 지우지 않는다. 그 row가 속한 - partition이 승인된 purge로 통째로 지워질 때만 함께 지워진다(§7). + ([0929] 05-01). reconcile은 그 row를 지우지 않는다. 그 partition이 승인된 + purge로 통째로 지워질 때 함께 지워지는 것은 partition 키(`logicalKey`)를 + 읽을 수 있는 row뿐이다(§7). 승인된 LOGOUT/ACCOUNT_DELETION purge는 그런 + row를 보면 그 partition을 지운다(2026-11-03 05-01). + - `unattributedCorruptTransactions`: 위 수 가운데 `logicalKey`까지 읽을 수 + 없는 row 수다. 어느 partition에도 속하지 않아 partition 삭제도 지우지 + 못한다. 그래서 따로 세고 `DEGRADED` 관측에서 뺀다(2026-11-03 05-03). 예전에는 + 이런 row 하나가 reconcile마다 `DEGRADED`를 남겨 경보가 늘 켜졌다. 같은 수로 + 남아 있으면 할 일이 없다. 수가 늘면 키 없는 row를 쓴 writer(다른 release, + 외부 도구)를 찾는다. 이 row도 수동으로 지우지 않는다(3번). 그 row의 + staging은 partition 디렉터리 안에 있어 그 partition이 지워지면 함께 + 지워지지만, 이 row는 purge의 partition 삭제 조건에 들지 않는다. 3. journal row를 수동으로 삭제하지 않는다. row가 사라지면 stale staging을 추적할 근거가 사라지고 quota만 누수된다. 4. OPFS root나 journal database를 통째로 삭제하거나 schema를 downgrade하지 @@ -466,8 +476,10 @@ VD-15 목표 runtime을 먼저 구현한다. - 범위: worker가 origin mutation lease를 쥔 채 `authorities///` 디렉터리를 지우고, 이어서 journal에서 그 partition 키로 시작하는 object·journal·chunk - reference row와 budget row를 지운다. 값을 읽지 않고 키로 찾으므로 손상 row도 - 지워진다. scope binding row는 남긴다. 다른 partition은 건드리지 않는다. + reference row와 budget row를 지운다. 값을 읽지 않고 키로 찾으므로 키가 남은 + 손상 row도 지워진다. journal row는 `logicalKey` 인덱스로 찾으므로 그 값까지 + 깨진 row는 남는다(§0 2번 `unattributedCorruptTransactions`). scope binding + row는 남긴다. 다른 partition은 건드리지 않는다. - 결과: `partitionDeleted: true`, `moreObjectsAvailable: false`, 건너뛴 row 수 `corruptObjects`. 이 report면 그 partition의 bytes는 모두 지워진 것이다. `chunkSweep`은 없다. 디렉터리는 지웠는데 journal 정리가 실패하면 실패를 diff --git a/src/adapters/storage/opfs/indexeddb-opfs-journal.ts b/src/adapters/storage/opfs/indexeddb-opfs-journal.ts index b449952f..ea4b49c9 100644 --- a/src/adapters/storage/opfs/indexeddb-opfs-journal.ts +++ b/src/adapters/storage/opfs/indexeddb-opfs-journal.ts @@ -699,6 +699,7 @@ export function createIndexedDbOpfsJournal( (transaction, context) => { const rows: OpfsJournalTransaction[] = []; let corruptRows = 0; + let unattributedRows = 0; walkIndexedDbCursor({ request: transaction .objectStore(JOURNAL_STORE) @@ -720,6 +721,8 @@ export function createIndexedDbOpfsJournal( rows.push(cursor.value); } else { corruptRows += 1; + // 2026-11-03 05-03. No partition's key range holds it. + if (!hasPartitionKey(cursor.value)) unattributedRows += 1; } return { kind: "CONTINUE" }; }, @@ -730,6 +733,7 @@ export function createIndexedDbOpfsJournal( moreAvailable: summary.reason === "STOPPED" && rows.length === limit, corruptRows, + ...(unattributedRows ? { unattributedRows } : {}), }), ); }, @@ -969,6 +973,27 @@ export function createIndexedDbOpfsJournal( return Object.freeze(journal); + /** + * 2026-11-03 05-03. Whether a row's `logicalKey` lies in the key range + * `removePartition` deletes for some partition of this authority. + */ + function hasPartitionKey(value: unknown): boolean { + const key = + value && typeof value === "object" && "logicalKey" in value + ? value.logicalKey + : undefined; + if (typeof key !== "string") return false; + const [authority, namespaceToken, partitionToken, objectId] = + key.split("|"); + return ( + authority === dependencies.authorityToken && + objectId !== undefined && + [namespaceToken, partitionToken].every( + (token) => token !== undefined && SAFE_BOUNDARY_ID.test(token), + ) + ); + } + async function commitMutation( transactionId: string, fencingToken: string, diff --git a/src/adapters/storage/opfs/opfs-byte-store-adapter.ts b/src/adapters/storage/opfs/opfs-byte-store-adapter.ts index 3912218f..9c918f51 100644 --- a/src/adapters/storage/opfs/opfs-byte-store-adapter.ts +++ b/src/adapters/storage/opfs/opfs-byte-store-adapter.ts @@ -639,6 +639,7 @@ export function createOpfsByteStoreAdapter( // skipped. The page is read again under those locks: a row listed while // its owner still ran may have moved on before the owner let go. const held = new Map void>(); + let unattributed: number; try { for (const transaction of listed.value.transactions) { let release: (() => void) | null; @@ -663,6 +664,7 @@ export function createOpfsByteStoreAdapter( report.moreTransactionsAvailable = page.value.moreAvailable; // [0929] 05-01. The journal skipped these rows; they stop nothing now. report.corruptTransactions = page.value.corruptRows; + unattributed = page.value.unattributedRows ?? 0; for (const transaction of page.value.transactions) { if (!held.has(transaction.transactionId)) continue; if (request.signal?.aborted) { @@ -709,12 +711,19 @@ export function createOpfsByteStoreAdapter( operation: "OBJECT_RECONCILE", transactionBucket: transactionBucket(report.inspectedTransactions), // A skipped corrupt row is observed, not hidden behind a success. - ...(report.corruptTransactions + // 2026-11-03 05-03. One no partition owns is reported, not observed: + // nothing clears it, so it would keep the signal on for good. + ...(report.corruptTransactions > unattributed ? { outcome: "DEGRADED", failureCode: "CORRUPT_DATA" } : { outcome: "SUCCEEDED" }), }); return browserDataSuccess( - Object.freeze({ ...report }), + Object.freeze({ + ...report, + ...(unattributed + ? { unattributedCorruptTransactions: unattributed } + : {}), + }), ); }, diff --git a/src/application/ports/browser-file-storage/opfs-ports.ts b/src/application/ports/browser-file-storage/opfs-ports.ts index a43ddffd..7c638698 100644 --- a/src/application/ports/browser-file-storage/opfs-ports.ts +++ b/src/application/ports/browser-file-storage/opfs-ports.ts @@ -131,6 +131,13 @@ export type OpfsReconciliationReport = Readonly<{ * zero is storage damage for an operator to look at. */ corruptTransactions: number; + /** + * 2026-11-03 05-03. Of `corruptTransactions`, the rows whose partition key + * cannot be read either. No approved partition delete takes them, so they + * stay; they are not observed as `DEGRADED`, which is kept for damage a + * purge can clear. Present only when there were some. + */ + unattributedCorruptTransactions?: number; }>; export type OpfsPolicyMaintenanceReport = Readonly<{ @@ -324,6 +331,12 @@ export type OpfsJournalPage = Readonly<{ transactions: readonly OpfsJournalTransaction[]; moreAvailable: boolean; corruptRows: number; + /** + * 2026-11-03 05-03. Of `corruptRows`, the rows without a readable partition + * key, which `removePartition` never reaches. Absent means none, or a + * journal that does not tell them apart. + */ + unattributedRows?: number; }>; export type OpfsCommittedObjectPage = Readonly<{ diff --git a/tests/unit/indexeddb-opfs-journal.test.ts b/tests/unit/indexeddb-opfs-journal.test.ts index ed3dab4d..bd193f6e 100644 --- a/tests/unit/indexeddb-opfs-journal.test.ts +++ b/tests/unit/indexeddb-opfs-journal.test.ts @@ -508,7 +508,12 @@ describe("IndexedDB OPFS journal", () => { // [0929] 05-01. The row is counted, not returned and not deleted. expect(await journal.listIncomplete(10)).toEqual({ ok: true, - value: { transactions: [], moreAvailable: false, corruptRows: 1 }, + value: { + transactions: [], + moreAvailable: false, + corruptRows: 1, + unattributedRows: 1, + }, }); expect(factory.readRaw("opfs-journal", "transaction_88888888")).toMatchObject({ corrupt: true, @@ -865,6 +870,54 @@ describe("[0929] 05-01 a corrupt row is skipped and counted; a partition's rows ).toMatchObject({ preparedObject: { corrupt: true } }); }); + // 2026-11-03 05-03. The partition delete finds journal rows by their + // `logicalKey`. A damaged row that still has one goes with its partition; a + // row without one belongs to no partition, so no delete takes it. The page + // tells the two apart so an operator can. + it("tells a damaged journal row its partition delete takes from one no partition owns", async () => { + const { factory, journal } = createHarness(); + // Opens the database the seeds below write into. + expect(await journal.listIncomplete(10)).toMatchObject({ ok: true }); + factory.seed("opfs-journal", { + transactionId: "transaction_keyed001", + logicalKey: `${scopeKeyOf(scope)}|object_aaaa0001`, + startedAtEpochMs: 1, + phase: "GARBLED", + }); + factory.seed("opfs-journal", { + transactionId: "transaction_keyless1", + startedAtEpochMs: 2, + corrupt: true, + }); + + expect(await journal.listIncomplete(10)).toEqual({ + ok: true, + value: { + transactions: [], + moreAvailable: false, + corruptRows: 2, + unattributedRows: 1, + }, + }); + expect(await journal.removePartition(scope)).toMatchObject({ ok: true }); + + expect( + factory.readRaw("opfs-journal", "transaction_keyed001"), + ).toBeUndefined(); + expect( + factory.readRaw("opfs-journal", "transaction_keyless1"), + ).toMatchObject({ corrupt: true }); + expect(await journal.listIncomplete(10)).toEqual({ + ok: true, + value: { + transactions: [], + moreAvailable: false, + corruptRows: 1, + unattributedRows: 1, + }, + }); + }); + it("counts the unfinished rows of one partition, a damaged one included (2026-11-03 05-01)", async () => { const { factory, journal } = createHarness(); await commitObject(journal, "transaction_obj00001", "object_aaaa0001", 1); diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 35d66fcd..a9699b72 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -2720,6 +2720,60 @@ describe("05-04 a LOGOUT or ACCOUNT_DELETION purge leaves no partition bytes beh }); }); + // 2026-11-03 05-03 X4. A damaged journal row without a readable partition + // key outlives every approved partition delete. It used to keep each + // reconcile DEGRADED for good, with nothing the runbook lets anyone do. It + // is now counted apart and kept out of the partition damage signal; a row + // its partition delete would take still raises it. + it("reports a damaged journal row no partition owns apart from partition damage", async () => { + const { root, idb, tab } = sharedRootTabs(); + const outcomes: string[] = []; + const store = tab("s", { + ...authority, + observer: (observation) => { + if ( + observation.operation === "OBJECT_RECONCILE" && + observation.outcome !== "STARTED" + ) { + outcomes.push(`${observation.outcome}:${observation.failureCode ?? ""}`); + } + }, + }); + expect(await putBytes(store, "object_secret_1", [1, 2, 3])).toMatchObject({ ok: true }); + idb.seed("opfs-objects", corruptObjectRow(scopeA, "object_secret_2")); + idb.seed("opfs-journal", { + transactionId: "transaction_corrupt1", + startedAtEpochMs: 1, + corrupt: true, + }); + + expect( + await store.maintenance.enforcePolicies({ reason: "ACCOUNT_DELETION" }), + ).toMatchObject({ ok: true, value: { partitionDeleted: true } }); + expect(filesUnder(root, scopeA)).toEqual([]); + expect(idb.readRaw("opfs-journal", "transaction_corrupt1")).toMatchObject({ + corrupt: true, + }); + + expect(await store.maintenance.reconcile()).toMatchObject({ + ok: true, + value: { corruptTransactions: 1, unattributedCorruptTransactions: 1 }, + }); + expect(outcomes).toEqual(["SUCCEEDED:"]); + + idb.seed("opfs-journal", { + transactionId: "transaction_keyed001", + logicalKey: `${scopeA.authorityToken}|${scopeA.namespaceToken}|${scopeA.partitionToken}|object_secret_3`, + startedAtEpochMs: 2, + phase: "GARBLED", + }); + expect(await store.maintenance.reconcile()).toMatchObject({ + ok: true, + value: { corruptTransactions: 2, unattributedCorruptTransactions: 1 }, + }); + expect(outcomes.at(-1)).toBe("DEGRADED:CORRUPT_DATA"); + }); + it("removes a partition directory only under the origin mutation lease", async () => { const root = new MemoryDirectory(); const leases = exclusiveLeases();