fix(storage): hold a per-transaction lock so reconcile skips a live OPFS put (05-01)

Reconcile cleaned up any incomplete journal row, including a put still
running in another tab, and the put then reported success for deleted files.
Put and delete now hold a Web Lock per transaction from begin to return;
reconcile takes those locks with ifAvailable, re-reads the page under them
and reports the rows it skipped (D-3).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-25 22:09:30 +09:00
co-authored by Claude Opus 5.5
parent 926f448fce
commit 8fc816ee25
5 changed files with 661 additions and 205 deletions
@@ -96,6 +96,13 @@ export type OpfsByteStoreDependencies = Readonly<{
* deliberately separate from any caller signal.
*/
compensationSignal?: AbortSignal;
/**
* 05-01 (D-3). Web Locks, `navigator.locks` by default. A put or delete
* holds an exclusive per-transaction lock from journal `begin` until it
* returns; reconcile only touches rows whose lock it can take at once, so a
* live transaction in any tab is never cleaned up underneath its owner.
*/
lockManager?: Pick<LockManager, "request">;
now?: () => number;
observer?: OpfsSafeObserver;
/**
@@ -152,6 +159,12 @@ export function createOpfsByteStoreAdapter(
*/
const compensationSignal = dependencies.compensationSignal;
const now = dependencies.now ?? Date.now;
const lockManager =
dependencies.lockManager ?? globalThis.navigator?.locks;
if (!lockManager || typeof lockManager.request !== "function") {
throw new TypeError("OPFS byte store requires Web Locks.");
}
const requestLock = lockManager.request.bind(lockManager);
const objects: DurableObjectStorePort = Object.freeze({
async capabilities() {
@@ -203,146 +216,152 @@ export function createOpfsByteStoreAdapter(
}
const targetGeneration = (current?.descriptor.generation ?? 0) + 1;
const transactionId = createTransactionId();
// STO-01. The logical generation is reused across transactions; this
// token makes the physical target unique so a late compensation can never
// delete a newer transaction's directory.
const physicalGenerationId = createPhysicalGenerationId();
notifyProgress(request, "PREPARING", 0);
const begun = await dependencies.journal.begin({
return await withTransactionLock(
transactionId,
mutation: "PUT",
scope: dependencies.scope,
objectId: request.objectId,
expectedGeneration: request.expectedGeneration,
targetGeneration,
targetByteLength: request.source.byteLength!,
targetStoragePolicy: dependencies.storagePolicy,
startedAtEpochMs: now(),
});
if (!begun.ok) {
return observeFailure(
rebaseFailure(begun.error, "OBJECT_WRITE"),
dependencies.observer,
request.source.byteLength!,
);
}
"OBJECT_WRITE",
async () => {
// STO-01. The logical generation is reused across transactions; this
// token makes the physical target unique so a late compensation can never
// delete a newer transaction's directory.
const physicalGenerationId = createPhysicalGenerationId();
notifyProgress(request, "PREPARING", 0);
const begun = await dependencies.journal.begin({
transactionId,
mutation: "PUT",
scope: dependencies.scope,
objectId: request.objectId,
expectedGeneration: request.expectedGeneration,
targetGeneration,
targetByteLength: request.source.byteLength!,
targetStoragePolicy: dependencies.storagePolicy,
startedAtEpochMs: now(),
});
if (!begun.ok) {
return observeFailure(
rebaseFailure(begun.error, "OBJECT_WRITE"),
dependencies.observer,
request.source.byteLength!,
);
}
const descriptor: Omit<DurableObjectDescriptor, "integrity"> =
Object.freeze({
objectId: request.objectId,
scope: dependencies.scope,
generation: targetGeneration,
byteLength: request.source.byteLength!,
mediaType: request.mediaType,
createdAtEpochMs: now(),
storagePolicy: dependencies.storagePolicy,
});
const prepared = await dependencies.worker.preparePut({
transactionId,
physicalGenerationId,
descriptor,
source: request.source,
signal: request.signal,
onProgress: request.onProgress,
});
if (!prepared.ok) {
return observeFailure(
await compensatePreparedPut(
begun.value,
const descriptor: Omit<DurableObjectDescriptor, "integrity"> =
Object.freeze({
objectId: request.objectId,
scope: dependencies.scope,
generation: targetGeneration,
byteLength: request.source.byteLength!,
mediaType: request.mediaType,
createdAtEpochMs: now(),
storagePolicy: dependencies.storagePolicy,
});
const prepared = await dependencies.worker.preparePut({
transactionId,
physicalGenerationId,
prepared.error,
),
dependencies.observer,
request.source.byteLength!,
);
}
descriptor,
source: request.source,
signal: request.signal,
onProgress: request.onProgress,
});
if (!prepared.ok) {
return observeFailure(
await compensatePreparedPut(
begun.value,
physicalGenerationId,
prepared.error,
),
dependencies.observer,
request.source.byteLength!,
);
}
const filesReady = await dependencies.journal.markFilesReady(
transactionId,
begun.value.fencingToken,
prepared.value,
);
if (!filesReady.ok) {
return observeFailure(
await compensatePreparedPut(
begun.value,
physicalGenerationId,
filesReady.error,
),
dependencies.observer,
request.source.byteLength!,
);
}
const filesReady = await dependencies.journal.markFilesReady(
transactionId,
begun.value.fencingToken,
prepared.value,
);
if (!filesReady.ok) {
return observeFailure(
await compensatePreparedPut(
begun.value,
physicalGenerationId,
filesReady.error,
),
dependencies.observer,
request.source.byteLength!,
);
}
const committed = await dependencies.journal.commitPut(
transactionId,
begun.value.fencingToken,
);
if (!committed.ok) {
// The commit response can be lost after IndexedDB commits. Reconciliation
// decides from the durable journal; rolling back here would be unsafe.
return observeFailure(
rebaseFailure(committed.error, "OBJECT_WRITE"),
dependencies.observer,
request.source.byteLength!,
);
}
const committed = await dependencies.journal.commitPut(
transactionId,
begun.value.fencingToken,
);
if (!committed.ok) {
// The commit response can be lost after IndexedDB commits. Reconciliation
// decides from the durable journal; rolling back here would be unsafe.
return observeFailure(
rebaseFailure(committed.error, "OBJECT_WRITE"),
dependencies.observer,
request.source.byteLength!,
);
}
notifyProgress(
request,
"FINALIZING",
prepared.value.descriptor.byteLength,
notifyProgress(
request,
"FINALIZING",
prepared.value.descriptor.byteLength,
);
const finalized = await dependencies.worker.finalizePut(
transactionId,
prepared.value,
request.signal,
);
if (!finalized.ok) {
// STO-RR-01. The commit fence already passed, so the payload is durable
// and the journal keeps its COMMITTED record for reconciliation to
// settle. What did not happen is finalization: the previous generation
// and the staging directory are still present. Reporting a plain
// success here would claim a settled state nobody observed, so the
// worker's own failure is surfaced and the record is left recoverable.
return observeFailure(
rebaseFailure(finalized.error, "OBJECT_WRITE"),
dependencies.observer,
request.source.byteLength!,
);
}
const completed = await dependencies.journal.complete(
transactionId,
begun.value.fencingToken,
);
if (!completed.ok) {
// NS-04. The payload is durable and finalized, so nothing is rolled
// back and the `COMMITTED` row stays the reconciler's authority. What
// did not happen is settling the transaction, and reporting a plain
// success for it claimed a state nobody observed while the reconcile
// backlog and its quota pressure grew unseen.
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_WRITE",
outcome: "DEGRADED",
failureCode: completed.error.code,
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
});
return {
ok: false as const,
error: Object.freeze({
...completed.error,
operation: "OBJECT_WRITE" as const,
retryable: true,
recovery: "RECONCILE" as const,
}),
};
}
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_WRITE",
outcome: "SUCCEEDED",
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
});
return browserDataSuccess(prepared.value.descriptor);
},
);
const finalized = await dependencies.worker.finalizePut(
transactionId,
prepared.value,
request.signal,
);
if (!finalized.ok) {
// STO-RR-01. The commit fence already passed, so the payload is durable
// and the journal keeps its COMMITTED record for reconciliation to
// settle. What did not happen is finalization: the previous generation
// and the staging directory are still present. Reporting a plain
// success here would claim a settled state nobody observed, so the
// worker's own failure is surfaced and the record is left recoverable.
return observeFailure(
rebaseFailure(finalized.error, "OBJECT_WRITE"),
dependencies.observer,
request.source.byteLength!,
);
}
const completed = await dependencies.journal.complete(
transactionId,
begun.value.fencingToken,
);
if (!completed.ok) {
// NS-04. The payload is durable and finalized, so nothing is rolled
// back and the `COMMITTED` row stays the reconciler's authority. What
// did not happen is settling the transaction, and reporting a plain
// success for it claimed a state nobody observed while the reconcile
// backlog and its quota pressure grew unseen.
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_WRITE",
outcome: "DEGRADED",
failureCode: completed.error.code,
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
});
return {
ok: false as const,
error: Object.freeze({
...completed.error,
operation: "OBJECT_WRITE" as const,
retryable: true,
recovery: "RECONCILE" as const,
}),
};
}
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_WRITE",
outcome: "SUCCEEDED",
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
});
return browserDataSuccess(prepared.value.descriptor);
},
async open(
@@ -469,63 +488,70 @@ export function createOpfsByteStoreAdapter(
);
}
const targetStoragePolicy = current.value.descriptor.storagePolicy;
const transactionId = createTransactionId();
const begun = await dependencies.journal.begin({
return await withTransactionLock(
transactionId,
mutation: "DELETE",
scope: dependencies.scope,
objectId: request.objectId,
expectedGeneration: request.expectedGeneration,
targetGeneration: request.expectedGeneration + 1,
targetByteLength: 0,
targetStoragePolicy: current.value.descriptor.storagePolicy,
startedAtEpochMs: now(),
});
if (!begun.ok) {
return observeFailure(
rebaseFailure(begun.error, "OBJECT_DELETE"),
dependencies.observer,
);
}
const committed = await dependencies.journal.commitDelete(
transactionId,
begun.value.fencingToken,
);
if (!committed.ok) {
return observeFailure(
rebaseFailure(committed.error, "OBJECT_DELETE"),
dependencies.observer,
);
}
const removed = await dependencies.worker.removeObject(
dependencies.scope,
request.objectId,
request.expectedGeneration,
request.signal,
);
const completed = removed.ok
? await dependencies.journal.complete(
"OBJECT_DELETE",
async () => {
const begun = await dependencies.journal.begin({
transactionId,
mutation: "DELETE",
scope: dependencies.scope,
objectId: request.objectId,
expectedGeneration: request.expectedGeneration,
targetGeneration: request.expectedGeneration + 1,
targetByteLength: 0,
targetStoragePolicy,
startedAtEpochMs: now(),
});
if (!begun.ok) {
return observeFailure(
rebaseFailure(begun.error, "OBJECT_DELETE"),
dependencies.observer,
);
}
const committed = await dependencies.journal.commitDelete(
transactionId,
begun.value.fencingToken,
)
: null;
// Logical deletion is already committed. Physical cleanup is retryable
// maintenance and must not make the caller repeat a non-idempotent delete.
// NS-04. It is still not a settled state: an unfinished physical removal
// or an unsettled journal row is maintenance debt the reconciler owns, so
// it is observed as such instead of as a clean success.
const settled = removed.ok && completed !== null && completed.ok;
const debtCode = removed.ok
? completed !== null && !completed.ok
? completed.error.code
: undefined
: removed.error.code;
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_DELETE",
outcome: settled ? "SUCCEEDED" : "DEGRADED",
...(debtCode ? { failureCode: debtCode } : {}),
});
return browserDataSuccess(undefined);
);
if (!committed.ok) {
return observeFailure(
rebaseFailure(committed.error, "OBJECT_DELETE"),
dependencies.observer,
);
}
const removed = await dependencies.worker.removeObject(
dependencies.scope,
request.objectId,
request.expectedGeneration,
request.signal,
);
const completed = removed.ok
? await dependencies.journal.complete(
transactionId,
begun.value.fencingToken,
)
: null;
// Logical deletion is already committed. Physical cleanup is retryable
// maintenance and must not make the caller repeat a non-idempotent delete.
// NS-04. It is still not a settled state: an unfinished physical removal
// or an unsettled journal row is maintenance debt the reconciler owns, so
// it is observed as such instead of as a clean success.
const settled = removed.ok && completed !== null && completed.ok;
const debtCode = removed.ok
? completed !== null && !completed.ok
? completed.error.code
: undefined
: removed.error.code;
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_DELETE",
outcome: settled ? "SUCCEEDED" : "DEGRADED",
...(debtCode ? { failureCode: debtCode } : {}),
});
return browserDataSuccess(undefined);
},
);
},
});
@@ -563,14 +589,14 @@ export function createOpfsByteStoreAdapter(
const maxTransactions =
request.maxTransactions ?? policy.reconciliationBatchSize;
const deadline = now() + budgetMs;
const page = await dependencies.journal.listIncomplete(maxTransactions);
if (!page.ok) {
return rebaseFailure(page.error, "OBJECT_RECONCILE");
const listed = await dependencies.journal.listIncomplete(maxTransactions);
if (!listed.ok) {
return rebaseFailure(listed.error, "OBJECT_RECONCILE");
}
observeOpfsSafely(dependencies.observer, {
operation: "OBJECT_RECONCILE",
outcome: "STARTED",
transactionBucket: transactionBucket(page.value.transactions.length),
transactionBucket: transactionBucket(listed.value.transactions.length),
});
const report: {
@@ -586,6 +612,7 @@ export function createOpfsByteStoreAdapter(
| "STAGING_STATE_UNREADABLE";
moreTransactionsAvailable: boolean;
deadlineReached: boolean;
skippedLiveTransactions: number;
} = {
inspectedTransactions: 0,
committedTransactions: 0,
@@ -594,26 +621,58 @@ export function createOpfsByteStoreAdapter(
inspectedOrphanChunks: 0,
deletedOrphanChunks: 0,
orphanGcStatus: "COMPLETED",
moreTransactionsAvailable: page.value.moreAvailable,
moreTransactionsAvailable: listed.value.moreAvailable,
deadlineReached: false,
skippedLiveTransactions: 0,
};
for (const transaction of page.value.transactions) {
if (request.signal?.aborted) {
return browserDataFailure("ABORTED", "OBJECT_RECONCILE");
// 05-01 (D-3). A row is reconciled only while this call holds its
// per-transaction lock, so a put or delete still running in any tab is
// skipped. The page is read again under those locks: a row listed while
// its owner still ran may have moved on before the owner let go.
const held = new Map<string, () => void>();
try {
for (const transaction of listed.value.transactions) {
let release: (() => void) | null;
try {
release = await holdTransactionLock(transaction.transactionId);
} catch {
return browserDataFailure("UNAVAILABLE", "OBJECT_RECONCILE", {
retryable: true,
recovery: "RETRY",
});
}
if (release) held.set(transaction.transactionId, release);
else report.skippedLiveTransactions += 1;
}
if (now() >= deadline) {
report.deadlineReached = true;
report.moreTransactionsAvailable = true;
break;
const page =
held.size === 0
? listed
: await dependencies.journal.listIncomplete(maxTransactions);
if (!page.ok) {
return rebaseFailure(page.error, "OBJECT_RECONCILE");
}
report.inspectedTransactions += 1;
const reconciled = await reconcileTransaction(transaction, request.signal);
if (!reconciled.ok) {
return observeFailure(reconciled, dependencies.observer);
report.moreTransactionsAvailable = page.value.moreAvailable;
for (const transaction of page.value.transactions) {
if (!held.has(transaction.transactionId)) continue;
if (request.signal?.aborted) {
return browserDataFailure("ABORTED", "OBJECT_RECONCILE");
}
if (now() >= deadline) {
report.deadlineReached = true;
report.moreTransactionsAvailable = true;
break;
}
report.inspectedTransactions += 1;
const reconciled = await reconcileTransaction(transaction, request.signal);
if (!reconciled.ok) {
return observeFailure(reconciled, dependencies.observer);
}
report.committedTransactions += reconciled.value.committed;
report.rolledBackTransactions += reconciled.value.rolledBack;
report.cleanedTransactions += reconciled.value.cleaned;
}
report.committedTransactions += reconciled.value.committed;
report.rolledBackTransactions += reconciled.value.rolledBack;
report.cleanedTransactions += reconciled.value.cleaned;
} finally {
for (const release of held.values()) release();
}
if (now() >= deadline) {
report.deadlineReached = true;
@@ -905,6 +964,56 @@ export function createOpfsByteStoreAdapter(
return Object.freeze({ objects, maintenance });
/**
* Takes the exclusive per-transaction Web Lock only if nobody holds it.
* Resolves with its release, or `null` while another context holds it.
*/
async function holdTransactionLock(
transactionId: string,
): Promise<(() => void) | null> {
let release!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
return await new Promise((resolve, reject) => {
requestLock(
`${policy.mutationLockName}:transaction:${scope.authorityToken}:${transactionId}`,
{ mode: "exclusive", ifAvailable: true },
async (lock) => {
if (!lock) {
resolve(null);
return;
}
resolve(release);
await held;
},
).catch(reject);
});
}
async function withTransactionLock<Value>(
transactionId: string,
operation: "OBJECT_WRITE" | "OBJECT_DELETE",
run: () => Promise<BrowserDataResult<Value>>,
): Promise<BrowserDataResult<Value>> {
let release: (() => void) | null;
try {
release = await holdTransactionLock(transactionId);
} catch {
return browserDataFailure("UNAVAILABLE", operation, {
retryable: true,
recovery: "RETRY",
});
}
// A fresh transaction id is never held; a holder means a duplicate id.
if (!release) return browserDataFailure("CONFLICT", operation);
try {
return await run();
} finally {
release();
}
}
/**
* STO-01. The compensating half of the put saga.
*
@@ -100,6 +100,11 @@ export type OpfsReconciliationReport = Readonly<{
| "STAGING_STATE_UNREADABLE";
moreTransactionsAvailable: boolean;
deadlineReached: boolean;
/**
* 05-01. Rows whose owner still holds the per-transaction lock. They are
* a live put or delete, not debris, and are left for a later reconcile.
*/
skippedLiveTransactions: number;
}>;
export type OpfsPolicyMaintenanceReport = Readonly<{