fix(storage): hold a per-transaction lock so reconcile skips a live OPFS put (05-01)
Reconcile cleaned up any incomplete journal row, including a put still running in another tab, and the put then reported success for deleted files. Put and delete now hold a Web Lock per transaction from begin to return; reconcile takes those locks with ifAvailable, re-reads the page under them and reports the rows it skipped (D-3). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
926f448fce
commit
8fc816ee25
@@ -96,6 +96,13 @@ export type OpfsByteStoreDependencies = Readonly<{
|
||||
* deliberately separate from any caller signal.
|
||||
*/
|
||||
compensationSignal?: AbortSignal;
|
||||
/**
|
||||
* 05-01 (D-3). Web Locks, `navigator.locks` by default. A put or delete
|
||||
* holds an exclusive per-transaction lock from journal `begin` until it
|
||||
* returns; reconcile only touches rows whose lock it can take at once, so a
|
||||
* live transaction in any tab is never cleaned up underneath its owner.
|
||||
*/
|
||||
lockManager?: Pick<LockManager, "request">;
|
||||
now?: () => number;
|
||||
observer?: OpfsSafeObserver;
|
||||
/**
|
||||
@@ -152,6 +159,12 @@ export function createOpfsByteStoreAdapter(
|
||||
*/
|
||||
const compensationSignal = dependencies.compensationSignal;
|
||||
const now = dependencies.now ?? Date.now;
|
||||
const lockManager =
|
||||
dependencies.lockManager ?? globalThis.navigator?.locks;
|
||||
if (!lockManager || typeof lockManager.request !== "function") {
|
||||
throw new TypeError("OPFS byte store requires Web Locks.");
|
||||
}
|
||||
const requestLock = lockManager.request.bind(lockManager);
|
||||
|
||||
const objects: DurableObjectStorePort = Object.freeze({
|
||||
async capabilities() {
|
||||
@@ -203,146 +216,152 @@ export function createOpfsByteStoreAdapter(
|
||||
}
|
||||
const targetGeneration = (current?.descriptor.generation ?? 0) + 1;
|
||||
const transactionId = createTransactionId();
|
||||
// STO-01. The logical generation is reused across transactions; this
|
||||
// token makes the physical target unique so a late compensation can never
|
||||
// delete a newer transaction's directory.
|
||||
const physicalGenerationId = createPhysicalGenerationId();
|
||||
notifyProgress(request, "PREPARING", 0);
|
||||
const begun = await dependencies.journal.begin({
|
||||
return await withTransactionLock(
|
||||
transactionId,
|
||||
mutation: "PUT",
|
||||
scope: dependencies.scope,
|
||||
objectId: request.objectId,
|
||||
expectedGeneration: request.expectedGeneration,
|
||||
targetGeneration,
|
||||
targetByteLength: request.source.byteLength!,
|
||||
targetStoragePolicy: dependencies.storagePolicy,
|
||||
startedAtEpochMs: now(),
|
||||
});
|
||||
if (!begun.ok) {
|
||||
return observeFailure(
|
||||
rebaseFailure(begun.error, "OBJECT_WRITE"),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
"OBJECT_WRITE",
|
||||
async () => {
|
||||
// STO-01. The logical generation is reused across transactions; this
|
||||
// token makes the physical target unique so a late compensation can never
|
||||
// delete a newer transaction's directory.
|
||||
const physicalGenerationId = createPhysicalGenerationId();
|
||||
notifyProgress(request, "PREPARING", 0);
|
||||
const begun = await dependencies.journal.begin({
|
||||
transactionId,
|
||||
mutation: "PUT",
|
||||
scope: dependencies.scope,
|
||||
objectId: request.objectId,
|
||||
expectedGeneration: request.expectedGeneration,
|
||||
targetGeneration,
|
||||
targetByteLength: request.source.byteLength!,
|
||||
targetStoragePolicy: dependencies.storagePolicy,
|
||||
startedAtEpochMs: now(),
|
||||
});
|
||||
if (!begun.ok) {
|
||||
return observeFailure(
|
||||
rebaseFailure(begun.error, "OBJECT_WRITE"),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
|
||||
const descriptor: Omit<DurableObjectDescriptor, "integrity"> =
|
||||
Object.freeze({
|
||||
objectId: request.objectId,
|
||||
scope: dependencies.scope,
|
||||
generation: targetGeneration,
|
||||
byteLength: request.source.byteLength!,
|
||||
mediaType: request.mediaType,
|
||||
createdAtEpochMs: now(),
|
||||
storagePolicy: dependencies.storagePolicy,
|
||||
});
|
||||
const prepared = await dependencies.worker.preparePut({
|
||||
transactionId,
|
||||
physicalGenerationId,
|
||||
descriptor,
|
||||
source: request.source,
|
||||
signal: request.signal,
|
||||
onProgress: request.onProgress,
|
||||
});
|
||||
if (!prepared.ok) {
|
||||
return observeFailure(
|
||||
await compensatePreparedPut(
|
||||
begun.value,
|
||||
const descriptor: Omit<DurableObjectDescriptor, "integrity"> =
|
||||
Object.freeze({
|
||||
objectId: request.objectId,
|
||||
scope: dependencies.scope,
|
||||
generation: targetGeneration,
|
||||
byteLength: request.source.byteLength!,
|
||||
mediaType: request.mediaType,
|
||||
createdAtEpochMs: now(),
|
||||
storagePolicy: dependencies.storagePolicy,
|
||||
});
|
||||
const prepared = await dependencies.worker.preparePut({
|
||||
transactionId,
|
||||
physicalGenerationId,
|
||||
prepared.error,
|
||||
),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
descriptor,
|
||||
source: request.source,
|
||||
signal: request.signal,
|
||||
onProgress: request.onProgress,
|
||||
});
|
||||
if (!prepared.ok) {
|
||||
return observeFailure(
|
||||
await compensatePreparedPut(
|
||||
begun.value,
|
||||
physicalGenerationId,
|
||||
prepared.error,
|
||||
),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
|
||||
const filesReady = await dependencies.journal.markFilesReady(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
prepared.value,
|
||||
);
|
||||
if (!filesReady.ok) {
|
||||
return observeFailure(
|
||||
await compensatePreparedPut(
|
||||
begun.value,
|
||||
physicalGenerationId,
|
||||
filesReady.error,
|
||||
),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
const filesReady = await dependencies.journal.markFilesReady(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
prepared.value,
|
||||
);
|
||||
if (!filesReady.ok) {
|
||||
return observeFailure(
|
||||
await compensatePreparedPut(
|
||||
begun.value,
|
||||
physicalGenerationId,
|
||||
filesReady.error,
|
||||
),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
|
||||
const committed = await dependencies.journal.commitPut(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
);
|
||||
if (!committed.ok) {
|
||||
// The commit response can be lost after IndexedDB commits. Reconciliation
|
||||
// decides from the durable journal; rolling back here would be unsafe.
|
||||
return observeFailure(
|
||||
rebaseFailure(committed.error, "OBJECT_WRITE"),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
const committed = await dependencies.journal.commitPut(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
);
|
||||
if (!committed.ok) {
|
||||
// The commit response can be lost after IndexedDB commits. Reconciliation
|
||||
// decides from the durable journal; rolling back here would be unsafe.
|
||||
return observeFailure(
|
||||
rebaseFailure(committed.error, "OBJECT_WRITE"),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
|
||||
notifyProgress(
|
||||
request,
|
||||
"FINALIZING",
|
||||
prepared.value.descriptor.byteLength,
|
||||
notifyProgress(
|
||||
request,
|
||||
"FINALIZING",
|
||||
prepared.value.descriptor.byteLength,
|
||||
);
|
||||
const finalized = await dependencies.worker.finalizePut(
|
||||
transactionId,
|
||||
prepared.value,
|
||||
request.signal,
|
||||
);
|
||||
if (!finalized.ok) {
|
||||
// STO-RR-01. The commit fence already passed, so the payload is durable
|
||||
// and the journal keeps its COMMITTED record for reconciliation to
|
||||
// settle. What did not happen is finalization: the previous generation
|
||||
// and the staging directory are still present. Reporting a plain
|
||||
// success here would claim a settled state nobody observed, so the
|
||||
// worker's own failure is surfaced and the record is left recoverable.
|
||||
return observeFailure(
|
||||
rebaseFailure(finalized.error, "OBJECT_WRITE"),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
const completed = await dependencies.journal.complete(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
);
|
||||
if (!completed.ok) {
|
||||
// NS-04. The payload is durable and finalized, so nothing is rolled
|
||||
// back and the `COMMITTED` row stays the reconciler's authority. What
|
||||
// did not happen is settling the transaction, and reporting a plain
|
||||
// success for it claimed a state nobody observed while the reconcile
|
||||
// backlog and its quota pressure grew unseen.
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_WRITE",
|
||||
outcome: "DEGRADED",
|
||||
failureCode: completed.error.code,
|
||||
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
|
||||
});
|
||||
return {
|
||||
ok: false as const,
|
||||
error: Object.freeze({
|
||||
...completed.error,
|
||||
operation: "OBJECT_WRITE" as const,
|
||||
retryable: true,
|
||||
recovery: "RECONCILE" as const,
|
||||
}),
|
||||
};
|
||||
}
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_WRITE",
|
||||
outcome: "SUCCEEDED",
|
||||
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
|
||||
});
|
||||
return browserDataSuccess(prepared.value.descriptor);
|
||||
},
|
||||
);
|
||||
const finalized = await dependencies.worker.finalizePut(
|
||||
transactionId,
|
||||
prepared.value,
|
||||
request.signal,
|
||||
);
|
||||
if (!finalized.ok) {
|
||||
// STO-RR-01. The commit fence already passed, so the payload is durable
|
||||
// and the journal keeps its COMMITTED record for reconciliation to
|
||||
// settle. What did not happen is finalization: the previous generation
|
||||
// and the staging directory are still present. Reporting a plain
|
||||
// success here would claim a settled state nobody observed, so the
|
||||
// worker's own failure is surfaced and the record is left recoverable.
|
||||
return observeFailure(
|
||||
rebaseFailure(finalized.error, "OBJECT_WRITE"),
|
||||
dependencies.observer,
|
||||
request.source.byteLength!,
|
||||
);
|
||||
}
|
||||
const completed = await dependencies.journal.complete(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
);
|
||||
if (!completed.ok) {
|
||||
// NS-04. The payload is durable and finalized, so nothing is rolled
|
||||
// back and the `COMMITTED` row stays the reconciler's authority. What
|
||||
// did not happen is settling the transaction, and reporting a plain
|
||||
// success for it claimed a state nobody observed while the reconcile
|
||||
// backlog and its quota pressure grew unseen.
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_WRITE",
|
||||
outcome: "DEGRADED",
|
||||
failureCode: completed.error.code,
|
||||
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
|
||||
});
|
||||
return {
|
||||
ok: false as const,
|
||||
error: Object.freeze({
|
||||
...completed.error,
|
||||
operation: "OBJECT_WRITE" as const,
|
||||
retryable: true,
|
||||
recovery: "RECONCILE" as const,
|
||||
}),
|
||||
};
|
||||
}
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_WRITE",
|
||||
outcome: "SUCCEEDED",
|
||||
byteBucket: byteBucket(prepared.value.descriptor.byteLength),
|
||||
});
|
||||
return browserDataSuccess(prepared.value.descriptor);
|
||||
},
|
||||
|
||||
async open(
|
||||
@@ -469,63 +488,70 @@ export function createOpfsByteStoreAdapter(
|
||||
);
|
||||
}
|
||||
|
||||
const targetStoragePolicy = current.value.descriptor.storagePolicy;
|
||||
const transactionId = createTransactionId();
|
||||
const begun = await dependencies.journal.begin({
|
||||
return await withTransactionLock(
|
||||
transactionId,
|
||||
mutation: "DELETE",
|
||||
scope: dependencies.scope,
|
||||
objectId: request.objectId,
|
||||
expectedGeneration: request.expectedGeneration,
|
||||
targetGeneration: request.expectedGeneration + 1,
|
||||
targetByteLength: 0,
|
||||
targetStoragePolicy: current.value.descriptor.storagePolicy,
|
||||
startedAtEpochMs: now(),
|
||||
});
|
||||
if (!begun.ok) {
|
||||
return observeFailure(
|
||||
rebaseFailure(begun.error, "OBJECT_DELETE"),
|
||||
dependencies.observer,
|
||||
);
|
||||
}
|
||||
const committed = await dependencies.journal.commitDelete(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
);
|
||||
if (!committed.ok) {
|
||||
return observeFailure(
|
||||
rebaseFailure(committed.error, "OBJECT_DELETE"),
|
||||
dependencies.observer,
|
||||
);
|
||||
}
|
||||
const removed = await dependencies.worker.removeObject(
|
||||
dependencies.scope,
|
||||
request.objectId,
|
||||
request.expectedGeneration,
|
||||
request.signal,
|
||||
);
|
||||
const completed = removed.ok
|
||||
? await dependencies.journal.complete(
|
||||
"OBJECT_DELETE",
|
||||
async () => {
|
||||
const begun = await dependencies.journal.begin({
|
||||
transactionId,
|
||||
mutation: "DELETE",
|
||||
scope: dependencies.scope,
|
||||
objectId: request.objectId,
|
||||
expectedGeneration: request.expectedGeneration,
|
||||
targetGeneration: request.expectedGeneration + 1,
|
||||
targetByteLength: 0,
|
||||
targetStoragePolicy,
|
||||
startedAtEpochMs: now(),
|
||||
});
|
||||
if (!begun.ok) {
|
||||
return observeFailure(
|
||||
rebaseFailure(begun.error, "OBJECT_DELETE"),
|
||||
dependencies.observer,
|
||||
);
|
||||
}
|
||||
const committed = await dependencies.journal.commitDelete(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
)
|
||||
: null;
|
||||
// Logical deletion is already committed. Physical cleanup is retryable
|
||||
// maintenance and must not make the caller repeat a non-idempotent delete.
|
||||
// NS-04. It is still not a settled state: an unfinished physical removal
|
||||
// or an unsettled journal row is maintenance debt the reconciler owns, so
|
||||
// it is observed as such instead of as a clean success.
|
||||
const settled = removed.ok && completed !== null && completed.ok;
|
||||
const debtCode = removed.ok
|
||||
? completed !== null && !completed.ok
|
||||
? completed.error.code
|
||||
: undefined
|
||||
: removed.error.code;
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_DELETE",
|
||||
outcome: settled ? "SUCCEEDED" : "DEGRADED",
|
||||
...(debtCode ? { failureCode: debtCode } : {}),
|
||||
});
|
||||
return browserDataSuccess(undefined);
|
||||
);
|
||||
if (!committed.ok) {
|
||||
return observeFailure(
|
||||
rebaseFailure(committed.error, "OBJECT_DELETE"),
|
||||
dependencies.observer,
|
||||
);
|
||||
}
|
||||
const removed = await dependencies.worker.removeObject(
|
||||
dependencies.scope,
|
||||
request.objectId,
|
||||
request.expectedGeneration,
|
||||
request.signal,
|
||||
);
|
||||
const completed = removed.ok
|
||||
? await dependencies.journal.complete(
|
||||
transactionId,
|
||||
begun.value.fencingToken,
|
||||
)
|
||||
: null;
|
||||
// Logical deletion is already committed. Physical cleanup is retryable
|
||||
// maintenance and must not make the caller repeat a non-idempotent delete.
|
||||
// NS-04. It is still not a settled state: an unfinished physical removal
|
||||
// or an unsettled journal row is maintenance debt the reconciler owns, so
|
||||
// it is observed as such instead of as a clean success.
|
||||
const settled = removed.ok && completed !== null && completed.ok;
|
||||
const debtCode = removed.ok
|
||||
? completed !== null && !completed.ok
|
||||
? completed.error.code
|
||||
: undefined
|
||||
: removed.error.code;
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_DELETE",
|
||||
outcome: settled ? "SUCCEEDED" : "DEGRADED",
|
||||
...(debtCode ? { failureCode: debtCode } : {}),
|
||||
});
|
||||
return browserDataSuccess(undefined);
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
@@ -563,14 +589,14 @@ export function createOpfsByteStoreAdapter(
|
||||
const maxTransactions =
|
||||
request.maxTransactions ?? policy.reconciliationBatchSize;
|
||||
const deadline = now() + budgetMs;
|
||||
const page = await dependencies.journal.listIncomplete(maxTransactions);
|
||||
if (!page.ok) {
|
||||
return rebaseFailure(page.error, "OBJECT_RECONCILE");
|
||||
const listed = await dependencies.journal.listIncomplete(maxTransactions);
|
||||
if (!listed.ok) {
|
||||
return rebaseFailure(listed.error, "OBJECT_RECONCILE");
|
||||
}
|
||||
observeOpfsSafely(dependencies.observer, {
|
||||
operation: "OBJECT_RECONCILE",
|
||||
outcome: "STARTED",
|
||||
transactionBucket: transactionBucket(page.value.transactions.length),
|
||||
transactionBucket: transactionBucket(listed.value.transactions.length),
|
||||
});
|
||||
|
||||
const report: {
|
||||
@@ -586,6 +612,7 @@ export function createOpfsByteStoreAdapter(
|
||||
| "STAGING_STATE_UNREADABLE";
|
||||
moreTransactionsAvailable: boolean;
|
||||
deadlineReached: boolean;
|
||||
skippedLiveTransactions: number;
|
||||
} = {
|
||||
inspectedTransactions: 0,
|
||||
committedTransactions: 0,
|
||||
@@ -594,26 +621,58 @@ export function createOpfsByteStoreAdapter(
|
||||
inspectedOrphanChunks: 0,
|
||||
deletedOrphanChunks: 0,
|
||||
orphanGcStatus: "COMPLETED",
|
||||
moreTransactionsAvailable: page.value.moreAvailable,
|
||||
moreTransactionsAvailable: listed.value.moreAvailable,
|
||||
deadlineReached: false,
|
||||
skippedLiveTransactions: 0,
|
||||
};
|
||||
for (const transaction of page.value.transactions) {
|
||||
if (request.signal?.aborted) {
|
||||
return browserDataFailure("ABORTED", "OBJECT_RECONCILE");
|
||||
// 05-01 (D-3). A row is reconciled only while this call holds its
|
||||
// per-transaction lock, so a put or delete still running in any tab is
|
||||
// skipped. The page is read again under those locks: a row listed while
|
||||
// its owner still ran may have moved on before the owner let go.
|
||||
const held = new Map<string, () => void>();
|
||||
try {
|
||||
for (const transaction of listed.value.transactions) {
|
||||
let release: (() => void) | null;
|
||||
try {
|
||||
release = await holdTransactionLock(transaction.transactionId);
|
||||
} catch {
|
||||
return browserDataFailure("UNAVAILABLE", "OBJECT_RECONCILE", {
|
||||
retryable: true,
|
||||
recovery: "RETRY",
|
||||
});
|
||||
}
|
||||
if (release) held.set(transaction.transactionId, release);
|
||||
else report.skippedLiveTransactions += 1;
|
||||
}
|
||||
if (now() >= deadline) {
|
||||
report.deadlineReached = true;
|
||||
report.moreTransactionsAvailable = true;
|
||||
break;
|
||||
const page =
|
||||
held.size === 0
|
||||
? listed
|
||||
: await dependencies.journal.listIncomplete(maxTransactions);
|
||||
if (!page.ok) {
|
||||
return rebaseFailure(page.error, "OBJECT_RECONCILE");
|
||||
}
|
||||
report.inspectedTransactions += 1;
|
||||
const reconciled = await reconcileTransaction(transaction, request.signal);
|
||||
if (!reconciled.ok) {
|
||||
return observeFailure(reconciled, dependencies.observer);
|
||||
report.moreTransactionsAvailable = page.value.moreAvailable;
|
||||
for (const transaction of page.value.transactions) {
|
||||
if (!held.has(transaction.transactionId)) continue;
|
||||
if (request.signal?.aborted) {
|
||||
return browserDataFailure("ABORTED", "OBJECT_RECONCILE");
|
||||
}
|
||||
if (now() >= deadline) {
|
||||
report.deadlineReached = true;
|
||||
report.moreTransactionsAvailable = true;
|
||||
break;
|
||||
}
|
||||
report.inspectedTransactions += 1;
|
||||
const reconciled = await reconcileTransaction(transaction, request.signal);
|
||||
if (!reconciled.ok) {
|
||||
return observeFailure(reconciled, dependencies.observer);
|
||||
}
|
||||
report.committedTransactions += reconciled.value.committed;
|
||||
report.rolledBackTransactions += reconciled.value.rolledBack;
|
||||
report.cleanedTransactions += reconciled.value.cleaned;
|
||||
}
|
||||
report.committedTransactions += reconciled.value.committed;
|
||||
report.rolledBackTransactions += reconciled.value.rolledBack;
|
||||
report.cleanedTransactions += reconciled.value.cleaned;
|
||||
} finally {
|
||||
for (const release of held.values()) release();
|
||||
}
|
||||
if (now() >= deadline) {
|
||||
report.deadlineReached = true;
|
||||
@@ -905,6 +964,56 @@ export function createOpfsByteStoreAdapter(
|
||||
|
||||
return Object.freeze({ objects, maintenance });
|
||||
|
||||
/**
|
||||
* Takes the exclusive per-transaction Web Lock only if nobody holds it.
|
||||
* Resolves with its release, or `null` while another context holds it.
|
||||
*/
|
||||
async function holdTransactionLock(
|
||||
transactionId: string,
|
||||
): Promise<(() => void) | null> {
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return await new Promise((resolve, reject) => {
|
||||
requestLock(
|
||||
`${policy.mutationLockName}:transaction:${scope.authorityToken}:${transactionId}`,
|
||||
{ mode: "exclusive", ifAvailable: true },
|
||||
async (lock) => {
|
||||
if (!lock) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
resolve(release);
|
||||
await held;
|
||||
},
|
||||
).catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function withTransactionLock<Value>(
|
||||
transactionId: string,
|
||||
operation: "OBJECT_WRITE" | "OBJECT_DELETE",
|
||||
run: () => Promise<BrowserDataResult<Value>>,
|
||||
): Promise<BrowserDataResult<Value>> {
|
||||
let release: (() => void) | null;
|
||||
try {
|
||||
release = await holdTransactionLock(transactionId);
|
||||
} catch {
|
||||
return browserDataFailure("UNAVAILABLE", operation, {
|
||||
retryable: true,
|
||||
recovery: "RETRY",
|
||||
});
|
||||
}
|
||||
// A fresh transaction id is never held; a holder means a duplicate id.
|
||||
if (!release) return browserDataFailure("CONFLICT", operation);
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* STO-01. The compensating half of the put saga.
|
||||
*
|
||||
|
||||
@@ -100,6 +100,11 @@ export type OpfsReconciliationReport = Readonly<{
|
||||
| "STAGING_STATE_UNREADABLE";
|
||||
moreTransactionsAvailable: boolean;
|
||||
deadlineReached: boolean;
|
||||
/**
|
||||
* 05-01. Rows whose owner still holds the per-transaction lock. They are
|
||||
* a live put or delete, not debris, and are left for a later reconcile.
|
||||
*/
|
||||
skippedLiveTransactions: number;
|
||||
}>;
|
||||
|
||||
export type OpfsPolicyMaintenanceReport = Readonly<{
|
||||
|
||||
@@ -219,3 +219,193 @@ test("runs put, open, remove and reconcile through a native DedicatedWorker, OPF
|
||||
expect(result.deletion).toBe("DELETED");
|
||||
expect(["DELETED", "ABSENT"]).toContain(result.opfsCleanup);
|
||||
});
|
||||
|
||||
type OpfsTabScope = Readonly<{
|
||||
namespace: string;
|
||||
authorityToken: string;
|
||||
namespaceToken: string;
|
||||
partitionToken: string;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* Installs an OPFS runtime for `scope` in this tab and keeps it on
|
||||
* `globalThis.__opfsTab` for later evaluate calls.
|
||||
*/
|
||||
async function installOpfsTab(
|
||||
page: import("@playwright/test").Page,
|
||||
scope: OpfsTabScope,
|
||||
): Promise<boolean> {
|
||||
return await page.evaluate(async (tabScope) => {
|
||||
const runtimeModulePath =
|
||||
"/src/adapters/storage/opfs/browser-opfs-runtime.ts";
|
||||
const policyModulePath = "/src/adapters/storage/opfs/opfs-policy.ts";
|
||||
const { createBrowserOpfsRuntime, inspectBrowserOpfsSupport } =
|
||||
(await import(
|
||||
/* @vite-ignore */ runtimeModulePath
|
||||
)) as typeof import("../../src/adapters/storage/opfs/browser-opfs-runtime.ts");
|
||||
const { resolveOpfsRuntimePolicy } = (await import(
|
||||
/* @vite-ignore */ policyModulePath
|
||||
)) as typeof import("../../src/adapters/storage/opfs/opfs-policy.ts");
|
||||
const policy = resolveOpfsRuntimePolicy();
|
||||
if (!inspectBrowserOpfsSupport(policy).ok) return false;
|
||||
(globalThis as unknown as { __opfsTab: unknown }).__opfsTab =
|
||||
createBrowserOpfsRuntime({
|
||||
workerUrl: new URL(
|
||||
"/tests/browser-capabilities/opfs-test.worker.ts",
|
||||
location.href,
|
||||
),
|
||||
workerName: `opfs-tab-${crypto.randomUUID()}`,
|
||||
scope: tabScope,
|
||||
storagePolicy: {
|
||||
owner: "browser-capability",
|
||||
namespace: tabScope.namespace,
|
||||
classification: "PERSONAL",
|
||||
authority: "LOCAL_FIRST",
|
||||
accountScope: "OPAQUE_PARTITION",
|
||||
retention: { kind: "EXPLICIT_DELETE" },
|
||||
softBudgetBytes: 1024 * 1024,
|
||||
hardBudgetBytes: 2 * 1024 * 1024,
|
||||
evictionPriority: "USER_AUTHORED",
|
||||
logoutAction: "EXPORT_THEN_PURGE",
|
||||
accountDeletionAction: "PURGE_PARTITION",
|
||||
pressureAction: "RETAIN",
|
||||
unavailableFallback: "EXPORT_REQUIRED",
|
||||
},
|
||||
policy,
|
||||
});
|
||||
return true;
|
||||
}, scope);
|
||||
}
|
||||
|
||||
test("05-01 a reconcile in another tab skips a live put and the put stays readable", async ({
|
||||
page,
|
||||
}) => {
|
||||
const secondPage = await page.context().newPage();
|
||||
const suffix = crypto.randomUUID().replaceAll("-", "");
|
||||
const scope: OpfsTabScope = {
|
||||
namespace: "durable-objects",
|
||||
authorityToken: `authority_${suffix}`,
|
||||
namespaceToken: `namespace_${suffix}`,
|
||||
partitionToken: `partition_${suffix}`,
|
||||
};
|
||||
type Tab = import("../../src/adapters/storage/opfs/browser-opfs-runtime.ts").BrowserOpfsRuntime;
|
||||
try {
|
||||
await Promise.all([page.goto("/"), secondPage.goto("/")]);
|
||||
if (!(await installOpfsTab(page, scope))) return;
|
||||
expect(await installOpfsTab(secondPage, scope)).toBe(true);
|
||||
|
||||
// Tab A starts a put whose source stalls, so its journal row stays
|
||||
// PREPARING and its worker holds the OPFS mutation lock.
|
||||
await page.evaluate(async () => {
|
||||
const state = globalThis as unknown as {
|
||||
__opfsTab: Tab;
|
||||
__release?: () => void;
|
||||
__put?: Promise<unknown>;
|
||||
};
|
||||
let streaming!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
streaming = resolve;
|
||||
});
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
state.__release = resolve;
|
||||
});
|
||||
state.__put = state.__opfsTab.objects.put({
|
||||
objectId: "object_tab_12345678",
|
||||
expectedGeneration: null,
|
||||
mediaType: "application/octet-stream",
|
||||
source: {
|
||||
byteLength: 4,
|
||||
async *stream() {
|
||||
streaming();
|
||||
await gate;
|
||||
yield { ok: true as const, value: new Uint8Array([2, 7, 1, 8]) };
|
||||
},
|
||||
},
|
||||
});
|
||||
await started;
|
||||
});
|
||||
|
||||
// Tab B reconciles while tab A's put is in flight.
|
||||
await secondPage.evaluate(() => {
|
||||
const state = globalThis as unknown as {
|
||||
__opfsTab: Tab;
|
||||
__reconcile?: Promise<unknown>;
|
||||
};
|
||||
state.__reconcile = state.__opfsTab.maintenance.reconcile();
|
||||
});
|
||||
await page.waitForTimeout(250);
|
||||
const put = await page.evaluate(async () => {
|
||||
const state = globalThis as unknown as {
|
||||
__release: () => void;
|
||||
__put: Promise<unknown>;
|
||||
};
|
||||
state.__release();
|
||||
return await state.__put;
|
||||
});
|
||||
const reconciled = await secondPage.evaluate(
|
||||
async () =>
|
||||
await (globalThis as unknown as { __reconcile: Promise<unknown> })
|
||||
.__reconcile,
|
||||
);
|
||||
const opened = await page.evaluate(async () => {
|
||||
const runtime = (globalThis as unknown as { __opfsTab: Tab }).__opfsTab;
|
||||
const result = await runtime.objects.open({
|
||||
objectId: "object_tab_12345678",
|
||||
});
|
||||
if (!result.ok) return result.error.code;
|
||||
const bytes: number[] = [];
|
||||
for await (const chunk of result.value.source.stream(
|
||||
new AbortController().signal,
|
||||
)) {
|
||||
if (!chunk.ok) return chunk.error.code;
|
||||
bytes.push(...chunk.value);
|
||||
}
|
||||
return bytes;
|
||||
});
|
||||
|
||||
expect(put).toMatchObject({ ok: true, value: { generation: 1 } });
|
||||
expect(reconciled).toMatchObject({
|
||||
ok: true,
|
||||
value: { skippedLiveTransactions: 1, rolledBackTransactions: 0 },
|
||||
});
|
||||
expect(opened).toEqual([2, 7, 1, 8]);
|
||||
} finally {
|
||||
for (const tab of [page, secondPage]) {
|
||||
await tab.evaluate(() =>
|
||||
(globalThis as unknown as { __opfsTab?: Tab }).__opfsTab?.close(),
|
||||
);
|
||||
}
|
||||
const cleanup = await page.evaluate(async (tabScope) => {
|
||||
const journalModulePath =
|
||||
"/src/adapters/storage/opfs/indexeddb-opfs-journal.ts";
|
||||
const policyModulePath = "/src/adapters/storage/opfs/opfs-policy.ts";
|
||||
const { opfsJournalDatabaseName } = (await import(
|
||||
/* @vite-ignore */ journalModulePath
|
||||
)) as typeof import("../../src/adapters/storage/opfs/indexeddb-opfs-journal.ts");
|
||||
const { DEFAULT_OPFS_RUNTIME_POLICY } = (await import(
|
||||
/* @vite-ignore */ policyModulePath
|
||||
)) as typeof import("../../src/adapters/storage/opfs/opfs-policy.ts");
|
||||
const deletion = await new Promise<string>((resolve) => {
|
||||
const request = indexedDB.deleteDatabase(
|
||||
opfsJournalDatabaseName(tabScope.authorityToken),
|
||||
);
|
||||
request.onsuccess = () => resolve("DELETED");
|
||||
request.onerror = () => resolve(request.error?.name ?? "UNKNOWN");
|
||||
request.onblocked = () => resolve("BLOCKED");
|
||||
});
|
||||
try {
|
||||
const root = await (
|
||||
await navigator.storage.getDirectory()
|
||||
).getDirectoryHandle(DEFAULT_OPFS_RUNTIME_POLICY.rootDirectoryName);
|
||||
await (
|
||||
await root.getDirectoryHandle("authorities")
|
||||
).removeEntry(tabScope.authorityToken, { recursive: true });
|
||||
} catch {
|
||||
// Nothing was written for this authority.
|
||||
}
|
||||
return deletion;
|
||||
}, scope);
|
||||
expect(cleanup).toBe("DELETED");
|
||||
await secondPage.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -666,6 +666,57 @@ describe("OPFS byte-store coordinator", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("05-01 decides a reconciled row from what the journal holds once its lock is taken", async () => {
|
||||
// Listed while its owner still ran, the row was PREPARING. By the time
|
||||
// the owner let go it had committed and only finalization failed, so the
|
||||
// staging receipt names the committed generation. Cleaning it up from the
|
||||
// stale snapshot would delete a committed object.
|
||||
const journal = createJournal();
|
||||
const row: OpfsJournalTransaction = {
|
||||
transactionId: "transaction_12345678",
|
||||
fencingToken: "fencing_12345678",
|
||||
mutation: "PUT",
|
||||
scope,
|
||||
phase: "PREPARING",
|
||||
objectId: "object_12345678",
|
||||
expectedGeneration: null,
|
||||
targetGeneration: 1,
|
||||
targetByteLength: 1,
|
||||
targetStoragePolicy: storagePolicy,
|
||||
budgetReservation: {
|
||||
namespace: scope.namespace,
|
||||
reservedBytes: 1,
|
||||
hardBudgetBytes: storagePolicy.hardBudgetBytes,
|
||||
},
|
||||
startedAtEpochMs: 100,
|
||||
};
|
||||
journal.transactions.set(row.transactionId, {
|
||||
...row,
|
||||
phase: "COMMITTED",
|
||||
preparedObject: committedObject("object_12345678", storagePolicy, 100),
|
||||
});
|
||||
vi.spyOn(journal, "listIncomplete").mockResolvedValueOnce(
|
||||
browserDataSuccess({ transactions: [row], moreAvailable: false }),
|
||||
);
|
||||
const worker = createWorker();
|
||||
const cleanupTransaction = vi.spyOn(worker, "cleanupTransaction");
|
||||
const finalizePut = vi.spyOn(worker, "finalizePut");
|
||||
const adapter = createOpfsByteStoreAdapter({
|
||||
journal,
|
||||
worker,
|
||||
scope,
|
||||
storagePolicy,
|
||||
now: () => 100,
|
||||
});
|
||||
|
||||
expect(await adapter.maintenance.reconcile()).toMatchObject({
|
||||
ok: true,
|
||||
value: { inspectedTransactions: 1, rolledBackTransactions: 0 },
|
||||
});
|
||||
expect(cleanupTransaction).not.toHaveBeenCalled();
|
||||
expect(finalizePut).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not roll back journal after an unknown worker mutation effect", async () => {
|
||||
const journal = createJournal();
|
||||
const worker = createWorker({
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
} from "../../src/application/ports/browser-file-storage/shared.ts";
|
||||
import { browserDataSuccess } from "../../src/adapters/browser-file-storage/result.ts";
|
||||
import { createOpfsByteStoreAdapter } from "../../src/adapters/storage/opfs/opfs-byte-store-adapter.ts";
|
||||
import { createIndexedDbOpfsJournal } from "../../src/adapters/storage/opfs/indexeddb-opfs-journal.ts";
|
||||
import {
|
||||
resolveOpfsRuntimePolicy,
|
||||
} from "../../src/adapters/storage/opfs/opfs-policy.ts";
|
||||
@@ -33,6 +34,7 @@ import {
|
||||
type OpfsWorkerMessageHost,
|
||||
type OpfsWorkerRuntime,
|
||||
} from "../../src/adapters/storage/opfs/opfs-worker-runtime.ts";
|
||||
import { MemoryIndexedDbFactory } from "../helpers/memory-indexeddb.ts";
|
||||
|
||||
const scopeA: OpfsStorageScope = Object.freeze({
|
||||
namespace: "durable-objects",
|
||||
@@ -2083,3 +2085,102 @@ describe("NS-05 a bootstrap failure answers the request it belongs to", () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Two runtimes ("tabs") share one OPFS root, one exclusive worker lock and
|
||||
* one journal database, the way two tabs of one origin do.
|
||||
*/
|
||||
function sharedRootTabs(policy = runtimePolicy) {
|
||||
const root = new MemoryDirectory();
|
||||
const leases = exclusiveLeases();
|
||||
const idb = new MemoryIndexedDbFactory();
|
||||
const tab = (tag: string) => {
|
||||
const runtime = createOpfsWorkerRuntime({
|
||||
root: root as unknown as FileSystemDirectoryHandle,
|
||||
crypto: globalThis.crypto,
|
||||
policy,
|
||||
leaseManager: leases,
|
||||
dedicatedWorker: true,
|
||||
supportsSynchronousAccessHandles: false,
|
||||
});
|
||||
let sequence = 0;
|
||||
return createOpfsByteStoreAdapter({
|
||||
journal: createIndexedDbOpfsJournal({
|
||||
authorityToken: scopeA.authorityToken,
|
||||
factory: idb.factory,
|
||||
keyRange: idb.keyRange,
|
||||
}),
|
||||
worker: loopbackGateway(runtime, policy),
|
||||
scope: scopeA,
|
||||
storagePolicy,
|
||||
policy,
|
||||
createTransactionId: () => `transaction_${tag}_${++sequence}`,
|
||||
});
|
||||
};
|
||||
return { root, leases, idb, tab };
|
||||
}
|
||||
|
||||
async function openedBytes(
|
||||
adapter: ReturnType<typeof createOpfsByteStoreAdapter>,
|
||||
objectId: string,
|
||||
): Promise<Readonly<{ generation: number; bytes: number[] }> | string> {
|
||||
const opened = await adapter.objects.open({ objectId });
|
||||
if (!opened.ok) return opened.error.code;
|
||||
const bytes: number[] = [];
|
||||
for await (const chunk of opened.value.source.stream(
|
||||
new AbortController().signal,
|
||||
)) {
|
||||
if (!chunk.ok) return chunk.error.code;
|
||||
bytes.push(...chunk.value);
|
||||
}
|
||||
return { generation: opened.value.descriptor.generation, bytes };
|
||||
}
|
||||
|
||||
describe("05-01 reconcile leaves a live put's journal row alone", () => {
|
||||
it.each([
|
||||
["a first put", null],
|
||||
["an overwrite", 1],
|
||||
] as const)(
|
||||
"keeps %s that reconcile overlapped in another tab",
|
||||
async (_label, expectedGeneration) => {
|
||||
const { tab } = sharedRootTabs();
|
||||
const tabA = tab("a");
|
||||
const tabB = tab("b");
|
||||
if (expectedGeneration !== null) {
|
||||
expect(
|
||||
await tabA.objects.put({
|
||||
objectId: "object_12345678",
|
||||
expectedGeneration: null,
|
||||
mediaType: "application/octet-stream",
|
||||
source: oneByteSource(),
|
||||
}),
|
||||
).toMatchObject({ ok: true, value: { generation: 1 } });
|
||||
}
|
||||
|
||||
const stalled = stalledSource();
|
||||
const putA = tabA.objects.put({
|
||||
objectId: "object_12345678",
|
||||
expectedGeneration,
|
||||
mediaType: "application/octet-stream",
|
||||
source: stalled.source,
|
||||
});
|
||||
// Tab A holds the worker lock and its journal row is PREPARING.
|
||||
await stalled.streaming;
|
||||
const reconcileB = tabB.maintenance.reconcile();
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
stalled.release();
|
||||
|
||||
const [put, reconciled] = await Promise.all([putA, reconcileB]);
|
||||
const generation = (expectedGeneration ?? 0) + 1;
|
||||
expect(put).toMatchObject({ ok: true, value: { generation } });
|
||||
expect(reconciled).toMatchObject({
|
||||
ok: true,
|
||||
value: { skippedLiveTransactions: 1, rolledBackTransactions: 0 },
|
||||
});
|
||||
expect(await openedBytes(tabA, "object_12345678")).toEqual({
|
||||
generation,
|
||||
bytes: [5],
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user