fix(storage): fail a put whose source stalls so the OPFS origin lock is freed (05-07)
A put holds the origin-wide mutation lock from BEGIN to FINISH, and the page waited on its source with no bound, so one stalled source blocked every tab's writes until their lock wait failed. Each wait for source bytes is now bounded by putIdleTimeoutMs; the put fails NOT_READABLE (retryable) and its compensation releases the lock. The port now says writes are one at a time. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7f45afac7d
commit
2b24f6d8f0
@@ -366,6 +366,11 @@ VD-15 목표 runtime을 먼저 구현한다.
|
||||
- committed logical row와 physical manifest/file 불일치
|
||||
- digest/length mismatch
|
||||
- worker crash, handle lock timeout, `NoModificationAllowedError`
|
||||
- put의 retryable `NOT_READABLE`: source가 `putIdleTimeoutMs`(기본 2분) 동안 다음
|
||||
bytes를 주지 않아 put을 끝내고 origin mutation lock을 풀었다(05-07,
|
||||
`opfs-worker-client.ts` `preparePut`, `opfs-policy.ts`). OPFS 쓰기는 origin
|
||||
전체에서 한 번에 하나라서, 이 bound가 없으면 멈춘 source 하나가 모든 tab의 쓰기를
|
||||
`mutationLockWaitTimeoutMs` 뒤 `BLOCKED`로 만든다
|
||||
- sensitive policy maintenance의 `POLICY_REJECTED`, proof replay/expiry 또는
|
||||
authority provider/consumer unavailable
|
||||
|
||||
|
||||
@@ -20,6 +20,13 @@ export type OpfsRuntimePolicy = Readonly<{
|
||||
* is bounded on its own and never counted against `rpcTimeoutMs`.
|
||||
*/
|
||||
mutationLockWaitTimeoutMs: number;
|
||||
/**
|
||||
* 05-07. How long a put may wait for its source's next bytes. The put holds
|
||||
* the origin mutation lock from BEGIN to FINISH, so a stalled source would
|
||||
* otherwise stop every tab's writes until their lock wait ran out. Keep it
|
||||
* below `mutationLockWaitTimeoutMs` so a queued writer outlasts it.
|
||||
*/
|
||||
putIdleTimeoutMs: number;
|
||||
reconciliationBudgetMs: number;
|
||||
reconciliationBatchSize: number;
|
||||
orphanGracePeriodMs: number;
|
||||
@@ -56,6 +63,7 @@ export const DEFAULT_OPFS_RUNTIME_POLICY: OpfsRuntimePolicy = Object.freeze({
|
||||
maxChunkCount: 512,
|
||||
rpcTimeoutMs: 60_000,
|
||||
mutationLockWaitTimeoutMs: 10 * 60_000,
|
||||
putIdleTimeoutMs: 2 * 60_000,
|
||||
reconciliationBudgetMs: 5_000,
|
||||
reconciliationBatchSize: 100,
|
||||
orphanGracePeriodMs: 24 * 60 * 60 * 1_000,
|
||||
@@ -94,6 +102,9 @@ export function assertOpfsRuntimePolicy(policy: OpfsRuntimePolicy): void {
|
||||
!Number.isSafeInteger(policy.mutationLockWaitTimeoutMs) ||
|
||||
policy.mutationLockWaitTimeoutMs < 1_000 ||
|
||||
policy.mutationLockWaitTimeoutMs > 60 * 60_000 ||
|
||||
!Number.isSafeInteger(policy.putIdleTimeoutMs) ||
|
||||
policy.putIdleTimeoutMs < 1_000 ||
|
||||
policy.putIdleTimeoutMs > 60 * 60_000 ||
|
||||
!Number.isSafeInteger(policy.reconciliationBudgetMs) ||
|
||||
policy.reconciliationBudgetMs < 1 ||
|
||||
policy.reconciliationBudgetMs > 60_000 ||
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
browserDataFailure,
|
||||
browserDataSuccess,
|
||||
} from "../../browser-file-storage/result.ts";
|
||||
import { createAbortableOperation } from "../../platform/abortable-operation.ts";
|
||||
import type { OpfsRuntimePolicy } from "./opfs-policy.ts";
|
||||
import { isPreparedObject } from "./opfs-prepared-object.ts";
|
||||
import {
|
||||
@@ -272,13 +273,36 @@ export function createOpfsWorkerGateway(
|
||||
let sequence = 0;
|
||||
let transferredBytes = 0;
|
||||
notifyProgress(request, "TRANSFERRING", 0);
|
||||
// 05-07. BEGIN took the origin mutation lock, so every wait for the
|
||||
// source is bounded by `putIdleTimeoutMs`. A source that stops fails the
|
||||
// put, and the coordinator's compensation releases the lock.
|
||||
const sourceStop = new AbortController();
|
||||
const stopSource = () => sourceStop.abort();
|
||||
request.signal?.addEventListener("abort", stopSource, { once: true });
|
||||
if (request.signal?.aborted) stopSource();
|
||||
const chunks = rechunk(
|
||||
request.source.stream(sourceStop.signal),
|
||||
dependencies.policy.chunkSizeBytes,
|
||||
dependencies.policy.maxObjectBytes,
|
||||
request.signal,
|
||||
);
|
||||
try {
|
||||
for await (const chunk of rechunk(
|
||||
request.source.stream(requiredSignal(request.signal)),
|
||||
dependencies.policy.chunkSizeBytes,
|
||||
dependencies.policy.maxObjectBytes,
|
||||
request.signal,
|
||||
)) {
|
||||
for (;;) {
|
||||
const idle = createAbortableOperation({
|
||||
signal: request.signal,
|
||||
timeoutMs: dependencies.policy.putIdleTimeoutMs,
|
||||
});
|
||||
const next = await idle.race(chunks.next());
|
||||
idle.close();
|
||||
if (next.kind === "REJECTED") throw next.reason;
|
||||
if (next.kind === "TERMINAL") {
|
||||
stopSource();
|
||||
throw new OpfsRpcError(
|
||||
next.terminal === "DEADLINE" ? "NOT_READABLE" : "ABORTED",
|
||||
);
|
||||
}
|
||||
if (next.value.done) break;
|
||||
const chunk = next.value.value;
|
||||
const chunkByteLength = chunk.byteLength;
|
||||
const append = await invoke<void>(
|
||||
"OBJECT_WRITE",
|
||||
@@ -320,6 +344,8 @@ export function createOpfsWorkerGateway(
|
||||
error instanceof OpfsRpcError ? error.code : "NOT_READABLE",
|
||||
"OBJECT_WRITE",
|
||||
);
|
||||
} finally {
|
||||
request.signal?.removeEventListener("abort", stopSource);
|
||||
}
|
||||
},
|
||||
|
||||
@@ -585,10 +611,6 @@ async function* rechunk(
|
||||
}
|
||||
}
|
||||
|
||||
function requiredSignal(signal: AbortSignal | undefined): AbortSignal {
|
||||
return signal ?? new AbortController().signal;
|
||||
}
|
||||
|
||||
function notifyProgress(
|
||||
request: PreparePhysicalObjectRequest,
|
||||
phase: "TRANSFERRING" | "VERIFYING",
|
||||
|
||||
@@ -76,6 +76,12 @@ export type OpfsCapabilities = Readonly<{
|
||||
|
||||
export interface DurableObjectStorePort {
|
||||
capabilities(): Promise<BrowserDataResult<OpfsCapabilities>>;
|
||||
/**
|
||||
* 05-07. OPFS writes are one at a time across the whole origin: a put holds
|
||||
* the origin mutation lock while it streams, and every put, remove and GC in
|
||||
* every tab waits for it. A source that stops for the runtime's put idle
|
||||
* bound fails the put with a retryable `NOT_READABLE` and frees the lock.
|
||||
*/
|
||||
put(
|
||||
request: PutDurableObjectRequest,
|
||||
): Promise<BrowserDataResult<DurableObjectDescriptor>>;
|
||||
|
||||
@@ -2398,3 +2398,48 @@ describe("05-04 a LOGOUT or ACCOUNT_DELETION purge leaves no partition bytes beh
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("05-07 a put whose source stops lets go of the origin lock", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("fails the stalled put at the idle bound so another tab's put runs", async () => {
|
||||
const policy = resolveOpfsRuntimePolicy({
|
||||
...runtimePolicy,
|
||||
putIdleTimeoutMs: 30_000,
|
||||
});
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { tab, leases } = sharedRootTabs(policy);
|
||||
const tabA = tab("a");
|
||||
const tabB = tab("b");
|
||||
const stalled = stalledSource();
|
||||
const putA = tabA.objects.put({
|
||||
objectId: "object_stalled_01",
|
||||
expectedGeneration: null,
|
||||
mediaType: "application/octet-stream",
|
||||
source: stalled.source,
|
||||
});
|
||||
await stalled.streaming;
|
||||
expect(leases.held).toBe(true);
|
||||
const putB = tabB.objects.put({
|
||||
objectId: "object_waiting_01",
|
||||
expectedGeneration: null,
|
||||
mediaType: "application/octet-stream",
|
||||
source: oneByteSource(),
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(policy.putIdleTimeoutMs + 1);
|
||||
vi.useRealTimers();
|
||||
const settledA = await Promise.race([
|
||||
putA,
|
||||
new Promise((resolve) => setTimeout(() => resolve("STILL_RUNNING"), 2_000)),
|
||||
]);
|
||||
expect(settledA).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "NOT_READABLE", retryable: true },
|
||||
});
|
||||
expect(await putB).toMatchObject({ ok: true, value: { generation: 1 } });
|
||||
stalled.release();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user