From 9f1dc51d894a5f1df9de02a159661782e3b045e1 Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Thu, 24 Sep 2026 13:07:45 +0900 Subject: [PATCH] perf(storage): open OPFS objects after a manifest-only check (Q-6.5-6) openObject used to send VERIFY_OBJECT, which read and hashed every chunk, and then streamed the object with READ_CHUNK, which read and hashed every chunk again. A large object did twice the I/O before the first byte and could exceed the RPC timeout during the pre-check. VERIFY_OBJECT now takes an optional `manifestOnly` flag. openObject sends it, so the pre-check only compares the stored manifest with the caller's copy. Integrity is kept on the stream: READ_CHUNK still hashes each chunk, and at the last chunk it also checks the chunk list against the root digest before releasing the bytes, so a stream never ends as a success on an unverified object. An empty object has no chunk to read, so its root digest is still checked at open. verifyObject (reconcile) keeps the full check. Verified: new tests in tests/unit/opfs-worker-runtime.test.ts were RED (open read each chunk; a corrupted chunk or forged root failed at open) and are GREEN (open reads no chunk bytes, each chunk is read once, a corrupted chunk and a forged root digest fail the stream, empty objects are checked at open, verifyObject still detects a corrupted chunk). All gates pass. Co-Authored-By: Claude Opus 5.5 --- .../storage/opfs/opfs-worker-client.ts | 2 +- .../storage/opfs/opfs-worker-protocol.ts | 6 + .../storage/opfs/opfs-worker-runtime.ts | 39 +++-- tests/unit/opfs-worker-runtime.test.ts | 161 ++++++++++++++++++ 4 files changed, 196 insertions(+), 12 deletions(-) diff --git a/src/adapters/storage/opfs/opfs-worker-client.ts b/src/adapters/storage/opfs/opfs-worker-client.ts index ba90fac0..9707ce79 100644 --- a/src/adapters/storage/opfs/opfs-worker-client.ts +++ b/src/adapters/storage/opfs/opfs-worker-client.ts @@ -342,7 +342,7 @@ export function createOpfsWorkerGateway( ) { const verified = await invoke( "OBJECT_READ", - { kind: "VERIFY_OBJECT", preparedObject }, + { kind: "VERIFY_OBJECT", preparedObject, manifestOnly: true }, signal, [], (value) => (typeof value === "boolean" ? value : null), diff --git a/src/adapters/storage/opfs/opfs-worker-protocol.ts b/src/adapters/storage/opfs/opfs-worker-protocol.ts index 67e68f38..69a7892a 100644 --- a/src/adapters/storage/opfs/opfs-worker-protocol.ts +++ b/src/adapters/storage/opfs/opfs-worker-protocol.ts @@ -77,6 +77,12 @@ export type OpfsWorkerRequest = protocolVersion: typeof OPFS_WORKER_PROTOCOL_VERSION; kind: "VERIFY_OBJECT"; preparedObject: OpfsPreparedObject; + /** + * Q-6.5-6. Compare the stored manifest only. `openObject` sends it: the + * stream then verifies each chunk as it reads it and the root digest + * before it releases the last chunk, so an object is not read twice. + */ + manifestOnly?: true; }> | Readonly<{ requestId: string; diff --git a/src/adapters/storage/opfs/opfs-worker-runtime.ts b/src/adapters/storage/opfs/opfs-worker-runtime.ts index 2ee07d46..68bf2691 100644 --- a/src/adapters/storage/opfs/opfs-worker-runtime.ts +++ b/src/adapters/storage/opfs/opfs-worker-runtime.ts @@ -248,7 +248,10 @@ export function createOpfsWorkerRuntime( return success( request.requestId, request.kind, - await verifyObject(request.preparedObject), + await verifyObject( + request.preparedObject, + request.manifestOnly === true, + ), ); case "READ_CHUNK": return success( @@ -693,6 +696,7 @@ export function createOpfsWorkerRuntime( async function verifyObject( expected: OpfsPreparedObject, + manifestOnly: boolean, ): Promise { if (!isPreparedObject(expected)) return false; let stored: unknown; @@ -708,15 +712,10 @@ export function createOpfsWorkerRuntime( ) { return false; } - const rootDigestHex = await treeDigestHex( - dependencies.crypto, - expected.descriptor.integrity.chunkSizeBytes, - expected.descriptor.byteLength, - expected.chunks, - ); - if (rootDigestHex !== expected.descriptor.integrity.rootDigestHex) { - return false; - } + // Q-6.5-6. READ_CHUNK checks the root digest at the last chunk; an empty + // object has no chunk to read, so its stream ends here. + if (manifestOnly && expected.chunks.length > 0) return true; + if (!(await rootDigestMatches(expected))) return false; for (const chunk of expected.chunks) { let bytes: Uint8Array; try { @@ -737,6 +736,19 @@ export function createOpfsWorkerRuntime( return true; } + async function rootDigestMatches( + expected: OpfsPreparedObject, + ): Promise { + return ( + (await treeDigestHex( + dependencies.crypto, + expected.descriptor.integrity.chunkSizeBytes, + expected.descriptor.byteLength, + expected.chunks, + )) === expected.descriptor.integrity.rootDigestHex + ); + } + async function readVerifiedChunk( expected: OpfsPreparedObject, sequence: number, @@ -757,7 +769,12 @@ export function createOpfsWorkerRuntime( ); if ( bytes.byteLength !== reference.byteLength || - (await sha256Hex(dependencies.crypto, bytes)) !== reference.digestHex + (await sha256Hex(dependencies.crypto, bytes)) !== reference.digestHex || + // Q-6.5-6. The last chunk is not released until the chunk list is + // proven against the root digest, so a stream never ends as a success + // on an unverified object. + (sequence === expected.chunks.length - 1 && + !(await rootDigestMatches(expected))) ) { throw new OpfsRuntimeFailure("INTEGRITY_FAILED"); } diff --git a/tests/unit/opfs-worker-runtime.test.ts b/tests/unit/opfs-worker-runtime.test.ts index 65ef8b10..71f7a117 100644 --- a/tests/unit/opfs-worker-runtime.test.ts +++ b/tests/unit/opfs-worker-runtime.test.ts @@ -72,6 +72,7 @@ class MemoryFile { readonly kind = "file"; bytes = new Uint8Array(); lastModified = 0; + reads = 0; } class MemoryDirectory { @@ -152,6 +153,7 @@ function fileHandle(file: MemoryFile): FileSystemFileHandle { kind: "file", name: "memory", async getFile() { + file.reads += 1; const blob = new Blob([Uint8Array.from(file.bytes)]); Object.defineProperty(blob, "lastModified", { value: file.lastModified, @@ -1136,6 +1138,165 @@ describe("R-5.5 stored objects are checked against their own chunk size", () => }); }); +describe("Q-6.5-6 openObject checks the manifest up front and the bytes while streaming", () => { + const twoChunkPolicy = resolveOpfsRuntimePolicy({ + chunkSizeBytes: 64 * 1024, + maxObjectBytes: 128 * 1024, + maxChunkCount: 2, + orphanGracePeriodMs: 60_000, + }); + const payload = Uint8Array.from( + { length: 64 * 1024 + 1 }, + (_, index) => index % 251, + ); + + async function writeObject(bytes: Uint8Array) { + const root = new MemoryDirectory(); + const gateway = loopbackGateway( + createOpfsWorkerRuntime({ + root: root as unknown as FileSystemDirectoryHandle, + crypto: globalThis.crypto, + policy: twoChunkPolicy, + leaseManager: immediateLeases({ count: 0 }), + dedicatedWorker: true, + supportsSynchronousAccessHandles: false, + }), + twoChunkPolicy, + ); + const written = await gateway.preparePut( + putRequest("e", bytesSource(bytes)), + ); + if (!written.ok) throw new Error(written.error.code); + return { root, gateway, stored: written.value }; + } + + function filesNamed( + directory: MemoryDirectory, + matches: (name: string) => boolean, + ): MemoryFile[] { + const found: MemoryFile[] = []; + for (const [name, node] of directory.children) { + if (node instanceof MemoryDirectory) { + found.push(...filesNamed(node, matches)); + } else if (matches(name)) { + found.push(node); + } + } + return found; + } + + function chunkFile(root: MemoryDirectory, digestHex: string): MemoryFile { + const [file] = filesNamed(root, (name) => name === `${digestHex}.bin`); + if (!file) throw new Error("Chunk file is missing."); + return file; + } + + /** Rewrites the root digest in both the stored manifest and the caller's copy. */ + function withForgedRootDigest( + root: MemoryDirectory, + stored: OpfsPreparedObject, + ): OpfsPreparedObject { + const forged: OpfsPreparedObject = { + ...stored, + descriptor: { + ...stored.descriptor, + integrity: { + ...stored.descriptor.integrity, + rootDigestHex: "0".repeat(64), + }, + }, + }; + const [manifest] = filesNamed(root, (name) => name === "manifest.json"); + if (!manifest) throw new Error("Manifest is missing."); + manifest.bytes = new TextEncoder().encode(JSON.stringify(forged)); + return forged; + } + + async function drain(source: ByteSource) { + const results: Array<{ ok: true; bytes: number } | { ok: false; code: string }> = []; + for await (const chunk of source.stream(new AbortController().signal)) { + results.push( + chunk.ok + ? { ok: true, bytes: chunk.value.byteLength } + : { ok: false, code: chunk.error.code }, + ); + } + return results; + } + + it("opens without reading chunk bytes and reads each chunk once while streaming", async () => { + const { root, gateway, stored } = await writeObject(payload); + const chunks = stored.chunks.map((chunk) => + chunkFile(root, chunk.digestHex), + ); + expect(chunks).toHaveLength(2); + for (const file of chunks) file.reads = 0; + + const opened = await gateway.openObject(stored); + if (!opened.ok) throw new Error(opened.error.code); + expect(chunks.map((file) => file.reads)).toEqual([0, 0]); + + const bytes: number[] = []; + for await (const chunk of opened.value.stream( + new AbortController().signal, + )) { + if (!chunk.ok) throw new Error(chunk.error.code); + bytes.push(...chunk.value); + } + expect(bytes).toEqual([...payload]); + expect(chunks.map((file) => file.reads)).toEqual([1, 1]); + }); + + it("still fails the read at a corrupted chunk", async () => { + const { root, gateway, stored } = await writeObject(payload); + const second = chunkFile(root, stored.chunks[1]!.digestHex); + second.bytes = Uint8Array.from(second.bytes, (byte) => byte ^ 0xff); + + const opened = await gateway.openObject(stored); + if (!opened.ok) throw new Error(opened.error.code); + expect(await drain(opened.value)).toEqual([ + { ok: true, bytes: 64 * 1024 }, + { ok: false, code: "INTEGRITY_FAILED" }, + ]); + }); + + it("checks the root digest before the last chunk is released", async () => { + const { root, gateway, stored } = await writeObject(payload); + const forged = withForgedRootDigest(root, stored); + + const opened = await gateway.openObject(forged); + if (!opened.ok) throw new Error(opened.error.code); + expect(await drain(opened.value)).toEqual([ + { ok: true, bytes: 64 * 1024 }, + { ok: false, code: "INTEGRITY_FAILED" }, + ]); + }); + + it("checks the root digest of an empty object at open, where its stream ends", async () => { + const { root, gateway, stored } = await writeObject(new Uint8Array()); + expect(stored.chunks).toHaveLength(0); + const intact = await gateway.openObject(stored); + expect(intact.ok && (await drain(intact.value))).toEqual([]); + + const forged = withForgedRootDigest(root, stored); + expect(await gateway.openObject(forged)).toMatchObject({ + ok: false, + error: { code: "INTEGRITY_FAILED" }, + }); + }); + + it("keeps verifyObject a full byte check", async () => { + const { root, gateway, stored } = await writeObject(payload); + const second = chunkFile(root, stored.chunks[1]!.digestHex); + second.bytes = Uint8Array.from(second.bytes, (byte) => byte ^ 0xff); + + expect(await gateway.verifyObject(stored)).toEqual({ + ok: true, + value: false, + }); + }); +}); + describe("OPFS worker client lifecycle", () => { it("rejects a colliding concurrent request id without replacing the first RPC", async () => { let listener: ((event: MessageEvent) => void) | undefined;