perf(storage): open OPFS objects after a manifest-only check (Q-6.5-6)
openObject used to send VERIFY_OBJECT, which read and hashed every chunk, and then streamed the object with READ_CHUNK, which read and hashed every chunk again. A large object did twice the I/O before the first byte and could exceed the RPC timeout during the pre-check. VERIFY_OBJECT now takes an optional `manifestOnly` flag. openObject sends it, so the pre-check only compares the stored manifest with the caller's copy. Integrity is kept on the stream: READ_CHUNK still hashes each chunk, and at the last chunk it also checks the chunk list against the root digest before releasing the bytes, so a stream never ends as a success on an unverified object. An empty object has no chunk to read, so its root digest is still checked at open. verifyObject (reconcile) keeps the full check. Verified: new tests in tests/unit/opfs-worker-runtime.test.ts were RED (open read each chunk; a corrupted chunk or forged root failed at open) and are GREEN (open reads no chunk bytes, each chunk is read once, a corrupted chunk and a forged root digest fail the stream, empty objects are checked at open, verifyObject still detects a corrupted chunk). All gates pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b95314245d
commit
9f1dc51d89
@@ -342,7 +342,7 @@ export function createOpfsWorkerGateway(
|
||||
) {
|
||||
const verified = await invoke(
|
||||
"OBJECT_READ",
|
||||
{ kind: "VERIFY_OBJECT", preparedObject },
|
||||
{ kind: "VERIFY_OBJECT", preparedObject, manifestOnly: true },
|
||||
signal,
|
||||
[],
|
||||
(value) => (typeof value === "boolean" ? value : null),
|
||||
|
||||
@@ -77,6 +77,12 @@ export type OpfsWorkerRequest =
|
||||
protocolVersion: typeof OPFS_WORKER_PROTOCOL_VERSION;
|
||||
kind: "VERIFY_OBJECT";
|
||||
preparedObject: OpfsPreparedObject;
|
||||
/**
|
||||
* Q-6.5-6. Compare the stored manifest only. `openObject` sends it: the
|
||||
* stream then verifies each chunk as it reads it and the root digest
|
||||
* before it releases the last chunk, so an object is not read twice.
|
||||
*/
|
||||
manifestOnly?: true;
|
||||
}>
|
||||
| Readonly<{
|
||||
requestId: string;
|
||||
|
||||
@@ -248,7 +248,10 @@ export function createOpfsWorkerRuntime(
|
||||
return success(
|
||||
request.requestId,
|
||||
request.kind,
|
||||
await verifyObject(request.preparedObject),
|
||||
await verifyObject(
|
||||
request.preparedObject,
|
||||
request.manifestOnly === true,
|
||||
),
|
||||
);
|
||||
case "READ_CHUNK":
|
||||
return success(
|
||||
@@ -693,6 +696,7 @@ export function createOpfsWorkerRuntime(
|
||||
|
||||
async function verifyObject(
|
||||
expected: OpfsPreparedObject,
|
||||
manifestOnly: boolean,
|
||||
): Promise<boolean> {
|
||||
if (!isPreparedObject(expected)) return false;
|
||||
let stored: unknown;
|
||||
@@ -708,15 +712,10 @@ export function createOpfsWorkerRuntime(
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const rootDigestHex = await treeDigestHex(
|
||||
dependencies.crypto,
|
||||
expected.descriptor.integrity.chunkSizeBytes,
|
||||
expected.descriptor.byteLength,
|
||||
expected.chunks,
|
||||
);
|
||||
if (rootDigestHex !== expected.descriptor.integrity.rootDigestHex) {
|
||||
return false;
|
||||
}
|
||||
// Q-6.5-6. READ_CHUNK checks the root digest at the last chunk; an empty
|
||||
// object has no chunk to read, so its stream ends here.
|
||||
if (manifestOnly && expected.chunks.length > 0) return true;
|
||||
if (!(await rootDigestMatches(expected))) return false;
|
||||
for (const chunk of expected.chunks) {
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
@@ -737,6 +736,19 @@ export function createOpfsWorkerRuntime(
|
||||
return true;
|
||||
}
|
||||
|
||||
async function rootDigestMatches(
|
||||
expected: OpfsPreparedObject,
|
||||
): Promise<boolean> {
|
||||
return (
|
||||
(await treeDigestHex(
|
||||
dependencies.crypto,
|
||||
expected.descriptor.integrity.chunkSizeBytes,
|
||||
expected.descriptor.byteLength,
|
||||
expected.chunks,
|
||||
)) === expected.descriptor.integrity.rootDigestHex
|
||||
);
|
||||
}
|
||||
|
||||
async function readVerifiedChunk(
|
||||
expected: OpfsPreparedObject,
|
||||
sequence: number,
|
||||
@@ -757,7 +769,12 @@ export function createOpfsWorkerRuntime(
|
||||
);
|
||||
if (
|
||||
bytes.byteLength !== reference.byteLength ||
|
||||
(await sha256Hex(dependencies.crypto, bytes)) !== reference.digestHex
|
||||
(await sha256Hex(dependencies.crypto, bytes)) !== reference.digestHex ||
|
||||
// Q-6.5-6. The last chunk is not released until the chunk list is
|
||||
// proven against the root digest, so a stream never ends as a success
|
||||
// on an unverified object.
|
||||
(sequence === expected.chunks.length - 1 &&
|
||||
!(await rootDigestMatches(expected)))
|
||||
) {
|
||||
throw new OpfsRuntimeFailure("INTEGRITY_FAILED");
|
||||
}
|
||||
|
||||
@@ -72,6 +72,7 @@ class MemoryFile {
|
||||
readonly kind = "file";
|
||||
bytes = new Uint8Array();
|
||||
lastModified = 0;
|
||||
reads = 0;
|
||||
}
|
||||
|
||||
class MemoryDirectory {
|
||||
@@ -152,6 +153,7 @@ function fileHandle(file: MemoryFile): FileSystemFileHandle {
|
||||
kind: "file",
|
||||
name: "memory",
|
||||
async getFile() {
|
||||
file.reads += 1;
|
||||
const blob = new Blob([Uint8Array.from(file.bytes)]);
|
||||
Object.defineProperty(blob, "lastModified", {
|
||||
value: file.lastModified,
|
||||
@@ -1136,6 +1138,165 @@ describe("R-5.5 stored objects are checked against their own chunk size", () =>
|
||||
});
|
||||
});
|
||||
|
||||
describe("Q-6.5-6 openObject checks the manifest up front and the bytes while streaming", () => {
|
||||
const twoChunkPolicy = resolveOpfsRuntimePolicy({
|
||||
chunkSizeBytes: 64 * 1024,
|
||||
maxObjectBytes: 128 * 1024,
|
||||
maxChunkCount: 2,
|
||||
orphanGracePeriodMs: 60_000,
|
||||
});
|
||||
const payload = Uint8Array.from(
|
||||
{ length: 64 * 1024 + 1 },
|
||||
(_, index) => index % 251,
|
||||
);
|
||||
|
||||
async function writeObject(bytes: Uint8Array) {
|
||||
const root = new MemoryDirectory();
|
||||
const gateway = loopbackGateway(
|
||||
createOpfsWorkerRuntime({
|
||||
root: root as unknown as FileSystemDirectoryHandle,
|
||||
crypto: globalThis.crypto,
|
||||
policy: twoChunkPolicy,
|
||||
leaseManager: immediateLeases({ count: 0 }),
|
||||
dedicatedWorker: true,
|
||||
supportsSynchronousAccessHandles: false,
|
||||
}),
|
||||
twoChunkPolicy,
|
||||
);
|
||||
const written = await gateway.preparePut(
|
||||
putRequest("e", bytesSource(bytes)),
|
||||
);
|
||||
if (!written.ok) throw new Error(written.error.code);
|
||||
return { root, gateway, stored: written.value };
|
||||
}
|
||||
|
||||
function filesNamed(
|
||||
directory: MemoryDirectory,
|
||||
matches: (name: string) => boolean,
|
||||
): MemoryFile[] {
|
||||
const found: MemoryFile[] = [];
|
||||
for (const [name, node] of directory.children) {
|
||||
if (node instanceof MemoryDirectory) {
|
||||
found.push(...filesNamed(node, matches));
|
||||
} else if (matches(name)) {
|
||||
found.push(node);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function chunkFile(root: MemoryDirectory, digestHex: string): MemoryFile {
|
||||
const [file] = filesNamed(root, (name) => name === `${digestHex}.bin`);
|
||||
if (!file) throw new Error("Chunk file is missing.");
|
||||
return file;
|
||||
}
|
||||
|
||||
/** Rewrites the root digest in both the stored manifest and the caller's copy. */
|
||||
function withForgedRootDigest(
|
||||
root: MemoryDirectory,
|
||||
stored: OpfsPreparedObject,
|
||||
): OpfsPreparedObject {
|
||||
const forged: OpfsPreparedObject = {
|
||||
...stored,
|
||||
descriptor: {
|
||||
...stored.descriptor,
|
||||
integrity: {
|
||||
...stored.descriptor.integrity,
|
||||
rootDigestHex: "0".repeat(64),
|
||||
},
|
||||
},
|
||||
};
|
||||
const [manifest] = filesNamed(root, (name) => name === "manifest.json");
|
||||
if (!manifest) throw new Error("Manifest is missing.");
|
||||
manifest.bytes = new TextEncoder().encode(JSON.stringify(forged));
|
||||
return forged;
|
||||
}
|
||||
|
||||
async function drain(source: ByteSource) {
|
||||
const results: Array<{ ok: true; bytes: number } | { ok: false; code: string }> = [];
|
||||
for await (const chunk of source.stream(new AbortController().signal)) {
|
||||
results.push(
|
||||
chunk.ok
|
||||
? { ok: true, bytes: chunk.value.byteLength }
|
||||
: { ok: false, code: chunk.error.code },
|
||||
);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
it("opens without reading chunk bytes and reads each chunk once while streaming", async () => {
|
||||
const { root, gateway, stored } = await writeObject(payload);
|
||||
const chunks = stored.chunks.map((chunk) =>
|
||||
chunkFile(root, chunk.digestHex),
|
||||
);
|
||||
expect(chunks).toHaveLength(2);
|
||||
for (const file of chunks) file.reads = 0;
|
||||
|
||||
const opened = await gateway.openObject(stored);
|
||||
if (!opened.ok) throw new Error(opened.error.code);
|
||||
expect(chunks.map((file) => file.reads)).toEqual([0, 0]);
|
||||
|
||||
const bytes: number[] = [];
|
||||
for await (const chunk of opened.value.stream(
|
||||
new AbortController().signal,
|
||||
)) {
|
||||
if (!chunk.ok) throw new Error(chunk.error.code);
|
||||
bytes.push(...chunk.value);
|
||||
}
|
||||
expect(bytes).toEqual([...payload]);
|
||||
expect(chunks.map((file) => file.reads)).toEqual([1, 1]);
|
||||
});
|
||||
|
||||
it("still fails the read at a corrupted chunk", async () => {
|
||||
const { root, gateway, stored } = await writeObject(payload);
|
||||
const second = chunkFile(root, stored.chunks[1]!.digestHex);
|
||||
second.bytes = Uint8Array.from(second.bytes, (byte) => byte ^ 0xff);
|
||||
|
||||
const opened = await gateway.openObject(stored);
|
||||
if (!opened.ok) throw new Error(opened.error.code);
|
||||
expect(await drain(opened.value)).toEqual([
|
||||
{ ok: true, bytes: 64 * 1024 },
|
||||
{ ok: false, code: "INTEGRITY_FAILED" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("checks the root digest before the last chunk is released", async () => {
|
||||
const { root, gateway, stored } = await writeObject(payload);
|
||||
const forged = withForgedRootDigest(root, stored);
|
||||
|
||||
const opened = await gateway.openObject(forged);
|
||||
if (!opened.ok) throw new Error(opened.error.code);
|
||||
expect(await drain(opened.value)).toEqual([
|
||||
{ ok: true, bytes: 64 * 1024 },
|
||||
{ ok: false, code: "INTEGRITY_FAILED" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("checks the root digest of an empty object at open, where its stream ends", async () => {
|
||||
const { root, gateway, stored } = await writeObject(new Uint8Array());
|
||||
expect(stored.chunks).toHaveLength(0);
|
||||
const intact = await gateway.openObject(stored);
|
||||
expect(intact.ok && (await drain(intact.value))).toEqual([]);
|
||||
|
||||
const forged = withForgedRootDigest(root, stored);
|
||||
expect(await gateway.openObject(forged)).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "INTEGRITY_FAILED" },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps verifyObject a full byte check", async () => {
|
||||
const { root, gateway, stored } = await writeObject(payload);
|
||||
const second = chunkFile(root, stored.chunks[1]!.digestHex);
|
||||
second.bytes = Uint8Array.from(second.bytes, (byte) => byte ^ 0xff);
|
||||
|
||||
expect(await gateway.verifyObject(stored)).toEqual({
|
||||
ok: true,
|
||||
value: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("OPFS worker client lifecycle", () => {
|
||||
it("rejects a colliding concurrent request id without replacing the first RPC", async () => {
|
||||
let listener: ((event: MessageEvent<unknown>) => void) | undefined;
|
||||
|
||||
Reference in New Issue
Block a user