import { describe, expect, it, vi } from "vitest"; import type { BrowserDataObservation } from "../../src/application/ports/browser-file-storage/shared.ts"; import type { PresignedUploadPartCapability } from "../../src/application/ports/browser-transfer/presigned-transfer.ts"; import type { ResumableUploadControlPlane, UploadPartExecutor, } from "../../src/application/ports/browser-transfer/resumable-upload.ts"; import { browserDataFailure, browserDataSuccess, } from "../../src/adapters/browser-file-storage/result.ts"; import { sha256Hex } from "../../src/adapters/browser-transfer/presigned/incremental-sha256.ts"; import { createPresignedUploadPartExecutor } from "../../src/adapters/browser-transfer/resumable-upload/presigned-upload-part-executor.ts"; import { createResumableUploadRuntime } from "../../src/adapters/browser-transfer/resumable-upload/resumable-upload-runtime.ts"; import { CONTROL_ENDPOINT, NOW, POLICY_HEADER, createHarness, jsonResponse, responseWithUrl, uploadCapabilityPayload, } from "./presigned-transfer-fixture.ts"; import { activeSignal, createControlHarness, executorFor, MemoryCheckpointStore, noContentionLock, rangeSource, runtimePolicy, type TestCapability, } from "./resumable-upload-runtime-fixture.ts"; const request = (uploadKey: string) => ({ uploadKey, purpose: "attachment", mediaType: "application/octet-stream", source: rangeSource(new Uint8Array([1, 2, 3, 4])), signal: activeSignal, }) as const; /** What `statusFailure(403)` hands the runtime through the part executor. */ const putForbidden = () => browserDataFailure("PERMISSION_DENIED", "PRESIGNED_TRANSFER", { retryable: false, recovery: "REISSUE_CAPABILITY", }); /** * The runtime composed with the real presigned provider, vault and executor, * over a fake BFF and object store. `putStatuses` are the object store's * answers to successive PUTs. */ function presignedRuntime( putStatuses: readonly number[], observer?: Readonly<{ record(observation: BrowserDataObservation): void }>, ) { const bytes = new Uint8Array([1, 2, 3, 4]); const checksum = sha256Hex(bytes); let issuance = 0; let puts = 0; const fetcher = vi.fn( async (input: RequestInfo | URL, init?: RequestInit) => { if (String(input) === CONTROL_ENDPOINT) { issuance += 1; const body = JSON.parse(String(init?.body)) as Readonly<{ binding: Readonly>; }>; return jsonResponse( uploadCapabilityPayload( { bytes, checksum }, { capabilityReceipt: `capability-upload-${issuance}`, binding: body.binding, maxBytes: bytes.byteLength, expiresAtEpochMs: NOW + 4_000, }, ), ); } const status = putStatuses[puts] ?? 200; puts += 1; return responseWithUrl( status === 200 ? new Response(null, { status: 200, headers: { [POLICY_HEADER]: "v1", "Content-Length": "0", ETag: "\"part-etag-reissued\"", }, }) : new Response(null, { status }), String(input), ); }, ) as unknown as typeof fetch; const { provider, executor } = createHarness({ fetcher, ...(observer ? { observer } : {}), }); const harness = createControlHarness({ now: NOW }); const control = harness.control as unknown as ResumableUploadControlPlane< PresignedUploadPartCapability >; control.issuePartCapability = async (input) => { const issued = await provider.issueUploadPart({ sessionId: input.sessionId, requestBindingSha256: input.requestBindingSha256, uploadBindingSha256: input.uploadBindingSha256, partNumber: input.part.partNumber, offset: input.part.offset, byteLength: input.part.byteLength, checksumSha256: input.part.checksumSha256, mediaType: input.mediaType, idempotencyKey: input.idempotencyKey, signal: input.signal, }); return issued.ok ? browserDataSuccess({ capability: issued.value, uploadBindingSha256: input.uploadBindingSha256, expiresAtEpochMs: issued.value.expiresAtEpochMs, }) : issued; }; const realPart = createPresignedUploadPartExecutor( executor.uploadParts, () => NOW, ); const runtime = createResumableUploadRuntime({ controlPlane: control, partExecutor: { async uploadPart(input) { const uploaded = await realPart.uploadPart(input); // The fake server lists what the object store accepted. if (uploaded.ok) { harness.accepted.set(uploaded.value.partNumber, uploaded.value); } return uploaded; }, }, checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 3 }), now: () => NOW, sleep: async () => {}, ...(observer ? { observer } : {}), }); return { runtime, harness, issuance: () => issuance, puts: () => puts, }; } describe("resumable upload capability reissue (01-01)", () => { it("reissues the capability after an object-store PUT 403 and completes", async () => { const harness = createControlHarness(); let attempts = 0; const partExecutor: UploadPartExecutor = { async uploadPart(input) { attempts += 1; if (attempts === 1) return putForbidden(); const receipt = Object.freeze({ ...input.part, receiptToken: "etag-after-reissue", }); harness.accepted.set(input.part.partNumber, receipt); return browserDataSuccess(receipt); }, }; const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor, checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 3 }), now: () => 1_000, sleep: async () => {}, }); const result = await runtime.upload(request("upload_key_put_403")); expect(result).toMatchObject({ ok: true, value: { state: "QUARANTINED" } }); expect(attempts).toBe(2); expect(harness.issued).toHaveBeenCalledTimes(2); }); it("stops after maxRetries when every PUT keeps refusing the capability", async () => { const harness = createControlHarness(); const partExecutor = { uploadPart: vi.fn(async () => putForbidden()) }; const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: partExecutor as unknown as UploadPartExecutor, checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 2 }), now: () => 1_000, sleep: async () => {}, }); const result = await runtime.upload(request("upload_key_put_403_always")); expect(result).toMatchObject({ ok: false, error: { code: "PERMISSION_DENIED", operation: "UPLOAD_PART" }, }); expect(partExecutor.uploadPart).toHaveBeenCalledTimes(3); expect(harness.issued).toHaveBeenCalledTimes(3); }); it("does not retry a capability issuance the BFF itself refuses with 403", async () => { const harness = createControlHarness({ issueCapability: () => browserDataFailure("PERMISSION_DENIED", "PRESIGNED_TRANSFER", { retryable: false, recovery: "REISSUE_CAPABILITY", }), }); const partExecutor = { uploadPart: vi.fn() }; const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: partExecutor as unknown as UploadPartExecutor, checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 3 }), now: () => 1_000, sleep: async () => {}, }); const result = await runtime.upload(request("upload_key_issue_403")); expect(result).toMatchObject({ ok: false, error: { code: "PERMISSION_DENIED", retryable: false }, }); expect(harness.issued).toHaveBeenCalledTimes(1); expect(partExecutor.uploadPart).not.toHaveBeenCalled(); }); it("recovers a real presigned PUT 403 through a freshly issued capability", async () => { const real = presignedRuntime([403, 200]); const result = await real.runtime.upload(request("upload_key_real_put_403")); expect(result).toMatchObject({ ok: true, value: { state: "QUARANTINED" } }); expect(real.harness.completedParts).toEqual([ [expect.objectContaining({ receiptToken: "part-etag-reissued" })], ]); expect(real.issuance()).toBe(2); expect(real.puts()).toBe(2); }); }); describe("resumable upload idempotency keys (01-04)", () => { const unavailable = ( operation: "UPLOAD_SESSION" | "UPLOAD_PART" | "UPLOAD_COMPLETE", ) => browserDataFailure("UNAVAILABLE", operation, { retryable: true, recovery: "RESUME", }); it("retries createSession with the same idempotency key", async () => { const harness = createControlHarness(); const original = harness.control.createSession.bind(harness.control); const createSession = vi.fn( async ( input: Parameters[0], ): ReturnType => createSession.mock.calls.length === 1 ? unavailable("UPLOAD_SESSION") : await original(input), ); harness.control.createSession = createSession; const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 1 }), now: () => 1_000, sleep: async () => {}, }); const result = await runtime.upload(request("upload_key_create_retry")); expect(result.ok).toBe(true); expect(createSession).toHaveBeenCalledTimes(2); expect(createSession.mock.calls[1]?.[0].idempotencyKey).toBe( createSession.mock.calls[0]?.[0].idempotencyKey, ); }); it("reuses the part key across issuance retries and a resumed upload()", async () => { const harness = createControlHarness({ issueCapability: (input, callIndex) => callIndex === 1 ? unavailable("UPLOAD_PART") : browserDataSuccess({ capability: Object.freeze({ id: `cap-${callIndex}` }), uploadBindingSha256: input.uploadBindingSha256, expiresAtEpochMs: 5_000, }), }); const partKeys: string[] = []; let puts = 0; const partExecutor: UploadPartExecutor = { async uploadPart(input) { puts += 1; partKeys.push(input.idempotencyKey); if (puts === 1) return unavailable("UPLOAD_PART"); return await executorFor(harness).uploadPart(input); }, }; const checkpoints = new MemoryCheckpointStore(); const runtimeWith = (maxRetries: number) => createResumableUploadRuntime({ controlPlane: harness.control, partExecutor, checkpoints, mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries }), now: () => 1_000, sleep: async () => {}, }); const key = "upload_key_part_retry"; // Issuance fails once and is retried; the PUT then fails with no budget. const first = await runtimeWith(1).upload(request(key)); expect(first.ok).toBe(false); const resumed = await runtimeWith(1).upload(request(key)); expect(resumed.ok).toBe(true); const issuedKeys = harness.issued.mock.calls.map( ([input]) => (input as Readonly<{ idempotencyKey: string }>).idempotencyKey, ); expect(issuedKeys).toHaveLength(3); expect(new Set([...issuedKeys, ...partKeys]).size).toBe(1); }); it("retries complete with the same idempotency key", async () => { const harness = createControlHarness(); const original = harness.control.complete.bind(harness.control); const complete = vi.fn( async (input: Parameters[0]): ReturnType => complete.mock.calls.length === 1 ? unavailable("UPLOAD_COMPLETE") : await original(input), ); harness.control.complete = complete; const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 1 }), now: () => 1_000, sleep: async () => {}, }); const result = await runtime.upload(request("upload_key_complete_retry")); expect(result.ok).toBe(true); expect(complete).toHaveBeenCalledTimes(2); expect(complete.mock.calls[1]?.[0].idempotencyKey).toBe( complete.mock.calls[0]?.[0].idempotencyKey, ); }); it("resends a lost complete from a new upload() with the same key", async () => { const harness = createControlHarness(); const original = harness.control.complete.bind(harness.control); const complete = vi.fn( async (input: Parameters[0]): ReturnType => complete.mock.calls.length === 1 ? unavailable("UPLOAD_COMPLETE") : await original(input), ); harness.control.complete = complete; const checkpoints = new MemoryCheckpointStore(); const runtime = () => createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints, mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 0 }), now: () => 1_000, sleep: async () => {}, }); const key = "upload_key_complete_lost"; const lost = await runtime().upload(request(key)); expect(lost).toMatchObject({ ok: false, error: { operation: "UPLOAD_COMPLETE" }, }); const resent = await runtime().upload(request(key)); expect(resent.ok).toBe(true); expect(complete).toHaveBeenCalledTimes(2); expect(complete.mock.calls[1]?.[0].idempotencyKey).toBe( complete.mock.calls[0]?.[0].idempotencyKey, ); }); }); describe("resumable upload completed-result reporting (01-02)", () => { /** A server whose complete commits the resource, reported by getStatus. */ function committingHarness( completeResult: "SUCCEEDS" | "LOST", ) { const harness = createControlHarness(); const original = harness.control.complete.bind(harness.control); let committed = false; harness.control.complete = async (input) => { committed = true; return completeResult === "SUCCEEDS" ? await original(input) : browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", { retryable: true, recovery: "RESUME", }); }; harness.control.getStatus = async () => { const session = harness.getSession(); if (!session) return browserDataFailure("NOT_FOUND", "UPLOAD_RECONCILE"); return committed ? browserDataSuccess({ state: "QUARANTINED", session, resourceId: "resource_01", }) : browserDataSuccess({ state: "ACTIVE", session, acceptedParts: [...harness.accepted.values()], }); }; harness.control.abort = async () => browserDataSuccess({ state: committed ? "ALREADY_COMPLETED" : "ABORTED", }); return harness; } it("reports a completed upload even when the local checkpoint cleanup fails", async () => { const harness = committingHarness("SUCCEEDS"); const checkpoints = new MemoryCheckpointStore(); const remove = checkpoints.remove.bind(checkpoints); let failRemove = true; checkpoints.remove = async (input) => failRemove ? browserDataFailure("UNAVAILABLE", "UPLOAD_RECONCILE", { retryable: true, recovery: "RESUME", }) : await remove(input); const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints, mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 0 }), now: () => 1_000, sleep: async () => {}, }); const key = "upload_key_cleanup_fails"; const completed = await runtime.upload(request(key)); expect(completed).toEqual({ ok: true, value: { state: "QUARANTINED", resourceId: "resource_01", byteLength: 4, replayed: false, }, }); // The stale row is the next call's to reconcile, not a reason to fail. expect(checkpoints.rows.has(key)).toBe(true); failRemove = false; const replayed = await runtime.upload(request(key)); expect(replayed).toMatchObject({ ok: true, value: { resourceId: "resource_01", replayed: true }, }); expect(checkpoints.rows.has(key)).toBe(false); }); it("returns the resourceId when abort() finds the lost complete already applied", async () => { const harness = committingHarness("LOST"); const checkpoints = new MemoryCheckpointStore(); const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints, mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 3 }), now: () => 1_000, sleep: async () => {}, }); const key = "upload_key_x2_lost_complete"; const lost = await runtime.upload(request(key)); expect(lost).toMatchObject({ ok: false, error: { code: "UNAVAILABLE", operation: "UPLOAD_COMPLETE" }, }); const aborted = await runtime.abort({ uploadKey: key, signal: activeSignal }); expect(aborted).toEqual({ ok: true, value: { state: "ALREADY_COMPLETED", resourceId: "resource_01" }, }); expect(checkpoints.rows.size).toBe(0); }); it("waits out an in-progress complete answered with 429 Retry-After", async () => { // The port's in-progress contract, as the fetch transport decodes it. const harness = createControlHarness(); const original = harness.control.complete.bind(harness.control); const complete = vi.fn( async (input: Parameters[0]): ReturnType => complete.mock.calls.length <= 2 ? Object.freeze({ ok: false as const, error: Object.freeze({ code: "UNAVAILABLE" as const, operation: "UPLOAD_COMPLETE" as const, retryable: true, recovery: "RESUME" as const, retryAfterMs: 40, }), }) : await original(input), ); harness.control.complete = complete; const delays: number[] = []; const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 3 }), now: () => 1_000, random: () => 0, sleep: async (delay) => { delays.push(delay); }, }); const result = await runtime.upload(request("upload_key_complete_in_progress")); expect(result).toMatchObject({ ok: true, value: { replayed: false } }); expect(delays).toEqual([40, 40]); expect( new Set(complete.mock.calls.map(([input]) => input.idempotencyKey)).size, ).toBe(1); }); it("keeps the checkpoint when the completed resource cannot be read yet", async () => { const harness = committingHarness("LOST"); const status = harness.control.getStatus.bind(harness.control); let statusDown = false; harness.control.getStatus = async (input) => statusDown ? browserDataFailure("UNAVAILABLE", "UPLOAD_RECONCILE", { retryable: true, recovery: "RESUME", }) : await status(input); const checkpoints = new MemoryCheckpointStore(); const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints, mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 0 }), now: () => 1_000, sleep: async () => {}, }); const key = "upload_key_status_unavailable"; await runtime.upload(request(key)); statusDown = true; const failed = await runtime.abort({ uploadKey: key, signal: activeSignal }); expect(failed).toMatchObject({ ok: false, error: { code: "UNAVAILABLE", operation: "UPLOAD_ABORT", retryable: true }, }); expect(checkpoints.rows.get(key)?.state).toBe("ABORT_PENDING"); statusDown = false; const retried = await runtime.abort({ uploadKey: key, signal: activeSignal }); expect(retried).toEqual({ ok: true, value: { state: "ALREADY_COMPLETED", resourceId: "resource_01" }, }); expect(checkpoints.rows.size).toBe(0); }); }); describe("resumable upload part time limit (01-03)", () => { const FIVE_MIB = 5 * 1024 * 1024; /** A part PUT that ends after `durationMs`, or never when it is null. */ function timedExecutor( harness: ReturnType, durationMs: number | null, ) { let markStarted: () => void = () => {}; const started = new Promise((resolve) => { markStarted = resolve; }); let attempts = 0; const executor: UploadPartExecutor = { async uploadPart(input) { attempts += 1; markStarted(); const finished = await new Promise((resolve) => { const timer = durationMs === null ? undefined : setTimeout(() => resolve(true), durationMs); input.signal.addEventListener("abort", () => { clearTimeout(timer); resolve(false); }); }); if (!finished) return browserDataFailure("ABORTED", "PRESIGNED_TRANSFER"); const receipt = Object.freeze({ ...input.part, receiptToken: "etag-slow" }); harness.accepted.set(input.part.partNumber, receipt); return browserDataSuccess(receipt); }, }; return { executor, started, attempts: () => attempts }; } function defaultPolicyRuntime( harness: ReturnType, executor: UploadPartExecutor, ) { return createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executor, checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, // Default part size, attempt deadline and throughput floor. policy: { capabilityRefreshSkewMs: 5, maxRetries: 0 }, now: () => 1_000, sleep: async () => {}, }); } it("lets a 5 MiB part finish at 1 Mbit/s under the default policy", async () => { vi.useFakeTimers(); try { const harness = createControlHarness({ serverMaxConcurrency: 1 }); // 5 MiB at 1 Mbit/s is about 42 s, past the 30 s base deadline. const slow = timedExecutor(harness, 42_000); const pending = defaultPolicyRuntime(harness, slow.executor).upload({ ...request("upload_key_slow_uplink"), source: rangeSource(new Uint8Array(FIVE_MIB)), }); await slow.started; await vi.advanceTimersByTimeAsync(43_000); expect(await pending).toMatchObject({ ok: true }); expect(slow.attempts()).toBe(1); } finally { vi.useRealTimers(); } }); it("ends a stalled 5 MiB part at the base deadline plus its byte allowance", async () => { vi.useFakeTimers(); try { const harness = createControlHarness({ serverMaxConcurrency: 1 }); const stalled = timedExecutor(harness, null); let settled = false; const pending = defaultPolicyRuntime(harness, stalled.executor) .upload({ ...request("upload_key_stalled_uplink"), source: rangeSource(new Uint8Array(FIVE_MIB)), }) .finally(() => { settled = true; }); await stalled.started; // 30 s base + 5 MiB at the default 64 KiB/s floor (80 s). await vi.advanceTimersByTimeAsync(109_000); expect(settled).toBe(false); await vi.advanceTimersByTimeAsync(2_000); expect(await pending).toMatchObject({ ok: false, error: { code: "UNAVAILABLE", operation: "UPLOAD_PART", retryable: true }, }); } finally { vi.useRealTimers(); } }); }); describe("upload() failure recovery maps to one caller action (01-05)", () => { /** The port's recovery → action table (`ResumableUploadPort.upload`). */ const UPLOAD_ACTIONS = new Set([ "RESUME", "RECONCILE", "RESTART", "RESELECT", "NONE", ]); function runtimeFor( harness: ReturnType, overrides: Partial< Parameters>[0] > = {}, ) { return createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 0 }), now: () => 1_000, sleep: async () => {}, ...overrides, }); } const scenarios = { "an abort is still pending": async () => { const harness = createControlHarness(); harness.control.complete = async () => browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", { retryable: true, recovery: "RESUME", }); harness.control.abort = async () => browserDataFailure("UNAVAILABLE", "UPLOAD_ABORT", { retryable: true, recovery: "RESUME", }); const runtime = runtimeFor(harness); const key = "upload_key_abort_pending"; await runtime.upload(request(key)); await runtime.abort({ uploadKey: key, signal: activeSignal }); return await runtime.upload(request(key)); }, "another context wrote the checkpoint first": async () => { const checkpoints = new MemoryCheckpointStore(); checkpoints.compareAndSwap = async () => browserDataFailure("CONFLICT", "UPLOAD_RECONCILE", { recovery: "RECONCILE", }); return await runtimeFor(createControlHarness(), { checkpoints }).upload( request("upload_key_cas_conflict"), ); }, "the server status disagrees with the file": async () => await runtimeFor( createControlHarness({ statusParts: (_session, accepted) => [...accepted.values()].map((part) => ({ ...part, checksumSha256: "e".repeat(64), })), }), ).upload(request("upload_key_status_mismatch")), "the checkpoint row is unreadable": async () => { const checkpoints = new MemoryCheckpointStore(); checkpoints.read = async () => browserDataFailure("CORRUPT_DATA", "UPLOAD_RECONCILE", { recovery: "RECONCILE", }); return await runtimeFor(createControlHarness(), { checkpoints }).upload( request("upload_key_unreadable"), ); }, "the server session outlives the policy": async () => { const harness = createControlHarness(); const create = harness.control.createSession.bind(harness.control); harness.control.createSession = async (input) => { const created = await create(input); return created.ok ? browserDataSuccess({ ...created.value, expiresAtEpochMs: 1_000 + 20_000, }) : created; }; return await runtimeFor(harness).upload(request("upload_key_long_session")); }, "every PUT refuses the capability": async () => await runtimeFor(createControlHarness(), { partExecutor: { uploadPart: async () => putForbidden() }, }).upload(request("upload_key_refused_put")), "the key belongs to another file": async () => { const checkpoints = new MemoryCheckpointStore(); const harness = createControlHarness(); harness.control.complete = async () => browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", { retryable: true, recovery: "RESUME", }); const runtime = runtimeFor(harness, { checkpoints }); const key = "upload_key_other_file"; await runtime.upload(request(key)); return await runtime.upload({ ...request(key), source: rangeSource(new Uint8Array([9, 9, 9, 9])), }); }, } as const; it.each([ ["an abort is still pending", "CONFLICT", "UPLOAD_ABORT", "RESTART"], [ "another context wrote the checkpoint first", "CONFLICT", "UPLOAD_RECONCILE", "RECONCILE", ], [ "the server status disagrees with the file", "INTEGRITY_FAILED", "UPLOAD_RECONCILE", "RESTART", ], [ "the checkpoint row is unreadable", "CORRUPT_DATA", "UPLOAD_RECONCILE", "RESTART", ], [ "the server session outlives the policy", "POLICY_REJECTED", "UPLOAD_SESSION", "NONE", ], [ "every PUT refuses the capability", "PERMISSION_DENIED", "UPLOAD_PART", "RESUME", ], [ "the key belongs to another file", "STALE_RESULT", "UPLOAD_RECONCILE", "RESELECT", ], ] as const)("%s → %s/%s/%s", async (scenario, code, operation, recovery) => { const result = await scenarios[scenario](); expect(result).toMatchObject({ ok: false, error: { code, operation, recovery }, }); if (!result.ok) expect(UPLOAD_ACTIONS.has(result.error.recovery)).toBe(true); }); }); describe("device clock skew on server lifetimes (01-06)", () => { /** A server whose clock runs `skewMs` ahead and gives full-length sessions. */ function skewedServer(skewMs: number, lifetimeMs: number) { const harness = createControlHarness(); const expiresAtEpochMs = 1_000 + skewMs + lifetimeMs; const create = harness.control.createSession.bind(harness.control); const status = harness.control.getStatus.bind(harness.control); harness.control.createSession = async (input) => { const created = await create(input); return created.ok ? browserDataSuccess({ ...created.value, expiresAtEpochMs }) : created; }; harness.control.getStatus = async (input) => { const read = await status(input); return read.ok && read.value.state === "ACTIVE" ? browserDataSuccess({ ...read.value, session: { ...read.value.session, expiresAtEpochMs }, }) : read; }; return harness; } it.each([ [500, true], [501, false], ] as const)( "accepts a max-length session from a server %i ms ahead: %s", async (skewMs, accepted) => { const harness = skewedServer(skewMs, 10_000); const runtime = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints: new MemoryCheckpointStore(), mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 0, maxSessionLifetimeMs: 10_000, maxClockSkewMs: 500, }), now: () => 1_000, sleep: async () => {}, }); const result = await runtime.upload(request(`upload_key_skew_${skewMs}`)); expect(result.ok).toBe(accepted); if (!accepted) { expect(result).toMatchObject({ error: { code: "POLICY_REJECTED", operation: "UPLOAD_SESSION" }, }); } }, ); }); describe("resume after a policy change (01-08)", () => { it("resumes a concurrency-3 checkpoint with a runtime lowered to 1", async () => { const harness = createControlHarness({ serverMaxConcurrency: 3 }); const checkpoints = new MemoryCheckpointStore(); const source = () => rangeSource(new Uint8Array(12)); const key = "upload_key_policy_lowered"; const partial = await createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: { async uploadPart(input) { if (input.part.partNumber === 2) { return browserDataFailure("UNAVAILABLE", "UPLOAD_PART", { recovery: "RESUME", }); } return await executorFor(harness).uploadPart(input); }, }, checkpoints, mutationLock: noContentionLock, crypto, policy: runtimePolicy({ maxRetries: 0, maxConcurrency: 3 }), now: () => 1_000, sleep: async () => {}, }).upload({ ...request(key), source: source() }); expect(partial.ok).toBe(false); expect(checkpoints.rows.get(key)?.sessionMaxConcurrency).toBe(3); const lowered = createResumableUploadRuntime({ controlPlane: harness.control, partExecutor: executorFor(harness), checkpoints, mutationLock: noContentionLock, crypto, // A later build lowers both limits the stored session was made under. policy: runtimePolicy({ maxRetries: 0, maxConcurrency: 1, maxSessionLifetimeMs: 1_000, }), now: () => 1_000, sleep: async () => {}, }); const resumed = await lowered.upload({ ...request(key), source: source() }); expect(resumed).toMatchObject({ ok: true, value: { replayed: false } }); expect(checkpoints.rows.size).toBe(0); }); }); describe("upload part observation (01-10)", () => { it("records a part that succeeded on its second attempt as one UPLOAD_PART", async () => { const observations: BrowserDataObservation[] = []; // One observer for both layers, as a composition naturally wires it. const real = presignedRuntime([503, 200], { record: (observation) => observations.push(observation), }); const result = await real.runtime.upload(request("upload_key_observed")); expect(result.ok).toBe(true); expect( observations.filter(({ operation }) => operation === "UPLOAD_PART"), ).toEqual([ { operation: "UPLOAD_PART", outcome: "SUCCEEDED", byteBucket: "LT1MIB", countBucket: "TWO_TO_TEN", }, ]); // The executor's own PUT attempts stay visible under their own name. expect( observations.filter( ({ operation, outcome }) => operation === "PRESIGNED_TRANSFER" && outcome === "FAILED", ), ).toHaveLength(1); }); });