A corrupt row, or one a newer build wrote with an unknown schemaVersion, failed both upload and abort with CORRUPT_DATA until the whole partition was deleted. The checkpoint store port gains an optional removeUnreadable(): in one transaction it deletes the row only if it does not parse as a checkpoint (a readable row is CONFLICT, an absent row is a success). Explicit abort calls it on a CORRUPT_DATA read and reports ORPHANED; the server session is left to expire because the row names no session. upload() still fails closed on such a row. The port change is additive (optional method); the architecture doc notes the exception. Verified: RED tests "removes an unreadable checkpoint row on abort and reports ORPHANED" (was CORRUPT_DATA) and "removes only an unreadable row without a revision" (method missing) + all gates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1156 lines
37 KiB
TypeScript
1156 lines
37 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
||
|
||
import type {
|
||
PresignedUploadPartCapability,
|
||
PresignedUploadPartPort,
|
||
} from "../../src/application/ports/browser-transfer/presigned-transfer.ts";
|
||
import type {
|
||
ResumableUploadCheckpoint,
|
||
ResumableUploadCheckpointStore,
|
||
ResumableUploadControlPlane,
|
||
ResumableUploadSource,
|
||
UploadPartExecutor,
|
||
UploadPartReceipt,
|
||
UploadProviderResult,
|
||
UploadSession,
|
||
} from "../../src/application/ports/browser-transfer/resumable-upload.ts";
|
||
import { RESUMABLE_UPLOAD_PROTOCOL } from "../../src/application/ports/browser-transfer/resumable-upload.ts";
|
||
import type {
|
||
BrowserDataObservation,
|
||
BrowserDataResult,
|
||
} from "../../src/application/ports/browser-file-storage/shared.ts";
|
||
import {
|
||
browserDataFailure,
|
||
browserDataSuccess,
|
||
} from "../../src/adapters/browser-file-storage/result.ts";
|
||
import { createIndexedDbResumableUploadCheckpointStore } from "../../src/adapters/browser-transfer/resumable-upload/indexeddb-checkpoint-store.ts";
|
||
import { createPresignedUploadPartExecutor } from "../../src/adapters/browser-transfer/resumable-upload/presigned-upload-part-executor.ts";
|
||
import { createResumableUploadRuntime } from "../../src/adapters/browser-transfer/resumable-upload/resumable-upload-runtime.ts";
|
||
import { resolveResumableUploadRuntimePolicy } from "../../src/adapters/browser-transfer/resumable-upload/runtime-policy.ts";
|
||
import type {
|
||
UploadCancellationChannel,
|
||
UploadCancellationListener,
|
||
} from "../../src/adapters/browser-transfer/resumable-upload/upload-cancellation-channel.ts";
|
||
import type { UploadMutationLock } from "../../src/adapters/browser-transfer/resumable-upload/upload-mutation-lock.ts";
|
||
import { MemoryIndexedDbFactory } from "../helpers/memory-indexeddb.ts";
|
||
|
||
import {
|
||
activeSignal,
|
||
byteStreamSource,
|
||
createControlHarness,
|
||
createMemoryCancellationPair,
|
||
createSerialMutationLock,
|
||
executorFor,
|
||
MemoryCheckpointStore,
|
||
noContentionLock,
|
||
rangeSource,
|
||
runtimePolicy,
|
||
type TestCapability,
|
||
} from "./resumable-upload-runtime-fixture.ts";
|
||
|
||
describe("production resumable upload runtime", () => {
|
||
it("disposes through one drain that proves quiescence", async () => {
|
||
// BT-UP-06. close() closes admission; dispose() awaits real settlement.
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
let releaseUpload: (() => void) | undefined;
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness, {
|
||
delay: async () => {
|
||
await new Promise<void>((resolve) => {
|
||
releaseUpload = resolve;
|
||
});
|
||
},
|
||
}),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
random: () => 0,
|
||
sleep: async () => {},
|
||
});
|
||
|
||
const uploading = runtime.upload({
|
||
uploadKey: "upload_key_01",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: byteStreamSource(new Uint8Array([1, 2, 3, 4, 5])),
|
||
signal: activeSignal,
|
||
});
|
||
await vi.waitFor(() => expect(releaseUpload).toBeDefined());
|
||
expect(runtime.lifecycle()).toBe("OPEN");
|
||
|
||
const first = runtime.dispose();
|
||
const second = runtime.dispose();
|
||
// Duplicate dispose is single-flight.
|
||
expect(first).toBe(second);
|
||
expect(runtime.lifecycle()).toBe("CLOSING");
|
||
|
||
// New admission is refused while draining.
|
||
await expect(
|
||
runtime.upload({
|
||
uploadKey: "upload_key_02",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: byteStreamSource(new Uint8Array([1])),
|
||
signal: activeSignal,
|
||
}),
|
||
).resolves.toMatchObject({ ok: false, error: { code: "UNAVAILABLE" } });
|
||
// The checkpoint store cannot close before the operation settles.
|
||
expect(checkpoints.closed).toBe(false);
|
||
|
||
releaseUpload?.();
|
||
await first;
|
||
await uploading;
|
||
expect(runtime.lifecycle()).toBe("CLOSED");
|
||
expect(checkpoints.closed).toBe(true);
|
||
});
|
||
|
||
it("has a valid production default policy", () => {
|
||
expect(() => resolveResumableUploadRuntimePolicy()).not.toThrow();
|
||
expect(() =>
|
||
resolveResumableUploadRuntimePolicy(
|
||
runtimePolicy({ maxInFlightBytes: 15 }),
|
||
),
|
||
).toThrow(TypeError);
|
||
// R-1.3. partSize 4 × concurrency 4 × copyFactor 4 = 64 > 48 is refused
|
||
// up front instead of being silently clamped to fewer workers.
|
||
expect(() =>
|
||
resolveResumableUploadRuntimePolicy(
|
||
runtimePolicy({ maxConcurrency: 4 }),
|
||
),
|
||
).toThrow(TypeError);
|
||
});
|
||
|
||
// R-1.3. The default memory budget (20 MiB / (5 MiB × 4)) runs one worker,
|
||
// but createSession asked the server for 3.
|
||
it("requests the concurrency the default policy can actually run", async () => {
|
||
const harness = createControlHarness({ serverMaxConcurrency: 1 });
|
||
const createSession = vi.spyOn(harness.control, "createSession");
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints: new MemoryCheckpointStore(),
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
// Only the capability skew differs from the default, to fit the harness.
|
||
policy: { capabilityRefreshSkewMs: 5 },
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_default_policy",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4, 5])),
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({ ok: true });
|
||
expect(createSession).toHaveBeenCalledWith(
|
||
expect.objectContaining({ requestedMaxConcurrency: 1 }),
|
||
);
|
||
});
|
||
|
||
it("uploads bounded range parts, obeys the server concurrency ceiling and completes as quarantined", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness({ serverMaxConcurrency: 1 });
|
||
let maxActive = 0;
|
||
const observations: BrowserDataObservation[] = [];
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness, {
|
||
delay: async () => await Promise.resolve(),
|
||
onActive(active) {
|
||
maxActive = Math.max(maxActive, active);
|
||
},
|
||
}),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
random: () => 0,
|
||
sleep: async () => {},
|
||
observer: {
|
||
record(observation) {
|
||
observations.push(observation);
|
||
},
|
||
},
|
||
});
|
||
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_01",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(
|
||
new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]),
|
||
),
|
||
signal: activeSignal,
|
||
});
|
||
|
||
expect(result).toMatchObject({
|
||
ok: true,
|
||
value: {
|
||
state: "QUARANTINED",
|
||
resourceId: "resource_01",
|
||
byteLength: 9,
|
||
},
|
||
});
|
||
if (result.ok) {
|
||
expect(Object.keys(result.value).sort()).toEqual([
|
||
"byteLength",
|
||
"replayed",
|
||
"resourceId",
|
||
"state",
|
||
]);
|
||
}
|
||
expect(maxActive).toBe(1);
|
||
expect(harness.completedParts[0]?.map((part) => part.partNumber)).toEqual(
|
||
[1, 2, 3],
|
||
);
|
||
expect(checkpoints.rows.size).toBe(0);
|
||
expect(
|
||
observations.map((observation) => observation.operation),
|
||
).toEqual(
|
||
expect.arrayContaining([
|
||
"UPLOAD_SESSION",
|
||
"UPLOAD_RECONCILE",
|
||
"UPLOAD_PART",
|
||
"UPLOAD_COMPLETE",
|
||
]),
|
||
);
|
||
const serialized = JSON.stringify(observations);
|
||
expect(serialized).not.toContain("upload_key_01");
|
||
expect(serialized).not.toContain("session_01");
|
||
expect(serialized).not.toContain("etag-part");
|
||
});
|
||
|
||
// Q-6.1-5. A sibling worker used to keep retrying its part until its own
|
||
// attempt deadline; the first failure now aborts the whole transfer.
|
||
it("aborts in-flight sibling workers as soon as one part fails", async () => {
|
||
const harness = createControlHarness({ serverMaxConcurrency: 2 });
|
||
let secondStarted: () => void = () => {};
|
||
const secondRunning = new Promise<void>((resolve) => {
|
||
secondStarted = resolve;
|
||
});
|
||
let siblingAborted = false;
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: {
|
||
async uploadPart(input) {
|
||
if (input.part.partNumber === 1) {
|
||
await secondRunning;
|
||
return browserDataFailure("POLICY_REJECTED", "UPLOAD_PART");
|
||
}
|
||
secondStarted();
|
||
return await new Promise((resolve) => {
|
||
input.signal.addEventListener(
|
||
"abort",
|
||
() => {
|
||
siblingAborted = true;
|
||
resolve(browserDataFailure("ABORTED", "UPLOAD_PART"));
|
||
},
|
||
{ once: true },
|
||
);
|
||
});
|
||
},
|
||
},
|
||
checkpoints: new MemoryCheckpointStore(),
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ providerAttemptTimeoutMs: 60_000 }),
|
||
now: () => 1_000,
|
||
random: () => 0,
|
||
sleep: async () => {},
|
||
});
|
||
|
||
const uploading = runtime.upload({
|
||
uploadKey: "upload_key_01",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9])),
|
||
signal: activeSignal,
|
||
});
|
||
|
||
await vi.waitFor(() => expect(siblingAborted).toBe(true));
|
||
// The caller sees the part that failed, not the sibling's ABORTED.
|
||
await expect(uploading).resolves.toMatchObject({
|
||
ok: false,
|
||
error: { code: "POLICY_REJECTED", operation: "UPLOAD_PART" },
|
||
});
|
||
});
|
||
|
||
it("supports replayable FileByteSource without retaining the whole file", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_02",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: byteStreamSource(new Uint8Array([1, 2, 3, 4, 5])),
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({
|
||
ok: true,
|
||
value: { state: "QUARANTINED", byteLength: 5 },
|
||
});
|
||
});
|
||
|
||
// R-1.4. The size check used to throw inside request validation and surface
|
||
// as INVALID_INPUT; a file over the policy limit is LIMIT_EXCEEDED.
|
||
it("reports a source over maxFileBytes as LIMIT_EXCEEDED", async () => {
|
||
const harness = createControlHarness();
|
||
const createSession = vi.spyOn(harness.control, "createSession");
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints: new MemoryCheckpointStore(),
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
// A valid policy already bounds the part count by maxFileBytes.
|
||
policy: runtimePolicy({ maxFileBytes: 4 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_too_large",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: byteStreamSource(new Uint8Array([1, 2, 3, 4, 5])),
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({
|
||
ok: false,
|
||
error: { code: "LIMIT_EXCEEDED" },
|
||
});
|
||
expect(createSession).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it("fails closed when server list-parts disagrees with the current source manifest", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness({
|
||
statusParts(session) {
|
||
return [
|
||
{
|
||
partNumber: 1,
|
||
offset: 0,
|
||
byteLength: Math.min(4, session.fingerprint.byteLength),
|
||
checksumSha256: "f".repeat(64),
|
||
receiptToken: "etag-wrong-part",
|
||
},
|
||
];
|
||
},
|
||
});
|
||
const partExecutor = { uploadPart: vi.fn() };
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: partExecutor as unknown as UploadPartExecutor<TestCapability>,
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_03",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({
|
||
ok: false,
|
||
error: {
|
||
code: "INTEGRITY_FAILED",
|
||
operation: "UPLOAD_RECONCILE",
|
||
recovery: "RECONCILE",
|
||
},
|
||
});
|
||
expect(partExecutor.uploadPart).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it.each(["NOT_FOUND", "EXPIRED_RESOURCE"] as const)(
|
||
"removes a definitive %s status checkpoint and creates a fresh session",
|
||
async (code) => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness({
|
||
sessionId: (index) => `session_terminal_${index}`,
|
||
});
|
||
const originalCreate = harness.control.createSession.bind(
|
||
harness.control,
|
||
);
|
||
const createSession = vi.fn(originalCreate);
|
||
harness.control.createSession = createSession;
|
||
let statusCalls = 0;
|
||
harness.control.getStatus = async () => {
|
||
statusCalls += 1;
|
||
const session = harness.getSession();
|
||
if (!session) {
|
||
return browserDataFailure(
|
||
"NOT_FOUND",
|
||
"UPLOAD_RECONCILE",
|
||
);
|
||
}
|
||
if (statusCalls === 1) {
|
||
return browserDataFailure(code, "UPLOAD_RECONCILE", {
|
||
recovery:
|
||
code === "NOT_FOUND" ? "RECONCILE" : "RESTART",
|
||
});
|
||
}
|
||
return browserDataSuccess({
|
||
state: "ACTIVE",
|
||
session,
|
||
acceptedParts: [...harness.accepted.values()],
|
||
});
|
||
};
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
|
||
const result = await runtime.upload({
|
||
uploadKey: `upload_key_terminal_${code.toLowerCase()}`,
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
|
||
expect(result.ok).toBe(true);
|
||
expect(createSession).toHaveBeenCalledTimes(2);
|
||
expect(
|
||
createSession.mock.calls[0]?.[0].idempotencyKey,
|
||
).not.toBe(
|
||
createSession.mock.calls[1]?.[0].idempotencyKey,
|
||
);
|
||
expect(checkpoints.rows.size).toBe(0);
|
||
},
|
||
);
|
||
|
||
it("removes the exact checkpoint revision when status becomes terminal after part transfer", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
let statusCalls = 0;
|
||
harness.control.getStatus = async () => {
|
||
statusCalls += 1;
|
||
const session = harness.getSession();
|
||
if (!session) {
|
||
return browserDataFailure("NOT_FOUND", "UPLOAD_RECONCILE");
|
||
}
|
||
return statusCalls === 1
|
||
? browserDataSuccess({
|
||
state: "ACTIVE",
|
||
session,
|
||
acceptedParts: [],
|
||
})
|
||
: browserDataFailure(
|
||
"EXPIRED_RESOURCE",
|
||
"UPLOAD_RECONCILE",
|
||
{ recovery: "RESTART" },
|
||
);
|
||
};
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_post_transfer_terminal",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
|
||
expect(result).toMatchObject({
|
||
ok: false,
|
||
error: {
|
||
code: "EXPIRED_RESOURCE",
|
||
operation: "UPLOAD_SESSION",
|
||
recovery: "RESTART",
|
||
},
|
||
});
|
||
expect(checkpoints.rows.size).toBe(0);
|
||
expect(harness.completedParts).toHaveLength(0);
|
||
});
|
||
|
||
it("reissues an expiring part capability before sending bytes", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const delays: number[] = [];
|
||
const harness = createControlHarness({
|
||
issueCapability(input, callIndex) {
|
||
return browserDataSuccess({
|
||
capability: { id: `cap-${callIndex}` },
|
||
uploadBindingSha256: input.uploadBindingSha256,
|
||
expiresAtEpochMs: callIndex === 1 ? 1_003 : 4_000,
|
||
});
|
||
},
|
||
});
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 1 }),
|
||
now: () => 1_000,
|
||
random: () => 0,
|
||
sleep: async (delay) => {
|
||
delays.push(delay);
|
||
},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_04",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
expect(result.ok).toBe(true);
|
||
expect(harness.issued).toHaveBeenCalledTimes(2);
|
||
expect(delays).toHaveLength(1);
|
||
});
|
||
|
||
it("honors a bounded Retry-After delay and reissues the single-use capability", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
const delays: number[] = [];
|
||
let uploadAttempts = 0;
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: {
|
||
async uploadPart(input) {
|
||
uploadAttempts += 1;
|
||
if (uploadAttempts === 1) {
|
||
return Object.freeze({
|
||
ok: false as const,
|
||
error: Object.freeze({
|
||
code: "UNAVAILABLE" as const,
|
||
operation: "UPLOAD_PART" as const,
|
||
retryable: true,
|
||
recovery: "REISSUE_CAPABILITY" as const,
|
||
retryAfterMs: 7,
|
||
}),
|
||
});
|
||
}
|
||
const receipt = Object.freeze({
|
||
...input.part,
|
||
receiptToken: "etag-retried-part",
|
||
});
|
||
harness.accepted.set(input.part.partNumber, receipt);
|
||
return browserDataSuccess(receipt);
|
||
},
|
||
},
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 1 }),
|
||
now: () => 1_000,
|
||
random: () => 0,
|
||
sleep: async (delay) => {
|
||
delays.push(delay);
|
||
},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_retry_after",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
|
||
expect(result.ok).toBe(true);
|
||
expect(uploadAttempts).toBe(2);
|
||
expect(harness.issued).toHaveBeenCalledTimes(2);
|
||
expect(delays).toEqual([7]);
|
||
});
|
||
|
||
it("detects source mutation between the fingerprint and transfer passes", async () => {
|
||
let reads = 0;
|
||
const source: ResumableUploadSource = {
|
||
kind: "RANGE_READER",
|
||
reader: {
|
||
byteLength: 4,
|
||
async readRange() {
|
||
reads += 1;
|
||
return browserDataSuccess(
|
||
reads === 1
|
||
? new Uint8Array([1, 2, 3, 4])
|
||
: new Uint8Array([9, 2, 3, 4]),
|
||
);
|
||
},
|
||
},
|
||
};
|
||
const harness = createControlHarness();
|
||
const executor = { uploadPart: vi.fn() };
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executor as unknown as UploadPartExecutor<TestCapability>,
|
||
checkpoints: new MemoryCheckpointStore(),
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_05",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source,
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({
|
||
ok: false,
|
||
error: { code: "STALE_RESULT", recovery: "RESELECT" },
|
||
});
|
||
expect(executor.uploadPart).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it("bounds a control-plane provider that ignores AbortSignal", async () => {
|
||
const harness = createControlHarness();
|
||
harness.control.createSession = (() =>
|
||
new Promise(() => {})) as typeof harness.control.createSession;
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints: new MemoryCheckpointStore(),
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({
|
||
maxRetries: 0,
|
||
providerAttemptTimeoutMs: 1,
|
||
}),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const result = await runtime.upload({
|
||
uploadKey: "upload_key_06",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({
|
||
ok: false,
|
||
error: { code: "UNAVAILABLE", operation: "UPLOAD_SESSION" },
|
||
});
|
||
});
|
||
|
||
it("persists ABORT_PENDING until backend abort reconciliation succeeds", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const failedUpload = await runtime.upload({
|
||
uploadKey: "upload_key_07",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: {
|
||
kind: "RANGE_READER",
|
||
reader: {
|
||
byteLength: 4,
|
||
async readRange() {
|
||
return browserDataFailure("NOT_READABLE", "FILE_READ");
|
||
},
|
||
},
|
||
},
|
||
signal: activeSignal,
|
||
});
|
||
expect(failedUpload.ok).toBe(false);
|
||
// Seed a valid checkpoint because source failure occurs before session create.
|
||
const seededHarness = createControlHarness();
|
||
let abortCalls = 0;
|
||
seededHarness.control.abort = async () => {
|
||
abortCalls += 1;
|
||
return abortCalls === 1
|
||
? browserDataFailure("UNAVAILABLE", "UPLOAD_ABORT", {
|
||
retryable: true,
|
||
recovery: "RESUME",
|
||
})
|
||
: browserDataSuccess({ state: "EXPIRED" });
|
||
};
|
||
const seededRuntime = createResumableUploadRuntime({
|
||
controlPlane: seededHarness.control,
|
||
partExecutor: {
|
||
async uploadPart() {
|
||
return browserDataFailure("UNAVAILABLE", "UPLOAD_PART");
|
||
},
|
||
},
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const fingerprint = {
|
||
algorithm: "SHA-256-PARTS-V1" as const,
|
||
digestHex: "a".repeat(64),
|
||
byteLength: 4,
|
||
partSizeBytes: 4,
|
||
partCount: 1,
|
||
};
|
||
checkpoints.rows.set("upload_key_08", {
|
||
schemaVersion: 1,
|
||
protocol: RESUMABLE_UPLOAD_PROTOCOL,
|
||
revision: 1,
|
||
state: "ACTIVE",
|
||
uploadKey: "upload_key_08",
|
||
requestBindingSha256: "b".repeat(64),
|
||
fingerprint,
|
||
sessionId: "session_08",
|
||
sessionExpiresAtEpochMs: 5_000,
|
||
sessionMaxConcurrency: 1,
|
||
acceptedParts: [],
|
||
updatedAtEpochMs: 1_000,
|
||
});
|
||
|
||
expect(
|
||
await seededRuntime.abort({
|
||
uploadKey: "upload_key_08",
|
||
signal: activeSignal,
|
||
}),
|
||
).toMatchObject({
|
||
ok: false,
|
||
error: { code: "UNAVAILABLE" },
|
||
});
|
||
expect(checkpoints.rows.get("upload_key_08")?.state).toBe(
|
||
"ABORT_PENDING",
|
||
);
|
||
expect(
|
||
await seededRuntime.abort({
|
||
uploadKey: "upload_key_08",
|
||
signal: activeSignal,
|
||
}),
|
||
).toEqual({ ok: true, value: { state: "ORPHANED" } });
|
||
expect(checkpoints.rows.has("upload_key_08")).toBe(false);
|
||
});
|
||
|
||
it("treats definitive abort 404/410 outcomes as orphan cleanup", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
harness.control.abort = async () =>
|
||
browserDataFailure("NOT_FOUND", "UPLOAD_ABORT", {
|
||
recovery: "RECONCILE",
|
||
});
|
||
checkpoints.rows.set("upload_key_abort_not_found", {
|
||
schemaVersion: 1,
|
||
protocol: RESUMABLE_UPLOAD_PROTOCOL,
|
||
revision: 1,
|
||
state: "ACTIVE",
|
||
uploadKey: "upload_key_abort_not_found",
|
||
requestBindingSha256: "a".repeat(64),
|
||
fingerprint: {
|
||
algorithm: "SHA-256-PARTS-V1",
|
||
digestHex: "b".repeat(64),
|
||
byteLength: 4,
|
||
partSizeBytes: 4,
|
||
partCount: 1,
|
||
},
|
||
sessionId: "session_abort_not_found",
|
||
sessionExpiresAtEpochMs: 5_000,
|
||
sessionMaxConcurrency: 1,
|
||
acceptedParts: [],
|
||
updatedAtEpochMs: 1_000,
|
||
});
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
|
||
const result = await runtime.abort({
|
||
uploadKey: "upload_key_abort_not_found",
|
||
signal: activeSignal,
|
||
});
|
||
|
||
expect(result).toEqual({
|
||
ok: true,
|
||
value: { state: "ORPHANED" },
|
||
});
|
||
expect(checkpoints.rows.has("upload_key_abort_not_found")).toBe(
|
||
false,
|
||
);
|
||
});
|
||
|
||
// Q-6.1-6. A row this build cannot read (here a newer schemaVersion left by
|
||
// a rolled-back release) used to fail both upload and abort with
|
||
// CORRUPT_DATA until the whole partition was deleted.
|
||
it("removes an unreadable checkpoint row on abort and reports ORPHANED", async () => {
|
||
const memory = new MemoryIndexedDbFactory();
|
||
const checkpoints = createIndexedDbResumableUploadCheckpointStore({
|
||
scope: {
|
||
authorityToken: "authority_token_01",
|
||
namespaceToken: "namespace_token_01",
|
||
partitionToken: "partition_token_01",
|
||
},
|
||
factory: memory.factory,
|
||
});
|
||
// Opens the database so the object stores exist before seeding.
|
||
expect(await checkpoints.read("upload_key_unreadable")).toEqual({
|
||
ok: true,
|
||
value: null,
|
||
});
|
||
memory.seed("checkpoints", {
|
||
schemaVersion: 2,
|
||
uploadKey: "upload_key_unreadable",
|
||
revision: 7,
|
||
state: "PAUSED",
|
||
});
|
||
const harness = createControlHarness();
|
||
const serverAbort = vi.spyOn(harness.control, "abort");
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy(),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
|
||
expect(
|
||
await runtime.abort({
|
||
uploadKey: "upload_key_unreadable",
|
||
signal: activeSignal,
|
||
}),
|
||
).toEqual({ ok: true, value: { state: "ORPHANED" } });
|
||
// No session id can be read from the row, so the server is not called.
|
||
expect(serverAbort).not.toHaveBeenCalled();
|
||
expect(memory.readRaw("checkpoints", "upload_key_unreadable")).toBe(
|
||
undefined,
|
||
);
|
||
});
|
||
|
||
it("interrupts a same-runtime active upload before explicit abort waits for the key lock", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
let markEntered!: () => void;
|
||
const entered = new Promise<void>((resolve) => {
|
||
markEntered = resolve;
|
||
});
|
||
const runtime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: {
|
||
async uploadPart(input) {
|
||
markEntered();
|
||
return await new Promise((resolve) => {
|
||
const abort = () =>
|
||
resolve(
|
||
browserDataFailure("ABORTED", "UPLOAD_PART"),
|
||
);
|
||
input.signal.addEventListener("abort", abort, {
|
||
once: true,
|
||
});
|
||
if (input.signal.aborted) abort();
|
||
});
|
||
},
|
||
},
|
||
checkpoints,
|
||
mutationLock: createSerialMutationLock(),
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const upload = runtime.upload({
|
||
uploadKey: "upload_key_local_interrupt",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
await entered;
|
||
|
||
const aborted = runtime.abort({
|
||
uploadKey: "upload_key_local_interrupt",
|
||
signal: activeSignal,
|
||
});
|
||
|
||
await expect(upload).resolves.toMatchObject({
|
||
ok: false,
|
||
error: { code: "ABORTED", operation: "UPLOAD_PART" },
|
||
});
|
||
await expect(aborted).resolves.toEqual({
|
||
ok: true,
|
||
value: { state: "ABORTED" },
|
||
});
|
||
expect(checkpoints.rows.size).toBe(0);
|
||
});
|
||
|
||
it("interrupts another context before explicit abort waits for the shared Web Lock", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
const mutationLock = createSerialMutationLock();
|
||
const [firstCancellation, secondCancellation] =
|
||
createMemoryCancellationPair();
|
||
let markEntered!: () => void;
|
||
const entered = new Promise<void>((resolve) => {
|
||
markEntered = resolve;
|
||
});
|
||
const dependencies = {
|
||
controlPlane: harness.control,
|
||
checkpoints,
|
||
mutationLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
} as const;
|
||
const firstContext = createResumableUploadRuntime({
|
||
...dependencies,
|
||
crossContextCancellation: firstCancellation,
|
||
partExecutor: {
|
||
async uploadPart(input) {
|
||
markEntered();
|
||
return await new Promise((resolve) => {
|
||
const abort = () =>
|
||
resolve(
|
||
browserDataFailure("ABORTED", "UPLOAD_PART"),
|
||
);
|
||
input.signal.addEventListener("abort", abort, {
|
||
once: true,
|
||
});
|
||
if (input.signal.aborted) abort();
|
||
});
|
||
},
|
||
},
|
||
});
|
||
const secondContext = createResumableUploadRuntime({
|
||
...dependencies,
|
||
crossContextCancellation: secondCancellation,
|
||
partExecutor: executorFor(harness),
|
||
});
|
||
const upload = firstContext.upload({
|
||
uploadKey: "upload_key_cross_context",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
});
|
||
await entered;
|
||
const abort = secondContext.abort({
|
||
uploadKey: "upload_key_cross_context",
|
||
signal: activeSignal,
|
||
});
|
||
|
||
await expect(upload).resolves.toMatchObject({
|
||
ok: false,
|
||
error: { code: "ABORTED", operation: "UPLOAD_PART" },
|
||
});
|
||
await expect(abort).resolves.toEqual({
|
||
ok: true,
|
||
value: { state: "ABORTED" },
|
||
});
|
||
expect(checkpoints.rows.size).toBe(0);
|
||
firstContext.close();
|
||
secondContext.close();
|
||
});
|
||
|
||
it("reconciles an ambiguous complete response to QUARANTINED even after session expiry", async () => {
|
||
const checkpoints = new MemoryCheckpointStore();
|
||
const harness = createControlHarness();
|
||
let committed = false;
|
||
const complete = vi.fn(
|
||
async (): ReturnType<
|
||
ResumableUploadControlPlane<TestCapability>["complete"]
|
||
> => {
|
||
committed = true;
|
||
return browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", {
|
||
retryable: true,
|
||
recovery: "RECONCILE",
|
||
});
|
||
},
|
||
);
|
||
harness.control.complete = complete;
|
||
harness.control.getStatus = async () => {
|
||
const session = harness.getSession();
|
||
if (!session) {
|
||
return browserDataFailure("NOT_FOUND", "UPLOAD_RECONCILE");
|
||
}
|
||
return committed
|
||
? browserDataSuccess({
|
||
state: "QUARANTINED",
|
||
session,
|
||
resourceId: "resource_ambiguous_01",
|
||
})
|
||
: browserDataSuccess({
|
||
state: "ACTIVE",
|
||
session,
|
||
acceptedParts: [...harness.accepted.values()].sort(
|
||
(left, right) => left.partNumber - right.partNumber,
|
||
),
|
||
});
|
||
};
|
||
|
||
const firstRuntime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor: executorFor(harness),
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 1_000,
|
||
sleep: async () => {},
|
||
});
|
||
const request = {
|
||
uploadKey: "upload_key_ambiguous_complete",
|
||
purpose: "attachment",
|
||
mediaType: "application/octet-stream",
|
||
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
||
signal: activeSignal,
|
||
} as const;
|
||
const ambiguous = await firstRuntime.upload(request);
|
||
|
||
expect(ambiguous).toMatchObject({
|
||
ok: false,
|
||
error: {
|
||
code: "UNAVAILABLE",
|
||
operation: "UPLOAD_COMPLETE",
|
||
recovery: "RECONCILE",
|
||
},
|
||
});
|
||
expect(checkpoints.rows.has(request.uploadKey)).toBe(true);
|
||
|
||
const resumedPartExecutor = { uploadPart: vi.fn() };
|
||
const resumedRuntime = createResumableUploadRuntime({
|
||
controlPlane: harness.control,
|
||
partExecutor:
|
||
resumedPartExecutor as unknown as UploadPartExecutor<TestCapability>,
|
||
checkpoints,
|
||
mutationLock: noContentionLock,
|
||
crypto,
|
||
policy: runtimePolicy({ maxRetries: 0 }),
|
||
now: () => 7_000,
|
||
sleep: async () => {},
|
||
});
|
||
const recovered = await resumedRuntime.upload(request);
|
||
|
||
expect(recovered).toMatchObject({
|
||
ok: true,
|
||
value: {
|
||
state: "QUARANTINED",
|
||
resourceId: "resource_ambiguous_01",
|
||
replayed: true,
|
||
},
|
||
});
|
||
expect(complete).toHaveBeenCalledTimes(1);
|
||
expect(resumedPartExecutor.uploadPart).not.toHaveBeenCalled();
|
||
expect(checkpoints.rows.has(request.uploadKey)).toBe(false);
|
||
});
|
||
|
||
it("bridges presigned PUT using the verified response receipt, not the capability receipt", async () => {
|
||
let forwardedUploadBinding = "";
|
||
let forwardedSessionId = "";
|
||
let forwardedRequestBinding = "";
|
||
const port: PresignedUploadPartPort = {
|
||
async put(input) {
|
||
forwardedUploadBinding = input.uploadBindingSha256;
|
||
forwardedSessionId = input.sessionId;
|
||
forwardedRequestBinding = input.requestBindingSha256;
|
||
return browserDataSuccess({
|
||
bytesWritten: input.byteLength,
|
||
checksumSha256: input.checksumSha256,
|
||
receiptToken: "etag.response-01",
|
||
});
|
||
},
|
||
};
|
||
const executor = createPresignedUploadPartExecutor(port, () => 1_000);
|
||
const capability = {
|
||
capabilityReceipt: "capability_receipt_01",
|
||
method: "PUT",
|
||
binding: {
|
||
kind: "UPLOAD_PART",
|
||
protocol: RESUMABLE_UPLOAD_PROTOCOL,
|
||
sessionId: "session_01",
|
||
requestBindingSha256: "c".repeat(64),
|
||
uploadBindingSha256: "b".repeat(64),
|
||
partNumber: 1,
|
||
offset: 0,
|
||
idempotencyKey: "upload-part-idempotency-01",
|
||
},
|
||
mediaType: "application/octet-stream",
|
||
byteLength: 4,
|
||
maxBytes: 4,
|
||
expectedSha256: "a".repeat(64),
|
||
expiresAtEpochMs: 2_000,
|
||
} as unknown as PresignedUploadPartCapability;
|
||
const result = await executor.uploadPart({
|
||
protocol: RESUMABLE_UPLOAD_PROTOCOL,
|
||
capability,
|
||
sessionId: "session_01",
|
||
requestBindingSha256: "c".repeat(64),
|
||
uploadBindingSha256: "b".repeat(64),
|
||
fingerprint: {
|
||
algorithm: "SHA-256-PARTS-V1",
|
||
digestHex: "d".repeat(64),
|
||
byteLength: 4,
|
||
partSizeBytes: 4,
|
||
partCount: 1,
|
||
},
|
||
mediaType: "application/octet-stream",
|
||
part: {
|
||
partNumber: 1,
|
||
offset: 0,
|
||
byteLength: 4,
|
||
checksumSha256: "a".repeat(64),
|
||
},
|
||
bytes: new Uint8Array([1, 2, 3, 4]),
|
||
idempotencyKey: "upload-part-idempotency-01",
|
||
signal: activeSignal,
|
||
});
|
||
expect(result).toMatchObject({
|
||
ok: true,
|
||
value: { receiptToken: "etag.response-01" },
|
||
});
|
||
expect(forwardedUploadBinding).toBe("b".repeat(64));
|
||
expect(forwardedSessionId).toBe("session_01");
|
||
expect(forwardedRequestBinding).toBe("c".repeat(64));
|
||
if (result.ok) {
|
||
expect(result.value.receiptToken).not.toBe(
|
||
capability.capabilityReceipt,
|
||
);
|
||
}
|
||
});
|
||
});
|