문제: complete 응답을 잃은 뒤 다음 upload()가 재조정해 서버 status
QUARANTINED를 찾았을 때, 로컬 checkpoint 삭제가 한 번 실패하면 runtime이
그 실패를 그대로 돌려준다({UNAVAILABLE, UPLOAD_RECONCILE, retryable:true,
RESUME}). 서버에는 자원이 이미 있는데 이 호출은 resourceId를 잃는다. 같은
상황의 complete 경로는 삭제 실패를 무시하고 성공을 준다.
고친 방법: 재조정의 QUARANTINED 분기도 complete 경로와 같이 삭제 결과와
상관없이 {QUARANTINED, resourceId, replayed:true}를 준다. 남은 행은 다음
upload()나 abort()가 같은 자원으로 정리한다. 설계 문서 4.3절의 같은 규칙
줄에 재조정 경우를 더했다.
위험·한계: 삭제가 계속 실패하는 저장소에서는 행이 남는다. 다음 호출마다
status를 한 번 읽고 같은 resourceId를 replayed로 준다. complete 경로와 같은
한계다.
검증:
- 새 테스트: 고치기 전 {ok:false, UNAVAILABLE, UPLOAD_RECONCILE} (리뷰 X4와
같다), 고친 뒤 {ok:true, QUARANTINED, resource_01, replayed:true}, 행 남음,
다음 호출이 정리
- 영역 단위 테스트 15파일 276건 통과
- check:types(test), eslint(바꾼 파일) 통과
Refs: docs/reviews/2026-11-03-maturity-review/01-adapters-browser-transfer.md 0929/01-02
1491 lines
50 KiB
TypeScript
1491 lines
50 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { BrowserDataObservation } from "../../src/application/ports/browser-file-storage/shared.ts";
|
|
import type { PresignedUploadPartCapability } from "../../src/application/ports/browser-transfer/presigned-transfer.ts";
|
|
import type {
|
|
ResumableUploadControlPlane,
|
|
UploadPartExecutor,
|
|
} from "../../src/application/ports/browser-transfer/resumable-upload.ts";
|
|
import {
|
|
browserDataFailure,
|
|
browserDataSuccess,
|
|
} from "../../src/adapters/browser-file-storage/result.ts";
|
|
import { sha256Hex } from "../../src/adapters/browser-transfer/presigned/incremental-sha256.ts";
|
|
import { createResumableUploadFetchJsonTransport } from "../../src/adapters/browser-transfer/resumable-upload/fetch-json-transport.ts";
|
|
import { createResumableUploadHttpControlPlane } from "../../src/adapters/browser-transfer/resumable-upload/http-control-plane-adapter.ts";
|
|
import { createPresignedUploadPartExecutor } from "../../src/adapters/browser-transfer/resumable-upload/presigned-upload-part-executor.ts";
|
|
import { createResumableUploadRuntime } from "../../src/adapters/browser-transfer/resumable-upload/resumable-upload-runtime.ts";
|
|
|
|
import {
|
|
CONTROL_ENDPOINT,
|
|
NOW,
|
|
POLICY_HEADER,
|
|
createHarness,
|
|
jsonResponse,
|
|
responseWithUrl,
|
|
uploadCapabilityPayload,
|
|
} from "./presigned-transfer-fixture.ts";
|
|
import {
|
|
activeSignal,
|
|
createControlHarness,
|
|
executorFor,
|
|
MemoryCheckpointStore,
|
|
noContentionLock,
|
|
rangeSource,
|
|
runtimePolicy,
|
|
type TestCapability,
|
|
} from "./resumable-upload-runtime-fixture.ts";
|
|
|
|
const request = (uploadKey: string) =>
|
|
({
|
|
uploadKey,
|
|
purpose: "attachment",
|
|
mediaType: "application/octet-stream",
|
|
source: rangeSource(new Uint8Array([1, 2, 3, 4])),
|
|
signal: activeSignal,
|
|
}) as const;
|
|
|
|
/** What `statusFailure(403)` hands the runtime through the part executor. */
|
|
const putForbidden = () =>
|
|
browserDataFailure("PERMISSION_DENIED", "PRESIGNED_TRANSFER", {
|
|
retryable: false,
|
|
recovery: "REISSUE_CAPABILITY",
|
|
});
|
|
|
|
/**
|
|
* The runtime composed with the real presigned provider, vault and executor,
|
|
* over a fake BFF and object store. `putStatuses` are the object store's
|
|
* answers to successive PUTs.
|
|
*/
|
|
function presignedRuntime(
|
|
putStatuses: readonly number[],
|
|
observer?: Readonly<{ record(observation: BrowserDataObservation): void }>,
|
|
faults: Readonly<{
|
|
/** The BFF's answers to successive issuances; past the list, 200. */
|
|
issueStatuses?: readonly number[];
|
|
/** `Retry-After` sent with every refusal from either server. */
|
|
retryAfter?: string;
|
|
maxRetryAfterMs?: number;
|
|
}> = {},
|
|
) {
|
|
const bytes = new Uint8Array([1, 2, 3, 4]);
|
|
const checksum = sha256Hex(bytes);
|
|
let issuance = 0;
|
|
let puts = 0;
|
|
const sleeps: number[] = [];
|
|
const refusal = (status: number) =>
|
|
new Response(null, {
|
|
status,
|
|
headers:
|
|
faults.retryAfter === undefined
|
|
? {}
|
|
: { "Retry-After": faults.retryAfter },
|
|
});
|
|
const fetcher = vi.fn(
|
|
async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
if (String(input) === CONTROL_ENDPOINT) {
|
|
issuance += 1;
|
|
const issueStatus = faults.issueStatuses?.[issuance - 1] ?? 200;
|
|
if (issueStatus !== 200) {
|
|
return responseWithUrl(refusal(issueStatus), CONTROL_ENDPOINT);
|
|
}
|
|
const body = JSON.parse(String(init?.body)) as Readonly<{
|
|
binding: Readonly<Record<string, unknown>>;
|
|
}>;
|
|
return jsonResponse(
|
|
uploadCapabilityPayload(
|
|
{ bytes, checksum },
|
|
{
|
|
capabilityReceipt: `capability-upload-${issuance}`,
|
|
binding: body.binding,
|
|
maxBytes: bytes.byteLength,
|
|
expiresAtEpochMs: NOW + 4_000,
|
|
},
|
|
),
|
|
);
|
|
}
|
|
const status = putStatuses[puts] ?? 200;
|
|
puts += 1;
|
|
return responseWithUrl(
|
|
status === 200
|
|
? new Response(null, {
|
|
status: 200,
|
|
headers: {
|
|
[POLICY_HEADER]: "v1",
|
|
"Content-Length": "0",
|
|
ETag: "\"part-etag-reissued\"",
|
|
},
|
|
})
|
|
: refusal(status),
|
|
String(input),
|
|
);
|
|
},
|
|
) as unknown as typeof fetch;
|
|
const { provider, executor } = createHarness({
|
|
fetcher,
|
|
...(observer ? { observer } : {}),
|
|
});
|
|
const harness = createControlHarness({ now: NOW });
|
|
const control = harness.control as unknown as ResumableUploadControlPlane<
|
|
PresignedUploadPartCapability
|
|
>;
|
|
control.issuePartCapability = async (input) => {
|
|
const issued = await provider.issueUploadPart({
|
|
sessionId: input.sessionId,
|
|
requestBindingSha256: input.requestBindingSha256,
|
|
uploadBindingSha256: input.uploadBindingSha256,
|
|
partNumber: input.part.partNumber,
|
|
offset: input.part.offset,
|
|
byteLength: input.part.byteLength,
|
|
checksumSha256: input.part.checksumSha256,
|
|
mediaType: input.mediaType,
|
|
idempotencyKey: input.idempotencyKey,
|
|
signal: input.signal,
|
|
});
|
|
return issued.ok
|
|
? browserDataSuccess({
|
|
capability: issued.value,
|
|
uploadBindingSha256: input.uploadBindingSha256,
|
|
expiresAtEpochMs: issued.value.expiresAtEpochMs,
|
|
})
|
|
: issued;
|
|
};
|
|
const realPart = createPresignedUploadPartExecutor(
|
|
executor.uploadParts,
|
|
() => NOW,
|
|
);
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: control,
|
|
partExecutor: {
|
|
async uploadPart(input) {
|
|
const uploaded = await realPart.uploadPart(input);
|
|
// The fake server lists what the object store accepted.
|
|
if (uploaded.ok) {
|
|
harness.accepted.set(uploaded.value.partNumber, uploaded.value);
|
|
}
|
|
return uploaded;
|
|
},
|
|
},
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({
|
|
maxRetries: 3,
|
|
...(faults.maxRetryAfterMs === undefined
|
|
? {}
|
|
: { maxRetryAfterMs: faults.maxRetryAfterMs }),
|
|
}),
|
|
now: () => NOW,
|
|
sleep: async (milliseconds: number) => {
|
|
sleeps.push(milliseconds);
|
|
},
|
|
...(observer ? { observer } : {}),
|
|
});
|
|
return {
|
|
runtime,
|
|
harness,
|
|
issuance: () => issuance,
|
|
puts: () => puts,
|
|
sleeps,
|
|
};
|
|
}
|
|
|
|
describe("resumable upload capability reissue (01-01)", () => {
|
|
it("reissues the capability after an object-store PUT 403 and completes", async () => {
|
|
const harness = createControlHarness();
|
|
let attempts = 0;
|
|
const partExecutor: UploadPartExecutor<TestCapability> = {
|
|
async uploadPart(input) {
|
|
attempts += 1;
|
|
if (attempts === 1) return putForbidden();
|
|
const receipt = Object.freeze({
|
|
...input.part,
|
|
receiptToken: "etag-after-reissue",
|
|
});
|
|
harness.accepted.set(input.part.partNumber, receipt);
|
|
return browserDataSuccess(receipt);
|
|
},
|
|
};
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor,
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 3 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
|
|
const result = await runtime.upload(request("upload_key_put_403"));
|
|
|
|
expect(result).toMatchObject({ ok: true, value: { state: "QUARANTINED" } });
|
|
expect(attempts).toBe(2);
|
|
expect(harness.issued).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("stops after maxRetries when every PUT keeps refusing the capability", async () => {
|
|
const harness = createControlHarness();
|
|
const partExecutor = { uploadPart: vi.fn(async () => putForbidden()) };
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor:
|
|
partExecutor as unknown as UploadPartExecutor<TestCapability>,
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 2 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
|
|
const result = await runtime.upload(request("upload_key_put_403_always"));
|
|
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { code: "PERMISSION_DENIED", operation: "UPLOAD_PART" },
|
|
});
|
|
expect(partExecutor.uploadPart).toHaveBeenCalledTimes(3);
|
|
expect(harness.issued).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("does not retry a capability issuance the BFF itself refuses with 403", async () => {
|
|
const harness = createControlHarness({
|
|
issueCapability: () =>
|
|
browserDataFailure("PERMISSION_DENIED", "PRESIGNED_TRANSFER", {
|
|
retryable: false,
|
|
recovery: "REISSUE_CAPABILITY",
|
|
}),
|
|
});
|
|
const partExecutor = { uploadPart: vi.fn() };
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor:
|
|
partExecutor as unknown as UploadPartExecutor<TestCapability>,
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 3 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
|
|
const result = await runtime.upload(request("upload_key_issue_403"));
|
|
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { code: "PERMISSION_DENIED", retryable: false },
|
|
});
|
|
expect(harness.issued).toHaveBeenCalledTimes(1);
|
|
expect(partExecutor.uploadPart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("recovers a real presigned PUT 403 through a freshly issued capability", async () => {
|
|
const real = presignedRuntime([403, 200]);
|
|
|
|
const result = await real.runtime.upload(request("upload_key_real_put_403"));
|
|
|
|
expect(result).toMatchObject({ ok: true, value: { state: "QUARANTINED" } });
|
|
expect(real.harness.completedParts).toEqual([
|
|
[expect.objectContaining({ receiptToken: "part-etag-reissued" })],
|
|
]);
|
|
expect(real.issuance()).toBe(2);
|
|
expect(real.puts()).toBe(2);
|
|
});
|
|
});
|
|
|
|
/**
|
|
* 0929/01-01 ④. Both servers' 401/403 reach the runtime as the same
|
|
* PERMISSION_DENIED/UPLOAD_PART/REISSUE_CAPABILITY. A refused issuance means
|
|
* the session expired or the permission was revoked: no new call succeeds
|
|
* before re-authentication. A refused PUT means the signature expired: the
|
|
* runtime's next capability recovers it.
|
|
*/
|
|
describe("issuance refusal versus PUT refusal (0929/01-01)", () => {
|
|
it.each([
|
|
["the BFF refuses issuance with 401", "NONE", [401], [], 0],
|
|
["the BFF refuses issuance with 403", "NONE", [403], [], 0],
|
|
["the object store refuses every PUT with 403", "RESUME", [], [403, 403, 403, 403], 4],
|
|
] as const)(
|
|
"%s → %s",
|
|
async (_scenario, recovery, issueStatuses, putStatuses, puts) => {
|
|
const real = presignedRuntime(putStatuses, undefined, { issueStatuses });
|
|
|
|
const result = await real.runtime.upload(
|
|
request("upload_key_permission_denied"),
|
|
);
|
|
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "PERMISSION_DENIED",
|
|
operation: "UPLOAD_PART",
|
|
retryable: false,
|
|
recovery,
|
|
},
|
|
});
|
|
expect(real.puts()).toBe(puts);
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("part-stage Retry-After (2026-11-01 01-01)", () => {
|
|
// 2026-11-03 01-03 (N2). A 503 carries the server's Retry-After as a 429
|
|
// does (`statusFailure` in presigned/http-common.ts).
|
|
const stages = [
|
|
{ stage: "BFF issuance", status: 429, issueStatuses: [429], putStatuses: [] },
|
|
{ stage: "object-store PUT", status: 429, issueStatuses: [], putStatuses: [429] },
|
|
{ stage: "BFF issuance", status: 503, issueStatuses: [503], putStatuses: [] },
|
|
{ stage: "object-store PUT", status: 503, issueStatuses: [], putStatuses: [503] },
|
|
] as const;
|
|
|
|
it.each(stages)(
|
|
"waits the server's Retry-After before retrying a $stage $status",
|
|
async ({ issueStatuses, putStatuses }) => {
|
|
const real = presignedRuntime(putStatuses, undefined, {
|
|
issueStatuses,
|
|
retryAfter: "5",
|
|
maxRetryAfterMs: 30_000,
|
|
});
|
|
|
|
const result = await real.runtime.upload(
|
|
request("upload_key_part_retry_after"),
|
|
);
|
|
|
|
expect(result).toMatchObject({ ok: true });
|
|
expect(real.sleeps).toHaveLength(1);
|
|
expect(real.sleeps[0]).toBeGreaterThanOrEqual(5_000);
|
|
},
|
|
);
|
|
|
|
it.each(stages)(
|
|
"does not retry a $stage $status whose Retry-After exceeds maxRetryAfterMs",
|
|
async ({ issueStatuses, putStatuses }) => {
|
|
const real = presignedRuntime(putStatuses, undefined, {
|
|
issueStatuses,
|
|
retryAfter: "5",
|
|
maxRetryAfterMs: 1_000,
|
|
});
|
|
|
|
const result = await real.runtime.upload(
|
|
request("upload_key_part_retry_after_over"),
|
|
);
|
|
|
|
// 2026-11-03 01-02. The server asked for a longer wait than the runtime
|
|
// gives, so the app is not invited to call again at once either.
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "UNAVAILABLE",
|
|
operation: "UPLOAD_PART",
|
|
retryable: false,
|
|
recovery: "RESUME",
|
|
},
|
|
});
|
|
expect(real.sleeps).toEqual([]);
|
|
expect(real.issuance()).toBe(1);
|
|
},
|
|
);
|
|
});
|
|
|
|
/**
|
|
* 2026-11-03 01-02. A control-plane 429 past the ceiling ends the same way
|
|
* as a part-stage one, whichever of the two factories' `maxRetryAfterMs` is
|
|
* larger: the transport's (it decides `retryable`) or the runtime policy's
|
|
* (it decides whether to wait).
|
|
*/
|
|
describe("control-plane Retry-After past the ceiling (2026-11-03 01-02)", () => {
|
|
const API = "https://api.example";
|
|
const endpoints = Object.freeze({
|
|
CREATE_SESSION: `${API}/uploads/create`,
|
|
GET_STATUS: `${API}/uploads/status`,
|
|
COMPLETE: `${API}/uploads/complete`,
|
|
ABORT: `${API}/uploads/abort`,
|
|
});
|
|
|
|
it.each([
|
|
{ transportMaxRetryAfterMs: 30_000 },
|
|
{ transportMaxRetryAfterMs: 60_000 },
|
|
])(
|
|
"createSession 429 Retry-After: 45 with transport ceiling $transportMaxRetryAfterMs and policy ceiling 30000",
|
|
async ({ transportMaxRetryAfterMs }) => {
|
|
const fetcher = vi.fn(async (input: RequestInfo | URL) =>
|
|
responseWithUrl(
|
|
new Response(null, { status: 429, headers: { "Retry-After": "45" } }),
|
|
String(input),
|
|
),
|
|
);
|
|
const sleeps: number[] = [];
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: createResumableUploadHttpControlPlane({
|
|
transport: createResumableUploadFetchJsonTransport({
|
|
endpoints,
|
|
allowedOrigins: [API],
|
|
credentials: "same-origin",
|
|
fetcher: fetcher as unknown as typeof fetch,
|
|
timeoutMs: 100,
|
|
maxRequestBytes: 4_096,
|
|
maxResponseBytes: 4_096,
|
|
maxRetryAfterMs: transportMaxRetryAfterMs,
|
|
nowEpochMs: () => NOW,
|
|
}),
|
|
partCapabilities: { issueUploadPart: vi.fn() },
|
|
}),
|
|
partExecutor: { uploadPart: vi.fn() },
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 3, maxRetryAfterMs: 30_000 }),
|
|
now: () => NOW,
|
|
sleep: async (milliseconds: number) => {
|
|
sleeps.push(milliseconds);
|
|
},
|
|
});
|
|
|
|
const result = await runtime.upload(
|
|
request("upload_key_control_retry_after_over"),
|
|
);
|
|
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
error: {
|
|
code: "UNAVAILABLE",
|
|
operation: "UPLOAD_SESSION",
|
|
retryable: false,
|
|
recovery: "RESUME",
|
|
},
|
|
});
|
|
expect(fetcher).toHaveBeenCalledTimes(1);
|
|
expect(sleeps).toEqual([]);
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("resumable upload idempotency keys (01-04)", () => {
|
|
const unavailable = (
|
|
operation:
|
|
| "UPLOAD_SESSION"
|
|
| "UPLOAD_PART"
|
|
| "UPLOAD_COMPLETE"
|
|
| "UPLOAD_ABORT",
|
|
) =>
|
|
browserDataFailure("UNAVAILABLE", operation, {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
});
|
|
|
|
it("retries createSession with the same idempotency key", async () => {
|
|
const harness = createControlHarness();
|
|
const original = harness.control.createSession.bind(harness.control);
|
|
const createSession = vi.fn(
|
|
async (
|
|
input: Parameters<typeof original>[0],
|
|
): ReturnType<typeof original> =>
|
|
createSession.mock.calls.length === 1
|
|
? unavailable("UPLOAD_SESSION")
|
|
: await original(input),
|
|
);
|
|
harness.control.createSession = createSession;
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 1 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
|
|
const result = await runtime.upload(request("upload_key_create_retry"));
|
|
|
|
expect(result.ok).toBe(true);
|
|
expect(createSession).toHaveBeenCalledTimes(2);
|
|
expect(createSession.mock.calls[1]?.[0].idempotencyKey).toBe(
|
|
createSession.mock.calls[0]?.[0].idempotencyKey,
|
|
);
|
|
});
|
|
|
|
it("reuses the part key across issuance retries and a resumed upload()", async () => {
|
|
const harness = createControlHarness({
|
|
issueCapability: (input, callIndex) =>
|
|
callIndex === 1
|
|
? unavailable("UPLOAD_PART")
|
|
: browserDataSuccess({
|
|
capability: Object.freeze({ id: `cap-${callIndex}` }),
|
|
uploadBindingSha256: input.uploadBindingSha256,
|
|
expiresAtEpochMs: 5_000,
|
|
}),
|
|
});
|
|
const partKeys: string[] = [];
|
|
let puts = 0;
|
|
const partExecutor: UploadPartExecutor<TestCapability> = {
|
|
async uploadPart(input) {
|
|
puts += 1;
|
|
partKeys.push(input.idempotencyKey);
|
|
if (puts === 1) return unavailable("UPLOAD_PART");
|
|
return await executorFor(harness).uploadPart(input);
|
|
},
|
|
};
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const runtimeWith = (maxRetries: number) =>
|
|
createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor,
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_part_retry";
|
|
|
|
// Issuance fails once and is retried; the PUT then fails with no budget.
|
|
const first = await runtimeWith(1).upload(request(key));
|
|
expect(first.ok).toBe(false);
|
|
const resumed = await runtimeWith(1).upload(request(key));
|
|
expect(resumed.ok).toBe(true);
|
|
|
|
const issuedKeys = harness.issued.mock.calls.map(
|
|
([input]) => (input as Readonly<{ idempotencyKey: string }>).idempotencyKey,
|
|
);
|
|
expect(issuedKeys).toHaveLength(3);
|
|
expect(new Set([...issuedKeys, ...partKeys]).size).toBe(1);
|
|
});
|
|
|
|
it("retries complete with the same idempotency key", async () => {
|
|
const harness = createControlHarness();
|
|
const original = harness.control.complete.bind(harness.control);
|
|
const complete = vi.fn(
|
|
async (input: Parameters<typeof original>[0]): ReturnType<typeof original> =>
|
|
complete.mock.calls.length === 1
|
|
? unavailable("UPLOAD_COMPLETE")
|
|
: await original(input),
|
|
);
|
|
harness.control.complete = complete;
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 1 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
|
|
const result = await runtime.upload(request("upload_key_complete_retry"));
|
|
|
|
expect(result.ok).toBe(true);
|
|
expect(complete).toHaveBeenCalledTimes(2);
|
|
expect(complete.mock.calls[1]?.[0].idempotencyKey).toBe(
|
|
complete.mock.calls[0]?.[0].idempotencyKey,
|
|
);
|
|
});
|
|
|
|
it("resends a lost complete from a new upload() with the same key", async () => {
|
|
const harness = createControlHarness();
|
|
const original = harness.control.complete.bind(harness.control);
|
|
const complete = vi.fn(
|
|
async (input: Parameters<typeof original>[0]): ReturnType<typeof original> =>
|
|
complete.mock.calls.length === 1
|
|
? unavailable("UPLOAD_COMPLETE")
|
|
: await original(input),
|
|
);
|
|
harness.control.complete = complete;
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const runtime = () =>
|
|
createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_complete_lost";
|
|
|
|
const lost = await runtime().upload(request(key));
|
|
expect(lost).toMatchObject({
|
|
ok: false,
|
|
error: { operation: "UPLOAD_COMPLETE" },
|
|
});
|
|
const resent = await runtime().upload(request(key));
|
|
|
|
expect(resent.ok).toBe(true);
|
|
expect(complete).toHaveBeenCalledTimes(2);
|
|
expect(complete.mock.calls[1]?.[0].idempotencyKey).toBe(
|
|
complete.mock.calls[0]?.[0].idempotencyKey,
|
|
);
|
|
});
|
|
|
|
// 2026-11-03 01-03 (N10). The same key is how the server recognises an
|
|
// abort whose answer was lost as the same abort.
|
|
it("retries abort with the same key, within one abort() and from the next", async () => {
|
|
const harness = createControlHarness();
|
|
const abort = vi.fn(
|
|
async (
|
|
_input: Parameters<typeof harness.control.abort>[0],
|
|
): ReturnType<typeof harness.control.abort> =>
|
|
abort.mock.calls.length <= 2
|
|
? unavailable("UPLOAD_ABORT")
|
|
: browserDataSuccess({ state: "ABORTED" as const }),
|
|
);
|
|
harness.control.abort = abort;
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
// Every PUT fails, so upload() leaves an ACTIVE checkpoint to abort.
|
|
partExecutor: { uploadPart: async () => unavailable("UPLOAD_PART") },
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 1 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_abort_retry";
|
|
expect((await runtime.upload(request(key))).ok).toBe(false);
|
|
|
|
// Two attempts inside the first abort() both fail; the next abort() wins.
|
|
const failed = await runtime.abort({ uploadKey: key, signal: activeSignal });
|
|
expect(failed).toMatchObject({
|
|
ok: false,
|
|
error: { operation: "UPLOAD_ABORT" },
|
|
});
|
|
const retried = await runtime.abort({ uploadKey: key, signal: activeSignal });
|
|
|
|
expect(retried).toEqual({ ok: true, value: { state: "ABORTED" } });
|
|
expect(abort).toHaveBeenCalledTimes(3);
|
|
const keys = abort.mock.calls.map(([input]) => input.idempotencyKey);
|
|
expect(keys[0]).toMatch(/\S/u);
|
|
expect(new Set(keys).size).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("resumable upload completed-result reporting (01-02)", () => {
|
|
/** A server whose complete commits the resource, reported by getStatus. */
|
|
function committingHarness(
|
|
completeResult: "SUCCEEDS" | "LOST",
|
|
) {
|
|
const harness = createControlHarness();
|
|
const original = harness.control.complete.bind(harness.control);
|
|
let committed = false;
|
|
harness.control.complete = async (input) => {
|
|
committed = true;
|
|
return completeResult === "SUCCEEDS"
|
|
? await original(input)
|
|
: browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
});
|
|
};
|
|
harness.control.getStatus = async () => {
|
|
const session = harness.getSession();
|
|
if (!session) return browserDataFailure("NOT_FOUND", "UPLOAD_RECONCILE");
|
|
return committed
|
|
? browserDataSuccess({
|
|
state: "QUARANTINED",
|
|
session,
|
|
resourceId: "resource_01",
|
|
})
|
|
: browserDataSuccess({
|
|
state: "ACTIVE",
|
|
session,
|
|
acceptedParts: [...harness.accepted.values()],
|
|
});
|
|
};
|
|
harness.control.abort = async () =>
|
|
browserDataSuccess({
|
|
state: committed ? "ALREADY_COMPLETED" : "ABORTED",
|
|
});
|
|
return harness;
|
|
}
|
|
|
|
it("reports a completed upload even when the local checkpoint cleanup fails", async () => {
|
|
const harness = committingHarness("SUCCEEDS");
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const remove = checkpoints.remove.bind(checkpoints);
|
|
let failRemove = true;
|
|
checkpoints.remove = async (input) =>
|
|
failRemove
|
|
? browserDataFailure("UNAVAILABLE", "UPLOAD_RECONCILE", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
})
|
|
: await remove(input);
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_cleanup_fails";
|
|
|
|
const completed = await runtime.upload(request(key));
|
|
|
|
expect(completed).toEqual({
|
|
ok: true,
|
|
value: {
|
|
state: "QUARANTINED",
|
|
resourceId: "resource_01",
|
|
byteLength: 4,
|
|
replayed: false,
|
|
},
|
|
});
|
|
// The stale row is the next call's to reconcile, not a reason to fail.
|
|
expect(checkpoints.rows.has(key)).toBe(true);
|
|
failRemove = false;
|
|
const replayed = await runtime.upload(request(key));
|
|
expect(replayed).toMatchObject({
|
|
ok: true,
|
|
value: { resourceId: "resource_01", replayed: true },
|
|
});
|
|
expect(checkpoints.rows.has(key)).toBe(false);
|
|
});
|
|
|
|
// 0929/01-02 (2026-11-03 리뷰 2절). The reconcile path finds the lost
|
|
// complete applied; a failed local cleanup must not cost the caller the
|
|
// resourceId, exactly as on the complete path above.
|
|
it("reports the reconciled resource even when the local checkpoint cleanup fails", async () => {
|
|
const harness = committingHarness("LOST");
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const remove = checkpoints.remove.bind(checkpoints);
|
|
let failRemove = false;
|
|
checkpoints.remove = async (input) =>
|
|
failRemove
|
|
? browserDataFailure("UNAVAILABLE", "UPLOAD_RECONCILE", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
})
|
|
: await remove(input);
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_reconcile_cleanup_fails";
|
|
expect((await runtime.upload(request(key))).ok).toBe(false);
|
|
|
|
failRemove = true;
|
|
const reconciled = await runtime.upload(request(key));
|
|
|
|
expect(reconciled).toEqual({
|
|
ok: true,
|
|
value: {
|
|
state: "QUARANTINED",
|
|
resourceId: "resource_01",
|
|
byteLength: 4,
|
|
replayed: true,
|
|
},
|
|
});
|
|
// The stale row is the next call's to clean up, with the same resource.
|
|
expect(checkpoints.rows.has(key)).toBe(true);
|
|
failRemove = false;
|
|
expect(await runtime.upload(request(key))).toMatchObject({
|
|
ok: true,
|
|
value: { resourceId: "resource_01", replayed: true },
|
|
});
|
|
expect(checkpoints.rows.has(key)).toBe(false);
|
|
});
|
|
|
|
it("returns the resourceId when abort() finds the lost complete already applied", async () => {
|
|
const harness = committingHarness("LOST");
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 3 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_x2_lost_complete";
|
|
|
|
const lost = await runtime.upload(request(key));
|
|
expect(lost).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", operation: "UPLOAD_COMPLETE" },
|
|
});
|
|
const aborted = await runtime.abort({ uploadKey: key, signal: activeSignal });
|
|
|
|
expect(aborted).toEqual({
|
|
ok: true,
|
|
value: { state: "ALREADY_COMPLETED", resourceId: "resource_01" },
|
|
});
|
|
expect(checkpoints.rows.size).toBe(0);
|
|
});
|
|
|
|
it("waits out an in-progress complete answered with 429 Retry-After", async () => {
|
|
// The port's in-progress contract, as the fetch transport decodes it.
|
|
const harness = createControlHarness();
|
|
const original = harness.control.complete.bind(harness.control);
|
|
const complete = vi.fn(
|
|
async (input: Parameters<typeof original>[0]): ReturnType<typeof original> =>
|
|
complete.mock.calls.length <= 2
|
|
? Object.freeze({
|
|
ok: false as const,
|
|
error: Object.freeze({
|
|
code: "UNAVAILABLE" as const,
|
|
operation: "UPLOAD_COMPLETE" as const,
|
|
retryable: true,
|
|
recovery: "RESUME" as const,
|
|
retryAfterMs: 40,
|
|
}),
|
|
})
|
|
: await original(input),
|
|
);
|
|
harness.control.complete = complete;
|
|
const delays: number[] = [];
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 3 }),
|
|
now: () => 1_000,
|
|
random: () => 0,
|
|
sleep: async (delay) => {
|
|
delays.push(delay);
|
|
},
|
|
});
|
|
|
|
const result = await runtime.upload(request("upload_key_complete_in_progress"));
|
|
|
|
expect(result).toMatchObject({ ok: true, value: { replayed: false } });
|
|
expect(delays).toEqual([40, 40]);
|
|
expect(
|
|
new Set(complete.mock.calls.map(([input]) => input.idempotencyKey)).size,
|
|
).toBe(1);
|
|
});
|
|
|
|
it("keeps the checkpoint when the completed resource cannot be read yet", async () => {
|
|
const harness = committingHarness("LOST");
|
|
const status = harness.control.getStatus.bind(harness.control);
|
|
let statusDown = false;
|
|
harness.control.getStatus = async (input) =>
|
|
statusDown
|
|
? browserDataFailure("UNAVAILABLE", "UPLOAD_RECONCILE", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
})
|
|
: await status(input);
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const key = "upload_key_status_unavailable";
|
|
await runtime.upload(request(key));
|
|
statusDown = true;
|
|
|
|
const failed = await runtime.abort({ uploadKey: key, signal: activeSignal });
|
|
expect(failed).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", operation: "UPLOAD_ABORT", retryable: true },
|
|
});
|
|
expect(checkpoints.rows.get(key)?.state).toBe("ABORT_PENDING");
|
|
|
|
statusDown = false;
|
|
const retried = await runtime.abort({ uploadKey: key, signal: activeSignal });
|
|
expect(retried).toEqual({
|
|
ok: true,
|
|
value: { state: "ALREADY_COMPLETED", resourceId: "resource_01" },
|
|
});
|
|
expect(checkpoints.rows.size).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe("resumable upload part time limit (01-03)", () => {
|
|
const FIVE_MIB = 5 * 1024 * 1024;
|
|
|
|
/** A part PUT that ends after `durationMs`, or never when it is null. */
|
|
function timedExecutor(
|
|
harness: ReturnType<typeof createControlHarness>,
|
|
durationMs: number | null,
|
|
) {
|
|
let markStarted: () => void = () => {};
|
|
const started = new Promise<void>((resolve) => {
|
|
markStarted = resolve;
|
|
});
|
|
let attempts = 0;
|
|
const executor: UploadPartExecutor<TestCapability> = {
|
|
async uploadPart(input) {
|
|
attempts += 1;
|
|
markStarted();
|
|
const finished = await new Promise<boolean>((resolve) => {
|
|
const timer =
|
|
durationMs === null
|
|
? undefined
|
|
: setTimeout(() => resolve(true), durationMs);
|
|
input.signal.addEventListener("abort", () => {
|
|
clearTimeout(timer);
|
|
resolve(false);
|
|
});
|
|
});
|
|
if (!finished) return browserDataFailure("ABORTED", "PRESIGNED_TRANSFER");
|
|
const receipt = Object.freeze({ ...input.part, receiptToken: "etag-slow" });
|
|
harness.accepted.set(input.part.partNumber, receipt);
|
|
return browserDataSuccess(receipt);
|
|
},
|
|
};
|
|
return { executor, started, attempts: () => attempts };
|
|
}
|
|
|
|
function defaultPolicyRuntime(
|
|
harness: ReturnType<typeof createControlHarness>,
|
|
executor: UploadPartExecutor<TestCapability>,
|
|
) {
|
|
return createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executor,
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
// Default part size, attempt deadline and throughput floor.
|
|
policy: { capabilityRefreshSkewMs: 5, maxRetries: 0 },
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
}
|
|
|
|
it("lets a 5 MiB part finish at 1 Mbit/s under the default policy", async () => {
|
|
vi.useFakeTimers();
|
|
try {
|
|
const harness = createControlHarness({ serverMaxConcurrency: 1 });
|
|
// 5 MiB at 1 Mbit/s is about 42 s, past the 30 s base deadline.
|
|
const slow = timedExecutor(harness, 42_000);
|
|
const pending = defaultPolicyRuntime(harness, slow.executor).upload({
|
|
...request("upload_key_slow_uplink"),
|
|
source: rangeSource(new Uint8Array(FIVE_MIB)),
|
|
});
|
|
await slow.started;
|
|
await vi.advanceTimersByTimeAsync(43_000);
|
|
|
|
expect(await pending).toMatchObject({ ok: true });
|
|
expect(slow.attempts()).toBe(1);
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("ends a stalled 5 MiB part at the base deadline plus its byte allowance", async () => {
|
|
vi.useFakeTimers();
|
|
try {
|
|
const harness = createControlHarness({ serverMaxConcurrency: 1 });
|
|
const stalled = timedExecutor(harness, null);
|
|
let settled = false;
|
|
const pending = defaultPolicyRuntime(harness, stalled.executor)
|
|
.upload({
|
|
...request("upload_key_stalled_uplink"),
|
|
source: rangeSource(new Uint8Array(FIVE_MIB)),
|
|
})
|
|
.finally(() => {
|
|
settled = true;
|
|
});
|
|
await stalled.started;
|
|
// 30 s base + 5 MiB at the default 64 KiB/s floor (80 s).
|
|
await vi.advanceTimersByTimeAsync(109_000);
|
|
expect(settled).toBe(false);
|
|
await vi.advanceTimersByTimeAsync(2_000);
|
|
|
|
expect(await pending).toMatchObject({
|
|
ok: false,
|
|
error: { code: "UNAVAILABLE", operation: "UPLOAD_PART", retryable: true },
|
|
});
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("upload() failure recovery maps to one caller action (01-05)", () => {
|
|
/** The port's recovery → action table (`ResumableUploadPort.upload`). */
|
|
const UPLOAD_ACTIONS = new Set([
|
|
"RESUME",
|
|
"RECONCILE",
|
|
"RESTART",
|
|
"RESELECT",
|
|
"NONE",
|
|
]);
|
|
|
|
function runtimeFor(
|
|
harness: ReturnType<typeof createControlHarness>,
|
|
overrides: Partial<
|
|
Parameters<typeof createResumableUploadRuntime<TestCapability>>[0]
|
|
> = {},
|
|
) {
|
|
return createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
...overrides,
|
|
});
|
|
}
|
|
|
|
const scenarios = {
|
|
"an abort is still pending": async () => {
|
|
const harness = createControlHarness();
|
|
harness.control.complete = async () =>
|
|
browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
});
|
|
harness.control.abort = async () =>
|
|
browserDataFailure("UNAVAILABLE", "UPLOAD_ABORT", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
});
|
|
const runtime = runtimeFor(harness);
|
|
const key = "upload_key_abort_pending";
|
|
await runtime.upload(request(key));
|
|
await runtime.abort({ uploadKey: key, signal: activeSignal });
|
|
return await runtime.upload(request(key));
|
|
},
|
|
"another context wrote the checkpoint first": async () => {
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
checkpoints.compareAndSwap = async () =>
|
|
browserDataFailure("CONFLICT", "UPLOAD_RECONCILE", {
|
|
recovery: "RECONCILE",
|
|
});
|
|
return await runtimeFor(createControlHarness(), { checkpoints }).upload(
|
|
request("upload_key_cas_conflict"),
|
|
);
|
|
},
|
|
"the server status disagrees with the file": async () =>
|
|
await runtimeFor(
|
|
createControlHarness({
|
|
statusParts: (_session, accepted) =>
|
|
[...accepted.values()].map((part) => ({
|
|
...part,
|
|
checksumSha256: "e".repeat(64),
|
|
})),
|
|
}),
|
|
).upload(request("upload_key_status_mismatch")),
|
|
"the checkpoint row is unreadable": async () => {
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
checkpoints.read = async () =>
|
|
browserDataFailure("CORRUPT_DATA", "UPLOAD_RECONCILE", {
|
|
recovery: "RECONCILE",
|
|
});
|
|
return await runtimeFor(createControlHarness(), { checkpoints }).upload(
|
|
request("upload_key_unreadable"),
|
|
);
|
|
},
|
|
"the server session outlives the policy": async () => {
|
|
const harness = createControlHarness();
|
|
const create = harness.control.createSession.bind(harness.control);
|
|
harness.control.createSession = async (input) => {
|
|
const created = await create(input);
|
|
return created.ok
|
|
? browserDataSuccess({
|
|
...created.value,
|
|
expiresAtEpochMs: 1_000 + 20_000,
|
|
})
|
|
: created;
|
|
};
|
|
return await runtimeFor(harness).upload(request("upload_key_long_session"));
|
|
},
|
|
"every PUT refuses the capability": async () =>
|
|
await runtimeFor(createControlHarness(), {
|
|
partExecutor: { uploadPart: async () => putForbidden() },
|
|
}).upload(request("upload_key_refused_put")),
|
|
"the key belongs to another file": async () => {
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const harness = createControlHarness();
|
|
harness.control.complete = async () =>
|
|
browserDataFailure("UNAVAILABLE", "UPLOAD_COMPLETE", {
|
|
retryable: true,
|
|
recovery: "RESUME",
|
|
});
|
|
const runtime = runtimeFor(harness, { checkpoints });
|
|
const key = "upload_key_other_file";
|
|
await runtime.upload(request(key));
|
|
return await runtime.upload({
|
|
...request(key),
|
|
source: rangeSource(new Uint8Array([9, 9, 9, 9])),
|
|
});
|
|
},
|
|
} as const;
|
|
|
|
it.each([
|
|
["an abort is still pending", "CONFLICT", "UPLOAD_ABORT", "RESTART"],
|
|
[
|
|
"another context wrote the checkpoint first",
|
|
"CONFLICT",
|
|
"UPLOAD_RECONCILE",
|
|
"RECONCILE",
|
|
],
|
|
[
|
|
"the server status disagrees with the file",
|
|
"INTEGRITY_FAILED",
|
|
"UPLOAD_RECONCILE",
|
|
"RESTART",
|
|
],
|
|
[
|
|
"the checkpoint row is unreadable",
|
|
"CORRUPT_DATA",
|
|
"UPLOAD_RECONCILE",
|
|
"RESTART",
|
|
],
|
|
[
|
|
"the server session outlives the policy",
|
|
"POLICY_REJECTED",
|
|
"UPLOAD_SESSION",
|
|
"NONE",
|
|
],
|
|
[
|
|
"every PUT refuses the capability",
|
|
"PERMISSION_DENIED",
|
|
"UPLOAD_PART",
|
|
"RESUME",
|
|
],
|
|
[
|
|
"the key belongs to another file",
|
|
"STALE_RESULT",
|
|
"UPLOAD_RECONCILE",
|
|
"RESELECT",
|
|
],
|
|
] as const)("%s → %s/%s/%s", async (scenario, code, operation, recovery) => {
|
|
const result = await scenarios[scenario]();
|
|
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { code, operation, recovery },
|
|
});
|
|
if (!result.ok) expect(UPLOAD_ACTIONS.has(result.error.recovery)).toBe(true);
|
|
if (!result.ok && recovery === "NONE") {
|
|
expect(result.error.retryable).toBe(false);
|
|
}
|
|
});
|
|
|
|
/**
|
|
* 0929/01-01. Paths whose recovery vocabulary alone pointed at the wrong
|
|
* action. `retryable` is part of the answer: `NONE` and `RESELECT` are never
|
|
* retried automatically.
|
|
*/
|
|
const vocabularyScenarios = {
|
|
"the selected file can no longer be read": async () =>
|
|
await runtimeFor(createControlHarness()).upload({
|
|
...request("upload_key_unreadable_file"),
|
|
source: {
|
|
kind: "RANGE_READER",
|
|
reader: {
|
|
byteLength: 4,
|
|
async readRange() {
|
|
return browserDataFailure("NOT_READABLE", "FILE_READ", {
|
|
retryable: true,
|
|
recovery: "REOPEN",
|
|
});
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
"the checkpoint store fails transiently": async () => {
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
checkpoints.read = async () =>
|
|
browserDataFailure("UNAVAILABLE", "UPLOAD_RECONCILE", {
|
|
retryable: true,
|
|
recovery: "REOPEN",
|
|
});
|
|
return await runtimeFor(createControlHarness(), { checkpoints }).upload(
|
|
request("upload_key_store_busy"),
|
|
);
|
|
},
|
|
"the BFF no longer knows the session's part": async () =>
|
|
await runtimeFor(
|
|
createControlHarness({
|
|
issueCapability: () =>
|
|
browserDataFailure("NOT_FOUND", "PRESIGNED_TRANSFER"),
|
|
}),
|
|
).upload(request("upload_key_forgotten_session")),
|
|
"the runtime is closed": async () => {
|
|
const runtime = runtimeFor(createControlHarness());
|
|
runtime.close();
|
|
return await runtime.upload(request("upload_key_closed"));
|
|
},
|
|
"a capability outlives its session": async () =>
|
|
await runtimeFor(
|
|
createControlHarness({
|
|
issueCapability: (input, callIndex) =>
|
|
browserDataSuccess({
|
|
capability: Object.freeze({ id: `cap-${callIndex}` }),
|
|
uploadBindingSha256: input.uploadBindingSha256,
|
|
expiresAtEpochMs: 1_000 + 6_000,
|
|
}),
|
|
}),
|
|
).upload(request("upload_key_long_capability")),
|
|
} as const;
|
|
|
|
it.each([
|
|
["the selected file can no longer be read", "NOT_READABLE", "RESELECT", false],
|
|
["the checkpoint store fails transiently", "UNAVAILABLE", "RESUME", true],
|
|
["the BFF no longer knows the session's part", "NOT_FOUND", "RECONCILE", false],
|
|
["the runtime is closed", "UNAVAILABLE", "NONE", false],
|
|
["a capability outlives its session", "POLICY_REJECTED", "NONE", false],
|
|
] as const)(
|
|
"%s → %s/%s (retryable %s) (0929/01-01)",
|
|
async (scenario, code, recovery, retryable) => {
|
|
const result = await vocabularyScenarios[scenario]();
|
|
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { code, recovery, retryable },
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("device clock skew on server lifetimes (01-06)", () => {
|
|
/** A server whose clock runs `skewMs` ahead and gives full-length sessions. */
|
|
function skewedServer(skewMs: number, lifetimeMs: number) {
|
|
const harness = createControlHarness();
|
|
const expiresAtEpochMs = 1_000 + skewMs + lifetimeMs;
|
|
const create = harness.control.createSession.bind(harness.control);
|
|
const status = harness.control.getStatus.bind(harness.control);
|
|
harness.control.createSession = async (input) => {
|
|
const created = await create(input);
|
|
return created.ok
|
|
? browserDataSuccess({ ...created.value, expiresAtEpochMs })
|
|
: created;
|
|
};
|
|
harness.control.getStatus = async (input) => {
|
|
const read = await status(input);
|
|
return read.ok && read.value.state === "ACTIVE"
|
|
? browserDataSuccess({
|
|
...read.value,
|
|
session: { ...read.value.session, expiresAtEpochMs },
|
|
})
|
|
: read;
|
|
};
|
|
return harness;
|
|
}
|
|
|
|
it.each([
|
|
[500, true],
|
|
[501, false],
|
|
] as const)(
|
|
"accepts a max-length session from a server %i ms ahead: %s",
|
|
async (skewMs, accepted) => {
|
|
const harness = skewedServer(skewMs, 10_000);
|
|
const runtime = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints: new MemoryCheckpointStore(),
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({
|
|
maxRetries: 0,
|
|
maxSessionLifetimeMs: 10_000,
|
|
maxClockSkewMs: 500,
|
|
}),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
|
|
const result = await runtime.upload(request(`upload_key_skew_${skewMs}`));
|
|
|
|
expect(result.ok).toBe(accepted);
|
|
if (!accepted) {
|
|
expect(result).toMatchObject({
|
|
error: { code: "POLICY_REJECTED", operation: "UPLOAD_SESSION" },
|
|
});
|
|
}
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("resume after a policy change (01-08)", () => {
|
|
it("resumes a concurrency-3 checkpoint with a runtime lowered to 1", async () => {
|
|
const harness = createControlHarness({ serverMaxConcurrency: 3 });
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const source = () => rangeSource(new Uint8Array(12));
|
|
const key = "upload_key_policy_lowered";
|
|
const partial = await createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: {
|
|
async uploadPart(input) {
|
|
if (input.part.partNumber === 2) {
|
|
return browserDataFailure("UNAVAILABLE", "UPLOAD_PART", {
|
|
recovery: "RESUME",
|
|
});
|
|
}
|
|
return await executorFor(harness).uploadPart(input);
|
|
},
|
|
},
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0, maxConcurrency: 3 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
}).upload({ ...request(key), source: source() });
|
|
expect(partial.ok).toBe(false);
|
|
expect(checkpoints.rows.get(key)?.sessionMaxConcurrency).toBe(3);
|
|
|
|
const lowered = createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
// A later build lowers both limits the stored session was made under.
|
|
policy: runtimePolicy({
|
|
maxRetries: 0,
|
|
maxConcurrency: 1,
|
|
maxSessionLifetimeMs: 1_000,
|
|
}),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
});
|
|
const resumed = await lowered.upload({ ...request(key), source: source() });
|
|
|
|
expect(resumed).toMatchObject({ ok: true, value: { replayed: false } });
|
|
expect(checkpoints.rows.size).toBe(0);
|
|
});
|
|
});
|
|
|
|
/**
|
|
* 1101/01-02 (2026-11-03 리뷰 2절). A resumed FILE_BYTE_SOURCE upload reads
|
|
* the whole stream again, but hashes only the parts it still has to send,
|
|
* as the RANGE_READER path already does. The manifest hashes every part once.
|
|
*/
|
|
describe("resumed stream upload hashing (1101/01-02)", () => {
|
|
const PARTS = Array.from({ length: 8 }, (_, index) =>
|
|
Uint8Array.from([1, 2, 3, 4], (offset) => index * 4 + offset),
|
|
);
|
|
const FILE = Uint8Array.from(PARTS.flatMap((part) => [...part]));
|
|
|
|
const streamSource = () =>
|
|
Object.freeze({
|
|
kind: "FILE_BYTE_SOURCE" as const,
|
|
bytes: Object.freeze({
|
|
byteLength: FILE.byteLength,
|
|
async *stream() {
|
|
for (let offset = 0; offset < FILE.byteLength; offset += 5) {
|
|
yield browserDataSuccess(FILE.slice(offset, offset + 5));
|
|
}
|
|
},
|
|
}),
|
|
});
|
|
|
|
/** Counts SHA-256 digests whose input is exactly one part's bytes. */
|
|
function countingCrypto() {
|
|
let partDigests = 0;
|
|
const isPart = (data: Uint8Array) =>
|
|
PARTS.some(
|
|
(part) =>
|
|
part.byteLength === data.byteLength &&
|
|
part.every((byte, index) => byte === data[index]),
|
|
);
|
|
const counting = {
|
|
subtle: {
|
|
async digest(algorithm: AlgorithmIdentifier, data: Uint8Array<ArrayBuffer>) {
|
|
if (isPart(data)) partDigests += 1;
|
|
return await crypto.subtle.digest(algorithm, data);
|
|
},
|
|
},
|
|
} as unknown as Crypto;
|
|
return { crypto: counting, partDigests: () => partDigests };
|
|
}
|
|
|
|
it("hashes only the two parts the server has not accepted", async () => {
|
|
const harness = createControlHarness();
|
|
const checkpoints = new MemoryCheckpointStore();
|
|
const key = "upload_key_stream_resume_hash";
|
|
const partial = await createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: {
|
|
async uploadPart(input) {
|
|
return input.part.partNumber >= 7
|
|
? browserDataFailure("UNAVAILABLE", "UPLOAD_PART", {
|
|
recovery: "RESUME",
|
|
})
|
|
: await executorFor(harness).uploadPart(input);
|
|
},
|
|
},
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
}).upload({ ...request(key), source: streamSource() });
|
|
expect(partial.ok).toBe(false);
|
|
expect(harness.accepted.size).toBe(6);
|
|
|
|
const counted = countingCrypto();
|
|
const resumed = await createResumableUploadRuntime({
|
|
controlPlane: harness.control,
|
|
partExecutor: executorFor(harness),
|
|
checkpoints,
|
|
mutationLock: noContentionLock,
|
|
crypto: counted.crypto,
|
|
policy: runtimePolicy({ maxRetries: 0 }),
|
|
now: () => 1_000,
|
|
sleep: async () => {},
|
|
}).upload({ ...request(key), source: streamSource() });
|
|
|
|
expect(resumed).toMatchObject({ ok: true, value: { state: "QUARANTINED" } });
|
|
// 8 for the manifest, 2 for the parts sent. The old order made it 16.
|
|
expect(counted.partDigests()).toBe(10);
|
|
});
|
|
});
|
|
|
|
describe("upload part observation (01-10)", () => {
|
|
it("records a part that succeeded on its second attempt as one UPLOAD_PART", async () => {
|
|
const observations: BrowserDataObservation[] = [];
|
|
// One observer for both layers, as a composition naturally wires it.
|
|
const real = presignedRuntime([503, 200], {
|
|
record: (observation) => observations.push(observation),
|
|
});
|
|
|
|
const result = await real.runtime.upload(request("upload_key_observed"));
|
|
|
|
expect(result.ok).toBe(true);
|
|
expect(
|
|
observations.filter(({ operation }) => operation === "UPLOAD_PART"),
|
|
).toEqual([
|
|
{
|
|
operation: "UPLOAD_PART",
|
|
outcome: "SUCCEEDED",
|
|
byteBucket: "LT1MIB",
|
|
countBucket: "TWO_TO_TEN",
|
|
},
|
|
]);
|
|
// The executor's own PUT attempts stay visible under their own name.
|
|
expect(
|
|
observations.filter(
|
|
({ operation, outcome }) =>
|
|
operation === "PRESIGNED_TRANSFER" && outcome === "FAILED",
|
|
),
|
|
).toHaveLength(1);
|
|
});
|
|
});
|