fix(browser-transfer): abort sibling part workers on the first failure (Q-6.1-5)
A failing range worker only set firstFailure, which the other workers read on their next loop. An in-flight sibling kept retrying its part until its own attempt deadline, so the caller got the failure minutes late. The workers now share a transfer-scoped abort combined with the request signal; the first failure aborts it, and a sibling's resulting ABORTED never replaces the failure that won. Verified: new RED test "aborts in-flight sibling workers as soon as one part fails" (sibling signal was never aborted before the fix) + all gates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b3a68f70ca
commit
e83cda6f7f
@@ -811,6 +811,22 @@ async function transferMissingParts<Capability>(
|
||||
);
|
||||
let nextIndex = 0;
|
||||
let firstFailure: BrowserDataResult<never> | null = null;
|
||||
// Q-6.1-5. The first failure aborts every sibling worker at once, so an
|
||||
// in-flight part does not keep retrying until its own attempt deadline.
|
||||
// The sibling's resulting ABORTED never replaces the failure that won.
|
||||
const failFast = new AbortController();
|
||||
const transferScope = combineAbortSignals(
|
||||
request.signal,
|
||||
failFast.signal,
|
||||
);
|
||||
const transferRequest = Object.freeze({
|
||||
...request,
|
||||
signal: transferScope.signal,
|
||||
});
|
||||
const fail = (failure: BrowserDataResult<never>): void => {
|
||||
firstFailure ??= failure;
|
||||
failFast.abort();
|
||||
};
|
||||
// R-1.3. The resolved policy already fits maxConcurrency parts in
|
||||
// maxInFlightBytes, so the concurrency createSession requested is real.
|
||||
const workerCount = Math.min(
|
||||
@@ -831,15 +847,15 @@ async function transferMissingParts<Capability>(
|
||||
>,
|
||||
part,
|
||||
crypto: dependencies.crypto,
|
||||
signal: request.signal,
|
||||
signal: transferRequest.signal,
|
||||
});
|
||||
if (!bytes.ok) {
|
||||
firstFailure = bytes;
|
||||
fail(bytes);
|
||||
return;
|
||||
}
|
||||
const uploaded = await uploadPartWithRetry(
|
||||
dependencies,
|
||||
request,
|
||||
transferRequest,
|
||||
manifest,
|
||||
checkpoint,
|
||||
part,
|
||||
@@ -852,17 +868,21 @@ async function transferMissingParts<Capability>(
|
||||
part.byteLength,
|
||||
);
|
||||
if (!uploaded.ok) {
|
||||
firstFailure = uploaded;
|
||||
fail(uploaded);
|
||||
return;
|
||||
}
|
||||
const persisted = await persistReceipt(uploaded.value);
|
||||
if (!persisted.ok) {
|
||||
firstFailure = persisted;
|
||||
fail(persisted);
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
try {
|
||||
await Promise.all(workers);
|
||||
} finally {
|
||||
transferScope.release();
|
||||
}
|
||||
if (firstFailure) return firstFailure;
|
||||
} else {
|
||||
for await (const partResult of iterateUploadParts({
|
||||
|
||||
@@ -225,6 +225,61 @@ describe("production resumable upload runtime", () => {
|
||||
expect(serialized).not.toContain("etag-part");
|
||||
});
|
||||
|
||||
// Q-6.1-5. A sibling worker used to keep retrying its part until its own
|
||||
// attempt deadline; the first failure now aborts the whole transfer.
|
||||
it("aborts in-flight sibling workers as soon as one part fails", async () => {
|
||||
const harness = createControlHarness({ serverMaxConcurrency: 2 });
|
||||
let secondStarted: () => void = () => {};
|
||||
const secondRunning = new Promise<void>((resolve) => {
|
||||
secondStarted = resolve;
|
||||
});
|
||||
let siblingAborted = false;
|
||||
const runtime = createResumableUploadRuntime({
|
||||
controlPlane: harness.control,
|
||||
partExecutor: {
|
||||
async uploadPart(input) {
|
||||
if (input.part.partNumber === 1) {
|
||||
await secondRunning;
|
||||
return browserDataFailure("POLICY_REJECTED", "UPLOAD_PART");
|
||||
}
|
||||
secondStarted();
|
||||
return await new Promise((resolve) => {
|
||||
input.signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
siblingAborted = true;
|
||||
resolve(browserDataFailure("ABORTED", "UPLOAD_PART"));
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
},
|
||||
},
|
||||
checkpoints: new MemoryCheckpointStore(),
|
||||
mutationLock: noContentionLock,
|
||||
crypto,
|
||||
policy: runtimePolicy({ providerAttemptTimeoutMs: 60_000 }),
|
||||
now: () => 1_000,
|
||||
random: () => 0,
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
const uploading = runtime.upload({
|
||||
uploadKey: "upload_key_01",
|
||||
purpose: "attachment",
|
||||
mediaType: "application/octet-stream",
|
||||
source: rangeSource(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9])),
|
||||
signal: activeSignal,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(siblingAborted).toBe(true));
|
||||
// The caller sees the part that failed, not the sibling's ABORTED.
|
||||
await expect(uploading).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "POLICY_REJECTED", operation: "UPLOAD_PART" },
|
||||
});
|
||||
});
|
||||
|
||||
it("supports replayable FileByteSource without retaining the whole file", async () => {
|
||||
const checkpoints = new MemoryCheckpointStore();
|
||||
const harness = createControlHarness();
|
||||
|
||||
Reference in New Issue
Block a user