fix(browser-rpc): report sent command failures as MAYBE_APPLIED and fence retries (Q-6.3-9)
Three gaps in the unary/stream runtime: - A KEYED_COMMAND/NON_REPLAYABLE unary that failed after the transport was invoked (timeout, throw, UNAVAILABLE, ...) came back retryable with action "retry", although the server may have applied it. Every failure of a COMMAND after send now carries effect MAYBE_APPLIED (retryable false, contact-support). Pre-send rejections and queries are unchanged. - A retry did not re-check the scope generation, so after a logout the previous user's command could be sent again. Each retry now checks safeIsCurrent first and ends with SCOPE_GENERATION_CHANGED. - A stream whose consumer aborted but never called return() stayed suspended at a yield; its finally never ran and the operation stayed RPC_STREAM_DRAINING for good. The lease is now released (cancel + positive-close subscription) from the stream's own abort listener; finally triggers the same path through its abort. Verified: five new tests failed RED (no effect, second attempt sent, cancel never called) and pass now; a pin that pre-send command failures and query failures stay effect-free passes before and after. All lane gates pass except the known removal-fixture test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fcd37a9d16
commit
9b11d0dd2c
@@ -22,6 +22,7 @@ import {
|
||||
import type { InstalledBoundaryMapper } from "../../contracts/boundary-mapper.ts";
|
||||
import {
|
||||
createFailure,
|
||||
withFailureEffect,
|
||||
type AppFailure,
|
||||
type FailureKind,
|
||||
} from "../../contracts/errors.ts";
|
||||
@@ -292,6 +293,7 @@ async function executeUnary<Output>(
|
||||
const { operation, profile, encoder, transport } = bound;
|
||||
const linked = linkedAbortController(context.signal);
|
||||
let attemptCount = 0;
|
||||
let sent = false;
|
||||
|
||||
const finish = (
|
||||
result: Result<Output, AppFailure>,
|
||||
@@ -300,7 +302,11 @@ async function executeUnary<Output>(
|
||||
// R-06. Listener and timer cleanup happens exactly once, on every exit.
|
||||
linked.cleanup();
|
||||
observe(dependencies, operation, outcome, attemptCount, result.ok ? 1 : 0);
|
||||
return result;
|
||||
// Q-6.3-9. Once a command reached the transport, no failure proves the
|
||||
// server did not apply it, so it is never offered as a plain retry.
|
||||
return !result.ok && sent && operation.semantics === "COMMAND"
|
||||
? failureResult(withFailureEffect(result.error, "MAYBE_APPLIED"))
|
||||
: result;
|
||||
};
|
||||
|
||||
const generation = safeCapture(generationFence);
|
||||
@@ -350,6 +356,21 @@ async function executeUnary<Output>(
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < profile.maxAttempts; attempt += 1) {
|
||||
// Q-6.3-9. A retry never outlives the scope generation that started it,
|
||||
// so a previous user's request is not sent again after a logout.
|
||||
if (attempt > 0 && !safeIsCurrent(generationFence, generation)) {
|
||||
return finish(
|
||||
failureResult(
|
||||
callFailure(
|
||||
operation,
|
||||
attempt - 1,
|
||||
"SCOPE_GENERATION_CHANGED",
|
||||
"RPC_SCOPE_GENERATION_CHANGED",
|
||||
),
|
||||
),
|
||||
"ABORTED",
|
||||
);
|
||||
}
|
||||
attemptCount = attempt + 1;
|
||||
const attemptNow = safeNow(clock);
|
||||
if (attemptNow === CLOCK_FAILURE) {
|
||||
@@ -393,6 +414,7 @@ async function executeUnary<Output>(
|
||||
? { idempotencyKey: context.idempotencyKey }
|
||||
: {}),
|
||||
});
|
||||
sent = true;
|
||||
const timed = await raceWithin(
|
||||
Promise.resolve().then(() => transport.invokeUnary!(call)),
|
||||
remainingMs,
|
||||
@@ -549,10 +571,30 @@ async function* executeServerStream<Event>(
|
||||
const deadlineAt = safeNowOrDeadline(clock, 0) + operation.totalDeadlineMs;
|
||||
const linked = linkedAbortController(context.signal);
|
||||
let iterator: AsyncIterator<BrowserRpcStreamFrame> | null = null;
|
||||
let lease: BrowserRpcServerStreamLease | null = null;
|
||||
let registered: ActiveStreamLease | null = null;
|
||||
let observed = false;
|
||||
let messageCount = 0;
|
||||
|
||||
// R-01 / RPC-RR-01. Runs once, when the stream's own signal aborts. The
|
||||
// lease's `cancel` is what actually stops the physical stream, and the entry
|
||||
// is removed only once the transport confirms the physical stream closed.
|
||||
// Until then the operation stays DRAINING and admits nothing new — a bounded
|
||||
// wait here would re-open the very hole this registry exists to close.
|
||||
const releaseRegistered = () => {
|
||||
const entry = registered;
|
||||
if (!entry) return;
|
||||
try {
|
||||
entry.cancel("stream-closed");
|
||||
} catch {
|
||||
// A transport that refuses to cancel stays DRAINING below.
|
||||
}
|
||||
void entry.closed.then((confirmed) => {
|
||||
if (confirmed && activeStreams.get(operation.operationId) === entry) {
|
||||
activeStreams.delete(operation.operationId);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const finish = (outcome: BrowserRpcObservationOutcome) => {
|
||||
if (observed) return;
|
||||
observed = true;
|
||||
@@ -662,21 +704,28 @@ async function* executeServerStream<Event>(
|
||||
yield failureResult(protocolFailure(operation, 0));
|
||||
return;
|
||||
}
|
||||
lease = decodedLease;
|
||||
const lease = decodedLease;
|
||||
// Registered at the moment the physical stream exists, not after a timeout.
|
||||
activeStreams.set(
|
||||
operation.operationId,
|
||||
Object.freeze({
|
||||
streamId: lease.streamId,
|
||||
cancel: lease.cancel,
|
||||
// RPC-01. Only a fulfilled, contract-shaped receipt is evidence that the
|
||||
// physical stream closed. Absorbing a rejection, a synchronous throw or
|
||||
// a non-promise into `undefined` forged that evidence, and the registry
|
||||
// then admitted a second stream for the same operation while the first
|
||||
// was still running against the server.
|
||||
closed: confirmPhysicalClosure(lease),
|
||||
}),
|
||||
);
|
||||
registered = Object.freeze({
|
||||
streamId: lease.streamId,
|
||||
cancel: lease.cancel,
|
||||
// RPC-01. Only a fulfilled, contract-shaped receipt is evidence that the
|
||||
// physical stream closed. Absorbing a rejection, a synchronous throw or
|
||||
// a non-promise into `undefined` forged that evidence, and the registry
|
||||
// then admitted a second stream for the same operation while the first
|
||||
// was still running against the server.
|
||||
closed: confirmPhysicalClosure(lease),
|
||||
});
|
||||
activeStreams.set(operation.operationId, registered);
|
||||
// Q-6.3-9. A consumer that aborts and walks away without `return()` leaves
|
||||
// this generator suspended at a `yield`, so `finally` never runs. The
|
||||
// abort itself releases the physical stream.
|
||||
if (linked.controller.signal.aborted) releaseRegistered();
|
||||
else {
|
||||
linked.controller.signal.addEventListener("abort", releaseRegistered, {
|
||||
once: true,
|
||||
});
|
||||
}
|
||||
try {
|
||||
iterator = lease.frames[Symbol.asyncIterator]();
|
||||
} catch {
|
||||
@@ -842,38 +891,13 @@ async function* executeServerStream<Event>(
|
||||
}
|
||||
} finally {
|
||||
// R-01 / RPC-RR-01. The commit/admission generation is fenced immediately
|
||||
// and listeners are released without waiting for the transport.
|
||||
// `iterator.return()` is a cleanup request, not a lifecycle authority, so
|
||||
// the lease's own `cancel` is what actually stops the physical stream.
|
||||
// and listeners are released without waiting for the transport. The abort
|
||||
// runs `releaseRegistered` (cancel + positive-close subscription) unless
|
||||
// an earlier abort already did. RPC-02: that happens before any fallible
|
||||
// cleanup, so a throwing `return` accessor cannot strand the entry.
|
||||
// `iterator.return()` is a cleanup request, not a lifecycle authority.
|
||||
linked.controller.abort("stream-closed");
|
||||
linked.cleanup();
|
||||
const registered = lease
|
||||
? activeStreams.get(operation.operationId)
|
||||
: undefined;
|
||||
if (registered && registered.streamId === lease?.streamId) {
|
||||
try {
|
||||
registered.cancel("stream-closed");
|
||||
} catch {
|
||||
// A transport that refuses to cancel stays DRAINING below.
|
||||
}
|
||||
}
|
||||
if (registered && registered.streamId === lease?.streamId) {
|
||||
// RPC-02. The positive-close subscription is installed before any
|
||||
// fallible cleanup. Running cleanup first meant a throwing `return`
|
||||
// accessor could skip it entirely and strand the registry entry.
|
||||
// The entry is removed only once the transport confirms the physical
|
||||
// stream closed. Until then the operation stays DRAINING and admits
|
||||
// nothing new — a bounded wait here would re-open the very hole this
|
||||
// registry exists to close.
|
||||
void registered.closed.then((confirmed) => {
|
||||
if (
|
||||
confirmed &&
|
||||
activeStreams.get(operation.operationId) === registered
|
||||
) {
|
||||
activeStreams.delete(operation.operationId);
|
||||
}
|
||||
});
|
||||
}
|
||||
const returnIterator = safeIteratorReturn(iterator);
|
||||
if (returnIterator) {
|
||||
const cleanup = Promise.resolve()
|
||||
@@ -882,7 +906,7 @@ async function* executeServerStream<Event>(
|
||||
.catch(() => undefined);
|
||||
await boundedStreamCleanup(cleanup, clock, STREAM_CLEANUP_BOUND_MS);
|
||||
}
|
||||
if (registered && registered.streamId === lease?.streamId) {
|
||||
if (registered) {
|
||||
await boundedStreamCleanup(
|
||||
registered.closed.then(() => undefined),
|
||||
clock,
|
||||
|
||||
@@ -795,3 +795,229 @@ describe("RPC-03 registries are snapshotted before they are validated", () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Q-6.3-9. A command the transport already received may have been applied, a
|
||||
* retry must not outlive the scope generation that started it, and a stream the
|
||||
* consumer aborted without calling `return()` must not hold the operation in
|
||||
* DRAINING forever.
|
||||
*/
|
||||
describe("Q-6.3-9 command effect, retry scope and abandoned streams", () => {
|
||||
const COMMAND_KEY = "command-key-0001";
|
||||
const commandOperation = (
|
||||
overrides: Parameters<typeof unaryOperation>[0] = {},
|
||||
) =>
|
||||
unaryOperation({
|
||||
semantics: "COMMAND",
|
||||
replayPolicy: "KEYED_COMMAND",
|
||||
idempotencyKeyPolicy: "REQUIRED",
|
||||
idempotencyLevel: "IDEMPOTENT",
|
||||
serverStateProfileId: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
function commandRuntime(
|
||||
invokeUnary: BrowserRpcTransport["invokeUnary"],
|
||||
extra: Readonly<{
|
||||
operation?: ReturnType<typeof unaryOperation>;
|
||||
profile?: ReturnType<typeof unaryProfile>;
|
||||
generationFence?: Readonly<{
|
||||
capture(): unknown;
|
||||
isCurrent(token: unknown): boolean;
|
||||
}>;
|
||||
}> = {},
|
||||
) {
|
||||
return createBrowserRpcRuntime({
|
||||
operations: { GET_RPC_RESOURCE: extra.operation ?? commandOperation() },
|
||||
profiles: { CONNECT_REFERENCE_UNARY: extra.profile ?? unaryProfile() },
|
||||
schemaCodecs: SCHEMA_CODECS,
|
||||
mappers: MAPPERS,
|
||||
requestEncoders: { RpcResourceRequestEncoder: UNARY_ENCODER },
|
||||
transports: { CONNECT_REFERENCE_UNARY: unaryTransport(invokeUnary) },
|
||||
...(extra.generationFence === undefined
|
||||
? {}
|
||||
: { generationFence: extra.generationFence }),
|
||||
}).bindUnary("GET_RPC_RESOURCE", isResourceView);
|
||||
}
|
||||
|
||||
const afterSend: ReadonlyArray<
|
||||
Readonly<{
|
||||
label: string;
|
||||
invokeUnary: BrowserRpcTransport["invokeUnary"];
|
||||
operation?: ReturnType<typeof unaryOperation>;
|
||||
}>
|
||||
> = [
|
||||
{
|
||||
label: "answers UNAVAILABLE",
|
||||
invokeUnary: async () => ({
|
||||
ok: false,
|
||||
failure: { code: "UNAVAILABLE" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
label: "throws",
|
||||
invokeUnary: async () => {
|
||||
throw new TypeError("connection reset");
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "never answers",
|
||||
invokeUnary: () => new Promise<never>(() => {}),
|
||||
operation: commandOperation({ totalDeadlineMs: 25 }),
|
||||
},
|
||||
];
|
||||
|
||||
for (const { label, invokeUnary, operation } of afterSend) {
|
||||
it(`reports MAYBE_APPLIED when a sent command's transport ${label}`, async () => {
|
||||
const port = commandRuntime(invokeUnary, {
|
||||
...(operation === undefined ? {} : { operation }),
|
||||
});
|
||||
|
||||
await expect(
|
||||
port.execute({ resourceId: "r-1" }, { idempotencyKey: COMMAND_KEY }),
|
||||
).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
effect: "MAYBE_APPLIED",
|
||||
retryable: false,
|
||||
action: "contact-support",
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps a command rejected before send and a failed query effect-free", async () => {
|
||||
const unsent = await commandRuntime(async () => {
|
||||
throw new Error("must not be called");
|
||||
}).execute({ resourceId: "r-1" });
|
||||
expect(unsent).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "VALIDATION_REJECTED" },
|
||||
});
|
||||
expect(unsent.ok ? null : unsent.error).not.toHaveProperty("effect");
|
||||
|
||||
const query = await unaryRuntime(
|
||||
unaryTransport(async () => ({
|
||||
ok: false,
|
||||
failure: { code: "UNAVAILABLE" },
|
||||
})),
|
||||
)
|
||||
.bindUnary("GET_RPC_RESOURCE", isResourceView)
|
||||
.execute({ resourceId: "r-1" });
|
||||
expect(query).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "SERVER_FAILURE", retryable: true },
|
||||
});
|
||||
expect(query.ok ? null : query.error).not.toHaveProperty("effect");
|
||||
});
|
||||
|
||||
it("does not retry a command after its scope generation changed", async () => {
|
||||
let calls = 0;
|
||||
let generation = 1;
|
||||
const port = commandRuntime(
|
||||
async () => {
|
||||
calls += 1;
|
||||
// The session changes while the first attempt is in flight.
|
||||
generation = 2;
|
||||
return { ok: false, failure: { code: "UNAVAILABLE" } };
|
||||
},
|
||||
{
|
||||
profile: unaryProfile({
|
||||
retryProfileId: "RPC_RETRY_TWO",
|
||||
retryOwner: "FRONTEND_ADAPTER",
|
||||
maxAttempts: 2,
|
||||
backoffMs: [0],
|
||||
retryableFailures: ["UNAVAILABLE"],
|
||||
maxRetryAfterMs: 100,
|
||||
}),
|
||||
operation: commandOperation({ retryProfileId: "RPC_RETRY_TWO" }),
|
||||
generationFence: {
|
||||
capture: () => generation,
|
||||
isCurrent: (token) => token === generation,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const result = await port.execute(
|
||||
{ resourceId: "r-1" },
|
||||
{ idempotencyKey: COMMAND_KEY },
|
||||
);
|
||||
|
||||
expect(calls).toBe(1);
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "SCOPE_GENERATION_CHANGED",
|
||||
effect: "MAYBE_APPLIED",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("releases a stream the consumer aborted without calling return()", async () => {
|
||||
const cancels: string[] = [];
|
||||
let opened = 0;
|
||||
const transport = defineBrowserRpcTransport({
|
||||
runtimeProfileId: "CONNECT_REFERENCE_STREAM",
|
||||
providerId: "REFERENCE_RPC",
|
||||
protocol: "CONNECT_HTTP",
|
||||
rpcKind: "SERVER_STREAM",
|
||||
openServerStream: () => {
|
||||
opened += 1;
|
||||
let closed: () => void = () => {};
|
||||
const closure = new Promise<void>((resolve) => {
|
||||
closed = resolve;
|
||||
});
|
||||
let sent = false;
|
||||
return {
|
||||
streamId: `physical-${opened}`,
|
||||
frames: {
|
||||
[Symbol.asyncIterator]: () =>
|
||||
({
|
||||
next: () => {
|
||||
if (sent) return new Promise<never>(() => {});
|
||||
sent = true;
|
||||
return Promise.resolve({
|
||||
done: false,
|
||||
value: {
|
||||
kind: "MESSAGE",
|
||||
message: { id: "r-1", name: "Resource" },
|
||||
encodedBytes: 16,
|
||||
},
|
||||
});
|
||||
},
|
||||
}) as AsyncIterator<BrowserRpcStreamFrame>,
|
||||
},
|
||||
cancel(reason: string) {
|
||||
cancels.push(reason);
|
||||
closed();
|
||||
},
|
||||
waitClosed: () => closure,
|
||||
};
|
||||
},
|
||||
});
|
||||
const stream = streamingRuntime(transport).bindServerStream(
|
||||
"WATCH_RPC_RESOURCES",
|
||||
isResourceView,
|
||||
);
|
||||
const consumer = new AbortController();
|
||||
|
||||
const opening = stream.open(
|
||||
{ resourceId: "r-1" },
|
||||
{ signal: consumer.signal },
|
||||
);
|
||||
const first = opening[Symbol.asyncIterator]();
|
||||
await expect(first.next()).resolves.toMatchObject({
|
||||
value: { ok: true },
|
||||
});
|
||||
// The consumer walks away: it aborts but never calls next() or return().
|
||||
consumer.abort();
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
|
||||
expect(cancels).toHaveLength(1);
|
||||
const second = stream.open({ resourceId: "r-1" })[Symbol.asyncIterator]();
|
||||
await expect(second.next()).resolves.toMatchObject({
|
||||
value: { ok: true },
|
||||
});
|
||||
expect(opened).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user