fix: make Browser RPC and Realtime own the physical work they report on
A server stream's registration was pruned on any settled close receipt.
`waitClosed()` rejecting, throwing synchronously, or not returning a
promise at all was absorbed into a fulfilled `undefined`, so the runtime
opened a second physical stream for the same operation while the first was
still running against the server. Only a fulfilled, contract-shaped
receipt confirms closure now; every negative receipt keeps the operation
DRAINING.
Cleanup also read foreign state outside the result boundary. A throwing
iterator `return` accessor replaced the already selected timeout with a
native `TypeError` and skipped the rest of the teardown, and the exported
lease decoder threw on a hostile `Symbol.asyncIterator`. Both reads move
inside their own boundaries, and the positive-close subscription is
installed before any fallible cleanup.
Composition validated the caller's registries before snapshotting them, so
a hostile accessor ran twice during validation, and rows hiding fields
behind a prototype or a non-enumerable key installed. Transport results
were checked for allowed own keys only, so own `{ok,message,encodedBytes}`
plus a prototype `injected` was a success and a missing `message` reached
a permissive schema as `undefined`.
In Realtime the tracked task was registered after the collaborator
returned. An authority that re-entered `close()` from inside its own
invocation saw an empty registry and got `{ok:true}` while its effect was
pending. The task is now registered first and the collaborator is invoked
a microtask later. A `scheduleTimeout` that threw was worse: the caller's
own catch treated it as an apply failure and started a recovery beside the
still-running effect, and `close()` rejected with a native `TypeError`. An
uninstallable deadline now fails closed as an expired one.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
co-authored by
Claude Opus 5
Claude Opus 5
parent
632b230c82
commit
aa8ac35600
@@ -7,6 +7,7 @@ import {
|
||||
type BrowserRpcStreamFrame,
|
||||
type BrowserRpcTransport,
|
||||
} from "../../../src/adapters/browser-rpc/index.ts";
|
||||
import { decodeServerStreamLease } from "../../../src/adapters/browser-rpc/transport.ts";
|
||||
import { installBrowserRpcContractBindings } from "../../../src/contracts/browser-rpc.ts";
|
||||
import {
|
||||
MAPPERS,
|
||||
@@ -474,3 +475,316 @@ describe("RPC-RR-01 server stream leases and the DRAINING fence", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* RPC-01. A `waitClosed()` that rejects, throws, or does not even return a
|
||||
* promise is not evidence that the physical stream closed. Absorbing all three
|
||||
* into a fulfilled `undefined` let the registry prune a live stream and admit a
|
||||
* second one for the same operation against the same server.
|
||||
*/
|
||||
describe("RPC-01 only a positive receipt confirms physical closure", () => {
|
||||
const shortDeadline = () =>
|
||||
streamOperation({ totalDeadlineMs: 25, idleDeadlineMs: 25 });
|
||||
|
||||
function leaseProbeWith(
|
||||
waitClosed: () => unknown,
|
||||
): Readonly<{
|
||||
transport: BrowserRpcTransport;
|
||||
cancels: string[];
|
||||
opened(): number;
|
||||
}> {
|
||||
const cancels: string[] = [];
|
||||
const counter = { opened: 0 };
|
||||
const transport = defineBrowserRpcTransport({
|
||||
runtimeProfileId: "CONNECT_REFERENCE_STREAM",
|
||||
providerId: "REFERENCE_RPC",
|
||||
protocol: "CONNECT_HTTP",
|
||||
rpcKind: "SERVER_STREAM",
|
||||
openServerStream: () => {
|
||||
counter.opened += 1;
|
||||
return {
|
||||
streamId: `physical-${counter.opened}`,
|
||||
frames: {
|
||||
[Symbol.asyncIterator]: () =>
|
||||
({
|
||||
next: () => new Promise<never>(() => {}),
|
||||
}) as AsyncIterator<BrowserRpcStreamFrame>,
|
||||
},
|
||||
cancel(reason: string) {
|
||||
cancels.push(reason);
|
||||
},
|
||||
waitClosed: waitClosed as () => Promise<void>,
|
||||
};
|
||||
},
|
||||
});
|
||||
return Object.freeze({
|
||||
transport,
|
||||
cancels,
|
||||
opened: () => counter.opened,
|
||||
});
|
||||
}
|
||||
|
||||
const negativeReceipts = [
|
||||
{
|
||||
label: "rejects",
|
||||
waitClosed: () => Promise.reject(new Error("never closed")),
|
||||
},
|
||||
{
|
||||
label: "throws synchronously",
|
||||
waitClosed: () => {
|
||||
throw new TypeError("waitClosed exploded");
|
||||
},
|
||||
},
|
||||
{ label: "returns a non-promise", waitClosed: () => "closed" },
|
||||
{ label: "never settles", waitClosed: () => new Promise<void>(() => {}) },
|
||||
];
|
||||
|
||||
for (const { label, waitClosed } of negativeReceipts) {
|
||||
it(`keeps the operation DRAINING when waitClosed ${label}`, async () => {
|
||||
const probe = leaseProbeWith(waitClosed);
|
||||
const runtime = createBrowserRpcRuntime({
|
||||
operations: { WATCH_RPC_RESOURCES: shortDeadline() },
|
||||
profiles: { CONNECT_REFERENCE_STREAM: streamProfile() },
|
||||
schemaCodecs: SCHEMA_CODECS,
|
||||
mappers: MAPPERS,
|
||||
requestEncoders: { RpcResourceStreamRequestEncoder: STREAM_ENCODER },
|
||||
transports: { CONNECT_REFERENCE_STREAM: probe.transport },
|
||||
});
|
||||
const stream = runtime.bindServerStream(
|
||||
"WATCH_RPC_RESOURCES",
|
||||
isResourceView,
|
||||
);
|
||||
|
||||
await collect(stream.open({ resourceId: "scope-1" }));
|
||||
expect(probe.opened()).toBe(1);
|
||||
expect(probe.cancels).toHaveLength(1);
|
||||
|
||||
const second = await collect(stream.open({ resourceId: "scope-1" }));
|
||||
expect(second).toHaveLength(1);
|
||||
expect(second[0]).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "RPC_STREAM_DRAINING" },
|
||||
});
|
||||
// No second physical stream was ever opened.
|
||||
expect(probe.opened()).toBe(1);
|
||||
});
|
||||
}
|
||||
|
||||
it("admits a second stream once waitClosed fulfils", async () => {
|
||||
let release: (() => void) | undefined;
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const probe = leaseProbeWith(() => closed);
|
||||
const runtime = createBrowserRpcRuntime({
|
||||
operations: { WATCH_RPC_RESOURCES: shortDeadline() },
|
||||
profiles: { CONNECT_REFERENCE_STREAM: streamProfile() },
|
||||
schemaCodecs: SCHEMA_CODECS,
|
||||
mappers: MAPPERS,
|
||||
requestEncoders: { RpcResourceStreamRequestEncoder: STREAM_ENCODER },
|
||||
transports: { CONNECT_REFERENCE_STREAM: probe.transport },
|
||||
});
|
||||
const stream = runtime.bindServerStream(
|
||||
"WATCH_RPC_RESOURCES",
|
||||
isResourceView,
|
||||
);
|
||||
|
||||
await collect(stream.open({ resourceId: "scope-1" }));
|
||||
release?.();
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
await collect(stream.open({ resourceId: "scope-1" }));
|
||||
expect(probe.opened()).toBe(2);
|
||||
});
|
||||
|
||||
/**
|
||||
* RPC-02. Cleanup reads foreign state, so it must stay inside the result
|
||||
* boundary: a throwing `return` accessor replaced the already selected
|
||||
* outcome with a native rejection and skipped the rest of the teardown.
|
||||
*/
|
||||
it("keeps the selected outcome when the iterator return accessor throws", async () => {
|
||||
const cancels: string[] = [];
|
||||
const transport = defineBrowserRpcTransport({
|
||||
runtimeProfileId: "CONNECT_REFERENCE_STREAM",
|
||||
providerId: "REFERENCE_RPC",
|
||||
protocol: "CONNECT_HTTP",
|
||||
rpcKind: "SERVER_STREAM",
|
||||
openServerStream: () => ({
|
||||
streamId: "physical-hostile",
|
||||
frames: {
|
||||
[Symbol.asyncIterator]: () =>
|
||||
Object.defineProperty(
|
||||
{ next: () => new Promise<never>(() => {}) },
|
||||
"return",
|
||||
{
|
||||
enumerable: true,
|
||||
get() {
|
||||
throw new TypeError("return getter escaped");
|
||||
},
|
||||
},
|
||||
) as AsyncIterator<BrowserRpcStreamFrame>,
|
||||
},
|
||||
cancel(reason: string) {
|
||||
cancels.push(reason);
|
||||
},
|
||||
waitClosed: async () => {},
|
||||
}),
|
||||
});
|
||||
const runtime = createBrowserRpcRuntime({
|
||||
operations: { WATCH_RPC_RESOURCES: shortDeadline() },
|
||||
profiles: { CONNECT_REFERENCE_STREAM: streamProfile() },
|
||||
schemaCodecs: SCHEMA_CODECS,
|
||||
mappers: MAPPERS,
|
||||
requestEncoders: { RpcResourceStreamRequestEncoder: STREAM_ENCODER },
|
||||
transports: { CONNECT_REFERENCE_STREAM: transport },
|
||||
});
|
||||
|
||||
const results = await collect(
|
||||
runtime
|
||||
.bindServerStream("WATCH_RPC_RESOURCES", isResourceView)
|
||||
.open({ resourceId: "scope-1" }),
|
||||
);
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "RPC_TOTAL_DEADLINE_EXCEEDED" },
|
||||
});
|
||||
expect(cancels).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("decodes a hostile frames iterator to null instead of throwing", async () => {
|
||||
const hostile = {
|
||||
streamId: "physical-hostile",
|
||||
frames: Object.defineProperty({}, Symbol.asyncIterator, {
|
||||
enumerable: true,
|
||||
get() {
|
||||
throw new TypeError("async iterator getter escaped");
|
||||
},
|
||||
}),
|
||||
cancel() {},
|
||||
async waitClosed() {},
|
||||
};
|
||||
expect(decodeServerStreamLease(hostile)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* RPC-04. An exact union is exactly these own data keys on a plain object. A
|
||||
* custom prototype could otherwise carry metadata past the boundary while the
|
||||
* own shape still looked valid, and a missing wire field could reach a
|
||||
* permissive schema as `undefined`.
|
||||
*/
|
||||
describe("RPC-04 transport results are an exact union", () => {
|
||||
const hostileResults = [
|
||||
{
|
||||
label: "valid own fields with an inherited extra",
|
||||
result: () =>
|
||||
Object.assign(Object.create({ injected: "prototype" }), {
|
||||
ok: true,
|
||||
message: { id: "a", name: "A" },
|
||||
encodedBytes: 8,
|
||||
}),
|
||||
},
|
||||
{
|
||||
label: "a missing message",
|
||||
result: () => ({ ok: true, encodedBytes: 8 }),
|
||||
},
|
||||
{
|
||||
label: "a non-enumerable own extra",
|
||||
result: () =>
|
||||
Object.defineProperty(
|
||||
{ ok: true, message: { id: "a", name: "A" }, encodedBytes: 8 },
|
||||
"injected",
|
||||
{ enumerable: false, value: true },
|
||||
),
|
||||
},
|
||||
{
|
||||
label: "a failure with an inherited code",
|
||||
result: () => ({
|
||||
ok: false,
|
||||
failure: Object.create({ code: "SERVER_FAILURE" }) as object,
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
for (const { label, result } of hostileResults) {
|
||||
it(`rejects ${label} as a protocol mismatch`, async () => {
|
||||
const runtime = unaryRuntime(
|
||||
unaryTransport(async () => result() as never),
|
||||
);
|
||||
await expect(
|
||||
runtime
|
||||
.bindUnary("GET_RPC_RESOURCE", isResourceView)
|
||||
.execute({ resourceId: "scope-1" }),
|
||||
).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "RPC_PROTOCOL_MISMATCH" },
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* RPC-03. Every registry is snapshotted before anything validates it, so a
|
||||
* hostile accessor never runs, and a row that hides fields behind a prototype
|
||||
* or a non-enumerable key is refused rather than installed.
|
||||
*/
|
||||
describe("RPC-03 registries are snapshotted before they are validated", () => {
|
||||
const build = (operations: Record<string, unknown>) =>
|
||||
createBrowserRpcRuntime({
|
||||
operations: operations as never,
|
||||
profiles: { CONNECT_REFERENCE_UNARY: unaryProfile() },
|
||||
schemaCodecs: SCHEMA_CODECS,
|
||||
mappers: MAPPERS,
|
||||
requestEncoders: { RpcResourceRequestEncoder: UNARY_ENCODER },
|
||||
transports: {
|
||||
CONNECT_REFERENCE_UNARY: unaryTransport(async () => ({
|
||||
ok: true,
|
||||
message: { id: "a", name: "A" },
|
||||
encodedBytes: 8,
|
||||
})),
|
||||
},
|
||||
});
|
||||
|
||||
it("never invokes a registry accessor", () => {
|
||||
let reads = 0;
|
||||
const operations = Object.defineProperty({}, "GET_RPC_RESOURCE", {
|
||||
enumerable: true,
|
||||
get() {
|
||||
reads += 1;
|
||||
return unaryOperation();
|
||||
},
|
||||
});
|
||||
expect(() => build(operations)).toThrow(TypeError);
|
||||
expect(reads).toBe(0);
|
||||
});
|
||||
|
||||
const hostileRows = [
|
||||
{
|
||||
label: "an inherited extra field",
|
||||
row: () =>
|
||||
Object.assign(Object.create({ injected: true }), unaryOperation()),
|
||||
},
|
||||
{
|
||||
label: "a non-enumerable own extra field",
|
||||
row: () =>
|
||||
Object.defineProperty({ ...unaryOperation() }, "injected", {
|
||||
enumerable: false,
|
||||
value: true,
|
||||
}),
|
||||
},
|
||||
{
|
||||
label: "a symbol field",
|
||||
row: () => ({
|
||||
...unaryOperation(),
|
||||
[Symbol.for("injected")]: true,
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
for (const { label, row } of hostileRows) {
|
||||
it(`refuses to install a row with ${label}`, () => {
|
||||
expect(() => build({ GET_RPC_RESOURCE: row() })).toThrow(TypeError);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1068,6 +1068,8 @@ type HarnessOptions = Readonly<{
|
||||
recoveryTimeoutMs?: number;
|
||||
drainTimeoutMs?: number;
|
||||
}>;
|
||||
scheduleTimeout?: (callback: () => void, delayMs: number) => unknown;
|
||||
clearScheduledTimeout?: (handle: unknown) => void;
|
||||
}>;
|
||||
|
||||
function createHarness(options: HarnessOptions = {}) {
|
||||
@@ -1114,6 +1116,12 @@ function createHarness(options: HarnessOptions = {}) {
|
||||
now: () => 10_000,
|
||||
observe: options.observe,
|
||||
...(options.taskLimits ? { taskLimits: options.taskLimits } : {}),
|
||||
...(options.scheduleTimeout
|
||||
? { scheduleTimeout: options.scheduleTimeout }
|
||||
: {}),
|
||||
...(options.clearScheduledTimeout
|
||||
? { clearScheduledTimeout: options.clearScheduledTimeout }
|
||||
: {}),
|
||||
});
|
||||
|
||||
return {
|
||||
@@ -1176,3 +1184,168 @@ function deferred<Value>() {
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
/**
|
||||
* RT-01. The registry entry has to exist before the collaborator is called.
|
||||
* Registering after the invocation returned left a window in which an authority
|
||||
* that re-entered `close()` from inside its own callback saw an empty set, so
|
||||
* `close()` reported quiescence while its effect was still running.
|
||||
*/
|
||||
describe("RT-01 reentrant close cannot pass the registration window", () => {
|
||||
it("refuses quiescence when apply re-enters close during its invocation", async () => {
|
||||
const applyGate = deferred<RealtimeResult<void>>();
|
||||
let closeResult: RealtimeResult<void> | undefined;
|
||||
let harness: ReturnType<typeof createHarness> | undefined;
|
||||
harness = createHarness({
|
||||
apply: () => {
|
||||
// Re-entered from inside the invocation itself.
|
||||
void harness!.coordinator.close().then((result) => {
|
||||
closeResult = result;
|
||||
});
|
||||
return applyGate.promise;
|
||||
},
|
||||
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
||||
});
|
||||
await harness.initialize();
|
||||
|
||||
const applying = harness.coordinator.accept(harness.event());
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
|
||||
expect(closeResult?.ok).toBe(false);
|
||||
expect(closeResult?.ok ? null : closeResult?.error.kind).toBe(
|
||||
"IDLE_TIMEOUT",
|
||||
);
|
||||
|
||||
applyGate.resolve(realtimeSuccess(undefined));
|
||||
await applying;
|
||||
// Once the raw task settled, a second close does converge.
|
||||
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses quiescence when recovery re-enters close during its invocation", async () => {
|
||||
const recoveryGate = deferred<RealtimeResult<RealtimeRecoveryCommit>>();
|
||||
let closeResult: RealtimeResult<void> | undefined;
|
||||
let harness: ReturnType<typeof createHarness> | undefined;
|
||||
let recoveries = 0;
|
||||
harness = createHarness({
|
||||
recover: () => {
|
||||
recoveries += 1;
|
||||
if (recoveries === 1) {
|
||||
return Promise.resolve(realtimeSuccess(snapshotCommit("0")));
|
||||
}
|
||||
void harness!.coordinator.close().then((result) => {
|
||||
closeResult = result;
|
||||
});
|
||||
return recoveryGate.promise;
|
||||
},
|
||||
taskLimits: { recoveryTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
||||
});
|
||||
await harness.initialize();
|
||||
|
||||
const recovering = harness.coordinator.recover(STREAM_ID, "SEQUENCE_GAP");
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
|
||||
expect(closeResult?.ok).toBe(false);
|
||||
expect(closeResult?.ok ? null : closeResult?.error.kind).toBe(
|
||||
"IDLE_TIMEOUT",
|
||||
);
|
||||
|
||||
recoveryGate.resolve(realtimeSuccess(snapshotCommit("0")));
|
||||
await recovering;
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* RT-02. A scheduler that cannot install a deadline leaves the wait unbounded.
|
||||
* Letting the exception escape turned a typed realtime result into a native
|
||||
* rejection and, through the caller's own catch, started a recovery that
|
||||
* overlapped the effect still running.
|
||||
*/
|
||||
describe("RT-02 a throwing scheduler stays inside the result contract", () => {
|
||||
/** Installs deadlines normally until the stream is initialized. */
|
||||
function breakableScheduler() {
|
||||
const state = { broken: false };
|
||||
return {
|
||||
state,
|
||||
scheduleTimeout: (callback: () => void, delayMs: number) => {
|
||||
if (state.broken) throw new TypeError("scheduleTimeout exploded");
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearScheduledTimeout: (handle: unknown) => {
|
||||
clearTimeout(handle as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("does not start a recovery that overlaps a pending apply", async () => {
|
||||
const applyGate = deferred<RealtimeResult<void>>();
|
||||
const scheduler = breakableScheduler();
|
||||
let recoveriesAfterApply = 0;
|
||||
const harness = createHarness({
|
||||
apply: () => applyGate.promise,
|
||||
recover: async () => {
|
||||
recoveriesAfterApply += 1;
|
||||
return realtimeSuccess(snapshotCommit("0"));
|
||||
},
|
||||
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
||||
scheduleTimeout: scheduler.scheduleTimeout,
|
||||
clearScheduledTimeout: scheduler.clearScheduledTimeout,
|
||||
});
|
||||
await harness.initialize();
|
||||
recoveriesAfterApply = 0;
|
||||
scheduler.state.broken = true;
|
||||
|
||||
const accepted = await harness.coordinator.accept(harness.event());
|
||||
|
||||
// The apply is still pending, so no recovery may have run beside it.
|
||||
expect(recoveriesAfterApply).toBe(0);
|
||||
expect(accepted.ok).toBe(false);
|
||||
expect(harness.coordinator.lifecycle(STREAM_ID)).toBe("DRAINING");
|
||||
|
||||
applyGate.resolve(realtimeSuccess(undefined));
|
||||
});
|
||||
|
||||
it("returns a typed close failure instead of rejecting natively", async () => {
|
||||
const applyGate = deferred<RealtimeResult<void>>();
|
||||
const scheduler = breakableScheduler();
|
||||
const harness = createHarness({
|
||||
apply: () => applyGate.promise,
|
||||
taskLimits: { effectTimeoutMs: 10_000, drainTimeoutMs: 20 },
|
||||
scheduleTimeout: scheduler.scheduleTimeout,
|
||||
clearScheduledTimeout: scheduler.clearScheduledTimeout,
|
||||
});
|
||||
await harness.initialize();
|
||||
scheduler.state.broken = true;
|
||||
|
||||
const applying = harness.coordinator.accept(harness.event());
|
||||
// Let the apply reach the authority before the fence arrives.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
const closed = await harness.coordinator.close();
|
||||
|
||||
// A drain that could not be bounded is not proof of quiescence.
|
||||
expect(closed.ok).toBe(false);
|
||||
expect(closed.ok ? null : closed.error.kind).toBe("IDLE_TIMEOUT");
|
||||
|
||||
applyGate.resolve(realtimeSuccess(undefined));
|
||||
await applying;
|
||||
});
|
||||
|
||||
it("keeps the classified outcome when clearing a timer throws", async () => {
|
||||
const harness = createHarness({
|
||||
clearScheduledTimeout: () => {
|
||||
throw new TypeError("clearScheduledTimeout exploded");
|
||||
},
|
||||
});
|
||||
await harness.initialize();
|
||||
|
||||
await expect(
|
||||
harness.coordinator.accept(harness.event()),
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
await expect(harness.coordinator.close()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user