문제 브라우저는 online인데 서버·프록시만 닿지 않을 때 실패 이름이 transport마다 다르다. SSE는 fetch·reader 거절을 OFFLINE, poll은 execute 거절을 OFFLINE, WebSocket은 error를 PROVIDER_UNAVAILABLE로 낸다. reconnect coordinator는 transport OFFLINE을 backoff로 재시도하고 예산을 다 쓰면 online인데도 OFFLINE으로 끝낸다. 설계 13.1의 OFFLINE 행은 "online hint 뒤 bounded check"라 5.3과 다른 말을 한다. kind로 화면 문구나 조합 판단을 고르면 같은 서버 장애가 SSE에서는 "오프라인", WS에서는 "서비스 이용 불가"가 되고, "OFFLINE이면 online 이벤트를 기다린다"는 조합 코드는 다시 멈춘다(0929/04-01의 모양). 고른 방법 - 리뷰 5.3절 판단대로 어휘를 맞춘다. SSE fetch 거절(CONNECT)과 reader 거절(RECEIVE), poll execute 거절을 PROVIDER_UNAVAILABLE(retryable)로 바꾼다. 이제 OFFLINE은 환경 신호에서만 나온다. - poll은 PROVIDER_UNAVAILABLE을 lease 안에서 재시도하므로, 던지는 execute는 바로 끝나지 않고 backoff로 maxAttempts까지 다시 시도한다. executor가 PROVIDER_UNAVAILABLE을 돌려준 경우와 같은 처리다. - 설계 13.1의 OFFLINE·PROVIDER_UNAVAILABLE 행과 5.3 문단을 고친다. - 버린 대안: 13.1에 OFFLINE의 두 뜻을 적기만 하기. 비용은 1행이지만 조합자가 navigator.onLine을 따로 봐야 원인을 안다. 운영 중인 제품이 없는 지금 어휘를 바꾸는 비용이 가장 작다(리뷰 5.3). 위험·한계 - 배럴로 공개된 SSE 연결·poll coordinator의 결과 kind가 바뀐다. repo 안 소비자는 테스트뿐이다. - 던지는 execute(조합 코드의 버그 포함)는 이제 lease 예산만큼 다시 불린다. 상한은 maxAttempts·maxElapsed 그대로다. - 환경 신호가 늦게 offline이 되는 경우 첫 실패는 PROVIDER_UNAVAILABLE로 보인다. offline 대기는 kind가 아니라 환경 신호로 걸리므로 멈춤은 없다 (before/after 테스트 그대로 통과). 검증 - 바꾼 기대값 3건을 옛 코드에 돌림: 3건 실패(RED). 새 코드: realtime 단위 241건 통과. - check:types 통과, 바꾼 파일 eslint 통과. BREAKING CHANGE: SSE 연결의 fetch·reader 거절과 bounded poll의 execute 거절이 OFFLINE 대신 PROVIDER_UNAVAILABLE(retryable)로 끝난다. poll은 던지는 execute를 lease 안에서 재시도한다. OFFLINE을 서버 장애로 읽던 조합 코드는 PROVIDER_UNAVAILABLE을 함께 본다. Refs: docs/reviews/2026-11-03-maturity-review/04-adapters-realtime.md 04-04
1164 lines
31 KiB
TypeScript
1164 lines
31 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { ClockPort } from "../../../src/application/ports/clock-port.ts";
|
|
import {
|
|
definePollLeasePolicy,
|
|
type BoundedPollOperation,
|
|
} from "../../../src/application/policies/bounded-polling.ts";
|
|
import { createContractHttpExecutor } from "../../../src/adapters/http/index.ts";
|
|
import {
|
|
createBoundedPollCoordinator,
|
|
type BoundedPollAttemptResult,
|
|
type BoundedPollEnvironment,
|
|
type BoundedPollObservation,
|
|
} from "../../../src/adapters/realtime/polling/bounded-poll-coordinator.ts";
|
|
import { TEST_LIST_HTTP_CONTRACT } from "../../helpers/external-contract-fixture.ts";
|
|
|
|
type Sleeper = {
|
|
dueAt: number;
|
|
resolve(): void;
|
|
reject(): void;
|
|
signal?: AbortSignal;
|
|
onAbort?: () => void;
|
|
};
|
|
|
|
class ManualClock implements ClockPort {
|
|
current = 0;
|
|
readonly sleepers: Sleeper[] = [];
|
|
|
|
now(): number {
|
|
return this.current;
|
|
}
|
|
|
|
sleep(milliseconds: number, signal?: AbortSignal): Promise<void> {
|
|
return new Promise((resolve, reject) => {
|
|
if (signal?.aborted) {
|
|
reject(new DOMException("Aborted", "AbortError"));
|
|
return;
|
|
}
|
|
const sleeper: Sleeper = {
|
|
dueAt: this.current + milliseconds,
|
|
resolve: () => {
|
|
signal?.removeEventListener("abort", sleeper.onAbort!);
|
|
resolve();
|
|
},
|
|
reject: () => {
|
|
signal?.removeEventListener("abort", sleeper.onAbort!);
|
|
reject(new DOMException("Aborted", "AbortError"));
|
|
},
|
|
signal,
|
|
};
|
|
sleeper.onAbort = () => {
|
|
this.remove(sleeper);
|
|
sleeper.reject();
|
|
};
|
|
signal?.addEventListener("abort", sleeper.onAbort, {
|
|
once: true,
|
|
});
|
|
this.sleepers.push(sleeper);
|
|
});
|
|
}
|
|
|
|
advance(milliseconds: number): void {
|
|
this.current += milliseconds;
|
|
const ready = this.sleepers
|
|
.filter((sleeper) => sleeper.dueAt <= this.current)
|
|
.sort((left, right) => left.dueAt - right.dueAt);
|
|
for (const sleeper of ready) {
|
|
this.remove(sleeper);
|
|
sleeper.resolve();
|
|
}
|
|
}
|
|
|
|
private remove(target: Sleeper): void {
|
|
const index = this.sleepers.indexOf(target);
|
|
if (index >= 0) this.sleepers.splice(index, 1);
|
|
}
|
|
}
|
|
|
|
class MutableEnvironment implements BoundedPollEnvironment {
|
|
visible = true;
|
|
connected = true;
|
|
readonly visibilityListeners = new Set<
|
|
(visibility: "HIDDEN" | "VISIBLE") => void
|
|
>();
|
|
readonly onlineListeners = new Set<(online: boolean) => void>();
|
|
|
|
visibility(): "HIDDEN" | "VISIBLE" {
|
|
return this.visible ? "VISIBLE" : "HIDDEN";
|
|
}
|
|
|
|
online(): boolean {
|
|
return this.connected;
|
|
}
|
|
|
|
subscribeVisibility(
|
|
listener: (visibility: "HIDDEN" | "VISIBLE") => void,
|
|
): () => void {
|
|
this.visibilityListeners.add(listener);
|
|
return () => this.visibilityListeners.delete(listener);
|
|
}
|
|
|
|
subscribeOnline(listener: (online: boolean) => void): () => void {
|
|
this.onlineListeners.add(listener);
|
|
return () => this.onlineListeners.delete(listener);
|
|
}
|
|
|
|
hide(): void {
|
|
this.visible = false;
|
|
for (const listener of this.visibilityListeners) listener("HIDDEN");
|
|
}
|
|
}
|
|
|
|
const policy = definePollLeasePolicy({
|
|
operationId: "GET_JOB_STATUS",
|
|
owner: "reference-job",
|
|
minimumIntervalMs: 5_000,
|
|
successIntervalMs: 5_000,
|
|
maxIntervalMs: 60_000,
|
|
maxAttempts: 3,
|
|
maxElapsedMs: 60_000,
|
|
maxResponseBytes: 1_024,
|
|
visibility: "VISIBLE_ONLY",
|
|
fallbackReason: "CONVERGENCE",
|
|
terminalStates: ["COMPLETED", "FAILED"],
|
|
});
|
|
const operation: BoundedPollOperation = Object.freeze({
|
|
contract: Object.freeze({
|
|
...TEST_LIST_HTTP_CONTRACT.contract,
|
|
operationId: "GET_JOB_STATUS",
|
|
}),
|
|
frontend: Object.freeze({
|
|
...TEST_LIST_HTTP_CONTRACT.frontend,
|
|
responseByteLimit: 1_024,
|
|
retryBudget: 0 as const,
|
|
}),
|
|
}) as BoundedPollOperation;
|
|
|
|
function withResponseByteLimit(
|
|
responseByteLimit: number,
|
|
): BoundedPollOperation {
|
|
return {
|
|
...operation,
|
|
frontend: { ...operation.frontend, responseByteLimit },
|
|
};
|
|
}
|
|
|
|
async function flush(): Promise<void> {
|
|
for (let turn = 0; turn < 12; turn += 1) {
|
|
await Promise.resolve();
|
|
}
|
|
}
|
|
|
|
describe("bounded poll coordinator", () => {
|
|
it("observes lease start, each attempt and the stop reason", async () => {
|
|
const clock = new ManualClock();
|
|
const results: BoundedPollAttemptResult<string>[] = [
|
|
{ ok: true, value: { kind: "UNCHANGED", responseBytes: 0 } },
|
|
{
|
|
ok: false,
|
|
error: {
|
|
kind: "RATE_LIMITED",
|
|
operation: "POLL",
|
|
retryable: true,
|
|
},
|
|
},
|
|
{
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "done",
|
|
responseBytes: 8,
|
|
state: "COMPLETED",
|
|
},
|
|
},
|
|
];
|
|
const observations: unknown[] = [];
|
|
const coordinator = createBoundedPollCoordinator({
|
|
policy,
|
|
operation,
|
|
execute: async () => results.shift()!,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
random: () => 0.5,
|
|
observe: (observation) => {
|
|
observations.push(observation);
|
|
throw new Error("sink failure must not change the lease");
|
|
},
|
|
});
|
|
|
|
let settled = false;
|
|
const running = coordinator.run();
|
|
void running.then(() => {
|
|
settled = true;
|
|
});
|
|
for (let step = 0; step < 50 && !settled; step += 1) {
|
|
clock.advance(1_000);
|
|
await flush();
|
|
}
|
|
|
|
await expect(running).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { kind: "TERMINAL", attempts: 3 },
|
|
});
|
|
expect(observations).toEqual([
|
|
{ kind: "LEASE_STARTED" },
|
|
{ kind: "ATTEMPT_FINISHED", outcome: "UNCHANGED" },
|
|
{ kind: "ATTEMPT_FINISHED", outcome: "RATE_LIMITED" },
|
|
{ kind: "ATTEMPT_FINISHED", outcome: "VALUE" },
|
|
{ kind: "LEASE_STOPPED", outcome: "TERMINAL" },
|
|
]);
|
|
});
|
|
|
|
it("chains completed attempts without overlap and stops at a terminal state", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableEnvironment();
|
|
let active = 0;
|
|
let highWatermark = 0;
|
|
const execute = vi
|
|
.fn<
|
|
(
|
|
input: Readonly<{
|
|
operationId: string;
|
|
attempt: number;
|
|
maxResponseBytes: number;
|
|
signal: AbortSignal;
|
|
}>,
|
|
) => Promise<BoundedPollAttemptResult<string>>
|
|
>()
|
|
.mockImplementation(async ({ attempt }) => {
|
|
active += 1;
|
|
highWatermark = Math.max(highWatermark, active);
|
|
await Promise.resolve();
|
|
active -= 1;
|
|
return {
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: attempt === 1 ? "working" : "done",
|
|
responseBytes: 32,
|
|
state: attempt === 1 ? "RUNNING" : "COMPLETED",
|
|
},
|
|
};
|
|
});
|
|
const onValue = vi.fn();
|
|
const coordinator = createBoundedPollCoordinator({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
});
|
|
|
|
const result = coordinator.run({ onValue });
|
|
expect(execute).not.toHaveBeenCalled();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
expect(clock.sleepers).toHaveLength(1);
|
|
clock.advance(5_000);
|
|
await flush();
|
|
|
|
await expect(result).resolves.toEqual({
|
|
ok: true,
|
|
value: {
|
|
kind: "TERMINAL",
|
|
attempts: 2,
|
|
state: "COMPLETED",
|
|
value: "done",
|
|
},
|
|
});
|
|
expect(onValue.mock.calls.map(([value]) => value)).toEqual([
|
|
"working",
|
|
"done",
|
|
]);
|
|
expect(highWatermark).toBe(1);
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
expect(environment.visibilityListeners.size).toBe(0);
|
|
expect(environment.onlineListeners.size).toBe(0);
|
|
});
|
|
|
|
it("passes the strictest response ceiling to the executor before decode", async () => {
|
|
const clock = new ManualClock();
|
|
const execute = vi.fn(
|
|
async ({
|
|
maxResponseBytes,
|
|
}: Readonly<{ maxResponseBytes: number }>) => ({
|
|
ok: true as const,
|
|
value: {
|
|
kind: "VALUE" as const,
|
|
value: "done",
|
|
responseBytes: maxResponseBytes,
|
|
state: "COMPLETED",
|
|
},
|
|
}),
|
|
);
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation: withResponseByteLimit(4_096),
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
|
|
await expect(pending).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { state: "COMPLETED" },
|
|
});
|
|
expect(execute).toHaveBeenCalledWith(
|
|
expect.objectContaining({ maxResponseBytes: 1_024 }),
|
|
);
|
|
});
|
|
|
|
it("does not start another attempt while a request is unresolved", async () => {
|
|
const clock = new ManualClock();
|
|
const environment = new MutableEnvironment();
|
|
let resolveFirst:
|
|
| ((result: BoundedPollAttemptResult<string>) => void)
|
|
| undefined;
|
|
const execute = vi
|
|
.fn()
|
|
.mockImplementationOnce(
|
|
() =>
|
|
new Promise<BoundedPollAttemptResult<string>>((resolve) => {
|
|
resolveFirst = resolve;
|
|
}),
|
|
)
|
|
.mockResolvedValue({
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "done",
|
|
responseBytes: 10,
|
|
state: "COMPLETED",
|
|
},
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment,
|
|
clock,
|
|
random: () => 0.5,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
clock.advance(30_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
|
|
resolveFirst?.({
|
|
ok: true,
|
|
value: { kind: "UNCHANGED", responseBytes: 0 },
|
|
});
|
|
await flush();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(2);
|
|
await expect(pending).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { attempts: 2, state: "COMPLETED" },
|
|
});
|
|
});
|
|
|
|
it("ends the lease when an in-flight executor ignores abort and exceeds max elapsed", async () => {
|
|
const clock = new ManualClock();
|
|
let attemptSignal: AbortSignal | undefined;
|
|
let settleAttempt:
|
|
| ((result: BoundedPollAttemptResult<string>) => void)
|
|
| undefined;
|
|
const execute = vi.fn(
|
|
({ signal }: Readonly<{ signal: AbortSignal }>) => {
|
|
attemptSignal = signal;
|
|
return new Promise<BoundedPollAttemptResult<string>>(
|
|
(resolve) => {
|
|
settleAttempt = resolve;
|
|
},
|
|
);
|
|
},
|
|
);
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
expect(attemptSignal?.aborted).toBe(false);
|
|
|
|
clock.advance(54_999);
|
|
await flush();
|
|
expect(coordinator.getState()).toBe("RUNNING");
|
|
clock.advance(1);
|
|
await flush();
|
|
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "POLL_BUDGET_EXHAUSTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(attemptSignal?.aborted).toBe(true);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
await expect(coordinator.run()).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
settleAttempt?.({
|
|
ok: false,
|
|
error: {
|
|
kind: "ABORTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
await flush();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
|
|
it("returns promptly when a caller aborts an executor that ignores its signal", async () => {
|
|
const clock = new ManualClock();
|
|
const caller = new AbortController();
|
|
let settleAttempt:
|
|
| ((result: BoundedPollAttemptResult<string>) => void)
|
|
| undefined;
|
|
const execute = vi.fn(
|
|
() =>
|
|
new Promise<BoundedPollAttemptResult<string>>(
|
|
(resolve) => {
|
|
settleAttempt = resolve;
|
|
},
|
|
),
|
|
);
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
});
|
|
|
|
const pending = coordinator.run({ signal: caller.signal });
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
caller.abort();
|
|
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "ABORTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
settleAttempt?.({
|
|
ok: false,
|
|
error: {
|
|
kind: "ABORTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
await flush();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
|
|
it("fails closed and aborts the attempt when the lease deadline clock is unavailable", async () => {
|
|
let current = 0;
|
|
let sleeps = 0;
|
|
const clock: ClockPort = {
|
|
now: () => current,
|
|
sleep: async (milliseconds) => {
|
|
sleeps += 1;
|
|
if (sleeps > 1) throw new Error("deadline unavailable");
|
|
current += milliseconds;
|
|
},
|
|
};
|
|
let attemptSignal: AbortSignal | undefined;
|
|
const execute = vi.fn(
|
|
async ({ signal }: Readonly<{ signal: AbortSignal }>) => {
|
|
attemptSignal = signal;
|
|
return {
|
|
ok: true as const,
|
|
value: {
|
|
kind: "UNCHANGED" as const,
|
|
responseBytes: 0 as const,
|
|
},
|
|
};
|
|
},
|
|
);
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
});
|
|
|
|
await expect(coordinator.run()).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(execute).toHaveBeenCalledOnce();
|
|
expect(attemptSignal?.aborted).toBe(true);
|
|
});
|
|
|
|
it("fences a non-cooperative apply callback at the lease deadline", async () => {
|
|
const clock = new ManualClock();
|
|
let applyContext:
|
|
| Readonly<{
|
|
signal: AbortSignal;
|
|
isCurrent(): boolean;
|
|
}>
|
|
| undefined;
|
|
let releaseApply: (() => void) | undefined;
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute: async () => ({
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "working",
|
|
responseBytes: 10,
|
|
state: "RUNNING",
|
|
},
|
|
}),
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
});
|
|
|
|
const pending = coordinator.run({
|
|
onValue: (_value, context) => {
|
|
applyContext = context;
|
|
return new Promise<void>((resolve) => {
|
|
releaseApply = resolve;
|
|
});
|
|
},
|
|
});
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(applyContext?.isCurrent()).toBe(true);
|
|
|
|
clock.advance(55_000);
|
|
await flush();
|
|
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "POLL_BUDGET_EXHAUSTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(applyContext?.signal.aborted).toBe(true);
|
|
expect(applyContext?.isCurrent()).toBe(false);
|
|
expect(coordinator.getState()).toBe("DRAINING");
|
|
releaseApply?.();
|
|
await flush();
|
|
expect(coordinator.getState()).toBe("IDLE");
|
|
});
|
|
|
|
it("honors Retry-After as a floor and exhausts finite attempts", async () => {
|
|
const clock = new ManualClock();
|
|
const execute = vi
|
|
.fn()
|
|
.mockResolvedValueOnce({
|
|
ok: false,
|
|
error: {
|
|
kind: "RATE_LIMITED",
|
|
operation: "POLL",
|
|
retryable: true,
|
|
retryAfterMs: 10_000,
|
|
},
|
|
})
|
|
.mockResolvedValue({
|
|
ok: true,
|
|
value: { kind: "UNCHANGED", responseBytes: 0 },
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy: definePollLeasePolicy({
|
|
...policy,
|
|
maxAttempts: 2,
|
|
}),
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
random: () => 0,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
clock.advance(9_999);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
clock.advance(1);
|
|
await flush();
|
|
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "POLL_BUDGET_EXHAUSTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(execute).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it.each(["RATE_LIMITED", "PROVIDER_UNAVAILABLE"] as const)(
|
|
"retries retryable %s without a server hint on policy backoff",
|
|
async (kind) => {
|
|
const clock = new ManualClock();
|
|
const execute = vi
|
|
.fn()
|
|
.mockResolvedValueOnce({
|
|
ok: false,
|
|
error: {
|
|
kind,
|
|
operation: "POLL",
|
|
retryable: true,
|
|
},
|
|
})
|
|
.mockResolvedValue({
|
|
ok: false,
|
|
error: {
|
|
kind,
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
random: () => 0,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
clock.advance(5_000);
|
|
await flush();
|
|
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind,
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(execute).toHaveBeenCalledTimes(2);
|
|
},
|
|
);
|
|
|
|
it("ignores Retry-After for retryable failure kinds that cannot carry the hint", async () => {
|
|
const clock = new ManualClock();
|
|
const execute = vi
|
|
.fn()
|
|
.mockResolvedValueOnce({
|
|
ok: false,
|
|
error: {
|
|
kind: "CONNECT_TIMEOUT",
|
|
operation: "POLL",
|
|
retryable: true,
|
|
retryAfterMs: 10_000,
|
|
},
|
|
})
|
|
.mockResolvedValue({
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "done",
|
|
responseBytes: 10,
|
|
state: "COMPLETED",
|
|
},
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
random: () => 0,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
clock.advance(4_999);
|
|
await flush();
|
|
expect(execute).toHaveBeenCalledTimes(1);
|
|
clock.advance(1);
|
|
|
|
await expect(pending).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { attempts: 2, state: "COMPLETED" },
|
|
});
|
|
expect(execute).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it.each(["AUTH_REQUIRED", "FORBIDDEN"] as const)(
|
|
"never retries terminal %s failures even when the executor marks them retryable",
|
|
async (kind) => {
|
|
const clock = new ManualClock();
|
|
const execute = vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
error: {
|
|
kind,
|
|
operation: "POLL",
|
|
retryable: true,
|
|
retryAfterMs: 10_000,
|
|
},
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
execute,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
clock.advance(5_000);
|
|
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind,
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(execute).toHaveBeenCalledOnce();
|
|
},
|
|
);
|
|
|
|
it("aborts in-flight work on hidden lifecycle and rejects late scope results", async () => {
|
|
const hiddenClock = new ManualClock();
|
|
const hiddenEnvironment = new MutableEnvironment();
|
|
const hiddenCoordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
environment: hiddenEnvironment,
|
|
clock: hiddenClock,
|
|
execute: ({ signal }) =>
|
|
new Promise((resolve) => {
|
|
signal.addEventListener(
|
|
"abort",
|
|
() =>
|
|
resolve({
|
|
ok: false,
|
|
error: {
|
|
kind: "ABORTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
}),
|
|
{ once: true },
|
|
);
|
|
}),
|
|
});
|
|
const hidden = hiddenCoordinator.run();
|
|
hiddenClock.advance(5_000);
|
|
await flush();
|
|
hiddenEnvironment.hide();
|
|
await expect(hidden).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "ABORTED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
|
|
const scopeClock = new ManualClock();
|
|
let current = true;
|
|
let resolveAttempt:
|
|
| ((result: BoundedPollAttemptResult<string>) => void)
|
|
| undefined;
|
|
const onValue = vi.fn();
|
|
const scopeCoordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
environment: new MutableEnvironment(),
|
|
clock: scopeClock,
|
|
isCurrent: () => current,
|
|
execute: () =>
|
|
new Promise((resolve) => {
|
|
resolveAttempt = resolve;
|
|
}),
|
|
});
|
|
const fenced = scopeCoordinator.run({ onValue });
|
|
scopeClock.advance(5_000);
|
|
await flush();
|
|
current = false;
|
|
resolveAttempt?.({
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "late",
|
|
responseBytes: 10,
|
|
state: "COMPLETED",
|
|
},
|
|
});
|
|
await expect(fenced).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "SCOPE_FENCED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(onValue).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("rejects accessor and extra-key attempt results without re-reading them", async () => {
|
|
const accessorClock = new ManualClock();
|
|
const readOk = vi.fn(() => true);
|
|
const accessorResult = Object.defineProperties(
|
|
{},
|
|
{
|
|
ok: {
|
|
enumerable: true,
|
|
get: readOk,
|
|
},
|
|
value: {
|
|
enumerable: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "unsafe",
|
|
responseBytes: 1,
|
|
state: "COMPLETED",
|
|
},
|
|
},
|
|
},
|
|
) as BoundedPollAttemptResult<string>;
|
|
const accessorCoordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
environment: new MutableEnvironment(),
|
|
clock: accessorClock,
|
|
execute: async () => accessorResult,
|
|
});
|
|
const accessorRun = accessorCoordinator.run();
|
|
accessorClock.advance(5_000);
|
|
await expect(accessorRun).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
expect(readOk).not.toHaveBeenCalled();
|
|
|
|
const extraClock = new ManualClock();
|
|
const extraCoordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
environment: new MutableEnvironment(),
|
|
clock: extraClock,
|
|
execute: async () =>
|
|
({
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "unsafe",
|
|
responseBytes: 1,
|
|
state: "COMPLETED",
|
|
},
|
|
extra: true,
|
|
}) as BoundedPollAttemptResult<string>,
|
|
});
|
|
const extraRun = extraCoordinator.run();
|
|
extraClock.advance(5_000);
|
|
await expect(extraRun).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
});
|
|
|
|
// 1103/04-04. A rejected executor means the request did not reach a server
|
|
// that answered. It is PROVIDER_UNAVAILABLE, retried within the lease like
|
|
// one the executor reports; OFFLINE comes only from the environment.
|
|
it("retries a throwing executor as PROVIDER_UNAVAILABLE within the lease", async () => {
|
|
const clock = new ManualClock();
|
|
const observations: BoundedPollObservation[] = [];
|
|
const execute = vi.fn(async (): Promise<BoundedPollAttemptResult<string>> => {
|
|
throw new TypeError("Failed to fetch");
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
random: () => 0.5,
|
|
execute,
|
|
observe: (observation) => observations.push(observation),
|
|
});
|
|
|
|
let result: unknown = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
for (let turn = 0; turn < 120 && result === null; turn += 1) {
|
|
clock.advance(1_000);
|
|
await flush();
|
|
}
|
|
|
|
expect(execute).toHaveBeenCalledTimes(policy.maxAttempts);
|
|
expect(
|
|
observations
|
|
.filter((observation) => observation.kind === "ATTEMPT_FINISHED")
|
|
.map((observation) => observation.outcome),
|
|
).toEqual([
|
|
"PROVIDER_UNAVAILABLE",
|
|
"PROVIDER_UNAVAILABLE",
|
|
"PROVIDER_UNAVAILABLE",
|
|
]);
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "POLL_BUDGET_EXHAUSTED", operation: "POLL" },
|
|
});
|
|
});
|
|
|
|
it("reads attempt results with the shared realtime snapshot rules", async () => {
|
|
const runWith = async (result: unknown) => {
|
|
const clock = new ManualClock();
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
execute: async () => result as BoundedPollAttemptResult<string>,
|
|
});
|
|
const running = coordinator.run();
|
|
clock.advance(5_000);
|
|
return running;
|
|
};
|
|
const terminalValue = () => ({
|
|
kind: "VALUE",
|
|
value: "done",
|
|
responseBytes: 1,
|
|
state: "COMPLETED",
|
|
});
|
|
|
|
// A hidden (non-enumerable) field is not plain data: rejected.
|
|
await expect(
|
|
runWith(
|
|
Object.defineProperties(
|
|
{},
|
|
{
|
|
ok: { enumerable: false, value: true },
|
|
value: { enumerable: true, value: terminalValue() },
|
|
},
|
|
),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { kind: "PROTOCOL_MISMATCH" },
|
|
});
|
|
// A null-prototype record is plain data: accepted.
|
|
await expect(
|
|
runWith(
|
|
Object.assign(Object.create(null) as object, {
|
|
ok: true,
|
|
value: Object.assign(
|
|
Object.create(null) as object,
|
|
terminalValue(),
|
|
),
|
|
}),
|
|
),
|
|
).resolves.toMatchObject({
|
|
ok: true,
|
|
value: { kind: "TERMINAL", value: "done" },
|
|
});
|
|
});
|
|
|
|
it("fails closed on response ceilings, concurrent runs and close", async () => {
|
|
const clock = new ManualClock();
|
|
const coordinator = createBoundedPollCoordinator<string>({
|
|
policy,
|
|
operation: withResponseByteLimit(512),
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
execute: async () => ({
|
|
ok: true,
|
|
value: {
|
|
kind: "VALUE",
|
|
value: "oversized",
|
|
responseBytes: 513,
|
|
state: "RUNNING",
|
|
},
|
|
}),
|
|
});
|
|
const first = coordinator.run();
|
|
await expect(coordinator.run()).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROTOCOL_MISMATCH",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
clock.advance(5_000);
|
|
await expect(first).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "PROTOCOL_MISMATCH",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
|
|
const pending = coordinator.run();
|
|
coordinator.close();
|
|
coordinator.close();
|
|
await expect(pending).resolves.toEqual({
|
|
ok: false,
|
|
error: {
|
|
kind: "CLOSED",
|
|
operation: "POLL",
|
|
retryable: false,
|
|
},
|
|
});
|
|
expect(coordinator.getState()).toBe("CLOSED");
|
|
});
|
|
});
|
|
|
|
// 1103/04-01. The natural composition: `execute` runs the poll's descriptor on
|
|
// the app's V3 executor. The executor's own retries would turn one poll attempt
|
|
// into up to three requests, hidden from the poll's budget and observations.
|
|
describe("a poll attempt on the V3 executor", () => {
|
|
const pollPolicy = definePollLeasePolicy({
|
|
operationId: TEST_LIST_HTTP_CONTRACT.contract.operationId,
|
|
owner: "reference-job",
|
|
minimumIntervalMs: 5_000,
|
|
successIntervalMs: 5_000,
|
|
maxIntervalMs: 60_000,
|
|
maxAttempts: 3,
|
|
maxElapsedMs: 600_000,
|
|
maxResponseBytes: 32_768,
|
|
visibility: "VISIBLE_ONLY",
|
|
fallbackReason: "STREAM_UNAVAILABLE",
|
|
terminalStates: [],
|
|
});
|
|
const scope = Object.freeze({
|
|
generation: 1,
|
|
fingerprint: "scope-1",
|
|
identities: Object.freeze({}) as never,
|
|
signal: new AbortController().signal,
|
|
isCurrent: () => true,
|
|
});
|
|
|
|
function pollOn(descriptor: BoundedPollOperation, clock: ManualClock) {
|
|
const requests: string[] = [];
|
|
const executor = createContractHttpExecutor({
|
|
baseUrl: "https://api.example/",
|
|
// The app default (`MAX_RETRY_ATTEMPTS`).
|
|
maxRetryAttempts: 2,
|
|
attachCredentials: () => ({ kind: "READY", headers: {} }),
|
|
fetcher: async (request) => {
|
|
requests.push(String(request));
|
|
return new Response(
|
|
JSON.stringify({ type: "about:blank", title: "down", status: 503 }),
|
|
{
|
|
status: 503,
|
|
headers: { "content-type": "application/problem+json" },
|
|
},
|
|
);
|
|
},
|
|
sleep: async () => undefined,
|
|
random: () => 0.5,
|
|
createRequestId: () => "req-1",
|
|
});
|
|
const coordinator = createBoundedPollCoordinator<unknown>({
|
|
policy: pollPolicy,
|
|
operation: descriptor,
|
|
environment: new MutableEnvironment(),
|
|
clock,
|
|
random: () => 0.5,
|
|
execute: async ({ signal }) => {
|
|
const outcome = await executor.execute(
|
|
descriptor,
|
|
{ limit: 20 },
|
|
{ routeId: "TEST_ROUTE", scope, signal },
|
|
);
|
|
return outcome.kind === "SUCCESS"
|
|
? {
|
|
ok: true,
|
|
value: { kind: "VALUE", value: outcome.value, responseBytes: 1 },
|
|
}
|
|
: {
|
|
ok: false,
|
|
error: {
|
|
kind: "PROVIDER_UNAVAILABLE",
|
|
operation: "POLL",
|
|
retryable: true,
|
|
},
|
|
};
|
|
},
|
|
});
|
|
return { coordinator, requests };
|
|
}
|
|
|
|
it("refuses the default read descriptor, whose executor retries", () => {
|
|
expect(TEST_LIST_HTTP_CONTRACT.frontend.retryBudget).toBe(2);
|
|
expect(() =>
|
|
pollOn(TEST_LIST_HTTP_CONTRACT as BoundedPollOperation, new ManualClock()),
|
|
).toThrow(TypeError);
|
|
});
|
|
|
|
it("sends one request per poll attempt with a poll descriptor", async () => {
|
|
const clock = new ManualClock();
|
|
const pollOnly = Object.freeze({
|
|
...TEST_LIST_HTTP_CONTRACT,
|
|
frontend: Object.freeze({
|
|
...TEST_LIST_HTTP_CONTRACT.frontend,
|
|
retryBudget: 0 as const,
|
|
}),
|
|
}) as BoundedPollOperation;
|
|
const { coordinator, requests } = pollOn(pollOnly, clock);
|
|
|
|
let result: unknown = null;
|
|
void coordinator.run().then((value) => {
|
|
result = value;
|
|
});
|
|
for (let turn = 0; turn < 400 && result === null; turn += 1) {
|
|
clock.advance(1_000);
|
|
await flush();
|
|
}
|
|
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { kind: "POLL_BUDGET_EXHAUSTED" },
|
|
});
|
|
expect(requests).toHaveLength(pollPolicy.maxAttempts);
|
|
});
|
|
});
|