문제: bounded poll은 `execute`의 throw를 PROVIDER_UNAVAILABLE/retryable로
바꿔 lease 안에서 다시 부른다(af00067). 설계 9.2가 권하는 조합은
`execute` 안에서 V3 실행기를 부르는 것이고, 실행기는 fetch 거절을 던지지
않고 NETWORK_FAILURE 결과로 돌려준다. 그래서 그 조합에서 throw는 결과
매핑·디코딩 같은 조합 코드 결함이다. 지금은 그 결함이 maxAttempts번의
요청과 ATTEMPT_FINISHED: PROVIDER_UNAVAILABLE, 끝으로
POLL_BUDGET_EXHAUSTED가 되어 서버 장애로 보인다. 원인 예외는 남지 않는다.
고른 방법: 리뷰 04-02의 선택지 (b). throw를 모양이 틀린 결과와 같은
계약 위반으로 본다. coordinator는 이미 `parseAttemptResult`가 거절한
결과를 재시도 없이 PROTOCOL_MISMATCH로 끝낸다. throw도 그 길로 보낸다.
lease 자신의 abort로 생긴 throw는 지금처럼 abort의 이유(ABORTED,
OFFLINE, 예산 소진 등)로 끝난다. `execute` 타입 주석, 설계 9.2와 13.1의
PROVIDER_UNAVAILABLE·PROTOCOL_MISMATCH 행을 같이 고쳤다.
버린 대안:
- (c) 주석과 13.1에만 "throw는 서버 장애로 재시도된다"를 적기. 완료
기준은 "결과를 보고 조합 결함과 서버 장애를 구분한다"인데, (c)는 읽는
사람만 알고 결과는 그대로라 구분할 수 없다.
- 관측에 throw 표시 칸을 더하기. 닫힌 관측 어휘를 넓히고, 결과 kind는
여전히 서버 장애다.
위험·한계:
- 배럴로 공개된 bounded poll의 결과가 바뀐다. repo 안 소비자는 테스트뿐이다.
- `fetch`를 직접 부르는 조합은 네트워크 실패를 결과로 바꾸지 않으면 첫
거절에서 lease가 끝난다. 설계가 권하지 않는 조합이고, 타입 주석과
9.2가 "실패도 결과로 돌려준다"를 적는다.
- reconnect coordinator는 던지는 `attempt`를 계속 PROVIDER_UNAVAILABLE로
재시도한다. 그쪽 attempt는 브라우저 연결 API를 감싸는 자리라 이번
범위 밖이다.
- CHANGELOG.md 미출시 [계약] 행의 "poll은 던지는 execute도 lease 안에서
다시 시도한다"는 이 커밋과 맞지 않는다. CHANGELOG는 오케스트레이터가
고친다.
검증: desktop batch에서 확인.
- node scripts/run-vitest.ts run tests/unit/realtime/
- RED: "ends the lease with PROTOCOL_MISMATCH for an async execute that
rejects"·"... throws before returning"가 옛 코드에서 execute 3회,
POLL_BUDGET_EXHAUSTED로 실패한다. "keeps ABORTED ..."는 양쪽에서 통과한다
(보존 조건).
BREAKING CHANGE: createBoundedPollCoordinator의 `execute`가 던지면(abort로
생긴 것 제외) 재시도 없이 PROTOCOL_MISMATCH(retryable: false)로 lease가
끝난다. 전에는 PROVIDER_UNAVAILABLE로 maxAttempts까지 재시도했다.
네트워크 실패를 throw로 내던 `execute`는 그것을 retryable
PROVIDER_UNAVAILABLE 결과로 바꿔 돌려준다.
Refs: 1105/04-02
855 lines
24 KiB
TypeScript
855 lines
24 KiB
TypeScript
import {
|
|
assertBoundedPollOperation,
|
|
definePollLeasePolicy,
|
|
type BoundedPollOperation,
|
|
type PollLeasePolicy,
|
|
} from "../../../application/policies/bounded-polling.ts";
|
|
import type { ClockPort } from "../../../application/ports/clock-port.ts";
|
|
import {
|
|
REALTIME_FAILURE_KINDS,
|
|
type RealtimeFailure,
|
|
type RealtimeFailureKind,
|
|
type RealtimeResult,
|
|
} from "../../../application/ports/realtime/shared.ts";
|
|
import { systemClock } from "../../platform/system-clock.ts";
|
|
import {
|
|
captureRealtimeDataSnapshot,
|
|
hasExactSnapshotKeys,
|
|
observeRealtime,
|
|
raceRealtimeTask,
|
|
realtimeFailure,
|
|
realtimeSuccess,
|
|
type RealtimeRace,
|
|
} from "../result.ts";
|
|
|
|
/**
|
|
* Provider-private Retry-After metadata is consumed by this adapter and is
|
|
* deliberately stripped before a failure crosses the realtime boundary.
|
|
*/
|
|
export type BoundedPollAttemptFailure = RealtimeFailure & Readonly<{
|
|
retryAfterMs?: number;
|
|
}>;
|
|
|
|
type BoundedPollAttemptSuccess<Value> =
|
|
| Readonly<{
|
|
kind: "UNCHANGED";
|
|
responseBytes: 0;
|
|
}>
|
|
| Readonly<{
|
|
kind: "VALUE";
|
|
value: Value;
|
|
responseBytes: number;
|
|
state?: string;
|
|
}>;
|
|
|
|
export type BoundedPollAttemptResult<Value> =
|
|
| Extract<
|
|
RealtimeResult<BoundedPollAttemptSuccess<Value>>,
|
|
{ ok: true }
|
|
>
|
|
| Readonly<{ ok: false; error: BoundedPollAttemptFailure }>;
|
|
|
|
export type BoundedPollResult<Value> =
|
|
RealtimeResult<
|
|
Readonly<{
|
|
kind: "TERMINAL";
|
|
attempts: number;
|
|
state: string;
|
|
value: Value;
|
|
}>
|
|
>;
|
|
|
|
export type BoundedPollEnvironment = Readonly<{
|
|
visibility(): "HIDDEN" | "VISIBLE";
|
|
online(): boolean;
|
|
subscribeVisibility?(
|
|
listener: (visibility: "HIDDEN" | "VISIBLE") => void,
|
|
): () => void;
|
|
subscribeOnline?(listener: (online: boolean) => void): () => void;
|
|
}>;
|
|
|
|
export type BoundedPollRunInput<Value> = Readonly<{
|
|
signal?: AbortSignal;
|
|
onValue?: (
|
|
value: Value,
|
|
context: Readonly<{ signal: AbortSignal; isCurrent(): boolean }>,
|
|
) => void | Promise<void>;
|
|
}>;
|
|
|
|
export type BoundedPollCoordinator<Value> = Readonly<{
|
|
run(input?: BoundedPollRunInput<Value>): Promise<
|
|
BoundedPollResult<Value>
|
|
>;
|
|
getState(): "CLOSED" | "DRAINING" | "IDLE" | "RUNNING";
|
|
close(): void;
|
|
}>;
|
|
|
|
/**
|
|
* Closed diagnostics for design 14.1 `bounded_poll_*`. `outcome` is VALUE,
|
|
* UNCHANGED or the failure kind for ATTEMPT_FINISHED, and TERMINAL or the
|
|
* failure kind for LEASE_STOPPED.
|
|
*/
|
|
export type BoundedPollObservation = Readonly<{
|
|
kind: "LEASE_STARTED" | "ATTEMPT_FINISHED" | "LEASE_STOPPED";
|
|
outcome?: "VALUE" | "UNCHANGED" | "TERMINAL" | RealtimeFailureKind;
|
|
}>;
|
|
|
|
export type BoundedPollCoordinatorDependencies<Value> = Readonly<{
|
|
policy: PollLeasePolicy;
|
|
/**
|
|
* The V3 descriptor `execute` runs (`BoundedPollOperation`). Creation
|
|
* refuses one whose `frontend.retryBudget` is not 0.
|
|
*/
|
|
operation: BoundedPollOperation;
|
|
/**
|
|
* Returns every outcome as a result, server and network failures included
|
|
* (the V3 executor returns a rejected fetch as NETWORK_FAILURE). A throw,
|
|
* like a result of the wrong shape, ends the lease with PROTOCOL_MISMATCH
|
|
* and is not retried: it is a composition defect, not a server outage.
|
|
*/
|
|
execute(input: Readonly<{
|
|
operationId: string;
|
|
attempt: number;
|
|
/**
|
|
* Hard response-body ceiling that must be enforced before decoding.
|
|
*/
|
|
maxResponseBytes: number;
|
|
signal: AbortSignal;
|
|
}>): Promise<BoundedPollAttemptResult<Value>>;
|
|
environment: BoundedPollEnvironment;
|
|
isCurrent?: () => boolean;
|
|
clock?: ClockPort;
|
|
random?: () => number;
|
|
observe?(observation: BoundedPollObservation): void;
|
|
}>;
|
|
|
|
const SAFE_STATE = /^[A-Z][A-Z0-9_]{0,63}$/u;
|
|
const POLL_RETRYABLE_FAILURE_KINDS = Object.freeze([
|
|
"CONNECT_TIMEOUT",
|
|
"RATE_LIMITED",
|
|
"PROVIDER_UNAVAILABLE",
|
|
] as const satisfies readonly RealtimeFailureKind[]);
|
|
const POLL_RETRY_AFTER_FAILURE_KINDS = Object.freeze([
|
|
"RATE_LIMITED",
|
|
"PROVIDER_UNAVAILABLE",
|
|
] as const satisfies readonly RealtimeFailureKind[]);
|
|
|
|
export function createBoundedPollCoordinator<Value>(
|
|
dependencies: BoundedPollCoordinatorDependencies<Value>,
|
|
): BoundedPollCoordinator<Value> {
|
|
const policy = definePollLeasePolicy(dependencies.policy);
|
|
assertBoundedPollOperation(policy, dependencies.operation);
|
|
const maxResponseBytes = Math.min(
|
|
policy.maxResponseBytes,
|
|
dependencies.operation.frontend.responseByteLimit,
|
|
);
|
|
const clock = dependencies.clock ?? systemClock;
|
|
const random = dependencies.random ?? Math.random;
|
|
const isCurrent = dependencies.isCurrent ?? (() => true);
|
|
let state: "CLOSED" | "DRAINING" | "IDLE" | "RUNNING" = "IDLE";
|
|
let activeController: AbortController | null = null;
|
|
let generation = 0;
|
|
let pendingWork = 0;
|
|
const observe = (observation: BoundedPollObservation) =>
|
|
observeRealtime(dependencies.observe, observation);
|
|
|
|
async function run(
|
|
input?: BoundedPollRunInput<Value>,
|
|
): Promise<BoundedPollResult<Value>> {
|
|
const startedGeneration = generation;
|
|
const result = await runLease(input);
|
|
// A call rejected before starting leaves the generation unchanged.
|
|
if (generation !== startedGeneration) {
|
|
observe({
|
|
kind: "LEASE_STOPPED",
|
|
outcome: result.ok ? "TERMINAL" : result.error.kind,
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
|
|
async function runLease(
|
|
input: BoundedPollRunInput<Value> = {},
|
|
): Promise<BoundedPollResult<Value>> {
|
|
if (state === "CLOSED") return pollFailure("CLOSED");
|
|
if (state !== "IDLE") {
|
|
return pollFailure("PROTOCOL_MISMATCH");
|
|
}
|
|
if (input.signal?.aborted) return pollFailure("ABORTED");
|
|
if (safeVisibility(dependencies.environment) !== "VISIBLE") {
|
|
return pollFailure("ABORTED");
|
|
}
|
|
if (safeOnline(dependencies.environment) !== true) {
|
|
return pollFailure("OFFLINE", true);
|
|
}
|
|
if (!safeIsCurrent(isCurrent)) {
|
|
return pollFailure("SCOPE_FENCED");
|
|
}
|
|
|
|
state = "RUNNING";
|
|
const runGeneration = ++generation;
|
|
observe({ kind: "LEASE_STARTED" });
|
|
const controller = new AbortController();
|
|
activeController = controller;
|
|
let stopKind: RealtimeFailureKind | null = null;
|
|
let attempts = 0;
|
|
let consecutiveFailures = 0;
|
|
let nextDelayMs = policy.minimumIntervalMs;
|
|
const startedAtMs = safeNow(clock);
|
|
|
|
const stop = (kind: RealtimeFailureKind) => {
|
|
if (stopKind !== null) return;
|
|
stopKind = kind;
|
|
controller.abort();
|
|
};
|
|
const onCallerAbort = () => stop("ABORTED");
|
|
input.signal?.addEventListener("abort", onCallerAbort, {
|
|
once: true,
|
|
});
|
|
let unsubscribeVisibility: (() => void) | undefined;
|
|
let unsubscribeOnline: (() => void) | undefined;
|
|
try {
|
|
unsubscribeVisibility =
|
|
dependencies.environment.subscribeVisibility?.((visibility) => {
|
|
if (visibility !== "VISIBLE") stop("ABORTED");
|
|
});
|
|
} catch {
|
|
stop("ABORTED");
|
|
}
|
|
try {
|
|
unsubscribeOnline =
|
|
dependencies.environment.subscribeOnline?.((online) => {
|
|
if (!online) stop("OFFLINE");
|
|
});
|
|
} catch {
|
|
stop("OFFLINE");
|
|
}
|
|
|
|
try {
|
|
if (startedAtMs === null) {
|
|
return pollFailure("PROVIDER_UNAVAILABLE");
|
|
}
|
|
while (true) {
|
|
const lifecycleFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
if (lifecycleFailure) {
|
|
return pollFailure(
|
|
lifecycleFailure,
|
|
lifecycleFailure === "OFFLINE",
|
|
);
|
|
}
|
|
if (attempts >= policy.maxAttempts) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
const nowBeforeSleep = safeNow(clock);
|
|
if (
|
|
nowBeforeSleep === null ||
|
|
nowBeforeSleep < startedAtMs
|
|
) {
|
|
return pollFailure("PROVIDER_UNAVAILABLE");
|
|
}
|
|
if (
|
|
nowBeforeSleep - startedAtMs + nextDelayMs >=
|
|
policy.maxElapsedMs
|
|
) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
const cadenceOutcome = await raceRealtimeTask(
|
|
() => clock.sleep(nextDelayMs, controller.signal),
|
|
controller.signal,
|
|
);
|
|
if (cadenceOutcome.kind !== "VALUE") {
|
|
const afterSleepFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
return pollFailure(
|
|
afterSleepFailure ??
|
|
(cadenceOutcome.kind === "REJECTED"
|
|
? "PROVIDER_UNAVAILABLE"
|
|
: "ABORTED"),
|
|
afterSleepFailure === "OFFLINE",
|
|
);
|
|
}
|
|
|
|
const beforeAttemptFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
if (beforeAttemptFailure) {
|
|
return pollFailure(
|
|
beforeAttemptFailure,
|
|
beforeAttemptFailure === "OFFLINE",
|
|
);
|
|
}
|
|
const attemptStartedAt = safeNow(clock);
|
|
if (
|
|
attemptStartedAt === null ||
|
|
attemptStartedAt < startedAtMs ||
|
|
attemptStartedAt - startedAtMs >= policy.maxElapsedMs
|
|
) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
|
|
attempts += 1;
|
|
let result: BoundedPollAttemptResult<Value> | null;
|
|
const attemptOutcome = await awaitWithinLease(
|
|
() =>
|
|
trackWork(
|
|
dependencies.execute({
|
|
operationId: policy.operationId,
|
|
attempt: attempts,
|
|
maxResponseBytes,
|
|
signal: controller.signal,
|
|
}),
|
|
runGeneration,
|
|
),
|
|
policy.maxElapsedMs - (attemptStartedAt - startedAtMs),
|
|
clock,
|
|
controller.signal,
|
|
() => stop("POLL_BUDGET_EXHAUSTED"),
|
|
);
|
|
if (attemptOutcome.kind === "VALUE") {
|
|
result = attemptOutcome.value;
|
|
} else if (attemptOutcome.kind === "REJECTED") {
|
|
// 1105/04-02. `execute` returns every failure as a result; the V3
|
|
// executor already returns a rejected fetch as NETWORK_FAILURE. A
|
|
// throw is a composition defect, read below like a malformed
|
|
// result, not retried as a server outage. One caused by this
|
|
// lease's abort ends with the abort's reason.
|
|
result = controller.signal.aborted
|
|
? realtimeFailure("ABORTED", "POLL", false)
|
|
: null;
|
|
} else {
|
|
if (attemptOutcome.kind === "CLOCK_FAILED") {
|
|
controller.abort();
|
|
}
|
|
const interruptedFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
return pollFailure(
|
|
interruptedFailure ??
|
|
(attemptOutcome.kind === "CLOCK_FAILED"
|
|
? "PROVIDER_UNAVAILABLE"
|
|
: "ABORTED"),
|
|
interruptedFailure === "OFFLINE",
|
|
);
|
|
}
|
|
|
|
const afterAttemptFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
if (afterAttemptFailure) {
|
|
return pollFailure(
|
|
afterAttemptFailure,
|
|
afterAttemptFailure === "OFFLINE",
|
|
);
|
|
}
|
|
const attemptFinishedAt = safeNow(clock);
|
|
if (
|
|
attemptFinishedAt === null ||
|
|
attemptFinishedAt < attemptStartedAt
|
|
) {
|
|
return pollFailure("PROVIDER_UNAVAILABLE");
|
|
}
|
|
if (
|
|
attemptFinishedAt - startedAtMs >= policy.maxElapsedMs
|
|
) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
|
|
const parsedResult =
|
|
result === null ? null : parseAttemptResult<Value>(result);
|
|
if (!parsedResult) {
|
|
return pollFailure("PROTOCOL_MISMATCH");
|
|
}
|
|
result = parsedResult;
|
|
observe({
|
|
kind: "ATTEMPT_FINISHED",
|
|
outcome: result.ok ? result.value.kind : result.error.kind,
|
|
});
|
|
if (!result.ok) {
|
|
// A missing server hint falls back to policy backoff; an executor
|
|
// that requires one reports `retryable: false` instead.
|
|
if (
|
|
!result.error.retryable ||
|
|
!isPollRetryableFailureKind(result.error.kind)
|
|
) {
|
|
// OFFLINE ends this lease but stays retryable, like every other
|
|
// OFFLINE here.
|
|
return pollFailure(
|
|
result.error.kind,
|
|
result.error.kind === "OFFLINE" && result.error.retryable,
|
|
);
|
|
}
|
|
consecutiveFailures += 1;
|
|
const failureDelay = retryDelay(
|
|
policy,
|
|
consecutiveFailures,
|
|
isPollRetryAfterFailureKind(result.error.kind)
|
|
? result.error.retryAfterMs
|
|
: undefined,
|
|
random,
|
|
);
|
|
if (failureDelay === null) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
nextDelayMs = failureDelay;
|
|
continue;
|
|
}
|
|
|
|
consecutiveFailures = 0;
|
|
if (
|
|
result.value.responseBytes > maxResponseBytes
|
|
) {
|
|
return pollFailure("PROTOCOL_MISMATCH");
|
|
}
|
|
if (result.value.kind === "UNCHANGED") {
|
|
const delay = successDelay(policy, random);
|
|
if (delay === null) {
|
|
return pollFailure("PROTOCOL_MISMATCH");
|
|
}
|
|
nextDelayMs = delay;
|
|
continue;
|
|
}
|
|
const valueResult = result.value;
|
|
|
|
if (input.onValue) {
|
|
const beforeApplyAt = safeNow(clock);
|
|
if (
|
|
beforeApplyAt === null ||
|
|
beforeApplyAt < attemptFinishedAt ||
|
|
beforeApplyAt - startedAtMs >= policy.maxElapsedMs
|
|
) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
const applyOutcome = await awaitWithinLease(
|
|
() =>
|
|
trackWork(
|
|
Promise.resolve(
|
|
input.onValue!(valueResult.value, {
|
|
signal: controller.signal,
|
|
isCurrent: () =>
|
|
state === "RUNNING" &&
|
|
generation === runGeneration &&
|
|
stopKind === null &&
|
|
!controller.signal.aborted &&
|
|
safeIsCurrent(isCurrent),
|
|
}),
|
|
),
|
|
runGeneration,
|
|
),
|
|
policy.maxElapsedMs - (beforeApplyAt - startedAtMs),
|
|
clock,
|
|
controller.signal,
|
|
() => stop("POLL_BUDGET_EXHAUSTED"),
|
|
);
|
|
if (applyOutcome.kind === "REJECTED") {
|
|
return applyFailure();
|
|
}
|
|
if (applyOutcome.kind !== "VALUE") {
|
|
if (applyOutcome.kind === "CLOCK_FAILED") {
|
|
controller.abort();
|
|
}
|
|
const interruptedFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
return pollFailure(
|
|
interruptedFailure ??
|
|
(applyOutcome.kind === "CLOCK_FAILED"
|
|
? "PROVIDER_UNAVAILABLE"
|
|
: "ABORTED"),
|
|
interruptedFailure === "OFFLINE",
|
|
);
|
|
}
|
|
}
|
|
const afterApplyFailure = currentFailure(
|
|
stopKind,
|
|
state,
|
|
runGeneration,
|
|
generation,
|
|
isCurrent,
|
|
dependencies.environment,
|
|
);
|
|
if (afterApplyFailure) {
|
|
return pollFailure(
|
|
afterApplyFailure,
|
|
afterApplyFailure === "OFFLINE",
|
|
);
|
|
}
|
|
const applyFinishedAt = safeNow(clock);
|
|
if (
|
|
applyFinishedAt === null ||
|
|
applyFinishedAt < attemptFinishedAt
|
|
) {
|
|
return pollFailure("PROVIDER_UNAVAILABLE");
|
|
}
|
|
if (applyFinishedAt - startedAtMs >= policy.maxElapsedMs) {
|
|
return pollFailure("POLL_BUDGET_EXHAUSTED");
|
|
}
|
|
if (
|
|
valueResult.state &&
|
|
policy.terminalStates.includes(valueResult.state)
|
|
) {
|
|
return realtimeSuccess(
|
|
Object.freeze({
|
|
kind: "TERMINAL" as const,
|
|
attempts,
|
|
state: valueResult.state,
|
|
value: valueResult.value,
|
|
}),
|
|
);
|
|
}
|
|
const delay = successDelay(policy, random);
|
|
if (delay === null) {
|
|
return pollFailure("PROTOCOL_MISMATCH");
|
|
}
|
|
nextDelayMs = delay;
|
|
}
|
|
} finally {
|
|
input.signal?.removeEventListener("abort", onCallerAbort);
|
|
safelyUnsubscribe(unsubscribeVisibility);
|
|
safelyUnsubscribe(unsubscribeOnline);
|
|
if (activeController === controller) {
|
|
activeController = null;
|
|
}
|
|
if (generation === runGeneration) {
|
|
state = pendingWork === 0 ? "IDLE" : "DRAINING";
|
|
}
|
|
}
|
|
}
|
|
|
|
function trackWork<WorkValue>(
|
|
work: Promise<WorkValue>,
|
|
workGeneration: number,
|
|
): Promise<WorkValue> {
|
|
pendingWork += 1;
|
|
void work.then(
|
|
() => releaseWork(workGeneration),
|
|
() => releaseWork(workGeneration),
|
|
);
|
|
return work;
|
|
}
|
|
|
|
function releaseWork(workGeneration: number): void {
|
|
pendingWork = Math.max(0, pendingWork - 1);
|
|
if (
|
|
pendingWork === 0 &&
|
|
state === "DRAINING" &&
|
|
generation === workGeneration
|
|
) {
|
|
state = "IDLE";
|
|
}
|
|
}
|
|
|
|
function close(): void {
|
|
if (state === "CLOSED") return;
|
|
state = "CLOSED";
|
|
generation += 1;
|
|
activeController?.abort();
|
|
}
|
|
|
|
return Object.freeze({
|
|
run,
|
|
getState: () => state,
|
|
close,
|
|
});
|
|
}
|
|
|
|
function currentFailure(
|
|
requested: RealtimeFailureKind | null,
|
|
state: "CLOSED" | "DRAINING" | "IDLE" | "RUNNING",
|
|
runGeneration: number,
|
|
currentGeneration: number,
|
|
isCurrent: () => boolean,
|
|
environment: BoundedPollEnvironment,
|
|
): RealtimeFailureKind | null {
|
|
if (requested) return requested;
|
|
if (
|
|
state === "CLOSED" ||
|
|
state === "DRAINING" ||
|
|
runGeneration !== currentGeneration
|
|
) {
|
|
return "CLOSED";
|
|
}
|
|
if (!safeIsCurrent(isCurrent)) return "SCOPE_FENCED";
|
|
if (safeVisibility(environment) !== "VISIBLE") return "ABORTED";
|
|
if (safeOnline(environment) !== true) return "OFFLINE";
|
|
return null;
|
|
}
|
|
|
|
const LEASE_EXPIRED = Object.freeze({ kind: "TIMEOUT" as const });
|
|
|
|
/** Bounds work by the lease's remaining time; expiry stops the whole run. */
|
|
async function awaitWithinLease<Value>(
|
|
task: () => Promise<Value>,
|
|
remainingMs: number,
|
|
clock: ClockPort,
|
|
signal: AbortSignal,
|
|
onLeaseExpired: () => void,
|
|
): Promise<RealtimeRace<Value>> {
|
|
const outcome =
|
|
Number.isFinite(remainingMs) && remainingMs > 0
|
|
? await raceRealtimeTask(task, signal, (lease) =>
|
|
clock.sleep(remainingMs, lease),
|
|
)
|
|
: LEASE_EXPIRED;
|
|
if (outcome.kind === "TIMEOUT") onLeaseExpired();
|
|
return outcome;
|
|
}
|
|
|
|
function parseAttemptResult<Value>(
|
|
result: unknown,
|
|
): BoundedPollAttemptResult<Value> | null {
|
|
const outer = snapshotDataRecord(result, [
|
|
["error", "ok"],
|
|
["ok", "value"],
|
|
]);
|
|
if (!outer) return null;
|
|
if (outer.ok === false) {
|
|
const error = snapshotDataRecord(outer.error, [
|
|
["kind", "operation", "retryable"],
|
|
["kind", "operation", "retryable", "retryAfterMs"],
|
|
]);
|
|
if (
|
|
!error ||
|
|
!REALTIME_FAILURE_KINDS.includes(
|
|
error.kind as RealtimeFailureKind,
|
|
) ||
|
|
error.operation !== "POLL" ||
|
|
typeof error.retryable !== "boolean" ||
|
|
(Object.hasOwn(error, "retryAfterMs") &&
|
|
(!Number.isSafeInteger(error.retryAfterMs) ||
|
|
(error.retryAfterMs as number) < 0))
|
|
) {
|
|
return null;
|
|
}
|
|
const canonicalError = Object.freeze({
|
|
kind: error.kind as RealtimeFailureKind,
|
|
operation: "POLL" as const,
|
|
retryable: error.retryable,
|
|
...(Object.hasOwn(error, "retryAfterMs")
|
|
? { retryAfterMs: error.retryAfterMs as number }
|
|
: {}),
|
|
});
|
|
return Object.freeze({
|
|
ok: false as const,
|
|
error: canonicalError,
|
|
});
|
|
}
|
|
if (outer.ok !== true) return null;
|
|
const value = snapshotDataRecord(outer.value, [
|
|
["kind", "responseBytes"],
|
|
["kind", "responseBytes", "state", "value"],
|
|
["kind", "responseBytes", "value"],
|
|
]);
|
|
if (
|
|
!value ||
|
|
!Number.isSafeInteger(value.responseBytes) ||
|
|
(value.responseBytes as number) < 0
|
|
) {
|
|
return null;
|
|
}
|
|
if (value.kind === "UNCHANGED") {
|
|
return value.responseBytes === 0
|
|
? Object.freeze({
|
|
ok: true as const,
|
|
value: Object.freeze({
|
|
kind: "UNCHANGED" as const,
|
|
responseBytes: 0 as const,
|
|
}),
|
|
})
|
|
: null;
|
|
}
|
|
if (
|
|
value.kind !== "VALUE" ||
|
|
(Object.hasOwn(value, "state") &&
|
|
(typeof value.state !== "string" ||
|
|
!SAFE_STATE.test(value.state)))
|
|
) {
|
|
return null;
|
|
}
|
|
return Object.freeze({
|
|
ok: true as const,
|
|
value: Object.freeze({
|
|
kind: "VALUE" as const,
|
|
value: value.value as Value,
|
|
responseBytes: value.responseBytes as number,
|
|
...(Object.hasOwn(value, "state")
|
|
? { state: value.state as string }
|
|
: {}),
|
|
}),
|
|
});
|
|
}
|
|
|
|
function snapshotDataRecord(
|
|
value: unknown,
|
|
allowedKeySets: readonly (readonly string[])[],
|
|
): Readonly<Record<string, unknown>> | null {
|
|
const snapshot = captureRealtimeDataSnapshot(value);
|
|
return snapshot && hasExactSnapshotKeys(snapshot, ...allowedKeySets)
|
|
? snapshot.values
|
|
: null;
|
|
}
|
|
|
|
function retryDelay(
|
|
policy: PollLeasePolicy,
|
|
consecutiveFailures: number,
|
|
retryAfterMs: number | undefined,
|
|
random: () => number,
|
|
): number | null {
|
|
const sample = safeRandom(random);
|
|
if (sample === null) return null;
|
|
if (
|
|
retryAfterMs !== undefined &&
|
|
(!Number.isSafeInteger(retryAfterMs) ||
|
|
retryAfterMs < 0 ||
|
|
retryAfterMs > policy.maxIntervalMs)
|
|
) {
|
|
return null;
|
|
}
|
|
const ceiling = Math.min(
|
|
policy.maxIntervalMs,
|
|
policy.minimumIntervalMs * 2 ** Math.max(0, consecutiveFailures - 1),
|
|
);
|
|
return Math.max(
|
|
policy.minimumIntervalMs,
|
|
Math.floor(ceiling * sample),
|
|
retryAfterMs ?? 0,
|
|
);
|
|
}
|
|
|
|
function isPollRetryableFailureKind(
|
|
kind: RealtimeFailureKind,
|
|
): boolean {
|
|
return POLL_RETRYABLE_FAILURE_KINDS.includes(
|
|
kind as (typeof POLL_RETRYABLE_FAILURE_KINDS)[number],
|
|
);
|
|
}
|
|
|
|
function isPollRetryAfterFailureKind(
|
|
kind: RealtimeFailureKind,
|
|
): boolean {
|
|
return POLL_RETRY_AFTER_FAILURE_KINDS.includes(
|
|
kind as (typeof POLL_RETRY_AFTER_FAILURE_KINDS)[number],
|
|
);
|
|
}
|
|
|
|
function successDelay(
|
|
policy: PollLeasePolicy,
|
|
random: () => number,
|
|
): number | null {
|
|
const sample = safeRandom(random);
|
|
if (sample === null) return null;
|
|
const spread = Math.floor(policy.successIntervalMs * 0.1);
|
|
return Math.min(
|
|
policy.maxIntervalMs,
|
|
Math.max(
|
|
policy.minimumIntervalMs,
|
|
policy.successIntervalMs -
|
|
spread +
|
|
Math.floor(2 * spread * sample),
|
|
),
|
|
);
|
|
}
|
|
|
|
function safeRandom(random: () => number): number | null {
|
|
try {
|
|
const value = random();
|
|
return Number.isFinite(value) && value >= 0 && value < 1
|
|
? value
|
|
: null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function safeNow(clock: ClockPort): number | null {
|
|
try {
|
|
const value = clock.now();
|
|
return Number.isFinite(value) ? value : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function safeIsCurrent(isCurrent: () => boolean): boolean {
|
|
try {
|
|
return isCurrent() === true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function safeVisibility(
|
|
environment: BoundedPollEnvironment,
|
|
): "HIDDEN" | "VISIBLE" | null {
|
|
try {
|
|
const value = environment.visibility();
|
|
return value === "HIDDEN" || value === "VISIBLE" ? value : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function safeOnline(
|
|
environment: BoundedPollEnvironment,
|
|
): boolean | null {
|
|
try {
|
|
const value = environment.online();
|
|
return typeof value === "boolean" ? value : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function safelyUnsubscribe(
|
|
unsubscribe: (() => void) | undefined,
|
|
): void {
|
|
try {
|
|
unsubscribe?.();
|
|
} catch {
|
|
// Lifecycle cleanup remains terminal even for a throwing host.
|
|
}
|
|
}
|
|
|
|
function pollFailure(
|
|
kind: RealtimeFailureKind,
|
|
retryable = false,
|
|
): BoundedPollResult<never> {
|
|
return realtimeFailure(kind, "POLL", retryable);
|
|
}
|
|
|
|
function applyFailure(): BoundedPollResult<never> {
|
|
return realtimeFailure("APPLY_FAILED", "APPLY", false);
|
|
}
|