review-followup Wave 2: 정합성 수정 통합과 검증, 현황표·재개 프롬프트

13개 작업 묶음(P1–P13)의 W2 커밋을 합치고 전체 게이트로 검증했다.

- 묶음별 정합성 수정(notification dispatch·recovery, object storage staged
  publication, fileserver 복구·정리, gRPC 플랫폼, Redis 분류·lease TTL, JPA
  outbox claim 순서·durable operation HTTP, web 요청 id·category 필수, graphql
  에러 mapper, Mongo profile 바인딩, build 소스 보존 게이트·compose smoke info 확인)
- 통합 때 드러난 3건 수정: outbox claim 테스트 fixture의 가짜 envelope hash,
  sample scan이 새 operation controller를 등록하던 문제, SpotBugs aux classpath
  파일이 truncate되지 않아 로컬에서 ".jar.jar"로 실패하던 문제
- 현황표: 묶음 결과 반영, 소유자 결정 21–27, 후속 발견 결함 47행, 요약 갱신
- docs/capability-status.md 「제거된 코드」에 85행 추가
- 묶음 결과 파일·에이전트 지시·스크립트를 docs로 옮기고 desktop 재개 프롬프트 추가

검증: check, ci verifyDependencyLocks, mainCheck, architecture/public path/env
keys/config contract, bootstrap·sample integration/system, build-logic·build-tools
test, optional-platforms ci, qualificationBootJar 통과. integrationCheck는 이번
작업 전에도 실패하던 lane(Redis topology 모드 인자, httpclient failure
injection, AWS 보호 lane)만 실패.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
DongHyeonka
2026-09-28 14:46:30 +09:00
co-authored by Claude Opus 5.5
parent 2c7d1d58f3
commit 2c612cb95b
28 changed files with 763 additions and 134 deletions
+3 -1
View File
@@ -35,7 +35,9 @@ tasks.register('ci') {
description = 'Runs the optional gRPC platform verification surface.'
dependsOn tasks.named('checkAll')
dependsOn tasks.named('qualityCheck')
dependsOn tasks.named('verifyCleanArchitectureDependencies')
// architectureCheck, not only the direction gate: it also runs verifyNoIgnoredSourcePackages, so
// a gRPC source Git would drop fails this build's ci the way it fails the main build's.
dependsOn tasks.named('architectureCheck')
}
// A strict integration lane that no workflow invokes is not coverage; it is a file that looks like
@@ -29,13 +29,18 @@ static_resources:
- match: { prefix: "/" }
route:
cluster: grpc_backend
timeout: 30s
# No route timeout: it would cut every server stream at that age. The
# caller's grpc-timeout header bounds each call instead.
timeout: 0s
max_stream_duration:
grpc_timeout_header_max: 0s
cors:
allow_origin_string_match:
- exact: "https://app.example.com"
allow_methods: "POST,OPTIONS"
allow_headers: "content-type,x-grpc-web,x-correlation-id,authorization"
expose_headers: "grpc-status,grpc-message,error-code,error-category"
allow_headers: "content-type,x-grpc-web,x-user-agent,grpc-timeout,x-correlation-id,authorization"
# Every trailer the platform sends (GrpcErrorMapper, GrpcValidationInterceptor).
expose_headers: "grpc-status,grpc-message,grpc-status-details-bin,error-reason,error-execution-id,completion-outcome,validation-violations"
max_age: "1728000"
http_filters:
- name: envoy.filters.http.grpc_web
@@ -51,6 +56,11 @@ static_resources:
name: envoy.transport_sockets.tls
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
# Placeholder paths: mount the proxy's certificate and key here.
common_tls_context:
tls_certificates:
- certificate_chain: { filename: /etc/envoy/tls/tls.crt }
private_key: { filename: /etc/envoy/tls/tls.key }
clusters:
- name: grpc_backend
connect_timeout: 1s
@@ -7,11 +7,15 @@ import dev.caskeleton.grpc.advanced.web.GrpcWebProfile;
import dev.caskeleton.grpc.advanced.web.GrpcWebProxyContract;
import dev.caskeleton.grpc.advanced.web.GrpcWebRpcSupport;
import dev.caskeleton.grpc.core.RpcType;
import dev.caskeleton.grpc.error.GrpcErrorMapper;
import dev.caskeleton.grpc.validation.GrpcValidationInterceptor;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -110,6 +114,58 @@ class GrpcWebCompatibilityTest {
.contains("envoy.filters.http.grpc_web");
}
@Test
void theReferenceProxyExposesEveryTrailerThePlatformSends() {
// when
Set<String> exposed = headerList(resource("envoy/envoy.yaml"), "expose_headers");
// then
assertThat(exposed)
.contains(
"grpc-status",
"grpc-message",
"grpc-status-details-bin",
GrpcErrorMapper.REASON_KEY.name(),
GrpcErrorMapper.EXECUTION_ID_KEY.name(),
GrpcErrorMapper.COMPLETION_OUTCOME_KEY.name(),
GrpcValidationInterceptor.VIOLATIONS_KEY.name());
}
@Test
void theReferenceProxyLetsTheBrowserSendItsDeadlineAndClientHeaders() {
// when
Set<String> allowed = headerList(resource("envoy/envoy.yaml"), "allow_headers");
// then
assertThat(allowed).contains("grpc-timeout", "x-user-agent", "x-grpc-web", "content-type");
}
@Test
void theReferenceRouteLeavesStreamLengthToTheCallersDeadline() {
// when
String envoy = resource("envoy/envoy.yaml");
// then
assertThat(envoy).contains("timeout: 0s").contains("grpc_timeout_header_max: 0s");
}
@Test
void theReferenceProxyTerminatesTlsWithACertificate() {
// when
String envoy = resource("envoy/envoy.yaml");
// then
assertThat(envoy).contains("certificate_chain:").contains("private_key:");
}
private static Set<String> headerList(String envoy, String field) {
Matcher matcher = Pattern.compile(field + ": \"([^\"]*)\"").matcher(envoy);
if (!matcher.find()) {
throw new IllegalStateException("the reference proxy has no " + field);
}
return Set.of(matcher.group(1).split(","));
}
private static String resource(String path) {
try (InputStream stream =
GrpcWebCompatibilityTest.class.getClassLoader().getResourceAsStream(path)) {
@@ -22,7 +22,9 @@ public final class GrpcServiceHealthRegistry {
private final ConcurrentMap<String, GrpcHealthState> states = new ConcurrentHashMap<>();
private final ConcurrentMap<String, Boolean> dependencyHealth = new ConcurrentHashMap<>();
private final GrpcHealthPolicy policy;
private volatile boolean draining;
// Every write holds this registry's lock, so a state change cannot land between the draining
// check and its write and undo a drain that started in between.
private boolean draining;
/** Binds a registry to its health policy. */
public GrpcServiceHealthRegistry(GrpcHealthPolicy policy) {
@@ -34,7 +36,7 @@ public final class GrpcServiceHealthRegistry {
}
/** Registers a service, unknown until something says otherwise. */
public void register(String serviceName) {
public synchronized void register(String serviceName) {
requireServiceName(serviceName);
states.putIfAbsent(serviceName, GrpcHealthState.UNKNOWN);
}
@@ -45,7 +47,7 @@ public final class GrpcServiceHealthRegistry {
* <p>Ignored while draining: a service that reports itself healthy after the drain has started
* would be routed traffic the instance has already promised not to take.
*/
public void markServing(String serviceName) {
public synchronized void markServing(String serviceName) {
requireServiceName(serviceName);
if (draining) {
return;
@@ -55,7 +57,7 @@ public final class GrpcServiceHealthRegistry {
}
/** Records that a service is not serving. */
public void markNotServing(String serviceName) {
public synchronized void markNotServing(String serviceName) {
requireServiceName(serviceName);
if (draining) {
return;
@@ -65,7 +67,7 @@ public final class GrpcServiceHealthRegistry {
}
/** Records a dependency's health, which only matters when the policy says it does. */
public void recordDependencyHealth(String dependencyName, boolean healthy) {
public synchronized void recordDependencyHealth(String dependencyName, boolean healthy) {
if (dependencyName == null || dependencyName.isBlank()) {
throw new IllegalArgumentException("a dependency needs a name");
}
@@ -81,7 +83,7 @@ public final class GrpcServiceHealthRegistry {
* <p>Called first in a shutdown, before admission stops, so that a load balancer has a chance to
* stop sending before the server starts refusing.
*/
public void beginDraining() {
public synchronized void beginDraining() {
draining = true;
states.replaceAll((service, state) -> GrpcHealthState.DRAINING);
}
@@ -4,6 +4,11 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -88,4 +93,47 @@ class GrpcServiceHealthRegistryTest {
registry.markServing(DOCUMENTS);
assertThat(registry.stateOf(DOCUMENTS)).isEqualTo(GrpcHealthState.DRAINING);
}
@Test
void aServiceMarkedServingWhileDrainBeginsNeverReturnsTheInstanceToServing() throws Exception {
// when
Set<String> outcomes = outcomesOfServingRacingDrain();
// then
assertThat(outcomes).containsExactly("DRAINING/DRAINING");
}
/** Races markServing against beginDraining, round after round, recording service/global. */
private static Set<String> outcomesOfServingRacingDrain() throws Exception {
ExecutorService pool = Executors.newFixedThreadPool(2);
try {
Set<String> outcomes = new TreeSet<>();
for (int round = 0; round < 1000; round++) {
GrpcServiceHealthRegistry registry =
new GrpcServiceHealthRegistry(GrpcHealthPolicy.standalone());
registry.register(DOCUMENTS);
CyclicBarrier start = new CyclicBarrier(2);
Future<?> serving =
pool.submit(
() -> {
start.await();
registry.markServing(DOCUMENTS);
return null;
});
Future<?> drain =
pool.submit(
() -> {
start.await();
registry.beginDraining();
return null;
});
serving.get();
drain.get();
outcomes.add(registry.stateOf(DOCUMENTS) + "/" + registry.globalState());
}
return outcomes;
} finally {
pool.shutdownNow();
}
}
}
@@ -3,7 +3,9 @@ package dev.caskeleton.grpc.client;
import dev.caskeleton.grpc.core.GrpcChannelProfileName;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -24,8 +26,9 @@ public final class GrpcChannelRuntimeRegistry {
private final ConcurrentMap<GrpcChannelProfileName, AtomicReference<GrpcChannelRuntime>> current =
new ConcurrentHashMap<>();
private final ConcurrentMap<GrpcChannelProfileName, List<GrpcChannelRuntime>> draining =
new ConcurrentHashMap<>();
// Guarded by this registry's lock, like every swap: rotations are rare, and one lock keeps the
// current pointer and the draining list from disagreeing.
private final Map<GrpcChannelProfileName, List<GrpcChannelRuntime>> draining = new HashMap<>();
private final GrpcChannelDrainPolicy drainPolicy;
/** Binds a registry to the drain policy its swaps use. */
@@ -37,7 +40,7 @@ public final class GrpcChannelRuntimeRegistry {
}
/** Installs the first generation for a profile. */
public GrpcChannelRuntime install(GrpcChannelGeneration generation) {
public synchronized GrpcChannelRuntime install(GrpcChannelGeneration generation) {
if (generation == null) {
throw new IllegalArgumentException("a generation is required");
}
@@ -80,7 +83,7 @@ public final class GrpcChannelRuntimeRegistry {
*
* @throws IllegalArgumentException when {@code next} does not supersede what is installed
*/
public GrpcChannelRuntime rotate(GrpcChannelGeneration next, Instant now) {
public synchronized GrpcChannelRuntime rotate(GrpcChannelGeneration next, Instant now) {
if (next == null || now == null) {
throw new IllegalArgumentException("a rotation needs a generation and a moment");
}
@@ -102,15 +105,12 @@ public final class GrpcChannelRuntimeRegistry {
GrpcChannelRuntime replacement = new GrpcChannelRuntime(next);
holder.set(replacement);
previous.beginDrain();
draining
.computeIfAbsent(
next.profileName(), key -> java.util.Collections.synchronizedList(new ArrayList<>()))
.add(previous);
draining.computeIfAbsent(next.profileName(), key -> new ArrayList<>()).add(previous);
return replacement;
}
/** The generations still draining for a profile. */
public List<GrpcChannelRuntime> draining(GrpcChannelProfileName profileName) {
public synchronized List<GrpcChannelRuntime> draining(GrpcChannelProfileName profileName) {
return List.copyOf(draining.getOrDefault(profileName, List.of()));
}
@@ -119,7 +119,7 @@ public final class GrpcChannelRuntimeRegistry {
*
* @return how many were retired
*/
public int retireQuiescent(GrpcChannelProfileName profileName) {
public synchronized int retireQuiescent(GrpcChannelProfileName profileName) {
List<GrpcChannelRuntime> runtimes = draining.get(profileName);
if (runtimes == null) {
return 0;
@@ -6,6 +6,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.grpc.core.GrpcChannelProfileName;
import java.time.Duration;
import java.time.Instant;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -125,4 +131,52 @@ class GrpcChannelRuntimeRegistryTest {
.extracting(Class::getName)
.noneMatch(name -> name.startsWith("io.grpc."));
}
@Test
void racingRotationsDrainEveryReplacedRuntimeAndKeepTheNewestCurrent() throws Exception {
// when
Set<String> outcomes = outcomesOfRacingRotations();
// then
assertThat(outcomes).isSubsetOf("3 draining [1]", "3 draining [1, 2]");
}
/** Races rotate(2) against rotate(3) on a profile at generation 1, round after round. */
private static Set<String> outcomesOfRacingRotations() throws Exception {
ExecutorService pool = Executors.newFixedThreadPool(2);
try {
Set<String> outcomes = new TreeSet<>();
for (int round = 0; round < 1000; round++) {
GrpcChannelRuntimeRegistry registry =
new GrpcChannelRuntimeRegistry(GrpcChannelDrainPolicy.stable());
registry.install(generation(1));
CyclicBarrier start = new CyclicBarrier(2);
Future<?> a = pool.submit(() -> rotateIgnoringRefusal(registry, start, 2));
Future<?> b = pool.submit(() -> rotateIgnoringRefusal(registry, start, 3));
a.get();
b.get();
outcomes.add(
registry.require(PROFILE).generation().generation()
+ " draining "
+ registry.draining(PROFILE).stream()
.map(runtime -> runtime.generation().generation())
.sorted()
.toList());
}
return outcomes;
} finally {
pool.shutdownNow();
}
}
private static Void rotateIgnoringRefusal(
GrpcChannelRuntimeRegistry registry, CyclicBarrier start, long next) throws Exception {
start.await();
try {
registry.rotate(generation(next), NOW);
} catch (IllegalArgumentException refused) {
// the other rotator installed a newer generation first
}
return null;
}
}
@@ -122,8 +122,10 @@ class GrpcTypedStubFactoryTest {
GrpcStubPolicyApplier.CallPolicy generous = applier.applyTo(GET, Duration.ofSeconds(30));
GrpcStubPolicyApplier.CallPolicy tight = applier.applyTo(GET, Duration.ofMillis(300));
assertThat(generous.deadline().remaining()).isEqualTo(Duration.ofMillis(1800));
assertThat(tight.deadline().remaining()).isEqualTo(Duration.ofMillis(100));
// The budget is live: a few milliseconds have passed since applyTo computed it.
assertThat(generous.deadline().remaining())
.isBetween(Duration.ofMillis(1700), Duration.ofMillis(1800));
assertThat(tight.deadline().remaining()).isBetween(Duration.ZERO, Duration.ofMillis(100));
assertThat(tight.waitForReady().queue()).isFalse();
}
}
@@ -1,6 +1,7 @@
package dev.caskeleton.grpc.api.deadline;
import java.time.Duration;
import java.util.function.LongSupplier;
/**
* How much time a call actually has, after the parent deadline and the method profile have both had
@@ -9,17 +10,40 @@ import java.time.Duration;
* <p>The rule is "shorter of the two, minus the reserve", and it is here rather than at each call
* site because the failure mode of getting it wrong is silent: a downstream timeout longer than the
* inbound deadline never fires, and the caller sees a deadline the server has no idea it missed.
*
* <p>The budget is an absolute deadline on a monotonic nanosecond clock, so {@link #remaining()}
* shrinks and {@link #expired()} turns true as time passes. A request context carries one budget
* for the whole call; a budget that remembered only the duration it started with would hand every
* later dependency call the full original time.
*/
public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile) {
public final class GrpcDeadlineBudget {
/** Rejects a budget with no time in it. */
public GrpcDeadlineBudget {
if (remaining == null || profile == null) {
throw new IllegalArgumentException("a deadline budget needs a remaining time and a profile");
private final long deadlineNanos;
private final GrpcDeadlineProfile profile;
private final LongSupplier nanoClock;
/** A budget of {@code remaining} from now on the system's monotonic clock. */
public GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile) {
this(remaining, profile, System::nanoTime);
}
/**
* A budget of {@code remaining} from now on {@code nanoClock}.
*
* @param nanoClock a monotonic nanosecond source, {@code System::nanoTime} outside tests
*/
public GrpcDeadlineBudget(
Duration remaining, GrpcDeadlineProfile profile, LongSupplier nanoClock) {
if (remaining == null || profile == null || nanoClock == null) {
throw new IllegalArgumentException(
"a deadline budget needs a remaining time, a profile and a clock");
}
if (remaining.isNegative()) {
throw new IllegalArgumentException("remaining time must not be negative; got " + remaining);
}
this.deadlineNanos = nanoClock.getAsLong() + remaining.toNanos();
this.profile = profile;
this.nanoClock = nanoClock;
}
/**
@@ -31,6 +55,12 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile
*/
public static GrpcDeadlineBudget forEntryPoint(
Duration parentRemaining, GrpcDeadlineProfile profile) {
return forEntryPoint(parentRemaining, profile, System::nanoTime);
}
/** {@link #forEntryPoint(Duration, GrpcDeadlineProfile)} on {@code nanoClock}. */
public static GrpcDeadlineBudget forEntryPoint(
Duration parentRemaining, GrpcDeadlineProfile profile, LongSupplier nanoClock) {
if (profile == null) {
throw new IllegalArgumentException("a deadline budget needs a profile");
}
@@ -42,17 +72,13 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile
throw new IllegalArgumentException(
"a Stable unary call requires a positive deadline; got " + parentRemaining);
}
Duration bounded =
parentRemaining.compareTo(profile.total()) <= 0 ? parentRemaining : profile.total();
Duration afterReserve = bounded.minus(profile.safetyMargin());
return new GrpcDeadlineBudget(
afterReserve.isNegative() ? Duration.ZERO : afterReserve, profile);
return new GrpcDeadlineBudget(afterReserve(parentRemaining, profile), profile, nanoClock);
}
/**
* Derives the budget for a downstream dependency call under {@code downstream}.
*
* <p>Takes the shorter of what is left here and what the downstream profile allows, then
* <p>Takes the shorter of what is left here now and what the downstream profile allows, then
* subtracts the downstream reserve. A dependency can never be given more time than its caller
* has.
*/
@@ -60,11 +86,18 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile
if (downstream == null) {
throw new IllegalArgumentException("a downstream budget needs a profile");
}
Duration bounded =
remaining.compareTo(downstream.total()) <= 0 ? remaining : downstream.total();
Duration afterReserve = bounded.minus(downstream.safetyMargin());
return new GrpcDeadlineBudget(
afterReserve.isNegative() ? Duration.ZERO : afterReserve, downstream);
return new GrpcDeadlineBudget(afterReserve(remaining(), downstream), downstream, nanoClock);
}
/** What is left now; zero once the deadline has passed. */
public Duration remaining() {
long left = deadlineNanos - nanoClock.getAsLong();
return left <= 0 ? Duration.ZERO : Duration.ofNanos(left);
}
/** The profile this budget was cut from. */
public GrpcDeadlineProfile profile() {
return profile;
}
/**
@@ -75,11 +108,23 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile
* investigates the wrong system.
*/
public boolean canStartDependencyCall() {
return remaining.compareTo(profile.minimumAttemptBudget()) >= 0 && !remaining.isZero();
Duration left = remaining();
return !left.isZero() && left.compareTo(profile.minimumAttemptBudget()) >= 0;
}
/** Whether any time is left at all. */
public boolean expired() {
return remaining.isZero() || remaining.isNegative();
return remaining().isZero();
}
@Override
public String toString() {
return "GrpcDeadlineBudget[remaining=" + remaining() + ", profile=" + profile + "]";
}
private static Duration afterReserve(Duration available, GrpcDeadlineProfile profile) {
Duration bounded = available.compareTo(profile.total()) <= 0 ? available : profile.total();
Duration afterReserve = bounded.minus(profile.safetyMargin());
return afterReserve.isNegative() ? Duration.ZERO : afterReserve;
}
}
@@ -16,6 +16,13 @@ import dev.caskeleton.grpc.core.GrpcMethodName;
public record GrpcOperationIdentity(
String callerFingerprint, GrpcMethodName method, String idempotencyKeyHash) {
// Column widths of grpc_operation_ledger (V001). An identity that does not fit is refused here,
// not by the database in the middle of a claim.
private static final int MAX_CALLER_FINGERPRINT = 256;
private static final int MAX_METHOD = 256;
private static final int MAX_KEY_HASH = 80;
private static final int MAX_STORAGE_KEY = 512;
/** Requires all three parts, with the key already hashed. */
public GrpcOperationIdentity {
if (callerFingerprint == null || callerFingerprint.isBlank()) {
@@ -30,6 +37,24 @@ public record GrpcOperationIdentity(
throw new IllegalArgumentException(
"the idempotency key is stored hashed; got '" + idempotencyKeyHash + "'");
}
requireAtMost("caller fingerprint", callerFingerprint, MAX_CALLER_FINGERPRINT);
requireAtMost("full method name", method.canonical(), MAX_METHOD);
requireAtMost("idempotency key hash", idempotencyKeyHash, MAX_KEY_HASH);
requireAtMost(
"storage key",
callerFingerprint + "|" + method.canonical() + "|" + idempotencyKeyHash,
MAX_STORAGE_KEY);
}
private static void requireAtMost(String what, String value, int max) {
if (value.length() > max) {
throw new IllegalArgumentException(
what
+ " must be at most "
+ max
+ " characters to fit the ledger; got "
+ value.length());
}
}
/** The stable storage key: the three parts joined, safe to index and safe to log. */
@@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -27,9 +28,11 @@ class GrpcDeadlineBudgetTest {
@DisplayName("the budget takes the shorter of the parent remaining and the method default")
void budgetTakesTheShorterOfParentAndMethodDefault() {
GrpcDeadlineBudget fromShortParent =
GrpcDeadlineBudget.forEntryPoint(Duration.ofMillis(500), TWO_SECONDS);
GrpcDeadlineBudget.forEntryPoint(
Duration.ofMillis(500), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos);
GrpcDeadlineBudget fromLongParent =
GrpcDeadlineBudget.forEntryPoint(Duration.ofSeconds(30), TWO_SECONDS);
GrpcDeadlineBudget.forEntryPoint(
Duration.ofSeconds(30), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos);
assertThat(fromShortParent.remaining())
.isEqualTo(Duration.ofMillis(500).minus(TWO_SECONDS.safetyMargin()));
@@ -44,7 +47,9 @@ class GrpcDeadlineBudgetTest {
new GrpcDeadlineProfile(
Duration.ofSeconds(10), Duration.ofSeconds(1), Duration.ofMillis(100));
GrpcDeadlineBudget budget = GrpcDeadlineBudget.forEntryPoint(Duration.ofSeconds(10), profile);
GrpcDeadlineBudget budget =
GrpcDeadlineBudget.forEntryPoint(
Duration.ofSeconds(10), profile, GrpcDeadlineBudgetTest::frozenNanos);
assertThat(budget.remaining()).isEqualTo(Duration.ofSeconds(9));
assertThat(profile.usable()).isEqualTo(Duration.ofSeconds(9));
@@ -54,7 +59,8 @@ class GrpcDeadlineBudgetTest {
@DisplayName("a downstream call never gets more time than its caller has left")
void downstreamNeverExceedsTheCaller() {
GrpcDeadlineBudget caller =
GrpcDeadlineBudget.forEntryPoint(Duration.ofMillis(400), TWO_SECONDS);
GrpcDeadlineBudget.forEntryPoint(
Duration.ofMillis(400), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos);
GrpcDeadlineProfile generousDownstream =
new GrpcDeadlineProfile(Duration.ofSeconds(30), Duration.ZERO, Duration.ofMillis(10));
@@ -77,6 +83,39 @@ class GrpcDeadlineBudgetTest {
assertThat(new GrpcDeadlineBudget(Duration.ZERO, profile).canStartDependencyCall()).isFalse();
}
@Test
void aBudgetRunsOutAsTimePassesAfterItWasCreated() {
// given
AtomicLong nanos = new AtomicLong();
GrpcDeadlineBudget budget =
new GrpcDeadlineBudget(Duration.ofMillis(300), TWO_SECONDS, nanos::get);
// when
nanos.addAndGet(Duration.ofMillis(300).toNanos());
// then
assertThat(budget)
.extracting(GrpcDeadlineBudget::remaining, GrpcDeadlineBudget::expired)
.containsExactly(Duration.ZERO, true);
}
@Test
void aDownstreamDerivedLaterGetsOnlyWhatIsLeftThen() {
// given
AtomicLong nanos = new AtomicLong();
GrpcDeadlineBudget caller =
new GrpcDeadlineBudget(Duration.ofSeconds(1), TWO_SECONDS, nanos::get);
nanos.addAndGet(Duration.ofMillis(700).toNanos());
// when
GrpcDeadlineBudget downstream =
caller.deriveDownstream(
new GrpcDeadlineProfile(Duration.ofSeconds(30), Duration.ZERO, Duration.ofMillis(10)));
// then
assertThat(downstream.remaining()).isEqualTo(Duration.ofMillis(300));
}
@Test
@DisplayName("a profile whose reserve leaves no usable time is refused")
void profileWithNoUsableTimeIsRefused() {
@@ -121,4 +160,8 @@ class GrpcDeadlineBudgetTest {
.hasMessageContaining("charge-card")
.hasMessageContaining("deadline-exceeded");
}
private static long frozenNanos() {
return 0L;
}
}
@@ -0,0 +1,44 @@
package dev.caskeleton.grpc.api.ledger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.catchThrowable;
import dev.caskeleton.grpc.core.GrpcMethodName;
import dev.caskeleton.grpc.core.GrpcServiceName;
import org.junit.jupiter.api.Test;
class GrpcOperationIdentityTest {
private static final GrpcMethodName CREATE =
GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument");
private static final String KEY_HASH =
"sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899";
@Test
void aCallerFingerprintWiderThanItsLedgerColumnIsRefused() {
// when
Throwable failure =
catchThrowable(() -> new GrpcOperationIdentity("c".repeat(257), CREATE, KEY_HASH));
// then
assertThat(failure)
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("caller fingerprint must be at most 256");
}
@Test
void anIdentityWhoseStorageKeyWouldNotFitTheLedgerIsRefused() {
// given
GrpcMethodName longMethod =
new GrpcMethodName(new GrpcServiceName("p.S" + "s".repeat(197)), "M" + "m".repeat(54));
// when
Throwable failure =
catchThrowable(() -> new GrpcOperationIdentity("c".repeat(256), longMethod, KEY_HASH));
// then
assertThat(failure)
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("storage key must be at most 512");
}
}
@@ -50,7 +50,19 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger {
if (inserted != 0) {
throw new IllegalStateException("claim insert affected an unexpected row count: " + inserted);
}
return repository.findById(identity.storageKey()).map(entity -> entity.toRecord(identity));
// Zero rows means another claim holds the key. An empty read now means that owner released it
// in between; treating that as "ours" would run the mutation without a claim row to guard it.
return Optional.of(
repository
.findById(identity.storageKey())
.orElseThrow(
() ->
new IllegalStateException(
"claim for '"
+ identity.storageKey()
+ "' conflicted, but there was no row to read; the other owner released"
+ " it in between. Retry the claim."))
.toRecord(identity));
}
@Override
@@ -2,6 +2,7 @@ package dev.caskeleton.grpc.ledger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.catchThrowable;
import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity;
import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger;
@@ -40,7 +41,7 @@ class GrpcOperationLedgerRepositoryTest {
* putIfAbsent} and therefore tested the database constraint we wished save() would hit rather
* than the merge behavior it actually uses.
*/
private static final class InMemoryRepository implements GrpcOperationLedgerRepository {
private static class InMemoryRepository implements GrpcOperationLedgerRepository {
private final ConcurrentMap<String, GrpcOperationLedgerEntity> rows = new ConcurrentHashMap<>();
@Override
@@ -124,6 +125,34 @@ class GrpcOperationLedgerRepositoryTest {
assertThat(second.get().sameRequestAs(FINGERPRINT)).isTrue();
}
@Test
void aConflictWhoseRowVanishedBeforeTheReadIsNotTakenAsOwnership() {
// given
InMemoryRepository conflictThenReleased =
new InMemoryRepository() {
@Override
public int tryInsertClaim(
String storageKey,
String callerFingerprint,
String fullMethodName,
String idempotencyKeyHash,
String requestFingerprint,
Instant claimedAt) {
return 0; // another owner held the key and released it before this read
}
};
GrpcOperationLedger racing = new JpaGrpcOperationLedger(conflictThenReleased);
// when
Throwable failure =
catchThrowable(() -> racing.claim(identity("tenant-1.actor-1"), FINGERPRINT, NOW));
// then
assertThat(failure)
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("no row to read");
}
@Test
@DisplayName("one caller's key does not collide with another's")
void callersDoNotCollide() {
@@ -87,15 +87,14 @@ public final class GrpcContextBinder {
if (work == null) {
throw new IllegalArgumentException("work must not be null");
}
GrpcContextSnapshot captured = SNAPSHOT.get();
if (captured == null && policy.failClosedWithoutContext()) {
// The whole io.grpc Context, not just the snapshot value: it also carries the call's
// cancellation and deadline, which the other thread must see.
Context captured = Context.current();
if (SNAPSHOT.get(captured) == null && policy.failClosedWithoutContext()) {
throw new IllegalStateException(
"refusing to hand work to another thread with no call context to carry");
}
if (captured == null) {
return work;
}
return () -> runWith(captured, work);
return captured.wrap(work);
}
/** An {@link Executor} that carries the submitting thread's context to every task. */
@@ -78,13 +78,15 @@ public final class GrpcRetryCoordinator {
}
Duration backoff = jittered(config.backoffBefore(completedAttempt + 1), config.jitterFactor());
if (remaining.remaining().compareTo(backoff) <= 0) {
// Read once: the budget is live, and a second read could fall below the backoff.
Duration left = remaining.remaining();
if (left.compareTo(backoff) <= 0) {
return GrpcRetryDecision.refuse(
GrpcRetryDecision.Verdict.TERMINAL,
"the remaining deadline is shorter than the backoff, so a retry could not finish");
}
GrpcDeadlineBudget afterBackoff =
new GrpcDeadlineBudget(remaining.remaining().minus(backoff), remaining.profile());
new GrpcDeadlineBudget(left.minus(backoff), remaining.profile());
if (!afterBackoff.canStartDependencyCall()) {
return GrpcRetryDecision.refuse(
GrpcRetryDecision.Verdict.TERMINAL,
@@ -35,7 +35,9 @@ public final class GrpcWaitForReadyValidator {
+ policy.method().canonical()
+ "' must carry one");
}
if (remaining.expired()) {
// Read once: the budget is live and keeps shrinking between reads.
Duration left = remaining.remaining();
if (left.isZero()) {
return GrpcWaitForReadyDecision.failFast("no deadline remains to queue inside");
}
if (policy.waitForReady() != profile.policy()) {
@@ -47,9 +49,7 @@ public final class GrpcWaitForReadyValidator {
+ "; the stricter of the two wins");
}
Duration allowance =
profile.maxQueueWait().compareTo(remaining.remaining()) <= 0
? profile.maxQueueWait()
: remaining.remaining();
profile.maxQueueWait().compareTo(left) <= 0 ? profile.maxQueueWait() : left;
return GrpcWaitForReadyDecision.queueFor(
allowance, "queueing under " + profile.policy() + " within the queue budget");
}
@@ -86,22 +86,29 @@ public final class GrpcCredentialRotationManager {
if (next == null || now == null) {
throw new IllegalArgumentException("a rotation needs a generation and a moment");
}
State observed = state.get();
if (observed.current().generation() == next.generation()) {
// Idempotent: re-applying the generation that is already current is a no-op rather than a
// second drain, because a retried rotation must not cancel the calls the first one admitted.
return new RotationPlan(observed.current(), Optional.empty(), Optional.empty());
}
if (!observed.current().supersededBy(next)) {
throw new IllegalArgumentException(
"credential generation "
+ next.generation()
+ " does not supersede the current generation "
+ observed.current().generation());
}
Instant deadline = now.plus(drainWindow);
state.set(new State(next, observed.current(), deadline));
return new RotationPlan(next, Optional.of(observed.current()), Optional.of(deadline));
// Compare-and-set loop: a racing rotator or completeDrain re-runs the checks against the state
// it actually replaces, so a rotation never goes backwards and never loses a draining
// generation.
while (true) {
State observed = state.get();
if (observed.current().generation() == next.generation()) {
// Idempotent: re-applying the generation that is already current is a no-op rather than a
// second drain, because a retried rotation must not cancel the calls the first one
// admitted.
return new RotationPlan(observed.current(), Optional.empty(), Optional.empty());
}
if (!observed.current().supersededBy(next)) {
throw new IllegalArgumentException(
"credential generation "
+ next.generation()
+ " does not supersede the current generation "
+ observed.current().generation());
}
if (state.compareAndSet(observed, new State(next, observed.current(), deadline))) {
return new RotationPlan(next, Optional.of(observed.current()), Optional.of(deadline));
}
}
}
/**
@@ -116,7 +123,6 @@ public final class GrpcCredentialRotationManager {
/** Forgets the draining generation once its work has finished or been cancelled. */
public void completeDrain() {
State observed = state.get();
state.set(new State(observed.current(), null, null));
state.updateAndGet(observed -> new State(observed.current(), null, null));
}
}
@@ -4,8 +4,9 @@ import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import java.util.Optional;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.LongSupplier;
import java.util.function.ToLongFunction;
/**
* The single writer every server stream goes through.
@@ -27,7 +28,7 @@ public final class GrpcSerializedStreamWriter<T> {
private final GrpcStreamSequence sequence;
private final GrpcFlowControlPolicy flowControl;
private final Consumer<GrpcStreamEnvelope<T>> transport;
private final LongSupplier payloadSizer;
private final ToLongFunction<? super T> payloadSizer;
private final Deque<QueuedEnvelope<T>> queue = new ArrayDeque<>();
private GrpcStreamWriterState state = GrpcStreamWriterState.OPEN;
@@ -37,15 +38,15 @@ public final class GrpcSerializedStreamWriter<T> {
/**
* @param transport the single consumer that actually calls the observer. Invoked only from {@link
* #flush}, which is synchronized, so the observer sees one thread.
* @param payloadSizer the serialized size of the next payload, for the byte bound
* #flush} and {@link #terminate}, which are synchronized, so the observer sees one thread.
* @param payloadSizer the serialized size of a payload, for the byte bound
*/
public GrpcSerializedStreamWriter(
GrpcStreamId streamId,
GrpcStreamSequence sequence,
GrpcFlowControlPolicy flowControl,
Consumer<GrpcStreamEnvelope<T>> transport,
LongSupplier payloadSizer) {
ToLongFunction<? super T> payloadSizer) {
if (streamId == null
|| sequence == null
|| flowControl == null
@@ -70,7 +71,7 @@ public final class GrpcSerializedStreamWriter<T> {
if (!state.acceptsMessages()) {
return GrpcStreamWriteResult.REJECTED_CLOSED;
}
long nextBytes = payloadSizer.getAsLong();
long nextBytes = payloadSizer.applyAsLong(payload);
GrpcFlowControlDecision decision =
flowControl.decide(queue.size(), queuedBytes, nextBytes, transportReady);
return switch (decision.action()) {
@@ -97,7 +98,11 @@ public final class GrpcSerializedStreamWriter<T> {
}
}
}
case PAUSE, PROCEED -> {
case PAUSE -> {
enqueue(kind, payload, snapshotVersion, resumeToken, nextBytes);
yield GrpcStreamWriteResult.ACCEPTED_PAUSE;
}
case PROCEED -> {
enqueue(kind, payload, snapshotVersion, resumeToken, nextBytes);
yield GrpcStreamWriteResult.ACCEPTED;
}
@@ -119,20 +124,22 @@ public final class GrpcSerializedStreamWriter<T> {
}
/**
* Hands every queued message to the transport, in order, from this one thread.
* Hands queued messages to the transport, in order, from this one thread, while it is ready.
*
* @param transportReady asked before each message ({@code ServerCallStreamObserver::isReady});
* flushing stops at the first false and the rest stay queued for the next onReady
* @return the sequence numbers flushed
*/
public synchronized List<Long> flush() {
public synchronized List<Long> flush(BooleanSupplier transportReady) {
java.util.List<Long> flushed = new java.util.ArrayList<>();
while (!queue.isEmpty()) {
GrpcStreamEnvelope<T> envelope = queue.pollFirst().envelope();
transport.accept(envelope);
lastFlushedSequence = envelope.sequence();
flushed.add(envelope.sequence());
while (!queue.isEmpty() && transportReady.getAsBoolean()) {
QueuedEnvelope<T> queued = queue.pollFirst();
transport.accept(queued.envelope());
queuedBytes = Math.max(0L, queuedBytes - queued.serializedBytes());
lastFlushedSequence = queued.envelope().sequence();
flushed.add(lastFlushedSequence);
}
queuedBytes = 0L;
if (state == GrpcStreamWriterState.DRAINING) {
if (queue.isEmpty() && state == GrpcStreamWriterState.DRAINING) {
state = GrpcStreamWriterState.TERMINATED;
}
return List.copyOf(flushed);
@@ -39,14 +39,15 @@ public final class GrpcStreamAdmission {
if (callerFingerprint == null || callerFingerprint.isBlank()) {
throw new IllegalArgumentException("stream admission needs a caller fingerprint");
}
// Global bound first: a caller refused here must not leave an empty per-caller entry behind.
if (openStreams.get() >= maxConcurrentStreams) {
return false;
}
AtomicInteger callerCount =
perCaller.computeIfAbsent(callerFingerprint, key -> new AtomicInteger());
if (callerCount.get() >= maxStreamsPerCaller) {
return false;
}
if (openStreams.get() >= maxConcurrentStreams) {
return false;
}
callerCount.incrementAndGet();
openStreams.incrementAndGet();
return true;
@@ -11,6 +11,12 @@ package dev.caskeleton.grpc.streaming;
public enum GrpcStreamWriteResult {
/** Queued for the single writer. Not a delivery guarantee. */
ACCEPTED,
/**
* Queued, and the producer should stop until the transport is ready again (the queue reached its
* high-water mark or the transport is not ready). Writing on regardless fills the queue to its
* bound and ends the stream.
*/
ACCEPTED_PAUSE,
/** The bounded queue is full and the profile drops rather than terminates. */
DROPPED,
/** The queue overflowed and the stream is being terminated. */
@@ -20,6 +26,6 @@ public enum GrpcStreamWriteResult {
/** Whether the message entered the queue. */
public boolean queued() {
return this == ACCEPTED;
return this == ACCEPTED || this == ACCEPTED_PAUSE;
}
}
@@ -11,6 +11,7 @@ import dev.caskeleton.grpc.api.deadline.GrpcDeadlineBudget;
import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile;
import dev.caskeleton.grpc.core.GrpcMethodName;
import dev.caskeleton.grpc.core.RpcType;
import io.grpc.Context;
import java.time.Duration;
import java.time.Instant;
import java.util.Map;
@@ -19,6 +20,7 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -156,4 +158,29 @@ class GrpcContextBinderTest {
assertThat(actor).isEqualTo("actor-1");
assertThat(binder.current()).isEmpty();
}
@Test
void wrappedWorkSeesTheSubmittingCallsCancellationOnAnotherThread() throws Exception {
// given
GrpcContextBinder binder = new GrpcContextBinder(GrpcContextPropagationPolicy.stable());
Context.CancellableContext call = Context.current().withCancellation();
AtomicBoolean cancelledInside = new AtomicBoolean();
AtomicReference<Runnable> wrapped = new AtomicReference<>();
call.run(
() ->
binder.runWith(
snapshot(null),
() ->
wrapped.set(
binder.wrap(() -> cancelledInside.set(Context.current().isCancelled())))));
call.cancel(new IllegalStateException("the client went away"));
Thread worker = new Thread(wrapped.get());
// when
worker.start();
worker.join();
// then
assertThat(cancelledInside).isTrue();
}
}
@@ -91,7 +91,7 @@ class GrpcWaitForReadyValidatorTest {
GrpcWaitForReadyValidator.decide(
policyWith(WaitForReadyPolicy.WORKER_OPT_IN),
GrpcWaitForReadyProfile.worker(Duration.ofSeconds(30)),
new GrpcDeadlineBudget(Duration.ofSeconds(5), TEN_SECONDS));
new GrpcDeadlineBudget(Duration.ofSeconds(5), TEN_SECONDS, () -> 0L));
GrpcWaitForReadyDecision tightBudget =
GrpcWaitForReadyValidator.decide(
policyWith(WaitForReadyPolicy.WORKER_OPT_IN),
@@ -0,0 +1,91 @@
package dev.caskeleton.grpc.security;
import static org.assertj.core.api.Assertions.assertThat;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
class GrpcCredentialRotationManagerTest {
private static final Instant NOW = Instant.parse("2026-08-30T10:00:00Z");
private static final int ROUNDS = 20_000;
@Test
void racingRotationsNeverLeaveAnOlderGenerationCurrent() throws Exception {
// when
long lowest =
lowestCurrentAfterRacing(
manager -> rotateIgnoringRefusal(manager, 3L), manager -> manager.rotate(gen(4L), NOW));
// then
assertThat(lowest).isEqualTo(4L);
}
@Test
void aDrainCompletedDuringARotationNeverRevertsTheNewGeneration() throws Exception {
// when
long lowest =
lowestCurrentAfterRacing(
manager -> manager.rotate(gen(3L), NOW), GrpcCredentialRotationManager::completeDrain);
// then
assertThat(lowest).isEqualTo(3L);
}
/**
* Starts each round at generation 2 (draining 1), runs {@code first} and {@code second} at the
* same moment, and returns the lowest generation left current over all rounds.
*/
private static long lowestCurrentAfterRacing(
Consumer<GrpcCredentialRotationManager> first, Consumer<GrpcCredentialRotationManager> second)
throws Exception {
ExecutorService pool = Executors.newFixedThreadPool(2);
try {
long lowest = Long.MAX_VALUE;
for (int round = 0; round < ROUNDS; round++) {
GrpcCredentialRotationManager manager =
new GrpcCredentialRotationManager(gen(1L), Duration.ofSeconds(30));
manager.rotate(gen(2L), NOW);
CyclicBarrier start = new CyclicBarrier(2);
Future<?> a = pool.submit(() -> run(start, first, manager));
Future<?> b = pool.submit(() -> run(start, second, manager));
a.get();
b.get();
lowest = Math.min(lowest, manager.current().generation());
}
return lowest;
} finally {
pool.shutdownNow();
}
}
private static Void run(
CyclicBarrier start,
Consumer<GrpcCredentialRotationManager> action,
GrpcCredentialRotationManager manager)
throws Exception {
start.await();
action.accept(manager);
return null;
}
/** The losing rotator of a race is refused, which is the correct outcome, not a failure. */
private static void rotateIgnoringRefusal(GrpcCredentialRotationManager manager, long next) {
try {
manager.rotate(gen(next), NOW);
} catch (IllegalArgumentException refused) {
// superseded by the other rotator
}
}
private static GrpcCredentialGeneration gen(long generation) {
return new GrpcCredentialGeneration(
generation, "vault://v" + generation, NOW, NOW.plusSeconds(3600));
}
}
@@ -9,6 +9,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -23,7 +24,7 @@ class GrpcSerializedStreamWriterTest {
private GrpcSerializedStreamWriter<String> writer(
GrpcFlowControlPolicy policy, long messageSize) {
return new GrpcSerializedStreamWriter<>(
STREAM, GrpcStreamSequence.forNewStream(STREAM), policy, sent::add, () -> messageSize);
STREAM, GrpcStreamSequence.forNewStream(STREAM), policy, sent::add, payload -> messageSize);
}
@Test
@@ -38,7 +39,7 @@ class GrpcSerializedStreamWriterTest {
assertThat(result.queued()).isTrue();
assertThat(sent).isEmpty();
assertThat(writer.flush()).containsExactly(1L);
assertThat(writer.flush(() -> true)).containsExactly(1L);
assertThat(sent).hasSize(1);
}
@@ -76,7 +77,7 @@ class GrpcSerializedStreamWriterTest {
}
assertThat(sent).isEmpty();
List<Long> flushed = writer.flush();
List<Long> flushed = writer.flush(() -> true);
assertThat(flushed).hasSize(producers * perProducer);
assertThat(flushed).isSorted();
assertThat(flushed).doesNotHaveDuplicates();
@@ -121,7 +122,7 @@ class GrpcSerializedStreamWriterTest {
GrpcStreamSequence.forNewStream(STREAM),
new GrpcFlowControlPolicy(10, 1000L, 9, GrpcSlowConsumerPolicy.DROP_OLDEST),
sent::add,
nextSize::get);
payload -> nextSize.get());
nextSize.set(900L);
assertThat(writer.write(GrpcStreamEnvelope.Kind.LIVE, "large", null, null, true))
@@ -149,7 +150,7 @@ class GrpcSerializedStreamWriterTest {
GrpcStreamSequence.forNewStream(STREAM),
new GrpcFlowControlPolicy(10, 700L, 9, GrpcSlowConsumerPolicy.DROP_OLDEST),
sent::add,
nextSize::get);
payload -> nextSize.get());
writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true);
writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true);
@@ -179,7 +180,7 @@ class GrpcSerializedStreamWriterTest {
void cancellingKeepsTheLastFlushedPositionAsEvidence() {
GrpcSerializedStreamWriter<String> writer = writer(GrpcFlowControlPolicy.stable(), 8L);
writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true);
writer.flush();
writer.flush(() -> true);
writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true);
long lastFlushed = writer.discardQueued();
@@ -189,4 +190,54 @@ class GrpcSerializedStreamWriterTest {
assertThat(writer.queuedMessages()).isZero();
assertThat(sent).hasSize(1);
}
@Test
void theByteBoundCountsEachPayloadsOwnSize() {
// given
GrpcSerializedStreamWriter<String> writer =
new GrpcSerializedStreamWriter<>(
STREAM,
GrpcStreamSequence.forNewStream(STREAM),
new GrpcFlowControlPolicy(10, 10L, 9, GrpcSlowConsumerPolicy.TERMINATE),
sent::add,
payload -> payload.length());
writer.write(GrpcStreamEnvelope.Kind.LIVE, "eight-by", null, null, true);
// when
GrpcStreamWriteResult result =
writer.write(GrpcStreamEnvelope.Kind.LIVE, "four", null, null, true);
// then
assertThat(result).isEqualTo(GrpcStreamWriteResult.OVERFLOW_TERMINATING);
}
@Test
void aWriteWhileTheTransportIsNotReadyTellsTheProducerToPause() {
// given
GrpcSerializedStreamWriter<String> writer = writer(GrpcFlowControlPolicy.stable(), 8L);
// when
GrpcStreamWriteResult result =
writer.write(GrpcStreamEnvelope.Kind.LIVE, "row", null, null, false);
// then
assertThat(result)
.extracting(r -> r, GrpcStreamWriteResult::queued)
.containsExactly(GrpcStreamWriteResult.ACCEPTED_PAUSE, true);
}
@Test
void flushStopsWhenTheTransportStopsBeingReady() {
// given
GrpcSerializedStreamWriter<String> writer = writer(GrpcFlowControlPolicy.stable(), 8L);
writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true);
writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true);
AtomicInteger readyChecks = new AtomicInteger();
// when
List<Long> flushed = writer.flush(() -> readyChecks.getAndIncrement() == 0);
// then
assertThat(flushed).containsExactly(1L);
}
}
@@ -45,4 +45,17 @@ class GrpcStreamAdmissionConcurrencyTest {
assertThat(admission.trackedCallers()).isZero();
}
}
@Test
void aCallerRefusedByTheGlobalBoundLeavesNoTrackedEntry() {
// given
GrpcStreamAdmission admission = new GrpcStreamAdmission(1, 1);
admission.tryAdmit("first");
// when
admission.tryAdmit("refused");
// then
assertThat(admission.trackedCallers()).isOne();
}
}
@@ -1,7 +1,5 @@
package dev.caskeleton.grpc.server;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Refuses work the server cannot do, before it spends anything on it.
*
@@ -14,9 +12,12 @@ public final class GrpcAdmissionController {
private final int maxConcurrentCalls;
private final int maxQueuedCalls;
private final AtomicInteger inFlight = new AtomicInteger();
private final AtomicInteger queued = new AtomicInteger();
private final AtomicInteger rejected = new AtomicInteger();
// One lock for both counters: every decision reads one and changes the other, and a check on an
// atomic followed by an increment lets racing callers all pass the same check.
private int inFlight;
private int queued;
private int rejected;
/** Bounds concurrency and queueing separately. */
public GrpcAdmissionController(int maxConcurrentCalls, int maxQueuedCalls) {
@@ -48,56 +49,54 @@ public final class GrpcAdmissionController {
}
/** Admits a call, queues it, or refuses it. */
public Decision tryAdmit() {
int running = inFlight.get();
if (running < maxConcurrentCalls) {
inFlight.incrementAndGet();
return new Decision(true, inFlight.get(), queued.get(), "within the concurrency bound");
public synchronized Decision tryAdmit() {
if (inFlight < maxConcurrentCalls) {
inFlight++;
return new Decision(true, inFlight, queued, "within the concurrency bound");
}
int waiting = queued.get();
if (waiting < maxQueuedCalls) {
queued.incrementAndGet();
return new Decision(true, running, queued.get(), "queued within the admission bound");
if (queued < maxQueuedCalls) {
queued++;
return new Decision(true, inFlight, queued, "queued within the admission bound");
}
rejected.incrementAndGet();
rejected++;
return new Decision(
false,
running,
waiting,
inFlight,
queued,
"at capacity: "
+ running
+ inFlight
+ " in flight and "
+ waiting
+ queued
+ " queued. Refusing is better than accepting work whose caller will have given up.");
}
/** Moves a queued call into execution. */
public void promoteFromQueue() {
if (queued.get() > 0) {
queued.decrementAndGet();
inFlight.incrementAndGet();
public synchronized void promoteFromQueue() {
if (queued > 0) {
queued--;
inFlight++;
}
}
/** Releases a finished call. */
public void release() {
if (inFlight.get() > 0) {
inFlight.decrementAndGet();
public synchronized void release() {
if (inFlight > 0) {
inFlight--;
}
}
/** How many calls are executing. */
public int inFlight() {
return inFlight.get();
public synchronized int inFlight() {
return inFlight;
}
/** How many calls are waiting. */
public int queued() {
return queued.get();
public synchronized int queued() {
return queued;
}
/** How many calls have been refused. */
public int rejected() {
return rejected.get();
public synchronized int rejected() {
return rejected;
}
}
@@ -0,0 +1,55 @@
package dev.caskeleton.grpc.server;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.junit.jupiter.api.Test;
class GrpcAdmissionControllerTest {
private static final int CALLERS = 8;
private static final int ROUNDS = 500;
@Test
void racingCallersNeverGetPastTheConcurrencyAndQueueBounds() throws Exception {
// when
int mostAdmitted = mostAdmittedOverRounds();
// then
assertThat(mostAdmitted).isEqualTo(2);
}
/** Races {@link #CALLERS} callers at a 1-running, 1-queued controller, round after round. */
private static int mostAdmittedOverRounds() throws Exception {
ExecutorService pool = Executors.newFixedThreadPool(CALLERS);
try {
int most = 0;
for (int round = 0; round < ROUNDS; round++) {
GrpcAdmissionController controller = new GrpcAdmissionController(1, 1);
CyclicBarrier start = new CyclicBarrier(CALLERS);
List<Future<Boolean>> admitted = new ArrayList<>();
for (int caller = 0; caller < CALLERS; caller++) {
admitted.add(
pool.submit(
() -> {
start.await();
return controller.tryAdmit().admitted();
}));
}
int count = 0;
for (Future<Boolean> result : admitted) {
count += result.get() ? 1 : 0;
}
most = Math.max(most, count);
}
return most;
} finally {
pool.shutdownNow();
}
}
}