From 2c612cb95b7e0d9a8905313ba6462012eae975da Mon Sep 17 00:00:00 2001 From: DongHyeonka Date: Mon, 28 Sep 2026 14:46:30 +0900 Subject: [PATCH] =?UTF-8?q?review-followup=20Wave=202:=20=EC=A0=95?= =?UTF-8?q?=ED=95=A9=EC=84=B1=20=EC=88=98=EC=A0=95=20=ED=86=B5=ED=95=A9?= =?UTF-8?q?=EA=B3=BC=20=EA=B2=80=EC=A6=9D,=20=ED=98=84=ED=99=A9=ED=91=9C?= =?UTF-8?q?=C2=B7=EC=9E=AC=EA=B0=9C=20=ED=94=84=EB=A1=AC=ED=94=84=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 13개 작업 묶음(P1–P13)의 W2 커밋을 합치고 전체 게이트로 검증했다. - 묶음별 정합성 수정(notification dispatch·recovery, object storage staged publication, fileserver 복구·정리, gRPC 플랫폼, Redis 분류·lease TTL, JPA outbox claim 순서·durable operation HTTP, web 요청 id·category 필수, graphql 에러 mapper, Mongo profile 바인딩, build 소스 보존 게이트·compose smoke info 확인) - 통합 때 드러난 3건 수정: outbox claim 테스트 fixture의 가짜 envelope hash, sample scan이 새 operation controller를 등록하던 문제, SpotBugs aux classpath 파일이 truncate되지 않아 로컬에서 ".jar.jar"로 실패하던 문제 - 현황표: 묶음 결과 반영, 소유자 결정 21–27, 후속 발견 결함 47행, 요약 갱신 - docs/capability-status.md 「제거된 코드」에 85행 추가 - 묶음 결과 파일·에이전트 지시·스크립트를 docs로 옮기고 desktop 재개 프롬프트 추가 검증: check, ci verifyDependencyLocks, mainCheck, architecture/public path/env keys/config contract, bootstrap·sample integration/system, build-logic·build-tools test, optional-platforms ci, qualificationBootJar 통과. integrationCheck는 이번 작업 전에도 실패하던 lane(Redis topology 모드 인자, httpclient failure injection, AWS 보호 lane)만 실패. Co-Authored-By: Claude Opus 5.5 (1M context) --- build.gradle | 4 +- .../src/main/resources/envoy/envoy.yaml | 16 +++- .../compat/GrpcWebCompatibilityTest.java | 56 ++++++++++++ .../grpc/admin/GrpcServiceHealthRegistry.java | 14 +-- .../admin/GrpcServiceHealthRegistryTest.java | 48 ++++++++++ .../client/GrpcChannelRuntimeRegistry.java | 20 ++-- .../GrpcChannelRuntimeRegistryTest.java | 54 +++++++++++ .../grpc/client/GrpcTypedStubFactoryTest.java | 6 +- .../grpc/api/deadline/GrpcDeadlineBudget.java | 81 +++++++++++++---- .../api/ledger/GrpcOperationIdentity.java | 25 +++++ .../api/deadline/GrpcDeadlineBudgetTest.java | 51 ++++++++++- .../api/ledger/GrpcOperationIdentityTest.java | 44 +++++++++ .../grpc/ledger/JpaGrpcOperationLedger.java | 14 ++- .../GrpcOperationLedgerRepositoryTest.java | 31 ++++++- .../grpc/context/GrpcContextBinder.java | 11 +-- .../grpc/resilience/GrpcRetryCoordinator.java | 6 +- .../resilience/GrpcWaitForReadyValidator.java | 8 +- .../GrpcCredentialRotationManager.java | 40 ++++---- .../streaming/GrpcSerializedStreamWriter.java | 39 ++++---- .../grpc/streaming/GrpcStreamAdmission.java | 7 +- .../grpc/streaming/GrpcStreamWriteResult.java | 8 +- .../grpc/context/GrpcContextBinderTest.java | 27 ++++++ .../GrpcWaitForReadyValidatorTest.java | 2 +- .../GrpcCredentialRotationManagerTest.java | 91 +++++++++++++++++++ .../GrpcSerializedStreamWriterTest.java | 63 +++++++++++-- .../GrpcStreamAdmissionConcurrencyTest.java | 13 +++ .../grpc/server/GrpcAdmissionController.java | 63 +++++++------ .../server/GrpcAdmissionControllerTest.java | 55 +++++++++++ 28 files changed, 763 insertions(+), 134 deletions(-) create mode 100644 grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java create mode 100644 grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java create mode 100644 grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java diff --git a/build.gradle b/build.gradle index 001bb2d..e91af3c 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,9 @@ tasks.register('ci') { description = 'Runs the optional gRPC platform verification surface.' dependsOn tasks.named('checkAll') dependsOn tasks.named('qualityCheck') - dependsOn tasks.named('verifyCleanArchitectureDependencies') + // architectureCheck, not only the direction gate: it also runs verifyNoIgnoredSourcePackages, so + // a gRPC source Git would drop fails this build's ci the way it fails the main build's. + dependsOn tasks.named('architectureCheck') } // A strict integration lane that no workflow invokes is not coverage; it is a file that looks like diff --git a/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml b/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml index 0c5ec86..ae3cab3 100644 --- a/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml +++ b/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml @@ -29,13 +29,18 @@ static_resources: - match: { prefix: "/" } route: cluster: grpc_backend - timeout: 30s + # No route timeout: it would cut every server stream at that age. The + # caller's grpc-timeout header bounds each call instead. + timeout: 0s + max_stream_duration: + grpc_timeout_header_max: 0s cors: allow_origin_string_match: - exact: "https://app.example.com" allow_methods: "POST,OPTIONS" - allow_headers: "content-type,x-grpc-web,x-correlation-id,authorization" - expose_headers: "grpc-status,grpc-message,error-code,error-category" + allow_headers: "content-type,x-grpc-web,x-user-agent,grpc-timeout,x-correlation-id,authorization" + # Every trailer the platform sends (GrpcErrorMapper, GrpcValidationInterceptor). + expose_headers: "grpc-status,grpc-message,grpc-status-details-bin,error-reason,error-execution-id,completion-outcome,validation-violations" max_age: "1728000" http_filters: - name: envoy.filters.http.grpc_web @@ -51,6 +56,11 @@ static_resources: name: envoy.transport_sockets.tls typed_config: "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + # Placeholder paths: mount the proxy's certificate and key here. + common_tls_context: + tls_certificates: + - certificate_chain: { filename: /etc/envoy/tls/tls.crt } + private_key: { filename: /etc/envoy/tls/tls.key } clusters: - name: grpc_backend connect_timeout: 1s diff --git a/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java b/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java index 0a215f8..a34f687 100644 --- a/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java +++ b/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java @@ -7,11 +7,15 @@ import dev.caskeleton.grpc.advanced.web.GrpcWebProfile; import dev.caskeleton.grpc.advanced.web.GrpcWebProxyContract; import dev.caskeleton.grpc.advanced.web.GrpcWebRpcSupport; import dev.caskeleton.grpc.core.RpcType; +import dev.caskeleton.grpc.error.GrpcErrorMapper; +import dev.caskeleton.grpc.validation.GrpcValidationInterceptor; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -110,6 +114,58 @@ class GrpcWebCompatibilityTest { .contains("envoy.filters.http.grpc_web"); } + @Test + void theReferenceProxyExposesEveryTrailerThePlatformSends() { + // when + Set exposed = headerList(resource("envoy/envoy.yaml"), "expose_headers"); + + // then + assertThat(exposed) + .contains( + "grpc-status", + "grpc-message", + "grpc-status-details-bin", + GrpcErrorMapper.REASON_KEY.name(), + GrpcErrorMapper.EXECUTION_ID_KEY.name(), + GrpcErrorMapper.COMPLETION_OUTCOME_KEY.name(), + GrpcValidationInterceptor.VIOLATIONS_KEY.name()); + } + + @Test + void theReferenceProxyLetsTheBrowserSendItsDeadlineAndClientHeaders() { + // when + Set allowed = headerList(resource("envoy/envoy.yaml"), "allow_headers"); + + // then + assertThat(allowed).contains("grpc-timeout", "x-user-agent", "x-grpc-web", "content-type"); + } + + @Test + void theReferenceRouteLeavesStreamLengthToTheCallersDeadline() { + // when + String envoy = resource("envoy/envoy.yaml"); + + // then + assertThat(envoy).contains("timeout: 0s").contains("grpc_timeout_header_max: 0s"); + } + + @Test + void theReferenceProxyTerminatesTlsWithACertificate() { + // when + String envoy = resource("envoy/envoy.yaml"); + + // then + assertThat(envoy).contains("certificate_chain:").contains("private_key:"); + } + + private static Set headerList(String envoy, String field) { + Matcher matcher = Pattern.compile(field + ": \"([^\"]*)\"").matcher(envoy); + if (!matcher.find()) { + throw new IllegalStateException("the reference proxy has no " + field); + } + return Set.of(matcher.group(1).split(",")); + } + private static String resource(String path) { try (InputStream stream = GrpcWebCompatibilityTest.class.getClassLoader().getResourceAsStream(path)) { diff --git a/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java b/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java index ee4ad40..4be13bd 100644 --- a/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java +++ b/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java @@ -22,7 +22,9 @@ public final class GrpcServiceHealthRegistry { private final ConcurrentMap states = new ConcurrentHashMap<>(); private final ConcurrentMap dependencyHealth = new ConcurrentHashMap<>(); private final GrpcHealthPolicy policy; - private volatile boolean draining; + // Every write holds this registry's lock, so a state change cannot land between the draining + // check and its write and undo a drain that started in between. + private boolean draining; /** Binds a registry to its health policy. */ public GrpcServiceHealthRegistry(GrpcHealthPolicy policy) { @@ -34,7 +36,7 @@ public final class GrpcServiceHealthRegistry { } /** Registers a service, unknown until something says otherwise. */ - public void register(String serviceName) { + public synchronized void register(String serviceName) { requireServiceName(serviceName); states.putIfAbsent(serviceName, GrpcHealthState.UNKNOWN); } @@ -45,7 +47,7 @@ public final class GrpcServiceHealthRegistry { *

Ignored while draining: a service that reports itself healthy after the drain has started * would be routed traffic the instance has already promised not to take. */ - public void markServing(String serviceName) { + public synchronized void markServing(String serviceName) { requireServiceName(serviceName); if (draining) { return; @@ -55,7 +57,7 @@ public final class GrpcServiceHealthRegistry { } /** Records that a service is not serving. */ - public void markNotServing(String serviceName) { + public synchronized void markNotServing(String serviceName) { requireServiceName(serviceName); if (draining) { return; @@ -65,7 +67,7 @@ public final class GrpcServiceHealthRegistry { } /** Records a dependency's health, which only matters when the policy says it does. */ - public void recordDependencyHealth(String dependencyName, boolean healthy) { + public synchronized void recordDependencyHealth(String dependencyName, boolean healthy) { if (dependencyName == null || dependencyName.isBlank()) { throw new IllegalArgumentException("a dependency needs a name"); } @@ -81,7 +83,7 @@ public final class GrpcServiceHealthRegistry { *

Called first in a shutdown, before admission stops, so that a load balancer has a chance to * stop sending before the server starts refusing. */ - public void beginDraining() { + public synchronized void beginDraining() { draining = true; states.replaceAll((service, state) -> GrpcHealthState.DRAINING); } diff --git a/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java b/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java index d85f7c3..7454b5a 100644 --- a/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java +++ b/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java @@ -4,6 +4,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -88,4 +93,47 @@ class GrpcServiceHealthRegistryTest { registry.markServing(DOCUMENTS); assertThat(registry.stateOf(DOCUMENTS)).isEqualTo(GrpcHealthState.DRAINING); } + + @Test + void aServiceMarkedServingWhileDrainBeginsNeverReturnsTheInstanceToServing() throws Exception { + // when + Set outcomes = outcomesOfServingRacingDrain(); + + // then + assertThat(outcomes).containsExactly("DRAINING/DRAINING"); + } + + /** Races markServing against beginDraining, round after round, recording service/global. */ + private static Set outcomesOfServingRacingDrain() throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Set outcomes = new TreeSet<>(); + for (int round = 0; round < 1000; round++) { + GrpcServiceHealthRegistry registry = + new GrpcServiceHealthRegistry(GrpcHealthPolicy.standalone()); + registry.register(DOCUMENTS); + CyclicBarrier start = new CyclicBarrier(2); + Future serving = + pool.submit( + () -> { + start.await(); + registry.markServing(DOCUMENTS); + return null; + }); + Future drain = + pool.submit( + () -> { + start.await(); + registry.beginDraining(); + return null; + }); + serving.get(); + drain.get(); + outcomes.add(registry.stateOf(DOCUMENTS) + "/" + registry.globalState()); + } + return outcomes; + } finally { + pool.shutdownNow(); + } + } } diff --git a/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java b/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java index b25dec2..188ad5f 100644 --- a/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java +++ b/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java @@ -3,7 +3,9 @@ package dev.caskeleton.grpc.client; import dev.caskeleton.grpc.core.GrpcChannelProfileName; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -24,8 +26,9 @@ public final class GrpcChannelRuntimeRegistry { private final ConcurrentMap> current = new ConcurrentHashMap<>(); - private final ConcurrentMap> draining = - new ConcurrentHashMap<>(); + // Guarded by this registry's lock, like every swap: rotations are rare, and one lock keeps the + // current pointer and the draining list from disagreeing. + private final Map> draining = new HashMap<>(); private final GrpcChannelDrainPolicy drainPolicy; /** Binds a registry to the drain policy its swaps use. */ @@ -37,7 +40,7 @@ public final class GrpcChannelRuntimeRegistry { } /** Installs the first generation for a profile. */ - public GrpcChannelRuntime install(GrpcChannelGeneration generation) { + public synchronized GrpcChannelRuntime install(GrpcChannelGeneration generation) { if (generation == null) { throw new IllegalArgumentException("a generation is required"); } @@ -80,7 +83,7 @@ public final class GrpcChannelRuntimeRegistry { * * @throws IllegalArgumentException when {@code next} does not supersede what is installed */ - public GrpcChannelRuntime rotate(GrpcChannelGeneration next, Instant now) { + public synchronized GrpcChannelRuntime rotate(GrpcChannelGeneration next, Instant now) { if (next == null || now == null) { throw new IllegalArgumentException("a rotation needs a generation and a moment"); } @@ -102,15 +105,12 @@ public final class GrpcChannelRuntimeRegistry { GrpcChannelRuntime replacement = new GrpcChannelRuntime(next); holder.set(replacement); previous.beginDrain(); - draining - .computeIfAbsent( - next.profileName(), key -> java.util.Collections.synchronizedList(new ArrayList<>())) - .add(previous); + draining.computeIfAbsent(next.profileName(), key -> new ArrayList<>()).add(previous); return replacement; } /** The generations still draining for a profile. */ - public List draining(GrpcChannelProfileName profileName) { + public synchronized List draining(GrpcChannelProfileName profileName) { return List.copyOf(draining.getOrDefault(profileName, List.of())); } @@ -119,7 +119,7 @@ public final class GrpcChannelRuntimeRegistry { * * @return how many were retired */ - public int retireQuiescent(GrpcChannelProfileName profileName) { + public synchronized int retireQuiescent(GrpcChannelProfileName profileName) { List runtimes = draining.get(profileName); if (runtimes == null) { return 0; diff --git a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java index 796e4a1..b1ceb3e 100644 --- a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java +++ b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java @@ -6,6 +6,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import dev.caskeleton.grpc.core.GrpcChannelProfileName; import java.time.Duration; import java.time.Instant; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -125,4 +131,52 @@ class GrpcChannelRuntimeRegistryTest { .extracting(Class::getName) .noneMatch(name -> name.startsWith("io.grpc.")); } + + @Test + void racingRotationsDrainEveryReplacedRuntimeAndKeepTheNewestCurrent() throws Exception { + // when + Set outcomes = outcomesOfRacingRotations(); + + // then + assertThat(outcomes).isSubsetOf("3 draining [1]", "3 draining [1, 2]"); + } + + /** Races rotate(2) against rotate(3) on a profile at generation 1, round after round. */ + private static Set outcomesOfRacingRotations() throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Set outcomes = new TreeSet<>(); + for (int round = 0; round < 1000; round++) { + GrpcChannelRuntimeRegistry registry = + new GrpcChannelRuntimeRegistry(GrpcChannelDrainPolicy.stable()); + registry.install(generation(1)); + CyclicBarrier start = new CyclicBarrier(2); + Future a = pool.submit(() -> rotateIgnoringRefusal(registry, start, 2)); + Future b = pool.submit(() -> rotateIgnoringRefusal(registry, start, 3)); + a.get(); + b.get(); + outcomes.add( + registry.require(PROFILE).generation().generation() + + " draining " + + registry.draining(PROFILE).stream() + .map(runtime -> runtime.generation().generation()) + .sorted() + .toList()); + } + return outcomes; + } finally { + pool.shutdownNow(); + } + } + + private static Void rotateIgnoringRefusal( + GrpcChannelRuntimeRegistry registry, CyclicBarrier start, long next) throws Exception { + start.await(); + try { + registry.rotate(generation(next), NOW); + } catch (IllegalArgumentException refused) { + // the other rotator installed a newer generation first + } + return null; + } } diff --git a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java index 8203a61..15cdf29 100644 --- a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java +++ b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java @@ -122,8 +122,10 @@ class GrpcTypedStubFactoryTest { GrpcStubPolicyApplier.CallPolicy generous = applier.applyTo(GET, Duration.ofSeconds(30)); GrpcStubPolicyApplier.CallPolicy tight = applier.applyTo(GET, Duration.ofMillis(300)); - assertThat(generous.deadline().remaining()).isEqualTo(Duration.ofMillis(1800)); - assertThat(tight.deadline().remaining()).isEqualTo(Duration.ofMillis(100)); + // The budget is live: a few milliseconds have passed since applyTo computed it. + assertThat(generous.deadline().remaining()) + .isBetween(Duration.ofMillis(1700), Duration.ofMillis(1800)); + assertThat(tight.deadline().remaining()).isBetween(Duration.ZERO, Duration.ofMillis(100)); assertThat(tight.waitForReady().queue()).isFalse(); } } diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java index b41258a..02a8cbb 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java @@ -1,6 +1,7 @@ package dev.caskeleton.grpc.api.deadline; import java.time.Duration; +import java.util.function.LongSupplier; /** * How much time a call actually has, after the parent deadline and the method profile have both had @@ -9,17 +10,40 @@ import java.time.Duration; *

The rule is "shorter of the two, minus the reserve", and it is here rather than at each call * site because the failure mode of getting it wrong is silent: a downstream timeout longer than the * inbound deadline never fires, and the caller sees a deadline the server has no idea it missed. + * + *

The budget is an absolute deadline on a monotonic nanosecond clock, so {@link #remaining()} + * shrinks and {@link #expired()} turns true as time passes. A request context carries one budget + * for the whole call; a budget that remembered only the duration it started with would hand every + * later dependency call the full original time. */ -public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile) { +public final class GrpcDeadlineBudget { - /** Rejects a budget with no time in it. */ - public GrpcDeadlineBudget { - if (remaining == null || profile == null) { - throw new IllegalArgumentException("a deadline budget needs a remaining time and a profile"); + private final long deadlineNanos; + private final GrpcDeadlineProfile profile; + private final LongSupplier nanoClock; + + /** A budget of {@code remaining} from now on the system's monotonic clock. */ + public GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile) { + this(remaining, profile, System::nanoTime); + } + + /** + * A budget of {@code remaining} from now on {@code nanoClock}. + * + * @param nanoClock a monotonic nanosecond source, {@code System::nanoTime} outside tests + */ + public GrpcDeadlineBudget( + Duration remaining, GrpcDeadlineProfile profile, LongSupplier nanoClock) { + if (remaining == null || profile == null || nanoClock == null) { + throw new IllegalArgumentException( + "a deadline budget needs a remaining time, a profile and a clock"); } if (remaining.isNegative()) { throw new IllegalArgumentException("remaining time must not be negative; got " + remaining); } + this.deadlineNanos = nanoClock.getAsLong() + remaining.toNanos(); + this.profile = profile; + this.nanoClock = nanoClock; } /** @@ -31,6 +55,12 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile */ public static GrpcDeadlineBudget forEntryPoint( Duration parentRemaining, GrpcDeadlineProfile profile) { + return forEntryPoint(parentRemaining, profile, System::nanoTime); + } + + /** {@link #forEntryPoint(Duration, GrpcDeadlineProfile)} on {@code nanoClock}. */ + public static GrpcDeadlineBudget forEntryPoint( + Duration parentRemaining, GrpcDeadlineProfile profile, LongSupplier nanoClock) { if (profile == null) { throw new IllegalArgumentException("a deadline budget needs a profile"); } @@ -42,17 +72,13 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile throw new IllegalArgumentException( "a Stable unary call requires a positive deadline; got " + parentRemaining); } - Duration bounded = - parentRemaining.compareTo(profile.total()) <= 0 ? parentRemaining : profile.total(); - Duration afterReserve = bounded.minus(profile.safetyMargin()); - return new GrpcDeadlineBudget( - afterReserve.isNegative() ? Duration.ZERO : afterReserve, profile); + return new GrpcDeadlineBudget(afterReserve(parentRemaining, profile), profile, nanoClock); } /** * Derives the budget for a downstream dependency call under {@code downstream}. * - *

Takes the shorter of what is left here and what the downstream profile allows, then + *

Takes the shorter of what is left here now and what the downstream profile allows, then * subtracts the downstream reserve. A dependency can never be given more time than its caller * has. */ @@ -60,11 +86,18 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile if (downstream == null) { throw new IllegalArgumentException("a downstream budget needs a profile"); } - Duration bounded = - remaining.compareTo(downstream.total()) <= 0 ? remaining : downstream.total(); - Duration afterReserve = bounded.minus(downstream.safetyMargin()); - return new GrpcDeadlineBudget( - afterReserve.isNegative() ? Duration.ZERO : afterReserve, downstream); + return new GrpcDeadlineBudget(afterReserve(remaining(), downstream), downstream, nanoClock); + } + + /** What is left now; zero once the deadline has passed. */ + public Duration remaining() { + long left = deadlineNanos - nanoClock.getAsLong(); + return left <= 0 ? Duration.ZERO : Duration.ofNanos(left); + } + + /** The profile this budget was cut from. */ + public GrpcDeadlineProfile profile() { + return profile; } /** @@ -75,11 +108,23 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile * investigates the wrong system. */ public boolean canStartDependencyCall() { - return remaining.compareTo(profile.minimumAttemptBudget()) >= 0 && !remaining.isZero(); + Duration left = remaining(); + return !left.isZero() && left.compareTo(profile.minimumAttemptBudget()) >= 0; } /** Whether any time is left at all. */ public boolean expired() { - return remaining.isZero() || remaining.isNegative(); + return remaining().isZero(); + } + + @Override + public String toString() { + return "GrpcDeadlineBudget[remaining=" + remaining() + ", profile=" + profile + "]"; + } + + private static Duration afterReserve(Duration available, GrpcDeadlineProfile profile) { + Duration bounded = available.compareTo(profile.total()) <= 0 ? available : profile.total(); + Duration afterReserve = bounded.minus(profile.safetyMargin()); + return afterReserve.isNegative() ? Duration.ZERO : afterReserve; } } diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java index b4dab91..bfd5bec 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java @@ -16,6 +16,13 @@ import dev.caskeleton.grpc.core.GrpcMethodName; public record GrpcOperationIdentity( String callerFingerprint, GrpcMethodName method, String idempotencyKeyHash) { + // Column widths of grpc_operation_ledger (V001). An identity that does not fit is refused here, + // not by the database in the middle of a claim. + private static final int MAX_CALLER_FINGERPRINT = 256; + private static final int MAX_METHOD = 256; + private static final int MAX_KEY_HASH = 80; + private static final int MAX_STORAGE_KEY = 512; + /** Requires all three parts, with the key already hashed. */ public GrpcOperationIdentity { if (callerFingerprint == null || callerFingerprint.isBlank()) { @@ -30,6 +37,24 @@ public record GrpcOperationIdentity( throw new IllegalArgumentException( "the idempotency key is stored hashed; got '" + idempotencyKeyHash + "'"); } + requireAtMost("caller fingerprint", callerFingerprint, MAX_CALLER_FINGERPRINT); + requireAtMost("full method name", method.canonical(), MAX_METHOD); + requireAtMost("idempotency key hash", idempotencyKeyHash, MAX_KEY_HASH); + requireAtMost( + "storage key", + callerFingerprint + "|" + method.canonical() + "|" + idempotencyKeyHash, + MAX_STORAGE_KEY); + } + + private static void requireAtMost(String what, String value, int max) { + if (value.length() > max) { + throw new IllegalArgumentException( + what + + " must be at most " + + max + + " characters to fit the ledger; got " + + value.length()); + } } /** The stable storage key: the three parts joined, safe to index and safe to log. */ diff --git a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java index 58fd57a..a23bc99 100644 --- a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java +++ b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.time.Duration; import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -27,9 +28,11 @@ class GrpcDeadlineBudgetTest { @DisplayName("the budget takes the shorter of the parent remaining and the method default") void budgetTakesTheShorterOfParentAndMethodDefault() { GrpcDeadlineBudget fromShortParent = - GrpcDeadlineBudget.forEntryPoint(Duration.ofMillis(500), TWO_SECONDS); + GrpcDeadlineBudget.forEntryPoint( + Duration.ofMillis(500), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos); GrpcDeadlineBudget fromLongParent = - GrpcDeadlineBudget.forEntryPoint(Duration.ofSeconds(30), TWO_SECONDS); + GrpcDeadlineBudget.forEntryPoint( + Duration.ofSeconds(30), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos); assertThat(fromShortParent.remaining()) .isEqualTo(Duration.ofMillis(500).minus(TWO_SECONDS.safetyMargin())); @@ -44,7 +47,9 @@ class GrpcDeadlineBudgetTest { new GrpcDeadlineProfile( Duration.ofSeconds(10), Duration.ofSeconds(1), Duration.ofMillis(100)); - GrpcDeadlineBudget budget = GrpcDeadlineBudget.forEntryPoint(Duration.ofSeconds(10), profile); + GrpcDeadlineBudget budget = + GrpcDeadlineBudget.forEntryPoint( + Duration.ofSeconds(10), profile, GrpcDeadlineBudgetTest::frozenNanos); assertThat(budget.remaining()).isEqualTo(Duration.ofSeconds(9)); assertThat(profile.usable()).isEqualTo(Duration.ofSeconds(9)); @@ -54,7 +59,8 @@ class GrpcDeadlineBudgetTest { @DisplayName("a downstream call never gets more time than its caller has left") void downstreamNeverExceedsTheCaller() { GrpcDeadlineBudget caller = - GrpcDeadlineBudget.forEntryPoint(Duration.ofMillis(400), TWO_SECONDS); + GrpcDeadlineBudget.forEntryPoint( + Duration.ofMillis(400), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos); GrpcDeadlineProfile generousDownstream = new GrpcDeadlineProfile(Duration.ofSeconds(30), Duration.ZERO, Duration.ofMillis(10)); @@ -77,6 +83,39 @@ class GrpcDeadlineBudgetTest { assertThat(new GrpcDeadlineBudget(Duration.ZERO, profile).canStartDependencyCall()).isFalse(); } + @Test + void aBudgetRunsOutAsTimePassesAfterItWasCreated() { + // given + AtomicLong nanos = new AtomicLong(); + GrpcDeadlineBudget budget = + new GrpcDeadlineBudget(Duration.ofMillis(300), TWO_SECONDS, nanos::get); + + // when + nanos.addAndGet(Duration.ofMillis(300).toNanos()); + + // then + assertThat(budget) + .extracting(GrpcDeadlineBudget::remaining, GrpcDeadlineBudget::expired) + .containsExactly(Duration.ZERO, true); + } + + @Test + void aDownstreamDerivedLaterGetsOnlyWhatIsLeftThen() { + // given + AtomicLong nanos = new AtomicLong(); + GrpcDeadlineBudget caller = + new GrpcDeadlineBudget(Duration.ofSeconds(1), TWO_SECONDS, nanos::get); + nanos.addAndGet(Duration.ofMillis(700).toNanos()); + + // when + GrpcDeadlineBudget downstream = + caller.deriveDownstream( + new GrpcDeadlineProfile(Duration.ofSeconds(30), Duration.ZERO, Duration.ofMillis(10))); + + // then + assertThat(downstream.remaining()).isEqualTo(Duration.ofMillis(300)); + } + @Test @DisplayName("a profile whose reserve leaves no usable time is refused") void profileWithNoUsableTimeIsRefused() { @@ -121,4 +160,8 @@ class GrpcDeadlineBudgetTest { .hasMessageContaining("charge-card") .hasMessageContaining("deadline-exceeded"); } + + private static long frozenNanos() { + return 0L; + } } diff --git a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java new file mode 100644 index 0000000..c210e21 --- /dev/null +++ b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java @@ -0,0 +1,44 @@ +package dev.caskeleton.grpc.api.ledger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.grpc.core.GrpcMethodName; +import dev.caskeleton.grpc.core.GrpcServiceName; +import org.junit.jupiter.api.Test; + +class GrpcOperationIdentityTest { + + private static final GrpcMethodName CREATE = + GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"); + private static final String KEY_HASH = + "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; + + @Test + void aCallerFingerprintWiderThanItsLedgerColumnIsRefused() { + // when + Throwable failure = + catchThrowable(() -> new GrpcOperationIdentity("c".repeat(257), CREATE, KEY_HASH)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("caller fingerprint must be at most 256"); + } + + @Test + void anIdentityWhoseStorageKeyWouldNotFitTheLedgerIsRefused() { + // given + GrpcMethodName longMethod = + new GrpcMethodName(new GrpcServiceName("p.S" + "s".repeat(197)), "M" + "m".repeat(54)); + + // when + Throwable failure = + catchThrowable(() -> new GrpcOperationIdentity("c".repeat(256), longMethod, KEY_HASH)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("storage key must be at most 512"); + } +} diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 443847a..7cd1afc 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -50,7 +50,19 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { if (inserted != 0) { throw new IllegalStateException("claim insert affected an unexpected row count: " + inserted); } - return repository.findById(identity.storageKey()).map(entity -> entity.toRecord(identity)); + // Zero rows means another claim holds the key. An empty read now means that owner released it + // in between; treating that as "ours" would run the mutation without a claim row to guard it. + return Optional.of( + repository + .findById(identity.storageKey()) + .orElseThrow( + () -> + new IllegalStateException( + "claim for '" + + identity.storageKey() + + "' conflicted, but there was no row to read; the other owner released" + + " it in between. Retry the claim.")) + .toRecord(identity)); } @Override diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index e1457b5..84862d5 100644 --- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.ledger; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; @@ -40,7 +41,7 @@ class GrpcOperationLedgerRepositoryTest { * putIfAbsent} and therefore tested the database constraint we wished save() would hit rather * than the merge behavior it actually uses. */ - private static final class InMemoryRepository implements GrpcOperationLedgerRepository { + private static class InMemoryRepository implements GrpcOperationLedgerRepository { private final ConcurrentMap rows = new ConcurrentHashMap<>(); @Override @@ -124,6 +125,34 @@ class GrpcOperationLedgerRepositoryTest { assertThat(second.get().sameRequestAs(FINGERPRINT)).isTrue(); } + @Test + void aConflictWhoseRowVanishedBeforeTheReadIsNotTakenAsOwnership() { + // given + InMemoryRepository conflictThenReleased = + new InMemoryRepository() { + @Override + public int tryInsertClaim( + String storageKey, + String callerFingerprint, + String fullMethodName, + String idempotencyKeyHash, + String requestFingerprint, + Instant claimedAt) { + return 0; // another owner held the key and released it before this read + } + }; + GrpcOperationLedger racing = new JpaGrpcOperationLedger(conflictThenReleased); + + // when + Throwable failure = + catchThrowable(() -> racing.claim(identity("tenant-1.actor-1"), FINGERPRINT, NOW)); + + // then + assertThat(failure) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no row to read"); + } + @Test @DisplayName("one caller's key does not collide with another's") void callersDoNotCollide() { diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java index 5f0f67c..5eef3bd 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java @@ -87,15 +87,14 @@ public final class GrpcContextBinder { if (work == null) { throw new IllegalArgumentException("work must not be null"); } - GrpcContextSnapshot captured = SNAPSHOT.get(); - if (captured == null && policy.failClosedWithoutContext()) { + // The whole io.grpc Context, not just the snapshot value: it also carries the call's + // cancellation and deadline, which the other thread must see. + Context captured = Context.current(); + if (SNAPSHOT.get(captured) == null && policy.failClosedWithoutContext()) { throw new IllegalStateException( "refusing to hand work to another thread with no call context to carry"); } - if (captured == null) { - return work; - } - return () -> runWith(captured, work); + return captured.wrap(work); } /** An {@link Executor} that carries the submitting thread's context to every task. */ diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java index b967e90..5da90b5 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java @@ -78,13 +78,15 @@ public final class GrpcRetryCoordinator { } Duration backoff = jittered(config.backoffBefore(completedAttempt + 1), config.jitterFactor()); - if (remaining.remaining().compareTo(backoff) <= 0) { + // Read once: the budget is live, and a second read could fall below the backoff. + Duration left = remaining.remaining(); + if (left.compareTo(backoff) <= 0) { return GrpcRetryDecision.refuse( GrpcRetryDecision.Verdict.TERMINAL, "the remaining deadline is shorter than the backoff, so a retry could not finish"); } GrpcDeadlineBudget afterBackoff = - new GrpcDeadlineBudget(remaining.remaining().minus(backoff), remaining.profile()); + new GrpcDeadlineBudget(left.minus(backoff), remaining.profile()); if (!afterBackoff.canStartDependencyCall()) { return GrpcRetryDecision.refuse( GrpcRetryDecision.Verdict.TERMINAL, diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java index f41151d..6becd96 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java @@ -35,7 +35,9 @@ public final class GrpcWaitForReadyValidator { + policy.method().canonical() + "' must carry one"); } - if (remaining.expired()) { + // Read once: the budget is live and keeps shrinking between reads. + Duration left = remaining.remaining(); + if (left.isZero()) { return GrpcWaitForReadyDecision.failFast("no deadline remains to queue inside"); } if (policy.waitForReady() != profile.policy()) { @@ -47,9 +49,7 @@ public final class GrpcWaitForReadyValidator { + "; the stricter of the two wins"); } Duration allowance = - profile.maxQueueWait().compareTo(remaining.remaining()) <= 0 - ? profile.maxQueueWait() - : remaining.remaining(); + profile.maxQueueWait().compareTo(left) <= 0 ? profile.maxQueueWait() : left; return GrpcWaitForReadyDecision.queueFor( allowance, "queueing under " + profile.policy() + " within the queue budget"); } diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java index bcf033f..dc13a6c 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java @@ -86,22 +86,29 @@ public final class GrpcCredentialRotationManager { if (next == null || now == null) { throw new IllegalArgumentException("a rotation needs a generation and a moment"); } - State observed = state.get(); - if (observed.current().generation() == next.generation()) { - // Idempotent: re-applying the generation that is already current is a no-op rather than a - // second drain, because a retried rotation must not cancel the calls the first one admitted. - return new RotationPlan(observed.current(), Optional.empty(), Optional.empty()); - } - if (!observed.current().supersededBy(next)) { - throw new IllegalArgumentException( - "credential generation " - + next.generation() - + " does not supersede the current generation " - + observed.current().generation()); - } Instant deadline = now.plus(drainWindow); - state.set(new State(next, observed.current(), deadline)); - return new RotationPlan(next, Optional.of(observed.current()), Optional.of(deadline)); + // Compare-and-set loop: a racing rotator or completeDrain re-runs the checks against the state + // it actually replaces, so a rotation never goes backwards and never loses a draining + // generation. + while (true) { + State observed = state.get(); + if (observed.current().generation() == next.generation()) { + // Idempotent: re-applying the generation that is already current is a no-op rather than a + // second drain, because a retried rotation must not cancel the calls the first one + // admitted. + return new RotationPlan(observed.current(), Optional.empty(), Optional.empty()); + } + if (!observed.current().supersededBy(next)) { + throw new IllegalArgumentException( + "credential generation " + + next.generation() + + " does not supersede the current generation " + + observed.current().generation()); + } + if (state.compareAndSet(observed, new State(next, observed.current(), deadline))) { + return new RotationPlan(next, Optional.of(observed.current()), Optional.of(deadline)); + } + } } /** @@ -116,7 +123,6 @@ public final class GrpcCredentialRotationManager { /** Forgets the draining generation once its work has finished or been cancelled. */ public void completeDrain() { - State observed = state.get(); - state.set(new State(observed.current(), null, null)); + state.updateAndGet(observed -> new State(observed.current(), null, null)); } } diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java index 166638a..c949d2f 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java @@ -4,8 +4,9 @@ import java.util.ArrayDeque; import java.util.Deque; import java.util.List; import java.util.Optional; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; -import java.util.function.LongSupplier; +import java.util.function.ToLongFunction; /** * The single writer every server stream goes through. @@ -27,7 +28,7 @@ public final class GrpcSerializedStreamWriter { private final GrpcStreamSequence sequence; private final GrpcFlowControlPolicy flowControl; private final Consumer> transport; - private final LongSupplier payloadSizer; + private final ToLongFunction payloadSizer; private final Deque> queue = new ArrayDeque<>(); private GrpcStreamWriterState state = GrpcStreamWriterState.OPEN; @@ -37,15 +38,15 @@ public final class GrpcSerializedStreamWriter { /** * @param transport the single consumer that actually calls the observer. Invoked only from {@link - * #flush}, which is synchronized, so the observer sees one thread. - * @param payloadSizer the serialized size of the next payload, for the byte bound + * #flush} and {@link #terminate}, which are synchronized, so the observer sees one thread. + * @param payloadSizer the serialized size of a payload, for the byte bound */ public GrpcSerializedStreamWriter( GrpcStreamId streamId, GrpcStreamSequence sequence, GrpcFlowControlPolicy flowControl, Consumer> transport, - LongSupplier payloadSizer) { + ToLongFunction payloadSizer) { if (streamId == null || sequence == null || flowControl == null @@ -70,7 +71,7 @@ public final class GrpcSerializedStreamWriter { if (!state.acceptsMessages()) { return GrpcStreamWriteResult.REJECTED_CLOSED; } - long nextBytes = payloadSizer.getAsLong(); + long nextBytes = payloadSizer.applyAsLong(payload); GrpcFlowControlDecision decision = flowControl.decide(queue.size(), queuedBytes, nextBytes, transportReady); return switch (decision.action()) { @@ -97,7 +98,11 @@ public final class GrpcSerializedStreamWriter { } } } - case PAUSE, PROCEED -> { + case PAUSE -> { + enqueue(kind, payload, snapshotVersion, resumeToken, nextBytes); + yield GrpcStreamWriteResult.ACCEPTED_PAUSE; + } + case PROCEED -> { enqueue(kind, payload, snapshotVersion, resumeToken, nextBytes); yield GrpcStreamWriteResult.ACCEPTED; } @@ -119,20 +124,22 @@ public final class GrpcSerializedStreamWriter { } /** - * Hands every queued message to the transport, in order, from this one thread. + * Hands queued messages to the transport, in order, from this one thread, while it is ready. * + * @param transportReady asked before each message ({@code ServerCallStreamObserver::isReady}); + * flushing stops at the first false and the rest stay queued for the next onReady * @return the sequence numbers flushed */ - public synchronized List flush() { + public synchronized List flush(BooleanSupplier transportReady) { java.util.List flushed = new java.util.ArrayList<>(); - while (!queue.isEmpty()) { - GrpcStreamEnvelope envelope = queue.pollFirst().envelope(); - transport.accept(envelope); - lastFlushedSequence = envelope.sequence(); - flushed.add(envelope.sequence()); + while (!queue.isEmpty() && transportReady.getAsBoolean()) { + QueuedEnvelope queued = queue.pollFirst(); + transport.accept(queued.envelope()); + queuedBytes = Math.max(0L, queuedBytes - queued.serializedBytes()); + lastFlushedSequence = queued.envelope().sequence(); + flushed.add(lastFlushedSequence); } - queuedBytes = 0L; - if (state == GrpcStreamWriterState.DRAINING) { + if (queue.isEmpty() && state == GrpcStreamWriterState.DRAINING) { state = GrpcStreamWriterState.TERMINATED; } return List.copyOf(flushed); diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java index 2c58baf..4bfb7e7 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java @@ -39,14 +39,15 @@ public final class GrpcStreamAdmission { if (callerFingerprint == null || callerFingerprint.isBlank()) { throw new IllegalArgumentException("stream admission needs a caller fingerprint"); } + // Global bound first: a caller refused here must not leave an empty per-caller entry behind. + if (openStreams.get() >= maxConcurrentStreams) { + return false; + } AtomicInteger callerCount = perCaller.computeIfAbsent(callerFingerprint, key -> new AtomicInteger()); if (callerCount.get() >= maxStreamsPerCaller) { return false; } - if (openStreams.get() >= maxConcurrentStreams) { - return false; - } callerCount.incrementAndGet(); openStreams.incrementAndGet(); return true; diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java index 802acdb..0063222 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java @@ -11,6 +11,12 @@ package dev.caskeleton.grpc.streaming; public enum GrpcStreamWriteResult { /** Queued for the single writer. Not a delivery guarantee. */ ACCEPTED, + /** + * Queued, and the producer should stop until the transport is ready again (the queue reached its + * high-water mark or the transport is not ready). Writing on regardless fills the queue to its + * bound and ends the stream. + */ + ACCEPTED_PAUSE, /** The bounded queue is full and the profile drops rather than terminates. */ DROPPED, /** The queue overflowed and the stream is being terminated. */ @@ -20,6 +26,6 @@ public enum GrpcStreamWriteResult { /** Whether the message entered the queue. */ public boolean queued() { - return this == ACCEPTED; + return this == ACCEPTED || this == ACCEPTED_PAUSE; } } diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java index a97f73c..f45ecf2 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java @@ -11,6 +11,7 @@ import dev.caskeleton.grpc.api.deadline.GrpcDeadlineBudget; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; import dev.caskeleton.grpc.core.GrpcMethodName; import dev.caskeleton.grpc.core.RpcType; +import io.grpc.Context; import java.time.Duration; import java.time.Instant; import java.util.Map; @@ -19,6 +20,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -156,4 +158,29 @@ class GrpcContextBinderTest { assertThat(actor).isEqualTo("actor-1"); assertThat(binder.current()).isEmpty(); } + + @Test + void wrappedWorkSeesTheSubmittingCallsCancellationOnAnotherThread() throws Exception { + // given + GrpcContextBinder binder = new GrpcContextBinder(GrpcContextPropagationPolicy.stable()); + Context.CancellableContext call = Context.current().withCancellation(); + AtomicBoolean cancelledInside = new AtomicBoolean(); + AtomicReference wrapped = new AtomicReference<>(); + call.run( + () -> + binder.runWith( + snapshot(null), + () -> + wrapped.set( + binder.wrap(() -> cancelledInside.set(Context.current().isCancelled()))))); + call.cancel(new IllegalStateException("the client went away")); + Thread worker = new Thread(wrapped.get()); + + // when + worker.start(); + worker.join(); + + // then + assertThat(cancelledInside).isTrue(); + } } diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java index a5bec3b..5bac289 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java @@ -91,7 +91,7 @@ class GrpcWaitForReadyValidatorTest { GrpcWaitForReadyValidator.decide( policyWith(WaitForReadyPolicy.WORKER_OPT_IN), GrpcWaitForReadyProfile.worker(Duration.ofSeconds(30)), - new GrpcDeadlineBudget(Duration.ofSeconds(5), TEN_SECONDS)); + new GrpcDeadlineBudget(Duration.ofSeconds(5), TEN_SECONDS, () -> 0L)); GrpcWaitForReadyDecision tightBudget = GrpcWaitForReadyValidator.decide( policyWith(WaitForReadyPolicy.WORKER_OPT_IN), diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java new file mode 100644 index 0000000..85c8cd0 --- /dev/null +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java @@ -0,0 +1,91 @@ +package dev.caskeleton.grpc.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; + +class GrpcCredentialRotationManagerTest { + + private static final Instant NOW = Instant.parse("2026-08-30T10:00:00Z"); + private static final int ROUNDS = 20_000; + + @Test + void racingRotationsNeverLeaveAnOlderGenerationCurrent() throws Exception { + // when + long lowest = + lowestCurrentAfterRacing( + manager -> rotateIgnoringRefusal(manager, 3L), manager -> manager.rotate(gen(4L), NOW)); + + // then + assertThat(lowest).isEqualTo(4L); + } + + @Test + void aDrainCompletedDuringARotationNeverRevertsTheNewGeneration() throws Exception { + // when + long lowest = + lowestCurrentAfterRacing( + manager -> manager.rotate(gen(3L), NOW), GrpcCredentialRotationManager::completeDrain); + + // then + assertThat(lowest).isEqualTo(3L); + } + + /** + * Starts each round at generation 2 (draining 1), runs {@code first} and {@code second} at the + * same moment, and returns the lowest generation left current over all rounds. + */ + private static long lowestCurrentAfterRacing( + Consumer first, Consumer second) + throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + long lowest = Long.MAX_VALUE; + for (int round = 0; round < ROUNDS; round++) { + GrpcCredentialRotationManager manager = + new GrpcCredentialRotationManager(gen(1L), Duration.ofSeconds(30)); + manager.rotate(gen(2L), NOW); + CyclicBarrier start = new CyclicBarrier(2); + Future a = pool.submit(() -> run(start, first, manager)); + Future b = pool.submit(() -> run(start, second, manager)); + a.get(); + b.get(); + lowest = Math.min(lowest, manager.current().generation()); + } + return lowest; + } finally { + pool.shutdownNow(); + } + } + + private static Void run( + CyclicBarrier start, + Consumer action, + GrpcCredentialRotationManager manager) + throws Exception { + start.await(); + action.accept(manager); + return null; + } + + /** The losing rotator of a race is refused, which is the correct outcome, not a failure. */ + private static void rotateIgnoringRefusal(GrpcCredentialRotationManager manager, long next) { + try { + manager.rotate(gen(next), NOW); + } catch (IllegalArgumentException refused) { + // superseded by the other rotator + } + } + + private static GrpcCredentialGeneration gen(long generation) { + return new GrpcCredentialGeneration( + generation, "vault://v" + generation, NOW, NOW.plusSeconds(3600)); + } +} diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java index 7976eea..767142e 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java @@ -9,6 +9,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -23,7 +24,7 @@ class GrpcSerializedStreamWriterTest { private GrpcSerializedStreamWriter writer( GrpcFlowControlPolicy policy, long messageSize) { return new GrpcSerializedStreamWriter<>( - STREAM, GrpcStreamSequence.forNewStream(STREAM), policy, sent::add, () -> messageSize); + STREAM, GrpcStreamSequence.forNewStream(STREAM), policy, sent::add, payload -> messageSize); } @Test @@ -38,7 +39,7 @@ class GrpcSerializedStreamWriterTest { assertThat(result.queued()).isTrue(); assertThat(sent).isEmpty(); - assertThat(writer.flush()).containsExactly(1L); + assertThat(writer.flush(() -> true)).containsExactly(1L); assertThat(sent).hasSize(1); } @@ -76,7 +77,7 @@ class GrpcSerializedStreamWriterTest { } assertThat(sent).isEmpty(); - List flushed = writer.flush(); + List flushed = writer.flush(() -> true); assertThat(flushed).hasSize(producers * perProducer); assertThat(flushed).isSorted(); assertThat(flushed).doesNotHaveDuplicates(); @@ -121,7 +122,7 @@ class GrpcSerializedStreamWriterTest { GrpcStreamSequence.forNewStream(STREAM), new GrpcFlowControlPolicy(10, 1000L, 9, GrpcSlowConsumerPolicy.DROP_OLDEST), sent::add, - nextSize::get); + payload -> nextSize.get()); nextSize.set(900L); assertThat(writer.write(GrpcStreamEnvelope.Kind.LIVE, "large", null, null, true)) @@ -149,7 +150,7 @@ class GrpcSerializedStreamWriterTest { GrpcStreamSequence.forNewStream(STREAM), new GrpcFlowControlPolicy(10, 700L, 9, GrpcSlowConsumerPolicy.DROP_OLDEST), sent::add, - nextSize::get); + payload -> nextSize.get()); writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true); writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true); @@ -179,7 +180,7 @@ class GrpcSerializedStreamWriterTest { void cancellingKeepsTheLastFlushedPositionAsEvidence() { GrpcSerializedStreamWriter writer = writer(GrpcFlowControlPolicy.stable(), 8L); writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true); - writer.flush(); + writer.flush(() -> true); writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true); long lastFlushed = writer.discardQueued(); @@ -189,4 +190,54 @@ class GrpcSerializedStreamWriterTest { assertThat(writer.queuedMessages()).isZero(); assertThat(sent).hasSize(1); } + + @Test + void theByteBoundCountsEachPayloadsOwnSize() { + // given + GrpcSerializedStreamWriter writer = + new GrpcSerializedStreamWriter<>( + STREAM, + GrpcStreamSequence.forNewStream(STREAM), + new GrpcFlowControlPolicy(10, 10L, 9, GrpcSlowConsumerPolicy.TERMINATE), + sent::add, + payload -> payload.length()); + writer.write(GrpcStreamEnvelope.Kind.LIVE, "eight-by", null, null, true); + + // when + GrpcStreamWriteResult result = + writer.write(GrpcStreamEnvelope.Kind.LIVE, "four", null, null, true); + + // then + assertThat(result).isEqualTo(GrpcStreamWriteResult.OVERFLOW_TERMINATING); + } + + @Test + void aWriteWhileTheTransportIsNotReadyTellsTheProducerToPause() { + // given + GrpcSerializedStreamWriter writer = writer(GrpcFlowControlPolicy.stable(), 8L); + + // when + GrpcStreamWriteResult result = + writer.write(GrpcStreamEnvelope.Kind.LIVE, "row", null, null, false); + + // then + assertThat(result) + .extracting(r -> r, GrpcStreamWriteResult::queued) + .containsExactly(GrpcStreamWriteResult.ACCEPTED_PAUSE, true); + } + + @Test + void flushStopsWhenTheTransportStopsBeingReady() { + // given + GrpcSerializedStreamWriter writer = writer(GrpcFlowControlPolicy.stable(), 8L); + writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true); + writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true); + AtomicInteger readyChecks = new AtomicInteger(); + + // when + List flushed = writer.flush(() -> readyChecks.getAndIncrement() == 0); + + // then + assertThat(flushed).containsExactly(1L); + } } diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java index 8849c52..f335220 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java @@ -45,4 +45,17 @@ class GrpcStreamAdmissionConcurrencyTest { assertThat(admission.trackedCallers()).isZero(); } } + + @Test + void aCallerRefusedByTheGlobalBoundLeavesNoTrackedEntry() { + // given + GrpcStreamAdmission admission = new GrpcStreamAdmission(1, 1); + admission.tryAdmit("first"); + + // when + admission.tryAdmit("refused"); + + // then + assertThat(admission.trackedCallers()).isOne(); + } } diff --git a/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java b/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java index c25f67a..a6b346c 100644 --- a/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java +++ b/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java @@ -1,7 +1,5 @@ package dev.caskeleton.grpc.server; -import java.util.concurrent.atomic.AtomicInteger; - /** * Refuses work the server cannot do, before it spends anything on it. * @@ -14,9 +12,12 @@ public final class GrpcAdmissionController { private final int maxConcurrentCalls; private final int maxQueuedCalls; - private final AtomicInteger inFlight = new AtomicInteger(); - private final AtomicInteger queued = new AtomicInteger(); - private final AtomicInteger rejected = new AtomicInteger(); + + // One lock for both counters: every decision reads one and changes the other, and a check on an + // atomic followed by an increment lets racing callers all pass the same check. + private int inFlight; + private int queued; + private int rejected; /** Bounds concurrency and queueing separately. */ public GrpcAdmissionController(int maxConcurrentCalls, int maxQueuedCalls) { @@ -48,56 +49,54 @@ public final class GrpcAdmissionController { } /** Admits a call, queues it, or refuses it. */ - public Decision tryAdmit() { - int running = inFlight.get(); - if (running < maxConcurrentCalls) { - inFlight.incrementAndGet(); - return new Decision(true, inFlight.get(), queued.get(), "within the concurrency bound"); + public synchronized Decision tryAdmit() { + if (inFlight < maxConcurrentCalls) { + inFlight++; + return new Decision(true, inFlight, queued, "within the concurrency bound"); } - int waiting = queued.get(); - if (waiting < maxQueuedCalls) { - queued.incrementAndGet(); - return new Decision(true, running, queued.get(), "queued within the admission bound"); + if (queued < maxQueuedCalls) { + queued++; + return new Decision(true, inFlight, queued, "queued within the admission bound"); } - rejected.incrementAndGet(); + rejected++; return new Decision( false, - running, - waiting, + inFlight, + queued, "at capacity: " - + running + + inFlight + " in flight and " - + waiting + + queued + " queued. Refusing is better than accepting work whose caller will have given up."); } /** Moves a queued call into execution. */ - public void promoteFromQueue() { - if (queued.get() > 0) { - queued.decrementAndGet(); - inFlight.incrementAndGet(); + public synchronized void promoteFromQueue() { + if (queued > 0) { + queued--; + inFlight++; } } /** Releases a finished call. */ - public void release() { - if (inFlight.get() > 0) { - inFlight.decrementAndGet(); + public synchronized void release() { + if (inFlight > 0) { + inFlight--; } } /** How many calls are executing. */ - public int inFlight() { - return inFlight.get(); + public synchronized int inFlight() { + return inFlight; } /** How many calls are waiting. */ - public int queued() { - return queued.get(); + public synchronized int queued() { + return queued; } /** How many calls have been refused. */ - public int rejected() { - return rejected.get(); + public synchronized int rejected() { + return rejected; } } diff --git a/grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java b/grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java new file mode 100644 index 0000000..895fdf5 --- /dev/null +++ b/grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java @@ -0,0 +1,55 @@ +package dev.caskeleton.grpc.server; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.junit.jupiter.api.Test; + +class GrpcAdmissionControllerTest { + + private static final int CALLERS = 8; + private static final int ROUNDS = 500; + + @Test + void racingCallersNeverGetPastTheConcurrencyAndQueueBounds() throws Exception { + // when + int mostAdmitted = mostAdmittedOverRounds(); + + // then + assertThat(mostAdmitted).isEqualTo(2); + } + + /** Races {@link #CALLERS} callers at a 1-running, 1-queued controller, round after round. */ + private static int mostAdmittedOverRounds() throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(CALLERS); + try { + int most = 0; + for (int round = 0; round < ROUNDS; round++) { + GrpcAdmissionController controller = new GrpcAdmissionController(1, 1); + CyclicBarrier start = new CyclicBarrier(CALLERS); + List> admitted = new ArrayList<>(); + for (int caller = 0; caller < CALLERS; caller++) { + admitted.add( + pool.submit( + () -> { + start.await(); + return controller.tryAdmit().admitted(); + })); + } + int count = 0; + for (Future result : admitted) { + count += result.get() ? 1 : 0; + } + most = Math.max(most, count); + } + return most; + } finally { + pool.shutdownNow(); + } + } +}