diff --git a/build.gradle b/build.gradle index 001bb2d..e91af3c 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,9 @@ tasks.register('ci') { description = 'Runs the optional gRPC platform verification surface.' dependsOn tasks.named('checkAll') dependsOn tasks.named('qualityCheck') - dependsOn tasks.named('verifyCleanArchitectureDependencies') + // architectureCheck, not only the direction gate: it also runs verifyNoIgnoredSourcePackages, so + // a gRPC source Git would drop fails this build's ci the way it fails the main build's. + dependsOn tasks.named('architectureCheck') } // A strict integration lane that no workflow invokes is not coverage; it is a file that looks like diff --git a/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml b/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml index 0c5ec86..ae3cab3 100644 --- a/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml +++ b/grpc-advanced/grpc-advanced-compat/src/main/resources/envoy/envoy.yaml @@ -29,13 +29,18 @@ static_resources: - match: { prefix: "/" } route: cluster: grpc_backend - timeout: 30s + # No route timeout: it would cut every server stream at that age. The + # caller's grpc-timeout header bounds each call instead. + timeout: 0s + max_stream_duration: + grpc_timeout_header_max: 0s cors: allow_origin_string_match: - exact: "https://app.example.com" allow_methods: "POST,OPTIONS" - allow_headers: "content-type,x-grpc-web,x-correlation-id,authorization" - expose_headers: "grpc-status,grpc-message,error-code,error-category" + allow_headers: "content-type,x-grpc-web,x-user-agent,grpc-timeout,x-correlation-id,authorization" + # Every trailer the platform sends (GrpcErrorMapper, GrpcValidationInterceptor). + expose_headers: "grpc-status,grpc-message,grpc-status-details-bin,error-reason,error-execution-id,completion-outcome,validation-violations" max_age: "1728000" http_filters: - name: envoy.filters.http.grpc_web @@ -51,6 +56,11 @@ static_resources: name: envoy.transport_sockets.tls typed_config: "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + # Placeholder paths: mount the proxy's certificate and key here. + common_tls_context: + tls_certificates: + - certificate_chain: { filename: /etc/envoy/tls/tls.crt } + private_key: { filename: /etc/envoy/tls/tls.key } clusters: - name: grpc_backend connect_timeout: 1s diff --git a/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java b/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java index 0a215f8..a34f687 100644 --- a/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java +++ b/grpc-advanced/grpc-advanced-compat/src/test/java/dev/caskeleton/grpc/advanced/compat/GrpcWebCompatibilityTest.java @@ -7,11 +7,15 @@ import dev.caskeleton.grpc.advanced.web.GrpcWebProfile; import dev.caskeleton.grpc.advanced.web.GrpcWebProxyContract; import dev.caskeleton.grpc.advanced.web.GrpcWebRpcSupport; import dev.caskeleton.grpc.core.RpcType; +import dev.caskeleton.grpc.error.GrpcErrorMapper; +import dev.caskeleton.grpc.validation.GrpcValidationInterceptor; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -110,6 +114,58 @@ class GrpcWebCompatibilityTest { .contains("envoy.filters.http.grpc_web"); } + @Test + void theReferenceProxyExposesEveryTrailerThePlatformSends() { + // when + Set exposed = headerList(resource("envoy/envoy.yaml"), "expose_headers"); + + // then + assertThat(exposed) + .contains( + "grpc-status", + "grpc-message", + "grpc-status-details-bin", + GrpcErrorMapper.REASON_KEY.name(), + GrpcErrorMapper.EXECUTION_ID_KEY.name(), + GrpcErrorMapper.COMPLETION_OUTCOME_KEY.name(), + GrpcValidationInterceptor.VIOLATIONS_KEY.name()); + } + + @Test + void theReferenceProxyLetsTheBrowserSendItsDeadlineAndClientHeaders() { + // when + Set allowed = headerList(resource("envoy/envoy.yaml"), "allow_headers"); + + // then + assertThat(allowed).contains("grpc-timeout", "x-user-agent", "x-grpc-web", "content-type"); + } + + @Test + void theReferenceRouteLeavesStreamLengthToTheCallersDeadline() { + // when + String envoy = resource("envoy/envoy.yaml"); + + // then + assertThat(envoy).contains("timeout: 0s").contains("grpc_timeout_header_max: 0s"); + } + + @Test + void theReferenceProxyTerminatesTlsWithACertificate() { + // when + String envoy = resource("envoy/envoy.yaml"); + + // then + assertThat(envoy).contains("certificate_chain:").contains("private_key:"); + } + + private static Set headerList(String envoy, String field) { + Matcher matcher = Pattern.compile(field + ": \"([^\"]*)\"").matcher(envoy); + if (!matcher.find()) { + throw new IllegalStateException("the reference proxy has no " + field); + } + return Set.of(matcher.group(1).split(",")); + } + private static String resource(String path) { try (InputStream stream = GrpcWebCompatibilityTest.class.getClassLoader().getResourceAsStream(path)) { diff --git a/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java b/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java index ee4ad40..4be13bd 100644 --- a/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java +++ b/grpc/grpc-admin/src/main/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistry.java @@ -22,7 +22,9 @@ public final class GrpcServiceHealthRegistry { private final ConcurrentMap states = new ConcurrentHashMap<>(); private final ConcurrentMap dependencyHealth = new ConcurrentHashMap<>(); private final GrpcHealthPolicy policy; - private volatile boolean draining; + // Every write holds this registry's lock, so a state change cannot land between the draining + // check and its write and undo a drain that started in between. + private boolean draining; /** Binds a registry to its health policy. */ public GrpcServiceHealthRegistry(GrpcHealthPolicy policy) { @@ -34,7 +36,7 @@ public final class GrpcServiceHealthRegistry { } /** Registers a service, unknown until something says otherwise. */ - public void register(String serviceName) { + public synchronized void register(String serviceName) { requireServiceName(serviceName); states.putIfAbsent(serviceName, GrpcHealthState.UNKNOWN); } @@ -45,7 +47,7 @@ public final class GrpcServiceHealthRegistry { *

Ignored while draining: a service that reports itself healthy after the drain has started * would be routed traffic the instance has already promised not to take. */ - public void markServing(String serviceName) { + public synchronized void markServing(String serviceName) { requireServiceName(serviceName); if (draining) { return; @@ -55,7 +57,7 @@ public final class GrpcServiceHealthRegistry { } /** Records that a service is not serving. */ - public void markNotServing(String serviceName) { + public synchronized void markNotServing(String serviceName) { requireServiceName(serviceName); if (draining) { return; @@ -65,7 +67,7 @@ public final class GrpcServiceHealthRegistry { } /** Records a dependency's health, which only matters when the policy says it does. */ - public void recordDependencyHealth(String dependencyName, boolean healthy) { + public synchronized void recordDependencyHealth(String dependencyName, boolean healthy) { if (dependencyName == null || dependencyName.isBlank()) { throw new IllegalArgumentException("a dependency needs a name"); } @@ -81,7 +83,7 @@ public final class GrpcServiceHealthRegistry { *

Called first in a shutdown, before admission stops, so that a load balancer has a chance to * stop sending before the server starts refusing. */ - public void beginDraining() { + public synchronized void beginDraining() { draining = true; states.replaceAll((service, state) -> GrpcHealthState.DRAINING); } diff --git a/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java b/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java index d85f7c3..7454b5a 100644 --- a/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java +++ b/grpc/grpc-admin/src/test/java/dev/caskeleton/grpc/admin/GrpcServiceHealthRegistryTest.java @@ -4,6 +4,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -88,4 +93,47 @@ class GrpcServiceHealthRegistryTest { registry.markServing(DOCUMENTS); assertThat(registry.stateOf(DOCUMENTS)).isEqualTo(GrpcHealthState.DRAINING); } + + @Test + void aServiceMarkedServingWhileDrainBeginsNeverReturnsTheInstanceToServing() throws Exception { + // when + Set outcomes = outcomesOfServingRacingDrain(); + + // then + assertThat(outcomes).containsExactly("DRAINING/DRAINING"); + } + + /** Races markServing against beginDraining, round after round, recording service/global. */ + private static Set outcomesOfServingRacingDrain() throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Set outcomes = new TreeSet<>(); + for (int round = 0; round < 1000; round++) { + GrpcServiceHealthRegistry registry = + new GrpcServiceHealthRegistry(GrpcHealthPolicy.standalone()); + registry.register(DOCUMENTS); + CyclicBarrier start = new CyclicBarrier(2); + Future serving = + pool.submit( + () -> { + start.await(); + registry.markServing(DOCUMENTS); + return null; + }); + Future drain = + pool.submit( + () -> { + start.await(); + registry.beginDraining(); + return null; + }); + serving.get(); + drain.get(); + outcomes.add(registry.stateOf(DOCUMENTS) + "/" + registry.globalState()); + } + return outcomes; + } finally { + pool.shutdownNow(); + } + } } diff --git a/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java b/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java index b25dec2..188ad5f 100644 --- a/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java +++ b/grpc/grpc-client/src/main/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistry.java @@ -3,7 +3,9 @@ package dev.caskeleton.grpc.client; import dev.caskeleton.grpc.core.GrpcChannelProfileName; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -24,8 +26,9 @@ public final class GrpcChannelRuntimeRegistry { private final ConcurrentMap> current = new ConcurrentHashMap<>(); - private final ConcurrentMap> draining = - new ConcurrentHashMap<>(); + // Guarded by this registry's lock, like every swap: rotations are rare, and one lock keeps the + // current pointer and the draining list from disagreeing. + private final Map> draining = new HashMap<>(); private final GrpcChannelDrainPolicy drainPolicy; /** Binds a registry to the drain policy its swaps use. */ @@ -37,7 +40,7 @@ public final class GrpcChannelRuntimeRegistry { } /** Installs the first generation for a profile. */ - public GrpcChannelRuntime install(GrpcChannelGeneration generation) { + public synchronized GrpcChannelRuntime install(GrpcChannelGeneration generation) { if (generation == null) { throw new IllegalArgumentException("a generation is required"); } @@ -80,7 +83,7 @@ public final class GrpcChannelRuntimeRegistry { * * @throws IllegalArgumentException when {@code next} does not supersede what is installed */ - public GrpcChannelRuntime rotate(GrpcChannelGeneration next, Instant now) { + public synchronized GrpcChannelRuntime rotate(GrpcChannelGeneration next, Instant now) { if (next == null || now == null) { throw new IllegalArgumentException("a rotation needs a generation and a moment"); } @@ -102,15 +105,12 @@ public final class GrpcChannelRuntimeRegistry { GrpcChannelRuntime replacement = new GrpcChannelRuntime(next); holder.set(replacement); previous.beginDrain(); - draining - .computeIfAbsent( - next.profileName(), key -> java.util.Collections.synchronizedList(new ArrayList<>())) - .add(previous); + draining.computeIfAbsent(next.profileName(), key -> new ArrayList<>()).add(previous); return replacement; } /** The generations still draining for a profile. */ - public List draining(GrpcChannelProfileName profileName) { + public synchronized List draining(GrpcChannelProfileName profileName) { return List.copyOf(draining.getOrDefault(profileName, List.of())); } @@ -119,7 +119,7 @@ public final class GrpcChannelRuntimeRegistry { * * @return how many were retired */ - public int retireQuiescent(GrpcChannelProfileName profileName) { + public synchronized int retireQuiescent(GrpcChannelProfileName profileName) { List runtimes = draining.get(profileName); if (runtimes == null) { return 0; diff --git a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java index 796e4a1..b1ceb3e 100644 --- a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java +++ b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcChannelRuntimeRegistryTest.java @@ -6,6 +6,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import dev.caskeleton.grpc.core.GrpcChannelProfileName; import java.time.Duration; import java.time.Instant; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -125,4 +131,52 @@ class GrpcChannelRuntimeRegistryTest { .extracting(Class::getName) .noneMatch(name -> name.startsWith("io.grpc.")); } + + @Test + void racingRotationsDrainEveryReplacedRuntimeAndKeepTheNewestCurrent() throws Exception { + // when + Set outcomes = outcomesOfRacingRotations(); + + // then + assertThat(outcomes).isSubsetOf("3 draining [1]", "3 draining [1, 2]"); + } + + /** Races rotate(2) against rotate(3) on a profile at generation 1, round after round. */ + private static Set outcomesOfRacingRotations() throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Set outcomes = new TreeSet<>(); + for (int round = 0; round < 1000; round++) { + GrpcChannelRuntimeRegistry registry = + new GrpcChannelRuntimeRegistry(GrpcChannelDrainPolicy.stable()); + registry.install(generation(1)); + CyclicBarrier start = new CyclicBarrier(2); + Future a = pool.submit(() -> rotateIgnoringRefusal(registry, start, 2)); + Future b = pool.submit(() -> rotateIgnoringRefusal(registry, start, 3)); + a.get(); + b.get(); + outcomes.add( + registry.require(PROFILE).generation().generation() + + " draining " + + registry.draining(PROFILE).stream() + .map(runtime -> runtime.generation().generation()) + .sorted() + .toList()); + } + return outcomes; + } finally { + pool.shutdownNow(); + } + } + + private static Void rotateIgnoringRefusal( + GrpcChannelRuntimeRegistry registry, CyclicBarrier start, long next) throws Exception { + start.await(); + try { + registry.rotate(generation(next), NOW); + } catch (IllegalArgumentException refused) { + // the other rotator installed a newer generation first + } + return null; + } } diff --git a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java index 8203a61..15cdf29 100644 --- a/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java +++ b/grpc/grpc-client/src/test/java/dev/caskeleton/grpc/client/GrpcTypedStubFactoryTest.java @@ -122,8 +122,10 @@ class GrpcTypedStubFactoryTest { GrpcStubPolicyApplier.CallPolicy generous = applier.applyTo(GET, Duration.ofSeconds(30)); GrpcStubPolicyApplier.CallPolicy tight = applier.applyTo(GET, Duration.ofMillis(300)); - assertThat(generous.deadline().remaining()).isEqualTo(Duration.ofMillis(1800)); - assertThat(tight.deadline().remaining()).isEqualTo(Duration.ofMillis(100)); + // The budget is live: a few milliseconds have passed since applyTo computed it. + assertThat(generous.deadline().remaining()) + .isBetween(Duration.ofMillis(1700), Duration.ofMillis(1800)); + assertThat(tight.deadline().remaining()).isBetween(Duration.ZERO, Duration.ofMillis(100)); assertThat(tight.waitForReady().queue()).isFalse(); } } diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java index b41258a..02a8cbb 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudget.java @@ -1,6 +1,7 @@ package dev.caskeleton.grpc.api.deadline; import java.time.Duration; +import java.util.function.LongSupplier; /** * How much time a call actually has, after the parent deadline and the method profile have both had @@ -9,17 +10,40 @@ import java.time.Duration; *

The rule is "shorter of the two, minus the reserve", and it is here rather than at each call * site because the failure mode of getting it wrong is silent: a downstream timeout longer than the * inbound deadline never fires, and the caller sees a deadline the server has no idea it missed. + * + *

The budget is an absolute deadline on a monotonic nanosecond clock, so {@link #remaining()} + * shrinks and {@link #expired()} turns true as time passes. A request context carries one budget + * for the whole call; a budget that remembered only the duration it started with would hand every + * later dependency call the full original time. */ -public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile) { +public final class GrpcDeadlineBudget { - /** Rejects a budget with no time in it. */ - public GrpcDeadlineBudget { - if (remaining == null || profile == null) { - throw new IllegalArgumentException("a deadline budget needs a remaining time and a profile"); + private final long deadlineNanos; + private final GrpcDeadlineProfile profile; + private final LongSupplier nanoClock; + + /** A budget of {@code remaining} from now on the system's monotonic clock. */ + public GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile) { + this(remaining, profile, System::nanoTime); + } + + /** + * A budget of {@code remaining} from now on {@code nanoClock}. + * + * @param nanoClock a monotonic nanosecond source, {@code System::nanoTime} outside tests + */ + public GrpcDeadlineBudget( + Duration remaining, GrpcDeadlineProfile profile, LongSupplier nanoClock) { + if (remaining == null || profile == null || nanoClock == null) { + throw new IllegalArgumentException( + "a deadline budget needs a remaining time, a profile and a clock"); } if (remaining.isNegative()) { throw new IllegalArgumentException("remaining time must not be negative; got " + remaining); } + this.deadlineNanos = nanoClock.getAsLong() + remaining.toNanos(); + this.profile = profile; + this.nanoClock = nanoClock; } /** @@ -31,6 +55,12 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile */ public static GrpcDeadlineBudget forEntryPoint( Duration parentRemaining, GrpcDeadlineProfile profile) { + return forEntryPoint(parentRemaining, profile, System::nanoTime); + } + + /** {@link #forEntryPoint(Duration, GrpcDeadlineProfile)} on {@code nanoClock}. */ + public static GrpcDeadlineBudget forEntryPoint( + Duration parentRemaining, GrpcDeadlineProfile profile, LongSupplier nanoClock) { if (profile == null) { throw new IllegalArgumentException("a deadline budget needs a profile"); } @@ -42,17 +72,13 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile throw new IllegalArgumentException( "a Stable unary call requires a positive deadline; got " + parentRemaining); } - Duration bounded = - parentRemaining.compareTo(profile.total()) <= 0 ? parentRemaining : profile.total(); - Duration afterReserve = bounded.minus(profile.safetyMargin()); - return new GrpcDeadlineBudget( - afterReserve.isNegative() ? Duration.ZERO : afterReserve, profile); + return new GrpcDeadlineBudget(afterReserve(parentRemaining, profile), profile, nanoClock); } /** * Derives the budget for a downstream dependency call under {@code downstream}. * - *

Takes the shorter of what is left here and what the downstream profile allows, then + *

Takes the shorter of what is left here now and what the downstream profile allows, then * subtracts the downstream reserve. A dependency can never be given more time than its caller * has. */ @@ -60,11 +86,18 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile if (downstream == null) { throw new IllegalArgumentException("a downstream budget needs a profile"); } - Duration bounded = - remaining.compareTo(downstream.total()) <= 0 ? remaining : downstream.total(); - Duration afterReserve = bounded.minus(downstream.safetyMargin()); - return new GrpcDeadlineBudget( - afterReserve.isNegative() ? Duration.ZERO : afterReserve, downstream); + return new GrpcDeadlineBudget(afterReserve(remaining(), downstream), downstream, nanoClock); + } + + /** What is left now; zero once the deadline has passed. */ + public Duration remaining() { + long left = deadlineNanos - nanoClock.getAsLong(); + return left <= 0 ? Duration.ZERO : Duration.ofNanos(left); + } + + /** The profile this budget was cut from. */ + public GrpcDeadlineProfile profile() { + return profile; } /** @@ -75,11 +108,23 @@ public record GrpcDeadlineBudget(Duration remaining, GrpcDeadlineProfile profile * investigates the wrong system. */ public boolean canStartDependencyCall() { - return remaining.compareTo(profile.minimumAttemptBudget()) >= 0 && !remaining.isZero(); + Duration left = remaining(); + return !left.isZero() && left.compareTo(profile.minimumAttemptBudget()) >= 0; } /** Whether any time is left at all. */ public boolean expired() { - return remaining.isZero() || remaining.isNegative(); + return remaining().isZero(); + } + + @Override + public String toString() { + return "GrpcDeadlineBudget[remaining=" + remaining() + ", profile=" + profile + "]"; + } + + private static Duration afterReserve(Duration available, GrpcDeadlineProfile profile) { + Duration bounded = available.compareTo(profile.total()) <= 0 ? available : profile.total(); + Duration afterReserve = bounded.minus(profile.safetyMargin()); + return afterReserve.isNegative() ? Duration.ZERO : afterReserve; } } diff --git a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java index b4dab91..bfd5bec 100644 --- a/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java +++ b/grpc/grpc-core-api/src/main/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentity.java @@ -16,6 +16,13 @@ import dev.caskeleton.grpc.core.GrpcMethodName; public record GrpcOperationIdentity( String callerFingerprint, GrpcMethodName method, String idempotencyKeyHash) { + // Column widths of grpc_operation_ledger (V001). An identity that does not fit is refused here, + // not by the database in the middle of a claim. + private static final int MAX_CALLER_FINGERPRINT = 256; + private static final int MAX_METHOD = 256; + private static final int MAX_KEY_HASH = 80; + private static final int MAX_STORAGE_KEY = 512; + /** Requires all three parts, with the key already hashed. */ public GrpcOperationIdentity { if (callerFingerprint == null || callerFingerprint.isBlank()) { @@ -30,6 +37,24 @@ public record GrpcOperationIdentity( throw new IllegalArgumentException( "the idempotency key is stored hashed; got '" + idempotencyKeyHash + "'"); } + requireAtMost("caller fingerprint", callerFingerprint, MAX_CALLER_FINGERPRINT); + requireAtMost("full method name", method.canonical(), MAX_METHOD); + requireAtMost("idempotency key hash", idempotencyKeyHash, MAX_KEY_HASH); + requireAtMost( + "storage key", + callerFingerprint + "|" + method.canonical() + "|" + idempotencyKeyHash, + MAX_STORAGE_KEY); + } + + private static void requireAtMost(String what, String value, int max) { + if (value.length() > max) { + throw new IllegalArgumentException( + what + + " must be at most " + + max + + " characters to fit the ledger; got " + + value.length()); + } } /** The stable storage key: the three parts joined, safe to index and safe to log. */ diff --git a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java index 58fd57a..a23bc99 100644 --- a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java +++ b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/deadline/GrpcDeadlineBudgetTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.time.Duration; import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -27,9 +28,11 @@ class GrpcDeadlineBudgetTest { @DisplayName("the budget takes the shorter of the parent remaining and the method default") void budgetTakesTheShorterOfParentAndMethodDefault() { GrpcDeadlineBudget fromShortParent = - GrpcDeadlineBudget.forEntryPoint(Duration.ofMillis(500), TWO_SECONDS); + GrpcDeadlineBudget.forEntryPoint( + Duration.ofMillis(500), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos); GrpcDeadlineBudget fromLongParent = - GrpcDeadlineBudget.forEntryPoint(Duration.ofSeconds(30), TWO_SECONDS); + GrpcDeadlineBudget.forEntryPoint( + Duration.ofSeconds(30), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos); assertThat(fromShortParent.remaining()) .isEqualTo(Duration.ofMillis(500).minus(TWO_SECONDS.safetyMargin())); @@ -44,7 +47,9 @@ class GrpcDeadlineBudgetTest { new GrpcDeadlineProfile( Duration.ofSeconds(10), Duration.ofSeconds(1), Duration.ofMillis(100)); - GrpcDeadlineBudget budget = GrpcDeadlineBudget.forEntryPoint(Duration.ofSeconds(10), profile); + GrpcDeadlineBudget budget = + GrpcDeadlineBudget.forEntryPoint( + Duration.ofSeconds(10), profile, GrpcDeadlineBudgetTest::frozenNanos); assertThat(budget.remaining()).isEqualTo(Duration.ofSeconds(9)); assertThat(profile.usable()).isEqualTo(Duration.ofSeconds(9)); @@ -54,7 +59,8 @@ class GrpcDeadlineBudgetTest { @DisplayName("a downstream call never gets more time than its caller has left") void downstreamNeverExceedsTheCaller() { GrpcDeadlineBudget caller = - GrpcDeadlineBudget.forEntryPoint(Duration.ofMillis(400), TWO_SECONDS); + GrpcDeadlineBudget.forEntryPoint( + Duration.ofMillis(400), TWO_SECONDS, GrpcDeadlineBudgetTest::frozenNanos); GrpcDeadlineProfile generousDownstream = new GrpcDeadlineProfile(Duration.ofSeconds(30), Duration.ZERO, Duration.ofMillis(10)); @@ -77,6 +83,39 @@ class GrpcDeadlineBudgetTest { assertThat(new GrpcDeadlineBudget(Duration.ZERO, profile).canStartDependencyCall()).isFalse(); } + @Test + void aBudgetRunsOutAsTimePassesAfterItWasCreated() { + // given + AtomicLong nanos = new AtomicLong(); + GrpcDeadlineBudget budget = + new GrpcDeadlineBudget(Duration.ofMillis(300), TWO_SECONDS, nanos::get); + + // when + nanos.addAndGet(Duration.ofMillis(300).toNanos()); + + // then + assertThat(budget) + .extracting(GrpcDeadlineBudget::remaining, GrpcDeadlineBudget::expired) + .containsExactly(Duration.ZERO, true); + } + + @Test + void aDownstreamDerivedLaterGetsOnlyWhatIsLeftThen() { + // given + AtomicLong nanos = new AtomicLong(); + GrpcDeadlineBudget caller = + new GrpcDeadlineBudget(Duration.ofSeconds(1), TWO_SECONDS, nanos::get); + nanos.addAndGet(Duration.ofMillis(700).toNanos()); + + // when + GrpcDeadlineBudget downstream = + caller.deriveDownstream( + new GrpcDeadlineProfile(Duration.ofSeconds(30), Duration.ZERO, Duration.ofMillis(10))); + + // then + assertThat(downstream.remaining()).isEqualTo(Duration.ofMillis(300)); + } + @Test @DisplayName("a profile whose reserve leaves no usable time is refused") void profileWithNoUsableTimeIsRefused() { @@ -121,4 +160,8 @@ class GrpcDeadlineBudgetTest { .hasMessageContaining("charge-card") .hasMessageContaining("deadline-exceeded"); } + + private static long frozenNanos() { + return 0L; + } } diff --git a/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java new file mode 100644 index 0000000..c210e21 --- /dev/null +++ b/grpc/grpc-core-api/src/test/java/dev/caskeleton/grpc/api/ledger/GrpcOperationIdentityTest.java @@ -0,0 +1,44 @@ +package dev.caskeleton.grpc.api.ledger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.grpc.core.GrpcMethodName; +import dev.caskeleton.grpc.core.GrpcServiceName; +import org.junit.jupiter.api.Test; + +class GrpcOperationIdentityTest { + + private static final GrpcMethodName CREATE = + GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument"); + private static final String KEY_HASH = + "sha256:aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; + + @Test + void aCallerFingerprintWiderThanItsLedgerColumnIsRefused() { + // when + Throwable failure = + catchThrowable(() -> new GrpcOperationIdentity("c".repeat(257), CREATE, KEY_HASH)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("caller fingerprint must be at most 256"); + } + + @Test + void anIdentityWhoseStorageKeyWouldNotFitTheLedgerIsRefused() { + // given + GrpcMethodName longMethod = + new GrpcMethodName(new GrpcServiceName("p.S" + "s".repeat(197)), "M" + "m".repeat(54)); + + // when + Throwable failure = + catchThrowable(() -> new GrpcOperationIdentity("c".repeat(256), longMethod, KEY_HASH)); + + // then + assertThat(failure) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("storage key must be at most 512"); + } +} diff --git a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java index 443847a..7cd1afc 100644 --- a/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java +++ b/grpc/grpc-operation-ledger-jpa/src/main/java/dev/caskeleton/grpc/ledger/JpaGrpcOperationLedger.java @@ -50,7 +50,19 @@ public final class JpaGrpcOperationLedger implements GrpcOperationLedger { if (inserted != 0) { throw new IllegalStateException("claim insert affected an unexpected row count: " + inserted); } - return repository.findById(identity.storageKey()).map(entity -> entity.toRecord(identity)); + // Zero rows means another claim holds the key. An empty read now means that owner released it + // in between; treating that as "ours" would run the mutation without a claim row to guard it. + return Optional.of( + repository + .findById(identity.storageKey()) + .orElseThrow( + () -> + new IllegalStateException( + "claim for '" + + identity.storageKey() + + "' conflicted, but there was no row to read; the other owner released" + + " it in between. Retry the claim.")) + .toRecord(identity)); } @Override diff --git a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java index e1457b5..84862d5 100644 --- a/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java +++ b/grpc/grpc-operation-ledger-jpa/src/test/java/dev/caskeleton/grpc/ledger/GrpcOperationLedgerRepositoryTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.grpc.ledger; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.grpc.api.ledger.GrpcOperationIdentity; import dev.caskeleton.grpc.api.ledger.GrpcOperationLedger; @@ -40,7 +41,7 @@ class GrpcOperationLedgerRepositoryTest { * putIfAbsent} and therefore tested the database constraint we wished save() would hit rather * than the merge behavior it actually uses. */ - private static final class InMemoryRepository implements GrpcOperationLedgerRepository { + private static class InMemoryRepository implements GrpcOperationLedgerRepository { private final ConcurrentMap rows = new ConcurrentHashMap<>(); @Override @@ -124,6 +125,34 @@ class GrpcOperationLedgerRepositoryTest { assertThat(second.get().sameRequestAs(FINGERPRINT)).isTrue(); } + @Test + void aConflictWhoseRowVanishedBeforeTheReadIsNotTakenAsOwnership() { + // given + InMemoryRepository conflictThenReleased = + new InMemoryRepository() { + @Override + public int tryInsertClaim( + String storageKey, + String callerFingerprint, + String fullMethodName, + String idempotencyKeyHash, + String requestFingerprint, + Instant claimedAt) { + return 0; // another owner held the key and released it before this read + } + }; + GrpcOperationLedger racing = new JpaGrpcOperationLedger(conflictThenReleased); + + // when + Throwable failure = + catchThrowable(() -> racing.claim(identity("tenant-1.actor-1"), FINGERPRINT, NOW)); + + // then + assertThat(failure) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("no row to read"); + } + @Test @DisplayName("one caller's key does not collide with another's") void callersDoNotCollide() { diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java index 5f0f67c..5eef3bd 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/context/GrpcContextBinder.java @@ -87,15 +87,14 @@ public final class GrpcContextBinder { if (work == null) { throw new IllegalArgumentException("work must not be null"); } - GrpcContextSnapshot captured = SNAPSHOT.get(); - if (captured == null && policy.failClosedWithoutContext()) { + // The whole io.grpc Context, not just the snapshot value: it also carries the call's + // cancellation and deadline, which the other thread must see. + Context captured = Context.current(); + if (SNAPSHOT.get(captured) == null && policy.failClosedWithoutContext()) { throw new IllegalStateException( "refusing to hand work to another thread with no call context to carry"); } - if (captured == null) { - return work; - } - return () -> runWith(captured, work); + return captured.wrap(work); } /** An {@link Executor} that carries the submitting thread's context to every task. */ diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java index b967e90..5da90b5 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcRetryCoordinator.java @@ -78,13 +78,15 @@ public final class GrpcRetryCoordinator { } Duration backoff = jittered(config.backoffBefore(completedAttempt + 1), config.jitterFactor()); - if (remaining.remaining().compareTo(backoff) <= 0) { + // Read once: the budget is live, and a second read could fall below the backoff. + Duration left = remaining.remaining(); + if (left.compareTo(backoff) <= 0) { return GrpcRetryDecision.refuse( GrpcRetryDecision.Verdict.TERMINAL, "the remaining deadline is shorter than the backoff, so a retry could not finish"); } GrpcDeadlineBudget afterBackoff = - new GrpcDeadlineBudget(remaining.remaining().minus(backoff), remaining.profile()); + new GrpcDeadlineBudget(left.minus(backoff), remaining.profile()); if (!afterBackoff.canStartDependencyCall()) { return GrpcRetryDecision.refuse( GrpcRetryDecision.Verdict.TERMINAL, diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java index f41151d..6becd96 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidator.java @@ -35,7 +35,9 @@ public final class GrpcWaitForReadyValidator { + policy.method().canonical() + "' must carry one"); } - if (remaining.expired()) { + // Read once: the budget is live and keeps shrinking between reads. + Duration left = remaining.remaining(); + if (left.isZero()) { return GrpcWaitForReadyDecision.failFast("no deadline remains to queue inside"); } if (policy.waitForReady() != profile.policy()) { @@ -47,9 +49,7 @@ public final class GrpcWaitForReadyValidator { + "; the stricter of the two wins"); } Duration allowance = - profile.maxQueueWait().compareTo(remaining.remaining()) <= 0 - ? profile.maxQueueWait() - : remaining.remaining(); + profile.maxQueueWait().compareTo(left) <= 0 ? profile.maxQueueWait() : left; return GrpcWaitForReadyDecision.queueFor( allowance, "queueing under " + profile.policy() + " within the queue budget"); } diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java index bcf033f..dc13a6c 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManager.java @@ -86,22 +86,29 @@ public final class GrpcCredentialRotationManager { if (next == null || now == null) { throw new IllegalArgumentException("a rotation needs a generation and a moment"); } - State observed = state.get(); - if (observed.current().generation() == next.generation()) { - // Idempotent: re-applying the generation that is already current is a no-op rather than a - // second drain, because a retried rotation must not cancel the calls the first one admitted. - return new RotationPlan(observed.current(), Optional.empty(), Optional.empty()); - } - if (!observed.current().supersededBy(next)) { - throw new IllegalArgumentException( - "credential generation " - + next.generation() - + " does not supersede the current generation " - + observed.current().generation()); - } Instant deadline = now.plus(drainWindow); - state.set(new State(next, observed.current(), deadline)); - return new RotationPlan(next, Optional.of(observed.current()), Optional.of(deadline)); + // Compare-and-set loop: a racing rotator or completeDrain re-runs the checks against the state + // it actually replaces, so a rotation never goes backwards and never loses a draining + // generation. + while (true) { + State observed = state.get(); + if (observed.current().generation() == next.generation()) { + // Idempotent: re-applying the generation that is already current is a no-op rather than a + // second drain, because a retried rotation must not cancel the calls the first one + // admitted. + return new RotationPlan(observed.current(), Optional.empty(), Optional.empty()); + } + if (!observed.current().supersededBy(next)) { + throw new IllegalArgumentException( + "credential generation " + + next.generation() + + " does not supersede the current generation " + + observed.current().generation()); + } + if (state.compareAndSet(observed, new State(next, observed.current(), deadline))) { + return new RotationPlan(next, Optional.of(observed.current()), Optional.of(deadline)); + } + } } /** @@ -116,7 +123,6 @@ public final class GrpcCredentialRotationManager { /** Forgets the draining generation once its work has finished or been cancelled. */ public void completeDrain() { - State observed = state.get(); - state.set(new State(observed.current(), null, null)); + state.updateAndGet(observed -> new State(observed.current(), null, null)); } } diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java index 166638a..c949d2f 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriter.java @@ -4,8 +4,9 @@ import java.util.ArrayDeque; import java.util.Deque; import java.util.List; import java.util.Optional; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; -import java.util.function.LongSupplier; +import java.util.function.ToLongFunction; /** * The single writer every server stream goes through. @@ -27,7 +28,7 @@ public final class GrpcSerializedStreamWriter { private final GrpcStreamSequence sequence; private final GrpcFlowControlPolicy flowControl; private final Consumer> transport; - private final LongSupplier payloadSizer; + private final ToLongFunction payloadSizer; private final Deque> queue = new ArrayDeque<>(); private GrpcStreamWriterState state = GrpcStreamWriterState.OPEN; @@ -37,15 +38,15 @@ public final class GrpcSerializedStreamWriter { /** * @param transport the single consumer that actually calls the observer. Invoked only from {@link - * #flush}, which is synchronized, so the observer sees one thread. - * @param payloadSizer the serialized size of the next payload, for the byte bound + * #flush} and {@link #terminate}, which are synchronized, so the observer sees one thread. + * @param payloadSizer the serialized size of a payload, for the byte bound */ public GrpcSerializedStreamWriter( GrpcStreamId streamId, GrpcStreamSequence sequence, GrpcFlowControlPolicy flowControl, Consumer> transport, - LongSupplier payloadSizer) { + ToLongFunction payloadSizer) { if (streamId == null || sequence == null || flowControl == null @@ -70,7 +71,7 @@ public final class GrpcSerializedStreamWriter { if (!state.acceptsMessages()) { return GrpcStreamWriteResult.REJECTED_CLOSED; } - long nextBytes = payloadSizer.getAsLong(); + long nextBytes = payloadSizer.applyAsLong(payload); GrpcFlowControlDecision decision = flowControl.decide(queue.size(), queuedBytes, nextBytes, transportReady); return switch (decision.action()) { @@ -97,7 +98,11 @@ public final class GrpcSerializedStreamWriter { } } } - case PAUSE, PROCEED -> { + case PAUSE -> { + enqueue(kind, payload, snapshotVersion, resumeToken, nextBytes); + yield GrpcStreamWriteResult.ACCEPTED_PAUSE; + } + case PROCEED -> { enqueue(kind, payload, snapshotVersion, resumeToken, nextBytes); yield GrpcStreamWriteResult.ACCEPTED; } @@ -119,20 +124,22 @@ public final class GrpcSerializedStreamWriter { } /** - * Hands every queued message to the transport, in order, from this one thread. + * Hands queued messages to the transport, in order, from this one thread, while it is ready. * + * @param transportReady asked before each message ({@code ServerCallStreamObserver::isReady}); + * flushing stops at the first false and the rest stay queued for the next onReady * @return the sequence numbers flushed */ - public synchronized List flush() { + public synchronized List flush(BooleanSupplier transportReady) { java.util.List flushed = new java.util.ArrayList<>(); - while (!queue.isEmpty()) { - GrpcStreamEnvelope envelope = queue.pollFirst().envelope(); - transport.accept(envelope); - lastFlushedSequence = envelope.sequence(); - flushed.add(envelope.sequence()); + while (!queue.isEmpty() && transportReady.getAsBoolean()) { + QueuedEnvelope queued = queue.pollFirst(); + transport.accept(queued.envelope()); + queuedBytes = Math.max(0L, queuedBytes - queued.serializedBytes()); + lastFlushedSequence = queued.envelope().sequence(); + flushed.add(lastFlushedSequence); } - queuedBytes = 0L; - if (state == GrpcStreamWriterState.DRAINING) { + if (queue.isEmpty() && state == GrpcStreamWriterState.DRAINING) { state = GrpcStreamWriterState.TERMINATED; } return List.copyOf(flushed); diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java index 2c58baf..4bfb7e7 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmission.java @@ -39,14 +39,15 @@ public final class GrpcStreamAdmission { if (callerFingerprint == null || callerFingerprint.isBlank()) { throw new IllegalArgumentException("stream admission needs a caller fingerprint"); } + // Global bound first: a caller refused here must not leave an empty per-caller entry behind. + if (openStreams.get() >= maxConcurrentStreams) { + return false; + } AtomicInteger callerCount = perCaller.computeIfAbsent(callerFingerprint, key -> new AtomicInteger()); if (callerCount.get() >= maxStreamsPerCaller) { return false; } - if (openStreams.get() >= maxConcurrentStreams) { - return false; - } callerCount.incrementAndGet(); openStreams.incrementAndGet(); return true; diff --git a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java index 802acdb..0063222 100644 --- a/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java +++ b/grpc/grpc-policy/src/main/java/dev/caskeleton/grpc/streaming/GrpcStreamWriteResult.java @@ -11,6 +11,12 @@ package dev.caskeleton.grpc.streaming; public enum GrpcStreamWriteResult { /** Queued for the single writer. Not a delivery guarantee. */ ACCEPTED, + /** + * Queued, and the producer should stop until the transport is ready again (the queue reached its + * high-water mark or the transport is not ready). Writing on regardless fills the queue to its + * bound and ends the stream. + */ + ACCEPTED_PAUSE, /** The bounded queue is full and the profile drops rather than terminates. */ DROPPED, /** The queue overflowed and the stream is being terminated. */ @@ -20,6 +26,6 @@ public enum GrpcStreamWriteResult { /** Whether the message entered the queue. */ public boolean queued() { - return this == ACCEPTED; + return this == ACCEPTED || this == ACCEPTED_PAUSE; } } diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java index a97f73c..f45ecf2 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/context/GrpcContextBinderTest.java @@ -11,6 +11,7 @@ import dev.caskeleton.grpc.api.deadline.GrpcDeadlineBudget; import dev.caskeleton.grpc.api.deadline.GrpcDeadlineProfile; import dev.caskeleton.grpc.core.GrpcMethodName; import dev.caskeleton.grpc.core.RpcType; +import io.grpc.Context; import java.time.Duration; import java.time.Instant; import java.util.Map; @@ -19,6 +20,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -156,4 +158,29 @@ class GrpcContextBinderTest { assertThat(actor).isEqualTo("actor-1"); assertThat(binder.current()).isEmpty(); } + + @Test + void wrappedWorkSeesTheSubmittingCallsCancellationOnAnotherThread() throws Exception { + // given + GrpcContextBinder binder = new GrpcContextBinder(GrpcContextPropagationPolicy.stable()); + Context.CancellableContext call = Context.current().withCancellation(); + AtomicBoolean cancelledInside = new AtomicBoolean(); + AtomicReference wrapped = new AtomicReference<>(); + call.run( + () -> + binder.runWith( + snapshot(null), + () -> + wrapped.set( + binder.wrap(() -> cancelledInside.set(Context.current().isCancelled()))))); + call.cancel(new IllegalStateException("the client went away")); + Thread worker = new Thread(wrapped.get()); + + // when + worker.start(); + worker.join(); + + // then + assertThat(cancelledInside).isTrue(); + } } diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java index a5bec3b..5bac289 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/resilience/GrpcWaitForReadyValidatorTest.java @@ -91,7 +91,7 @@ class GrpcWaitForReadyValidatorTest { GrpcWaitForReadyValidator.decide( policyWith(WaitForReadyPolicy.WORKER_OPT_IN), GrpcWaitForReadyProfile.worker(Duration.ofSeconds(30)), - new GrpcDeadlineBudget(Duration.ofSeconds(5), TEN_SECONDS)); + new GrpcDeadlineBudget(Duration.ofSeconds(5), TEN_SECONDS, () -> 0L)); GrpcWaitForReadyDecision tightBudget = GrpcWaitForReadyValidator.decide( policyWith(WaitForReadyPolicy.WORKER_OPT_IN), diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java new file mode 100644 index 0000000..85c8cd0 --- /dev/null +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/security/GrpcCredentialRotationManagerTest.java @@ -0,0 +1,91 @@ +package dev.caskeleton.grpc.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; + +class GrpcCredentialRotationManagerTest { + + private static final Instant NOW = Instant.parse("2026-08-30T10:00:00Z"); + private static final int ROUNDS = 20_000; + + @Test + void racingRotationsNeverLeaveAnOlderGenerationCurrent() throws Exception { + // when + long lowest = + lowestCurrentAfterRacing( + manager -> rotateIgnoringRefusal(manager, 3L), manager -> manager.rotate(gen(4L), NOW)); + + // then + assertThat(lowest).isEqualTo(4L); + } + + @Test + void aDrainCompletedDuringARotationNeverRevertsTheNewGeneration() throws Exception { + // when + long lowest = + lowestCurrentAfterRacing( + manager -> manager.rotate(gen(3L), NOW), GrpcCredentialRotationManager::completeDrain); + + // then + assertThat(lowest).isEqualTo(3L); + } + + /** + * Starts each round at generation 2 (draining 1), runs {@code first} and {@code second} at the + * same moment, and returns the lowest generation left current over all rounds. + */ + private static long lowestCurrentAfterRacing( + Consumer first, Consumer second) + throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + long lowest = Long.MAX_VALUE; + for (int round = 0; round < ROUNDS; round++) { + GrpcCredentialRotationManager manager = + new GrpcCredentialRotationManager(gen(1L), Duration.ofSeconds(30)); + manager.rotate(gen(2L), NOW); + CyclicBarrier start = new CyclicBarrier(2); + Future a = pool.submit(() -> run(start, first, manager)); + Future b = pool.submit(() -> run(start, second, manager)); + a.get(); + b.get(); + lowest = Math.min(lowest, manager.current().generation()); + } + return lowest; + } finally { + pool.shutdownNow(); + } + } + + private static Void run( + CyclicBarrier start, + Consumer action, + GrpcCredentialRotationManager manager) + throws Exception { + start.await(); + action.accept(manager); + return null; + } + + /** The losing rotator of a race is refused, which is the correct outcome, not a failure. */ + private static void rotateIgnoringRefusal(GrpcCredentialRotationManager manager, long next) { + try { + manager.rotate(gen(next), NOW); + } catch (IllegalArgumentException refused) { + // superseded by the other rotator + } + } + + private static GrpcCredentialGeneration gen(long generation) { + return new GrpcCredentialGeneration( + generation, "vault://v" + generation, NOW, NOW.plusSeconds(3600)); + } +} diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java index 7976eea..767142e 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcSerializedStreamWriterTest.java @@ -9,6 +9,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -23,7 +24,7 @@ class GrpcSerializedStreamWriterTest { private GrpcSerializedStreamWriter writer( GrpcFlowControlPolicy policy, long messageSize) { return new GrpcSerializedStreamWriter<>( - STREAM, GrpcStreamSequence.forNewStream(STREAM), policy, sent::add, () -> messageSize); + STREAM, GrpcStreamSequence.forNewStream(STREAM), policy, sent::add, payload -> messageSize); } @Test @@ -38,7 +39,7 @@ class GrpcSerializedStreamWriterTest { assertThat(result.queued()).isTrue(); assertThat(sent).isEmpty(); - assertThat(writer.flush()).containsExactly(1L); + assertThat(writer.flush(() -> true)).containsExactly(1L); assertThat(sent).hasSize(1); } @@ -76,7 +77,7 @@ class GrpcSerializedStreamWriterTest { } assertThat(sent).isEmpty(); - List flushed = writer.flush(); + List flushed = writer.flush(() -> true); assertThat(flushed).hasSize(producers * perProducer); assertThat(flushed).isSorted(); assertThat(flushed).doesNotHaveDuplicates(); @@ -121,7 +122,7 @@ class GrpcSerializedStreamWriterTest { GrpcStreamSequence.forNewStream(STREAM), new GrpcFlowControlPolicy(10, 1000L, 9, GrpcSlowConsumerPolicy.DROP_OLDEST), sent::add, - nextSize::get); + payload -> nextSize.get()); nextSize.set(900L); assertThat(writer.write(GrpcStreamEnvelope.Kind.LIVE, "large", null, null, true)) @@ -149,7 +150,7 @@ class GrpcSerializedStreamWriterTest { GrpcStreamSequence.forNewStream(STREAM), new GrpcFlowControlPolicy(10, 700L, 9, GrpcSlowConsumerPolicy.DROP_OLDEST), sent::add, - nextSize::get); + payload -> nextSize.get()); writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true); writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true); @@ -179,7 +180,7 @@ class GrpcSerializedStreamWriterTest { void cancellingKeepsTheLastFlushedPositionAsEvidence() { GrpcSerializedStreamWriter writer = writer(GrpcFlowControlPolicy.stable(), 8L); writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true); - writer.flush(); + writer.flush(() -> true); writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true); long lastFlushed = writer.discardQueued(); @@ -189,4 +190,54 @@ class GrpcSerializedStreamWriterTest { assertThat(writer.queuedMessages()).isZero(); assertThat(sent).hasSize(1); } + + @Test + void theByteBoundCountsEachPayloadsOwnSize() { + // given + GrpcSerializedStreamWriter writer = + new GrpcSerializedStreamWriter<>( + STREAM, + GrpcStreamSequence.forNewStream(STREAM), + new GrpcFlowControlPolicy(10, 10L, 9, GrpcSlowConsumerPolicy.TERMINATE), + sent::add, + payload -> payload.length()); + writer.write(GrpcStreamEnvelope.Kind.LIVE, "eight-by", null, null, true); + + // when + GrpcStreamWriteResult result = + writer.write(GrpcStreamEnvelope.Kind.LIVE, "four", null, null, true); + + // then + assertThat(result).isEqualTo(GrpcStreamWriteResult.OVERFLOW_TERMINATING); + } + + @Test + void aWriteWhileTheTransportIsNotReadyTellsTheProducerToPause() { + // given + GrpcSerializedStreamWriter writer = writer(GrpcFlowControlPolicy.stable(), 8L); + + // when + GrpcStreamWriteResult result = + writer.write(GrpcStreamEnvelope.Kind.LIVE, "row", null, null, false); + + // then + assertThat(result) + .extracting(r -> r, GrpcStreamWriteResult::queued) + .containsExactly(GrpcStreamWriteResult.ACCEPTED_PAUSE, true); + } + + @Test + void flushStopsWhenTheTransportStopsBeingReady() { + // given + GrpcSerializedStreamWriter writer = writer(GrpcFlowControlPolicy.stable(), 8L); + writer.write(GrpcStreamEnvelope.Kind.LIVE, "first", null, null, true); + writer.write(GrpcStreamEnvelope.Kind.LIVE, "second", null, null, true); + AtomicInteger readyChecks = new AtomicInteger(); + + // when + List flushed = writer.flush(() -> readyChecks.getAndIncrement() == 0); + + // then + assertThat(flushed).containsExactly(1L); + } } diff --git a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java index 8849c52..f335220 100644 --- a/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java +++ b/grpc/grpc-policy/src/test/java/dev/caskeleton/grpc/streaming/GrpcStreamAdmissionConcurrencyTest.java @@ -45,4 +45,17 @@ class GrpcStreamAdmissionConcurrencyTest { assertThat(admission.trackedCallers()).isZero(); } } + + @Test + void aCallerRefusedByTheGlobalBoundLeavesNoTrackedEntry() { + // given + GrpcStreamAdmission admission = new GrpcStreamAdmission(1, 1); + admission.tryAdmit("first"); + + // when + admission.tryAdmit("refused"); + + // then + assertThat(admission.trackedCallers()).isOne(); + } } diff --git a/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java b/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java index c25f67a..a6b346c 100644 --- a/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java +++ b/grpc/grpc-server/src/main/java/dev/caskeleton/grpc/server/GrpcAdmissionController.java @@ -1,7 +1,5 @@ package dev.caskeleton.grpc.server; -import java.util.concurrent.atomic.AtomicInteger; - /** * Refuses work the server cannot do, before it spends anything on it. * @@ -14,9 +12,12 @@ public final class GrpcAdmissionController { private final int maxConcurrentCalls; private final int maxQueuedCalls; - private final AtomicInteger inFlight = new AtomicInteger(); - private final AtomicInteger queued = new AtomicInteger(); - private final AtomicInteger rejected = new AtomicInteger(); + + // One lock for both counters: every decision reads one and changes the other, and a check on an + // atomic followed by an increment lets racing callers all pass the same check. + private int inFlight; + private int queued; + private int rejected; /** Bounds concurrency and queueing separately. */ public GrpcAdmissionController(int maxConcurrentCalls, int maxQueuedCalls) { @@ -48,56 +49,54 @@ public final class GrpcAdmissionController { } /** Admits a call, queues it, or refuses it. */ - public Decision tryAdmit() { - int running = inFlight.get(); - if (running < maxConcurrentCalls) { - inFlight.incrementAndGet(); - return new Decision(true, inFlight.get(), queued.get(), "within the concurrency bound"); + public synchronized Decision tryAdmit() { + if (inFlight < maxConcurrentCalls) { + inFlight++; + return new Decision(true, inFlight, queued, "within the concurrency bound"); } - int waiting = queued.get(); - if (waiting < maxQueuedCalls) { - queued.incrementAndGet(); - return new Decision(true, running, queued.get(), "queued within the admission bound"); + if (queued < maxQueuedCalls) { + queued++; + return new Decision(true, inFlight, queued, "queued within the admission bound"); } - rejected.incrementAndGet(); + rejected++; return new Decision( false, - running, - waiting, + inFlight, + queued, "at capacity: " - + running + + inFlight + " in flight and " - + waiting + + queued + " queued. Refusing is better than accepting work whose caller will have given up."); } /** Moves a queued call into execution. */ - public void promoteFromQueue() { - if (queued.get() > 0) { - queued.decrementAndGet(); - inFlight.incrementAndGet(); + public synchronized void promoteFromQueue() { + if (queued > 0) { + queued--; + inFlight++; } } /** Releases a finished call. */ - public void release() { - if (inFlight.get() > 0) { - inFlight.decrementAndGet(); + public synchronized void release() { + if (inFlight > 0) { + inFlight--; } } /** How many calls are executing. */ - public int inFlight() { - return inFlight.get(); + public synchronized int inFlight() { + return inFlight; } /** How many calls are waiting. */ - public int queued() { - return queued.get(); + public synchronized int queued() { + return queued; } /** How many calls have been refused. */ - public int rejected() { - return rejected.get(); + public synchronized int rejected() { + return rejected; } } diff --git a/grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java b/grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java new file mode 100644 index 0000000..895fdf5 --- /dev/null +++ b/grpc/grpc-server/src/test/java/dev/caskeleton/grpc/server/GrpcAdmissionControllerTest.java @@ -0,0 +1,55 @@ +package dev.caskeleton.grpc.server; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.junit.jupiter.api.Test; + +class GrpcAdmissionControllerTest { + + private static final int CALLERS = 8; + private static final int ROUNDS = 500; + + @Test + void racingCallersNeverGetPastTheConcurrencyAndQueueBounds() throws Exception { + // when + int mostAdmitted = mostAdmittedOverRounds(); + + // then + assertThat(mostAdmitted).isEqualTo(2); + } + + /** Races {@link #CALLERS} callers at a 1-running, 1-queued controller, round after round. */ + private static int mostAdmittedOverRounds() throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(CALLERS); + try { + int most = 0; + for (int round = 0; round < ROUNDS; round++) { + GrpcAdmissionController controller = new GrpcAdmissionController(1, 1); + CyclicBarrier start = new CyclicBarrier(CALLERS); + List> admitted = new ArrayList<>(); + for (int caller = 0; caller < CALLERS; caller++) { + admitted.add( + pool.submit( + () -> { + start.await(); + return controller.tryAdmit().admitted(); + })); + } + int count = 0; + for (Future result : admitted) { + count += result.get() ? 1 : 0; + } + most = Math.max(most, count); + } + return most; + } finally { + pool.shutdownNow(); + } + } +}