feat: grpc 기능 deep 구현

This commit is contained in:
DongHyeonka
2026-08-31 07:28:39 +09:00
parent a24ece9cf7
commit 21234e38cd
400 changed files with 40217 additions and 4 deletions
@@ -0,0 +1,11 @@
apply plugin: 'java-library'
// Resilience and discovery capabilities that Stable refuses: read-only unary hedging, the custom
// name resolver SPI, the custom load balancer SPI, and the proxyless xDS experimental profile.
dependencies {
api project(':grpc:grpc-core-api')
api project(':grpc:grpc-policy')
api project(':grpc:grpc-client')
api project(':grpc:grpc-discovery')
api project(':grpc-advanced:grpc-advanced-bootstrap')
}
@@ -0,0 +1,91 @@
# This is a Gradle generated file for dependency locking.
# Manual edits can break the build and are not advised.
# This file is expected to be part of source control.
com.github.ben-manes.caffeine:caffeine:3.2.4=annotationProcessor,testAnnotationProcessor
com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor
com.github.spotbugs:spotbugs-annotations:4.10.2=spotbugs
com.github.spotbugs:spotbugs:4.10.2=spotbugs
com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs
com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,testAnnotationProcessor
com.google.auto.value:auto-value-annotations:1.9=annotationProcessor,testAnnotationProcessor
com.google.auto:auto-common:1.2.2=annotationProcessor,testAnnotationProcessor
com.google.code.findbugs:jsr305:3.0.2=checkstyle,compileClasspath,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
com.google.code.gson:gson:2.13.2=spotbugs
com.google.errorprone:error_prone_annotation:2.49.0=annotationProcessor,testAnnotationProcessor
com.google.errorprone:error_prone_annotations:2.28.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.google.errorprone:error_prone_annotations:2.41.0=spotbugs
com.google.errorprone:error_prone_annotations:2.47.0=checkstyle
com.google.errorprone:error_prone_annotations:2.49.0=annotationProcessor,testAnnotationProcessor
com.google.errorprone:error_prone_check_api:2.49.0=annotationProcessor,testAnnotationProcessor
com.google.errorprone:error_prone_core:2.49.0=annotationProcessor,testAnnotationProcessor
com.google.googlejavaformat:google-java-format:1.35.0=annotationProcessor,testAnnotationProcessor
com.google.guava:failureaccess:1.0.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.google.guava:failureaccess:1.0.3=annotationProcessor,checkstyle,testAnnotationProcessor
com.google.guava:guava:33.2.1-android=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.google.guava:guava:33.5.0-jre=annotationProcessor,testAnnotationProcessor
com.google.guava:guava:33.6.0-jre=checkstyle
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
com.google.j2objc:j2objc-annotations:3.0.0=compileClasspath,testCompileClasspath
com.google.j2objc:j2objc-annotations:3.1=annotationProcessor,checkstyle,testAnnotationProcessor
com.google.protobuf:protobuf-java:4.33.2=annotationProcessor,testAnnotationProcessor
com.h3xstream.findsecbugs:findsecbugs-plugin:1.14.0=spotbugsPlugins
com.puppycrawl.tools:checkstyle:13.5.0=checkstyle
commons-beanutils:commons-beanutils:1.11.0=checkstyle
commons-collections:commons-collections:3.2.2=checkstyle
commons-io:commons-io:2.21.0=spotbugs
info.picocli:picocli:4.7.7=checkstyle
io.github.eisop:dataflow-errorprone:3.41.0-eisop1=annotationProcessor,testAnnotationProcessor
io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,testAnnotationProcessor
io.grpc:grpc-api:1.68.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.grpc:grpc-stub:1.68.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
javax.inject:javax.inject:1=annotationProcessor,testAnnotationProcessor
jaxen:jaxen:2.0.6=spotbugs
net.bytebuddy:byte-buddy:1.17.8=testCompileClasspath,testRuntimeClasspath
net.sf.saxon:Saxon-HE:12.9=checkstyle,spotbugs
org.antlr:antlr4-runtime:4.13.2=checkstyle
org.apache.bcel:bcel:6.12.0=spotbugs
org.apache.commons:commons-lang3:3.20.0=checkstyle,spotbugs
org.apache.commons:commons-text:1.15.0=spotbugs
org.apache.commons:commons-text:1.3=checkstyle
org.apache.httpcomponents:httpclient:4.5.13=checkstyle
org.apache.httpcomponents:httpcore:4.4.16=checkstyle
org.apache.logging.log4j:log4j-api:2.25.5=spotbugs
org.apache.logging.log4j:log4j-core:2.25.5=spotbugs
org.apache.maven.doxia:doxia-core:1.12.0=checkstyle
org.apache.maven.doxia:doxia-logging-api:1.12.0=checkstyle
org.apache.maven.doxia:doxia-module-xdoc:1.12.0=checkstyle
org.apache.maven.doxia:doxia-sink-api:1.12.0=checkstyle
org.apache.xbean:xbean-reflect:3.7=checkstyle
org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
org.assertj:assertj-core:3.27.7=testCompileClasspath,testRuntimeClasspath
org.checkerframework:checker-qual:3.42.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.codehaus.plexus:plexus-classworlds:2.6.0=checkstyle
org.codehaus.plexus:plexus-component-annotations:2.1.0=checkstyle
org.codehaus.plexus:plexus-container-default:2.1.0=checkstyle
org.codehaus.plexus:plexus-utils:3.3.0=checkstyle
org.dom4j:dom4j:2.2.0=spotbugs
org.javassist:javassist:3.28.0-GA=checkstyle
org.jspecify:jspecify:1.0.1=annotationProcessor,checkstyle,testAnnotationProcessor,testCompileClasspath
org.junit.jupiter:junit-jupiter-api:6.0.3=testCompileClasspath,testRuntimeClasspath
org.junit.jupiter:junit-jupiter-engine:6.0.3=testRuntimeClasspath
org.junit.jupiter:junit-jupiter-params:6.0.3=testCompileClasspath,testRuntimeClasspath
org.junit.jupiter:junit-jupiter:6.0.3=testCompileClasspath,testRuntimeClasspath
org.junit.platform:junit-platform-commons:6.0.3=testCompileClasspath,testRuntimeClasspath
org.junit.platform:junit-platform-engine:6.0.3=testRuntimeClasspath
org.junit.platform:junit-platform-launcher:6.0.3=testRuntimeClasspath
org.junit:junit-bom:6.0.3=testCompileClasspath,testRuntimeClasspath
org.junit:junit-bom:6.1.0=spotbugs
org.mockito:mockito-core:5.20.0=mockitoAgent
org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath
org.ow2.asm:asm-analysis:9.10.1=spotbugs
org.ow2.asm:asm-commons:9.10.1=spotbugs
org.ow2.asm:asm-tree:9.10.1=spotbugs
org.ow2.asm:asm-util:9.10.1=spotbugs
org.ow2.asm:asm:9.10.1=spotbugs
org.pcollections:pcollections:4.0.1=annotationProcessor,testAnnotationProcessor
org.reflections:reflections:0.10.2=checkstyle
org.slf4j:slf4j-api:2.0.18=spotbugs,spotbugsSlf4j
org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j
org.slf4j:slf4j-simple:2.0.18=checkstyle
org.xmlresolver:xmlresolver:5.3.3=checkstyle,spotbugs
empty=
@@ -0,0 +1,75 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
/**
* A custom name resolver, with the safety rules applied on the way in.
*
* <p>Everything an update can do wrong is checked here rather than by the listener, because the
* listener is the channel and the channel will believe whatever it is told. Stale revisions and
* empty endpoint sets are dropped rather than propagated, and once closed the resolver accepts
* nothing at all.
*/
public final class GrpcCustomResolver implements AutoCloseable {
private final String authority;
private final Consumer<GrpcResolverUpdate> listener;
private final AtomicReference<GrpcEndpointSnapshot> applied = new AtomicReference<>();
private final AtomicBoolean closed = new AtomicBoolean();
/** Binds a resolver to the authority it resolves and the listener it feeds. */
public GrpcCustomResolver(String authority, Consumer<GrpcResolverUpdate> listener) {
if (authority == null || authority.isBlank()) {
throw new IllegalArgumentException("a resolver names the authority it resolves");
}
if (listener == null) {
throw new IllegalArgumentException("a resolver needs a listener to deliver updates to");
}
this.authority = authority;
this.listener = listener;
}
/**
* Offers an update.
*
* @return the violations that stopped it, empty when it was applied
*/
public List<String> offer(GrpcResolverUpdate update) {
if (closed.get()) {
return List.of(
"the resolver is closed; an update after close resurrects routing for a channel nobody "
+ "is using");
}
List<String> violations = GrpcResolverSafetyPolicy.violations(update, applied.get());
if (!violations.isEmpty()) {
return violations;
}
applied.set(update.snapshot());
listener.accept(update);
return List.of();
}
/** The snapshot currently in force. */
public Optional<GrpcEndpointSnapshot> currentSnapshot() {
return Optional.ofNullable(applied.get());
}
/** The authority this resolver answers for. */
public String authority() {
return authority;
}
/** Whether the resolver has been closed. */
public boolean closed() {
return closed.get();
}
@Override
public void close() {
closed.set(true);
}
}
@@ -0,0 +1,36 @@
package dev.caskeleton.grpc.advanced.discovery;
/**
* One endpoint a picker may choose, described only by what routing is allowed to consider.
*
* <p>The field list is the allowlist. Health, connectivity, weight and ejection are properties of
* the endpoint; a tenant id or a request field is a property of the caller, and routing on those
* turns a load balancer into a router with an authorization decision buried in it.
*/
public record GrpcEndpointCandidate(
String address, boolean healthy, boolean connected, int weight, boolean ejected) {
/** Requires an address and a sane weight. */
public GrpcEndpointCandidate {
if (address == null || address.isBlank()) {
throw new IllegalArgumentException("an endpoint candidate needs an address");
}
if (weight < 0) {
throw new IllegalArgumentException("a weight must not be negative");
}
if (weight > 1000) {
throw new IllegalArgumentException(
"a weight above 1000 is a scale nobody can reason about against the others");
}
}
/** A healthy, connected endpoint at the default weight. */
public static GrpcEndpointCandidate ready(String address) {
return new GrpcEndpointCandidate(address, true, true, 100, false);
}
/** Whether this endpoint may receive a request. */
public boolean selectable() {
return healthy && connected && !ejected && weight > 0;
}
}
@@ -0,0 +1,46 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.List;
import java.util.Set;
/**
* One resolver update: a monotonic revision and the complete endpoint set at that revision.
*
* <p>Complete, not a delta. A delta protocol needs both sides to agree on what they last saw, and a
* resolver that reconnects to its discovery source has no way to establish that; a full set at each
* revision makes a missed update harmless.
*
* <p>The revision is what makes a late update safe to drop. Without it, an update that arrives out
* of order replaces newer endpoints with older ones, and the channel routes to instances that were
* removed.
*/
public record GrpcEndpointSnapshot(long revision, String authority, List<String> endpoints) {
/** Requires a positive revision, an authority and a non-empty endpoint set. */
public GrpcEndpointSnapshot {
if (revision < 1) {
throw new IllegalArgumentException("resolver revisions are 1-based; got " + revision);
}
if (authority == null || authority.isBlank()) {
throw new IllegalArgumentException("a snapshot names the authority it resolves");
}
if (endpoints == null || endpoints.isEmpty()) {
throw new IllegalArgumentException(
"an empty endpoint set is refused; a resolver that reports zero endpoints during its own "
+ "outage would take the channel down with it");
}
if (endpoints.stream().anyMatch(endpoint -> endpoint == null || endpoint.isBlank())) {
throw new IllegalArgumentException("every endpoint must be a non-blank address");
}
endpoints = List.copyOf(endpoints);
if (Set.copyOf(endpoints).size() != endpoints.size()) {
throw new IllegalArgumentException(
"duplicate endpoints skew a round-robin picker towards whichever address is repeated");
}
}
/** Whether this snapshot supersedes {@code other}. */
public boolean supersedes(GrpcEndpointSnapshot other) {
return other == null || (authority.equals(other.authority()) && revision > other.revision());
}
}
@@ -0,0 +1,58 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.Optional;
/**
* Which endpoint a picker chose, or why it chose none.
*
* <p>{@link Verdict#DETERMINISTIC_FALLBACK} is separate from {@link Verdict#NO_ENDPOINT_AVAILABLE}
* because they say different things about the picker. The first means custom logic failed and the
* platform took over, which is a defect to fix; the second means there was genuinely nowhere to
* send the request, which is an outage. A picker that reports both the same way hides its own bugs
* inside the backend's.
*/
public record GrpcLoadBalancerDecision(
Verdict verdict, Optional<GrpcEndpointCandidate> chosen, String reason) {
/** What the picker decided. */
public enum Verdict {
/** The picker chose an endpoint. */
PICKED,
/** The picker failed; the platform chose deterministically instead. */
DETERMINISTIC_FALLBACK,
/** Nothing was selectable. */
NO_ENDPOINT_AVAILABLE
}
/** Requires an endpoint on the two verdicts that have one. */
public GrpcLoadBalancerDecision {
if (verdict == null || chosen == null) {
throw new IllegalArgumentException("a picker decision has a verdict and the Optional");
}
if (reason == null || reason.isBlank()) {
throw new IllegalArgumentException("a picker decision explains itself");
}
if (verdict == Verdict.NO_ENDPOINT_AVAILABLE && chosen.isPresent()) {
throw new IllegalArgumentException("a decision with no endpoint available carries none");
}
if (verdict != Verdict.NO_ENDPOINT_AVAILABLE && chosen.isEmpty()) {
throw new IllegalArgumentException("a decision that picked carries the endpoint it picked");
}
}
/** The picker's own choice. */
public static GrpcLoadBalancerDecision picked(GrpcEndpointCandidate endpoint, String reason) {
return new GrpcLoadBalancerDecision(Verdict.PICKED, Optional.of(endpoint), reason);
}
/** The platform's fallback after a picker failure. */
public static GrpcLoadBalancerDecision fallback(GrpcEndpointCandidate endpoint, String reason) {
return new GrpcLoadBalancerDecision(
Verdict.DETERMINISTIC_FALLBACK, Optional.of(endpoint), reason);
}
/** Nothing was selectable. */
public static GrpcLoadBalancerDecision none(String reason) {
return new GrpcLoadBalancerDecision(Verdict.NO_ENDPOINT_AVAILABLE, Optional.empty(), reason);
}
}
@@ -0,0 +1,29 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.List;
/**
* Chooses one endpoint from the candidates the resolver supplied.
*
* <p>The signature is the safety property. A picker receives a list of candidates and nothing else:
* no request, no metadata, no caller. It therefore cannot route on a tenant, and the rule "business
* data is not a routing input" is enforced by there being no business data to reach.
*/
@FunctionalInterface
public interface GrpcLoadBalancerPicker {
/**
* Picks an endpoint.
*
* @param candidates the selectable endpoints, never empty
* @return the chosen endpoint, which must be one of {@code candidates}
*/
GrpcEndpointCandidate pick(List<GrpcEndpointCandidate> candidates);
/** Round-robin, as the deterministic default and the fallback. */
static GrpcLoadBalancerPicker roundRobin() {
java.util.concurrent.atomic.AtomicInteger cursor =
new java.util.concurrent.atomic.AtomicInteger();
return candidates -> candidates.get(Math.floorMod(cursor.getAndIncrement(), candidates.size()));
}
}
@@ -0,0 +1,76 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.List;
/**
* Runs a custom picker and refuses to let it do something the resolver did not authorise.
*
* <p>Two rules. A picker may only return an endpoint the resolver supplied, because a picker that
* can invent an address can send a request anywhere; and a picker that throws produces a
* deterministic fallback rather than a failed call, because a picker bug should degrade the
* balancing rather than the availability.
*
* <p>Weighted and load-aware pickers are not refused here, but the plan requires evidence before
* they ship: {@link #requiresLoadEvidence} names which shapes those are.
*/
public final class GrpcLoadBalancerSafetyPolicy {
private final GrpcLoadBalancerPicker picker;
private final GrpcLoadBalancerPicker fallback;
/** Wraps a custom picker with the platform's deterministic fallback. */
public GrpcLoadBalancerSafetyPolicy(GrpcLoadBalancerPicker picker) {
this(picker, GrpcLoadBalancerPicker.roundRobin());
}
/** Wraps a custom picker with an explicit fallback. */
public GrpcLoadBalancerSafetyPolicy(
GrpcLoadBalancerPicker picker, GrpcLoadBalancerPicker fallback) {
if (picker == null || fallback == null) {
throw new IllegalArgumentException("a safety policy needs a picker and a fallback");
}
this.picker = picker;
this.fallback = fallback;
}
/** Picks an endpoint, or explains why none was chosen. */
public GrpcLoadBalancerDecision pick(List<GrpcEndpointCandidate> candidates) {
if (candidates == null) {
throw new IllegalArgumentException("a candidate list is required");
}
List<GrpcEndpointCandidate> selectable =
candidates.stream().filter(GrpcEndpointCandidate::selectable).toList();
if (selectable.isEmpty()) {
return GrpcLoadBalancerDecision.none(
"no endpoint is healthy, connected, un-ejected and non-zero weight");
}
GrpcEndpointCandidate chosen;
try {
chosen = picker.pick(selectable);
} catch (RuntimeException pickerFailure) {
return GrpcLoadBalancerDecision.fallback(
fallback.pick(selectable),
"the custom picker threw ("
+ pickerFailure.getClass().getSimpleName()
+ "); falling back deterministically rather than failing the call");
}
if (chosen == null || !selectable.contains(chosen)) {
return GrpcLoadBalancerDecision.fallback(
fallback.pick(selectable),
"the custom picker returned an endpoint the resolver did not supply; a picker that can "
+ "invent an address can send a request anywhere");
}
return GrpcLoadBalancerDecision.picked(chosen, "chosen by the custom picker");
}
/**
* Whether a picker of this shape needs performance, fairness and failover evidence before it
* ships.
*
* @param loadAware whether the picker uses reported load or latency
* @param weighted whether the picker uses endpoint weights
*/
public static boolean requiresLoadEvidence(boolean loadAware, boolean weighted) {
return loadAware || weighted;
}
}
@@ -0,0 +1,92 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Pattern;
/**
* What a custom resolver is allowed to say, and what it may never carry.
*
* <p>A resolver runs inside the channel and speaks to something outside the deployment. Everything
* it can put into an update is therefore attacker-influenced in the worst case and
* operator-influenced in the ordinary one, so the safety rules are about limiting what an update
* can change: addresses and a validated service config, never a credential and never business
* metadata.
*/
public final class GrpcResolverSafetyPolicy {
private static final Pattern AUTHORITY =
Pattern.compile("[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:\\d{1,5})?");
private static final Pattern CREDENTIAL_SHAPED =
Pattern.compile("(?i).*(authorization|bearer|password|secret|token|api[_-]?key).*");
private GrpcResolverSafetyPolicy() {}
/**
* Every problem with an update, given what was last accepted.
*
* @param lastAccepted the newest snapshot already applied, or null when none has been
* @return an empty list when the update is safe to apply
*/
public static List<String> violations(
GrpcResolverUpdate update, GrpcEndpointSnapshot lastAccepted) {
if (update == null) {
throw new IllegalArgumentException("an update is required");
}
List<String> violations = new ArrayList<>();
GrpcEndpointSnapshot snapshot = update.snapshot();
if (!AUTHORITY.matcher(snapshot.authority()).matches()) {
violations.add(
"authority '"
+ snapshot.authority()
+ "' is not a plain host or host:port; a resolver that can change the authority can "
+ "change which certificate the channel accepts");
}
if (!snapshot.supersedes(lastAccepted)) {
violations.add(
"revision "
+ snapshot.revision()
+ " does not supersede the applied revision "
+ (lastAccepted == null ? "none" : lastAccepted.revision())
+ "; applying it would replace newer endpoints with older ones");
}
update
.serviceConfigJson()
.ifPresent(
config -> {
if (CREDENTIAL_SHAPED.matcher(config).find()) {
violations.add(
"the pushed service config contains a credential-shaped field; a resolver "
+ "supplies addresses and policy, never authentication material");
}
});
return List.copyOf(violations);
}
/**
* Whether a closed resolver's update should be applied.
*
* <p>Always false. A resolver that keeps delivering after close is one whose discovery source has
* not noticed the channel is gone, and applying its updates resurrects routing for a channel
* nobody is using.
*/
public static boolean acceptAfterClose() {
return false;
}
/** Whether a resolver may supply caller identity or business metadata. Always false. */
public static boolean mayCarryBusinessMetadata() {
return false;
}
/** The service config an update may contribute, once validated. */
public static Optional<String> acceptedServiceConfig(
GrpcResolverUpdate update, GrpcEndpointSnapshot lastAccepted) {
return violations(update, lastAccepted).isEmpty()
? update.serviceConfigJson()
: Optional.empty();
}
}
@@ -0,0 +1,34 @@
package dev.caskeleton.grpc.advanced.discovery;
import java.util.Optional;
/**
* A resolver's report: endpoints, and optionally the service config that goes with them.
*
* <p>The service config is optional and, when present, is validated as if a human had written it. A
* resolver that can push retry policy is a resolver that can turn on retries for a non-idempotent
* method from outside the codebase, and the fact that a control plane sent it is not evidence that
* anyone reviewed it.
*/
public record GrpcResolverUpdate(
GrpcEndpointSnapshot snapshot, Optional<String> serviceConfigJson) {
/** Requires a snapshot and the Optional. */
public GrpcResolverUpdate {
if (snapshot == null || serviceConfigJson == null) {
throw new IllegalArgumentException("a resolver update carries a snapshot and the Optional");
}
serviceConfigJson.ifPresent(
config -> {
if (config.isBlank()) {
throw new IllegalArgumentException(
"a present service config must not be blank; absent and empty are different states");
}
});
}
/** An update with endpoints only. */
public static GrpcResolverUpdate endpointsOnly(GrpcEndpointSnapshot snapshot) {
return new GrpcResolverUpdate(snapshot, Optional.empty());
}
}
@@ -0,0 +1,70 @@
package dev.caskeleton.grpc.advanced.resilience;
import java.util.concurrent.atomic.AtomicLong;
/**
* Caps duplicate attempts as a fraction of real traffic.
*
* <p>Necessary for the same reason a retry budget is, and more urgently. A retry happens after a
* failure; a hedge happens on a call that might have succeeded, so a fleet that hedges without a
* budget doubles its backend load in the steady state and doubles it again the moment latency
* rises.
*/
public final class GrpcHedgingBudget {
private final long maxTokens;
private final long tokensPerHedge;
private final AtomicLong tokens;
/**
* A budget that starts full.
*
* @param ratio hedges permitted per completed call, e.g. 0.1 for one hedge in ten
* @param maxTokens how much credit may accumulate, which bounds a burst after a quiet period
*/
public static GrpcHedgingBudget of(double ratio, long maxTokens) {
if (ratio <= 0.0d || ratio > 0.5d) {
throw new IllegalArgumentException(
"a hedging ratio above 0.5 means more than half of all calls are duplicated, which is a "
+ "load decision rather than a latency one");
}
if (maxTokens < 1) {
throw new IllegalArgumentException("a budget needs at least one token");
}
return new GrpcHedgingBudget(maxTokens, Math.round(1.0d / ratio));
}
private GrpcHedgingBudget(long maxTokens, long tokensPerHedge) {
this.maxTokens = maxTokens;
this.tokensPerHedge = tokensPerHedge;
this.tokens = new AtomicLong(maxTokens);
}
/** Takes the credit for one hedge, if there is any. */
public boolean tryConsume() {
while (true) {
long observed = tokens.get();
if (observed < tokensPerHedge) {
return false;
}
if (tokens.compareAndSet(observed, observed - tokensPerHedge)) {
return true;
}
}
}
/** Records a completed call, which earns credit back. */
public void recordCompletion() {
tokens.updateAndGet(observed -> Math.min(maxTokens, observed + 1L));
}
/** How much credit is left. */
public long availableTokens() {
return tokens.get();
}
/** Whether another hedge could be afforded. */
public boolean exhausted() {
return tokens.get() < tokensPerHedge;
}
}
@@ -0,0 +1,72 @@
package dev.caskeleton.grpc.advanced.resilience;
import dev.caskeleton.grpc.core.RpcType;
import dev.caskeleton.grpc.policy.GrpcMethodPolicy;
import dev.caskeleton.grpc.policy.RpcIdempotencyProfile;
import dev.caskeleton.grpc.resilience.GrpcRetryOwner;
import java.util.Optional;
/**
* Whether a method may be hedged at all.
*
* <p>Read-only unary, and nothing else. A hedged mutation runs twice by design rather than by
* accident — both attempts are in flight, both may reach the server, and an idempotency key does
* not help because the second attempt is not a retry of a failure but a duplicate of a success in
* progress. A hedged stream is worse still: two streams deliver two prefixes.
*/
public final class GrpcHedgingEligibility {
private GrpcHedgingEligibility() {}
/**
* Why {@code policy} may not be hedged, or empty when it may.
*
* @return a refusal reason, or empty when hedging is permitted
*/
public static Optional<String> refusalReason(GrpcMethodPolicy policy, GrpcRetryOwner retryOwner) {
if (policy == null || retryOwner == null) {
throw new IllegalArgumentException("eligibility needs a method policy and a retry owner");
}
if (policy.rpcType() != RpcType.UNARY) {
return Optional.of(
"method '"
+ policy.method().canonical()
+ "' is "
+ policy.rpcType()
+ "; two hedged streams deliver two prefixes");
}
if (policy.idempotency() != RpcIdempotencyProfile.READ_ONLY) {
return Optional.of(
"method '"
+ policy.method().canonical()
+ "' is "
+ policy.idempotency()
+ "; a hedged mutation runs twice by design, and an idempotency key does not help "
+ "because the second attempt duplicates a success in progress rather than retrying a "
+ "failure");
}
if (!retryOwner.hedgingAllowed()) {
return Optional.of(
"retry owner is " + retryOwner + ", which does not permit in-process hedging");
}
return Optional.empty();
}
/** Whether {@code policy} may be hedged. */
public static boolean eligible(GrpcMethodPolicy policy, GrpcRetryOwner retryOwner) {
return refusalReason(policy, retryOwner).isEmpty();
}
/**
* Fails when a method may not be hedged.
*
* @throws IllegalStateException with the reason
*/
public static void require(GrpcMethodPolicy policy, GrpcRetryOwner retryOwner) {
refusalReason(policy, retryOwner)
.ifPresent(
reason -> {
throw new IllegalStateException("hedging refused: " + reason);
});
}
}
@@ -0,0 +1,51 @@
package dev.caskeleton.grpc.advanced.resilience;
import java.time.Duration;
/**
* Duplicate in-flight attempts for a read, and the two bounds that keep them affordable.
*
* <p>Hedging trades backend load for tail latency: a second attempt goes out before the first has
* failed, so a slow replica stops mattering. The cost is that every hedged call may cost two, and
* it costs two precisely when the backend is already slow — which is why the attempt cap starts at
* 2 and the delay is required to be meaningfully above the median.
*/
public record GrpcHedgingPolicy(int maxAttempts, Duration hedgingDelay, Duration totalDeadline) {
/** The initial cap. Raising it is a deliberate decision with load evidence behind it. */
public static final int INITIAL_MAX_ATTEMPTS = 2;
/** Refuses a policy whose duplicate load is unbounded or whose delay is meaningless. */
public GrpcHedgingPolicy {
if (maxAttempts < 2) {
throw new IllegalArgumentException("hedging means at least two attempts; got " + maxAttempts);
}
if (maxAttempts > INITIAL_MAX_ATTEMPTS) {
throw new IllegalArgumentException(
"hedging is capped at "
+ INITIAL_MAX_ATTEMPTS
+ " attempts until load evidence justifies more; each extra attempt multiplies "
+ "backend load exactly when the backend is already slow");
}
if (hedgingDelay == null || hedgingDelay.isNegative()) {
throw new IllegalArgumentException("a hedging delay must be present and non-negative");
}
if (hedgingDelay.isZero()) {
throw new IllegalArgumentException(
"a zero hedging delay sends every attempt at once, which doubles load for every call "
+ "rather than for the slow ones");
}
if (totalDeadline == null || totalDeadline.isZero() || totalDeadline.isNegative()) {
throw new IllegalArgumentException("hedging needs a total deadline to fit inside");
}
if (hedgingDelay.compareTo(totalDeadline) >= 0) {
throw new IllegalArgumentException(
"the hedging delay is at or above the total deadline, so the second attempt never starts");
}
}
/** A policy that hedges once after {@code hedgingDelay}. */
public static GrpcHedgingPolicy hedgeOnce(Duration hedgingDelay, Duration totalDeadline) {
return new GrpcHedgingPolicy(INITIAL_MAX_ATTEMPTS, hedgingDelay, totalDeadline);
}
}
@@ -0,0 +1,49 @@
package dev.caskeleton.grpc.advanced.resilience;
import java.time.Duration;
/**
* What a hedged call cost and what it saved.
*
* <p>Both numbers, because hedging is a trade and a dashboard that shows only the latency
* improvement makes it look free. {@code duplicateBackendCalls} is what the backend team sees, and
* {@code cancelledLoserAttempts} is how much of that work was thrown away.
*/
public record GrpcHedgingResult(
int attemptsIssued,
int winningAttempt,
int cancelledLoserAttempts,
int duplicateBackendCalls,
Duration observedLatency) {
/** Requires coherent counts. */
public GrpcHedgingResult {
if (attemptsIssued < 1) {
throw new IllegalArgumentException("a hedged call issues at least one attempt");
}
if (winningAttempt < 1 || winningAttempt > attemptsIssued) {
throw new IllegalArgumentException("the winning attempt is one of the attempts issued");
}
if (cancelledLoserAttempts < 0 || cancelledLoserAttempts > attemptsIssued - 1) {
throw new IllegalArgumentException(
"at most every attempt but the winner can be a cancelled loser");
}
if (duplicateBackendCalls < 0 || duplicateBackendCalls > attemptsIssued - 1) {
throw new IllegalArgumentException(
"duplicate backend calls are the attempts beyond the first");
}
if (observedLatency == null || observedLatency.isNegative()) {
throw new IllegalArgumentException("a hedged call records its latency");
}
}
/** A call that did not need to hedge. */
public static GrpcHedgingResult firstAttemptWon(Duration latency) {
return new GrpcHedgingResult(1, 1, 0, 0, latency);
}
/** Whether this call actually issued a duplicate. */
public boolean hedged() {
return attemptsIssued > 1;
}
}
@@ -0,0 +1,60 @@
package dev.caskeleton.grpc.advanced.xds;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
/**
* What happens when the control plane goes away.
*
* <p>Last-known-good, with a bound. Serving forever from a stale snapshot means a decommissioned
* backend keeps receiving traffic indefinitely; failing immediately means a control-plane restart
* takes every client down with it. The bound is where the deployment decides which risk it prefers,
* and it has to be stated rather than inherited.
*/
public record GrpcXdsFailurePolicy(
Duration maxStaleness, boolean failFastOnMissingResource, Duration initialFetchTimeout) {
/** Refuses a policy without a staleness bound. */
public GrpcXdsFailurePolicy {
if (maxStaleness == null || maxStaleness.isZero() || maxStaleness.isNegative()) {
throw new IllegalArgumentException(
"last-known-good needs a staleness bound; without one a decommissioned backend keeps "
+ "receiving traffic indefinitely");
}
if (initialFetchTimeout == null
|| initialFetchTimeout.isZero()
|| initialFetchTimeout.isNegative()) {
throw new IllegalArgumentException(
"a client with no snapshot yet needs a bound on how long it waits before failing");
}
}
/** The default: fifteen minutes of last-known-good, fail fast on a resource that vanished. */
public static GrpcXdsFailurePolicy standard() {
return new GrpcXdsFailurePolicy(Duration.ofMinutes(15), true, Duration.ofSeconds(15));
}
/** What a client should do given the newest snapshot it holds. */
public Decision decide(Optional<GrpcXdsResourceSnapshot> snapshot, Instant now) {
if (snapshot == null || now == null) {
throw new IllegalArgumentException("a decision needs the snapshot Optional and a moment");
}
if (snapshot.isEmpty()) {
return Decision.NO_SNAPSHOT_YET;
}
return snapshot.get().ageAt(now).compareTo(maxStaleness) > 0
? Decision.STALE_BEYOND_BOUND
: Decision.SERVE_LAST_KNOWN_GOOD;
}
/** What the client does about a control-plane outage. */
public enum Decision {
/** Nothing has arrived yet; wait until the initial fetch timeout, then fail. */
NO_SNAPSHOT_YET,
/** Keep routing on the snapshot in hand. */
SERVE_LAST_KNOWN_GOOD,
/** The snapshot is older than the bound; stop trusting it. */
STALE_BEYOND_BOUND
}
}
@@ -0,0 +1,52 @@
package dev.caskeleton.grpc.advanced.xds;
import dev.caskeleton.grpc.resilience.GrpcRetryOwner;
import java.net.URI;
/**
* A proxyless xDS deployment's configuration.
*
* <p>xDS moves routing, load balancing, retries and often mTLS out of the application and into a
* control plane. The consequence this profile encodes is that the application must stop configuring
* them: retry policy defined in both places is defined twice, and which one wins depends on
* resolution order rather than on anyone's decision.
*/
public record GrpcXdsProfile(
URI target,
String bootstrapReference,
String resourceNamespace,
GrpcRetryOwner retryOwner,
boolean controlPlaneMutualTls) {
/** The only scheme an xDS target may use. */
public static final String XDS_SCHEME = "xds";
/** Refuses a profile that would leave retries or routing owned in two places. */
public GrpcXdsProfile {
if (target == null || !XDS_SCHEME.equals(target.getScheme())) {
throw new IllegalArgumentException(
"an xDS profile needs an 'xds:///' target; got '" + target + "'");
}
if (bootstrapReference == null || bootstrapReference.isBlank()) {
throw new IllegalArgumentException(
"xDS needs a bootstrap reference; without one the client has no control plane to ask");
}
if (resourceNamespace == null || resourceNamespace.isBlank()) {
throw new IllegalArgumentException(
"an xDS profile names its resource namespace; a client that subscribes to everything "
+ "receives another team's routing");
}
if (retryOwner != GrpcRetryOwner.SERVICE_MESH) {
throw new IllegalArgumentException(
"xDS routing means the control plane owns retries; a retry owner of "
+ retryOwner
+ " would define retry policy in two places, and which wins depends on resolution "
+ "order rather than on a decision");
}
if (!controlPlaneMutualTls) {
throw new IllegalArgumentException(
"the control-plane connection carries routing and often certificates; it is authenticated "
+ "in both directions or it is a channel that can be impersonated");
}
}
}
@@ -0,0 +1,40 @@
package dev.caskeleton.grpc.advanced.xds;
import java.time.Instant;
import java.util.List;
/**
* What the control plane last said, and when.
*
* <p>The timestamp is what makes last-known-good usable. A snapshot with no age cannot answer
* whether the control plane has been silent for a minute or a day, and those are a transient blip
* and a serious incident.
*/
public record GrpcXdsResourceSnapshot(
String versionInfo, List<String> resourceNames, Instant receivedAt) {
/** Requires a version, at least one resource and a receipt time. */
public GrpcXdsResourceSnapshot {
if (versionInfo == null || versionInfo.isBlank()) {
throw new IllegalArgumentException("an xDS snapshot carries the control plane's version");
}
if (resourceNames == null || resourceNames.isEmpty()) {
throw new IllegalArgumentException(
"an empty resource set is not a snapshot; a control plane that returns nothing has not "
+ "told the client its routing was removed");
}
if (receivedAt == null) {
throw new IllegalArgumentException(
"a snapshot records when it arrived; without it, last-known-good cannot say how old it is");
}
resourceNames = List.copyOf(resourceNames);
}
/** How old this snapshot is at {@code now}. */
public java.time.Duration ageAt(Instant now) {
if (now == null) {
throw new IllegalArgumentException("an age needs a moment");
}
return java.time.Duration.between(receivedAt, now);
}
}
@@ -0,0 +1,100 @@
package dev.caskeleton.grpc.advanced.xds;
import dev.caskeleton.grpc.advanced.bootstrap.GrpcAdvancedCapability;
import dev.caskeleton.grpc.advanced.bootstrap.GrpcAdvancedFeatureFlags;
import dev.caskeleton.grpc.advanced.bootstrap.GrpcAdvancedModuleGuard;
import java.util.ArrayList;
import java.util.List;
/**
* Refuses to start an xDS channel that is not fully configured, and refuses to let xDS be described
* as Stable support.
*
* <p>The second refusal is the one worth having in code. xDS working in a deployment is not the
* same claim as the platform supporting it: it brings a control plane, its outage modes, its own
* security boundary and its own version skew, and the Stable support statement covers DNS and
* static targets. A support matrix that quietly widens is a support matrix nobody can rely on.
*/
public final class GrpcXdsStartupGuard {
private GrpcXdsStartupGuard() {}
/**
* Every reason an xDS channel may not start.
*
* @return an empty list when the profile and flags permit it
*/
public static List<String> startupBlockers(
GrpcXdsProfile profile, GrpcAdvancedFeatureFlags flags, boolean applicationDefinesRetries) {
if (profile == null || flags == null) {
throw new IllegalArgumentException("startup validation needs a profile and the flags");
}
List<String> blockers = new ArrayList<>();
if (!GrpcAdvancedModuleGuard.available(flags, GrpcAdvancedCapability.XDS)) {
blockers.add(
"the xds capability is not available; it is "
+ flags.gradeOf(GrpcAdvancedCapability.XDS)
+ " and "
+ (flags.flagSet(GrpcAdvancedCapability.XDS)
? "production has not approved it"
: "its flag is not set"));
}
if (applicationDefinesRetries) {
blockers.add(
"the application also defines retry policy; with xDS the control plane owns it, and "
+ "defining it in both places makes the winner depend on resolution order");
}
return List.copyOf(blockers);
}
/**
* Every disagreement between a deployment's profile and the bootstrap file its client will read.
*
* <p>Checked because the two are written by different people in different repositories, and the
* failure is silent: a client whose bootstrap names a namespace the deployment did not configure
* subscribes successfully and receives another team's routing. Nothing errors — the control plane
* answers, the resources parse, and traffic goes somewhere nobody chose.
*
* <p>Matched textually rather than with a JSON parser, deliberately. This leaf's test classpath
* is plain JUnit and AssertJ, and adding a JSON library to check three fields would put a parser
* on the runtime classpath of every deployment that enables xDS.
*
* @param bootstrapJson the bootstrap document's contents
* @return an empty list when the bootstrap and the profile agree
*/
public static List<String> bootstrapMismatches(GrpcXdsProfile profile, String bootstrapJson) {
if (profile == null) {
throw new IllegalArgumentException("a profile is required");
}
if (bootstrapJson == null || bootstrapJson.isBlank()) {
throw new IllegalArgumentException(
"the bootstrap document is required; a client with no bootstrap has no control plane to ask");
}
List<String> mismatches = new ArrayList<>();
if (!bootstrapJson.contains("\"xds_servers\"")) {
mismatches.add("the bootstrap declares no xds_servers");
}
if (!bootstrapJson.contains("\"channel_creds\"") || !bootstrapJson.contains("\"tls\"")) {
mismatches.add(
"the bootstrap's control-plane channel is not TLS; that connection carries routing and "
+ "often certificates, so an unauthenticated one can be impersonated");
}
if (!bootstrapJson.contains(profile.resourceNamespace())) {
mismatches.add(
"the bootstrap does not name the profile's resource namespace '"
+ profile.resourceNamespace()
+ "'; a client that subscribes outside its namespace receives another team's routing, "
+ "and nothing about that fails");
}
return List.copyOf(mismatches);
}
/**
* Whether xDS may be advertised as part of Stable discovery support.
*
* <p>Always false. Stable support is DNS and static.
*/
public static boolean advertisableAsStableSupport() {
return false;
}
}
@@ -0,0 +1,146 @@
package dev.caskeleton.grpc.advanced.resilience;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.grpc.core.GrpcMethodName;
import dev.caskeleton.grpc.core.RpcType;
import dev.caskeleton.grpc.deadline.GrpcDeadlineProfile;
import dev.caskeleton.grpc.policy.GrpcMethodPolicy;
import dev.caskeleton.grpc.policy.RpcIdempotencyProfile;
import dev.caskeleton.grpc.policy.WaitForReadyPolicy;
import dev.caskeleton.grpc.resilience.GrpcRetryOwner;
import java.time.Duration;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
class GrpcHedgingEligibilityTest {
private static final GrpcMethodName GET =
GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/GetDocument");
private static final GrpcMethodName CREATE =
GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/CreateDocument");
private static final GrpcMethodName WATCH =
GrpcMethodName.parse("hyeonworks.document.v1.DocumentService/WatchDocuments");
private static final GrpcDeadlineProfile TWO_SECONDS =
GrpcDeadlineProfile.of(Duration.ofSeconds(2));
@Test
@DisplayName("only a read-only unary method may be hedged")
void onlyReadOnlyUnaryMayBeHedged() {
assertThat(
GrpcHedgingEligibility.eligible(
GrpcMethodPolicy.readOnlyUnary(GET, TWO_SECONDS), GrpcRetryOwner.GRPC_PLATFORM))
.isTrue();
assertThat(
GrpcHedgingEligibility.refusalReason(
GrpcMethodPolicy.nonIdempotentUnary(CREATE, TWO_SECONDS),
GrpcRetryOwner.GRPC_PLATFORM))
.hasValueSatisfying(
reason -> assertThat(reason).contains("duplicates a success in progress"));
}
@Test
@DisplayName("a keyed mutation is still refused; an idempotency key does not make hedging safe")
void aKeyedMutationIsStillRefused() {
GrpcMethodPolicy keyed =
new GrpcMethodPolicy(
CREATE,
RpcType.UNARY,
RpcIdempotencyProfile.IDEMPOTENCY_KEY_REQUIRED,
TWO_SECONDS,
WaitForReadyPolicy.DISABLED,
false,
1024,
1024);
assertThat(GrpcHedgingEligibility.eligible(keyed, GrpcRetryOwner.GRPC_PLATFORM)).isFalse();
}
@Test
@DisplayName("a streaming method may not be hedged")
void streamingMayNotBeHedged() {
GrpcMethodPolicy streaming =
new GrpcMethodPolicy(
WATCH,
RpcType.SERVER_STREAMING,
RpcIdempotencyProfile.READ_ONLY,
TWO_SECONDS,
WaitForReadyPolicy.DISABLED,
false,
1024,
1024);
assertThatThrownBy(
() -> GrpcHedgingEligibility.require(streaming, GrpcRetryOwner.GRPC_PLATFORM))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("two prefixes");
}
@Test
@DisplayName("a mesh-owned channel may not hedge in-process")
void aMeshOwnedChannelMayNotHedge() {
assertThat(
GrpcHedgingEligibility.eligible(
GrpcMethodPolicy.readOnlyUnary(GET, TWO_SECONDS), GrpcRetryOwner.SERVICE_MESH))
.isFalse();
assertThat(
GrpcHedgingEligibility.eligible(
GrpcMethodPolicy.readOnlyUnary(GET, TWO_SECONDS), GrpcRetryOwner.APPLICATION))
.isFalse();
}
@Test
@DisplayName("hedging is capped at two attempts and needs a meaningful delay")
void hedgingIsCappedAndDelayed() {
assertThat(
GrpcHedgingPolicy.hedgeOnce(Duration.ofMillis(50), Duration.ofSeconds(2)).maxAttempts())
.isEqualTo(2);
assertThatThrownBy(() -> new GrpcHedgingPolicy(3, Duration.ofMillis(50), Duration.ofSeconds(2)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("capped at 2");
assertThatThrownBy(() -> new GrpcHedgingPolicy(2, Duration.ZERO, Duration.ofSeconds(2)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("doubles load for every call");
assertThatThrownBy(() -> new GrpcHedgingPolicy(2, Duration.ofSeconds(3), Duration.ofSeconds(2)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("never starts");
}
@Test
@DisplayName("the hedging budget bounds duplicate load and refuses an absurd ratio")
void theHedgingBudgetBoundsDuplicateLoad() {
// ratio 0.5 costs two tokens per hedge, so a four-token budget affords two hedges and then
// needs two completions before it can afford another.
GrpcHedgingBudget budget = GrpcHedgingBudget.of(0.5d, 4L);
assertThat(budget.tryConsume()).isTrue();
assertThat(budget.tryConsume()).isTrue();
assertThat(budget.tryConsume()).isFalse();
assertThat(budget.availableTokens()).isZero();
budget.recordCompletion();
assertThat(budget.exhausted()).isTrue();
budget.recordCompletion();
assertThat(budget.exhausted()).isFalse();
assertThatThrownBy(() -> GrpcHedgingBudget.of(0.9d, 10L))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("load decision rather than a latency one");
}
@Test
@DisplayName("a hedged result records both the saving and the duplicate load")
void aHedgedResultRecordsBothSides() {
GrpcHedgingResult hedged = new GrpcHedgingResult(2, 2, 1, 1, Duration.ofMillis(40));
assertThat(hedged.hedged()).isTrue();
assertThat(hedged.duplicateBackendCalls()).isEqualTo(1);
assertThat(hedged.cancelledLoserAttempts()).isEqualTo(1);
assertThat(GrpcHedgingResult.firstAttemptWon(Duration.ofMillis(10)).hedged()).isFalse();
assertThatThrownBy(() -> new GrpcHedgingResult(2, 3, 0, 0, Duration.ZERO))
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(() -> new GrpcHedgingResult(2, 1, 2, 0, Duration.ZERO))
.isInstanceOf(IllegalArgumentException.class);
}
}
@@ -0,0 +1,124 @@
package dev.caskeleton.grpc.advanced.resilience;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.grpc.advanced.discovery.GrpcEndpointCandidate;
import dev.caskeleton.grpc.advanced.discovery.GrpcLoadBalancerDecision;
import dev.caskeleton.grpc.advanced.discovery.GrpcLoadBalancerPicker;
import dev.caskeleton.grpc.advanced.discovery.GrpcLoadBalancerSafetyPolicy;
import java.util.List;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
class GrpcLoadBalancerSafetyPolicyTest {
@Test
@DisplayName("a picker may only choose an endpoint the resolver supplied")
void aPickerMayNotInventAnEndpoint() {
GrpcLoadBalancerSafetyPolicy policy =
new GrpcLoadBalancerSafetyPolicy(candidates -> GrpcEndpointCandidate.ready("10.9.9.9"));
GrpcLoadBalancerDecision decision =
policy.pick(List.of(GrpcEndpointCandidate.ready("10.0.0.1")));
assertThat(decision.verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.DETERMINISTIC_FALLBACK);
assertThat(decision.reason()).contains("can send a request anywhere");
assertThat(decision.chosen())
.hasValueSatisfying(endpoint -> assertThat(endpoint.address()).isEqualTo("10.0.0.1"));
}
@Test
@DisplayName("a picker that throws degrades balancing, not availability")
void aThrowingPickerFallsBack() {
GrpcLoadBalancerSafetyPolicy policy =
new GrpcLoadBalancerSafetyPolicy(
candidates -> {
throw new IllegalStateException("picker bug");
});
GrpcLoadBalancerDecision decision =
policy.pick(List.of(GrpcEndpointCandidate.ready("10.0.0.1")));
assertThat(decision.verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.DETERMINISTIC_FALLBACK);
assertThat(decision.reason()).contains("IllegalStateException");
}
@Test
@DisplayName("a picker returning null falls back rather than failing the call")
void aNullPickFallsBack() {
GrpcLoadBalancerSafetyPolicy policy = new GrpcLoadBalancerSafetyPolicy(candidates -> null);
assertThat(policy.pick(List.of(GrpcEndpointCandidate.ready("10.0.0.1"))).verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.DETERMINISTIC_FALLBACK);
}
@Test
@DisplayName("no selectable endpoint is a different verdict from a picker failure")
void noEndpointDiffersFromAPickerFailure() {
GrpcLoadBalancerSafetyPolicy policy =
new GrpcLoadBalancerSafetyPolicy(GrpcLoadBalancerPicker.roundRobin());
assertThat(
policy
.pick(List.of(new GrpcEndpointCandidate("10.0.0.1", false, true, 100, false)))
.verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.NO_ENDPOINT_AVAILABLE);
assertThat(
policy
.pick(List.of(new GrpcEndpointCandidate("10.0.0.1", true, true, 100, true)))
.verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.NO_ENDPOINT_AVAILABLE);
assertThat(
policy
.pick(List.of(new GrpcEndpointCandidate("10.0.0.1", true, true, 0, false)))
.verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.NO_ENDPOINT_AVAILABLE);
}
@Test
@DisplayName("a healthy endpoint is picked and reported as the picker's own choice")
void aHealthyEndpointIsPicked() {
GrpcLoadBalancerSafetyPolicy policy =
new GrpcLoadBalancerSafetyPolicy(GrpcLoadBalancerPicker.roundRobin());
assertThat(
policy
.pick(
List.of(
GrpcEndpointCandidate.ready("10.0.0.1"),
GrpcEndpointCandidate.ready("10.0.0.2")))
.verdict())
.isEqualTo(GrpcLoadBalancerDecision.Verdict.PICKED);
}
@Test
@DisplayName("a picker sees endpoints only, never a request or a caller")
void aPickerSeesEndpointsOnly() {
assertThat(GrpcLoadBalancerPicker.class.getMethods())
.filteredOn(method -> "pick".equals(method.getName()))
.singleElement()
.satisfies(method -> assertThat(method.getParameterCount()).isEqualTo(1));
}
@Test
@DisplayName("a candidate carries only routing-relevant state, bounded")
void aCandidateCarriesOnlyRoutingState() {
assertThat(GrpcEndpointCandidate.ready("10.0.0.1").selectable()).isTrue();
assertThatThrownBy(() -> new GrpcEndpointCandidate("10.0.0.1", true, true, 5000, false))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("scale nobody can reason about");
assertThatThrownBy(() -> new GrpcEndpointCandidate(" ", true, true, 100, false))
.isInstanceOf(IllegalArgumentException.class);
}
@Test
@DisplayName("a load-aware or weighted picker requires evidence before it ships")
void loadAwarePickersRequireEvidence() {
assertThat(GrpcLoadBalancerSafetyPolicy.requiresLoadEvidence(true, false)).isTrue();
assertThat(GrpcLoadBalancerSafetyPolicy.requiresLoadEvidence(false, true)).isTrue();
assertThat(GrpcLoadBalancerSafetyPolicy.requiresLoadEvidence(false, false)).isFalse();
}
}
@@ -0,0 +1,119 @@
package dev.caskeleton.grpc.advanced.resilience;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.grpc.advanced.discovery.GrpcCustomResolver;
import dev.caskeleton.grpc.advanced.discovery.GrpcEndpointSnapshot;
import dev.caskeleton.grpc.advanced.discovery.GrpcResolverSafetyPolicy;
import dev.caskeleton.grpc.advanced.discovery.GrpcResolverUpdate;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
class GrpcResolverSafetyPolicyTest {
@Test
@DisplayName("an empty or duplicate endpoint set is refused")
void anEmptyOrDuplicateEndpointSetIsRefused() {
assertThatThrownBy(() -> new GrpcEndpointSnapshot(1L, "documents", List.of()))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("take the channel down with it");
assertThatThrownBy(
() -> new GrpcEndpointSnapshot(1L, "documents", List.of("10.0.0.1", "10.0.0.1")))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("skew a round-robin picker");
assertThatThrownBy(() -> new GrpcEndpointSnapshot(0L, "documents", List.of("10.0.0.1")))
.isInstanceOf(IllegalArgumentException.class);
}
@Test
@DisplayName("a stale revision is dropped rather than applied")
void aStaleRevisionIsDropped() {
List<GrpcResolverUpdate> delivered = new ArrayList<>();
try (GrpcCustomResolver resolver = new GrpcCustomResolver("documents", delivered::add)) {
assertThat(
resolver.offer(
GrpcResolverUpdate.endpointsOnly(
new GrpcEndpointSnapshot(2L, "documents", List.of("10.0.0.1")))))
.isEmpty();
assertThat(
resolver.offer(
GrpcResolverUpdate.endpointsOnly(
new GrpcEndpointSnapshot(1L, "documents", List.of("10.0.0.9")))))
.anySatisfy(violation -> assertThat(violation).contains("older ones"));
assertThat(delivered).hasSize(1);
assertThat(resolver.currentSnapshot())
.hasValueSatisfying(snapshot -> assertThat(snapshot.revision()).isEqualTo(2L));
assertThat(resolver.authority()).isEqualTo("documents");
}
}
@Test
@DisplayName("a resolver accepts nothing after close")
void aClosedResolverAcceptsNothing() {
List<GrpcResolverUpdate> delivered = new ArrayList<>();
GrpcCustomResolver resolver = new GrpcCustomResolver("documents", delivered::add);
resolver.close();
assertThat(resolver.closed()).isTrue();
assertThat(
resolver.offer(
GrpcResolverUpdate.endpointsOnly(
new GrpcEndpointSnapshot(1L, "documents", List.of("10.0.0.1")))))
.anySatisfy(violation -> assertThat(violation).contains("resolver is closed"));
assertThat(delivered).isEmpty();
assertThat(GrpcResolverSafetyPolicy.acceptAfterClose()).isFalse();
}
@Test
@DisplayName("a resolver may not push credentials or business metadata")
void aResolverMayNotPushCredentials() {
GrpcResolverUpdate withCredential =
new GrpcResolverUpdate(
new GrpcEndpointSnapshot(1L, "documents", List.of("10.0.0.1")),
Optional.of("{\"authorization\":\"Bearer abc\"}"));
assertThat(GrpcResolverSafetyPolicy.violations(withCredential, null))
.anySatisfy(violation -> assertThat(violation).contains("never authentication material"));
assertThat(GrpcResolverSafetyPolicy.mayCarryBusinessMetadata()).isFalse();
assertThat(GrpcResolverSafetyPolicy.acceptedServiceConfig(withCredential, null)).isEmpty();
}
@Test
@DisplayName("a clean service config survives validation")
void aCleanServiceConfigSurvives() {
GrpcResolverUpdate clean =
new GrpcResolverUpdate(
new GrpcEndpointSnapshot(1L, "documents", List.of("10.0.0.1")),
Optional.of("{\"loadBalancingConfig\":[{\"round_robin\":{}}]}"));
assertThat(GrpcResolverSafetyPolicy.violations(clean, null)).isEmpty();
assertThat(GrpcResolverSafetyPolicy.acceptedServiceConfig(clean, null)).isPresent();
}
@Test
@DisplayName("an invalid authority is refused, because it decides which certificate is accepted")
void anInvalidAuthorityIsRefused() {
assertThat(
GrpcResolverSafetyPolicy.violations(
GrpcResolverUpdate.endpointsOnly(
new GrpcEndpointSnapshot(1L, "Documents Service", List.of("10.0.0.1"))),
null))
.anySatisfy(violation -> assertThat(violation).contains("which certificate"));
}
@Test
@DisplayName("a present service config may not be blank")
void aPresentServiceConfigMayNotBeBlank() {
assertThatThrownBy(
() ->
new GrpcResolverUpdate(
new GrpcEndpointSnapshot(1L, "documents", List.of("10.0.0.1")),
Optional.of(" ")))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("different states");
}
}
@@ -0,0 +1,188 @@
package dev.caskeleton.grpc.advanced.resilience;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.grpc.advanced.bootstrap.GrpcAdvancedCapability;
import dev.caskeleton.grpc.advanced.bootstrap.GrpcAdvancedFeatureFlags;
import dev.caskeleton.grpc.advanced.xds.GrpcXdsFailurePolicy;
import dev.caskeleton.grpc.advanced.xds.GrpcXdsProfile;
import dev.caskeleton.grpc.advanced.xds.GrpcXdsResourceSnapshot;
import dev.caskeleton.grpc.advanced.xds.GrpcXdsStartupGuard;
import dev.caskeleton.grpc.resilience.GrpcRetryOwner;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
class GrpcXdsStartupGuardTest {
private static final Instant NOW = Instant.parse("2026-08-30T10:00:00Z");
private static GrpcXdsProfile profile(String namespace) {
return new GrpcXdsProfile(
URI.create("xds:///documents"),
"classpath:/xds/bootstrap.json",
namespace,
GrpcRetryOwner.SERVICE_MESH,
true);
}
@Test
@DisplayName("an xDS profile requires an xds target, a namespace, mesh retries and mTLS")
void anXdsProfileRequiresItsFourConditions() {
assertThat(profile("hyeonworks/documents").target().getScheme()).isEqualTo("xds");
assertThatThrownBy(
() ->
new GrpcXdsProfile(
URI.create("dns:///documents"),
"file:/etc/grpc/bootstrap.json",
"ns",
GrpcRetryOwner.SERVICE_MESH,
true))
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(
() ->
new GrpcXdsProfile(
URI.create("xds:///documents"), " ", "ns", GrpcRetryOwner.SERVICE_MESH, true))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("no control plane to ask");
assertThatThrownBy(
() ->
new GrpcXdsProfile(
URI.create("xds:///documents"),
"file:/etc/grpc/bootstrap.json",
"ns",
GrpcRetryOwner.APPLICATION,
true))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("two places");
assertThatThrownBy(
() ->
new GrpcXdsProfile(
URI.create("xds:///documents"),
"file:/etc/grpc/bootstrap.json",
"ns",
GrpcRetryOwner.SERVICE_MESH,
false))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("impersonated");
}
@Test
@DisplayName("xDS needs its capability approved and refuses duplicate retry ownership")
void startupRequiresApprovalAndSingleRetryOwner() {
GrpcAdvancedFeatureFlags unapproved =
GrpcAdvancedFeatureFlags.forProduction(Set.of()).enable(GrpcAdvancedCapability.XDS);
GrpcAdvancedFeatureFlags approved =
GrpcAdvancedFeatureFlags.forProduction(Set.of(GrpcAdvancedCapability.XDS))
.enable(GrpcAdvancedCapability.XDS);
assertThat(
GrpcXdsStartupGuard.startupBlockers(profile("hyeonworks/documents"), unapproved, false))
.anySatisfy(blocker -> assertThat(blocker).contains("production has not approved it"));
assertThat(
GrpcXdsStartupGuard.startupBlockers(profile("hyeonworks/documents"), approved, false))
.isEmpty();
assertThat(GrpcXdsStartupGuard.startupBlockers(profile("hyeonworks/documents"), approved, true))
.anySatisfy(blocker -> assertThat(blocker).contains("resolution order"));
}
@Test
@DisplayName("an unflagged capability is reported as unflagged rather than unapproved")
void anUnflaggedCapabilityIsReportedAsSuch() {
assertThat(
GrpcXdsStartupGuard.startupBlockers(
profile("hyeonworks/documents"),
GrpcAdvancedFeatureFlags.forProduction(Set.of()),
false))
.anySatisfy(blocker -> assertThat(blocker).contains("its flag is not set"));
}
@Test
@DisplayName("the committed bootstrap fixture agrees with the profile it is meant to serve")
void theBootstrapFixtureAgreesWithItsProfile() {
assertThat(
GrpcXdsStartupGuard.bootstrapMismatches(
profile("hyeonworks/documents"), resource("xds/bootstrap.json")))
.isEmpty();
}
@Test
@DisplayName("a bootstrap naming another namespace is reported, since nothing else would fail")
void aNamespaceMismatchIsReported() {
assertThat(
GrpcXdsStartupGuard.bootstrapMismatches(
profile("hyeonworks/billing"), resource("xds/bootstrap.json")))
.anySatisfy(mismatch -> assertThat(mismatch).contains("another team's routing"));
}
@Test
@DisplayName("a bootstrap with an unauthenticated control-plane channel is reported")
void anUnauthenticatedControlPlaneIsReported() {
String insecure =
resource("xds/bootstrap.json")
.replace("{ \"type\": \"tls\" }", "{ \"type\": \"insecure\" }");
assertThat(GrpcXdsStartupGuard.bootstrapMismatches(profile("hyeonworks/documents"), insecure))
.anySatisfy(mismatch -> assertThat(mismatch).contains("can be impersonated"));
}
@Test
@DisplayName("a missing bootstrap is refused rather than treated as an empty one")
void aMissingBootstrapIsRefused() {
assertThatThrownBy(
() -> GrpcXdsStartupGuard.bootstrapMismatches(profile("hyeonworks/documents"), " "))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("no control plane to ask");
}
@Test
@DisplayName("last-known-good is bounded, and an empty resource set is not a snapshot")
void lastKnownGoodIsBounded() {
GrpcXdsFailurePolicy policy = GrpcXdsFailurePolicy.standard();
GrpcXdsResourceSnapshot snapshot =
new GrpcXdsResourceSnapshot("v7", List.of("documents-cluster"), NOW);
assertThat(policy.decide(Optional.of(snapshot), NOW.plusSeconds(60)))
.isEqualTo(GrpcXdsFailurePolicy.Decision.SERVE_LAST_KNOWN_GOOD);
assertThat(policy.decide(Optional.of(snapshot), NOW.plusSeconds(1000)))
.isEqualTo(GrpcXdsFailurePolicy.Decision.STALE_BEYOND_BOUND);
assertThat(policy.decide(Optional.empty(), NOW))
.isEqualTo(GrpcXdsFailurePolicy.Decision.NO_SNAPSHOT_YET);
assertThat(snapshot.ageAt(NOW.plusSeconds(60))).isEqualTo(java.time.Duration.ofSeconds(60));
assertThatThrownBy(() -> new GrpcXdsResourceSnapshot("v7", List.of(), NOW))
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(
() ->
new GrpcXdsFailurePolicy(
java.time.Duration.ZERO, true, java.time.Duration.ofSeconds(1)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("receiving traffic indefinitely");
}
@Test
@DisplayName("xDS is not part of the Stable discovery support statement")
void xdsIsNotStableSupport() {
assertThat(GrpcXdsStartupGuard.advertisableAsStableSupport()).isFalse();
}
private static String resource(String path) {
try (InputStream stream =
GrpcXdsStartupGuardTest.class.getClassLoader().getResourceAsStream(path)) {
if (stream == null) {
throw new IllegalStateException("missing test resource " + path);
}
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
@@ -0,0 +1,40 @@
{
"_comment": [
"An xDS bootstrap fixture. The client reads this file to learn where its control plane is and",
"who it claims to be; GrpcXdsProfile validates the deployment settings that must agree with it.",
"The two fields the profile actually checks against are server_uri (the control plane must be",
"reached over authenticated mTLS) and the node id's namespace (a client that subscribes outside",
"its namespace receives another team's routing)."
],
"xds_servers": [
{
"server_uri": "xds-control-plane.hyeonworks.internal:15010",
"channel_creds": [
{ "type": "tls" }
],
"server_features": ["xds_v3"]
}
],
"node": {
"id": "hyeonworks/documents/documents-7f9c4",
"cluster": "documents",
"metadata": {
"NAMESPACE": "hyeonworks/documents"
},
"locality": {
"region": "ap-northeast-2",
"zone": "ap-northeast-2a"
}
},
"authorities": {
"hyeonworks.internal": {
"xds_servers": [
{
"server_uri": "xds-control-plane.hyeonworks.internal:15010",
"channel_creds": [{ "type": "tls" }],
"server_features": ["xds_v3"]
}
]
}
}
}