This commit is contained in:
+9
-6
@@ -43,8 +43,10 @@ import org.springframework.http.ResponseEntity;
|
||||
@AutoConfigureTestRestTemplate
|
||||
class GraphQlRequestDeadlineTest {
|
||||
|
||||
private static final Duration BUDGET = Duration.ofSeconds(1);
|
||||
private static final int SLOW_RESOLVER_MILLIS = 4_000;
|
||||
// Two seconds so the resolver has started before the budget runs out even on a loaded machine
|
||||
// (SLOW_STARTED below); the resolver itself holds until the test releases it.
|
||||
private static final Duration BUDGET = Duration.ofSeconds(2);
|
||||
private static final int SLOW_RESOLVER_MILLIS = 30_000;
|
||||
|
||||
@LocalServerPort int port;
|
||||
|
||||
@@ -70,6 +72,7 @@ class GraphQlRequestDeadlineTest {
|
||||
|
||||
// then
|
||||
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
|
||||
SlowWork.release.countDown();
|
||||
assertThat(response.getBody())
|
||||
.contains("\"code\":\"REQUEST_TIMEOUT\"")
|
||||
.contains("\"category\":\"TIMEOUT\"")
|
||||
@@ -78,9 +81,9 @@ class GraphQlRequestDeadlineTest {
|
||||
.as("the budget must have run out during execution, not before it")
|
||||
.hasValue(1);
|
||||
assertThat(elapsedMillis)
|
||||
.as("answered at the deadline, not when the %d ms resolver returned", SLOW_RESOLVER_MILLIS)
|
||||
.as("answered at the 2 s deadline, well before the 10 s default budget would have ended it")
|
||||
.isGreaterThanOrEqualTo(BUDGET.toMillis())
|
||||
.isLessThan(SLOW_RESOLVER_MILLIS - 1_000L);
|
||||
.isLessThan(BUDGET.plusSeconds(6).toMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -91,8 +94,8 @@ class GraphQlRequestDeadlineTest {
|
||||
http, port, "{ slowNode(millis: " + SLOW_RESOLVER_MILLIS + ") { id detail } }");
|
||||
|
||||
// when
|
||||
boolean finished =
|
||||
SlowWork.slowFinished.await(SLOW_RESOLVER_MILLIS * 2L, TimeUnit.MILLISECONDS);
|
||||
SlowWork.release.countDown();
|
||||
boolean finished = SlowWork.slowFinished.await(10, TimeUnit.SECONDS);
|
||||
|
||||
// then
|
||||
assertThat(finished)
|
||||
|
||||
+12
-1
@@ -12,6 +12,7 @@ import graphql.execution.instrumentation.parameters.InstrumentationExecutionPara
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.springframework.boot.graphql.autoconfigure.GraphQlSourceBuilderCustomizer;
|
||||
import org.springframework.boot.resttestclient.TestRestTemplate;
|
||||
@@ -92,6 +93,13 @@ final class RuntimeLimitsTestSupport {
|
||||
static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger();
|
||||
static volatile CountDownLatch slowFinished = new CountDownLatch(1);
|
||||
|
||||
/**
|
||||
* Opened by a test to let the slow resolver return before its {@code millis} cap. Until then
|
||||
* the resolver holds as a slow dependency would, so "answered before the resolver returned"
|
||||
* does not depend on how fast a loaded machine runs the rest of the request.
|
||||
*/
|
||||
static volatile CountDownLatch release = new CountDownLatch(1);
|
||||
|
||||
/**
|
||||
* Counted down when a {@code slowNode} execution begun after the last {@link #reset()} has
|
||||
* ended. The point after which a child field can no longer start, so an absence asserted after
|
||||
@@ -108,6 +116,7 @@ final class RuntimeLimitsTestSupport {
|
||||
DETAIL_INVOCATIONS.set(0);
|
||||
slowFinished = new CountDownLatch(1);
|
||||
slowExecutionEnded = new CountDownLatch(1);
|
||||
release = new CountDownLatch(1);
|
||||
GENERATION.incrementAndGet();
|
||||
}
|
||||
|
||||
@@ -141,8 +150,10 @@ final class RuntimeLimitsTestSupport {
|
||||
@QueryMapping
|
||||
SlowNode slowNode(@Argument int millis) throws InterruptedException {
|
||||
SlowWork.SLOW_STARTED.incrementAndGet();
|
||||
CountDownLatch release = SlowWork.release;
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
// Released by the test, or the cap ran out; either way the resolver returns normally.
|
||||
var unused = release.await(millis, TimeUnit.MILLISECONDS);
|
||||
} finally {
|
||||
SlowWork.slowFinished.countDown();
|
||||
}
|
||||
|
||||
+99
-9
@@ -73,6 +73,7 @@ class GrpcP1BoundaryWireTest {
|
||||
private static final MethodDescriptor<String, String> CLOSE_HANDLER_METHOD =
|
||||
unaryMethod("CloseHandler");
|
||||
private static final MethodDescriptor<String, String> SLOW_METHOD = unaryMethod("Slow");
|
||||
private static final MethodDescriptor<String, String> GATED_METHOD = unaryMethod("Gated");
|
||||
private static final CountDownLatch CLOSE_HANDLER_RAN = new CountDownLatch(1);
|
||||
|
||||
/** The platform's GrpcTransportLimits.DEFAULT_MAX_MESSAGE_BYTES, which this adapter mirrors. */
|
||||
@@ -81,6 +82,17 @@ class GrpcP1BoundaryWireTest {
|
||||
/** How long the slow handler holds its thread unless its gRPC Context is cancelled first. */
|
||||
private static final long SLOW_HANDLER_HOLD_MILLIS = 400;
|
||||
|
||||
/**
|
||||
* Holds every gated handler until a test releases it or the call's Context is cancelled. A gate,
|
||||
* not a clock: what a test means by "while the call is still running" must not depend on how fast
|
||||
* a loaded machine dispatches it (review 2026-11-13: the main stage runs beside other builds).
|
||||
* Each test that uses it sets a fresh latch and releases it before it ends.
|
||||
*/
|
||||
private static volatile CountDownLatch gate = new CountDownLatch(0);
|
||||
|
||||
/** The longest a gated handler waits, so a test that forgets to release cannot hang the JVM. */
|
||||
private static final long GATE_CAP_SECONDS = 30;
|
||||
|
||||
private static final AtomicInteger SLOW_IN_FLIGHT = new AtomicInteger();
|
||||
private static final AtomicInteger SLOW_MAX_IN_FLIGHT = new AtomicInteger();
|
||||
|
||||
@@ -264,17 +276,30 @@ class GrpcP1BoundaryWireTest {
|
||||
contextRunner.withPropertyValues("app.grpc.server.default-deadline=150ms"),
|
||||
channel -> {
|
||||
// given
|
||||
// The handler holds until its Context is cancelled and then answers nothing, so the
|
||||
// only way the call can end is the server deadline closing it. waitForReady keeps a
|
||||
// slow first connection from failing the call before it reaches the server.
|
||||
gate = new CountDownLatch(1);
|
||||
long started = System.nanoTime();
|
||||
ListenableFuture<String> withoutDeadline =
|
||||
ClientCalls.futureUnaryCall(
|
||||
authenticated(channel).newCall(SLOW_METHOD, CallOptions.DEFAULT), "slow");
|
||||
authenticated(channel)
|
||||
.newCall(GATED_METHOD, CallOptions.DEFAULT.withWaitForReady()),
|
||||
"gated");
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> withoutDeadline.get(5, TimeUnit.SECONDS));
|
||||
Throwable failure =
|
||||
catchThrowable(() -> withoutDeadline.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
|
||||
|
||||
// then
|
||||
gate.countDown();
|
||||
assertThat(failure).as("the call must fail, not answer").isNotNull();
|
||||
assertThat(Status.fromThrowable(failure).getCode())
|
||||
.as("status of %s", failure)
|
||||
.isEqualTo(Status.Code.DEADLINE_EXCEEDED);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.as("the configured 150 ms ended it, not the 10 s default it would fall back to")
|
||||
.isLessThan(Duration.ofSeconds(5));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -285,6 +310,9 @@ class GrpcP1BoundaryWireTest {
|
||||
contextRunner.withPropertyValues("app.grpc.server.default-deadline=10s"),
|
||||
channel -> {
|
||||
// given
|
||||
// A gated handler never answers before the gate opens, so the client deadline is the
|
||||
// only thing that can end the call: the 10 s server default must not have replaced it.
|
||||
gate = new CountDownLatch(1);
|
||||
long started = System.nanoTime();
|
||||
|
||||
// when
|
||||
@@ -293,14 +321,18 @@ class GrpcP1BoundaryWireTest {
|
||||
() ->
|
||||
ClientCalls.blockingUnaryCall(
|
||||
authenticated(channel),
|
||||
SLOW_METHOD,
|
||||
GATED_METHOD,
|
||||
CallOptions.DEFAULT.withDeadlineAfter(100, TimeUnit.MILLISECONDS),
|
||||
"slow"));
|
||||
"gated"));
|
||||
|
||||
// then
|
||||
gate.countDown();
|
||||
assertThat(failure).as("the call must fail, not answer").isNotNull();
|
||||
assertThat(Status.fromThrowable(failure).getCode())
|
||||
.as("status of %s", failure)
|
||||
.isEqualTo(Status.Code.DEADLINE_EXCEEDED);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.as("ended by the 100 ms client deadline")
|
||||
.isLessThan(Duration.ofSeconds(5));
|
||||
});
|
||||
}
|
||||
@@ -338,18 +370,25 @@ class GrpcP1BoundaryWireTest {
|
||||
"app.grpc.server.executor-queue-capacity=1"),
|
||||
channel -> {
|
||||
// given
|
||||
// Gated handlers hold the one thread and the one queue slot until the refusal is
|
||||
// seen, so the third call meets a full server however slowly the calls are dispatched.
|
||||
gate = new CountDownLatch(1);
|
||||
Channel caller = authenticated(channel);
|
||||
CallOptions bounded = CallOptions.DEFAULT.withDeadlineAfter(5, TimeUnit.SECONDS);
|
||||
CallOptions bounded =
|
||||
CallOptions.DEFAULT.withDeadlineAfter(GATE_CAP_SECONDS, TimeUnit.SECONDS);
|
||||
List<ListenableFuture<String>> calls =
|
||||
List.of(
|
||||
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"));
|
||||
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"));
|
||||
|
||||
// when
|
||||
boolean refusedWhileHeld = awaitAnyDone(calls);
|
||||
gate.countDown();
|
||||
List<Status> outcomes = calls.stream().map(GrpcP1BoundaryWireTest::outcome).toList();
|
||||
|
||||
// then
|
||||
assertThat(refusedWhileHeld).as("one call is refused while the others hold").isTrue();
|
||||
assertThat(outcomes)
|
||||
.extracting(Status::getCode)
|
||||
.containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE);
|
||||
@@ -361,10 +400,27 @@ class GrpcP1BoundaryWireTest {
|
||||
}
|
||||
|
||||
private static Status outcome(ListenableFuture<String> call) {
|
||||
Throwable failure = catchThrowable(() -> call.get(5, TimeUnit.SECONDS));
|
||||
Throwable failure = catchThrowable(() -> call.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
|
||||
return failure == null ? Status.OK : Status.fromThrowable(failure);
|
||||
}
|
||||
|
||||
/** Whether one of {@code calls} completes while the gate still holds the others. */
|
||||
private static boolean awaitAnyDone(List<ListenableFuture<String>> calls) {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
|
||||
while (System.nanoTime() < until) {
|
||||
if (calls.stream().anyMatch(ListenableFuture::isDone)) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(10);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String getUnchecked(ListenableFuture<String> future) {
|
||||
try {
|
||||
return future.get(5, TimeUnit.SECONDS);
|
||||
@@ -550,6 +606,17 @@ class GrpcP1BoundaryWireTest {
|
||||
observer.onNext(request + "-done");
|
||||
observer.onCompleted();
|
||||
}))
|
||||
.addMethod(
|
||||
GATED_METHOD,
|
||||
ServerCalls.asyncUnaryCall(
|
||||
(String request, StreamObserver<String> observer) -> {
|
||||
if (holdUntilReleased()) {
|
||||
observer.onNext(request + "-done");
|
||||
observer.onCompleted();
|
||||
}
|
||||
// Cancelled: a cooperative handler answers nothing; whoever cancelled
|
||||
// the call (the server deadline) has already closed it.
|
||||
}))
|
||||
.addMethod(
|
||||
CLOSE_HANDLER_METHOD,
|
||||
ServerCalls.asyncUnaryCall(
|
||||
@@ -601,6 +668,29 @@ class GrpcP1BoundaryWireTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds the handler thread until the test opens {@link #gate}.
|
||||
*
|
||||
* @return true once released; false when the call's Context was cancelled first (or the cap ran
|
||||
* out)
|
||||
*/
|
||||
private static boolean holdUntilReleased() {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
|
||||
try {
|
||||
while (System.nanoTime() < until) {
|
||||
if (Context.current().isCancelled()) {
|
||||
return false;
|
||||
}
|
||||
if (gate.await(10, TimeUnit.MILLISECONDS)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private enum StringMarshaller implements MethodDescriptor.Marshaller<String> {
|
||||
INSTANCE;
|
||||
|
||||
|
||||
+23
-8
@@ -98,7 +98,9 @@ class GrpcServerRunnerBootTest {
|
||||
void stoppingKeepsTheListenerUpReportingNotServingDuringTheDrainDelay() {
|
||||
contextRunner
|
||||
.withPropertyValues(
|
||||
"app.grpc.server.shutdown-drain-seconds=2", "app.grpc.server.shutdown-grace-seconds=4")
|
||||
// The drain is long enough that a loaded machine still sees it: the probe window below
|
||||
// must close before the listener does, and both are seconds, not a scheduling slice.
|
||||
"app.grpc.server.shutdown-drain-seconds=6", "app.grpc.server.shutdown-grace-seconds=8")
|
||||
.run(
|
||||
context -> {
|
||||
// given
|
||||
@@ -107,6 +109,15 @@ class GrpcServerRunnerBootTest {
|
||||
ManagedChannelBuilder.forAddress("localhost", runner.getListeningPort())
|
||||
.usePlaintext()
|
||||
.build();
|
||||
// Connect before shutdown begins, so the drain window is not spent on the first
|
||||
// connection.
|
||||
assertThat(
|
||||
HealthGrpc.newBlockingStub(channel)
|
||||
.withWaitForReady()
|
||||
.withDeadlineAfter(10, TimeUnit.SECONDS)
|
||||
.check(HealthCheckRequest.newBuilder().build())
|
||||
.getStatus())
|
||||
.isEqualTo(ServingStatus.SERVING);
|
||||
Thread stopping = Thread.ofVirtual().start(runner::stop);
|
||||
try {
|
||||
// when
|
||||
@@ -115,7 +126,7 @@ class GrpcServerRunnerBootTest {
|
||||
// then
|
||||
assertThat(duringDrain).isEqualTo(ServingStatus.NOT_SERVING);
|
||||
} finally {
|
||||
stopping.join(Duration.ofSeconds(10));
|
||||
stopping.join(Duration.ofSeconds(20));
|
||||
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
|
||||
}
|
||||
});
|
||||
@@ -127,13 +138,9 @@ class GrpcServerRunnerBootTest {
|
||||
*/
|
||||
private static ServingStatus firstStatusOtherThanServing(ManagedChannel channel)
|
||||
throws InterruptedException {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(4);
|
||||
while (System.nanoTime() < until) {
|
||||
ServingStatus status =
|
||||
HealthGrpc.newBlockingStub(channel)
|
||||
.withDeadlineAfter(1, TimeUnit.SECONDS)
|
||||
.check(HealthCheckRequest.newBuilder().build())
|
||||
.getStatus();
|
||||
ServingStatus status = health(channel);
|
||||
if (status != ServingStatus.SERVING) {
|
||||
return status;
|
||||
}
|
||||
@@ -142,6 +149,14 @@ class GrpcServerRunnerBootTest {
|
||||
return ServingStatus.SERVING;
|
||||
}
|
||||
|
||||
// Two seconds per probe: the four-second window plus one probe still ends inside the 6 s drain.
|
||||
private static ServingStatus health(ManagedChannel channel) {
|
||||
return HealthGrpc.newBlockingStub(channel)
|
||||
.withDeadlineAfter(2, TimeUnit.SECONDS)
|
||||
.check(HealthCheckRequest.newBuilder().build())
|
||||
.getStatus();
|
||||
}
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
static class UnqualifiedServiceConfiguration {
|
||||
|
||||
|
||||
+3
-1
@@ -217,7 +217,9 @@ class LiveRedisSemanticPortsTest {
|
||||
"api-default",
|
||||
"v1",
|
||||
RateLimitAlgorithm.FIXED_WINDOW,
|
||||
new RateParameters.FixedWindow(3, Duration.ofSeconds(10)),
|
||||
// An hour-long window: the four calls below must land in one window, and a 10 s window
|
||||
// aligned to Redis TIME let a slow run straddle its boundary.
|
||||
new RateParameters.FixedWindow(3, Duration.ofHours(1)),
|
||||
1,
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofMillis(250),
|
||||
|
||||
+5
-1
@@ -48,9 +48,13 @@ class LiveRedisDistributedLeaseContractTest extends DistributedLeaseContract {
|
||||
Duration.ZERO);
|
||||
}
|
||||
|
||||
// A live server waits for real, so the TTL is seconds rather than the fake's 30: long enough that
|
||||
// the calls a test makes while it holds a lease never outlast it on a loaded machine. Each of the
|
||||
// four displaced-holder cases sleeps 1.5 TTL, so this costs about 12 s per run (it was about 2 s
|
||||
// with a 400 ms TTL that a slow script call could outlast).
|
||||
@Override
|
||||
protected Duration leaseTtl() {
|
||||
return Duration.ofMillis(400);
|
||||
return Duration.ofSeconds(2);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+29
-6
@@ -43,6 +43,21 @@ import org.testcontainers.utility.DockerImageName;
|
||||
class FailureInjectionContractTest {
|
||||
|
||||
private static final Duration SLOW_DRIP_BUDGET = Duration.ofSeconds(1);
|
||||
private static final int SLOW_DRIP_BYTES = 4096;
|
||||
private static final int SLOW_DRIP_SLICE_BYTES = 16;
|
||||
private static final Duration SLOW_DRIP_GAP = Duration.ofMillis(50);
|
||||
|
||||
/** How long the drip alone takes to deliver the body: 256 slices, 50 ms apart (12.8 s). */
|
||||
private static final Duration SLOW_DRIP_TOTAL =
|
||||
SLOW_DRIP_GAP.multipliedBy(SLOW_DRIP_BYTES / SLOW_DRIP_SLICE_BYTES);
|
||||
|
||||
/**
|
||||
* Time a loaded machine may add between the deadline firing and the caller seeing the failure.
|
||||
* The main stage runs beside other builds; the bound only has to tell "cut at the budget" from
|
||||
* "waited for the drip", which the total leaves room for (budget + slack is well under half of
|
||||
* it).
|
||||
*/
|
||||
private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(4);
|
||||
|
||||
private static final DockerImageName UPSTREAM_IMAGE =
|
||||
DockerImageName.parse(
|
||||
@@ -129,11 +144,11 @@ class FailureInjectionContractTest {
|
||||
try (ToxiproxyFixture toxiproxy = ToxiproxyFixture.start(network)) {
|
||||
// given
|
||||
Proxy proxy = toxiproxy.proxyTo("slow-drip", "upstream", 80);
|
||||
toxiproxy.slowDrip(proxy, 16, Duration.ofMillis(50));
|
||||
toxiproxy.slowDrip(proxy, SLOW_DRIP_SLICE_BYTES, SLOW_DRIP_GAP);
|
||||
TimeoutSettings base = ClientProfiles.timeouts();
|
||||
URI baseUrl =
|
||||
URI.create("http://" + toxiproxy.proxiedHost() + ":" + toxiproxy.proxiedPort() + "/");
|
||||
long started;
|
||||
Duration elapsed;
|
||||
Throwable failure;
|
||||
|
||||
// when
|
||||
@@ -151,7 +166,7 @@ class FailureInjectionContractTest {
|
||||
base.readIdle(),
|
||||
SLOW_DRIP_BUDGET))
|
||||
.build())) {
|
||||
started = System.nanoTime();
|
||||
long started = System.nanoTime();
|
||||
failure =
|
||||
catchThrowable(
|
||||
() ->
|
||||
@@ -160,14 +175,22 @@ class FailureInjectionContractTest {
|
||||
.exchange(
|
||||
harness.profile().name(),
|
||||
HttpOperation.get(
|
||||
new OperationName("fault-probe"), "/bytes/4096", Map.of()),
|
||||
new OperationName("fault-probe"),
|
||||
"/bytes/" + SLOW_DRIP_BYTES,
|
||||
Map.of()),
|
||||
ResponseType.of(byte[].class)));
|
||||
// The call alone: closing the harness afterwards is not part of the deadline.
|
||||
elapsed = Duration.ofNanos(System.nanoTime() - started);
|
||||
}
|
||||
|
||||
// then
|
||||
Duration elapsed = Duration.ofNanos(System.nanoTime() - started);
|
||||
assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class);
|
||||
assertThat(elapsed).isLessThan(SLOW_DRIP_BUDGET.plus(Duration.ofMillis(500)));
|
||||
assertThat(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK))
|
||||
.as("the bound must stay well under the drip, or it proves nothing")
|
||||
.isLessThan(SLOW_DRIP_TOTAL.dividedBy(2));
|
||||
assertThat(elapsed)
|
||||
.as("the deadline, not the drip, ended the call")
|
||||
.isLessThan(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-5
@@ -29,15 +29,17 @@ class AttemptDeadlineTest {
|
||||
@Test
|
||||
@DisplayName("남은 deadline이 단계 timeout보다 짧으면 단계 timeout을 남은 시간으로 줄인다")
|
||||
void aShorterRemainingDeadlineBoundsTheStageTimeout() {
|
||||
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(300))) {
|
||||
// Seconds rather than milliseconds, so the deadline has not already passed by the time bound()
|
||||
// runs on a loaded machine; the stage is longer still, so bound() must cut it.
|
||||
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(5))) {
|
||||
// given
|
||||
Duration stage = Duration.ofSeconds(2);
|
||||
Duration stage = Duration.ofSeconds(30);
|
||||
|
||||
// when
|
||||
Duration bounded = AttemptDeadline.bound(stage);
|
||||
|
||||
// then
|
||||
assertThat(bounded).isLessThanOrEqualTo(Duration.ofMillis(300)).isPositive();
|
||||
assertThat(bounded).isLessThanOrEqualTo(Duration.ofSeconds(5)).isPositive();
|
||||
assertThat(scope.expired()).isFalse();
|
||||
}
|
||||
}
|
||||
@@ -89,14 +91,16 @@ class AttemptDeadlineTest {
|
||||
void closingTheScopeBeforeTheDeadlineCancelsTheAbort() throws Exception {
|
||||
// given
|
||||
AtomicInteger aborts = new AtomicInteger();
|
||||
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(100));
|
||||
// One second, so registering and closing always happen before it on a loaded machine; the
|
||||
// wait after the close outlasts it, so an abort that was not cancelled would have run.
|
||||
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(1));
|
||||
AttemptDeadline.abortOnExpiry(aborts::incrementAndGet);
|
||||
|
||||
// when
|
||||
scope.close();
|
||||
|
||||
// then
|
||||
Thread.sleep(300);
|
||||
Thread.sleep(1_500);
|
||||
assertThat(aborts).hasValue(0);
|
||||
assertThat(AttemptDeadline.remaining()).isEmpty();
|
||||
}
|
||||
|
||||
+11
-1
@@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName;
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.error.PessimisticLockTimeoutException;
|
||||
import dev.caskeleton.adapter.outbound.persistence.postgresql.lock.PostgreSqlLockExceptionTranslator;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
@@ -59,14 +60,23 @@ class PostgreSqlPessimisticLockContractTest {
|
||||
lockRow(holder);
|
||||
|
||||
contender.setAutoCommit(false);
|
||||
try (Statement statement = contender.createStatement();
|
||||
ResultSet setting = statement.executeQuery("show lock_timeout")) {
|
||||
assertThat(setting.next()).isTrue();
|
||||
assertThat(setting.getString(1))
|
||||
.as("without a lock_timeout a lock that waited would wait for the holder for good")
|
||||
.isEqualTo("0");
|
||||
}
|
||||
Instant startedAt = Instant.now();
|
||||
SQLException refused = lockRowNowait(contender);
|
||||
Duration waited = Duration.between(startedAt, Instant.now());
|
||||
|
||||
assertThat(refused.getSQLState()).isEqualTo("55P03");
|
||||
// The contender has no lock_timeout, so a lock that waited would wait for the holder, who
|
||||
// never lets go here. Seconds of slack cannot be mistaken for that.
|
||||
assertThat(waited)
|
||||
.as("NOWAIT must not wait for the holder")
|
||||
.isLessThan(Duration.ofSeconds(1));
|
||||
.isLessThan(Duration.ofSeconds(10));
|
||||
|
||||
var translated = translator.translate(refused, OPERATION, 1, waited, null).orElseThrow();
|
||||
assertThat(translated).isInstanceOf(PessimisticLockTimeoutException.class);
|
||||
|
||||
+25
-4
@@ -12,6 +12,7 @@ import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -49,13 +50,17 @@ class PostgreSqlLifecycleIntegrationTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(10)
|
||||
// The method starts its own PostgreSQL container; on a loaded machine that alone can take tens of
|
||||
// seconds. The bound that matters, the exhausted pool's wait, is measured on its own below.
|
||||
@Timeout(value = 2, unit = TimeUnit.MINUTES)
|
||||
void poolCapacityExhaustionAndShutdownAreBoundedAndObservable() throws Exception {
|
||||
PostgreSqlReadinessSupport bounded = PostgreSqlReadinessSupport.start(2, 300);
|
||||
HikariDataSource dataSource = bounded.dataSource();
|
||||
try {
|
||||
try (Connection first = bounded.connection();
|
||||
Connection second = bounded.connection()) {
|
||||
// Borrowing a connection the pool still has to open (TCP, SCRAM, backend fork) must not be
|
||||
// held to the 300 ms timeout this test measures, so the two are taken patiently.
|
||||
try (Connection first = patiently(bounded);
|
||||
Connection second = patiently(bounded)) {
|
||||
PoolSnapshot saturated = snapshot(dataSource);
|
||||
assertThat(saturated.state()).isEqualTo(PoolState.SATURATED);
|
||||
assertThat(saturated.activeConnections()).isEqualTo(2);
|
||||
@@ -66,8 +71,10 @@ class PostgreSqlLifecycleIntegrationTest {
|
||||
|
||||
long started = System.nanoTime();
|
||||
assertThatThrownBy(() -> bounded.connection().close()).isInstanceOf(SQLException.class);
|
||||
// At least the 300 ms connectionTimeout, and nowhere near an unbounded wait: the slack
|
||||
// above it is for a loaded machine, not for the pool.
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(2));
|
||||
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(5));
|
||||
}
|
||||
|
||||
assertThat(snapshot(dataSource).activeConnections()).isZero();
|
||||
@@ -81,6 +88,20 @@ class PostgreSqlLifecycleIntegrationTest {
|
||||
assertThat(closed.activeConnections()).isZero();
|
||||
}
|
||||
|
||||
/** A pool connection, retrying while a new physical connection takes longer than the timeout. */
|
||||
private static Connection patiently(PostgreSqlReadinessSupport support) throws SQLException {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
|
||||
while (true) {
|
||||
try {
|
||||
return support.connection();
|
||||
} catch (SQLException slow) {
|
||||
if (System.nanoTime() > until) {
|
||||
throw slow;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static PoolSnapshot snapshot(HikariDataSource dataSource) {
|
||||
int maximum = dataSource.getMaximumPoolSize();
|
||||
if (dataSource.isClosed()) {
|
||||
|
||||
+8
-3
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.readiness;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
@@ -168,9 +169,13 @@ class PostgreSqlMigrationIntegrationTest {
|
||||
.initSql("set lock_timeout = '250ms'; set statement_timeout = '2s'")
|
||||
.load();
|
||||
long started = System.nanoTime();
|
||||
assertThatThrownBy(blockedExpansion::migrate).isInstanceOf(FlywayException.class);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3));
|
||||
Throwable refused = catchThrowable(blockedExpansion::migrate);
|
||||
Duration waited = Duration.ofNanos(System.nanoTime() - started);
|
||||
// The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
|
||||
// says which, so the clock only has to show the migration did not wait for the holder.
|
||||
assertThat(refused).isInstanceOf(FlywayException.class);
|
||||
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
|
||||
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
|
||||
blocker.rollback();
|
||||
}
|
||||
|
||||
|
||||
+8
-4
@@ -1,7 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.readiness;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Statement;
|
||||
@@ -175,9 +175,13 @@ final class PostgreSqlOptionalStreamLifecycle {
|
||||
blocker.setAutoCommit(false);
|
||||
lock.execute("lock table capability_schema_registry in access exclusive mode");
|
||||
long started = System.nanoTime();
|
||||
assertThatThrownBy(flyway::migrate).isInstanceOf(FlywayException.class);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3));
|
||||
Throwable refused = catchThrowable(flyway::migrate);
|
||||
Duration waited = Duration.ofNanos(System.nanoTime() - started);
|
||||
// The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
|
||||
// says which, so the clock only has to show the migration did not wait for the holder.
|
||||
assertThat(refused).isInstanceOf(FlywayException.class);
|
||||
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
|
||||
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
|
||||
blocker.rollback();
|
||||
}
|
||||
|
||||
|
||||
+14
@@ -28,6 +28,20 @@ final class PostgreSqlReadinessSupport implements AutoCloseable {
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
/** Every SQLState in {@code failure}'s cause chain, outermost first. */
|
||||
static java.util.List<String> sqlStates(Throwable failure) {
|
||||
java.util.List<String> states = new java.util.ArrayList<>();
|
||||
for (Throwable cause = failure; cause != null; cause = cause.getCause()) {
|
||||
if (cause instanceof java.sql.SQLException sql && sql.getSQLState() != null) {
|
||||
states.add(sql.getSQLState());
|
||||
}
|
||||
if (cause.getCause() == cause) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
static PostgreSqlReadinessSupport start() {
|
||||
return start(5, 1_000);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user