diff --git a/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/GraphQlRequestDeadlineTest.java b/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/GraphQlRequestDeadlineTest.java index dc59e246..6c056ce0 100644 --- a/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/GraphQlRequestDeadlineTest.java +++ b/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/GraphQlRequestDeadlineTest.java @@ -43,8 +43,10 @@ import org.springframework.http.ResponseEntity; @AutoConfigureTestRestTemplate class GraphQlRequestDeadlineTest { - private static final Duration BUDGET = Duration.ofSeconds(1); - private static final int SLOW_RESOLVER_MILLIS = 4_000; + // Two seconds so the resolver has started before the budget runs out even on a loaded machine + // (SLOW_STARTED below); the resolver itself holds until the test releases it. + private static final Duration BUDGET = Duration.ofSeconds(2); + private static final int SLOW_RESOLVER_MILLIS = 30_000; @LocalServerPort int port; @@ -70,6 +72,7 @@ class GraphQlRequestDeadlineTest { // then long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started); + SlowWork.release.countDown(); assertThat(response.getBody()) .contains("\"code\":\"REQUEST_TIMEOUT\"") .contains("\"category\":\"TIMEOUT\"") @@ -78,9 +81,9 @@ class GraphQlRequestDeadlineTest { .as("the budget must have run out during execution, not before it") .hasValue(1); assertThat(elapsedMillis) - .as("answered at the deadline, not when the %d ms resolver returned", SLOW_RESOLVER_MILLIS) + .as("answered at the 2 s deadline, well before the 10 s default budget would have ended it") .isGreaterThanOrEqualTo(BUDGET.toMillis()) - .isLessThan(SLOW_RESOLVER_MILLIS - 1_000L); + .isLessThan(BUDGET.plusSeconds(6).toMillis()); } @Test @@ -91,8 +94,8 @@ class GraphQlRequestDeadlineTest { http, port, "{ slowNode(millis: " + SLOW_RESOLVER_MILLIS + ") { id detail } }"); // when - boolean finished = - SlowWork.slowFinished.await(SLOW_RESOLVER_MILLIS * 2L, TimeUnit.MILLISECONDS); + SlowWork.release.countDown(); + boolean finished = SlowWork.slowFinished.await(10, TimeUnit.SECONDS); // then assertThat(finished) diff --git a/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java b/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java index 9f79efd6..00b48c15 100644 --- a/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java +++ b/src/adapter/inbound/graphql/src/systemTest/java/dev/caskeleton/adapter/inbound/graphql/runtime/RuntimeLimitsTestSupport.java @@ -12,6 +12,7 @@ import graphql.execution.instrumentation.parameters.InstrumentationExecutionPara import java.net.URI; import java.time.Duration; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.springframework.boot.graphql.autoconfigure.GraphQlSourceBuilderCustomizer; import org.springframework.boot.resttestclient.TestRestTemplate; @@ -92,6 +93,13 @@ final class RuntimeLimitsTestSupport { static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger(); static volatile CountDownLatch slowFinished = new CountDownLatch(1); + /** + * Opened by a test to let the slow resolver return before its {@code millis} cap. Until then + * the resolver holds as a slow dependency would, so "answered before the resolver returned" + * does not depend on how fast a loaded machine runs the rest of the request. + */ + static volatile CountDownLatch release = new CountDownLatch(1); + /** * Counted down when a {@code slowNode} execution begun after the last {@link #reset()} has * ended. The point after which a child field can no longer start, so an absence asserted after @@ -108,6 +116,7 @@ final class RuntimeLimitsTestSupport { DETAIL_INVOCATIONS.set(0); slowFinished = new CountDownLatch(1); slowExecutionEnded = new CountDownLatch(1); + release = new CountDownLatch(1); GENERATION.incrementAndGet(); } @@ -141,8 +150,10 @@ final class RuntimeLimitsTestSupport { @QueryMapping SlowNode slowNode(@Argument int millis) throws InterruptedException { SlowWork.SLOW_STARTED.incrementAndGet(); + CountDownLatch release = SlowWork.release; try { - Thread.sleep(millis); + // Released by the test, or the cap ran out; either way the resolver returns normally. + var unused = release.await(millis, TimeUnit.MILLISECONDS); } finally { SlowWork.slowFinished.countDown(); } diff --git a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java index fedee48b..3c36bdfb 100644 --- a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java +++ b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcP1BoundaryWireTest.java @@ -73,6 +73,7 @@ class GrpcP1BoundaryWireTest { private static final MethodDescriptor CLOSE_HANDLER_METHOD = unaryMethod("CloseHandler"); private static final MethodDescriptor SLOW_METHOD = unaryMethod("Slow"); + private static final MethodDescriptor GATED_METHOD = unaryMethod("Gated"); private static final CountDownLatch CLOSE_HANDLER_RAN = new CountDownLatch(1); /** The platform's GrpcTransportLimits.DEFAULT_MAX_MESSAGE_BYTES, which this adapter mirrors. */ @@ -81,6 +82,17 @@ class GrpcP1BoundaryWireTest { /** How long the slow handler holds its thread unless its gRPC Context is cancelled first. */ private static final long SLOW_HANDLER_HOLD_MILLIS = 400; + /** + * Holds every gated handler until a test releases it or the call's Context is cancelled. A gate, + * not a clock: what a test means by "while the call is still running" must not depend on how fast + * a loaded machine dispatches it (review 2026-11-13: the main stage runs beside other builds). + * Each test that uses it sets a fresh latch and releases it before it ends. + */ + private static volatile CountDownLatch gate = new CountDownLatch(0); + + /** The longest a gated handler waits, so a test that forgets to release cannot hang the JVM. */ + private static final long GATE_CAP_SECONDS = 30; + private static final AtomicInteger SLOW_IN_FLIGHT = new AtomicInteger(); private static final AtomicInteger SLOW_MAX_IN_FLIGHT = new AtomicInteger(); @@ -264,17 +276,30 @@ class GrpcP1BoundaryWireTest { contextRunner.withPropertyValues("app.grpc.server.default-deadline=150ms"), channel -> { // given + // The handler holds until its Context is cancelled and then answers nothing, so the + // only way the call can end is the server deadline closing it. waitForReady keeps a + // slow first connection from failing the call before it reaches the server. + gate = new CountDownLatch(1); + long started = System.nanoTime(); ListenableFuture withoutDeadline = ClientCalls.futureUnaryCall( - authenticated(channel).newCall(SLOW_METHOD, CallOptions.DEFAULT), "slow"); + authenticated(channel) + .newCall(GATED_METHOD, CallOptions.DEFAULT.withWaitForReady()), + "gated"); // when - Throwable failure = catchThrowable(() -> withoutDeadline.get(5, TimeUnit.SECONDS)); + Throwable failure = + catchThrowable(() -> withoutDeadline.get(GATE_CAP_SECONDS, TimeUnit.SECONDS)); // then + gate.countDown(); + assertThat(failure).as("the call must fail, not answer").isNotNull(); assertThat(Status.fromThrowable(failure).getCode()) .as("status of %s", failure) .isEqualTo(Status.Code.DEADLINE_EXCEEDED); + assertThat(Duration.ofNanos(System.nanoTime() - started)) + .as("the configured 150 ms ended it, not the 10 s default it would fall back to") + .isLessThan(Duration.ofSeconds(5)); }); } @@ -285,6 +310,9 @@ class GrpcP1BoundaryWireTest { contextRunner.withPropertyValues("app.grpc.server.default-deadline=10s"), channel -> { // given + // A gated handler never answers before the gate opens, so the client deadline is the + // only thing that can end the call: the 10 s server default must not have replaced it. + gate = new CountDownLatch(1); long started = System.nanoTime(); // when @@ -293,14 +321,18 @@ class GrpcP1BoundaryWireTest { () -> ClientCalls.blockingUnaryCall( authenticated(channel), - SLOW_METHOD, + GATED_METHOD, CallOptions.DEFAULT.withDeadlineAfter(100, TimeUnit.MILLISECONDS), - "slow")); + "gated")); // then + gate.countDown(); + assertThat(failure).as("the call must fail, not answer").isNotNull(); assertThat(Status.fromThrowable(failure).getCode()) + .as("status of %s", failure) .isEqualTo(Status.Code.DEADLINE_EXCEEDED); assertThat(Duration.ofNanos(System.nanoTime() - started)) + .as("ended by the 100 ms client deadline") .isLessThan(Duration.ofSeconds(5)); }); } @@ -338,18 +370,25 @@ class GrpcP1BoundaryWireTest { "app.grpc.server.executor-queue-capacity=1"), channel -> { // given + // Gated handlers hold the one thread and the one queue slot until the refusal is + // seen, so the third call meets a full server however slowly the calls are dispatched. + gate = new CountDownLatch(1); Channel caller = authenticated(channel); - CallOptions bounded = CallOptions.DEFAULT.withDeadlineAfter(5, TimeUnit.SECONDS); + CallOptions bounded = + CallOptions.DEFAULT.withDeadlineAfter(GATE_CAP_SECONDS, TimeUnit.SECONDS); List> calls = List.of( - ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"), - ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"), - ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow")); + ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"), + ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"), + ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated")); // when + boolean refusedWhileHeld = awaitAnyDone(calls); + gate.countDown(); List outcomes = calls.stream().map(GrpcP1BoundaryWireTest::outcome).toList(); // then + assertThat(refusedWhileHeld).as("one call is refused while the others hold").isTrue(); assertThat(outcomes) .extracting(Status::getCode) .containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE); @@ -361,10 +400,27 @@ class GrpcP1BoundaryWireTest { } private static Status outcome(ListenableFuture call) { - Throwable failure = catchThrowable(() -> call.get(5, TimeUnit.SECONDS)); + Throwable failure = catchThrowable(() -> call.get(GATE_CAP_SECONDS, TimeUnit.SECONDS)); return failure == null ? Status.OK : Status.fromThrowable(failure); } + /** Whether one of {@code calls} completes while the gate still holds the others. */ + private static boolean awaitAnyDone(List> calls) { + long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS); + while (System.nanoTime() < until) { + if (calls.stream().anyMatch(ListenableFuture::isDone)) { + return true; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return false; + } + private static String getUnchecked(ListenableFuture future) { try { return future.get(5, TimeUnit.SECONDS); @@ -550,6 +606,17 @@ class GrpcP1BoundaryWireTest { observer.onNext(request + "-done"); observer.onCompleted(); })) + .addMethod( + GATED_METHOD, + ServerCalls.asyncUnaryCall( + (String request, StreamObserver observer) -> { + if (holdUntilReleased()) { + observer.onNext(request + "-done"); + observer.onCompleted(); + } + // Cancelled: a cooperative handler answers nothing; whoever cancelled + // the call (the server deadline) has already closed it. + })) .addMethod( CLOSE_HANDLER_METHOD, ServerCalls.asyncUnaryCall( @@ -601,6 +668,29 @@ class GrpcP1BoundaryWireTest { } } + /** + * Holds the handler thread until the test opens {@link #gate}. + * + * @return true once released; false when the call's Context was cancelled first (or the cap ran + * out) + */ + private static boolean holdUntilReleased() { + long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS); + try { + while (System.nanoTime() < until) { + if (Context.current().isCancelled()) { + return false; + } + if (gate.await(10, TimeUnit.MILLISECONDS)) { + return true; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return false; + } + private enum StringMarshaller implements MethodDescriptor.Marshaller { INSTANCE; diff --git a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcServerRunnerBootTest.java b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcServerRunnerBootTest.java index e6249b01..28c3d28d 100644 --- a/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcServerRunnerBootTest.java +++ b/src/adapter/inbound/grpc/src/test/java/dev/caskeleton/adapter/inbound/grpc/GrpcServerRunnerBootTest.java @@ -98,7 +98,9 @@ class GrpcServerRunnerBootTest { void stoppingKeepsTheListenerUpReportingNotServingDuringTheDrainDelay() { contextRunner .withPropertyValues( - "app.grpc.server.shutdown-drain-seconds=2", "app.grpc.server.shutdown-grace-seconds=4") + // The drain is long enough that a loaded machine still sees it: the probe window below + // must close before the listener does, and both are seconds, not a scheduling slice. + "app.grpc.server.shutdown-drain-seconds=6", "app.grpc.server.shutdown-grace-seconds=8") .run( context -> { // given @@ -107,6 +109,15 @@ class GrpcServerRunnerBootTest { ManagedChannelBuilder.forAddress("localhost", runner.getListeningPort()) .usePlaintext() .build(); + // Connect before shutdown begins, so the drain window is not spent on the first + // connection. + assertThat( + HealthGrpc.newBlockingStub(channel) + .withWaitForReady() + .withDeadlineAfter(10, TimeUnit.SECONDS) + .check(HealthCheckRequest.newBuilder().build()) + .getStatus()) + .isEqualTo(ServingStatus.SERVING); Thread stopping = Thread.ofVirtual().start(runner::stop); try { // when @@ -115,7 +126,7 @@ class GrpcServerRunnerBootTest { // then assertThat(duringDrain).isEqualTo(ServingStatus.NOT_SERVING); } finally { - stopping.join(Duration.ofSeconds(10)); + stopping.join(Duration.ofSeconds(20)); channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); } }); @@ -127,13 +138,9 @@ class GrpcServerRunnerBootTest { */ private static ServingStatus firstStatusOtherThanServing(ManagedChannel channel) throws InterruptedException { - long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(4); while (System.nanoTime() < until) { - ServingStatus status = - HealthGrpc.newBlockingStub(channel) - .withDeadlineAfter(1, TimeUnit.SECONDS) - .check(HealthCheckRequest.newBuilder().build()) - .getStatus(); + ServingStatus status = health(channel); if (status != ServingStatus.SERVING) { return status; } @@ -142,6 +149,14 @@ class GrpcServerRunnerBootTest { return ServingStatus.SERVING; } + // Two seconds per probe: the four-second window plus one probe still ends inside the 6 s drain. + private static ServingStatus health(ManagedChannel channel) { + return HealthGrpc.newBlockingStub(channel) + .withDeadlineAfter(2, TimeUnit.SECONDS) + .check(HealthCheckRequest.newBuilder().build()) + .getStatus(); + } + @Configuration(proxyBeanMethods = false) static class UnqualifiedServiceConfiguration { diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisSemanticPortsTest.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisSemanticPortsTest.java index 75960d15..b36b1334 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisSemanticPortsTest.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/LiveRedisSemanticPortsTest.java @@ -217,7 +217,9 @@ class LiveRedisSemanticPortsTest { "api-default", "v1", RateLimitAlgorithm.FIXED_WINDOW, - new RateParameters.FixedWindow(3, Duration.ofSeconds(10)), + // An hour-long window: the four calls below must land in one window, and a 10 s window + // aligned to Redis TIME let a slow run straddle its boundary. + new RateParameters.FixedWindow(3, Duration.ofHours(1)), 1, Duration.ofSeconds(5), Duration.ofMillis(250), diff --git a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/lease/LiveRedisDistributedLeaseContractTest.java b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/lease/LiveRedisDistributedLeaseContractTest.java index f8199492..a72488a2 100644 --- a/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/lease/LiveRedisDistributedLeaseContractTest.java +++ b/src/adapter/outbound/cache-redis/src/test/java/dev/caskeleton/adapter/outbound/cache/redis/lease/LiveRedisDistributedLeaseContractTest.java @@ -48,9 +48,13 @@ class LiveRedisDistributedLeaseContractTest extends DistributedLeaseContract { Duration.ZERO); } + // A live server waits for real, so the TTL is seconds rather than the fake's 30: long enough that + // the calls a test makes while it holds a lease never outlast it on a loaded machine. Each of the + // four displaced-holder cases sleeps 1.5 TTL, so this costs about 12 s per run (it was about 2 s + // with a 400 ms TTL that a slow script call could outlast). @Override protected Duration leaseTtl() { - return Duration.ofMillis(400); + return Duration.ofSeconds(2); } @Override diff --git a/src/adapter/outbound/httpclient/src/integrationTest/java/dev/caskeleton/adapter/outbound/httpclient/contract/FailureInjectionContractTest.java b/src/adapter/outbound/httpclient/src/integrationTest/java/dev/caskeleton/adapter/outbound/httpclient/contract/FailureInjectionContractTest.java index 0bf00d98..64948ec1 100644 --- a/src/adapter/outbound/httpclient/src/integrationTest/java/dev/caskeleton/adapter/outbound/httpclient/contract/FailureInjectionContractTest.java +++ b/src/adapter/outbound/httpclient/src/integrationTest/java/dev/caskeleton/adapter/outbound/httpclient/contract/FailureInjectionContractTest.java @@ -43,6 +43,21 @@ import org.testcontainers.utility.DockerImageName; class FailureInjectionContractTest { private static final Duration SLOW_DRIP_BUDGET = Duration.ofSeconds(1); + private static final int SLOW_DRIP_BYTES = 4096; + private static final int SLOW_DRIP_SLICE_BYTES = 16; + private static final Duration SLOW_DRIP_GAP = Duration.ofMillis(50); + + /** How long the drip alone takes to deliver the body: 256 slices, 50 ms apart (12.8 s). */ + private static final Duration SLOW_DRIP_TOTAL = + SLOW_DRIP_GAP.multipliedBy(SLOW_DRIP_BYTES / SLOW_DRIP_SLICE_BYTES); + + /** + * Time a loaded machine may add between the deadline firing and the caller seeing the failure. + * The main stage runs beside other builds; the bound only has to tell "cut at the budget" from + * "waited for the drip", which the total leaves room for (budget + slack is well under half of + * it). + */ + private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(4); private static final DockerImageName UPSTREAM_IMAGE = DockerImageName.parse( @@ -129,11 +144,11 @@ class FailureInjectionContractTest { try (ToxiproxyFixture toxiproxy = ToxiproxyFixture.start(network)) { // given Proxy proxy = toxiproxy.proxyTo("slow-drip", "upstream", 80); - toxiproxy.slowDrip(proxy, 16, Duration.ofMillis(50)); + toxiproxy.slowDrip(proxy, SLOW_DRIP_SLICE_BYTES, SLOW_DRIP_GAP); TimeoutSettings base = ClientProfiles.timeouts(); URI baseUrl = URI.create("http://" + toxiproxy.proxiedHost() + ":" + toxiproxy.proxiedPort() + "/"); - long started; + Duration elapsed; Throwable failure; // when @@ -151,7 +166,7 @@ class FailureInjectionContractTest { base.readIdle(), SLOW_DRIP_BUDGET)) .build())) { - started = System.nanoTime(); + long started = System.nanoTime(); failure = catchThrowable( () -> @@ -160,14 +175,22 @@ class FailureInjectionContractTest { .exchange( harness.profile().name(), HttpOperation.get( - new OperationName("fault-probe"), "/bytes/4096", Map.of()), + new OperationName("fault-probe"), + "/bytes/" + SLOW_DRIP_BYTES, + Map.of()), ResponseType.of(byte[].class))); + // The call alone: closing the harness afterwards is not part of the deadline. + elapsed = Duration.ofNanos(System.nanoTime() - started); } // then - Duration elapsed = Duration.ofNanos(System.nanoTime() - started); assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class); - assertThat(elapsed).isLessThan(SLOW_DRIP_BUDGET.plus(Duration.ofMillis(500))); + assertThat(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK)) + .as("the bound must stay well under the drip, or it proves nothing") + .isLessThan(SLOW_DRIP_TOTAL.dividedBy(2)); + assertThat(elapsed) + .as("the deadline, not the drip, ended the call") + .isLessThan(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK)); } } } diff --git a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadlineTest.java b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadlineTest.java index fa2cfc77..98e04f74 100644 --- a/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadlineTest.java +++ b/src/adapter/outbound/httpclient/src/test/java/dev/caskeleton/adapter/outbound/httpclient/transport/AttemptDeadlineTest.java @@ -29,15 +29,17 @@ class AttemptDeadlineTest { @Test @DisplayName("남은 deadline이 단계 timeout보다 짧으면 단계 timeout을 남은 시간으로 줄인다") void aShorterRemainingDeadlineBoundsTheStageTimeout() { - try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(300))) { + // Seconds rather than milliseconds, so the deadline has not already passed by the time bound() + // runs on a loaded machine; the stage is longer still, so bound() must cut it. + try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(5))) { // given - Duration stage = Duration.ofSeconds(2); + Duration stage = Duration.ofSeconds(30); // when Duration bounded = AttemptDeadline.bound(stage); // then - assertThat(bounded).isLessThanOrEqualTo(Duration.ofMillis(300)).isPositive(); + assertThat(bounded).isLessThanOrEqualTo(Duration.ofSeconds(5)).isPositive(); assertThat(scope.expired()).isFalse(); } } @@ -89,14 +91,16 @@ class AttemptDeadlineTest { void closingTheScopeBeforeTheDeadlineCancelsTheAbort() throws Exception { // given AtomicInteger aborts = new AtomicInteger(); - AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(100)); + // One second, so registering and closing always happen before it on a loaded machine; the + // wait after the close outlasts it, so an abort that was not cancelled would have run. + AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(1)); AttemptDeadline.abortOnExpiry(aborts::incrementAndGet); // when scope.close(); // then - Thread.sleep(300); + Thread.sleep(1_500); assertThat(aborts).hasValue(0); assertThat(AttemptDeadline.remaining()).isEmpty(); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/PostgreSqlPessimisticLockContractTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/PostgreSqlPessimisticLockContractTest.java index 01c42e49..4b377fac 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/PostgreSqlPessimisticLockContractTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/platform/PostgreSqlPessimisticLockContractTest.java @@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName; import dev.caskeleton.adapter.outbound.persistence.api.error.PessimisticLockTimeoutException; import dev.caskeleton.adapter.outbound.persistence.postgresql.lock.PostgreSqlLockExceptionTranslator; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.Duration; @@ -59,14 +60,23 @@ class PostgreSqlPessimisticLockContractTest { lockRow(holder); contender.setAutoCommit(false); + try (Statement statement = contender.createStatement(); + ResultSet setting = statement.executeQuery("show lock_timeout")) { + assertThat(setting.next()).isTrue(); + assertThat(setting.getString(1)) + .as("without a lock_timeout a lock that waited would wait for the holder for good") + .isEqualTo("0"); + } Instant startedAt = Instant.now(); SQLException refused = lockRowNowait(contender); Duration waited = Duration.between(startedAt, Instant.now()); assertThat(refused.getSQLState()).isEqualTo("55P03"); + // The contender has no lock_timeout, so a lock that waited would wait for the holder, who + // never lets go here. Seconds of slack cannot be mistaken for that. assertThat(waited) .as("NOWAIT must not wait for the holder") - .isLessThan(Duration.ofSeconds(1)); + .isLessThan(Duration.ofSeconds(10)); var translated = translator.translate(refused, OPERATION, 1, waited, null).orElseThrow(); assertThat(translated).isInstanceOf(PessimisticLockTimeoutException.class); diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlLifecycleIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlLifecycleIntegrationTest.java index 85192b92..0bf63f0e 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlLifecycleIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlLifecycleIntegrationTest.java @@ -12,6 +12,7 @@ import java.sql.Statement; import java.time.Duration; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -49,13 +50,17 @@ class PostgreSqlLifecycleIntegrationTest { } @Test - @Timeout(10) + // The method starts its own PostgreSQL container; on a loaded machine that alone can take tens of + // seconds. The bound that matters, the exhausted pool's wait, is measured on its own below. + @Timeout(value = 2, unit = TimeUnit.MINUTES) void poolCapacityExhaustionAndShutdownAreBoundedAndObservable() throws Exception { PostgreSqlReadinessSupport bounded = PostgreSqlReadinessSupport.start(2, 300); HikariDataSource dataSource = bounded.dataSource(); try { - try (Connection first = bounded.connection(); - Connection second = bounded.connection()) { + // Borrowing a connection the pool still has to open (TCP, SCRAM, backend fork) must not be + // held to the 300 ms timeout this test measures, so the two are taken patiently. + try (Connection first = patiently(bounded); + Connection second = patiently(bounded)) { PoolSnapshot saturated = snapshot(dataSource); assertThat(saturated.state()).isEqualTo(PoolState.SATURATED); assertThat(saturated.activeConnections()).isEqualTo(2); @@ -66,8 +71,10 @@ class PostgreSqlLifecycleIntegrationTest { long started = System.nanoTime(); assertThatThrownBy(() -> bounded.connection().close()).isInstanceOf(SQLException.class); + // At least the 300 ms connectionTimeout, and nowhere near an unbounded wait: the slack + // above it is for a loaded machine, not for the pool. assertThat(Duration.ofNanos(System.nanoTime() - started)) - .isBetween(Duration.ofMillis(250), Duration.ofSeconds(2)); + .isBetween(Duration.ofMillis(250), Duration.ofSeconds(5)); } assertThat(snapshot(dataSource).activeConnections()).isZero(); @@ -81,6 +88,20 @@ class PostgreSqlLifecycleIntegrationTest { assertThat(closed.activeConnections()).isZero(); } + /** A pool connection, retrying while a new physical connection takes longer than the timeout. */ + private static Connection patiently(PostgreSqlReadinessSupport support) throws SQLException { + long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (true) { + try { + return support.connection(); + } catch (SQLException slow) { + if (System.nanoTime() > until) { + throw slow; + } + } + } + } + private static PoolSnapshot snapshot(HikariDataSource dataSource) { int maximum = dataSource.getMaximumPoolSize(); if (dataSource.isClosed()) { diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java index bf5117a8..825ac309 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlMigrationIntegrationTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.readiness; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import java.nio.charset.StandardCharsets; import java.sql.Connection; @@ -168,9 +169,13 @@ class PostgreSqlMigrationIntegrationTest { .initSql("set lock_timeout = '250ms'; set statement_timeout = '2s'") .load(); long started = System.nanoTime(); - assertThatThrownBy(blockedExpansion::migrate).isInstanceOf(FlywayException.class); - assertThat(Duration.ofNanos(System.nanoTime() - started)) - .isBetween(Duration.ofMillis(200), Duration.ofSeconds(3)); + Throwable refused = catchThrowable(blockedExpansion::migrate); + Duration waited = Duration.ofNanos(System.nanoTime() - started); + // The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause + // says which, so the clock only has to show the migration did not wait for the holder. + assertThat(refused).isInstanceOf(FlywayException.class); + assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03"); + assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30)); blocker.rollback(); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java index f25dfaee..f7c8f434 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlOptionalStreamLifecycle.java @@ -1,7 +1,7 @@ package dev.caskeleton.adapter.outbound.persistence.readiness; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import java.sql.Connection; import java.sql.Statement; @@ -175,9 +175,13 @@ final class PostgreSqlOptionalStreamLifecycle { blocker.setAutoCommit(false); lock.execute("lock table capability_schema_registry in access exclusive mode"); long started = System.nanoTime(); - assertThatThrownBy(flyway::migrate).isInstanceOf(FlywayException.class); - assertThat(Duration.ofNanos(System.nanoTime() - started)) - .isBetween(Duration.ofMillis(200), Duration.ofSeconds(3)); + Throwable refused = catchThrowable(flyway::migrate); + Duration waited = Duration.ofNanos(System.nanoTime() - started); + // The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause + // says which, so the clock only has to show the migration did not wait for the holder. + assertThat(refused).isInstanceOf(FlywayException.class); + assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03"); + assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30)); blocker.rollback(); } diff --git a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlReadinessSupport.java b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlReadinessSupport.java index 312bd9be..fda534fe 100644 --- a/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlReadinessSupport.java +++ b/src/adapter/outbound/persistence-jpa/src/postgresqlIntegrationTest/java/dev/caskeleton/adapter/outbound/persistence/readiness/PostgreSqlReadinessSupport.java @@ -28,6 +28,20 @@ final class PostgreSqlReadinessSupport implements AutoCloseable { this.dataSource = dataSource; } + /** Every SQLState in {@code failure}'s cause chain, outermost first. */ + static java.util.List sqlStates(Throwable failure) { + java.util.List states = new java.util.ArrayList<>(); + for (Throwable cause = failure; cause != null; cause = cause.getCause()) { + if (cause instanceof java.sql.SQLException sql && sql.getSQLState() != null) { + states.add(sql.getSQLState()); + } + if (cause.getCause() == cause) { + break; + } + } + return states; + } + static PostgreSqlReadinessSupport start() { return start(5, 1_000); }