test: 부하 아래에서 흔들리던 벽시계 시험을 원인 단언과 gate로 고친다 #34

Merged
donghyeon.kang merged 3 commits from claude/r9-wallclock-tests into main 2026-11-13 07:58:20 +00:00
13 changed files with 255 additions and 49 deletions
@@ -43,8 +43,10 @@ import org.springframework.http.ResponseEntity;
@AutoConfigureTestRestTemplate
class GraphQlRequestDeadlineTest {
private static final Duration BUDGET = Duration.ofSeconds(1);
private static final int SLOW_RESOLVER_MILLIS = 4_000;
// Two seconds so the resolver has started before the budget runs out even on a loaded machine
// (SLOW_STARTED below); the resolver itself holds until the test releases it.
private static final Duration BUDGET = Duration.ofSeconds(2);
private static final int SLOW_RESOLVER_MILLIS = 30_000;
@LocalServerPort int port;
@@ -70,6 +72,7 @@ class GraphQlRequestDeadlineTest {
// then
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
SlowWork.release.countDown();
assertThat(response.getBody())
.contains("\"code\":\"REQUEST_TIMEOUT\"")
.contains("\"category\":\"TIMEOUT\"")
@@ -78,9 +81,9 @@ class GraphQlRequestDeadlineTest {
.as("the budget must have run out during execution, not before it")
.hasValue(1);
assertThat(elapsedMillis)
.as("answered at the deadline, not when the %d ms resolver returned", SLOW_RESOLVER_MILLIS)
.as("answered at the 2 s deadline, well before the 10 s default budget would have ended it")
.isGreaterThanOrEqualTo(BUDGET.toMillis())
.isLessThan(SLOW_RESOLVER_MILLIS - 1_000L);
.isLessThan(BUDGET.plusSeconds(6).toMillis());
}
@Test
@@ -91,8 +94,8 @@ class GraphQlRequestDeadlineTest {
http, port, "{ slowNode(millis: " + SLOW_RESOLVER_MILLIS + ") { id detail } }");
// when
boolean finished =
SlowWork.slowFinished.await(SLOW_RESOLVER_MILLIS * 2L, TimeUnit.MILLISECONDS);
SlowWork.release.countDown();
boolean finished = SlowWork.slowFinished.await(10, TimeUnit.SECONDS);
// then
assertThat(finished)
@@ -12,6 +12,7 @@ import graphql.execution.instrumentation.parameters.InstrumentationExecutionPara
import java.net.URI;
import java.time.Duration;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.springframework.boot.graphql.autoconfigure.GraphQlSourceBuilderCustomizer;
import org.springframework.boot.resttestclient.TestRestTemplate;
@@ -92,6 +93,13 @@ final class RuntimeLimitsTestSupport {
static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger();
static volatile CountDownLatch slowFinished = new CountDownLatch(1);
/**
* Opened by a test to let the slow resolver return before its {@code millis} cap. Until then
* the resolver holds as a slow dependency would, so "answered before the resolver returned"
* does not depend on how fast a loaded machine runs the rest of the request.
*/
static volatile CountDownLatch release = new CountDownLatch(1);
/**
* Counted down when a {@code slowNode} execution begun after the last {@link #reset()} has
* ended. The point after which a child field can no longer start, so an absence asserted after
@@ -108,6 +116,7 @@ final class RuntimeLimitsTestSupport {
DETAIL_INVOCATIONS.set(0);
slowFinished = new CountDownLatch(1);
slowExecutionEnded = new CountDownLatch(1);
release = new CountDownLatch(1);
GENERATION.incrementAndGet();
}
@@ -141,8 +150,10 @@ final class RuntimeLimitsTestSupport {
@QueryMapping
SlowNode slowNode(@Argument int millis) throws InterruptedException {
SlowWork.SLOW_STARTED.incrementAndGet();
CountDownLatch release = SlowWork.release;
try {
Thread.sleep(millis);
// Released by the test, or the cap ran out; either way the resolver returns normally.
var unused = release.await(millis, TimeUnit.MILLISECONDS);
} finally {
SlowWork.slowFinished.countDown();
}
@@ -73,6 +73,7 @@ class GrpcP1BoundaryWireTest {
private static final MethodDescriptor<String, String> CLOSE_HANDLER_METHOD =
unaryMethod("CloseHandler");
private static final MethodDescriptor<String, String> SLOW_METHOD = unaryMethod("Slow");
private static final MethodDescriptor<String, String> GATED_METHOD = unaryMethod("Gated");
private static final CountDownLatch CLOSE_HANDLER_RAN = new CountDownLatch(1);
/** The platform's GrpcTransportLimits.DEFAULT_MAX_MESSAGE_BYTES, which this adapter mirrors. */
@@ -81,6 +82,17 @@ class GrpcP1BoundaryWireTest {
/** How long the slow handler holds its thread unless its gRPC Context is cancelled first. */
private static final long SLOW_HANDLER_HOLD_MILLIS = 400;
/**
* Holds every gated handler until a test releases it or the call's Context is cancelled. A gate,
* not a clock: what a test means by "while the call is still running" must not depend on how fast
* a loaded machine dispatches it (review 2026-11-13: the main stage runs beside other builds).
* Each test that uses it sets a fresh latch and releases it before it ends.
*/
private static volatile CountDownLatch gate = new CountDownLatch(0);
/** The longest a gated handler waits, so a test that forgets to release cannot hang the JVM. */
private static final long GATE_CAP_SECONDS = 30;
private static final AtomicInteger SLOW_IN_FLIGHT = new AtomicInteger();
private static final AtomicInteger SLOW_MAX_IN_FLIGHT = new AtomicInteger();
@@ -264,17 +276,30 @@ class GrpcP1BoundaryWireTest {
contextRunner.withPropertyValues("app.grpc.server.default-deadline=150ms"),
channel -> {
// given
// The handler holds until its Context is cancelled and then answers nothing, so the
// only way the call can end is the server deadline closing it. waitForReady keeps a
// slow first connection from failing the call before it reaches the server.
gate = new CountDownLatch(1);
long started = System.nanoTime();
ListenableFuture<String> withoutDeadline =
ClientCalls.futureUnaryCall(
authenticated(channel).newCall(SLOW_METHOD, CallOptions.DEFAULT), "slow");
authenticated(channel)
.newCall(GATED_METHOD, CallOptions.DEFAULT.withWaitForReady()),
"gated");
// when
Throwable failure = catchThrowable(() -> withoutDeadline.get(5, TimeUnit.SECONDS));
Throwable failure =
catchThrowable(() -> withoutDeadline.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
// then
gate.countDown();
assertThat(failure).as("the call must fail, not answer").isNotNull();
assertThat(Status.fromThrowable(failure).getCode())
.as("status of %s", failure)
.isEqualTo(Status.Code.DEADLINE_EXCEEDED);
assertThat(Duration.ofNanos(System.nanoTime() - started))
.as("the configured 150 ms ended it, not the 10 s default it would fall back to")
.isLessThan(Duration.ofSeconds(5));
});
}
@@ -285,6 +310,9 @@ class GrpcP1BoundaryWireTest {
contextRunner.withPropertyValues("app.grpc.server.default-deadline=10s"),
channel -> {
// given
// A gated handler never answers before the gate opens, so the client deadline is the
// only thing that can end the call: the 10 s server default must not have replaced it.
gate = new CountDownLatch(1);
long started = System.nanoTime();
// when
@@ -293,14 +321,18 @@ class GrpcP1BoundaryWireTest {
() ->
ClientCalls.blockingUnaryCall(
authenticated(channel),
SLOW_METHOD,
GATED_METHOD,
CallOptions.DEFAULT.withDeadlineAfter(100, TimeUnit.MILLISECONDS),
"slow"));
"gated"));
// then
gate.countDown();
assertThat(failure).as("the call must fail, not answer").isNotNull();
assertThat(Status.fromThrowable(failure).getCode())
.as("status of %s", failure)
.isEqualTo(Status.Code.DEADLINE_EXCEEDED);
assertThat(Duration.ofNanos(System.nanoTime() - started))
.as("ended by the 100 ms client deadline")
.isLessThan(Duration.ofSeconds(5));
});
}
@@ -338,18 +370,25 @@ class GrpcP1BoundaryWireTest {
"app.grpc.server.executor-queue-capacity=1"),
channel -> {
// given
// Gated handlers hold the one thread and the one queue slot until the refusal is
// seen, so the third call meets a full server however slowly the calls are dispatched.
gate = new CountDownLatch(1);
Channel caller = authenticated(channel);
CallOptions bounded = CallOptions.DEFAULT.withDeadlineAfter(5, TimeUnit.SECONDS);
CallOptions bounded =
CallOptions.DEFAULT.withDeadlineAfter(GATE_CAP_SECONDS, TimeUnit.SECONDS);
List<ListenableFuture<String>> calls =
List.of(
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"),
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"),
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"));
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"));
// when
boolean refusedWhileHeld = awaitAnyDone(calls);
gate.countDown();
List<Status> outcomes = calls.stream().map(GrpcP1BoundaryWireTest::outcome).toList();
// then
assertThat(refusedWhileHeld).as("one call is refused while the others hold").isTrue();
assertThat(outcomes)
.extracting(Status::getCode)
.containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE);
@@ -361,10 +400,27 @@ class GrpcP1BoundaryWireTest {
}
private static Status outcome(ListenableFuture<String> call) {
Throwable failure = catchThrowable(() -> call.get(5, TimeUnit.SECONDS));
Throwable failure = catchThrowable(() -> call.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
return failure == null ? Status.OK : Status.fromThrowable(failure);
}
/** Whether one of {@code calls} completes while the gate still holds the others. */
private static boolean awaitAnyDone(List<ListenableFuture<String>> calls) {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
while (System.nanoTime() < until) {
if (calls.stream().anyMatch(ListenableFuture::isDone)) {
return true;
}
try {
Thread.sleep(10);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
}
return false;
}
private static String getUnchecked(ListenableFuture<String> future) {
try {
return future.get(5, TimeUnit.SECONDS);
@@ -550,6 +606,17 @@ class GrpcP1BoundaryWireTest {
observer.onNext(request + "-done");
observer.onCompleted();
}))
.addMethod(
GATED_METHOD,
ServerCalls.asyncUnaryCall(
(String request, StreamObserver<String> observer) -> {
if (holdUntilReleased()) {
observer.onNext(request + "-done");
observer.onCompleted();
}
// Cancelled: a cooperative handler answers nothing; whoever cancelled
// the call (the server deadline) has already closed it.
}))
.addMethod(
CLOSE_HANDLER_METHOD,
ServerCalls.asyncUnaryCall(
@@ -601,6 +668,29 @@ class GrpcP1BoundaryWireTest {
}
}
/**
* Holds the handler thread until the test opens {@link #gate}.
*
* @return true once released; false when the call's Context was cancelled first (or the cap ran
* out)
*/
private static boolean holdUntilReleased() {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
try {
while (System.nanoTime() < until) {
if (Context.current().isCancelled()) {
return false;
}
if (gate.await(10, TimeUnit.MILLISECONDS)) {
return true;
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return false;
}
private enum StringMarshaller implements MethodDescriptor.Marshaller<String> {
INSTANCE;
@@ -98,7 +98,9 @@ class GrpcServerRunnerBootTest {
void stoppingKeepsTheListenerUpReportingNotServingDuringTheDrainDelay() {
contextRunner
.withPropertyValues(
"app.grpc.server.shutdown-drain-seconds=2", "app.grpc.server.shutdown-grace-seconds=4")
// The drain is long enough that a loaded machine still sees it: the probe window below
// must close before the listener does, and both are seconds, not a scheduling slice.
"app.grpc.server.shutdown-drain-seconds=6", "app.grpc.server.shutdown-grace-seconds=8")
.run(
context -> {
// given
@@ -107,6 +109,15 @@ class GrpcServerRunnerBootTest {
ManagedChannelBuilder.forAddress("localhost", runner.getListeningPort())
.usePlaintext()
.build();
// Connect before shutdown begins, so the drain window is not spent on the first
// connection.
assertThat(
HealthGrpc.newBlockingStub(channel)
.withWaitForReady()
.withDeadlineAfter(10, TimeUnit.SECONDS)
.check(HealthCheckRequest.newBuilder().build())
.getStatus())
.isEqualTo(ServingStatus.SERVING);
Thread stopping = Thread.ofVirtual().start(runner::stop);
try {
// when
@@ -115,7 +126,7 @@ class GrpcServerRunnerBootTest {
// then
assertThat(duringDrain).isEqualTo(ServingStatus.NOT_SERVING);
} finally {
stopping.join(Duration.ofSeconds(10));
stopping.join(Duration.ofSeconds(20));
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
}
});
@@ -127,13 +138,9 @@ class GrpcServerRunnerBootTest {
*/
private static ServingStatus firstStatusOtherThanServing(ManagedChannel channel)
throws InterruptedException {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(4);
while (System.nanoTime() < until) {
ServingStatus status =
HealthGrpc.newBlockingStub(channel)
.withDeadlineAfter(1, TimeUnit.SECONDS)
.check(HealthCheckRequest.newBuilder().build())
.getStatus();
ServingStatus status = health(channel);
if (status != ServingStatus.SERVING) {
return status;
}
@@ -142,6 +149,14 @@ class GrpcServerRunnerBootTest {
return ServingStatus.SERVING;
}
// Two seconds per probe: the four-second window plus one probe still ends inside the 6 s drain.
private static ServingStatus health(ManagedChannel channel) {
return HealthGrpc.newBlockingStub(channel)
.withDeadlineAfter(2, TimeUnit.SECONDS)
.check(HealthCheckRequest.newBuilder().build())
.getStatus();
}
@Configuration(proxyBeanMethods = false)
static class UnqualifiedServiceConfiguration {
@@ -217,7 +217,9 @@ class LiveRedisSemanticPortsTest {
"api-default",
"v1",
RateLimitAlgorithm.FIXED_WINDOW,
new RateParameters.FixedWindow(3, Duration.ofSeconds(10)),
// An hour-long window: the four calls below must land in one window, and a 10 s window
// aligned to Redis TIME let a slow run straddle its boundary.
new RateParameters.FixedWindow(3, Duration.ofHours(1)),
1,
Duration.ofSeconds(5),
Duration.ofMillis(250),
@@ -48,9 +48,13 @@ class LiveRedisDistributedLeaseContractTest extends DistributedLeaseContract {
Duration.ZERO);
}
// A live server waits for real, so the TTL is seconds rather than the fake's 30: long enough that
// the calls a test makes while it holds a lease never outlast it on a loaded machine. Each of the
// four displaced-holder cases sleeps 1.5 TTL, so this costs about 12 s per run (it was about 2 s
// with a 400 ms TTL that a slow script call could outlast).
@Override
protected Duration leaseTtl() {
return Duration.ofMillis(400);
return Duration.ofSeconds(2);
}
@Override
@@ -43,6 +43,21 @@ import org.testcontainers.utility.DockerImageName;
class FailureInjectionContractTest {
private static final Duration SLOW_DRIP_BUDGET = Duration.ofSeconds(1);
private static final int SLOW_DRIP_BYTES = 4096;
private static final int SLOW_DRIP_SLICE_BYTES = 16;
private static final Duration SLOW_DRIP_GAP = Duration.ofMillis(50);
/** How long the drip alone takes to deliver the body: 256 slices, 50 ms apart (12.8 s). */
private static final Duration SLOW_DRIP_TOTAL =
SLOW_DRIP_GAP.multipliedBy(SLOW_DRIP_BYTES / SLOW_DRIP_SLICE_BYTES);
/**
* Time a loaded machine may add between the deadline firing and the caller seeing the failure.
* The main stage runs beside other builds; the bound only has to tell "cut at the budget" from
* "waited for the drip", which the total leaves room for (budget + slack is well under half of
* it).
*/
private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(4);
private static final DockerImageName UPSTREAM_IMAGE =
DockerImageName.parse(
@@ -129,11 +144,11 @@ class FailureInjectionContractTest {
try (ToxiproxyFixture toxiproxy = ToxiproxyFixture.start(network)) {
// given
Proxy proxy = toxiproxy.proxyTo("slow-drip", "upstream", 80);
toxiproxy.slowDrip(proxy, 16, Duration.ofMillis(50));
toxiproxy.slowDrip(proxy, SLOW_DRIP_SLICE_BYTES, SLOW_DRIP_GAP);
TimeoutSettings base = ClientProfiles.timeouts();
URI baseUrl =
URI.create("http://" + toxiproxy.proxiedHost() + ":" + toxiproxy.proxiedPort() + "/");
long started;
Duration elapsed;
Throwable failure;
// when
@@ -151,7 +166,7 @@ class FailureInjectionContractTest {
base.readIdle(),
SLOW_DRIP_BUDGET))
.build())) {
started = System.nanoTime();
long started = System.nanoTime();
failure =
catchThrowable(
() ->
@@ -160,14 +175,22 @@ class FailureInjectionContractTest {
.exchange(
harness.profile().name(),
HttpOperation.get(
new OperationName("fault-probe"), "/bytes/4096", Map.of()),
new OperationName("fault-probe"),
"/bytes/" + SLOW_DRIP_BYTES,
Map.of()),
ResponseType.of(byte[].class)));
// The call alone: closing the harness afterwards is not part of the deadline.
elapsed = Duration.ofNanos(System.nanoTime() - started);
}
// then
Duration elapsed = Duration.ofNanos(System.nanoTime() - started);
assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class);
assertThat(elapsed).isLessThan(SLOW_DRIP_BUDGET.plus(Duration.ofMillis(500)));
assertThat(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK))
.as("the bound must stay well under the drip, or it proves nothing")
.isLessThan(SLOW_DRIP_TOTAL.dividedBy(2));
assertThat(elapsed)
.as("the deadline, not the drip, ended the call")
.isLessThan(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK));
}
}
}
@@ -29,15 +29,17 @@ class AttemptDeadlineTest {
@Test
@DisplayName("남은 deadline이 단계 timeout보다 짧으면 단계 timeout을 남은 시간으로 줄인다")
void aShorterRemainingDeadlineBoundsTheStageTimeout() {
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(300))) {
// Seconds rather than milliseconds, so the deadline has not already passed by the time bound()
// runs on a loaded machine; the stage is longer still, so bound() must cut it.
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(5))) {
// given
Duration stage = Duration.ofSeconds(2);
Duration stage = Duration.ofSeconds(30);
// when
Duration bounded = AttemptDeadline.bound(stage);
// then
assertThat(bounded).isLessThanOrEqualTo(Duration.ofMillis(300)).isPositive();
assertThat(bounded).isLessThanOrEqualTo(Duration.ofSeconds(5)).isPositive();
assertThat(scope.expired()).isFalse();
}
}
@@ -89,14 +91,16 @@ class AttemptDeadlineTest {
void closingTheScopeBeforeTheDeadlineCancelsTheAbort() throws Exception {
// given
AtomicInteger aborts = new AtomicInteger();
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(100));
// One second, so registering and closing always happen before it on a loaded machine; the
// wait after the close outlasts it, so an abort that was not cancelled would have run.
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(1));
AttemptDeadline.abortOnExpiry(aborts::incrementAndGet);
// when
scope.close();
// then
Thread.sleep(300);
Thread.sleep(1_500);
assertThat(aborts).hasValue(0);
assertThat(AttemptDeadline.remaining()).isEmpty();
}
@@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName;
import dev.caskeleton.adapter.outbound.persistence.api.error.PessimisticLockTimeoutException;
import dev.caskeleton.adapter.outbound.persistence.postgresql.lock.PostgreSqlLockExceptionTranslator;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
@@ -59,14 +60,23 @@ class PostgreSqlPessimisticLockContractTest {
lockRow(holder);
contender.setAutoCommit(false);
try (Statement statement = contender.createStatement();
ResultSet setting = statement.executeQuery("show lock_timeout")) {
assertThat(setting.next()).isTrue();
assertThat(setting.getString(1))
.as("without a lock_timeout a lock that waited would wait for the holder for good")
.isEqualTo("0");
}
Instant startedAt = Instant.now();
SQLException refused = lockRowNowait(contender);
Duration waited = Duration.between(startedAt, Instant.now());
assertThat(refused.getSQLState()).isEqualTo("55P03");
// The contender has no lock_timeout, so a lock that waited would wait for the holder, who
// never lets go here. Seconds of slack cannot be mistaken for that.
assertThat(waited)
.as("NOWAIT must not wait for the holder")
.isLessThan(Duration.ofSeconds(1));
.isLessThan(Duration.ofSeconds(10));
var translated = translator.translate(refused, OPERATION, 1, waited, null).orElseThrow();
assertThat(translated).isInstanceOf(PessimisticLockTimeoutException.class);
@@ -12,6 +12,7 @@ import java.sql.Statement;
import java.time.Duration;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -49,13 +50,17 @@ class PostgreSqlLifecycleIntegrationTest {
}
@Test
@Timeout(10)
// The method starts its own PostgreSQL container; on a loaded machine that alone can take tens of
// seconds. The bound that matters, the exhausted pool's wait, is measured on its own below.
@Timeout(value = 2, unit = TimeUnit.MINUTES)
void poolCapacityExhaustionAndShutdownAreBoundedAndObservable() throws Exception {
PostgreSqlReadinessSupport bounded = PostgreSqlReadinessSupport.start(2, 300);
HikariDataSource dataSource = bounded.dataSource();
try {
try (Connection first = bounded.connection();
Connection second = bounded.connection()) {
// Borrowing a connection the pool still has to open (TCP, SCRAM, backend fork) must not be
// held to the 300 ms timeout this test measures, so the two are taken patiently.
try (Connection first = patiently(bounded);
Connection second = patiently(bounded)) {
PoolSnapshot saturated = snapshot(dataSource);
assertThat(saturated.state()).isEqualTo(PoolState.SATURATED);
assertThat(saturated.activeConnections()).isEqualTo(2);
@@ -66,8 +71,10 @@ class PostgreSqlLifecycleIntegrationTest {
long started = System.nanoTime();
assertThatThrownBy(() -> bounded.connection().close()).isInstanceOf(SQLException.class);
// At least the 300 ms connectionTimeout, and nowhere near an unbounded wait: the slack
// above it is for a loaded machine, not for the pool.
assertThat(Duration.ofNanos(System.nanoTime() - started))
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(2));
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(5));
}
assertThat(snapshot(dataSource).activeConnections()).isZero();
@@ -81,6 +88,20 @@ class PostgreSqlLifecycleIntegrationTest {
assertThat(closed.activeConnections()).isZero();
}
/** A pool connection, retrying while a new physical connection takes longer than the timeout. */
private static Connection patiently(PostgreSqlReadinessSupport support) throws SQLException {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
while (true) {
try {
return support.connection();
} catch (SQLException slow) {
if (System.nanoTime() > until) {
throw slow;
}
}
}
}
private static PoolSnapshot snapshot(HikariDataSource dataSource) {
int maximum = dataSource.getMaximumPoolSize();
if (dataSource.isClosed()) {
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.readiness;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.catchThrowable;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
@@ -168,9 +169,13 @@ class PostgreSqlMigrationIntegrationTest {
.initSql("set lock_timeout = '250ms'; set statement_timeout = '2s'")
.load();
long started = System.nanoTime();
assertThatThrownBy(blockedExpansion::migrate).isInstanceOf(FlywayException.class);
assertThat(Duration.ofNanos(System.nanoTime() - started))
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3));
Throwable refused = catchThrowable(blockedExpansion::migrate);
Duration waited = Duration.ofNanos(System.nanoTime() - started);
// The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
// says which, so the clock only has to show the migration did not wait for the holder.
assertThat(refused).isInstanceOf(FlywayException.class);
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
blocker.rollback();
}
@@ -1,7 +1,7 @@
package dev.caskeleton.adapter.outbound.persistence.readiness;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.catchThrowable;
import java.sql.Connection;
import java.sql.Statement;
@@ -175,9 +175,13 @@ final class PostgreSqlOptionalStreamLifecycle {
blocker.setAutoCommit(false);
lock.execute("lock table capability_schema_registry in access exclusive mode");
long started = System.nanoTime();
assertThatThrownBy(flyway::migrate).isInstanceOf(FlywayException.class);
assertThat(Duration.ofNanos(System.nanoTime() - started))
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3));
Throwable refused = catchThrowable(flyway::migrate);
Duration waited = Duration.ofNanos(System.nanoTime() - started);
// The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
// says which, so the clock only has to show the migration did not wait for the holder.
assertThat(refused).isInstanceOf(FlywayException.class);
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
blocker.rollback();
}
@@ -28,6 +28,20 @@ final class PostgreSqlReadinessSupport implements AutoCloseable {
this.dataSource = dataSource;
}
/** Every SQLState in {@code failure}'s cause chain, outermost first. */
static java.util.List<String> sqlStates(Throwable failure) {
java.util.List<String> states = new java.util.ArrayList<>();
for (Throwable cause = failure; cause != null; cause = cause.getCause()) {
if (cause instanceof java.sql.SQLException sql && sql.getSQLState() != null) {
states.add(sql.getSQLState());
}
if (cause.getCause() == cause) {
break;
}
}
return states;
}
static PostgreSqlReadinessSupport start() {
return start(5, 1_000);
}