Merge pull request 'test: 부하 아래에서 흔들리던 벽시계 시험을 원인 단언과 gate로 고친다' (#34) from claude/r9-wallclock-tests into main
main / main (push) Failing after 24m51s
main / build-logic (push) Successful in 2m6s
main / links (push) Successful in 11s
main / configuration-cache (push) Successful in 1m42s
main / vulnerabilities (push) Successful in 1m6s
main / http3-experimental (push) Successful in 1m21s
main / jpa-postgresql (16) (push) Skipped
main / jpa-postgresql (17) (push) Skipped
main / jpa-postgresql (18) (push) Skipped
main / httpclient-transport (push) Skipped
main / redis-topology (push) Skipped
main / fileserver-nfs-ambiguity (push) Skipped
main / notification-chaos (push) Skipped
main / jpa-next (hibernate-8, Hibernate 8 is not resolvable from this build, so nothing has been compiled or run against it, Hibernate 8, *Hibernate8CompatibilityTest) (push) Skipped
main / jpa-next (jakarta-persistence-4, the JPA 4 API is not on any configuration this build resolves, so nothing has been compiled against it, Jakarta Persistence 4, *Jpa4CompatibilityTest) (push) Skipped
main / jpa-next (postgresql-19, no PostgreSQL 19 image is published yet, so no container of that major has ever been started by this lane, PostgreSQL 19, *PostgreSql19CompatibilityTest) (push) Skipped
main / object-storage-aws (push) Skipped
main / compose-smoke (push) Skipped
main / done (push) Skipped
main / performance (push) Skipped

This commit was merged in pull request #34.
This commit is contained in:
2026-11-13 07:58:19 +00:00
13 changed files with 255 additions and 49 deletions
@@ -43,8 +43,10 @@ import org.springframework.http.ResponseEntity;
@AutoConfigureTestRestTemplate @AutoConfigureTestRestTemplate
class GraphQlRequestDeadlineTest { class GraphQlRequestDeadlineTest {
private static final Duration BUDGET = Duration.ofSeconds(1); // Two seconds so the resolver has started before the budget runs out even on a loaded machine
private static final int SLOW_RESOLVER_MILLIS = 4_000; // (SLOW_STARTED below); the resolver itself holds until the test releases it.
private static final Duration BUDGET = Duration.ofSeconds(2);
private static final int SLOW_RESOLVER_MILLIS = 30_000;
@LocalServerPort int port; @LocalServerPort int port;
@@ -70,6 +72,7 @@ class GraphQlRequestDeadlineTest {
// then // then
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started); long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
SlowWork.release.countDown();
assertThat(response.getBody()) assertThat(response.getBody())
.contains("\"code\":\"REQUEST_TIMEOUT\"") .contains("\"code\":\"REQUEST_TIMEOUT\"")
.contains("\"category\":\"TIMEOUT\"") .contains("\"category\":\"TIMEOUT\"")
@@ -78,9 +81,9 @@ class GraphQlRequestDeadlineTest {
.as("the budget must have run out during execution, not before it") .as("the budget must have run out during execution, not before it")
.hasValue(1); .hasValue(1);
assertThat(elapsedMillis) assertThat(elapsedMillis)
.as("answered at the deadline, not when the %d ms resolver returned", SLOW_RESOLVER_MILLIS) .as("answered at the 2 s deadline, well before the 10 s default budget would have ended it")
.isGreaterThanOrEqualTo(BUDGET.toMillis()) .isGreaterThanOrEqualTo(BUDGET.toMillis())
.isLessThan(SLOW_RESOLVER_MILLIS - 1_000L); .isLessThan(BUDGET.plusSeconds(6).toMillis());
} }
@Test @Test
@@ -91,8 +94,8 @@ class GraphQlRequestDeadlineTest {
http, port, "{ slowNode(millis: " + SLOW_RESOLVER_MILLIS + ") { id detail } }"); http, port, "{ slowNode(millis: " + SLOW_RESOLVER_MILLIS + ") { id detail } }");
// when // when
boolean finished = SlowWork.release.countDown();
SlowWork.slowFinished.await(SLOW_RESOLVER_MILLIS * 2L, TimeUnit.MILLISECONDS); boolean finished = SlowWork.slowFinished.await(10, TimeUnit.SECONDS);
// then // then
assertThat(finished) assertThat(finished)
@@ -12,6 +12,7 @@ import graphql.execution.instrumentation.parameters.InstrumentationExecutionPara
import java.net.URI; import java.net.URI;
import java.time.Duration; import java.time.Duration;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.springframework.boot.graphql.autoconfigure.GraphQlSourceBuilderCustomizer; import org.springframework.boot.graphql.autoconfigure.GraphQlSourceBuilderCustomizer;
import org.springframework.boot.resttestclient.TestRestTemplate; import org.springframework.boot.resttestclient.TestRestTemplate;
@@ -92,6 +93,13 @@ final class RuntimeLimitsTestSupport {
static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger(); static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger();
static volatile CountDownLatch slowFinished = new CountDownLatch(1); static volatile CountDownLatch slowFinished = new CountDownLatch(1);
/**
* Opened by a test to let the slow resolver return before its {@code millis} cap. Until then
* the resolver holds as a slow dependency would, so "answered before the resolver returned"
* does not depend on how fast a loaded machine runs the rest of the request.
*/
static volatile CountDownLatch release = new CountDownLatch(1);
/** /**
* Counted down when a {@code slowNode} execution begun after the last {@link #reset()} has * Counted down when a {@code slowNode} execution begun after the last {@link #reset()} has
* ended. The point after which a child field can no longer start, so an absence asserted after * ended. The point after which a child field can no longer start, so an absence asserted after
@@ -108,6 +116,7 @@ final class RuntimeLimitsTestSupport {
DETAIL_INVOCATIONS.set(0); DETAIL_INVOCATIONS.set(0);
slowFinished = new CountDownLatch(1); slowFinished = new CountDownLatch(1);
slowExecutionEnded = new CountDownLatch(1); slowExecutionEnded = new CountDownLatch(1);
release = new CountDownLatch(1);
GENERATION.incrementAndGet(); GENERATION.incrementAndGet();
} }
@@ -141,8 +150,10 @@ final class RuntimeLimitsTestSupport {
@QueryMapping @QueryMapping
SlowNode slowNode(@Argument int millis) throws InterruptedException { SlowNode slowNode(@Argument int millis) throws InterruptedException {
SlowWork.SLOW_STARTED.incrementAndGet(); SlowWork.SLOW_STARTED.incrementAndGet();
CountDownLatch release = SlowWork.release;
try { try {
Thread.sleep(millis); // Released by the test, or the cap ran out; either way the resolver returns normally.
var unused = release.await(millis, TimeUnit.MILLISECONDS);
} finally { } finally {
SlowWork.slowFinished.countDown(); SlowWork.slowFinished.countDown();
} }
@@ -73,6 +73,7 @@ class GrpcP1BoundaryWireTest {
private static final MethodDescriptor<String, String> CLOSE_HANDLER_METHOD = private static final MethodDescriptor<String, String> CLOSE_HANDLER_METHOD =
unaryMethod("CloseHandler"); unaryMethod("CloseHandler");
private static final MethodDescriptor<String, String> SLOW_METHOD = unaryMethod("Slow"); private static final MethodDescriptor<String, String> SLOW_METHOD = unaryMethod("Slow");
private static final MethodDescriptor<String, String> GATED_METHOD = unaryMethod("Gated");
private static final CountDownLatch CLOSE_HANDLER_RAN = new CountDownLatch(1); private static final CountDownLatch CLOSE_HANDLER_RAN = new CountDownLatch(1);
/** The platform's GrpcTransportLimits.DEFAULT_MAX_MESSAGE_BYTES, which this adapter mirrors. */ /** The platform's GrpcTransportLimits.DEFAULT_MAX_MESSAGE_BYTES, which this adapter mirrors. */
@@ -81,6 +82,17 @@ class GrpcP1BoundaryWireTest {
/** How long the slow handler holds its thread unless its gRPC Context is cancelled first. */ /** How long the slow handler holds its thread unless its gRPC Context is cancelled first. */
private static final long SLOW_HANDLER_HOLD_MILLIS = 400; private static final long SLOW_HANDLER_HOLD_MILLIS = 400;
/**
* Holds every gated handler until a test releases it or the call's Context is cancelled. A gate,
* not a clock: what a test means by "while the call is still running" must not depend on how fast
* a loaded machine dispatches it (review 2026-11-13: the main stage runs beside other builds).
* Each test that uses it sets a fresh latch and releases it before it ends.
*/
private static volatile CountDownLatch gate = new CountDownLatch(0);
/** The longest a gated handler waits, so a test that forgets to release cannot hang the JVM. */
private static final long GATE_CAP_SECONDS = 30;
private static final AtomicInteger SLOW_IN_FLIGHT = new AtomicInteger(); private static final AtomicInteger SLOW_IN_FLIGHT = new AtomicInteger();
private static final AtomicInteger SLOW_MAX_IN_FLIGHT = new AtomicInteger(); private static final AtomicInteger SLOW_MAX_IN_FLIGHT = new AtomicInteger();
@@ -264,17 +276,30 @@ class GrpcP1BoundaryWireTest {
contextRunner.withPropertyValues("app.grpc.server.default-deadline=150ms"), contextRunner.withPropertyValues("app.grpc.server.default-deadline=150ms"),
channel -> { channel -> {
// given // given
// The handler holds until its Context is cancelled and then answers nothing, so the
// only way the call can end is the server deadline closing it. waitForReady keeps a
// slow first connection from failing the call before it reaches the server.
gate = new CountDownLatch(1);
long started = System.nanoTime();
ListenableFuture<String> withoutDeadline = ListenableFuture<String> withoutDeadline =
ClientCalls.futureUnaryCall( ClientCalls.futureUnaryCall(
authenticated(channel).newCall(SLOW_METHOD, CallOptions.DEFAULT), "slow"); authenticated(channel)
.newCall(GATED_METHOD, CallOptions.DEFAULT.withWaitForReady()),
"gated");
// when // when
Throwable failure = catchThrowable(() -> withoutDeadline.get(5, TimeUnit.SECONDS)); Throwable failure =
catchThrowable(() -> withoutDeadline.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
// then // then
gate.countDown();
assertThat(failure).as("the call must fail, not answer").isNotNull();
assertThat(Status.fromThrowable(failure).getCode()) assertThat(Status.fromThrowable(failure).getCode())
.as("status of %s", failure) .as("status of %s", failure)
.isEqualTo(Status.Code.DEADLINE_EXCEEDED); .isEqualTo(Status.Code.DEADLINE_EXCEEDED);
assertThat(Duration.ofNanos(System.nanoTime() - started))
.as("the configured 150 ms ended it, not the 10 s default it would fall back to")
.isLessThan(Duration.ofSeconds(5));
}); });
} }
@@ -285,6 +310,9 @@ class GrpcP1BoundaryWireTest {
contextRunner.withPropertyValues("app.grpc.server.default-deadline=10s"), contextRunner.withPropertyValues("app.grpc.server.default-deadline=10s"),
channel -> { channel -> {
// given // given
// A gated handler never answers before the gate opens, so the client deadline is the
// only thing that can end the call: the 10 s server default must not have replaced it.
gate = new CountDownLatch(1);
long started = System.nanoTime(); long started = System.nanoTime();
// when // when
@@ -293,14 +321,18 @@ class GrpcP1BoundaryWireTest {
() -> () ->
ClientCalls.blockingUnaryCall( ClientCalls.blockingUnaryCall(
authenticated(channel), authenticated(channel),
SLOW_METHOD, GATED_METHOD,
CallOptions.DEFAULT.withDeadlineAfter(100, TimeUnit.MILLISECONDS), CallOptions.DEFAULT.withDeadlineAfter(100, TimeUnit.MILLISECONDS),
"slow")); "gated"));
// then // then
gate.countDown();
assertThat(failure).as("the call must fail, not answer").isNotNull();
assertThat(Status.fromThrowable(failure).getCode()) assertThat(Status.fromThrowable(failure).getCode())
.as("status of %s", failure)
.isEqualTo(Status.Code.DEADLINE_EXCEEDED); .isEqualTo(Status.Code.DEADLINE_EXCEEDED);
assertThat(Duration.ofNanos(System.nanoTime() - started)) assertThat(Duration.ofNanos(System.nanoTime() - started))
.as("ended by the 100 ms client deadline")
.isLessThan(Duration.ofSeconds(5)); .isLessThan(Duration.ofSeconds(5));
}); });
} }
@@ -338,18 +370,25 @@ class GrpcP1BoundaryWireTest {
"app.grpc.server.executor-queue-capacity=1"), "app.grpc.server.executor-queue-capacity=1"),
channel -> { channel -> {
// given // given
// Gated handlers hold the one thread and the one queue slot until the refusal is
// seen, so the third call meets a full server however slowly the calls are dispatched.
gate = new CountDownLatch(1);
Channel caller = authenticated(channel); Channel caller = authenticated(channel);
CallOptions bounded = CallOptions.DEFAULT.withDeadlineAfter(5, TimeUnit.SECONDS); CallOptions bounded =
CallOptions.DEFAULT.withDeadlineAfter(GATE_CAP_SECONDS, TimeUnit.SECONDS);
List<ListenableFuture<String>> calls = List<ListenableFuture<String>> calls =
List.of( List.of(
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"), ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"), ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow")); ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"));
// when // when
boolean refusedWhileHeld = awaitAnyDone(calls);
gate.countDown();
List<Status> outcomes = calls.stream().map(GrpcP1BoundaryWireTest::outcome).toList(); List<Status> outcomes = calls.stream().map(GrpcP1BoundaryWireTest::outcome).toList();
// then // then
assertThat(refusedWhileHeld).as("one call is refused while the others hold").isTrue();
assertThat(outcomes) assertThat(outcomes)
.extracting(Status::getCode) .extracting(Status::getCode)
.containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE); .containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE);
@@ -361,10 +400,27 @@ class GrpcP1BoundaryWireTest {
} }
private static Status outcome(ListenableFuture<String> call) { private static Status outcome(ListenableFuture<String> call) {
Throwable failure = catchThrowable(() -> call.get(5, TimeUnit.SECONDS)); Throwable failure = catchThrowable(() -> call.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
return failure == null ? Status.OK : Status.fromThrowable(failure); return failure == null ? Status.OK : Status.fromThrowable(failure);
} }
/** Whether one of {@code calls} completes while the gate still holds the others. */
private static boolean awaitAnyDone(List<ListenableFuture<String>> calls) {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
while (System.nanoTime() < until) {
if (calls.stream().anyMatch(ListenableFuture::isDone)) {
return true;
}
try {
Thread.sleep(10);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
}
return false;
}
private static String getUnchecked(ListenableFuture<String> future) { private static String getUnchecked(ListenableFuture<String> future) {
try { try {
return future.get(5, TimeUnit.SECONDS); return future.get(5, TimeUnit.SECONDS);
@@ -550,6 +606,17 @@ class GrpcP1BoundaryWireTest {
observer.onNext(request + "-done"); observer.onNext(request + "-done");
observer.onCompleted(); observer.onCompleted();
})) }))
.addMethod(
GATED_METHOD,
ServerCalls.asyncUnaryCall(
(String request, StreamObserver<String> observer) -> {
if (holdUntilReleased()) {
observer.onNext(request + "-done");
observer.onCompleted();
}
// Cancelled: a cooperative handler answers nothing; whoever cancelled
// the call (the server deadline) has already closed it.
}))
.addMethod( .addMethod(
CLOSE_HANDLER_METHOD, CLOSE_HANDLER_METHOD,
ServerCalls.asyncUnaryCall( ServerCalls.asyncUnaryCall(
@@ -601,6 +668,29 @@ class GrpcP1BoundaryWireTest {
} }
} }
/**
* Holds the handler thread until the test opens {@link #gate}.
*
* @return true once released; false when the call's Context was cancelled first (or the cap ran
* out)
*/
private static boolean holdUntilReleased() {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
try {
while (System.nanoTime() < until) {
if (Context.current().isCancelled()) {
return false;
}
if (gate.await(10, TimeUnit.MILLISECONDS)) {
return true;
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return false;
}
private enum StringMarshaller implements MethodDescriptor.Marshaller<String> { private enum StringMarshaller implements MethodDescriptor.Marshaller<String> {
INSTANCE; INSTANCE;
@@ -98,7 +98,9 @@ class GrpcServerRunnerBootTest {
void stoppingKeepsTheListenerUpReportingNotServingDuringTheDrainDelay() { void stoppingKeepsTheListenerUpReportingNotServingDuringTheDrainDelay() {
contextRunner contextRunner
.withPropertyValues( .withPropertyValues(
"app.grpc.server.shutdown-drain-seconds=2", "app.grpc.server.shutdown-grace-seconds=4") // The drain is long enough that a loaded machine still sees it: the probe window below
// must close before the listener does, and both are seconds, not a scheduling slice.
"app.grpc.server.shutdown-drain-seconds=6", "app.grpc.server.shutdown-grace-seconds=8")
.run( .run(
context -> { context -> {
// given // given
@@ -107,6 +109,15 @@ class GrpcServerRunnerBootTest {
ManagedChannelBuilder.forAddress("localhost", runner.getListeningPort()) ManagedChannelBuilder.forAddress("localhost", runner.getListeningPort())
.usePlaintext() .usePlaintext()
.build(); .build();
// Connect before shutdown begins, so the drain window is not spent on the first
// connection.
assertThat(
HealthGrpc.newBlockingStub(channel)
.withWaitForReady()
.withDeadlineAfter(10, TimeUnit.SECONDS)
.check(HealthCheckRequest.newBuilder().build())
.getStatus())
.isEqualTo(ServingStatus.SERVING);
Thread stopping = Thread.ofVirtual().start(runner::stop); Thread stopping = Thread.ofVirtual().start(runner::stop);
try { try {
// when // when
@@ -115,7 +126,7 @@ class GrpcServerRunnerBootTest {
// then // then
assertThat(duringDrain).isEqualTo(ServingStatus.NOT_SERVING); assertThat(duringDrain).isEqualTo(ServingStatus.NOT_SERVING);
} finally { } finally {
stopping.join(Duration.ofSeconds(10)); stopping.join(Duration.ofSeconds(20));
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
} }
}); });
@@ -127,13 +138,9 @@ class GrpcServerRunnerBootTest {
*/ */
private static ServingStatus firstStatusOtherThanServing(ManagedChannel channel) private static ServingStatus firstStatusOtherThanServing(ManagedChannel channel)
throws InterruptedException { throws InterruptedException {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(4);
while (System.nanoTime() < until) { while (System.nanoTime() < until) {
ServingStatus status = ServingStatus status = health(channel);
HealthGrpc.newBlockingStub(channel)
.withDeadlineAfter(1, TimeUnit.SECONDS)
.check(HealthCheckRequest.newBuilder().build())
.getStatus();
if (status != ServingStatus.SERVING) { if (status != ServingStatus.SERVING) {
return status; return status;
} }
@@ -142,6 +149,14 @@ class GrpcServerRunnerBootTest {
return ServingStatus.SERVING; return ServingStatus.SERVING;
} }
// Two seconds per probe: the four-second window plus one probe still ends inside the 6 s drain.
private static ServingStatus health(ManagedChannel channel) {
return HealthGrpc.newBlockingStub(channel)
.withDeadlineAfter(2, TimeUnit.SECONDS)
.check(HealthCheckRequest.newBuilder().build())
.getStatus();
}
@Configuration(proxyBeanMethods = false) @Configuration(proxyBeanMethods = false)
static class UnqualifiedServiceConfiguration { static class UnqualifiedServiceConfiguration {
@@ -217,7 +217,9 @@ class LiveRedisSemanticPortsTest {
"api-default", "api-default",
"v1", "v1",
RateLimitAlgorithm.FIXED_WINDOW, RateLimitAlgorithm.FIXED_WINDOW,
new RateParameters.FixedWindow(3, Duration.ofSeconds(10)), // An hour-long window: the four calls below must land in one window, and a 10 s window
// aligned to Redis TIME let a slow run straddle its boundary.
new RateParameters.FixedWindow(3, Duration.ofHours(1)),
1, 1,
Duration.ofSeconds(5), Duration.ofSeconds(5),
Duration.ofMillis(250), Duration.ofMillis(250),
@@ -48,9 +48,13 @@ class LiveRedisDistributedLeaseContractTest extends DistributedLeaseContract {
Duration.ZERO); Duration.ZERO);
} }
// A live server waits for real, so the TTL is seconds rather than the fake's 30: long enough that
// the calls a test makes while it holds a lease never outlast it on a loaded machine. Each of the
// four displaced-holder cases sleeps 1.5 TTL, so this costs about 12 s per run (it was about 2 s
// with a 400 ms TTL that a slow script call could outlast).
@Override @Override
protected Duration leaseTtl() { protected Duration leaseTtl() {
return Duration.ofMillis(400); return Duration.ofSeconds(2);
} }
@Override @Override
@@ -43,6 +43,21 @@ import org.testcontainers.utility.DockerImageName;
class FailureInjectionContractTest { class FailureInjectionContractTest {
private static final Duration SLOW_DRIP_BUDGET = Duration.ofSeconds(1); private static final Duration SLOW_DRIP_BUDGET = Duration.ofSeconds(1);
private static final int SLOW_DRIP_BYTES = 4096;
private static final int SLOW_DRIP_SLICE_BYTES = 16;
private static final Duration SLOW_DRIP_GAP = Duration.ofMillis(50);
/** How long the drip alone takes to deliver the body: 256 slices, 50 ms apart (12.8 s). */
private static final Duration SLOW_DRIP_TOTAL =
SLOW_DRIP_GAP.multipliedBy(SLOW_DRIP_BYTES / SLOW_DRIP_SLICE_BYTES);
/**
* Time a loaded machine may add between the deadline firing and the caller seeing the failure.
* The main stage runs beside other builds; the bound only has to tell "cut at the budget" from
* "waited for the drip", which the total leaves room for (budget + slack is well under half of
* it).
*/
private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(4);
private static final DockerImageName UPSTREAM_IMAGE = private static final DockerImageName UPSTREAM_IMAGE =
DockerImageName.parse( DockerImageName.parse(
@@ -129,11 +144,11 @@ class FailureInjectionContractTest {
try (ToxiproxyFixture toxiproxy = ToxiproxyFixture.start(network)) { try (ToxiproxyFixture toxiproxy = ToxiproxyFixture.start(network)) {
// given // given
Proxy proxy = toxiproxy.proxyTo("slow-drip", "upstream", 80); Proxy proxy = toxiproxy.proxyTo("slow-drip", "upstream", 80);
toxiproxy.slowDrip(proxy, 16, Duration.ofMillis(50)); toxiproxy.slowDrip(proxy, SLOW_DRIP_SLICE_BYTES, SLOW_DRIP_GAP);
TimeoutSettings base = ClientProfiles.timeouts(); TimeoutSettings base = ClientProfiles.timeouts();
URI baseUrl = URI baseUrl =
URI.create("http://" + toxiproxy.proxiedHost() + ":" + toxiproxy.proxiedPort() + "/"); URI.create("http://" + toxiproxy.proxiedHost() + ":" + toxiproxy.proxiedPort() + "/");
long started; Duration elapsed;
Throwable failure; Throwable failure;
// when // when
@@ -151,7 +166,7 @@ class FailureInjectionContractTest {
base.readIdle(), base.readIdle(),
SLOW_DRIP_BUDGET)) SLOW_DRIP_BUDGET))
.build())) { .build())) {
started = System.nanoTime(); long started = System.nanoTime();
failure = failure =
catchThrowable( catchThrowable(
() -> () ->
@@ -160,14 +175,22 @@ class FailureInjectionContractTest {
.exchange( .exchange(
harness.profile().name(), harness.profile().name(),
HttpOperation.get( HttpOperation.get(
new OperationName("fault-probe"), "/bytes/4096", Map.of()), new OperationName("fault-probe"),
"/bytes/" + SLOW_DRIP_BYTES,
Map.of()),
ResponseType.of(byte[].class))); ResponseType.of(byte[].class)));
// The call alone: closing the harness afterwards is not part of the deadline.
elapsed = Duration.ofNanos(System.nanoTime() - started);
} }
// then // then
Duration elapsed = Duration.ofNanos(System.nanoTime() - started);
assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class); assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class);
assertThat(elapsed).isLessThan(SLOW_DRIP_BUDGET.plus(Duration.ofMillis(500))); assertThat(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK))
.as("the bound must stay well under the drip, or it proves nothing")
.isLessThan(SLOW_DRIP_TOTAL.dividedBy(2));
assertThat(elapsed)
.as("the deadline, not the drip, ended the call")
.isLessThan(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK));
} }
} }
} }
@@ -29,15 +29,17 @@ class AttemptDeadlineTest {
@Test @Test
@DisplayName("남은 deadline이 단계 timeout보다 짧으면 단계 timeout을 남은 시간으로 줄인다") @DisplayName("남은 deadline이 단계 timeout보다 짧으면 단계 timeout을 남은 시간으로 줄인다")
void aShorterRemainingDeadlineBoundsTheStageTimeout() { void aShorterRemainingDeadlineBoundsTheStageTimeout() {
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(300))) { // Seconds rather than milliseconds, so the deadline has not already passed by the time bound()
// runs on a loaded machine; the stage is longer still, so bound() must cut it.
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(5))) {
// given // given
Duration stage = Duration.ofSeconds(2); Duration stage = Duration.ofSeconds(30);
// when // when
Duration bounded = AttemptDeadline.bound(stage); Duration bounded = AttemptDeadline.bound(stage);
// then // then
assertThat(bounded).isLessThanOrEqualTo(Duration.ofMillis(300)).isPositive(); assertThat(bounded).isLessThanOrEqualTo(Duration.ofSeconds(5)).isPositive();
assertThat(scope.expired()).isFalse(); assertThat(scope.expired()).isFalse();
} }
} }
@@ -89,14 +91,16 @@ class AttemptDeadlineTest {
void closingTheScopeBeforeTheDeadlineCancelsTheAbort() throws Exception { void closingTheScopeBeforeTheDeadlineCancelsTheAbort() throws Exception {
// given // given
AtomicInteger aborts = new AtomicInteger(); AtomicInteger aborts = new AtomicInteger();
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(100)); // One second, so registering and closing always happen before it on a loaded machine; the
// wait after the close outlasts it, so an abort that was not cancelled would have run.
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(1));
AttemptDeadline.abortOnExpiry(aborts::incrementAndGet); AttemptDeadline.abortOnExpiry(aborts::incrementAndGet);
// when // when
scope.close(); scope.close();
// then // then
Thread.sleep(300); Thread.sleep(1_500);
assertThat(aborts).hasValue(0); assertThat(aborts).hasValue(0);
assertThat(AttemptDeadline.remaining()).isEmpty(); assertThat(AttemptDeadline.remaining()).isEmpty();
} }
@@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName;
import dev.caskeleton.adapter.outbound.persistence.api.error.PessimisticLockTimeoutException; import dev.caskeleton.adapter.outbound.persistence.api.error.PessimisticLockTimeoutException;
import dev.caskeleton.adapter.outbound.persistence.postgresql.lock.PostgreSqlLockExceptionTranslator; import dev.caskeleton.adapter.outbound.persistence.postgresql.lock.PostgreSqlLockExceptionTranslator;
import java.sql.Connection; import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.time.Duration; import java.time.Duration;
@@ -59,14 +60,23 @@ class PostgreSqlPessimisticLockContractTest {
lockRow(holder); lockRow(holder);
contender.setAutoCommit(false); contender.setAutoCommit(false);
try (Statement statement = contender.createStatement();
ResultSet setting = statement.executeQuery("show lock_timeout")) {
assertThat(setting.next()).isTrue();
assertThat(setting.getString(1))
.as("without a lock_timeout a lock that waited would wait for the holder for good")
.isEqualTo("0");
}
Instant startedAt = Instant.now(); Instant startedAt = Instant.now();
SQLException refused = lockRowNowait(contender); SQLException refused = lockRowNowait(contender);
Duration waited = Duration.between(startedAt, Instant.now()); Duration waited = Duration.between(startedAt, Instant.now());
assertThat(refused.getSQLState()).isEqualTo("55P03"); assertThat(refused.getSQLState()).isEqualTo("55P03");
// The contender has no lock_timeout, so a lock that waited would wait for the holder, who
// never lets go here. Seconds of slack cannot be mistaken for that.
assertThat(waited) assertThat(waited)
.as("NOWAIT must not wait for the holder") .as("NOWAIT must not wait for the holder")
.isLessThan(Duration.ofSeconds(1)); .isLessThan(Duration.ofSeconds(10));
var translated = translator.translate(refused, OPERATION, 1, waited, null).orElseThrow(); var translated = translator.translate(refused, OPERATION, 1, waited, null).orElseThrow();
assertThat(translated).isInstanceOf(PessimisticLockTimeoutException.class); assertThat(translated).isInstanceOf(PessimisticLockTimeoutException.class);
@@ -12,6 +12,7 @@ import java.sql.Statement;
import java.time.Duration; import java.time.Duration;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -49,13 +50,17 @@ class PostgreSqlLifecycleIntegrationTest {
} }
@Test @Test
@Timeout(10) // The method starts its own PostgreSQL container; on a loaded machine that alone can take tens of
// seconds. The bound that matters, the exhausted pool's wait, is measured on its own below.
@Timeout(value = 2, unit = TimeUnit.MINUTES)
void poolCapacityExhaustionAndShutdownAreBoundedAndObservable() throws Exception { void poolCapacityExhaustionAndShutdownAreBoundedAndObservable() throws Exception {
PostgreSqlReadinessSupport bounded = PostgreSqlReadinessSupport.start(2, 300); PostgreSqlReadinessSupport bounded = PostgreSqlReadinessSupport.start(2, 300);
HikariDataSource dataSource = bounded.dataSource(); HikariDataSource dataSource = bounded.dataSource();
try { try {
try (Connection first = bounded.connection(); // Borrowing a connection the pool still has to open (TCP, SCRAM, backend fork) must not be
Connection second = bounded.connection()) { // held to the 300 ms timeout this test measures, so the two are taken patiently.
try (Connection first = patiently(bounded);
Connection second = patiently(bounded)) {
PoolSnapshot saturated = snapshot(dataSource); PoolSnapshot saturated = snapshot(dataSource);
assertThat(saturated.state()).isEqualTo(PoolState.SATURATED); assertThat(saturated.state()).isEqualTo(PoolState.SATURATED);
assertThat(saturated.activeConnections()).isEqualTo(2); assertThat(saturated.activeConnections()).isEqualTo(2);
@@ -66,8 +71,10 @@ class PostgreSqlLifecycleIntegrationTest {
long started = System.nanoTime(); long started = System.nanoTime();
assertThatThrownBy(() -> bounded.connection().close()).isInstanceOf(SQLException.class); assertThatThrownBy(() -> bounded.connection().close()).isInstanceOf(SQLException.class);
// At least the 300 ms connectionTimeout, and nowhere near an unbounded wait: the slack
// above it is for a loaded machine, not for the pool.
assertThat(Duration.ofNanos(System.nanoTime() - started)) assertThat(Duration.ofNanos(System.nanoTime() - started))
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(2)); .isBetween(Duration.ofMillis(250), Duration.ofSeconds(5));
} }
assertThat(snapshot(dataSource).activeConnections()).isZero(); assertThat(snapshot(dataSource).activeConnections()).isZero();
@@ -81,6 +88,20 @@ class PostgreSqlLifecycleIntegrationTest {
assertThat(closed.activeConnections()).isZero(); assertThat(closed.activeConnections()).isZero();
} }
/** A pool connection, retrying while a new physical connection takes longer than the timeout. */
private static Connection patiently(PostgreSqlReadinessSupport support) throws SQLException {
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
while (true) {
try {
return support.connection();
} catch (SQLException slow) {
if (System.nanoTime() > until) {
throw slow;
}
}
}
}
private static PoolSnapshot snapshot(HikariDataSource dataSource) { private static PoolSnapshot snapshot(HikariDataSource dataSource) {
int maximum = dataSource.getMaximumPoolSize(); int maximum = dataSource.getMaximumPoolSize();
if (dataSource.isClosed()) { if (dataSource.isClosed()) {
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.readiness;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.catchThrowable;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.sql.Connection; import java.sql.Connection;
@@ -168,9 +169,13 @@ class PostgreSqlMigrationIntegrationTest {
.initSql("set lock_timeout = '250ms'; set statement_timeout = '2s'") .initSql("set lock_timeout = '250ms'; set statement_timeout = '2s'")
.load(); .load();
long started = System.nanoTime(); long started = System.nanoTime();
assertThatThrownBy(blockedExpansion::migrate).isInstanceOf(FlywayException.class); Throwable refused = catchThrowable(blockedExpansion::migrate);
assertThat(Duration.ofNanos(System.nanoTime() - started)) Duration waited = Duration.ofNanos(System.nanoTime() - started);
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3)); // The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
// says which, so the clock only has to show the migration did not wait for the holder.
assertThat(refused).isInstanceOf(FlywayException.class);
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
blocker.rollback(); blocker.rollback();
} }
@@ -1,7 +1,7 @@
package dev.caskeleton.adapter.outbound.persistence.readiness; package dev.caskeleton.adapter.outbound.persistence.readiness;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable;
import java.sql.Connection; import java.sql.Connection;
import java.sql.Statement; import java.sql.Statement;
@@ -175,9 +175,13 @@ final class PostgreSqlOptionalStreamLifecycle {
blocker.setAutoCommit(false); blocker.setAutoCommit(false);
lock.execute("lock table capability_schema_registry in access exclusive mode"); lock.execute("lock table capability_schema_registry in access exclusive mode");
long started = System.nanoTime(); long started = System.nanoTime();
assertThatThrownBy(flyway::migrate).isInstanceOf(FlywayException.class); Throwable refused = catchThrowable(flyway::migrate);
assertThat(Duration.ofNanos(System.nanoTime() - started)) Duration waited = Duration.ofNanos(System.nanoTime() - started);
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3)); // The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
// says which, so the clock only has to show the migration did not wait for the holder.
assertThat(refused).isInstanceOf(FlywayException.class);
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
blocker.rollback(); blocker.rollback();
} }
@@ -28,6 +28,20 @@ final class PostgreSqlReadinessSupport implements AutoCloseable {
this.dataSource = dataSource; this.dataSource = dataSource;
} }
/** Every SQLState in {@code failure}'s cause chain, outermost first. */
static java.util.List<String> sqlStates(Throwable failure) {
java.util.List<String> states = new java.util.ArrayList<>();
for (Throwable cause = failure; cause != null; cause = cause.getCause()) {
if (cause instanceof java.sql.SQLException sql && sql.getSQLState() != null) {
states.add(sql.getSQLState());
}
if (cause.getCause() == cause) {
break;
}
}
return states;
}
static PostgreSqlReadinessSupport start() { static PostgreSqlReadinessSupport start() {
return start(5, 1_000); return start(5, 1_000);
} }