문제: main 단계 CI는 desktop heavy lock 밖에서 부하 14~25 아래 돈다. main push run 221은 gRPC GrpcP1BoundaryWireTest 서버 기본 deadline 시험이 다른 status를, run 228은 httpclient FailureInjectionContractTest slow drip이 1.92s(상한 1.5s)를 내 빨갰다. main 단계 lane을 전수로 보니 같은 부류가 더 있었다. 상한이 설정 예산에 바짝 붙었거나(예산+0.5s), 잰 구간에 준비 작업(컨테이너 기동, Flyway 준비, 첫 연결)이 들어가거나, 「아직 도는 동안」을 짧은 sleep·점유 시간으로 만든 시험들이다. 고친 것(시험만): - httpclient slow drip: 상한 예산+4s(drip 전체 12.8s의 절반 아래임을 함께 단언), 잰 구간에서 harness 닫기를 뺐다. AttemptDeadlineTest 둘은 deadline을 초 단위로. - gRPC: 서버·client deadline 시험과 포화(SERVER_BUSY) 시험이 시험이 여는 gate로 handler를 붙잡는다(취소되면 응답하지 않음). 서버 deadline 시험은 걸린 시간이 10s 기본값보다 확실히 짧음(5s 아래)을 함께 단언해 설정한 150ms가 걸렸음을 본다. waitForReady, 대기 상한 30s. drain 시험은 연결을 먼저 맺고 drain·폴링 창 6s·4s. - graphql deadline: sleep resolver를 gate로(상한 30s), 예산 2s, 응답 상한 예산+6s(기본 예산 10s보다 짧다). - persistence-jpa: lifecycle의 @Timeout이 컨테이너 기동을 덮던 것을 2분으로, 준비 연결은 재시도해 받고 풀 고갈 상한 5s. migration 둘은 원인을 SQLState 55P03 (lock_timeout)으로 단언하고 시간 상한 30s. NOWAIT는 contender의 lock_timeout이 0임을 먼저 단언하고 상한 10s. - cache-redis: live lease TTL 2s(밀려난 holder 케이스 넷이 1.5 TTL씩 잔다), fixed window 1시간. 버린 대안: 가짜 시계(deadline의 주인이 JDK HttpClient·Hikari·PostgreSQL·gRPC라 시험이 시계를 못 바꾼다), 상한 일률 배수(증명할 것과 가르지 못하는 값이 된다). 검증: desktop, CPU 부하(yes 24개, 12코어) 아래 바뀐 lane 전부 1084건, 리뷰 수정 뒤 다시 676건 실패 0. httpclient·gRPC lane 3회 반복 통과. verify.sh pr --no-cache 6805건 실패 0. RED: gRPC는 main·#25 이전·부하 유무 24회, httpclient는 main 부하 아래 4회 모두 통과해 desktop에서 재현되지 않았다. 고침은 CI 실패 둘과 코드 분석에 근거한다.
This commit is contained in:
+9
-6
@@ -43,8 +43,10 @@ import org.springframework.http.ResponseEntity;
|
||||
@AutoConfigureTestRestTemplate
|
||||
class GraphQlRequestDeadlineTest {
|
||||
|
||||
private static final Duration BUDGET = Duration.ofSeconds(1);
|
||||
private static final int SLOW_RESOLVER_MILLIS = 4_000;
|
||||
// Two seconds so the resolver has started before the budget runs out even on a loaded machine
|
||||
// (SLOW_STARTED below); the resolver itself holds until the test releases it.
|
||||
private static final Duration BUDGET = Duration.ofSeconds(2);
|
||||
private static final int SLOW_RESOLVER_MILLIS = 30_000;
|
||||
|
||||
@LocalServerPort int port;
|
||||
|
||||
@@ -70,6 +72,7 @@ class GraphQlRequestDeadlineTest {
|
||||
|
||||
// then
|
||||
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
|
||||
SlowWork.release.countDown();
|
||||
assertThat(response.getBody())
|
||||
.contains("\"code\":\"REQUEST_TIMEOUT\"")
|
||||
.contains("\"category\":\"TIMEOUT\"")
|
||||
@@ -78,9 +81,9 @@ class GraphQlRequestDeadlineTest {
|
||||
.as("the budget must have run out during execution, not before it")
|
||||
.hasValue(1);
|
||||
assertThat(elapsedMillis)
|
||||
.as("answered at the deadline, not when the %d ms resolver returned", SLOW_RESOLVER_MILLIS)
|
||||
.as("answered at the 2 s deadline, well before the 10 s default budget would have ended it")
|
||||
.isGreaterThanOrEqualTo(BUDGET.toMillis())
|
||||
.isLessThan(SLOW_RESOLVER_MILLIS - 1_000L);
|
||||
.isLessThan(BUDGET.plusSeconds(6).toMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -91,8 +94,8 @@ class GraphQlRequestDeadlineTest {
|
||||
http, port, "{ slowNode(millis: " + SLOW_RESOLVER_MILLIS + ") { id detail } }");
|
||||
|
||||
// when
|
||||
boolean finished =
|
||||
SlowWork.slowFinished.await(SLOW_RESOLVER_MILLIS * 2L, TimeUnit.MILLISECONDS);
|
||||
SlowWork.release.countDown();
|
||||
boolean finished = SlowWork.slowFinished.await(10, TimeUnit.SECONDS);
|
||||
|
||||
// then
|
||||
assertThat(finished)
|
||||
|
||||
+12
-1
@@ -12,6 +12,7 @@ import graphql.execution.instrumentation.parameters.InstrumentationExecutionPara
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.springframework.boot.graphql.autoconfigure.GraphQlSourceBuilderCustomizer;
|
||||
import org.springframework.boot.resttestclient.TestRestTemplate;
|
||||
@@ -92,6 +93,13 @@ final class RuntimeLimitsTestSupport {
|
||||
static final AtomicInteger DETAIL_INVOCATIONS = new AtomicInteger();
|
||||
static volatile CountDownLatch slowFinished = new CountDownLatch(1);
|
||||
|
||||
/**
|
||||
* Opened by a test to let the slow resolver return before its {@code millis} cap. Until then
|
||||
* the resolver holds as a slow dependency would, so "answered before the resolver returned"
|
||||
* does not depend on how fast a loaded machine runs the rest of the request.
|
||||
*/
|
||||
static volatile CountDownLatch release = new CountDownLatch(1);
|
||||
|
||||
/**
|
||||
* Counted down when a {@code slowNode} execution begun after the last {@link #reset()} has
|
||||
* ended. The point after which a child field can no longer start, so an absence asserted after
|
||||
@@ -108,6 +116,7 @@ final class RuntimeLimitsTestSupport {
|
||||
DETAIL_INVOCATIONS.set(0);
|
||||
slowFinished = new CountDownLatch(1);
|
||||
slowExecutionEnded = new CountDownLatch(1);
|
||||
release = new CountDownLatch(1);
|
||||
GENERATION.incrementAndGet();
|
||||
}
|
||||
|
||||
@@ -141,8 +150,10 @@ final class RuntimeLimitsTestSupport {
|
||||
@QueryMapping
|
||||
SlowNode slowNode(@Argument int millis) throws InterruptedException {
|
||||
SlowWork.SLOW_STARTED.incrementAndGet();
|
||||
CountDownLatch release = SlowWork.release;
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
// Released by the test, or the cap ran out; either way the resolver returns normally.
|
||||
var unused = release.await(millis, TimeUnit.MILLISECONDS);
|
||||
} finally {
|
||||
SlowWork.slowFinished.countDown();
|
||||
}
|
||||
|
||||
+99
-9
@@ -73,6 +73,7 @@ class GrpcP1BoundaryWireTest {
|
||||
private static final MethodDescriptor<String, String> CLOSE_HANDLER_METHOD =
|
||||
unaryMethod("CloseHandler");
|
||||
private static final MethodDescriptor<String, String> SLOW_METHOD = unaryMethod("Slow");
|
||||
private static final MethodDescriptor<String, String> GATED_METHOD = unaryMethod("Gated");
|
||||
private static final CountDownLatch CLOSE_HANDLER_RAN = new CountDownLatch(1);
|
||||
|
||||
/** The platform's GrpcTransportLimits.DEFAULT_MAX_MESSAGE_BYTES, which this adapter mirrors. */
|
||||
@@ -81,6 +82,17 @@ class GrpcP1BoundaryWireTest {
|
||||
/** How long the slow handler holds its thread unless its gRPC Context is cancelled first. */
|
||||
private static final long SLOW_HANDLER_HOLD_MILLIS = 400;
|
||||
|
||||
/**
|
||||
* Holds every gated handler until a test releases it or the call's Context is cancelled. A gate,
|
||||
* not a clock: what a test means by "while the call is still running" must not depend on how fast
|
||||
* a loaded machine dispatches it (review 2026-11-13: the main stage runs beside other builds).
|
||||
* Each test that uses it sets a fresh latch and releases it before it ends.
|
||||
*/
|
||||
private static volatile CountDownLatch gate = new CountDownLatch(0);
|
||||
|
||||
/** The longest a gated handler waits, so a test that forgets to release cannot hang the JVM. */
|
||||
private static final long GATE_CAP_SECONDS = 30;
|
||||
|
||||
private static final AtomicInteger SLOW_IN_FLIGHT = new AtomicInteger();
|
||||
private static final AtomicInteger SLOW_MAX_IN_FLIGHT = new AtomicInteger();
|
||||
|
||||
@@ -264,17 +276,30 @@ class GrpcP1BoundaryWireTest {
|
||||
contextRunner.withPropertyValues("app.grpc.server.default-deadline=150ms"),
|
||||
channel -> {
|
||||
// given
|
||||
// The handler holds until its Context is cancelled and then answers nothing, so the
|
||||
// only way the call can end is the server deadline closing it. waitForReady keeps a
|
||||
// slow first connection from failing the call before it reaches the server.
|
||||
gate = new CountDownLatch(1);
|
||||
long started = System.nanoTime();
|
||||
ListenableFuture<String> withoutDeadline =
|
||||
ClientCalls.futureUnaryCall(
|
||||
authenticated(channel).newCall(SLOW_METHOD, CallOptions.DEFAULT), "slow");
|
||||
authenticated(channel)
|
||||
.newCall(GATED_METHOD, CallOptions.DEFAULT.withWaitForReady()),
|
||||
"gated");
|
||||
|
||||
// when
|
||||
Throwable failure = catchThrowable(() -> withoutDeadline.get(5, TimeUnit.SECONDS));
|
||||
Throwable failure =
|
||||
catchThrowable(() -> withoutDeadline.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
|
||||
|
||||
// then
|
||||
gate.countDown();
|
||||
assertThat(failure).as("the call must fail, not answer").isNotNull();
|
||||
assertThat(Status.fromThrowable(failure).getCode())
|
||||
.as("status of %s", failure)
|
||||
.isEqualTo(Status.Code.DEADLINE_EXCEEDED);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.as("the configured 150 ms ended it, not the 10 s default it would fall back to")
|
||||
.isLessThan(Duration.ofSeconds(5));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -285,6 +310,9 @@ class GrpcP1BoundaryWireTest {
|
||||
contextRunner.withPropertyValues("app.grpc.server.default-deadline=10s"),
|
||||
channel -> {
|
||||
// given
|
||||
// A gated handler never answers before the gate opens, so the client deadline is the
|
||||
// only thing that can end the call: the 10 s server default must not have replaced it.
|
||||
gate = new CountDownLatch(1);
|
||||
long started = System.nanoTime();
|
||||
|
||||
// when
|
||||
@@ -293,14 +321,18 @@ class GrpcP1BoundaryWireTest {
|
||||
() ->
|
||||
ClientCalls.blockingUnaryCall(
|
||||
authenticated(channel),
|
||||
SLOW_METHOD,
|
||||
GATED_METHOD,
|
||||
CallOptions.DEFAULT.withDeadlineAfter(100, TimeUnit.MILLISECONDS),
|
||||
"slow"));
|
||||
"gated"));
|
||||
|
||||
// then
|
||||
gate.countDown();
|
||||
assertThat(failure).as("the call must fail, not answer").isNotNull();
|
||||
assertThat(Status.fromThrowable(failure).getCode())
|
||||
.as("status of %s", failure)
|
||||
.isEqualTo(Status.Code.DEADLINE_EXCEEDED);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.as("ended by the 100 ms client deadline")
|
||||
.isLessThan(Duration.ofSeconds(5));
|
||||
});
|
||||
}
|
||||
@@ -338,18 +370,25 @@ class GrpcP1BoundaryWireTest {
|
||||
"app.grpc.server.executor-queue-capacity=1"),
|
||||
channel -> {
|
||||
// given
|
||||
// Gated handlers hold the one thread and the one queue slot until the refusal is
|
||||
// seen, so the third call meets a full server however slowly the calls are dispatched.
|
||||
gate = new CountDownLatch(1);
|
||||
Channel caller = authenticated(channel);
|
||||
CallOptions bounded = CallOptions.DEFAULT.withDeadlineAfter(5, TimeUnit.SECONDS);
|
||||
CallOptions bounded =
|
||||
CallOptions.DEFAULT.withDeadlineAfter(GATE_CAP_SECONDS, TimeUnit.SECONDS);
|
||||
List<ListenableFuture<String>> calls =
|
||||
List.of(
|
||||
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(SLOW_METHOD, bounded), "slow"));
|
||||
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"),
|
||||
ClientCalls.futureUnaryCall(caller.newCall(GATED_METHOD, bounded), "gated"));
|
||||
|
||||
// when
|
||||
boolean refusedWhileHeld = awaitAnyDone(calls);
|
||||
gate.countDown();
|
||||
List<Status> outcomes = calls.stream().map(GrpcP1BoundaryWireTest::outcome).toList();
|
||||
|
||||
// then
|
||||
assertThat(refusedWhileHeld).as("one call is refused while the others hold").isTrue();
|
||||
assertThat(outcomes)
|
||||
.extracting(Status::getCode)
|
||||
.containsExactlyInAnyOrder(Status.Code.OK, Status.Code.OK, Status.Code.UNAVAILABLE);
|
||||
@@ -361,10 +400,27 @@ class GrpcP1BoundaryWireTest {
|
||||
}
|
||||
|
||||
private static Status outcome(ListenableFuture<String> call) {
|
||||
Throwable failure = catchThrowable(() -> call.get(5, TimeUnit.SECONDS));
|
||||
Throwable failure = catchThrowable(() -> call.get(GATE_CAP_SECONDS, TimeUnit.SECONDS));
|
||||
return failure == null ? Status.OK : Status.fromThrowable(failure);
|
||||
}
|
||||
|
||||
/** Whether one of {@code calls} completes while the gate still holds the others. */
|
||||
private static boolean awaitAnyDone(List<ListenableFuture<String>> calls) {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
|
||||
while (System.nanoTime() < until) {
|
||||
if (calls.stream().anyMatch(ListenableFuture::isDone)) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(10);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String getUnchecked(ListenableFuture<String> future) {
|
||||
try {
|
||||
return future.get(5, TimeUnit.SECONDS);
|
||||
@@ -550,6 +606,17 @@ class GrpcP1BoundaryWireTest {
|
||||
observer.onNext(request + "-done");
|
||||
observer.onCompleted();
|
||||
}))
|
||||
.addMethod(
|
||||
GATED_METHOD,
|
||||
ServerCalls.asyncUnaryCall(
|
||||
(String request, StreamObserver<String> observer) -> {
|
||||
if (holdUntilReleased()) {
|
||||
observer.onNext(request + "-done");
|
||||
observer.onCompleted();
|
||||
}
|
||||
// Cancelled: a cooperative handler answers nothing; whoever cancelled
|
||||
// the call (the server deadline) has already closed it.
|
||||
}))
|
||||
.addMethod(
|
||||
CLOSE_HANDLER_METHOD,
|
||||
ServerCalls.asyncUnaryCall(
|
||||
@@ -601,6 +668,29 @@ class GrpcP1BoundaryWireTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds the handler thread until the test opens {@link #gate}.
|
||||
*
|
||||
* @return true once released; false when the call's Context was cancelled first (or the cap ran
|
||||
* out)
|
||||
*/
|
||||
private static boolean holdUntilReleased() {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(GATE_CAP_SECONDS);
|
||||
try {
|
||||
while (System.nanoTime() < until) {
|
||||
if (Context.current().isCancelled()) {
|
||||
return false;
|
||||
}
|
||||
if (gate.await(10, TimeUnit.MILLISECONDS)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private enum StringMarshaller implements MethodDescriptor.Marshaller<String> {
|
||||
INSTANCE;
|
||||
|
||||
|
||||
+23
-8
@@ -98,7 +98,9 @@ class GrpcServerRunnerBootTest {
|
||||
void stoppingKeepsTheListenerUpReportingNotServingDuringTheDrainDelay() {
|
||||
contextRunner
|
||||
.withPropertyValues(
|
||||
"app.grpc.server.shutdown-drain-seconds=2", "app.grpc.server.shutdown-grace-seconds=4")
|
||||
// The drain is long enough that a loaded machine still sees it: the probe window below
|
||||
// must close before the listener does, and both are seconds, not a scheduling slice.
|
||||
"app.grpc.server.shutdown-drain-seconds=6", "app.grpc.server.shutdown-grace-seconds=8")
|
||||
.run(
|
||||
context -> {
|
||||
// given
|
||||
@@ -107,6 +109,15 @@ class GrpcServerRunnerBootTest {
|
||||
ManagedChannelBuilder.forAddress("localhost", runner.getListeningPort())
|
||||
.usePlaintext()
|
||||
.build();
|
||||
// Connect before shutdown begins, so the drain window is not spent on the first
|
||||
// connection.
|
||||
assertThat(
|
||||
HealthGrpc.newBlockingStub(channel)
|
||||
.withWaitForReady()
|
||||
.withDeadlineAfter(10, TimeUnit.SECONDS)
|
||||
.check(HealthCheckRequest.newBuilder().build())
|
||||
.getStatus())
|
||||
.isEqualTo(ServingStatus.SERVING);
|
||||
Thread stopping = Thread.ofVirtual().start(runner::stop);
|
||||
try {
|
||||
// when
|
||||
@@ -115,7 +126,7 @@ class GrpcServerRunnerBootTest {
|
||||
// then
|
||||
assertThat(duringDrain).isEqualTo(ServingStatus.NOT_SERVING);
|
||||
} finally {
|
||||
stopping.join(Duration.ofSeconds(10));
|
||||
stopping.join(Duration.ofSeconds(20));
|
||||
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
|
||||
}
|
||||
});
|
||||
@@ -127,13 +138,9 @@ class GrpcServerRunnerBootTest {
|
||||
*/
|
||||
private static ServingStatus firstStatusOtherThanServing(ManagedChannel channel)
|
||||
throws InterruptedException {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(4);
|
||||
while (System.nanoTime() < until) {
|
||||
ServingStatus status =
|
||||
HealthGrpc.newBlockingStub(channel)
|
||||
.withDeadlineAfter(1, TimeUnit.SECONDS)
|
||||
.check(HealthCheckRequest.newBuilder().build())
|
||||
.getStatus();
|
||||
ServingStatus status = health(channel);
|
||||
if (status != ServingStatus.SERVING) {
|
||||
return status;
|
||||
}
|
||||
@@ -142,6 +149,14 @@ class GrpcServerRunnerBootTest {
|
||||
return ServingStatus.SERVING;
|
||||
}
|
||||
|
||||
// Two seconds per probe: the four-second window plus one probe still ends inside the 6 s drain.
|
||||
private static ServingStatus health(ManagedChannel channel) {
|
||||
return HealthGrpc.newBlockingStub(channel)
|
||||
.withDeadlineAfter(2, TimeUnit.SECONDS)
|
||||
.check(HealthCheckRequest.newBuilder().build())
|
||||
.getStatus();
|
||||
}
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
static class UnqualifiedServiceConfiguration {
|
||||
|
||||
|
||||
+3
-1
@@ -217,7 +217,9 @@ class LiveRedisSemanticPortsTest {
|
||||
"api-default",
|
||||
"v1",
|
||||
RateLimitAlgorithm.FIXED_WINDOW,
|
||||
new RateParameters.FixedWindow(3, Duration.ofSeconds(10)),
|
||||
// An hour-long window: the four calls below must land in one window, and a 10 s window
|
||||
// aligned to Redis TIME let a slow run straddle its boundary.
|
||||
new RateParameters.FixedWindow(3, Duration.ofHours(1)),
|
||||
1,
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofMillis(250),
|
||||
|
||||
+5
-1
@@ -48,9 +48,13 @@ class LiveRedisDistributedLeaseContractTest extends DistributedLeaseContract {
|
||||
Duration.ZERO);
|
||||
}
|
||||
|
||||
// A live server waits for real, so the TTL is seconds rather than the fake's 30: long enough that
|
||||
// the calls a test makes while it holds a lease never outlast it on a loaded machine. Each of the
|
||||
// four displaced-holder cases sleeps 1.5 TTL, so this costs about 12 s per run (it was about 2 s
|
||||
// with a 400 ms TTL that a slow script call could outlast).
|
||||
@Override
|
||||
protected Duration leaseTtl() {
|
||||
return Duration.ofMillis(400);
|
||||
return Duration.ofSeconds(2);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+29
-6
@@ -43,6 +43,21 @@ import org.testcontainers.utility.DockerImageName;
|
||||
class FailureInjectionContractTest {
|
||||
|
||||
private static final Duration SLOW_DRIP_BUDGET = Duration.ofSeconds(1);
|
||||
private static final int SLOW_DRIP_BYTES = 4096;
|
||||
private static final int SLOW_DRIP_SLICE_BYTES = 16;
|
||||
private static final Duration SLOW_DRIP_GAP = Duration.ofMillis(50);
|
||||
|
||||
/** How long the drip alone takes to deliver the body: 256 slices, 50 ms apart (12.8 s). */
|
||||
private static final Duration SLOW_DRIP_TOTAL =
|
||||
SLOW_DRIP_GAP.multipliedBy(SLOW_DRIP_BYTES / SLOW_DRIP_SLICE_BYTES);
|
||||
|
||||
/**
|
||||
* Time a loaded machine may add between the deadline firing and the caller seeing the failure.
|
||||
* The main stage runs beside other builds; the bound only has to tell "cut at the budget" from
|
||||
* "waited for the drip", which the total leaves room for (budget + slack is well under half of
|
||||
* it).
|
||||
*/
|
||||
private static final Duration SCHEDULING_SLACK = Duration.ofSeconds(4);
|
||||
|
||||
private static final DockerImageName UPSTREAM_IMAGE =
|
||||
DockerImageName.parse(
|
||||
@@ -129,11 +144,11 @@ class FailureInjectionContractTest {
|
||||
try (ToxiproxyFixture toxiproxy = ToxiproxyFixture.start(network)) {
|
||||
// given
|
||||
Proxy proxy = toxiproxy.proxyTo("slow-drip", "upstream", 80);
|
||||
toxiproxy.slowDrip(proxy, 16, Duration.ofMillis(50));
|
||||
toxiproxy.slowDrip(proxy, SLOW_DRIP_SLICE_BYTES, SLOW_DRIP_GAP);
|
||||
TimeoutSettings base = ClientProfiles.timeouts();
|
||||
URI baseUrl =
|
||||
URI.create("http://" + toxiproxy.proxiedHost() + ":" + toxiproxy.proxiedPort() + "/");
|
||||
long started;
|
||||
Duration elapsed;
|
||||
Throwable failure;
|
||||
|
||||
// when
|
||||
@@ -151,7 +166,7 @@ class FailureInjectionContractTest {
|
||||
base.readIdle(),
|
||||
SLOW_DRIP_BUDGET))
|
||||
.build())) {
|
||||
started = System.nanoTime();
|
||||
long started = System.nanoTime();
|
||||
failure =
|
||||
catchThrowable(
|
||||
() ->
|
||||
@@ -160,14 +175,22 @@ class FailureInjectionContractTest {
|
||||
.exchange(
|
||||
harness.profile().name(),
|
||||
HttpOperation.get(
|
||||
new OperationName("fault-probe"), "/bytes/4096", Map.of()),
|
||||
new OperationName("fault-probe"),
|
||||
"/bytes/" + SLOW_DRIP_BYTES,
|
||||
Map.of()),
|
||||
ResponseType.of(byte[].class)));
|
||||
// The call alone: closing the harness afterwards is not part of the deadline.
|
||||
elapsed = Duration.ofNanos(System.nanoTime() - started);
|
||||
}
|
||||
|
||||
// then
|
||||
Duration elapsed = Duration.ofNanos(System.nanoTime() - started);
|
||||
assertThat(failure).isInstanceOf(HttpDeadlineExceededException.class);
|
||||
assertThat(elapsed).isLessThan(SLOW_DRIP_BUDGET.plus(Duration.ofMillis(500)));
|
||||
assertThat(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK))
|
||||
.as("the bound must stay well under the drip, or it proves nothing")
|
||||
.isLessThan(SLOW_DRIP_TOTAL.dividedBy(2));
|
||||
assertThat(elapsed)
|
||||
.as("the deadline, not the drip, ended the call")
|
||||
.isLessThan(SLOW_DRIP_BUDGET.plus(SCHEDULING_SLACK));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-5
@@ -29,15 +29,17 @@ class AttemptDeadlineTest {
|
||||
@Test
|
||||
@DisplayName("남은 deadline이 단계 timeout보다 짧으면 단계 timeout을 남은 시간으로 줄인다")
|
||||
void aShorterRemainingDeadlineBoundsTheStageTimeout() {
|
||||
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(300))) {
|
||||
// Seconds rather than milliseconds, so the deadline has not already passed by the time bound()
|
||||
// runs on a loaded machine; the stage is longer still, so bound() must cut it.
|
||||
try (AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(5))) {
|
||||
// given
|
||||
Duration stage = Duration.ofSeconds(2);
|
||||
Duration stage = Duration.ofSeconds(30);
|
||||
|
||||
// when
|
||||
Duration bounded = AttemptDeadline.bound(stage);
|
||||
|
||||
// then
|
||||
assertThat(bounded).isLessThanOrEqualTo(Duration.ofMillis(300)).isPositive();
|
||||
assertThat(bounded).isLessThanOrEqualTo(Duration.ofSeconds(5)).isPositive();
|
||||
assertThat(scope.expired()).isFalse();
|
||||
}
|
||||
}
|
||||
@@ -89,14 +91,16 @@ class AttemptDeadlineTest {
|
||||
void closingTheScopeBeforeTheDeadlineCancelsTheAbort() throws Exception {
|
||||
// given
|
||||
AtomicInteger aborts = new AtomicInteger();
|
||||
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofMillis(100));
|
||||
// One second, so registering and closing always happen before it on a loaded machine; the
|
||||
// wait after the close outlasts it, so an abort that was not cancelled would have run.
|
||||
AttemptDeadline.Scope scope = AttemptDeadline.open(Duration.ofSeconds(1));
|
||||
AttemptDeadline.abortOnExpiry(aborts::incrementAndGet);
|
||||
|
||||
// when
|
||||
scope.close();
|
||||
|
||||
// then
|
||||
Thread.sleep(300);
|
||||
Thread.sleep(1_500);
|
||||
assertThat(aborts).hasValue(0);
|
||||
assertThat(AttemptDeadline.remaining()).isEmpty();
|
||||
}
|
||||
|
||||
+11
-1
@@ -6,6 +6,7 @@ import dev.caskeleton.adapter.outbound.persistence.api.PersistenceOperationName;
|
||||
import dev.caskeleton.adapter.outbound.persistence.api.error.PessimisticLockTimeoutException;
|
||||
import dev.caskeleton.adapter.outbound.persistence.postgresql.lock.PostgreSqlLockExceptionTranslator;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
@@ -59,14 +60,23 @@ class PostgreSqlPessimisticLockContractTest {
|
||||
lockRow(holder);
|
||||
|
||||
contender.setAutoCommit(false);
|
||||
try (Statement statement = contender.createStatement();
|
||||
ResultSet setting = statement.executeQuery("show lock_timeout")) {
|
||||
assertThat(setting.next()).isTrue();
|
||||
assertThat(setting.getString(1))
|
||||
.as("without a lock_timeout a lock that waited would wait for the holder for good")
|
||||
.isEqualTo("0");
|
||||
}
|
||||
Instant startedAt = Instant.now();
|
||||
SQLException refused = lockRowNowait(contender);
|
||||
Duration waited = Duration.between(startedAt, Instant.now());
|
||||
|
||||
assertThat(refused.getSQLState()).isEqualTo("55P03");
|
||||
// The contender has no lock_timeout, so a lock that waited would wait for the holder, who
|
||||
// never lets go here. Seconds of slack cannot be mistaken for that.
|
||||
assertThat(waited)
|
||||
.as("NOWAIT must not wait for the holder")
|
||||
.isLessThan(Duration.ofSeconds(1));
|
||||
.isLessThan(Duration.ofSeconds(10));
|
||||
|
||||
var translated = translator.translate(refused, OPERATION, 1, waited, null).orElseThrow();
|
||||
assertThat(translated).isInstanceOf(PessimisticLockTimeoutException.class);
|
||||
|
||||
+25
-4
@@ -12,6 +12,7 @@ import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -49,13 +50,17 @@ class PostgreSqlLifecycleIntegrationTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@Timeout(10)
|
||||
// The method starts its own PostgreSQL container; on a loaded machine that alone can take tens of
|
||||
// seconds. The bound that matters, the exhausted pool's wait, is measured on its own below.
|
||||
@Timeout(value = 2, unit = TimeUnit.MINUTES)
|
||||
void poolCapacityExhaustionAndShutdownAreBoundedAndObservable() throws Exception {
|
||||
PostgreSqlReadinessSupport bounded = PostgreSqlReadinessSupport.start(2, 300);
|
||||
HikariDataSource dataSource = bounded.dataSource();
|
||||
try {
|
||||
try (Connection first = bounded.connection();
|
||||
Connection second = bounded.connection()) {
|
||||
// Borrowing a connection the pool still has to open (TCP, SCRAM, backend fork) must not be
|
||||
// held to the 300 ms timeout this test measures, so the two are taken patiently.
|
||||
try (Connection first = patiently(bounded);
|
||||
Connection second = patiently(bounded)) {
|
||||
PoolSnapshot saturated = snapshot(dataSource);
|
||||
assertThat(saturated.state()).isEqualTo(PoolState.SATURATED);
|
||||
assertThat(saturated.activeConnections()).isEqualTo(2);
|
||||
@@ -66,8 +71,10 @@ class PostgreSqlLifecycleIntegrationTest {
|
||||
|
||||
long started = System.nanoTime();
|
||||
assertThatThrownBy(() -> bounded.connection().close()).isInstanceOf(SQLException.class);
|
||||
// At least the 300 ms connectionTimeout, and nowhere near an unbounded wait: the slack
|
||||
// above it is for a loaded machine, not for the pool.
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(2));
|
||||
.isBetween(Duration.ofMillis(250), Duration.ofSeconds(5));
|
||||
}
|
||||
|
||||
assertThat(snapshot(dataSource).activeConnections()).isZero();
|
||||
@@ -81,6 +88,20 @@ class PostgreSqlLifecycleIntegrationTest {
|
||||
assertThat(closed.activeConnections()).isZero();
|
||||
}
|
||||
|
||||
/** A pool connection, retrying while a new physical connection takes longer than the timeout. */
|
||||
private static Connection patiently(PostgreSqlReadinessSupport support) throws SQLException {
|
||||
long until = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
|
||||
while (true) {
|
||||
try {
|
||||
return support.connection();
|
||||
} catch (SQLException slow) {
|
||||
if (System.nanoTime() > until) {
|
||||
throw slow;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static PoolSnapshot snapshot(HikariDataSource dataSource) {
|
||||
int maximum = dataSource.getMaximumPoolSize();
|
||||
if (dataSource.isClosed()) {
|
||||
|
||||
+8
-3
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.readiness;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
@@ -168,9 +169,13 @@ class PostgreSqlMigrationIntegrationTest {
|
||||
.initSql("set lock_timeout = '250ms'; set statement_timeout = '2s'")
|
||||
.load();
|
||||
long started = System.nanoTime();
|
||||
assertThatThrownBy(blockedExpansion::migrate).isInstanceOf(FlywayException.class);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3));
|
||||
Throwable refused = catchThrowable(blockedExpansion::migrate);
|
||||
Duration waited = Duration.ofNanos(System.nanoTime() - started);
|
||||
// The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
|
||||
// says which, so the clock only has to show the migration did not wait for the holder.
|
||||
assertThat(refused).isInstanceOf(FlywayException.class);
|
||||
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
|
||||
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
|
||||
blocker.rollback();
|
||||
}
|
||||
|
||||
|
||||
+8
-4
@@ -1,7 +1,7 @@
|
||||
package dev.caskeleton.adapter.outbound.persistence.readiness;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Statement;
|
||||
@@ -175,9 +175,13 @@ final class PostgreSqlOptionalStreamLifecycle {
|
||||
blocker.setAutoCommit(false);
|
||||
lock.execute("lock table capability_schema_registry in access exclusive mode");
|
||||
long started = System.nanoTime();
|
||||
assertThatThrownBy(flyway::migrate).isInstanceOf(FlywayException.class);
|
||||
assertThat(Duration.ofNanos(System.nanoTime() - started))
|
||||
.isBetween(Duration.ofMillis(200), Duration.ofSeconds(3));
|
||||
Throwable refused = catchThrowable(flyway::migrate);
|
||||
Duration waited = Duration.ofNanos(System.nanoTime() - started);
|
||||
// The lock_timeout (55P03), not the 2 s statement_timeout (57014), ended it: the cause
|
||||
// says which, so the clock only has to show the migration did not wait for the holder.
|
||||
assertThat(refused).isInstanceOf(FlywayException.class);
|
||||
assertThat(PostgreSqlReadinessSupport.sqlStates(refused)).contains("55P03");
|
||||
assertThat(waited).isBetween(Duration.ofMillis(200), Duration.ofSeconds(30));
|
||||
blocker.rollback();
|
||||
}
|
||||
|
||||
|
||||
+14
@@ -28,6 +28,20 @@ final class PostgreSqlReadinessSupport implements AutoCloseable {
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
/** Every SQLState in {@code failure}'s cause chain, outermost first. */
|
||||
static java.util.List<String> sqlStates(Throwable failure) {
|
||||
java.util.List<String> states = new java.util.ArrayList<>();
|
||||
for (Throwable cause = failure; cause != null; cause = cause.getCause()) {
|
||||
if (cause instanceof java.sql.SQLException sql && sql.getSQLState() != null) {
|
||||
states.add(sql.getSQLState());
|
||||
}
|
||||
if (cause.getCause() == cause) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
static PostgreSqlReadinessSupport start() {
|
||||
return start(5, 1_000);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user