diff --git a/.env.example b/.env.example
index 1c68b40e..09bb6b74 100644
--- a/.env.example
+++ b/.env.example
@@ -171,6 +171,7 @@ APP_CACHE_REDIS_COMMAND_TIMEOUT=200ms
APP_CACHE_REDIS_POSITIVE_SOFT_TTL=30s
APP_CACHE_REDIS_MINIMUM_HARD_TTL=1s
APP_CACHE_REDIS_SEMANTIC_REGION=default
+APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES=1048576
APP_MESSAGING_BROKER=kafka
APP_MESSAGING_PRODUCER_ID=
APP_MESSAGING_KAFKA_BROKERS=
@@ -293,7 +294,7 @@ APP_NOTIFICATION_PLATFORM_SMTP_ENABLED=false
APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY=true
APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT=local
APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE=default
-APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=10s
+APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=30s
APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY=4
APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND=10
APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY=no-reply@example.invalid
diff --git a/config/spotbugs/exclude.xml b/config/spotbugs/exclude.xml
index 2ee07473..156d6eae 100644
--- a/config/spotbugs/exclude.xml
+++ b/config/spotbugs/exclude.xml
@@ -359,6 +359,15 @@
+
+
+
+
+
+
+
} and followed by a {@code sql}
* fence. The lanes that run these blocks against real PostgreSQL read them from the document rather
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java
index 98739e4d..a483b2fe 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java
@@ -1,18 +1,34 @@
package dev.caskeleton.adapter.outbound.persistence.autoconfigure;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay;
import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository;
+import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper;
import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.springframework.boot.context.properties.bind.UnboundConfigurationPropertiesException;
import org.springframework.boot.convert.ApplicationConversionService;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.jdbc.core.JdbcOperations;
+import org.springframework.scheduling.config.FixedDelayTask;
+import org.springframework.scheduling.config.ScheduledTask;
+import org.springframework.scheduling.config.ScheduledTaskHolder;
class JpaLiveEventReplayActivationTest {
@@ -34,6 +50,110 @@ class JpaLiveEventReplayActivationTest {
});
}
+ @Test
+ void theReaperIsAssembledWithTheReplayStore() {
+ runner
+ .withPropertyValues(
+ "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H")
+ .run(
+ context ->
+ assertThat(context)
+ .as("without the reaper nothing removes a row past its retention")
+ .hasSingleBean(LiveEventReaper.class));
+ }
+
+ @Test
+ void noReaperIsAssembledWithoutTheReplayStore() {
+ runner.run(context -> assertThat(context).doesNotHaveBean(LiveEventReaper.class));
+ }
+
+ @Test
+ void aZeroReaperIntervalFailsStartup() {
+ runner
+ .withPropertyValues(
+ "app.realtime.replay.enabled=true",
+ "app.realtime.replay.retention=PT1H",
+ "app.realtime.replay.reaper-interval=PT0S")
+ .run(
+ context ->
+ assertThat(context.getStartupFailure())
+ .rootCause()
+ .hasMessageContaining("app.realtime.replay.reaper-interval must be positive"));
+ }
+
+ @Test
+ void aMisspeltReplayKeyFailsStartup() {
+ runner
+ .withPropertyValues(
+ "app.realtime.replay.enabled=true",
+ "app.realtime.replay.retention=PT1H",
+ "app.realtime.replay.reaper-intervall=PT1M")
+ .run(
+ context ->
+ assertThat(context.getStartupFailure())
+ .rootCause()
+ .isInstanceOf(UnboundConfigurationPropertiesException.class)
+ .hasMessageContaining("reaper-intervall"));
+ }
+
+ @ParameterizedTest(name = "[{index}] reaper-interval {0} → {1}")
+ @CsvSource(
+ value = {"UNSET, PT10M", "PT3M, PT3M"},
+ nullValues = "UNSET")
+ @DisplayName("reaper는 묶인 reaper-interval(기본 PT10M)마다 돌고, 첫 회도 그 간격 뒤다")
+ void theReaperRunsAtTheBoundInterval(String configured, Duration expected) {
+ // given
+ ApplicationContextRunner replay =
+ runner.withPropertyValues(
+ "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H");
+ if (configured != null) {
+ replay = replay.withPropertyValues("app.realtime.replay.reaper-interval=" + configured);
+ }
+
+ // when
+ List> schedules = new ArrayList<>();
+ replay.run(context -> schedules.addAll(fixedDelaySchedules(context)));
+
+ // then
+ assertThat(schedules)
+ .as("the schedule is the bound settings' interval, its default included")
+ .containsExactly(List.of(expected, expected));
+ }
+
+ @Test
+ void theScheduledTaskIsTheReaper() {
+ // given
+ ApplicationContextRunner replay =
+ runner.withPropertyValues(
+ "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H");
+
+ // when
+ List reapedThrough = new ArrayList<>();
+ replay.run(
+ context -> {
+ fixedDelayTasks(context).forEach(task -> task.getRunnable().run());
+ reapedThrough.add(context.getBean(LiveEventJpaRepository.class));
+ });
+
+ // then
+ verify(reapedThrough.getFirst()).deleteExpiredBatch(any(Instant.class), anyInt());
+ }
+
+ /** Interval and initial delay of every fixed-delay task the context scheduled. */
+ private static List> fixedDelaySchedules(ApplicationContext context) {
+ return fixedDelayTasks(context).stream()
+ .map(task -> List.of(task.getIntervalDuration(), task.getInitialDelayDuration()))
+ .toList();
+ }
+
+ private static List fixedDelayTasks(ApplicationContext context) {
+ return context.getBean(ScheduledTaskHolder.class).getScheduledTasks().stream()
+ .map(ScheduledTask::getTask)
+ .filter(FixedDelayTask.class::isInstance)
+ .map(FixedDelayTask.class::cast)
+ .toList();
+ }
+
@Test
void enabledReplayCapabilityRequiresExplicitRetention() {
runner
@@ -42,7 +162,8 @@ class JpaLiveEventReplayActivationTest {
context -> {
assertThat(context).hasFailed();
assertThat(context.getStartupFailure())
- .hasMessageContaining("app.realtime.replay.retention");
+ .rootCause()
+ .hasMessageContaining("app.realtime.replay.retention must be set");
});
}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java
index a560adf4..74694d1f 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java
@@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.experimental;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
import dev.caskeleton.adapter.outbound.persistence.experimental.database.TenantPoolBudget;
import dev.caskeleton.adapter.outbound.persistence.experimental.replica.ConsistencyAwareDataSourceRouter;
@@ -103,14 +104,28 @@ class ExperimentalFailClosedTest {
void tenantPoolBudgetIsBounded() {
var budget = new TenantPoolBudget(2, 20);
- assertThat(budget.hasCapacity(1, 10)).isTrue();
- assertThatThrownBy(() -> budget.requireCapacity(2, 10))
+ assertThat(budget.hasCapacity(1, 10, 10)).isTrue();
+ assertThatThrownBy(() -> budget.requireCapacity(2, 10, 1))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("tenant pool budget");
- assertThatThrownBy(() -> budget.requireCapacity(1, 20))
+ assertThatThrownBy(() -> budget.requireCapacity(1, 20, 1))
.isInstanceOf(IllegalStateException.class);
}
+ @Test
+ void aNewPoolLargerThanTheConnectionsLeftIsRefused() {
+ // given
+ var budget = new TenantPoolBudget(4, 20);
+
+ // when
+ Throwable refused = catchThrowable(() -> budget.requireCapacity(1, 15, 10));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("tenant pool budget exhausted");
+ }
+
/** A lag monitor with a fixed answer, including "I do not know". */
private record FixedLagMonitor(Optional replayedThrough, Optional lag)
implements ReplicaLagMonitor {}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java
new file mode 100644
index 00000000..1fb7f801
--- /dev/null
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java
@@ -0,0 +1,45 @@
+package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.Mockito.mock;
+
+import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
+import java.time.Duration;
+import java.util.Set;
+import org.hibernate.SessionFactory;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/**
+ * The stateless runner's transaction bound is the write bound every other transaction here has.
+ *
+ *
The runner restates the value instead of reading it, because the {@code hibernate} package may
+ * not depend on {@code transaction} (JpaModuleBoundaryTest). A test may depend on both, so it is
+ * the place the two copies are held to each other (round9 review 2026-11-12, RR9-01-25).
+ */
+class StatelessRunnerTimeoutTest {
+
+ @Test
+ void theRunnersDefaultIsSpringTransactionsWriteDefault() {
+ assertThat(HibernateStatelessSessionRunner.DEFAULT_TIMEOUT)
+ .isEqualTo(SpringTransaction.Timeouts.DEFAULT.write());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"PT0S", "PT-1S", "PT1.5S", "PT2147483648S"})
+ void aTimeoutThatIsNotAWholePositiveNumberOfSecondsIsTheAssemblysFault(String timeout) {
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ new HibernateStatelessSessionRunner(
+ mock(SessionFactory.class), Set.of(), Duration.parse(timeout)));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("must be a positive whole number of seconds");
+ }
+}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java
index 72235d67..e80b313d 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java
@@ -2,23 +2,61 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.time.Clock;
-import java.time.Instant;
-import java.time.ZoneOffset;
import org.junit.jupiter.api.Test;
+/**
+ * Unit tests for {@link IdempotencyReaper}: the delete runs in bounded batches until one comes back
+ * short, as the outbox reaper's does.
+ *
+ *
The repository is a Spring Data boundary, so Mockito stands in for it. The statement itself —
+ * expiry judged by the database clock — runs against PostgreSQL in {@code
+ * IdempotencyReaperIntegrationTest}.
+ */
class IdempotencyReaperTest {
- private static final Instant NOW = Instant.parse("2026-06-09T12:00:00Z");
+ private static final int BATCH = IdempotencyReaper.BATCH_SIZE;
+
+ private final IdempotencyRecordJpaRepository repository =
+ mock(IdempotencyRecordJpaRepository.class);
+ private final IdempotencyReaper reaper = new IdempotencyReaper(repository);
@Test
- void reapDeletesRecordsExpiredAsOfNow() {
- IdempotencyRecordJpaRepository repo = mock(IdempotencyRecordJpaRepository.class);
- when(repo.deleteExpired(NOW)).thenReturn(3);
- IdempotencyReaper reaper = new IdempotencyReaper(repo, Clock.fixed(NOW, ZoneOffset.UTC));
+ void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() {
+ // given
+ when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3);
- assertThat(reaper.reap()).isEqualTo(3);
+ // when
+ int purged = reaper.reap();
+
+ // then
+ assertThat(purged).isEqualTo(2 * BATCH + 3);
+ }
+
+ @Test
+ void eachBatchIsItsOwnDeleteStatement() {
+ // given
+ when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3);
+
+ // when
+ var unused = reaper.reap();
+
+ // then
+ verify(repository, times(3)).deleteExpiredBatch(BATCH);
+ }
+
+ @Test
+ void reapReturnsZeroWhenNothingHasExpired() {
+ // given
+ when(repository.deleteExpiredBatch(BATCH)).thenReturn(0);
+
+ // when
+ int purged = reaper.reap();
+
+ // then
+ assertThat(purged).isZero();
}
}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java
new file mode 100644
index 00000000..b43a316e
--- /dev/null
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java
@@ -0,0 +1,53 @@
+package dev.caskeleton.adapter.outbound.persistence.liveevent;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link LiveEventReaper}: rows retained no later than the clock's now go, in
+ * bounded batches until one comes back short.
+ *
+ *
The repository is a Spring Data boundary, so Mockito stands in for it. The statement runs
+ * against PostgreSQL in {@code JpaLiveEventReplayContractTest}.
+ */
+class LiveEventReaperTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T00:00:00Z");
+ private static final int BATCH = LiveEventReaper.BATCH_SIZE;
+
+ private final LiveEventJpaRepository repository = mock(LiveEventJpaRepository.class);
+ private final LiveEventReaper reaper =
+ new LiveEventReaper(repository, Clock.fixed(NOW, ZoneOffset.UTC));
+
+ @Test
+ void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() {
+ // given
+ when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4);
+
+ // when
+ int purged = reaper.reap();
+
+ // then
+ assertThat(purged).isEqualTo(BATCH + 4);
+ }
+
+ @Test
+ void eachBatchIsItsOwnDeleteStatement() {
+ // given
+ when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4);
+
+ // when
+ var unused = reaper.reap();
+
+ // then
+ verify(repository, times(2)).deleteExpiredBatch(NOW, BATCH);
+ }
+}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java
index 0d647c57..b6ed1ca0 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java
@@ -78,6 +78,20 @@ class DistributedLockPersistenceConfigTest {
});
}
+ @Test
+ @DisplayName("읽는 곳이 없던 app.lock.wait-time은 모르는 키로 기동을 실패시킨다")
+ void theRemovedWaitTimeKeyFailsStartup() {
+ // when
+ contextRunner
+ .withPropertyValues("app.lock.wait-time=3s")
+ .run(
+ context -> {
+ // then
+ assertThat(context).hasFailed();
+ assertThat(context.getStartupFailure()).hasStackTraceContaining("wait-time");
+ });
+ }
+
@Test
void multiInstanceBuildsTheJdbcProviderAndNotTheInProcessProvider() {
contextRunner
diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256
index 1eb57d14..93a51fe4 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256
+++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256
@@ -14,6 +14,7 @@
be51fabd07220f9ef885534ba36ada5f1491baf24cdfb54d25d93a19ddc29926 jpa/notification-platform/V11__notification_reconciliation_claim_lease.sql
43f9429f91c07528154ec01b721649b2c25fa98857ac154c7c08f4655634657f jpa/notification-platform/V12__notification_suppression_target_unique.sql
d01aaf59f66834358abf18246faa9a9eb8dfa9f42097ca9d968c44edff564630 jpa/notification-platform/V13__notification_request_acceptance.sql
+8a7dc3604d44f1438b39dfec3aa688f792e7363cd3c1b74bf0c799a6b53f9f0d jpa/notification-platform/V14__notification_provider_event_open_index.sql
1ac7cc1c2509868eb0ce202e1917bb36846f98f17ad4922abe3fac0ef128199b jpa/notification-platform/V1__notification_platform_core.sql
0add67aa23580e51712f84fabe9b09e08401ef708cfa1b3292b120fdb5c12fa2 jpa/notification-platform/V2__notification_platform_contact_template_policy.sql
f5b98598a59cc4376f5d233b410f8a5ae9f3cd94156291ebdf2d0da5a66f34ef jpa/notification-platform/V3__notification_platform_inbox_admin.sql
@@ -42,6 +43,7 @@ a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19
25db296e55444636c8a017d9b736eecf4a839a9caf1733c0c4a5f37bae5a3161 postgresql/V20__outbox_event_deferred_attempt_count.sql
21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql
20da5649a29bf2305f0e04ab14003b21b7f120d867439fb01f1658c98179dba0 postgresql/V22__outbox_event_last_failure_code.sql
+9492ace50ab828144792b77c72d92fce50bec29a028ebc917f7e55facf58d85a postgresql/V23__durable_operation_lease_fence.sql
8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql
6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql
c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java
index ef80fdb1..a435e5b5 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java
@@ -7,9 +7,14 @@ import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedExc
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
/**
* Bounds how many tenant clients exist at once (advanced plan Task 11).
@@ -18,6 +23,13 @@ import java.util.Set;
* connection pool and its own monitoring threads, so a thousand tenants is a thousand pools. The
* registry caps the number and evicts idle ones, and refuses rather than exceeding the cap — an
* unbounded registry fails later, as connection exhaustion on an unrelated request.
+ *
+ *
{@link #acquire} hands back a {@link Lease}; the client is in use until every lease on it is
+ * closed, and only a client nobody holds is idle. Acquire used to return nothing, so idle meant
+ * "acquired longest ago" and eviction could close a client a long-running caller was still using.
+ * Closing an evicted client is this registry's own clean-up: if it fails, the tenant whose acquire
+ * triggered the eviction is not told about another tenant's client — the failure is counted in
+ * {@link #evictionCloseFailures()} (round9 review 2026-11-12, RR9-01-27).
*/
@MongoAdvancedEntryPoint(MongoCapability.DATABASE_PER_TENANT)
public final class MongoTenantClientRegistry {
@@ -26,11 +38,21 @@ public final class MongoTenantClientRegistry {
private final Duration idleEviction;
- private final Map lastUsed = new java.util.concurrent.ConcurrentHashMap<>();
+ private final Map lastUsed = new ConcurrentHashMap<>();
- private final Map clients = new java.util.concurrent.ConcurrentHashMap<>();
+ private final Map clients = new ConcurrentHashMap<>();
- private final java.util.function.Function clientFactory;
+ private final Map leases = new HashMap<>();
+
+ /**
+ * Guards the cap, the lease counts and the eviction as one decision. A private monitor rather
+ * than the registry itself, so code outside cannot take it and stall every acquire.
+ */
+ private final Object lock = new Object();
+
+ private final AtomicLong evictionCloseFailures = new AtomicLong();
+
+ private final Function clientFactory;
private final Clock clock;
@@ -47,15 +69,15 @@ public final class MongoTenantClientRegistry {
MongoAdvancedCapabilityGuard guard,
int maximumActiveClients,
Duration idleEviction,
- java.util.function.Function clientFactory) {
+ Function clientFactory) {
this(guard, maximumActiveClients, idleEviction, clientFactory, Clock.systemUTC());
}
/**
* A registry that owns real clients.
*
- *
Idle eviction reads {@code clock}; {@link #acquire(String)} used to read {@code
- * Instant.now()} beside it.
+ *
Acquiring, closing a lease and idle eviction all read {@code clock}; {@link
+ * #acquire(String)} used to read {@code Instant.now()} beside it.
*
*
The factory is called at most once per tenant, inside the map's own computation, so two
* threads asking for the same tenant get one client rather than two — a per-tenant client carries
@@ -65,7 +87,7 @@ public final class MongoTenantClientRegistry {
MongoAdvancedCapabilityGuard guard,
int maximumActiveClients,
Duration idleEviction,
- java.util.function.Function clientFactory,
+ Function clientFactory,
Clock clock) {
Objects.requireNonNull(guard, "guard").require(MongoTenantClientRegistry.class);
this.clientFactory = Objects.requireNonNull(clientFactory, "clientFactory");
@@ -80,41 +102,128 @@ public final class MongoTenantClientRegistry {
/**
* Acquires a client for a tenant.
*
+ * @return the lease to close when the work with the client is done
* @throws MongoOperationRejectedException when the cap is reached and nothing can be evicted
*/
- public void acquire(String tenantKey) {
- acquire(tenantKey, clock.instant());
+ public Lease acquire(String tenantKey) {
+ Objects.requireNonNull(tenantKey, "tenantKey");
+ synchronized (lock) {
+ return acquireHeld(tenantKey);
+ }
}
- /** Acquires a client for a tenant at an explicit instant, so eviction is testable. */
- public synchronized void acquire(String tenantKey, Instant now) {
- Objects.requireNonNull(tenantKey, "tenantKey");
- Objects.requireNonNull(now, "now");
+ private Lease acquireHeld(String tenantKey) {
+ // One time source for the acquire, the close and the eviction: an overload that took the
+ // instant as an argument let a caller acquire in one timeline and close in the clock's.
+ Instant now = clock.instant();
// Synchronized rather than a bare ConcurrentHashMap: the cap, the eviction and the create are
// one decision. Checking the size on one thread while another inserts is how a cap of ten ends
// up holding twelve connection pools.
- if (lastUsed.containsKey(tenantKey)) {
- lastUsed.put(tenantKey, now);
- return;
+ if (!lastUsed.containsKey(tenantKey)) {
+ evictIdle(now);
+ if (lastUsed.size() >= maximumActiveClients) {
+ throw MongoOperationRejectedException.of(
+ "tenancy.client",
+ "the tenant client cap of "
+ + maximumActiveClients
+ + " is reached and no client is idle; each tenant client carries its own"
+ + " connection pool and monitoring threads, so the cap is what stops one process"
+ + " from exhausting the cluster's connections");
+ }
+ clients.computeIfAbsent(tenantKey, clientFactory);
}
- evictIdle(now);
- if (lastUsed.size() >= maximumActiveClients) {
- throw MongoOperationRejectedException.of(
- "tenancy.client",
- "the tenant client cap of "
- + maximumActiveClients
- + " is reached and no client is idle; each tenant client carries its own connection "
- + "pool and monitoring threads, so the cap is what stops one process from exhausting "
- + "the cluster's connections");
- }
- clients.computeIfAbsent(tenantKey, clientFactory);
lastUsed.put(tenantKey, now);
+ leases.merge(tenantKey, 1, Integer::sum);
+ return new HeldLease(tenantKey, clients.get(tenantKey));
}
- /** Releases a tenant's client and closes it. */
- public synchronized void release(String tenantKey) {
- lastUsed.remove(Objects.requireNonNull(tenantKey, "tenantKey"));
- close(tenantKey);
+ /**
+ * Releases a tenant's client and closes it.
+ *
+ * @throws IllegalStateException when a lease on the client is still open, or the close fails
+ */
+ public void release(String tenantKey) {
+ Objects.requireNonNull(tenantKey, "tenantKey");
+ synchronized (lock) {
+ if (leases.containsKey(tenantKey)) {
+ throw new IllegalStateException(
+ "the client for tenant '" + tenantKey + "' is still in use and cannot be released");
+ }
+ lastUsed.remove(tenantKey);
+ close(tenantKey);
+ }
+ }
+
+ /**
+ * How many evicted clients failed to close.
+ *
+ *
The registry has dropped each of them and freed its slot; the count is how an operator
+ * learns that clients — and their pools — may still be open behind it.
+ */
+ public long evictionCloseFailures() {
+ return evictionCloseFailures.get();
+ }
+
+ /**
+ * One use of a tenant client; close it when the work with that client is done.
+ *
+ *
An interface so the registry's own implementation stays private: only the registry makes a
+ * lease, and only through {@link #acquire}.
+ */
+ public interface Lease extends AutoCloseable {
+
+ /** The tenant's client, valid until this lease is closed. */
+ AutoCloseable client();
+
+ /** Ends this use. Closing twice is the same as closing once. */
+ @Override
+ void close();
+ }
+
+ private final class HeldLease implements Lease {
+
+ private final String tenantKey;
+
+ private final AutoCloseable client;
+
+ private boolean closed;
+
+ private HeldLease(String tenantKey, AutoCloseable client) {
+ this.tenantKey = tenantKey;
+ this.client = client;
+ }
+
+ @Override
+ public AutoCloseable client() {
+ return client;
+ }
+
+ @Override
+ public void close() {
+ synchronized (lock) {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ leases.computeIfPresent(tenantKey, (tenant, open) -> open == 1 ? null : open - 1);
+ // Idle from the end of the last use, not its start: a client held for half an hour was in
+ // use all that time, and measuring from the acquire evicted it the moment it was let go.
+ Instant released = clock.instant();
+ lastUsed.computeIfPresent(
+ tenantKey, (tenant, used) -> released.isAfter(used) ? released : used);
+ }
+ }
+ }
+
+ /**
+ * How many leases are open now, across all tenants.
+ *
+ *
A lease nobody closed keeps its client's slot for good; this is how that becomes visible.
+ */
+ public int openLeases() {
+ synchronized (lock) {
+ return leases.values().stream().mapToInt(Integer::intValue).sum();
+ }
}
/** The tenants with an active client. */
@@ -128,8 +237,9 @@ public final class MongoTenantClientRegistry {
}
private void evictIdle(Instant now) {
- java.util.List idle =
+ List idle =
lastUsed.entrySet().stream()
+ .filter(entry -> !leases.containsKey(entry.getKey()))
.filter(entry -> entry.getValue().plus(idleEviction).isBefore(now))
.map(Map.Entry::getKey)
.toList();
@@ -138,7 +248,11 @@ public final class MongoTenantClientRegistry {
lastUsed.remove(tenant);
// Evicting the bookkeeping entry without closing the client frees the slot and keeps the
// connection pool, which is the exhaustion this cap exists to prevent, delayed.
- close(tenant);
+ try {
+ close(tenant);
+ } catch (IllegalStateException closeFailed) {
+ evictionCloseFailures.incrementAndGet();
+ }
});
}
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java
index 8ed663b7..0f0a91d4 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java
@@ -6,20 +6,30 @@ import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer;
import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry;
import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget;
import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
+import java.util.stream.Stream;
+import org.bson.Document;
+import org.bson.RawBsonDocument;
+import org.bson.codecs.Codec;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
-import org.springframework.data.mongodb.core.aggregation.AggregationResults;
/**
* Runs an aggregation only after its stages, budget and lookups have been checked (design §17).
*
*
Every guard runs before the first stage reaches the server, because an aggregation that is
- * going to be refused should cost nothing. The one guard that runs afterwards is the result-count
- * check: {@code $limit} bounds what the pipeline emits, but a pipeline whose own shape produced
- * more than the budget allows has still told the caller something worth failing over.
+ * going to be refused should cost nothing. Two run while the results arrive: the result count and
+ * the result bytes. The pipeline gets a final {@code $limit} of one document past the count budget,
+ * so the server never produces more than that, and the results are read from the cursor one
+ * document at a time, the encoded size of each added up as it arrives. They used to be read whole —
+ * every document in memory before anything was counted — and the byte budget was declared and never
+ * checked (round9 review 2026-11-12, RR9-01-20; {@code docs/mongodb/query-aggregation-guide.md}
+ * 「집계에 예산을 둔다」).
*/
public final class PolicyAwareMongoAggregationExecutor {
@@ -53,6 +63,14 @@ public final class PolicyAwareMongoAggregationExecutor {
/**
* Executes a typed aggregation.
*
+ *
Results are read as {@link Document}, sized, then mapped with the template's converter. Two
+ * costs follow, both deliberate. Each document is encoded once more to be measured: the template
+ * hands back decoded documents, so the bytes the server sent are not available, and the budget
+ * has to be in bytes. And the mapping is the converter's alone: the template's typed aggregate
+ * would also publish {@code AfterLoadEvent}/{@code AfterConvertEvent} and run entity callbacks,
+ * and those do not run for an aggregation result read here. A listener that needs them maps from
+ * the returned type itself.
+ *
* @throws MongoOperationRejectedException when a stage, lookup, budget or result size is not
* permitted
*/
@@ -71,19 +89,38 @@ public final class PolicyAwareMongoAggregationExecutor {
MongoOperationBudget effective =
enforcer.narrow(budget, budget.narrowedTo(budgetFor(context, budget)));
+ List bounded = new ArrayList<>(plan.operations());
+ // One past the budget: enough to know the budget was exceeded, never more than that.
+ bounded.add(Aggregation.limit(effective.maxResults() + 1L));
Aggregation aggregation =
- Aggregation.newAggregation(plan.operations()).withOptions(optionsFor(profile, effective));
- AggregationResults results = operations.aggregate(aggregation, collection, outputType);
- List mapped = results.getMappedResults();
- if (mapped.size() > effective.maxResults()) {
- throw MongoOperationRejectedException.of(
- "aggregation.result",
- "the aggregation returned "
- + mapped.size()
- + " documents, above the budget of "
- + effective.maxResults());
+ Aggregation.newAggregation(bounded).withOptions(optionsFor(profile, effective));
+ Codec sizing = operations.getConverter().getCodecRegistry().get(Document.class);
+ List mapped = new ArrayList<>();
+ long bytes = 0;
+ try (Stream documents =
+ operations.aggregateStream(aggregation, collection, Document.class)) {
+ Iterator cursor = documents.iterator();
+ while (cursor.hasNext()) {
+ Document document = cursor.next();
+ if (mapped.size() == effective.maxResults()) {
+ throw MongoOperationRejectedException.of(
+ "aggregation.result",
+ "the aggregation returned more than the budget of "
+ + effective.maxResults()
+ + " documents");
+ }
+ bytes += new RawBsonDocument(document, sizing).getByteBuffer().remaining();
+ if (bytes > effective.maxResultBytes()) {
+ throw MongoOperationRejectedException.of(
+ "aggregation.result",
+ "the aggregation returned more than the budget of "
+ + effective.maxResultBytes()
+ + " bytes");
+ }
+ mapped.add(operations.getConverter().read(outputType, document));
+ }
}
- return mapped;
+ return List.copyOf(mapped);
}
private static MongoOperationBudget budgetFor(
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java
index 0466ca15..855db8ec 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java
@@ -36,20 +36,46 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
private static final String MIGRATION_ID = "migrationId";
+ private static final String FENCE = "fence";
+
private final MongoCollection ledger;
private final MongoCollection checkpoints;
+ private final MongoCollection locks;
+
+ /**
+ * A ledger over the default collections of {@code database}, the lock's included.
+ *
+ *
The lock collection is {@link MongoCollectionMigrationLock#DEFAULT_COLLECTION}. A deployment
+ * that keeps its runner lease in a collection of another name uses the three-collection
+ * constructor with that collection; with this one the ledger would read a lock that is never
+ * taken and refuse nothing on its account.
+ *
+ * @param database the database being migrated
+ */
public MongoCollectionMigrationLedger(MongoDatabase database) {
this(
Objects.requireNonNull(database, "database").getCollection(DEFAULT_COLLECTION),
- database.getCollection(DEFAULT_CHECKPOINT_COLLECTION));
+ database.getCollection(DEFAULT_CHECKPOINT_COLLECTION),
+ database.getCollection(MongoCollectionMigrationLock.DEFAULT_COLLECTION));
}
+ /**
+ * A ledger over explicit collections.
+ *
+ * @param ledger the applied migrations
+ * @param checkpoints the running migrations' positions
+ * @param locks the collection {@link MongoCollectionMigrationLock} keeps the runner lease in,
+ * read to refuse a replaced runner's ledger entry
+ */
public MongoCollectionMigrationLedger(
- MongoCollection ledger, MongoCollection checkpoints) {
+ MongoCollection ledger,
+ MongoCollection checkpoints,
+ MongoCollection locks) {
this.ledger = Objects.requireNonNull(ledger, "ledger");
this.checkpoints = Objects.requireNonNull(checkpoints, "checkpoints");
+ this.locks = Objects.requireNonNull(locks, "locks");
}
/**
@@ -90,13 +116,34 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
Objects.requireNonNull(checksum, "checksum");
Objects.requireNonNull(operator, "operator");
Objects.requireNonNull(appliedAt, "appliedAt");
- requireCurrentFence(fence, "ledger entry for " + migrationId.value());
+ requireFenced(fence, "ledger entry for " + migrationId.value());
+ // A superseded runner that wakes up after its replacement took the lease must not record the
+ // migration as applied under it (round9 review 2026-11-12, RR9-01-28). Two pieces of evidence:
+ // the lock's newest acquisition, which a replacement leaves the moment it takes the lease, and
+ // a checkpoint a newer runner wrote.
+ //
+ // A narrowing, not a guarantee: the reads and the insert are separate operations, so a runner
+ // replaced between them still records. What stops a superseded runner first is the lock's
+ // refresh, which refuses it; this check is the second line for a runner that skipped a
+ // refresh. Making it atomic needs the lock and the ledger in one transaction, which a
+ // standalone deployment does not have.
+ if (MongoCollectionMigrationLock.newestFence(locks) > fence
+ || checkpoints
+ .find(
+ Filters.and(
+ Filters.eq(MIGRATION_ID, migrationId.value()), Filters.gt(FENCE, fence)))
+ .first()
+ != null) {
+ throw MongoOperationRejectedException.of(
+ "migration.fence",
+ "a newer migration runner owns the lease; this runner's ledger entry was refused");
+ }
ledger.insertOne(
new Document(MIGRATION_ID, migrationId.value())
.append("checksum", checksum.value())
.append("operator", operator)
.append("appliedAt", Date.from(appliedAt))
- .append("fence", fence));
+ .append(FENCE, fence));
}
@Override
@@ -117,7 +164,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
@Override
public void saveCheckpoint(MongoMigrationCheckpoint checkpoint, long fence) {
Objects.requireNonNull(checkpoint, "checkpoint");
- requireCurrentFence(fence, "checkpoint for " + checkpoint.migrationId().value());
+ requireFenced(fence, "checkpoint for " + checkpoint.migrationId().value());
// Conditioned on the stored fence as well as matched by id: a superseded runner that wakes up
// mid-batch must not move the checkpoint of the runner that replaced it, which would send the
// live migration back over rows it has already done or forward past rows it has not.
@@ -126,7 +173,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
.append("resumePosition", checkpoint.resumePosition())
.append("processedCount", checkpoint.processedCount())
.append("updatedAt", Date.from(checkpoint.updatedAt()))
- .append("fence", fence);
+ .append(FENCE, fence);
// Replace-then-insert rather than one upsert, and the two cases this separates are the whole
// behaviour of the fence.
@@ -147,7 +194,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
.replaceOne(
Filters.and(
Filters.eq(MIGRATION_ID, checkpoint.migrationId().value()),
- Filters.or(Filters.exists("fence", false), Filters.lte("fence", fence))),
+ Filters.or(Filters.exists(FENCE, false), Filters.lte(FENCE, fence))),
document,
new ReplaceOptions().upsert(false))
.getMatchedCount();
@@ -172,9 +219,13 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
* Refuses a write from an unfenced lease.
*
*
An unfenced lease cannot prove the writer is still the owner, and the whole point of the
- * ledger is that it says what actually happened.
+ * ledger is that it says what actually happened. Whether a fenced writer is still the current one
+ * is decided in two places, not here: the runner refreshes its lease, conditioned on the fence,
+ * right before it writes, and each write is conditioned on the newest fence the ledger has seen
+ * for the migration. The method was called {@code requireCurrentFence} and checked neither
+ * (round9 review 2026-11-12, RR9-01-28).
*/
- private static void requireCurrentFence(long fence, String what) {
+ private static void requireFenced(long fence, String what) {
if (fence == MongoMigrationLock.UNFENCED) {
throw MongoOperationRejectedException.of(
"migration.fence",
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java
index cca219af..94c4ac26 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java
@@ -6,6 +6,7 @@ import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Updates;
+import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
@@ -125,6 +126,19 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock {
return fence;
}
+ /**
+ * The fence of the newest acquisition of the lock in {@code locks}, or {@link #UNFENCED} when the
+ * lock was never taken there.
+ *
+ *
For the ledger: a write whose fence is older than this one comes from a runner that has been
+ * replaced, whether or not its replacement has written anything yet.
+ */
+ static long newestFence(MongoCollection locks) {
+ Document current =
+ Objects.requireNonNull(locks, "locks").find(Filters.eq("_id", LOCK_ID)).first();
+ return current == null ? UNFENCED : fenceOf(current);
+ }
+
private static long fenceOf(Document lock) {
Object stored = lock.get("fence");
return stored instanceof Number number ? number.longValue() : 1L;
@@ -165,7 +179,11 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock {
.getMatchedCount();
if (stillOwned != 1) {
held = false;
- throw new IllegalStateException(
+ // The refusal the heartbeat promises (MongoMigrationHeartbeat#beat), so a migration that
+ // stops on it stops for the same reason the heartbeat's own ownership check gives. It was an
+ // IllegalStateException, which read as a server defect (round9 review 2026-11-12, RR9-01-28).
+ throw MongoOperationRejectedException.of(
+ "migration.lock",
"the migration lease was lost before it could be refreshed; another runner may have "
+ "taken it after this one appeared to stall");
}
diff --git a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java
index abbcad9c..edb28966 100644
--- a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java
+++ b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java
@@ -224,6 +224,95 @@ class MongoMigrationLaneTest {
assertThat(ledger.findCheckpoint(id).orElseThrow().processedCount()).isEqualTo(200);
}
+ @Test
+ void aSupersededRunnerCannotRecordTheMigrationAppliedUnderItsReplacement() {
+ // given
+ MongoDatabase database = freshDatabase();
+ MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database);
+ ledger.ensureIndexes();
+ MongoMigrationId id = new MongoMigrationId("20260811-011");
+ ledger.saveCheckpoint(
+ MongoMigrationCheckpoint.start(id, FIXED.instant())
+ .advancedTo("o-300", 300, FIXED.instant()),
+ 2L);
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ ledger.recordApplied(
+ id, new MongoMigrationChecksum("abc"), "operator", FIXED.instant(), 1L));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("a newer migration runner owns the lease");
+ }
+
+ @Test
+ void aReplacedRunnerCannotRecordTheMigrationBeforeItsReplacementWroteAnything() {
+ // given
+ MongoDatabase database = freshDatabase();
+ MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database);
+ ledger.ensureIndexes();
+ MongoMigrationId id = new MongoMigrationId("20260811-012");
+ Instant early = Instant.parse("2026-08-11T00:00:00Z");
+ MongoMigrationLock stalled =
+ MongoCollectionMigrationLock.tryAcquire(
+ database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC))
+ .orElseThrow();
+ var replacement =
+ MongoCollectionMigrationLock.tryAcquire(
+ database,
+ "next-runner",
+ LEASE,
+ Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC))
+ .orElseThrow();
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ ledger.recordApplied(
+ id,
+ new MongoMigrationChecksum("abc"),
+ "operator",
+ FIXED.instant(),
+ stalled.fence()));
+
+ // then
+ assertThat(refused)
+ .as("%s holds the lease with a newer fence and has not checkpointed yet", replacement)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("a newer migration runner owns the lease");
+ }
+
+ @Test
+ void aLeaseLostBeforeItsRefreshIsTheRefusalTheHeartbeatPromises() {
+ // given
+ MongoDatabase database = freshDatabase();
+ Instant early = Instant.parse("2026-08-11T00:00:00Z");
+ MongoMigrationLock stalled =
+ MongoCollectionMigrationLock.tryAcquire(
+ database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC))
+ .orElseThrow();
+ var unused =
+ MongoCollectionMigrationLock.tryAcquire(
+ database,
+ "next-runner",
+ LEASE,
+ Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC))
+ .orElseThrow();
+
+ // when
+ Throwable refused = catchThrowable(() -> stalled.refresh(LEASE));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("the migration lease was lost");
+ }
+
@Test
void theLedgerRefusesADoubleApplyOfTheSameId() {
MongoDatabase database = freshDatabase();
diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java
index 97309f29..06f1316e 100644
--- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java
+++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java
@@ -3,6 +3,7 @@ package dev.caskeleton.adapter.outbound.mongo.advanced.tenancy;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityFlags;
import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityGuard;
@@ -19,10 +20,25 @@ import dev.caskeleton.adapter.outbound.mongo.imperative.atomic.AtomicFilter;
import dev.caskeleton.adapter.outbound.mongo.migration.MongoMigrationCheckpoint;
import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoCollectionManifest;
import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoIndexManifest;
+import java.io.IOException;
+import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -37,6 +53,8 @@ class MongoTenancyContractTest {
MongoAdvancedCapabilityFlags.allDisabled()
.withEnabled(MongoCapability.DATABASE_PER_TENANT));
+ private static final Instant START = Instant.parse("2026-08-13T00:00:00Z");
+
@Test
void missingTenantContextFailsClosed() {
assertThatThrownBy(() -> injector.apply(Optional.empty(), AtomicFilter.id("o-1")))
@@ -116,29 +134,99 @@ class MongoTenancyContractTest {
@Test
void registryEnforcesMaximumActiveClients() {
+ // given
MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2);
- Instant now = Instant.parse("2026-08-13T00:00:00Z");
+ registry.acquire("t1");
+ registry.acquire("t2");
- registry.acquire("t1", now);
- registry.acquire("t2", now);
+ // when
+ Throwable refused = catchThrowable(() -> registry.acquire("t3"));
- assertThatThrownBy(() -> registry.acquire("t3", now))
- .isInstanceOf(MongoOperationRejectedException.class);
- assertThat(registry.activeCount()).isEqualTo(2);
+ // then
+ assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.activeCount()))
+ .containsExactly(true, 2);
}
@Test
void anIdleClientIsEvictedToMakeRoom() {
- MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(TENANCY, 1, Duration.ofMinutes(5));
- Instant start = Instant.parse("2026-08-13T00:00:00Z");
+ // given
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock);
+ registry.acquire("t1").close();
+ clock.advance(Duration.ofMinutes(10));
- registry.acquire("t1", start);
- registry.acquire("t2", start.plus(Duration.ofMinutes(10)));
+ // when
+ registry.acquire("t2");
+ // then
assertThat(registry.activeTenants()).containsExactly("t2");
}
+ @Test
+ void aClientIsIdleFromTheEndOfItsLastUseNotItsStart() {
+ // given
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock);
+ MongoTenantClientRegistry.Lease held = registry.acquire("t1");
+ clock.advance(Duration.ofMinutes(30));
+ held.close();
+ clock.advance(Duration.ofMinutes(1));
+
+ // when
+ Throwable refused = catchThrowable(() -> registry.acquire("t2"));
+
+ // then
+ assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.openLeases()))
+ .as("held for thirty minutes, it was in use all that time and is not idle yet")
+ .containsExactly(true, 0);
+ }
+
+ @Test
+ void anUnclosedLeaseIsCounted() {
+ // given
+ MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2);
+
+ // when
+ var unused = registry.acquire("t1");
+
+ // then
+ assertThat(registry.openLeases()).isEqualTo(1);
+ }
+
+ private static MongoTenantClientRegistry registry(
+ int cap, Function clients, Clock clock) {
+ return new MongoTenantClientRegistry(TENANCY, cap, Duration.ofMinutes(5), clients, clock);
+ }
+
+ /** A clock a test moves by hand: one time source for acquiring, closing and eviction. */
+ private static final class MovingClock extends Clock {
+
+ private final AtomicReference now;
+
+ private MovingClock(Instant start) {
+ this.now = new AtomicReference<>(start);
+ }
+
+ private void advance(Duration by) {
+ now.updateAndGet(instant -> instant.plus(by));
+ }
+
+ @Override
+ public ZoneId getZone() {
+ return ZoneOffset.UTC;
+ }
+
+ @Override
+ public Clock withZone(ZoneId zone) {
+ return this;
+ }
+
+ @Override
+ public Instant instant() {
+ return now.get();
+ }
+ }
+
@Test
void aTenantDatabaseCannotServeBeforeItsSchemaIsValidated() {
MongoTenantLifecyclePolicy policy = MongoTenantLifecyclePolicy.standard();
@@ -172,100 +260,112 @@ class MongoTenancyContractTest {
}
@Test
- @org.junit.jupiter.api.DisplayName("the same tenant is created once under concurrency")
- void theSameTenantIsCreatedOnce() throws Exception {
- java.util.concurrent.atomic.AtomicInteger created =
- new java.util.concurrent.atomic.AtomicInteger();
+ void theSameTenantIsCreatedOnceUnderConcurrency() throws Exception {
+ // given
+ AtomicInteger created = new AtomicInteger();
MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(
- TENANCY,
+ registry(
8,
- Duration.ofMinutes(10),
tenant -> {
created.incrementAndGet();
return () -> {};
- });
- java.util.concurrent.ExecutorService pool =
- java.util.concurrent.Executors.newFixedThreadPool(8);
+ },
+ new MovingClock(START));
+ List> work =
+ IntStream.range(0, 50)
+ .>mapToObj(
+ index ->
+ () -> {
+ registry.acquire("t1").close();
+ return null;
+ })
+ .toList();
+ ExecutorService pool = Executors.newFixedThreadPool(8);
+
+ // when
try {
- java.util.List> work =
- java.util.stream.IntStream.range(0, 50)
- .>mapToObj(
- index ->
- () -> {
- registry.acquire("t1", Instant.parse("2026-08-13T00:00:00Z"));
- return null;
- })
- .toList();
- for (java.util.concurrent.Future future : pool.invokeAll(work)) {
+ for (Future future : pool.invokeAll(work)) {
future.get();
}
} finally {
pool.shutdownNow();
}
- assertThat(created)
+ // then
+ assertThat(List.of(created.get(), registry.activeCount()))
.as("a per-tenant client carries a pool and monitoring threads; a second one doubles both")
- .hasValue(1);
- assertThat(registry.activeCount()).isEqualTo(1);
+ .containsExactly(1, 1);
}
@Test
- @org.junit.jupiter.api.DisplayName("acquire(tenant)는 주입된 clock으로 idle 여부를 판단한다")
- void anAcquireWithoutAnInstantReadsTheInjectedClock() {
+ void aClientStillHeldIsNotEvictedHoweverLongAgoItWasAcquired() {
// given
- java.util.List closed = new java.util.ArrayList<>();
- java.util.concurrent.atomic.AtomicReference now =
- new java.util.concurrent.atomic.AtomicReference<>(Instant.parse("2026-08-13T00:00:00Z"));
- java.time.Clock clock =
- new java.time.Clock() {
- @Override
- public java.time.ZoneId getZone() {
- return java.time.ZoneOffset.UTC;
- }
+ List closed = new ArrayList<>();
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock);
+ MongoTenantClientRegistry.Lease longRunning = registry.acquire("t1");
+ clock.advance(Duration.ofMinutes(10));
- @Override
- public java.time.Clock withZone(java.time.ZoneId zone) {
- return this;
- }
+ // when
+ Throwable refused = catchThrowable(() -> registry.acquire("t2"));
- @Override
- public Instant instant() {
- return now.get();
- }
- };
+ // then
+ assertThat(List.of(refused instanceof MongoOperationRejectedException, closed))
+ .as(
+ "t1 is in use by %s; closing it would pull the pool out from under that work",
+ longRunning)
+ .containsExactly(true, List.of());
+ }
+
+ @Test
+ void anotherTenantsClientThatFailsToCloseDoesNotFailThisAcquire() {
+ // given
+ MovingClock clock = new MovingClock(START);
MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(
- TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant), clock);
- registry.acquire("t1");
- now.set(now.get().plus(Duration.ofMinutes(10)));
+ registry(
+ 1,
+ tenant ->
+ () -> {
+ if (tenant.equals("t1")) {
+ throw new IOException("t1's pool would not shut down");
+ }
+ },
+ clock);
+ registry.acquire("t1").close();
+ clock.advance(Duration.ofMinutes(10));
+
+ // when
+ MongoTenantClientRegistry.Lease admitted = registry.acquire("t2");
+
+ // then
+ assertThat(
+ List.of(
+ admitted.client() != null,
+ registry.activeTenants(),
+ registry.evictionCloseFailures()))
+ .containsExactly(true, Set.of("t2"), 1L);
+ }
+
+ @Test
+ void evictionClosesTheClientItEvicts() {
+ // given
+ List closed = new ArrayList<>();
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock);
+ registry.acquire("t1").close();
+ clock.advance(Duration.ofMinutes(10));
// when
registry.acquire("t2");
// then
- assertThat(closed).containsExactly("t1");
- }
-
- @Test
- @org.junit.jupiter.api.DisplayName("eviction closes the client it evicts")
- void evictionClosesTheClient() {
- java.util.List closed = new java.util.ArrayList<>();
- MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(
- TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant));
- Instant start = Instant.parse("2026-08-13T00:00:00Z");
-
- registry.acquire("t1", start);
- registry.acquire("t2", start.plus(Duration.ofMinutes(10)));
-
assertThat(closed)
.as("freeing the slot without closing the client keeps the connection pool it counted")
.containsExactly("t1");
}
@Test
- @org.junit.jupiter.api.DisplayName("a coordinator resumes from its durable store after a restart")
+ @DisplayName("a coordinator resumes from its durable store after a restart")
void aCoordinatorResumesFromItsDurableStore() {
RecordingCheckpointStore store = new RecordingCheckpointStore();
MongoTenantMigrationCoordinator first =
diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java
index 89fd5ff0..bcf90f54 100644
--- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java
+++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java
@@ -3,11 +3,31 @@ package dev.caskeleton.adapter.outbound.mongo.aggregation;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import com.mongodb.MongoClientSettings;
+import dev.caskeleton.adapter.outbound.mongo.api.MongoOperationContext;
+import dev.caskeleton.adapter.outbound.mongo.api.consistency.MongoConsistencyProfile;
import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException;
+import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer;
+import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry;
+import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget;
+import java.time.Duration;
+import java.util.List;
import java.util.Set;
+import org.bson.Document;
+import org.bson.conversions.Bson;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.data.mongodb.core.MongoOperations;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.convert.MongoConverter;
+import org.springframework.data.mongodb.core.query.Criteria;
/** Design §17 — write stages are D4, and stage cost is graded rather than assumed. */
@Tag("mongodb-contract")
@@ -105,4 +125,141 @@ class PolicyAwareMongoAggregationExecutorTest {
assertThat(MongoAggregationProfile.stableRead().allowDiskUse()).isFalse();
assertThat(MongoAggregationProfile.stableRead().strictMapping()).isTrue();
}
+
+ private static final MongoOperationContext CONTEXT =
+ MongoOperationContext.of(
+ "orders.summary",
+ "primary",
+ "orders",
+ MongoConsistencyProfile.PRIMARY_MAJORITY,
+ Duration.ofSeconds(2));
+
+ private static final MongoAggregationPlan OPEN_ORDERS =
+ MongoAggregationPlan.builder()
+ .stage("$match", Aggregation.match(Criteria.where("status").is("OPEN")))
+ .build();
+
+ @Test
+ void thePipelineEndsWithALimitOnePastTheCountBudget() {
+ // given
+ ArgumentCaptor sent = ArgumentCaptor.forClass(Aggregation.class);
+ MongoOperations operations = returning(List.of(), sent);
+
+ // when
+ var unused =
+ executorOver(operations, budget(3, 1_048_576L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class);
+
+ // then
+ List pipeline = sent.getValue().toPipeline(Aggregation.DEFAULT_CONTEXT);
+ assertThat(pipeline.get(pipeline.size() - 1)).isEqualTo(new Document("$limit", 4L));
+ }
+
+ @Test
+ void moreDocumentsThanTheCountBudgetAreRefused() {
+ // given
+ MongoOperations operations =
+ returning(
+ List.of(order(1), order(2), order(3), order(4)),
+ ArgumentCaptor.forClass(Aggregation.class));
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ executorOver(operations, budget(3, 1_048_576L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("more than the budget of 3 documents");
+ }
+
+ @Test
+ void documentsLargerTogetherThanTheByteBudgetAreRefused() {
+ // given
+ // Each order encodes to well over 40 bytes: two of them pass any 80-byte budget.
+ MongoOperations operations =
+ returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class));
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ executorOver(operations, budget(10, 80L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("more than the budget of 80 bytes");
+ }
+
+ @Test
+ void resultsInsideBothBudgetsAreReturnedInCursorOrder() {
+ // given
+ MongoOperations operations =
+ returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class));
+
+ // when
+ List results =
+ executorOver(operations, budget(10, 1_048_576L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class);
+
+ // then
+ assertThat(results).containsExactly(order(1), order(2));
+ }
+
+ private static MongoOperationBudget budget(int maxResults, long maxResultBytes) {
+ return new MongoOperationBudget(maxResults, maxResultBytes, 500L, 50);
+ }
+
+ private static PolicyAwareMongoAggregationExecutor executorOver(
+ MongoOperations operations, MongoOperationBudget budget) {
+ return new PolicyAwareMongoAggregationExecutor(
+ operations,
+ MongoBudgetPolicyRegistry.builder().register("orders.summary", budget).build(),
+ new MongoBudgetEnforcer());
+ }
+
+ /** The driver boundary: a cursor over {@code documents}, mapped as the converter would. */
+ private static MongoOperations returning(
+ List documents, ArgumentCaptor sent) {
+ MongoOperations operations = mock(MongoOperations.class);
+ MongoConverter converter = mock(MongoConverter.class);
+ when(operations.getConverter()).thenReturn(converter);
+ when(converter.getCodecRegistry()).thenReturn(MongoClientSettings.getDefaultCodecRegistry());
+ when(converter.read(eq(Document.class), any(Bson.class)))
+ .thenAnswer(invocation -> invocation.getArgument(1));
+ when(operations.aggregateStream(sent.capture(), eq("orders"), eq(Document.class)))
+ .thenAnswer(invocation -> documents.stream());
+ return operations;
+ }
+
+ private static Document order(int number) {
+ return new Document("_id", "order-" + number)
+ .append("status", "OPEN")
+ .append("customer", "customer-" + number);
+ }
}
diff --git a/src/adapter/outbound/persistence-notification/AGENTS.md b/src/adapter/outbound/persistence-notification/AGENTS.md
index 94fa03ad..cb9ae8b1 100644
--- a/src/adapter/outbound/persistence-notification/AGENTS.md
+++ b/src/adapter/outbound/persistence-notification/AGENTS.md
@@ -8,7 +8,8 @@
- 패키지 루트: `dev.caskeleton.adapter.outbound.persistence.notification`. 패키지 규칙은
`persistence-fileserver`와 같아서 두 capability leaf를 같은 방식으로 읽는다:
- `platform.entity` — JPA 행만.
- - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`).
+ - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). `platform`의 store도
+ 이 SQL 조각을 쓴다.
- `platform` — store 어댑터(`Jpa*`, `Jdbc*`, inbox의 `JpaNotificationInbox`와 그 outbox 도우미),
`NotificationRecordMapper`, payload codec.
- `configuration` — store를 bean으로 등록하는 자동 구성 창구. leaf 루트에는 scan 설정과 schema stream이 있다.
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java
index 5a81e94b..72a52a8e 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java
@@ -1,13 +1,17 @@
package dev.caskeleton.adapter.outbound.persistence.notification.platform;
+import dev.caskeleton.adapter.outbound.persistence.notification.platform.repository.RecipientClaimSql;
import dev.caskeleton.application.notification.platform.observation.NotificationServingState;
import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort;
import java.sql.Timestamp;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.Map;
import java.util.Objects;
import org.springframework.jdbc.core.JdbcOperations;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
/**
* Measures the platform's serving state from the durable rows.
@@ -17,34 +21,44 @@ import org.springframework.jdbc.core.JdbcOperations;
* identical. These are the questions an operator actually asks during an incident: how far behind
* are we, how long has the oldest thing waited, and is anything stuck.
*
- *
One statement per question rather than one joined statement. They are read independently, a
- * slow one does not delay the others, and each index the queries rely on is the same index the
- * corresponding worker already uses to claim.
+ *
One statement per question rather than one joined statement. They are read independently and a
+ * slow one does not delay the others. Every statement is bounded by an index: the backlog and
+ * stuck-lease counts by the predicates the claim uses, the reconciliation count by the job's due
+ * time, and the projection lag by {@code ix_notification_provider_event_open} (V14), which holds
+ * only the events still waiting for something — the provider-event table itself is never purged, so
+ * the lag query used to read all of it (round9 review 2026-11-12, RR9-01-14).
+ *
+ *
The scheduler reads only {@link #backlogDepth()}, on every poll; the full snapshot is for the
+ * health endpoint.
*/
public final class JdbcNotificationServingState implements NotificationServingStatePort {
/**
- * Deliveries that are due and unclaimed.
+ * Deliveries the claim would take now.
*
- *
The lease predicate matches the claim's: a row whose lease has expired is available again,
- * so it is backlog. Counting it as in-flight is how a stuck worker hides a growing queue.
+ *
The {@code WHERE} is the claim's own fragment ({@link RecipientClaimSql#CLAIMABLE_AT_NOW}):
+ * the same states, due, not expired, and a lease that is absent or has run out. A row whose lease
+ * has expired is available again, so it is backlog; counting it as in-flight is how a stuck
+ * worker hides a growing queue. An expired delivery is not backlog: the expiry sweep retires it
+ * unsent. The fragment's parameter is named, so this statement is too.
*/
private static final String BACKLOG =
"""
SELECT count(*) AS depth,
- COALESCE(EXTRACT(EPOCH FROM (? - min(next_dispatch_at))), 0) AS oldest_seconds
+ COALESCE(EXTRACT(EPOCH FROM (:now - min(next_dispatch_at))), 0) AS oldest_seconds
FROM notification_recipient_delivery
- WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')
- AND next_dispatch_at IS NOT NULL
- AND next_dispatch_at <= ?
- AND (lease_until IS NULL OR lease_until <= ?)
- """;
+ WHERE\s"""
+ + RecipientClaimSql.CLAIMABLE_AT_NOW;
/**
* Leases whose holder never came back.
*
- *
Distinct from backlog: these rows are claimable again, but the fact that they had to expire
- * means a worker died mid-dispatch, and a provider call may have happened without a record.
+ *
Distinct from backlog: these rows are still dispatching, which the claim never takes; lease
+ * recovery requeues or reconciles them one lease after they ran out. A row left from before
+ * notification V4 (an owner but fence 0) is counted here and not recovered; {@code
+ * docs/upgrade/2026-11-round9-claim-fence.md}, row 5, says how to hand it to recovery. That a
+ * lease had to run out means a worker died or stalled mid-dispatch, and a provider call may have
+ * happened without a record.
*/
private static final String STUCK_LEASES =
"""
@@ -56,7 +70,21 @@ public final class JdbcNotificationServingState implements NotificationServingSt
AND delivery_state = 'DISPATCHING'
""";
- /** Provider events stored and not yet applied to a delivery. */
+ /**
+ * The provider events still waiting for something: the lag query's {@code WHERE}, and the
+ * predicate of the partial index {@code V14} builds for it. One string, held to the migration by
+ * {@code JdbcNotificationServingStateOpenEventsTest}: a query the index does not cover reads the
+ * whole table again, which is what the index was made to stop.
+ */
+ static final String OPEN_EVENTS =
+ "projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL";
+
+ /**
+ * Provider events stored and not yet applied to a delivery.
+ *
+ *
The {@code WHERE} is exactly the predicate of {@code ix_notification_provider_event_open}:
+ * every row any of the four counts can see, and none of the applied, matched history.
+ */
private static final String PROJECTION_LAG =
"""
SELECT
@@ -66,7 +94,9 @@ public final class JdbcNotificationServingState implements NotificationServingSt
COALESCE(EXTRACT(EPOCH FROM (? - min(received_at)
FILTER (WHERE projection_status = 'PENDING'))), 0) AS oldest_seconds
FROM notification_provider_event
- """;
+ WHERE
+ """
+ + OPEN_EVENTS;
/** Questions to providers that are due and unanswered. */
private static final String RECONCILIATION_LAG =
@@ -78,26 +108,36 @@ public final class JdbcNotificationServingState implements NotificationServingSt
""";
private final JdbcOperations jdbc;
+ private final NamedParameterJdbcOperations named;
private final Clock clock;
public JdbcNotificationServingState(JdbcOperations jdbc, Clock clock) {
this.jdbc = Objects.requireNonNull(jdbc, "jdbc");
+ this.named = new NamedParameterJdbcTemplate(jdbc);
this.clock = Objects.requireNonNull(clock, "clock");
}
+ @Override
+ public long backlogDepth() {
+ Measurement backlog = measureBacklog(Timestamp.from(clock.instant()));
+ return backlog == null ? 0 : backlog.count();
+ }
+
+ /** The backlog query alone: the one statement the scheduler's queue-depth gauge needs. */
+ private Measurement measureBacklog(Timestamp at) {
+ return named.queryForObject(
+ BACKLOG,
+ Map.of("now", at),
+ (rows, rowNumber) ->
+ new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")));
+ }
+
@Override
public NotificationServingState currentState() {
Instant now = clock.instant();
Timestamp at = Timestamp.from(now);
- Measurement backlog =
- jdbc.queryForObject(
- BACKLOG,
- (rows, rowNumber) ->
- new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")),
- at,
- at,
- at);
+ Measurement backlog = measureBacklog(at);
Long stuck = jdbc.queryForObject(STUCK_LEASES, Long.class, at);
ProjectionMeasurement projection =
jdbc.queryForObject(
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java
index 23fdb2dc..a2da23c2 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java
@@ -99,6 +99,23 @@ public final class JpaRecipientDeliveryStore implements RecipientDeliveryStorePo
return reread(written, record.id());
}
+ @Override
+ public Optional requeueAbandoned(
+ RecipientLease abandoned, Instant nextDispatchAt) {
+ CallerTransaction.require("recipient delivery requeueAbandoned");
+ Objects.requireNonNull(abandoned, "abandoned");
+ Objects.requireNonNull(nextDispatchAt, "nextDispatchAt");
+ int written =
+ recipients.requeueAbandoned(
+ abandoned.recipientDeliveryId().value(),
+ abandoned.owner(),
+ abandoned.fence(),
+ abandoned.leaseUntil(),
+ nextDispatchAt,
+ clock.instant());
+ return reread(written, abandoned.recipientDeliveryId());
+ }
+
@Override
public Optional transitionHeldBy(
RecipientDeliveryId id,
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java
index 65784bac..98cd4bef 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java
@@ -5,9 +5,11 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction
import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
import dev.caskeleton.application.notification.platform.dispatch.RecipientLease;
import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort;
+import java.sql.Timestamp;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.time.OffsetDateTime;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -109,13 +111,38 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort {
}
@Override
- public List expiredDispatching(int limit, Duration olderThan) {
+ public List expiredDispatching(int limit, Duration olderThan) {
Objects.requireNonNull(olderThan, "olderThan");
if (limit < 1) {
throw new IllegalArgumentException("limit");
}
return recipients.selectExpiredDispatching(clock.instant().minus(olderThan), limit).stream()
- .map(RecipientDeliveryId::new)
+ .map(
+ row ->
+ new RecipientLease(
+ new RecipientDeliveryId((UUID) row[0]),
+ (String) row[1],
+ ((Number) row[2]).longValue(),
+ instantOf(row[3])))
.toList();
}
+
+ /**
+ * A {@code timestamptz} column of a native row. Hibernate hands it back as an {@link
+ * OffsetDateTime} or a {@link Timestamp} depending on its JDBC type mapping, never null here
+ * because the select only returns rows that have a lease.
+ */
+ private static Instant instantOf(Object column) {
+ if (column instanceof Instant instant) {
+ return instant;
+ }
+ if (column instanceof OffsetDateTime offset) {
+ return offset.toInstant();
+ }
+ if (column instanceof Timestamp timestamp) {
+ return timestamp.toInstant();
+ }
+ throw new IllegalStateException(
+ "lease_until came back as " + (column == null ? "null" : column.getClass().getName()));
+ }
}
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java
index 15428027..e7ccfb32 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java
@@ -60,18 +60,16 @@ public interface InboxItemJpaRepository extends Repository{@code read_at IS NULL} in the predicate is what makes the update idempotent under
* concurrency: a second caller updates zero rows instead of overwriting the first caller's
* timestamp.
- */
- /**
- * Marks everything up to and including a cursor position as read.
*
*
The cutoff is a tuple, matching the list's own {@code (created_at DESC, id DESC)} ordering.
* It used to be a timestamp alone, and the list's order is not a total order on {@code
@@ -95,14 +93,23 @@ public interface InboxItemJpaRepository extends RepositoryThe lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code
* PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and
* the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The
- * named parameters are Spring Data's.
+ * named parameters are bound by Spring Data for the repository statements, and by a {@code
+ * NamedParameterJdbcTemplate} where the serving state reads {@link #CLAIMABLE_AT_NOW}.
*/
public final class RecipientClaimSql {
+ /**
+ * The states a delivery waits in to be sent, which the claim takes from, the expiry retires from
+ * and the serving state counts as backlog. One list so the statements cannot disagree about what
+ * is waiting.
+ */
+ public static final String CLAIMABLE_STATES = "('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')";
+
+ /**
+ * Which deliveries the claim takes at {@code :now}: due, in a waiting state, not expired, and
+ * with no lease or one that has run out. The serving state counts its backlog with this same
+ * fragment, so the backlog is exactly what a claim would take.
+ */
+ public static final String CLAIMABLE_AT_NOW =
+ "next_dispatch_at <= :now"
+ + " AND delivery_state IN "
+ + CLAIMABLE_STATES
+ + " AND (expires_at IS NULL OR expires_at > :now)"
+ + " AND (lease_until IS NULL OR lease_until < :now)";
+
/**
* Claims due deliveries and returns their fences.
*
@@ -28,10 +48,8 @@ public final class RecipientClaimSql {
public static final String CLAIM_BATCH =
"WITH claimable AS ("
+ " SELECT id FROM notification_recipient_delivery"
- + " WHERE next_dispatch_at <= :now"
- + " AND delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"
- + " AND (expires_at IS NULL OR expires_at > :now)"
- + " AND (lease_until IS NULL OR lease_until < :now)"
+ + " WHERE "
+ + CLAIMABLE_AT_NOW
+ " ORDER BY next_dispatch_at, id"
+ " FOR UPDATE SKIP LOCKED"
+ " LIMIT :batchSize"
@@ -51,7 +69,18 @@ public final class RecipientClaimSql {
+ " WHERE d.id = claimable.id"
+ " RETURNING d.id, d.lease_fence";
- /** Retires deliveries whose window closed before anyone claimed them. */
+ /**
+ * Retires deliveries whose window closed before anyone claimed them.
+ *
+ *
A delivery a worker claimed while this statement ran must not be retired to {@code EXPIRED}
+ * under the worker dispatching it, which is what the unlocked subquery did (round9 review
+ * 2026-11-12, RR9-01-13). Two things stop it. The subquery locks the rows it picks and skips rows
+ * another transaction holds, so a delivery being claimed is not picked. The state and expiry
+ * conditions are also on the {@code UPDATE} itself: under READ COMMITTED PostgreSQL re-checks
+ * only the outer {@code WHERE} against a row it had to wait for, so if the lock clause is ever
+ * lost the claimed row is still left alone. A delivery being claimed right now is not expiring
+ * either: the claim refuses expired rows in the same statement.
+ */
public static final String EXPIRE_OVERDUE =
"UPDATE notification_recipient_delivery SET"
+ " delivery_state = 'EXPIRED',"
@@ -61,11 +90,17 @@ public final class RecipientClaimSql {
+ " updated_at = :now"
+ " WHERE id IN ("
+ " SELECT id FROM notification_recipient_delivery"
- + " WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"
+ + " WHERE delivery_state IN "
+ + CLAIMABLE_STATES
+ " AND expires_at IS NOT NULL"
+ " AND expires_at <= :now"
+ " ORDER BY expires_at"
- + " LIMIT :batchSize)";
+ + " FOR UPDATE SKIP LOCKED"
+ + " LIMIT :batchSize)"
+ + " AND delivery_state IN "
+ + CLAIMABLE_STATES
+ + " AND expires_at IS NOT NULL"
+ + " AND expires_at <= :now";
private RecipientClaimSql() {}
}
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java
index b5f1ee18..1240cf0f 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java
@@ -183,13 +183,48 @@ public interface RecipientDeliveryJpaRepository extends RepositoryOnly rows a claim wrote: an owner and a fence of at least one. The requeue matches on both,
+ * and a row without them — one left dispatching before notification V4 added the fence — is no
+ * lease anyone holds. Read here it would be a lease the application refuses to build, and the one
+ * row would fail every recovery pass after it.
+ *
+ * @return one row per delivery: id, lease_owner, lease_fence, lease_until
+ */
@Query(
value =
- "SELECT id FROM notification_recipient_delivery "
+ "SELECT id, lease_owner, lease_fence, lease_until FROM notification_recipient_delivery "
+ "WHERE delivery_state = 'DISPATCHING' AND lease_until < :threshold "
+ + "AND lease_owner IS NOT NULL AND lease_fence > 0 "
+ "ORDER BY lease_until LIMIT :batchSize",
nativeQuery = true)
- List selectExpiredDispatching(
+ List