diff --git a/.env.example b/.env.example index 1c68b40e..09bb6b74 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,7 @@ APP_CACHE_REDIS_COMMAND_TIMEOUT=200ms APP_CACHE_REDIS_POSITIVE_SOFT_TTL=30s APP_CACHE_REDIS_MINIMUM_HARD_TTL=1s APP_CACHE_REDIS_SEMANTIC_REGION=default +APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES=1048576 APP_MESSAGING_BROKER=kafka APP_MESSAGING_PRODUCER_ID= APP_MESSAGING_KAFKA_BROKERS= @@ -293,7 +294,7 @@ APP_NOTIFICATION_PLATFORM_SMTP_ENABLED=false APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY=true APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT=local APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE=default -APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=10s +APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=30s APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY=4 APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND=10 APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY=no-reply@example.invalid diff --git a/config/spotbugs/exclude.xml b/config/spotbugs/exclude.xml index 2ee07473..156d6eae 100644 --- a/config/spotbugs/exclude.xml +++ b/config/spotbugs/exclude.xml @@ -359,6 +359,15 @@ + + + + + + + } and followed by a {@code sql} * fence. The lanes that run these blocks against real PostgreSQL read them from the document rather diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java index 98739e4d..a483b2fe 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java @@ -1,18 +1,34 @@ package dev.caskeleton.adapter.outbound.persistence.autoconfigure; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay; import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper; import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.boot.context.properties.bind.UnboundConfigurationPropertiesException; import org.springframework.boot.convert.ApplicationConversionService; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.scheduling.config.FixedDelayTask; +import org.springframework.scheduling.config.ScheduledTask; +import org.springframework.scheduling.config.ScheduledTaskHolder; class JpaLiveEventReplayActivationTest { @@ -34,6 +50,110 @@ class JpaLiveEventReplayActivationTest { }); } + @Test + void theReaperIsAssembledWithTheReplayStore() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H") + .run( + context -> + assertThat(context) + .as("without the reaper nothing removes a row past its retention") + .hasSingleBean(LiveEventReaper.class)); + } + + @Test + void noReaperIsAssembledWithoutTheReplayStore() { + runner.run(context -> assertThat(context).doesNotHaveBean(LiveEventReaper.class)); + } + + @Test + void aZeroReaperIntervalFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-interval=PT0S") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .hasMessageContaining("app.realtime.replay.reaper-interval must be positive")); + } + + @Test + void aMisspeltReplayKeyFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-intervall=PT1M") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .isInstanceOf(UnboundConfigurationPropertiesException.class) + .hasMessageContaining("reaper-intervall")); + } + + @ParameterizedTest(name = "[{index}] reaper-interval {0} → {1}") + @CsvSource( + value = {"UNSET, PT10M", "PT3M, PT3M"}, + nullValues = "UNSET") + @DisplayName("reaper는 묶인 reaper-interval(기본 PT10M)마다 돌고, 첫 회도 그 간격 뒤다") + void theReaperRunsAtTheBoundInterval(String configured, Duration expected) { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + if (configured != null) { + replay = replay.withPropertyValues("app.realtime.replay.reaper-interval=" + configured); + } + + // when + List> schedules = new ArrayList<>(); + replay.run(context -> schedules.addAll(fixedDelaySchedules(context))); + + // then + assertThat(schedules) + .as("the schedule is the bound settings' interval, its default included") + .containsExactly(List.of(expected, expected)); + } + + @Test + void theScheduledTaskIsTheReaper() { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + + // when + List reapedThrough = new ArrayList<>(); + replay.run( + context -> { + fixedDelayTasks(context).forEach(task -> task.getRunnable().run()); + reapedThrough.add(context.getBean(LiveEventJpaRepository.class)); + }); + + // then + verify(reapedThrough.getFirst()).deleteExpiredBatch(any(Instant.class), anyInt()); + } + + /** Interval and initial delay of every fixed-delay task the context scheduled. */ + private static List> fixedDelaySchedules(ApplicationContext context) { + return fixedDelayTasks(context).stream() + .map(task -> List.of(task.getIntervalDuration(), task.getInitialDelayDuration())) + .toList(); + } + + private static List fixedDelayTasks(ApplicationContext context) { + return context.getBean(ScheduledTaskHolder.class).getScheduledTasks().stream() + .map(ScheduledTask::getTask) + .filter(FixedDelayTask.class::isInstance) + .map(FixedDelayTask.class::cast) + .toList(); + } + @Test void enabledReplayCapabilityRequiresExplicitRetention() { runner @@ -42,7 +162,8 @@ class JpaLiveEventReplayActivationTest { context -> { assertThat(context).hasFailed(); assertThat(context.getStartupFailure()) - .hasMessageContaining("app.realtime.replay.retention"); + .rootCause() + .hasMessageContaining("app.realtime.replay.retention must be set"); }); } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java index a560adf4..74694d1f 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.experimental; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.experimental.database.TenantPoolBudget; import dev.caskeleton.adapter.outbound.persistence.experimental.replica.ConsistencyAwareDataSourceRouter; @@ -103,14 +104,28 @@ class ExperimentalFailClosedTest { void tenantPoolBudgetIsBounded() { var budget = new TenantPoolBudget(2, 20); - assertThat(budget.hasCapacity(1, 10)).isTrue(); - assertThatThrownBy(() -> budget.requireCapacity(2, 10)) + assertThat(budget.hasCapacity(1, 10, 10)).isTrue(); + assertThatThrownBy(() -> budget.requireCapacity(2, 10, 1)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("tenant pool budget"); - assertThatThrownBy(() -> budget.requireCapacity(1, 20)) + assertThatThrownBy(() -> budget.requireCapacity(1, 20, 1)) .isInstanceOf(IllegalStateException.class); } + @Test + void aNewPoolLargerThanTheConnectionsLeftIsRefused() { + // given + var budget = new TenantPoolBudget(4, 20); + + // when + Throwable refused = catchThrowable(() -> budget.requireCapacity(1, 15, 10)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("tenant pool budget exhausted"); + } + /** A lag monitor with a fixed answer, including "I do not know". */ private record FixedLagMonitor(Optional replayedThrough, Optional lag) implements ReplicaLagMonitor {} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java new file mode 100644 index 00000000..1fb7f801 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java @@ -0,0 +1,45 @@ +package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.Mockito.mock; + +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import java.time.Duration; +import java.util.Set; +import org.hibernate.SessionFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The stateless runner's transaction bound is the write bound every other transaction here has. + * + *

The runner restates the value instead of reading it, because the {@code hibernate} package may + * not depend on {@code transaction} (JpaModuleBoundaryTest). A test may depend on both, so it is + * the place the two copies are held to each other (round9 review 2026-11-12, RR9-01-25). + */ +class StatelessRunnerTimeoutTest { + + @Test + void theRunnersDefaultIsSpringTransactionsWriteDefault() { + assertThat(HibernateStatelessSessionRunner.DEFAULT_TIMEOUT) + .isEqualTo(SpringTransaction.Timeouts.DEFAULT.write()); + } + + @ParameterizedTest + @ValueSource(strings = {"PT0S", "PT-1S", "PT1.5S", "PT2147483648S"}) + void aTimeoutThatIsNotAWholePositiveNumberOfSecondsIsTheAssemblysFault(String timeout) { + // when + Throwable refused = + catchThrowable( + () -> + new HibernateStatelessSessionRunner( + mock(SessionFactory.class), Set.of(), Duration.parse(timeout))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("must be a positive whole number of seconds"); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java index 72235d67..e80b313d 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java @@ -2,23 +2,61 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.time.Clock; -import java.time.Instant; -import java.time.ZoneOffset; import org.junit.jupiter.api.Test; +/** + * Unit tests for {@link IdempotencyReaper}: the delete runs in bounded batches until one comes back + * short, as the outbox reaper's does. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement itself — + * expiry judged by the database clock — runs against PostgreSQL in {@code + * IdempotencyReaperIntegrationTest}. + */ class IdempotencyReaperTest { - private static final Instant NOW = Instant.parse("2026-06-09T12:00:00Z"); + private static final int BATCH = IdempotencyReaper.BATCH_SIZE; + + private final IdempotencyRecordJpaRepository repository = + mock(IdempotencyRecordJpaRepository.class); + private final IdempotencyReaper reaper = new IdempotencyReaper(repository); @Test - void reapDeletesRecordsExpiredAsOfNow() { - IdempotencyRecordJpaRepository repo = mock(IdempotencyRecordJpaRepository.class); - when(repo.deleteExpired(NOW)).thenReturn(3); - IdempotencyReaper reaper = new IdempotencyReaper(repo, Clock.fixed(NOW, ZoneOffset.UTC)); + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); - assertThat(reaper.reap()).isEqualTo(3); + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(2 * BATCH + 3); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(3)).deleteExpiredBatch(BATCH); + } + + @Test + void reapReturnsZeroWhenNothingHasExpired() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(0); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isZero(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java new file mode 100644 index 00000000..b43a316e --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java @@ -0,0 +1,53 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LiveEventReaper}: rows retained no later than the clock's now go, in + * bounded batches until one comes back short. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement runs + * against PostgreSQL in {@code JpaLiveEventReplayContractTest}. + */ +class LiveEventReaperTest { + + private static final Instant NOW = Instant.parse("2026-11-13T00:00:00Z"); + private static final int BATCH = LiveEventReaper.BATCH_SIZE; + + private final LiveEventJpaRepository repository = mock(LiveEventJpaRepository.class); + private final LiveEventReaper reaper = + new LiveEventReaper(repository, Clock.fixed(NOW, ZoneOffset.UTC)); + + @Test + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(BATCH + 4); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(2)).deleteExpiredBatch(NOW, BATCH); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java index 0d647c57..b6ed1ca0 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java @@ -78,6 +78,20 @@ class DistributedLockPersistenceConfigTest { }); } + @Test + @DisplayName("읽는 곳이 없던 app.lock.wait-time은 모르는 키로 기동을 실패시킨다") + void theRemovedWaitTimeKeyFailsStartup() { + // when + contextRunner + .withPropertyValues("app.lock.wait-time=3s") + .run( + context -> { + // then + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("wait-time"); + }); + } + @Test void multiInstanceBuildsTheJdbcProviderAndNotTheInProcessProvider() { contextRunner diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index 1eb57d14..93a51fe4 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -14,6 +14,7 @@ be51fabd07220f9ef885534ba36ada5f1491baf24cdfb54d25d93a19ddc29926 jpa/notification-platform/V11__notification_reconciliation_claim_lease.sql 43f9429f91c07528154ec01b721649b2c25fa98857ac154c7c08f4655634657f jpa/notification-platform/V12__notification_suppression_target_unique.sql d01aaf59f66834358abf18246faa9a9eb8dfa9f42097ca9d968c44edff564630 jpa/notification-platform/V13__notification_request_acceptance.sql +8a7dc3604d44f1438b39dfec3aa688f792e7363cd3c1b74bf0c799a6b53f9f0d jpa/notification-platform/V14__notification_provider_event_open_index.sql 1ac7cc1c2509868eb0ce202e1917bb36846f98f17ad4922abe3fac0ef128199b jpa/notification-platform/V1__notification_platform_core.sql 0add67aa23580e51712f84fabe9b09e08401ef708cfa1b3292b120fdb5c12fa2 jpa/notification-platform/V2__notification_platform_contact_template_policy.sql f5b98598a59cc4376f5d233b410f8a5ae9f3cd94156291ebdf2d0da5a66f34ef jpa/notification-platform/V3__notification_platform_inbox_admin.sql @@ -42,6 +43,7 @@ a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19 25db296e55444636c8a017d9b736eecf4a839a9caf1733c0c4a5f37bae5a3161 postgresql/V20__outbox_event_deferred_attempt_count.sql 21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql 20da5649a29bf2305f0e04ab14003b21b7f120d867439fb01f1658c98179dba0 postgresql/V22__outbox_event_last_failure_code.sql +9492ace50ab828144792b77c72d92fce50bec29a028ebc917f7e55facf58d85a postgresql/V23__durable_operation_lease_fence.sql 8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql 6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java index ef80fdb1..a435e5b5 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java @@ -7,9 +7,14 @@ import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedExc import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; /** * Bounds how many tenant clients exist at once (advanced plan Task 11). @@ -18,6 +23,13 @@ import java.util.Set; * connection pool and its own monitoring threads, so a thousand tenants is a thousand pools. The * registry caps the number and evicts idle ones, and refuses rather than exceeding the cap — an * unbounded registry fails later, as connection exhaustion on an unrelated request. + * + *

{@link #acquire} hands back a {@link Lease}; the client is in use until every lease on it is + * closed, and only a client nobody holds is idle. Acquire used to return nothing, so idle meant + * "acquired longest ago" and eviction could close a client a long-running caller was still using. + * Closing an evicted client is this registry's own clean-up: if it fails, the tenant whose acquire + * triggered the eviction is not told about another tenant's client — the failure is counted in + * {@link #evictionCloseFailures()} (round9 review 2026-11-12, RR9-01-27). */ @MongoAdvancedEntryPoint(MongoCapability.DATABASE_PER_TENANT) public final class MongoTenantClientRegistry { @@ -26,11 +38,21 @@ public final class MongoTenantClientRegistry { private final Duration idleEviction; - private final Map lastUsed = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map lastUsed = new ConcurrentHashMap<>(); - private final Map clients = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map clients = new ConcurrentHashMap<>(); - private final java.util.function.Function clientFactory; + private final Map leases = new HashMap<>(); + + /** + * Guards the cap, the lease counts and the eviction as one decision. A private monitor rather + * than the registry itself, so code outside cannot take it and stall every acquire. + */ + private final Object lock = new Object(); + + private final AtomicLong evictionCloseFailures = new AtomicLong(); + + private final Function clientFactory; private final Clock clock; @@ -47,15 +69,15 @@ public final class MongoTenantClientRegistry { MongoAdvancedCapabilityGuard guard, int maximumActiveClients, Duration idleEviction, - java.util.function.Function clientFactory) { + Function clientFactory) { this(guard, maximumActiveClients, idleEviction, clientFactory, Clock.systemUTC()); } /** * A registry that owns real clients. * - *

Idle eviction reads {@code clock}; {@link #acquire(String)} used to read {@code - * Instant.now()} beside it. + *

Acquiring, closing a lease and idle eviction all read {@code clock}; {@link + * #acquire(String)} used to read {@code Instant.now()} beside it. * *

The factory is called at most once per tenant, inside the map's own computation, so two * threads asking for the same tenant get one client rather than two — a per-tenant client carries @@ -65,7 +87,7 @@ public final class MongoTenantClientRegistry { MongoAdvancedCapabilityGuard guard, int maximumActiveClients, Duration idleEviction, - java.util.function.Function clientFactory, + Function clientFactory, Clock clock) { Objects.requireNonNull(guard, "guard").require(MongoTenantClientRegistry.class); this.clientFactory = Objects.requireNonNull(clientFactory, "clientFactory"); @@ -80,41 +102,128 @@ public final class MongoTenantClientRegistry { /** * Acquires a client for a tenant. * + * @return the lease to close when the work with the client is done * @throws MongoOperationRejectedException when the cap is reached and nothing can be evicted */ - public void acquire(String tenantKey) { - acquire(tenantKey, clock.instant()); + public Lease acquire(String tenantKey) { + Objects.requireNonNull(tenantKey, "tenantKey"); + synchronized (lock) { + return acquireHeld(tenantKey); + } } - /** Acquires a client for a tenant at an explicit instant, so eviction is testable. */ - public synchronized void acquire(String tenantKey, Instant now) { - Objects.requireNonNull(tenantKey, "tenantKey"); - Objects.requireNonNull(now, "now"); + private Lease acquireHeld(String tenantKey) { + // One time source for the acquire, the close and the eviction: an overload that took the + // instant as an argument let a caller acquire in one timeline and close in the clock's. + Instant now = clock.instant(); // Synchronized rather than a bare ConcurrentHashMap: the cap, the eviction and the create are // one decision. Checking the size on one thread while another inserts is how a cap of ten ends // up holding twelve connection pools. - if (lastUsed.containsKey(tenantKey)) { - lastUsed.put(tenantKey, now); - return; + if (!lastUsed.containsKey(tenantKey)) { + evictIdle(now); + if (lastUsed.size() >= maximumActiveClients) { + throw MongoOperationRejectedException.of( + "tenancy.client", + "the tenant client cap of " + + maximumActiveClients + + " is reached and no client is idle; each tenant client carries its own" + + " connection pool and monitoring threads, so the cap is what stops one process" + + " from exhausting the cluster's connections"); + } + clients.computeIfAbsent(tenantKey, clientFactory); } - evictIdle(now); - if (lastUsed.size() >= maximumActiveClients) { - throw MongoOperationRejectedException.of( - "tenancy.client", - "the tenant client cap of " - + maximumActiveClients - + " is reached and no client is idle; each tenant client carries its own connection " - + "pool and monitoring threads, so the cap is what stops one process from exhausting " - + "the cluster's connections"); - } - clients.computeIfAbsent(tenantKey, clientFactory); lastUsed.put(tenantKey, now); + leases.merge(tenantKey, 1, Integer::sum); + return new HeldLease(tenantKey, clients.get(tenantKey)); } - /** Releases a tenant's client and closes it. */ - public synchronized void release(String tenantKey) { - lastUsed.remove(Objects.requireNonNull(tenantKey, "tenantKey")); - close(tenantKey); + /** + * Releases a tenant's client and closes it. + * + * @throws IllegalStateException when a lease on the client is still open, or the close fails + */ + public void release(String tenantKey) { + Objects.requireNonNull(tenantKey, "tenantKey"); + synchronized (lock) { + if (leases.containsKey(tenantKey)) { + throw new IllegalStateException( + "the client for tenant '" + tenantKey + "' is still in use and cannot be released"); + } + lastUsed.remove(tenantKey); + close(tenantKey); + } + } + + /** + * How many evicted clients failed to close. + * + *

The registry has dropped each of them and freed its slot; the count is how an operator + * learns that clients — and their pools — may still be open behind it. + */ + public long evictionCloseFailures() { + return evictionCloseFailures.get(); + } + + /** + * One use of a tenant client; close it when the work with that client is done. + * + *

An interface so the registry's own implementation stays private: only the registry makes a + * lease, and only through {@link #acquire}. + */ + public interface Lease extends AutoCloseable { + + /** The tenant's client, valid until this lease is closed. */ + AutoCloseable client(); + + /** Ends this use. Closing twice is the same as closing once. */ + @Override + void close(); + } + + private final class HeldLease implements Lease { + + private final String tenantKey; + + private final AutoCloseable client; + + private boolean closed; + + private HeldLease(String tenantKey, AutoCloseable client) { + this.tenantKey = tenantKey; + this.client = client; + } + + @Override + public AutoCloseable client() { + return client; + } + + @Override + public void close() { + synchronized (lock) { + if (closed) { + return; + } + closed = true; + leases.computeIfPresent(tenantKey, (tenant, open) -> open == 1 ? null : open - 1); + // Idle from the end of the last use, not its start: a client held for half an hour was in + // use all that time, and measuring from the acquire evicted it the moment it was let go. + Instant released = clock.instant(); + lastUsed.computeIfPresent( + tenantKey, (tenant, used) -> released.isAfter(used) ? released : used); + } + } + } + + /** + * How many leases are open now, across all tenants. + * + *

A lease nobody closed keeps its client's slot for good; this is how that becomes visible. + */ + public int openLeases() { + synchronized (lock) { + return leases.values().stream().mapToInt(Integer::intValue).sum(); + } } /** The tenants with an active client. */ @@ -128,8 +237,9 @@ public final class MongoTenantClientRegistry { } private void evictIdle(Instant now) { - java.util.List idle = + List idle = lastUsed.entrySet().stream() + .filter(entry -> !leases.containsKey(entry.getKey())) .filter(entry -> entry.getValue().plus(idleEviction).isBefore(now)) .map(Map.Entry::getKey) .toList(); @@ -138,7 +248,11 @@ public final class MongoTenantClientRegistry { lastUsed.remove(tenant); // Evicting the bookkeeping entry without closing the client frees the slot and keeps the // connection pool, which is the exhaustion this cap exists to prevent, delayed. - close(tenant); + try { + close(tenant); + } catch (IllegalStateException closeFailed) { + evictionCloseFailures.incrementAndGet(); + } }); } diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java index 8ed663b7..0f0a91d4 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java @@ -6,20 +6,30 @@ import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer; import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry; import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget; import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.stream.Stream; +import org.bson.Document; +import org.bson.RawBsonDocument; +import org.bson.codecs.Codec; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.AggregationOperation; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; -import org.springframework.data.mongodb.core.aggregation.AggregationResults; /** * Runs an aggregation only after its stages, budget and lookups have been checked (design §17). * *

Every guard runs before the first stage reaches the server, because an aggregation that is - * going to be refused should cost nothing. The one guard that runs afterwards is the result-count - * check: {@code $limit} bounds what the pipeline emits, but a pipeline whose own shape produced - * more than the budget allows has still told the caller something worth failing over. + * going to be refused should cost nothing. Two run while the results arrive: the result count and + * the result bytes. The pipeline gets a final {@code $limit} of one document past the count budget, + * so the server never produces more than that, and the results are read from the cursor one + * document at a time, the encoded size of each added up as it arrives. They used to be read whole — + * every document in memory before anything was counted — and the byte budget was declared and never + * checked (round9 review 2026-11-12, RR9-01-20; {@code docs/mongodb/query-aggregation-guide.md} + * 「집계에 예산을 둔다」). */ public final class PolicyAwareMongoAggregationExecutor { @@ -53,6 +63,14 @@ public final class PolicyAwareMongoAggregationExecutor { /** * Executes a typed aggregation. * + *

Results are read as {@link Document}, sized, then mapped with the template's converter. Two + * costs follow, both deliberate. Each document is encoded once more to be measured: the template + * hands back decoded documents, so the bytes the server sent are not available, and the budget + * has to be in bytes. And the mapping is the converter's alone: the template's typed aggregate + * would also publish {@code AfterLoadEvent}/{@code AfterConvertEvent} and run entity callbacks, + * and those do not run for an aggregation result read here. A listener that needs them maps from + * the returned type itself. + * * @throws MongoOperationRejectedException when a stage, lookup, budget or result size is not * permitted */ @@ -71,19 +89,38 @@ public final class PolicyAwareMongoAggregationExecutor { MongoOperationBudget effective = enforcer.narrow(budget, budget.narrowedTo(budgetFor(context, budget))); + List bounded = new ArrayList<>(plan.operations()); + // One past the budget: enough to know the budget was exceeded, never more than that. + bounded.add(Aggregation.limit(effective.maxResults() + 1L)); Aggregation aggregation = - Aggregation.newAggregation(plan.operations()).withOptions(optionsFor(profile, effective)); - AggregationResults results = operations.aggregate(aggregation, collection, outputType); - List mapped = results.getMappedResults(); - if (mapped.size() > effective.maxResults()) { - throw MongoOperationRejectedException.of( - "aggregation.result", - "the aggregation returned " - + mapped.size() - + " documents, above the budget of " - + effective.maxResults()); + Aggregation.newAggregation(bounded).withOptions(optionsFor(profile, effective)); + Codec sizing = operations.getConverter().getCodecRegistry().get(Document.class); + List mapped = new ArrayList<>(); + long bytes = 0; + try (Stream documents = + operations.aggregateStream(aggregation, collection, Document.class)) { + Iterator cursor = documents.iterator(); + while (cursor.hasNext()) { + Document document = cursor.next(); + if (mapped.size() == effective.maxResults()) { + throw MongoOperationRejectedException.of( + "aggregation.result", + "the aggregation returned more than the budget of " + + effective.maxResults() + + " documents"); + } + bytes += new RawBsonDocument(document, sizing).getByteBuffer().remaining(); + if (bytes > effective.maxResultBytes()) { + throw MongoOperationRejectedException.of( + "aggregation.result", + "the aggregation returned more than the budget of " + + effective.maxResultBytes() + + " bytes"); + } + mapped.add(operations.getConverter().read(outputType, document)); + } } - return mapped; + return List.copyOf(mapped); } private static MongoOperationBudget budgetFor( diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java index 0466ca15..855db8ec 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java @@ -36,20 +36,46 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge private static final String MIGRATION_ID = "migrationId"; + private static final String FENCE = "fence"; + private final MongoCollection ledger; private final MongoCollection checkpoints; + private final MongoCollection locks; + + /** + * A ledger over the default collections of {@code database}, the lock's included. + * + *

The lock collection is {@link MongoCollectionMigrationLock#DEFAULT_COLLECTION}. A deployment + * that keeps its runner lease in a collection of another name uses the three-collection + * constructor with that collection; with this one the ledger would read a lock that is never + * taken and refuse nothing on its account. + * + * @param database the database being migrated + */ public MongoCollectionMigrationLedger(MongoDatabase database) { this( Objects.requireNonNull(database, "database").getCollection(DEFAULT_COLLECTION), - database.getCollection(DEFAULT_CHECKPOINT_COLLECTION)); + database.getCollection(DEFAULT_CHECKPOINT_COLLECTION), + database.getCollection(MongoCollectionMigrationLock.DEFAULT_COLLECTION)); } + /** + * A ledger over explicit collections. + * + * @param ledger the applied migrations + * @param checkpoints the running migrations' positions + * @param locks the collection {@link MongoCollectionMigrationLock} keeps the runner lease in, + * read to refuse a replaced runner's ledger entry + */ public MongoCollectionMigrationLedger( - MongoCollection ledger, MongoCollection checkpoints) { + MongoCollection ledger, + MongoCollection checkpoints, + MongoCollection locks) { this.ledger = Objects.requireNonNull(ledger, "ledger"); this.checkpoints = Objects.requireNonNull(checkpoints, "checkpoints"); + this.locks = Objects.requireNonNull(locks, "locks"); } /** @@ -90,13 +116,34 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge Objects.requireNonNull(checksum, "checksum"); Objects.requireNonNull(operator, "operator"); Objects.requireNonNull(appliedAt, "appliedAt"); - requireCurrentFence(fence, "ledger entry for " + migrationId.value()); + requireFenced(fence, "ledger entry for " + migrationId.value()); + // A superseded runner that wakes up after its replacement took the lease must not record the + // migration as applied under it (round9 review 2026-11-12, RR9-01-28). Two pieces of evidence: + // the lock's newest acquisition, which a replacement leaves the moment it takes the lease, and + // a checkpoint a newer runner wrote. + // + // A narrowing, not a guarantee: the reads and the insert are separate operations, so a runner + // replaced between them still records. What stops a superseded runner first is the lock's + // refresh, which refuses it; this check is the second line for a runner that skipped a + // refresh. Making it atomic needs the lock and the ledger in one transaction, which a + // standalone deployment does not have. + if (MongoCollectionMigrationLock.newestFence(locks) > fence + || checkpoints + .find( + Filters.and( + Filters.eq(MIGRATION_ID, migrationId.value()), Filters.gt(FENCE, fence))) + .first() + != null) { + throw MongoOperationRejectedException.of( + "migration.fence", + "a newer migration runner owns the lease; this runner's ledger entry was refused"); + } ledger.insertOne( new Document(MIGRATION_ID, migrationId.value()) .append("checksum", checksum.value()) .append("operator", operator) .append("appliedAt", Date.from(appliedAt)) - .append("fence", fence)); + .append(FENCE, fence)); } @Override @@ -117,7 +164,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge @Override public void saveCheckpoint(MongoMigrationCheckpoint checkpoint, long fence) { Objects.requireNonNull(checkpoint, "checkpoint"); - requireCurrentFence(fence, "checkpoint for " + checkpoint.migrationId().value()); + requireFenced(fence, "checkpoint for " + checkpoint.migrationId().value()); // Conditioned on the stored fence as well as matched by id: a superseded runner that wakes up // mid-batch must not move the checkpoint of the runner that replaced it, which would send the // live migration back over rows it has already done or forward past rows it has not. @@ -126,7 +173,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge .append("resumePosition", checkpoint.resumePosition()) .append("processedCount", checkpoint.processedCount()) .append("updatedAt", Date.from(checkpoint.updatedAt())) - .append("fence", fence); + .append(FENCE, fence); // Replace-then-insert rather than one upsert, and the two cases this separates are the whole // behaviour of the fence. @@ -147,7 +194,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge .replaceOne( Filters.and( Filters.eq(MIGRATION_ID, checkpoint.migrationId().value()), - Filters.or(Filters.exists("fence", false), Filters.lte("fence", fence))), + Filters.or(Filters.exists(FENCE, false), Filters.lte(FENCE, fence))), document, new ReplaceOptions().upsert(false)) .getMatchedCount(); @@ -172,9 +219,13 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge * Refuses a write from an unfenced lease. * *

An unfenced lease cannot prove the writer is still the owner, and the whole point of the - * ledger is that it says what actually happened. + * ledger is that it says what actually happened. Whether a fenced writer is still the current one + * is decided in two places, not here: the runner refreshes its lease, conditioned on the fence, + * right before it writes, and each write is conditioned on the newest fence the ledger has seen + * for the migration. The method was called {@code requireCurrentFence} and checked neither + * (round9 review 2026-11-12, RR9-01-28). */ - private static void requireCurrentFence(long fence, String what) { + private static void requireFenced(long fence, String what) { if (fence == MongoMigrationLock.UNFENCED) { throw MongoOperationRejectedException.of( "migration.fence", diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java index cca219af..94c4ac26 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java @@ -6,6 +6,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; +import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -125,6 +126,19 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock { return fence; } + /** + * The fence of the newest acquisition of the lock in {@code locks}, or {@link #UNFENCED} when the + * lock was never taken there. + * + *

For the ledger: a write whose fence is older than this one comes from a runner that has been + * replaced, whether or not its replacement has written anything yet. + */ + static long newestFence(MongoCollection locks) { + Document current = + Objects.requireNonNull(locks, "locks").find(Filters.eq("_id", LOCK_ID)).first(); + return current == null ? UNFENCED : fenceOf(current); + } + private static long fenceOf(Document lock) { Object stored = lock.get("fence"); return stored instanceof Number number ? number.longValue() : 1L; @@ -165,7 +179,11 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock { .getMatchedCount(); if (stillOwned != 1) { held = false; - throw new IllegalStateException( + // The refusal the heartbeat promises (MongoMigrationHeartbeat#beat), so a migration that + // stops on it stops for the same reason the heartbeat's own ownership check gives. It was an + // IllegalStateException, which read as a server defect (round9 review 2026-11-12, RR9-01-28). + throw MongoOperationRejectedException.of( + "migration.lock", "the migration lease was lost before it could be refreshed; another runner may have " + "taken it after this one appeared to stall"); } diff --git a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java index abbcad9c..edb28966 100644 --- a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java +++ b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java @@ -224,6 +224,95 @@ class MongoMigrationLaneTest { assertThat(ledger.findCheckpoint(id).orElseThrow().processedCount()).isEqualTo(200); } + @Test + void aSupersededRunnerCannotRecordTheMigrationAppliedUnderItsReplacement() { + // given + MongoDatabase database = freshDatabase(); + MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database); + ledger.ensureIndexes(); + MongoMigrationId id = new MongoMigrationId("20260811-011"); + ledger.saveCheckpoint( + MongoMigrationCheckpoint.start(id, FIXED.instant()) + .advancedTo("o-300", 300, FIXED.instant()), + 2L); + + // when + Throwable refused = + catchThrowable( + () -> + ledger.recordApplied( + id, new MongoMigrationChecksum("abc"), "operator", FIXED.instant(), 1L)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("a newer migration runner owns the lease"); + } + + @Test + void aReplacedRunnerCannotRecordTheMigrationBeforeItsReplacementWroteAnything() { + // given + MongoDatabase database = freshDatabase(); + MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database); + ledger.ensureIndexes(); + MongoMigrationId id = new MongoMigrationId("20260811-012"); + Instant early = Instant.parse("2026-08-11T00:00:00Z"); + MongoMigrationLock stalled = + MongoCollectionMigrationLock.tryAcquire( + database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC)) + .orElseThrow(); + var replacement = + MongoCollectionMigrationLock.tryAcquire( + database, + "next-runner", + LEASE, + Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC)) + .orElseThrow(); + + // when + Throwable refused = + catchThrowable( + () -> + ledger.recordApplied( + id, + new MongoMigrationChecksum("abc"), + "operator", + FIXED.instant(), + stalled.fence())); + + // then + assertThat(refused) + .as("%s holds the lease with a newer fence and has not checkpointed yet", replacement) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("a newer migration runner owns the lease"); + } + + @Test + void aLeaseLostBeforeItsRefreshIsTheRefusalTheHeartbeatPromises() { + // given + MongoDatabase database = freshDatabase(); + Instant early = Instant.parse("2026-08-11T00:00:00Z"); + MongoMigrationLock stalled = + MongoCollectionMigrationLock.tryAcquire( + database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC)) + .orElseThrow(); + var unused = + MongoCollectionMigrationLock.tryAcquire( + database, + "next-runner", + LEASE, + Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC)) + .orElseThrow(); + + // when + Throwable refused = catchThrowable(() -> stalled.refresh(LEASE)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("the migration lease was lost"); + } + @Test void theLedgerRefusesADoubleApplyOfTheSameId() { MongoDatabase database = freshDatabase(); diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java index 97309f29..06f1316e 100644 --- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java +++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java @@ -3,6 +3,7 @@ package dev.caskeleton.adapter.outbound.mongo.advanced.tenancy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityFlags; import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityGuard; @@ -19,10 +20,25 @@ import dev.caskeleton.adapter.outbound.mongo.imperative.atomic.AtomicFilter; import dev.caskeleton.adapter.outbound.mongo.migration.MongoMigrationCheckpoint; import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoCollectionManifest; import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoIndexManifest; +import java.io.IOException; +import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.IntStream; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -37,6 +53,8 @@ class MongoTenancyContractTest { MongoAdvancedCapabilityFlags.allDisabled() .withEnabled(MongoCapability.DATABASE_PER_TENANT)); + private static final Instant START = Instant.parse("2026-08-13T00:00:00Z"); + @Test void missingTenantContextFailsClosed() { assertThatThrownBy(() -> injector.apply(Optional.empty(), AtomicFilter.id("o-1"))) @@ -116,29 +134,99 @@ class MongoTenancyContractTest { @Test void registryEnforcesMaximumActiveClients() { + // given MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2); - Instant now = Instant.parse("2026-08-13T00:00:00Z"); + registry.acquire("t1"); + registry.acquire("t2"); - registry.acquire("t1", now); - registry.acquire("t2", now); + // when + Throwable refused = catchThrowable(() -> registry.acquire("t3")); - assertThatThrownBy(() -> registry.acquire("t3", now)) - .isInstanceOf(MongoOperationRejectedException.class); - assertThat(registry.activeCount()).isEqualTo(2); + // then + assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.activeCount())) + .containsExactly(true, 2); } @Test void anIdleClientIsEvictedToMakeRoom() { - MongoTenantClientRegistry registry = - new MongoTenantClientRegistry(TENANCY, 1, Duration.ofMinutes(5)); - Instant start = Instant.parse("2026-08-13T00:00:00Z"); + // given + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock); + registry.acquire("t1").close(); + clock.advance(Duration.ofMinutes(10)); - registry.acquire("t1", start); - registry.acquire("t2", start.plus(Duration.ofMinutes(10))); + // when + registry.acquire("t2"); + // then assertThat(registry.activeTenants()).containsExactly("t2"); } + @Test + void aClientIsIdleFromTheEndOfItsLastUseNotItsStart() { + // given + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock); + MongoTenantClientRegistry.Lease held = registry.acquire("t1"); + clock.advance(Duration.ofMinutes(30)); + held.close(); + clock.advance(Duration.ofMinutes(1)); + + // when + Throwable refused = catchThrowable(() -> registry.acquire("t2")); + + // then + assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.openLeases())) + .as("held for thirty minutes, it was in use all that time and is not idle yet") + .containsExactly(true, 0); + } + + @Test + void anUnclosedLeaseIsCounted() { + // given + MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2); + + // when + var unused = registry.acquire("t1"); + + // then + assertThat(registry.openLeases()).isEqualTo(1); + } + + private static MongoTenantClientRegistry registry( + int cap, Function clients, Clock clock) { + return new MongoTenantClientRegistry(TENANCY, cap, Duration.ofMinutes(5), clients, clock); + } + + /** A clock a test moves by hand: one time source for acquiring, closing and eviction. */ + private static final class MovingClock extends Clock { + + private final AtomicReference now; + + private MovingClock(Instant start) { + this.now = new AtomicReference<>(start); + } + + private void advance(Duration by) { + now.updateAndGet(instant -> instant.plus(by)); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return now.get(); + } + } + @Test void aTenantDatabaseCannotServeBeforeItsSchemaIsValidated() { MongoTenantLifecyclePolicy policy = MongoTenantLifecyclePolicy.standard(); @@ -172,100 +260,112 @@ class MongoTenancyContractTest { } @Test - @org.junit.jupiter.api.DisplayName("the same tenant is created once under concurrency") - void theSameTenantIsCreatedOnce() throws Exception { - java.util.concurrent.atomic.AtomicInteger created = - new java.util.concurrent.atomic.AtomicInteger(); + void theSameTenantIsCreatedOnceUnderConcurrency() throws Exception { + // given + AtomicInteger created = new AtomicInteger(); MongoTenantClientRegistry registry = - new MongoTenantClientRegistry( - TENANCY, + registry( 8, - Duration.ofMinutes(10), tenant -> { created.incrementAndGet(); return () -> {}; - }); - java.util.concurrent.ExecutorService pool = - java.util.concurrent.Executors.newFixedThreadPool(8); + }, + new MovingClock(START)); + List> work = + IntStream.range(0, 50) + .>mapToObj( + index -> + () -> { + registry.acquire("t1").close(); + return null; + }) + .toList(); + ExecutorService pool = Executors.newFixedThreadPool(8); + + // when try { - java.util.List> work = - java.util.stream.IntStream.range(0, 50) - .>mapToObj( - index -> - () -> { - registry.acquire("t1", Instant.parse("2026-08-13T00:00:00Z")); - return null; - }) - .toList(); - for (java.util.concurrent.Future future : pool.invokeAll(work)) { + for (Future future : pool.invokeAll(work)) { future.get(); } } finally { pool.shutdownNow(); } - assertThat(created) + // then + assertThat(List.of(created.get(), registry.activeCount())) .as("a per-tenant client carries a pool and monitoring threads; a second one doubles both") - .hasValue(1); - assertThat(registry.activeCount()).isEqualTo(1); + .containsExactly(1, 1); } @Test - @org.junit.jupiter.api.DisplayName("acquire(tenant)는 주입된 clock으로 idle 여부를 판단한다") - void anAcquireWithoutAnInstantReadsTheInjectedClock() { + void aClientStillHeldIsNotEvictedHoweverLongAgoItWasAcquired() { // given - java.util.List closed = new java.util.ArrayList<>(); - java.util.concurrent.atomic.AtomicReference now = - new java.util.concurrent.atomic.AtomicReference<>(Instant.parse("2026-08-13T00:00:00Z")); - java.time.Clock clock = - new java.time.Clock() { - @Override - public java.time.ZoneId getZone() { - return java.time.ZoneOffset.UTC; - } + List closed = new ArrayList<>(); + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock); + MongoTenantClientRegistry.Lease longRunning = registry.acquire("t1"); + clock.advance(Duration.ofMinutes(10)); - @Override - public java.time.Clock withZone(java.time.ZoneId zone) { - return this; - } + // when + Throwable refused = catchThrowable(() -> registry.acquire("t2")); - @Override - public Instant instant() { - return now.get(); - } - }; + // then + assertThat(List.of(refused instanceof MongoOperationRejectedException, closed)) + .as( + "t1 is in use by %s; closing it would pull the pool out from under that work", + longRunning) + .containsExactly(true, List.of()); + } + + @Test + void anotherTenantsClientThatFailsToCloseDoesNotFailThisAcquire() { + // given + MovingClock clock = new MovingClock(START); MongoTenantClientRegistry registry = - new MongoTenantClientRegistry( - TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant), clock); - registry.acquire("t1"); - now.set(now.get().plus(Duration.ofMinutes(10))); + registry( + 1, + tenant -> + () -> { + if (tenant.equals("t1")) { + throw new IOException("t1's pool would not shut down"); + } + }, + clock); + registry.acquire("t1").close(); + clock.advance(Duration.ofMinutes(10)); + + // when + MongoTenantClientRegistry.Lease admitted = registry.acquire("t2"); + + // then + assertThat( + List.of( + admitted.client() != null, + registry.activeTenants(), + registry.evictionCloseFailures())) + .containsExactly(true, Set.of("t2"), 1L); + } + + @Test + void evictionClosesTheClientItEvicts() { + // given + List closed = new ArrayList<>(); + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock); + registry.acquire("t1").close(); + clock.advance(Duration.ofMinutes(10)); // when registry.acquire("t2"); // then - assertThat(closed).containsExactly("t1"); - } - - @Test - @org.junit.jupiter.api.DisplayName("eviction closes the client it evicts") - void evictionClosesTheClient() { - java.util.List closed = new java.util.ArrayList<>(); - MongoTenantClientRegistry registry = - new MongoTenantClientRegistry( - TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant)); - Instant start = Instant.parse("2026-08-13T00:00:00Z"); - - registry.acquire("t1", start); - registry.acquire("t2", start.plus(Duration.ofMinutes(10))); - assertThat(closed) .as("freeing the slot without closing the client keeps the connection pool it counted") .containsExactly("t1"); } @Test - @org.junit.jupiter.api.DisplayName("a coordinator resumes from its durable store after a restart") + @DisplayName("a coordinator resumes from its durable store after a restart") void aCoordinatorResumesFromItsDurableStore() { RecordingCheckpointStore store = new RecordingCheckpointStore(); MongoTenantMigrationCoordinator first = diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java index 89fd5ff0..bcf90f54 100644 --- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java +++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java @@ -3,11 +3,31 @@ package dev.caskeleton.adapter.outbound.mongo.aggregation; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.mongodb.MongoClientSettings; +import dev.caskeleton.adapter.outbound.mongo.api.MongoOperationContext; +import dev.caskeleton.adapter.outbound.mongo.api.consistency.MongoConsistencyProfile; import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException; +import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer; +import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry; +import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget; +import java.time.Duration; +import java.util.List; import java.util.Set; +import org.bson.Document; +import org.bson.conversions.Bson; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.data.mongodb.core.MongoOperations; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.convert.MongoConverter; +import org.springframework.data.mongodb.core.query.Criteria; /** Design §17 — write stages are D4, and stage cost is graded rather than assumed. */ @Tag("mongodb-contract") @@ -105,4 +125,141 @@ class PolicyAwareMongoAggregationExecutorTest { assertThat(MongoAggregationProfile.stableRead().allowDiskUse()).isFalse(); assertThat(MongoAggregationProfile.stableRead().strictMapping()).isTrue(); } + + private static final MongoOperationContext CONTEXT = + MongoOperationContext.of( + "orders.summary", + "primary", + "orders", + MongoConsistencyProfile.PRIMARY_MAJORITY, + Duration.ofSeconds(2)); + + private static final MongoAggregationPlan OPEN_ORDERS = + MongoAggregationPlan.builder() + .stage("$match", Aggregation.match(Criteria.where("status").is("OPEN"))) + .build(); + + @Test + void thePipelineEndsWithALimitOnePastTheCountBudget() { + // given + ArgumentCaptor sent = ArgumentCaptor.forClass(Aggregation.class); + MongoOperations operations = returning(List.of(), sent); + + // when + var unused = + executorOver(operations, budget(3, 1_048_576L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class); + + // then + List pipeline = sent.getValue().toPipeline(Aggregation.DEFAULT_CONTEXT); + assertThat(pipeline.get(pipeline.size() - 1)).isEqualTo(new Document("$limit", 4L)); + } + + @Test + void moreDocumentsThanTheCountBudgetAreRefused() { + // given + MongoOperations operations = + returning( + List.of(order(1), order(2), order(3), order(4)), + ArgumentCaptor.forClass(Aggregation.class)); + + // when + Throwable refused = + catchThrowable( + () -> + executorOver(operations, budget(3, 1_048_576L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("more than the budget of 3 documents"); + } + + @Test + void documentsLargerTogetherThanTheByteBudgetAreRefused() { + // given + // Each order encodes to well over 40 bytes: two of them pass any 80-byte budget. + MongoOperations operations = + returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class)); + + // when + Throwable refused = + catchThrowable( + () -> + executorOver(operations, budget(10, 80L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("more than the budget of 80 bytes"); + } + + @Test + void resultsInsideBothBudgetsAreReturnedInCursorOrder() { + // given + MongoOperations operations = + returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class)); + + // when + List results = + executorOver(operations, budget(10, 1_048_576L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class); + + // then + assertThat(results).containsExactly(order(1), order(2)); + } + + private static MongoOperationBudget budget(int maxResults, long maxResultBytes) { + return new MongoOperationBudget(maxResults, maxResultBytes, 500L, 50); + } + + private static PolicyAwareMongoAggregationExecutor executorOver( + MongoOperations operations, MongoOperationBudget budget) { + return new PolicyAwareMongoAggregationExecutor( + operations, + MongoBudgetPolicyRegistry.builder().register("orders.summary", budget).build(), + new MongoBudgetEnforcer()); + } + + /** The driver boundary: a cursor over {@code documents}, mapped as the converter would. */ + private static MongoOperations returning( + List documents, ArgumentCaptor sent) { + MongoOperations operations = mock(MongoOperations.class); + MongoConverter converter = mock(MongoConverter.class); + when(operations.getConverter()).thenReturn(converter); + when(converter.getCodecRegistry()).thenReturn(MongoClientSettings.getDefaultCodecRegistry()); + when(converter.read(eq(Document.class), any(Bson.class))) + .thenAnswer(invocation -> invocation.getArgument(1)); + when(operations.aggregateStream(sent.capture(), eq("orders"), eq(Document.class))) + .thenAnswer(invocation -> documents.stream()); + return operations; + } + + private static Document order(int number) { + return new Document("_id", "order-" + number) + .append("status", "OPEN") + .append("customer", "customer-" + number); + } } diff --git a/src/adapter/outbound/persistence-notification/AGENTS.md b/src/adapter/outbound/persistence-notification/AGENTS.md index 94fa03ad..cb9ae8b1 100644 --- a/src/adapter/outbound/persistence-notification/AGENTS.md +++ b/src/adapter/outbound/persistence-notification/AGENTS.md @@ -8,7 +8,8 @@ - 패키지 루트: `dev.caskeleton.adapter.outbound.persistence.notification`. 패키지 규칙은 `persistence-fileserver`와 같아서 두 capability leaf를 같은 방식으로 읽는다: - `platform.entity` — JPA 행만. - - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). + - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). `platform`의 store도 + 이 SQL 조각을 쓴다. - `platform` — store 어댑터(`Jpa*`, `Jdbc*`, inbox의 `JpaNotificationInbox`와 그 outbox 도우미), `NotificationRecordMapper`, payload codec. - `configuration` — store를 bean으로 등록하는 자동 구성 창구. leaf 루트에는 scan 설정과 schema stream이 있다. diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java index 5a81e94b..72a52a8e 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java @@ -1,13 +1,17 @@ package dev.caskeleton.adapter.outbound.persistence.notification.platform; +import dev.caskeleton.adapter.outbound.persistence.notification.platform.repository.RecipientClaimSql; import dev.caskeleton.application.notification.platform.observation.NotificationServingState; import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Map; import java.util.Objects; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; /** * Measures the platform's serving state from the durable rows. @@ -17,34 +21,44 @@ import org.springframework.jdbc.core.JdbcOperations; * identical. These are the questions an operator actually asks during an incident: how far behind * are we, how long has the oldest thing waited, and is anything stuck. * - *

One statement per question rather than one joined statement. They are read independently, a - * slow one does not delay the others, and each index the queries rely on is the same index the - * corresponding worker already uses to claim. + *

One statement per question rather than one joined statement. They are read independently and a + * slow one does not delay the others. Every statement is bounded by an index: the backlog and + * stuck-lease counts by the predicates the claim uses, the reconciliation count by the job's due + * time, and the projection lag by {@code ix_notification_provider_event_open} (V14), which holds + * only the events still waiting for something — the provider-event table itself is never purged, so + * the lag query used to read all of it (round9 review 2026-11-12, RR9-01-14). + * + *

The scheduler reads only {@link #backlogDepth()}, on every poll; the full snapshot is for the + * health endpoint. */ public final class JdbcNotificationServingState implements NotificationServingStatePort { /** - * Deliveries that are due and unclaimed. + * Deliveries the claim would take now. * - *

The lease predicate matches the claim's: a row whose lease has expired is available again, - * so it is backlog. Counting it as in-flight is how a stuck worker hides a growing queue. + *

The {@code WHERE} is the claim's own fragment ({@link RecipientClaimSql#CLAIMABLE_AT_NOW}): + * the same states, due, not expired, and a lease that is absent or has run out. A row whose lease + * has expired is available again, so it is backlog; counting it as in-flight is how a stuck + * worker hides a growing queue. An expired delivery is not backlog: the expiry sweep retires it + * unsent. The fragment's parameter is named, so this statement is too. */ private static final String BACKLOG = """ SELECT count(*) AS depth, - COALESCE(EXTRACT(EPOCH FROM (? - min(next_dispatch_at))), 0) AS oldest_seconds + COALESCE(EXTRACT(EPOCH FROM (:now - min(next_dispatch_at))), 0) AS oldest_seconds FROM notification_recipient_delivery - WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING') - AND next_dispatch_at IS NOT NULL - AND next_dispatch_at <= ? - AND (lease_until IS NULL OR lease_until <= ?) - """; + WHERE\s""" + + RecipientClaimSql.CLAIMABLE_AT_NOW; /** * Leases whose holder never came back. * - *

Distinct from backlog: these rows are claimable again, but the fact that they had to expire - * means a worker died mid-dispatch, and a provider call may have happened without a record. + *

Distinct from backlog: these rows are still dispatching, which the claim never takes; lease + * recovery requeues or reconciles them one lease after they ran out. A row left from before + * notification V4 (an owner but fence 0) is counted here and not recovered; {@code + * docs/upgrade/2026-11-round9-claim-fence.md}, row 5, says how to hand it to recovery. That a + * lease had to run out means a worker died or stalled mid-dispatch, and a provider call may have + * happened without a record. */ private static final String STUCK_LEASES = """ @@ -56,7 +70,21 @@ public final class JdbcNotificationServingState implements NotificationServingSt AND delivery_state = 'DISPATCHING' """; - /** Provider events stored and not yet applied to a delivery. */ + /** + * The provider events still waiting for something: the lag query's {@code WHERE}, and the + * predicate of the partial index {@code V14} builds for it. One string, held to the migration by + * {@code JdbcNotificationServingStateOpenEventsTest}: a query the index does not cover reads the + * whole table again, which is what the index was made to stop. + */ + static final String OPEN_EVENTS = + "projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL"; + + /** + * Provider events stored and not yet applied to a delivery. + * + *

The {@code WHERE} is exactly the predicate of {@code ix_notification_provider_event_open}: + * every row any of the four counts can see, and none of the applied, matched history. + */ private static final String PROJECTION_LAG = """ SELECT @@ -66,7 +94,9 @@ public final class JdbcNotificationServingState implements NotificationServingSt COALESCE(EXTRACT(EPOCH FROM (? - min(received_at) FILTER (WHERE projection_status = 'PENDING'))), 0) AS oldest_seconds FROM notification_provider_event - """; + WHERE + """ + + OPEN_EVENTS; /** Questions to providers that are due and unanswered. */ private static final String RECONCILIATION_LAG = @@ -78,26 +108,36 @@ public final class JdbcNotificationServingState implements NotificationServingSt """; private final JdbcOperations jdbc; + private final NamedParameterJdbcOperations named; private final Clock clock; public JdbcNotificationServingState(JdbcOperations jdbc, Clock clock) { this.jdbc = Objects.requireNonNull(jdbc, "jdbc"); + this.named = new NamedParameterJdbcTemplate(jdbc); this.clock = Objects.requireNonNull(clock, "clock"); } + @Override + public long backlogDepth() { + Measurement backlog = measureBacklog(Timestamp.from(clock.instant())); + return backlog == null ? 0 : backlog.count(); + } + + /** The backlog query alone: the one statement the scheduler's queue-depth gauge needs. */ + private Measurement measureBacklog(Timestamp at) { + return named.queryForObject( + BACKLOG, + Map.of("now", at), + (rows, rowNumber) -> + new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds"))); + } + @Override public NotificationServingState currentState() { Instant now = clock.instant(); Timestamp at = Timestamp.from(now); - Measurement backlog = - jdbc.queryForObject( - BACKLOG, - (rows, rowNumber) -> - new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")), - at, - at, - at); + Measurement backlog = measureBacklog(at); Long stuck = jdbc.queryForObject(STUCK_LEASES, Long.class, at); ProjectionMeasurement projection = jdbc.queryForObject( diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java index 23fdb2dc..a2da23c2 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java @@ -99,6 +99,23 @@ public final class JpaRecipientDeliveryStore implements RecipientDeliveryStorePo return reread(written, record.id()); } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + CallerTransaction.require("recipient delivery requeueAbandoned"); + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + int written = + recipients.requeueAbandoned( + abandoned.recipientDeliveryId().value(), + abandoned.owner(), + abandoned.fence(), + abandoned.leaseUntil(), + nextDispatchAt, + clock.instant()); + return reread(written, abandoned.recipientDeliveryId()); + } + @Override public Optional transitionHeldBy( RecipientDeliveryId id, diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java index 65784bac..98cd4bef 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java @@ -5,9 +5,11 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import dev.caskeleton.application.notification.platform.dispatch.RecipientLease; import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.OffsetDateTime; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -109,13 +111,38 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort { } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); } return recipients.selectExpiredDispatching(clock.instant().minus(olderThan), limit).stream() - .map(RecipientDeliveryId::new) + .map( + row -> + new RecipientLease( + new RecipientDeliveryId((UUID) row[0]), + (String) row[1], + ((Number) row[2]).longValue(), + instantOf(row[3]))) .toList(); } + + /** + * A {@code timestamptz} column of a native row. Hibernate hands it back as an {@link + * OffsetDateTime} or a {@link Timestamp} depending on its JDBC type mapping, never null here + * because the select only returns rows that have a lease. + */ + private static Instant instantOf(Object column) { + if (column instanceof Instant instant) { + return instant; + } + if (column instanceof OffsetDateTime offset) { + return offset.toInstant(); + } + if (column instanceof Timestamp timestamp) { + return timestamp.toInstant(); + } + throw new IllegalStateException( + "lease_until came back as " + (column == null ? "null" : column.getClass().getName())); + } } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java index 15428027..e7ccfb32 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java @@ -60,18 +60,16 @@ public interface InboxItemJpaRepository extends Repository{@code read_at IS NULL} in the predicate is what makes the update idempotent under * concurrency: a second caller updates zero rows instead of overwriting the first caller's * timestamp. - */ - /** - * Marks everything up to and including a cursor position as read. * *

The cutoff is a tuple, matching the list's own {@code (created_at DESC, id DESC)} ordering. * It used to be a timestamp alone, and the list's order is not a total order on {@code @@ -95,14 +93,23 @@ public interface InboxItemJpaRepository extends RepositoryThe lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code * PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and * the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The - * named parameters are Spring Data's. + * named parameters are bound by Spring Data for the repository statements, and by a {@code + * NamedParameterJdbcTemplate} where the serving state reads {@link #CLAIMABLE_AT_NOW}. */ public final class RecipientClaimSql { + /** + * The states a delivery waits in to be sent, which the claim takes from, the expiry retires from + * and the serving state counts as backlog. One list so the statements cannot disagree about what + * is waiting. + */ + public static final String CLAIMABLE_STATES = "('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"; + + /** + * Which deliveries the claim takes at {@code :now}: due, in a waiting state, not expired, and + * with no lease or one that has run out. The serving state counts its backlog with this same + * fragment, so the backlog is exactly what a claim would take. + */ + public static final String CLAIMABLE_AT_NOW = + "next_dispatch_at <= :now" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND (expires_at IS NULL OR expires_at > :now)" + + " AND (lease_until IS NULL OR lease_until < :now)"; + /** * Claims due deliveries and returns their fences. * @@ -28,10 +48,8 @@ public final class RecipientClaimSql { public static final String CLAIM_BATCH = "WITH claimable AS (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE next_dispatch_at <= :now" - + " AND delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" - + " AND (expires_at IS NULL OR expires_at > :now)" - + " AND (lease_until IS NULL OR lease_until < :now)" + + " WHERE " + + CLAIMABLE_AT_NOW + " ORDER BY next_dispatch_at, id" + " FOR UPDATE SKIP LOCKED" + " LIMIT :batchSize" @@ -51,7 +69,18 @@ public final class RecipientClaimSql { + " WHERE d.id = claimable.id" + " RETURNING d.id, d.lease_fence"; - /** Retires deliveries whose window closed before anyone claimed them. */ + /** + * Retires deliveries whose window closed before anyone claimed them. + * + *

A delivery a worker claimed while this statement ran must not be retired to {@code EXPIRED} + * under the worker dispatching it, which is what the unlocked subquery did (round9 review + * 2026-11-12, RR9-01-13). Two things stop it. The subquery locks the rows it picks and skips rows + * another transaction holds, so a delivery being claimed is not picked. The state and expiry + * conditions are also on the {@code UPDATE} itself: under READ COMMITTED PostgreSQL re-checks + * only the outer {@code WHERE} against a row it had to wait for, so if the lock clause is ever + * lost the claimed row is still left alone. A delivery being claimed right now is not expiring + * either: the claim refuses expired rows in the same statement. + */ public static final String EXPIRE_OVERDUE = "UPDATE notification_recipient_delivery SET" + " delivery_state = 'EXPIRED'," @@ -61,11 +90,17 @@ public final class RecipientClaimSql { + " updated_at = :now" + " WHERE id IN (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" + + " WHERE delivery_state IN " + + CLAIMABLE_STATES + " AND expires_at IS NOT NULL" + " AND expires_at <= :now" + " ORDER BY expires_at" - + " LIMIT :batchSize)"; + + " FOR UPDATE SKIP LOCKED" + + " LIMIT :batchSize)" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND expires_at IS NOT NULL" + + " AND expires_at <= :now"; private RecipientClaimSql() {} } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java index b5f1ee18..1240cf0f 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java @@ -183,13 +183,48 @@ public interface RecipientDeliveryJpaRepository extends RepositoryOnly rows a claim wrote: an owner and a fence of at least one. The requeue matches on both, + * and a row without them — one left dispatching before notification V4 added the fence — is no + * lease anyone holds. Read here it would be a lease the application refuses to build, and the one + * row would fail every recovery pass after it. + * + * @return one row per delivery: id, lease_owner, lease_fence, lease_until + */ @Query( value = - "SELECT id FROM notification_recipient_delivery " + "SELECT id, lease_owner, lease_fence, lease_until FROM notification_recipient_delivery " + "WHERE delivery_state = 'DISPATCHING' AND lease_until < :threshold " + + "AND lease_owner IS NOT NULL AND lease_fence > 0 " + "ORDER BY lease_until LIMIT :batchSize", nativeQuery = true) - List selectExpiredDispatching( + List selectExpiredDispatching( @Param("threshold") Instant threshold, @Param("batchSize") int batchSize); + + /** + * Returns an abandoned dispatch to the queue while it is still the one recovery found. + * + *

Owner, fence, state and expiry are all conditions. A new claim raises the fence, a finished + * holder moves the state, another instance's recovery moves the state, and a holder that renewed + * in time moves the expiry past what recovery saw; in each case the job is no longer abandoned + * and this matches nothing. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query( + value = + "UPDATE notification_recipient_delivery " + + "SET delivery_state = 'READY_TO_DISPATCH', next_dispatch_at = :nextDispatchAt, " + + "version = version + 1, updated_at = :now " + + "WHERE id = :id AND lease_owner = :owner AND lease_fence = :fence " + + "AND delivery_state = 'DISPATCHING' AND lease_until <= :leaseUntil", + nativeQuery = true) + int requeueAbandoned( + @Param("id") UUID id, + @Param("owner") String owner, + @Param("fence") long fence, + @Param("leaseUntil") Instant leaseUntil, + @Param("nextDispatchAt") Instant nextDispatchAt, + @Param("now") Instant now); } diff --git a/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql new file mode 100644 index 00000000..2d2416a2 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql @@ -0,0 +1,26 @@ +-- An index over the provider events still waiting for something. +-- +-- The serving-state projection lag counts pending, failed and unmatched events. It had no WHERE and +-- read the whole notification_provider_event table, which nothing ever purges, and the scheduler +-- asked for it on every poll (round9 review 2026-11-12, RR9-01-14). The query now names exactly +-- this predicate, so it reads only these rows; applied, matched events — the history that grows — +-- are outside the index. +-- +-- Built inside this migration's transaction, not CONCURRENTLY, as V21 of the core stream builds its +-- index: a transactional file fails whole and leaves nothing to repair, where a failed concurrent +-- build leaves an INVALID index and a failed history row an operator must clear by hand +-- (docs/runbooks/migration-failed.md). The cost is that the build holds SHARE on +-- notification_provider_event: provider callbacks wait for it, and providers retry a callback that +-- does not get an answer. On a table already large in production, create the same index +-- CONCURRENTLY by hand before deploying; IF NOT EXISTS then makes this migration a no-op. + +-- Bound the wait for the lock (SHARE on notification_provider_event, held until commit). Unbounded, +-- it queues behind any open transaction that touched the table, and every callback insert queues +-- behind the migration. Five seconds, then the migration fails, rolls back whole and the stream +-- stays at V13: deploy again once that transaction has ended (docs/jpa/migration-guide.md "Bound +-- the lock wait"). +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX IF NOT EXISTS ix_notification_provider_event_open + ON notification_provider_event (received_at) + WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL; diff --git a/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java new file mode 100644 index 00000000..0805a0d6 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java @@ -0,0 +1,42 @@ +package dev.caskeleton.adapter.outbound.persistence.notification.platform; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import org.junit.jupiter.api.Test; + +/** + * The projection-lag query and the partial index built for it name the same events. + * + *

The index is what keeps the lag query, which the scheduler runs on every poll, from reading + * the whole provider-event history. A predicate that drifted on either side would leave the query + * outside the index and scanning the table again, with nothing failing. + */ +class JdbcNotificationServingStateOpenEventsTest { + + private static final String V14 = + "db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql"; + + @Test + void theLagQueryAndItsPartialIndexNameTheSameEvents() throws IOException { + // given + String migration; + try (InputStream sql = getClass().getClassLoader().getResourceAsStream(V14)) { + migration = new String(Objects.requireNonNull(sql, V14).readAllBytes(), UTF_8); + } + int index = migration.indexOf("CREATE INDEX"); + + // when + String indexPredicate = + migration + .substring( + migration.indexOf("WHERE ", index) + "WHERE ".length(), migration.lastIndexOf(';')) + .trim(); + + // then + assertThat(indexPredicate).isEqualTo(JdbcNotificationServingState.OPEN_EVENTS); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java index 535a146d..8c4a6de9 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java @@ -19,10 +19,15 @@ public interface IntegrationEventPublishPort { CompletionStage publish(ValidatedIntegrationEvent event); /** - * The longest one publish can take before the returned stage completes, when the publisher knows - * it. The outbox relay sizes its batch against it (see {@code + * The longest one publish can take, from the call to {@code publish} until the returned stage + * completes, when the publisher knows it. The outbox relay sizes its batch against it (see {@code * OutboxMessagePublishPort#longestAttempt}). * + *

A bound, not a hint: a caller may stop waiting once it has passed and treat the attempt as + * ambiguous, so a publisher that reports less than it can take turns slow answers into ambiguous + * ones. Positive when present. Empty means unknown, and then the stage's own completion is the + * only bound. + * * @return the longest attempt, or empty when unknown */ default Optional longestAttempt() { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java new file mode 100644 index 00000000..80bdbe0c --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java @@ -0,0 +1,45 @@ +package dev.caskeleton.application.operation; + +import java.util.Objects; + +/** + * One worker's claim on one durable operation, as returned by {@link + * DurableOperationStorePort#claimNext}. + * + *

The {@code fence} is what tells one run from the next. Every claim increments it, and a + * heartbeat, progress report or completion names the fence it holds, so a run that stalled past its + * lease finds nothing left to change instead of overwriting the run that took over. The worker id + * alone cannot do this: a worker that restarts with the same id, or one process running two + * executions under one id, would otherwise have its earlier run accepted as the current one. + * + *

Kept apart from {@link DurableOperationSubmission} on purpose, as the outbox keeps its lease + * apart from the claimed event: the work to execute is the submission, and the claim is store + * bookkeeping the work has no use for. + * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * + * @param submission the claimed work + * @param workerId the worker that holds the claim + * @param fence the fencing token this claim was granted; at least 1 + */ +public record DurableOperationClaim( + DurableOperationSubmission submission, String workerId, long fence) { + + public DurableOperationClaim { + Objects.requireNonNull(submission, "submission"); + if (workerId == null || workerId.isBlank()) { + throw new IllegalStateException( + "a claim needs an owner; an anonymous one cannot be verified when the work completes"); + } + if (fence < 1) { + throw new IllegalStateException("fence must be >= 1, was " + fence); + } + } + + /** The claimed operation. */ + public DurableOperationId operationId() { + return submission.operationId(); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java index 440d3fb2..66c718b2 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java @@ -11,9 +11,12 @@ import java.util.Optional; *

A port because the application owns what an operation is and nothing about how it is stored. * The JPA adapter behind it is one implementation; an in-memory one serves tests. * - *

Every method that changes a running operation takes the {@code workerId} and verifies it. A - * worker whose lease lapsed may still be alive and still finishing its work, and letting it report - * a result after another worker took over is how an operation ends up with two answers. + *

Every method that changes a running operation takes the {@link DurableOperationClaim} that + * {@link #claimNext} returned and verifies both its worker and its fence. A worker whose lease + * lapsed may still be alive and still finishing its work, and letting it report a result after + * another run took over is how an operation ends up with two answers. The worker id alone does not + * tell the runs apart: the same id may claim the operation again after a restart, so the fence — + * incremented by every claim — is what the store compares. */ public interface DurableOperationStorePort { @@ -43,48 +46,43 @@ public interface DurableOperationStorePort { * *

Runnable means PENDING, or RUNNING with a lapsed lease. The second case is the recovery * path: without it, an operation claimed by a worker that then died is never picked up again. + * Every claim gets a fence higher than any earlier claim of the same operation, whoever made it. * * @param workerId who is claiming * @param leaseDuration how long the claim holds without a heartbeat * @param now the current instant + * @return the claim the worker names on every later write, or empty when nothing is runnable */ - Optional claimNext( - String workerId, Duration leaseDuration, Instant now); + Optional claimNext(String workerId, Duration leaseDuration, Instant now); /** - * Extends a lease the worker still holds. + * Extends a lease the claim still holds. * - * @return false when the lease is no longer this worker's, which means it must stop + * @return false when the lease lapsed or another claim (by any worker, this one included) took + * the operation over, which means this run must stop */ - boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now); + boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now); /** - * Publishes progress for an operation this worker holds. + * Publishes progress for an operation the claim holds. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over or it is no longer running */ - boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress); + boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress); /** * Records success. * - * @return false when the lease is no longer this worker's, so the result is not recorded + * @return false when another claim took the operation over, so the result is not recorded */ - boolean succeed( - DurableOperationId operationId, String workerId, String resultReference, Instant completedAt); + boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt); /** * Records failure. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over, so the failure is not recorded */ - boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt); + boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt); /** * Requests cancellation. diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java index 86328d84..618f23c1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java @@ -11,6 +11,10 @@ import java.util.Objects; * ever with no worker behind it — the failure that makes a queue stop draining while every * dashboard reports it busy. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param workerId who holds it * @param acquiredAt when it was taken * @param expiresAt when another worker may take it @@ -22,11 +26,11 @@ public record OperationLease(String workerId, Instant acquiredAt, Instant expire Objects.requireNonNull(acquiredAt, "acquiredAt"); Objects.requireNonNull(expiresAt, "expiresAt"); if (workerId.isBlank()) { - throw new IllegalArgumentException( + throw new IllegalStateException( "a lease needs an owner; an anonymous one cannot be verified when the work completes"); } if (!expiresAt.isAfter(acquiredAt)) { - throw new IllegalArgumentException("a lease that expires when taken is not a lease"); + throw new IllegalStateException("a lease that expires when taken is not a lease"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java index bafa1b5e..05c0062e 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java @@ -12,6 +12,10 @@ import java.util.Objects; * publishes the bytes that were validated rather than a re-rendering of them. The accessors below * name the few identities the relay and its diagnostics read without reaching into the envelope. * + *

A store builds it from a row, never from a request, so a count it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the appended event * @param status the row's lifecycle status; {@link OutboxEventStatus#IN_FLIGHT} right after a claim * @param attemptCount claims made, counting the claim that returned this row; the retry delay grows @@ -30,10 +34,10 @@ public record ClaimedOutboxEvent( Objects.requireNonNull(event, "event must not be null"); Objects.requireNonNull(status, "status must not be null"); if (attemptCount < 0) { - throw new IllegalArgumentException("attemptCount must be >= 0, was " + attemptCount); + throw new IllegalStateException("attemptCount must be >= 0, was " + attemptCount); } if (deferredAttemptCount < 0 || deferredAttemptCount > attemptCount) { - throw new IllegalArgumentException( + throw new IllegalStateException( "deferredAttemptCount must be between 0 and attemptCount (" + attemptCount + "), was " diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java index eed397ab..ae783bc1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java @@ -13,6 +13,10 @@ import java.util.Objects; *

Kept apart from {@link ClaimedOutboxEvent} on purpose: the broker adapter receives the event, * and the claim is store bookkeeping it has no use for. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the claimed row * @param owner the relay instance that holds the claim * @param token the fencing token this claim was granted @@ -22,10 +26,10 @@ public record OutboxLease(ClaimedOutboxEvent event, String owner, long token) { public OutboxLease { Objects.requireNonNull(event, "event must not be null"); if (owner == null || owner.isBlank()) { - throw new IllegalArgumentException("owner must not be blank"); + throw new IllegalStateException("owner must not be blank"); } if (token < 1) { - throw new IllegalArgumentException("token must be >= 1, was " + token); + throw new IllegalStateException("token must be >= 1, was " + token); } } diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java index 628ee3ef..8c845cac 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java @@ -39,8 +39,9 @@ class CancelDurableOperationUseCaseTest { void cancelOfAFinishedOperationReportsNoChange() { // given operations.submit(submission(null)); - operations.claimNext("worker-a", Duration.ofMinutes(1), NOW); - operations.succeed(ID, "worker-a", "/results/1", NOW); + DurableOperationClaim claim = + operations.claimNext("worker-a", Duration.ofMinutes(1), NOW).orElseThrow(); + operations.succeed(claim, "/results/1", NOW); // when Optional canceled = diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..13f6efaf --- /dev/null +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java @@ -0,0 +1,80 @@ +package dev.caskeleton.application.operation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.outbox.ClaimedOutboxEvent; +import dev.caskeleton.application.outbox.OutboxEventStatus; +import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.testkit.OutboxEvents; +import java.time.Duration; +import java.time.Instant; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** The claims and leases a store builds from its rows refuse a bad value as each record says. */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + DurableOperationSubmission submission = + new DurableOperationSubmission( + new DurableOperationId("op-1"), + "reports.generate", + "alice", + "tenant-a", + new RequestFingerprint("a".repeat(64)), + "{}", + NOW, + Duration.ofHours(1)); + return Stream.of( + Arguments.of( + "owner 없는 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, " ", 1), + "a claim needs an owner"), + Arguments.of( + "fence 0인 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, "worker-1", 0), + "fence must be >= 1, was 0"), + Arguments.of( + "owner 없는 lease", + (ThrowingCallable) () -> new OperationLease(" ", NOW, NOW.plusSeconds(30)), + "a lease needs an owner"), + Arguments.of( + "잡는 순간 끝나는 lease", + (ThrowingCallable) () -> new OperationLease("worker-1", NOW, NOW), + "a lease that expires when taken is not a lease"), + Arguments.of( + "owner 없는 outbox lease", + (ThrowingCallable) () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), " ", 1), + "owner must not be blank"), + Arguments.of( + "token 0인 outbox lease", + (ThrowingCallable) + () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), "relay-1", 0), + "token must be >= 1, was 0"), + Arguments.of( + "미룬 claim이 claim보다 많은 outbox 행", + (ThrowingCallable) + () -> + new ClaimedOutboxEvent( + OutboxEvents.event("evt-2", NOW), OutboxEventStatus.IN_FLIGHT, 1, 2), + "deferredAttemptCount must be between 0 and attemptCount (1), was 2")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } +} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java index a5a6a3fa..b22ab546 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java @@ -49,7 +49,8 @@ class ClaimedOutboxEventContractTest { // then assertThat(thrown) - .isInstanceOf(IllegalArgumentException.class) + .as("a store builds the claim from its row, so a bad count is the store's fault") + .isInstanceOf(IllegalStateException.class) .hasMessage("attemptCount must be >= 0, was -1"); } } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java index b7fe8a5c..a913ffb4 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java @@ -4,11 +4,13 @@ import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; -import java.util.ArrayList; +import java.time.Duration; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; /** * The {@link IntegrationEventPublishPort} fake: records every event it is handed and answers with @@ -17,8 +19,55 @@ import java.util.concurrent.CompletionStage; */ public final class FakeIntegrationEventPublisher implements IntegrationEventPublishPort { - private final List published = new ArrayList<>(); + // A relay thread publishes while the test thread reads. + private final List published = new CopyOnWriteArrayList<>(); private OutboxPublishAttempt answer = OutboxPublishAttempt.confirmed(); + private boolean silent; + private Duration blocksFor = Duration.ZERO; + private RuntimeException fails; + private Optional longestAttempt = Optional.empty(); + + /** + * Never completes the stage of any later publish, and reports {@code longestAttempt} as the + * longest a publish takes: a broker that does not answer, behind a publisher that promised it + * would within that bound. + * + * @param longestAttempt what {@link #longestAttempt()} reports + * @return this fake + */ + public FakeIntegrationEventPublisher silentPast(Duration longestAttempt) { + this.longestAttempt = Optional.of(Objects.requireNonNull(longestAttempt, "longestAttempt")); + this.silent = true; + return this; + } + + /** + * Holds the caller for {@code blocking} inside every later {@code publish}, before the stage is + * handed back: a send that blocks its caller, as a producer waiting on cluster metadata does. + * + * @param blocking how long {@code publish} blocks + * @return this fake + */ + public FakeIntegrationEventPublisher blockingFor(Duration blocking) { + this.blocksFor = Objects.requireNonNull(blocking, "blocking"); + return this; + } + + /** + * Completes every later publish's stage exceptionally with {@code failure}. + * + * @param failure what the stage fails with + * @return this fake + */ + public FakeIntegrationEventPublisher failingWith(RuntimeException failure) { + this.fails = Objects.requireNonNull(failure, "failure"); + return this; + } + + @Override + public Optional longestAttempt() { + return longestAttempt; + } /** * Answers every later publish with {@code outcome}, and for anything but a confirmation the @@ -45,7 +94,17 @@ public final class FakeIntegrationEventPublisher implements IntegrationEventPubl @Override public CompletionStage publish(ValidatedIntegrationEvent event) { published.add(Objects.requireNonNull(event, "event must not be null")); - return CompletableFuture.completedFuture(answer); + if (!blocksFor.isZero()) { + try { + Thread.sleep(blocksFor); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + if (fails != null) { + return CompletableFuture.failedFuture(fails); + } + return silent ? new CompletableFuture<>() : CompletableFuture.completedFuture(answer); } /** Every event handed to this fake, in order. */ diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java index 4be958d2..38f896aa 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -25,6 +26,10 @@ import org.junit.jupiter.api.Test; * *

Every call goes through {@link #inTransaction}, because the real store refuses writes without * a caller transaction and because a resubmission must leave that transaction usable. + * + *

A lease write names the {@link DurableOperationClaim} it holds. The fence cases are why: the + * same worker id claiming an operation again must fence out its own earlier run, which a check on + * the worker id alone lets through. */ public abstract class DurableOperationStoreContract { @@ -33,6 +38,9 @@ public abstract class DurableOperationStoreContract { private static final Duration TTL = Duration.ofDays(1); private static final String HASH_A = "a".repeat(64); private static final String HASH_B = "b".repeat(64); + private static final String HASH_C = "c".repeat(64); + private static final OperationFailure TIMED_OUT = + new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); /** The store under test, empty at the start of each test. */ protected abstract DurableOperationStorePort store(); @@ -182,10 +190,36 @@ public abstract class DurableOperationStoreContract { submit(submission); // when - Optional claimed = claim("worker-a", NOW); + Optional claimed = claim("worker-a", NOW); // then - assertThat(claimed).contains(submission); + assertThat(claimed).contains(new DurableOperationClaim(submission, "worker-a", 1)); + } + + @Test + void everyClaimOfAnOperationGetsAHigherFence() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim first = claim("worker-a", NOW).orElseThrow(); + + // when + DurableOperationClaim second = claim("worker-a", NOW.plus(LEASE).plusSeconds(1)).orElseThrow(); + + // then + assertThat(second.fence()).isGreaterThan(first.fence()); + } + + @Test + void canceledOperationIsNotHandedToAWorker() { + // given + submit(submission("op-1", null, HASH_A)); + cancel("op-1", "alice", NOW.plusSeconds(5)); + + // when + Optional claimed = claim("worker-a", NOW.plusSeconds(6)); + + // then + assertThat(claimed).isEmpty(); } @Test @@ -195,7 +229,7 @@ public abstract class DurableOperationStoreContract { claim("worker-a", NOW); // when - Optional claimed = claim("worker-b", NOW.plusSeconds(30)); + Optional claimed = claim("worker-b", NOW.plusSeconds(30)); // then assertThat(claimed).isEmpty(); @@ -224,11 +258,11 @@ public abstract class DurableOperationStoreContract { void heartbeatFromTheLeaseHolderExtendsTheLease() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant beat = NOW.plusSeconds(60); // when - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, beat)); + inTransaction(() -> store().heartbeat(held, LEASE, beat)); // then assertThat(find("op-1", "alice", beat).orElseThrow().lease()) @@ -239,13 +273,12 @@ public abstract class DurableOperationStoreContract { void heartbeatFromAWorkerThatLostTheLeaseIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when - boolean extended = - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, later.plusSeconds(1))); + boolean extended = inTransaction(() -> store().heartbeat(stale, LEASE, later.plusSeconds(1))); // then assertThat(extended).isFalse(); @@ -255,12 +288,12 @@ public abstract class DurableOperationStoreContract { void progressFromTheLeaseHolderIsPublished() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationProgressSnapshot progress = new OperationProgressSnapshot(3, Optional.of(10L), Optional.of("extracting")); // when - inTransaction(() -> store().reportProgress(id("op-1"), "worker-a", progress)); + inTransaction(() -> store().reportProgress(held, progress)); // then assertThat(find("op-1", "alice", NOW).orElseThrow().progress()).contains(progress); @@ -270,12 +303,13 @@ public abstract class DurableOperationStoreContract { void progressFromAnotherWorkerIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + DurableOperationClaim impostor = + new DurableOperationClaim(held.submission(), "worker-b", held.fence()); // when boolean published = - inTransaction( - () -> store().reportProgress(id("op-1"), "worker-b", OperationProgressSnapshot.of(1))); + inTransaction(() -> store().reportProgress(impostor, OperationProgressSnapshot.of(1))); // then assertThat(published).isFalse(); @@ -285,11 +319,11 @@ public abstract class DurableOperationStoreContract { void successFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant done = NOW.plusSeconds(10); // when - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", done)); + inTransaction(() -> store().succeed(held, "/results/a", done)); // then assertThat(find("op-1", "alice", done).orElseThrow()) @@ -309,28 +343,106 @@ public abstract class DurableOperationStoreContract { void staleWorkerCannotRecordAResultOverItsSuccessor() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when boolean recorded = - inTransaction( - () -> store().succeed(id("op-1"), "worker-a", "/results/a", later.plusSeconds(1))); + inTransaction(() -> store().succeed(stale, "/results/a", later.plusSeconds(1))); // then assertThat(recorded).isFalse(); } + @Test + void heartbeatOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().heartbeat(earlier, LEASE, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void progressOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().reportProgress(earlier, OperationProgressSnapshot.of(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void successOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void failureOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().fail(earlier, TIMED_OUT, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void resultOfTheRunThatTookOverSurvivesItsSameWorkerPredecessor() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + DurableOperationClaim current = claim("worker-a", later).orElseThrow(); + inTransaction(() -> store().succeed(current, "/results/current", later.plusSeconds(1))); + + // when + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(2))); + + // then + assertThat(find("op-1", "alice", later.plusSeconds(2)).orElseThrow().resultReference()) + .contains("/results/current"); + } + @Test void failureFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationFailure failure = new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); // when - inTransaction(() -> store().fail(id("op-1"), "worker-a", failure, NOW.plusSeconds(10))); + inTransaction(() -> store().fail(held, failure, NOW.plusSeconds(10))); // then assertThat(find("op-1", "alice", NOW.plusSeconds(10)).orElseThrow()) @@ -369,8 +481,8 @@ public abstract class DurableOperationStoreContract { void cancelNeverReversesASuccess() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); // when cancel("op-1", "alice", NOW.plusSeconds(20)); @@ -407,19 +519,35 @@ public abstract class DurableOperationStoreContract { } @Test - void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + void reclaimLeavesALiveLeaseAlone() { // given submit(submission("op-1", null, HASH_A)); claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + + // when + List reclaimed = + inTransaction(() -> store().reclaimExpiredLeases(NOW.plusSeconds(30))); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); submit(submission("op-2", null, HASH_B)); + submit(submission("op-3", null, HASH_C)); + claim("worker-b", NOW.plusSeconds(20)); // when List expired = inTransaction(() -> store().expireStaleOperations(NOW.plus(TTL).plus(TTL))); // then - assertThat(expired).containsExactly(id("op-2")); + assertThat(expired).containsExactlyInAnyOrder(id("op-2"), id("op-3")); } private DurableOperationSubmitResult submit(DurableOperationSubmission submission) { @@ -430,7 +558,7 @@ public abstract class DurableOperationStoreContract { return inTransaction(() -> store().find(id(operationId), principal, now)); } - private Optional claim(String workerId, Instant now) { + private Optional claim(String workerId, Instant now) { return inTransaction(() -> store().claimNext(workerId, LEASE, now)); } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java index 019193c3..3c76c99a 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java @@ -1,6 +1,7 @@ package dev.caskeleton.application.operation.testkit; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -26,8 +27,10 @@ import java.util.concurrent.ConcurrentHashMap; *

Written to the same rules rather than to whatever makes the tests pass: a submission scope * that already exists returns the existing operation, a lease is checked on every write, and a * terminal operation is never reopened. A fake that was laxer than the adapter would let the use - * case tests certify behaviour the real store refuses. {@link DurableOperationStoreContract} holds - * this fake and the adapter to the same cases. + * case tests certify behaviour the real store refuses. Every claim increments the operation's + * fence, and a write is accepted only from the claim holding the current fence, as the adapter's + * {@code lease_fence} column does. {@link DurableOperationStoreContract} holds this fake and the + * adapter to the same cases. * *

Built over a {@link DirectTransactionPort}, every write made inside one of its transactions is * undone when that transaction rolls back, as the database undoes it; built without one, writes @@ -41,6 +44,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor private final Map byScope = new ConcurrentHashMap<>(); private final Map submissions = new ConcurrentHashMap<>(); + private final Map fences = new ConcurrentHashMap<>(); private final DirectTransactionPort transactions; /** A store whose writes stay as they are made. */ @@ -103,7 +107,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public Optional claimNext( + public Optional claimNext( String workerId, Duration leaseDuration, Instant now) { return byId.values().stream() .filter( @@ -126,19 +130,22 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty(), Optional.of(new OperationLease(workerId, now, now.plus(leaseDuration)))); write(claimed.operationId(), claimed); - return submissions.get(claimed.operationId()); + long previous = fences.getOrDefault(claimed.operationId(), 0L); + fences.put(claimed.operationId(), previous + 1); + onRollback(() -> fences.put(claimed.operationId(), previous)); + return new DurableOperationClaim( + submissions.get(claimed.operationId()), workerId, previous + 1); }); } @Override - public boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) { - DurableOperation operation = byId.get(operationId); - if (!holdsLease(operation, workerId, now)) { + public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) { + DurableOperation operation = byId.get(claim.operationId()); + if (!holdsLease(operation, claim, now)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -152,14 +159,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -173,17 +179,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean succeed( - DurableOperationId operationId, - String workerId, - String resultReference, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.SUCCEEDED, @@ -197,17 +199,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.FAILED, @@ -316,6 +314,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor byId.clear(); byScope.clear(); submissions.clear(); + fences.clear(); } // Same as the adapter: a poll after the TTL sees EXPIRED even before the sweeper runs. @@ -334,13 +333,15 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty()); } - private static boolean holdsLease(DurableOperation operation, String workerId, Instant now) { - return operation != null && ownedBy(operation, workerId) && !operation.leaseExpiredAt(now); + private boolean holdsLease(DurableOperation operation, DurableOperationClaim claim, Instant now) { + return heldBy(operation, claim) && !operation.leaseExpiredAt(now); } - private static boolean ownedBy(DurableOperation operation, String workerId) { - return operation.state() == DurableOperationState.RUNNING - && operation.lease().map(lease -> lease.workerId().equals(workerId)).orElse(false); + private boolean heldBy(DurableOperation operation, DurableOperationClaim claim) { + return operation != null + && operation.state() == DurableOperationState.RUNNING + && operation.lease().map(lease -> lease.workerId().equals(claim.workerId())).orElse(false) + && fences.getOrDefault(claim.operationId(), 0L) == claim.fence(); } private static DurableOperation replace( diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java index 4577fec0..ad90e67d 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java @@ -44,6 +44,16 @@ public interface CleanupQueuePort { /** Records a failed attempt and when to try again, if {@code item}'s claim is still current. */ void markFailed(CleanupItem item, String reasonCode, Instant nextAttemptAt); + /** + * Gives a claimed item back unattempted, if {@code item}'s claim is still current. + * + *

For an item the batch claimed and then did not start — its byte budget or its claim's lease + * ran out first. The claim ends and the item is claimable again at {@code nextAttemptAt}; the + * attempt counter does not move, because nothing was tried. Recorded as a failure instead, every + * such release spent an attempt, and a busy queue's tail was abandoned without ever being tried. + */ + void release(CleanupItem item, Instant nextAttemptAt); + /** * Parks an item for good, if {@code item}'s claim is still current. * diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java index d28a7d40..71033678 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java @@ -49,6 +49,20 @@ public final class DefaultCleanupService implements CleanupService { */ public static final Duration CLAIM_LEASE = Duration.ofMinutes(10); + /** + * No claimed item is started once less than this is left of the batch's claim lease. + * + *

A batch claims up to {@code maxItems} under one lease and works them one after another, so + * the last item starts long after the claim. Past the lease, the reaper may already have handed + * that item to another worker, and both delete it and settle its quota. The outbox relay checks + * its batch against its lease when it is assembled and the notification scheduler claims only + * what it can start; cleanup has no bound on one delete to check against, so it stops starting + * items instead, and releases the rest for the next batch the way the byte budget does (round9 + * review 2026-11-12, RR9-01-23). Two minutes is the time one delete against slow storage is + * allowed to finish inside its lease. + */ + static final Duration LAST_START_BEFORE_LEASE_ENDS = Duration.ofMinutes(2); + private final CleanupQueuePort queue; private final CleanupContentPort contentGateway; private final FileMetadataPort metadataStore; @@ -95,6 +109,7 @@ public final class DefaultCleanupService implements CleanupService { // claiming means the recovered items are eligible in this same pass. transactions.inWrite(() -> queue.reclaimExpiredClaims(now, maxItems)); List due = transactions.inWrite(() -> queue.claimDue(now, maxItems, CLAIM_LEASE)); + Instant lastStart = now.plus(CLAIM_LEASE).minus(LAST_START_BEFORE_LEASE_ENDS); int deleted = 0; int skippedActiveLease = 0; @@ -113,10 +128,16 @@ public final class DefaultCleanupService implements CleanupService { failed++; continue; } + if (!clock.instant().isBefore(lastStart)) { + // The claim's lease is nearly spent; an item started now could still be running when the + // reaper hands it to another worker. Given back untried, like the byte budget's. + release(item, now); + continue; + } if (reclaimed >= maxBytes) { - // The byte budget is spent; the remaining claims are released for the next batch rather - // than being marked done, so nothing is silently dropped. - markFailed(item, "BATCH_BYTE_BUDGET_EXHAUSTED", now); + // The byte budget is spent; the remaining claims are given back untried rather than + // being marked done, so nothing is silently dropped and no attempt is spent on them. + release(item, now); continue; } Outcome outcome = process(item, now); @@ -330,6 +351,11 @@ public final class DefaultCleanupService implements CleanupService { }); } + /** Gives a claimed item back untried, in a write transaction of its own. */ + private void release(CleanupItem item, Instant nextAttemptAt) { + transactions.inWrite(() -> queue.release(item, nextAttemptAt)); + } + /** * Reschedules an item, or parks it once this failure spends its retry budget. * diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java index dd0bd88d..ead9afb2 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java @@ -26,7 +26,10 @@ import dev.caskeleton.application.fileserver.testkit.ReadyFileFixture; import dev.caskeleton.application.fileserver.testkit.RecordingQuotaReclaimGateway; import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.List; import java.util.OptionalLong; import java.util.UUID; import org.junit.jupiter.api.DisplayName; @@ -321,7 +324,91 @@ class CleanupServiceTest { CleanupBatchResult result = cleanup.runBatch(100, 1); assertThat(result.deleted()).isEqualTo(1); - assertThat(queue.failed().values()).contains("BATCH_BYTE_BUDGET_EXHAUSTED"); + assertThat(List.of(queue.released().size(), queue.failed().size())) + .as("the deferred item is given back untried; recorded as failed it spent an attempt") + .containsExactly(1, 0); + } + + @Test + void itemsTheClaimLeaseNoLongerCoversAreReleasedNotStarted() { + // given + // Each staging delete takes just over half the time an item may still start in (the lease less + // its last two minutes): the first two items run, the third is handed back to the queue. + MovableClock slowStorage = new MovableClock(FileserverFixtures.NOW); + Duration perDelete = + DefaultCleanupService.CLAIM_LEASE + .minus(DefaultCleanupService.LAST_START_BEFORE_LEASE_ENDS) + .dividedBy(2) + .plusSeconds(1); + content.afterEachStagingDiscard(() -> slowStorage.advance(perDelete)); + CleanupService timed = + new DefaultCleanupService( + queue, + content, + metadata, + sessions, + quota, + Duration.ofMinutes(5), + FileserverMetricsPort.noop(), + transactions, + slowStorage); + for (int item = 0; item < 3; item++) { + cancelledStagingItem(); + } + + // when + CleanupBatchResult result = timed.runBatch(100, 1L << 30); + + // then + assertThat(List.of(result.deleted(), queue.released().size(), queue.failed().size())) + .as( + "the third item comes back untried rather than failed; that a release leaves the" + + " attempt counter where it was is the queue contract's") + .containsExactly(2, 1, 0); + } + + private void cancelledStagingItem() { + UploadId uploadId = UploadId.of(UUID.randomUUID()); + FileId fileId = FileId.of(UUID.randomUUID()); + sessions.create( + new UploadSessionDraft( + uploadId, + fileId, + UploadMode.SINGLE_SHOT, + OptionalLong.of(SIZE), + FileserverFixtures.NOW.plusSeconds(3600))); + sessions.terminate(uploadId); + content.storeStaging(uploadId); + queue.enqueue(CleanupRequest.forStaging(CleanupType.CANCELLED_STAGING, fileId, uploadId)); + } + + /** A clock the slow-storage gateway moves forward. */ + private static final class MovableClock extends Clock { + + private Instant now; + + private MovableClock(Instant start) { + this.now = start; + } + + private void advance(Duration duration) { + now = now.plus(duration); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return now; + } } @Test diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java index 0db36011..b9cc169b 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java @@ -119,6 +119,40 @@ public abstract class CleanupQueuePortContract { }); } + @Test + void aReleasedItemIsClaimableAgainWithItsAttemptUnchanged() { + // given + enqueue(contentCleanup()); + CleanupItem item = claim(due(), 10).get(0); + settle(() -> queue().release(item, due())); + + // when + List again = claim(due(), 10); + + // then + assertThat(again) + .singleElement() + .extracting(CleanupItem::cleanupId, CleanupItem::attempt) + .containsExactly(item.cleanupId(), 0); + } + + @Test + void aReleaseUnderAReclaimedTokenChangesNothing() { + // given + enqueue(contentCleanup()); + CleanupItem stale = claim(due(), 10).get(0); + settle(() -> queue().release(stale, due())); + var unused = claim(due(), 10); + + // when + settle(() -> queue().release(stale, due())); + + // then + assertThat(claim(due(), 10)) + .as("the stale release would hand the current claim's item to a second worker") + .isEmpty(); + } + @Test @DisplayName("완료된 항목은 다시 claim되지 않는다") void aDoneItemIsNeverClaimedAgain() { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java index 8a485a3c..3b842068 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java @@ -7,6 +7,7 @@ import dev.caskeleton.application.fileserver.api.content.DeleteResult; import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; /** Tracks which physical objects still exist so a test can prove nothing live was removed. */ @@ -16,6 +17,7 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { private final Set stagingObjects = new LinkedHashSet<>(); private final Set quarantined = new LinkedHashSet<>(); private final List preconditions = new java.util.ArrayList<>(); + private Runnable afterStagingDiscard = () -> {}; public void store(ContentKey key) { objects.add(key); @@ -46,9 +48,20 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { return DeleteResult.removed(precondition.expectedSize().orElse(0)); } + /** + * Runs {@code hook} after every staging discard, for a test that needs the storage to be slow — + * advancing its clock — or to fail mid-batch. + * + * @param hook what a discard does after removing the staging object + */ + public void afterEachStagingDiscard(Runnable hook) { + afterStagingDiscard = Objects.requireNonNull(hook, "hook"); + } + @Override public void discardStaging(UploadId uploadId) { stagingObjects.remove(uploadId); + afterStagingDiscard.run(); } public void quarantine(ContentKey key) { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java index 50edd600..0dc81d84 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java @@ -35,6 +35,7 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { private final Map rows = new LinkedHashMap<>(); private final List done = new ArrayList<>(); private final Map failed = new LinkedHashMap<>(); + private final List released = new ArrayList<>(); private final Map abandoned = new LinkedHashMap<>(); private final List reclaimSweeps = new ArrayList<>(); private final List claimSweeps = new ArrayList<>(); @@ -108,6 +109,15 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { } } + @Override + public void release(CleanupItem item, Instant nextAttemptAt) { + Row row = rows.get(item.cleanupId()); + if (row != null && item.claimToken().equals(row.claimToken)) { + row.release(nextAttemptAt); + released.add(item); + } + } + @Override public void markAbandoned(CleanupItem item, String reasonCode) { if (settle(item, Status.ABANDONED, clock.instant())) { @@ -151,6 +161,11 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { return Map.copyOf(failed); } + /** Items the caller gave back untried, in order. */ + public List released() { + return List.copyOf(released); + } + /** Items the caller parked for good, with the reason given. */ public Map abandoned() { return Map.copyOf(abandoned); @@ -180,6 +195,13 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { this.nextAttemptAt = nextAttemptAt; } + private void release(Instant nextAttemptAt) { + this.status = attempt == 0 ? Status.PENDING : Status.FAILED; + this.nextAttemptAt = nextAttemptAt; + this.claimToken = null; + this.leaseUntil = null; + } + private void settle(Status status, Instant nextAttemptAt) { this.status = status; this.attempt++; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java index bd902930..40006855 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java @@ -11,6 +11,10 @@ import java.util.Optional; * the work is done; and the same id with a different command is a mistake worth reporting rather * than an idempotent repeat. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param outcome what the caller may do * @param existing the earlier result, when there is one */ @@ -37,7 +41,7 @@ public record AdminOperationClaim(Outcome outcome, OptionalA requeue is one write conditional on the abandoned lease recovery found (owner, fence, state, + * expiry), not a read followed by an unconditional transition. The pass runs on every instance, and + * between finding a job and writing it another instance's recovery may have requeued it and a new + * worker claimed it, or the holder may have finished; the unconditional write put such a job back + * to {@code READY_TO_DISPATCH} and it was dispatched again (round9 review 2026-11-12, RR9-01-12). A + * requeue that matches nothing is not counted as handled. + * *

Every write opens its own transaction; the pass itself runs on a bare scheduler thread. The * stores behind these ports are modifying JPA queries, which throw outside a transaction — so the * pass used to fail on its first statement, every time, and the scheduler only logged it. It lives @@ -110,14 +116,14 @@ public final class LeaseRecoveryService { // Retiring expired deliveries rides along with recovery because both answer the same operator // question — "why is this row still here?" — and neither needs a scheduler of its own. int handled = transactions.inWrite(() -> leases.expireOverdue(batchSize)); - List abandoned = leases.expiredDispatching(batchSize, staleAfter); - for (var recipientDeliveryId : abandoned) { + List abandoned = leases.expiredDispatching(batchSize, staleAfter); + for (var lease : abandoned) { + RecipientDeliveryId recipientDeliveryId = lease.recipientDeliveryId(); var attemptsOfDelivery = attempts.attemptsOf(recipientDeliveryId); if (attemptsOfDelivery.isEmpty()) { // Crash before the attempt row: no provider call can have happened, so this is the one // safe requeue. Leaving it DISPATCHING stranded the delivery permanently. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } if (attemptsOfDelivery.stream().allMatch(attempt -> attempt.completedAt().isPresent())) { @@ -127,8 +133,7 @@ public final class LeaseRecoveryService { // the delivery, and dispatch derives the next step from exactly those facts — an accepted // submission stops as COMPLETED, an ambiguous one goes to reconciliation — so handing it // back to dispatch cannot send a second copy of something the provider accepted. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } for (var attempt : attemptsOfDelivery) { @@ -141,25 +146,25 @@ public final class LeaseRecoveryService { // identically, so every abandoned attempt had to be reconciled. The pre-call row the // dispatcher writes never lands here: it records the start as unknown, because it is // still the stored state while the provider call runs. - requeue(recipientDeliveryId); + handled += requeue(lease); } else { // Outside any transaction: reconciliation asks the provider over the network and opens // its own boundaries for what it writes. reconciliation.reconcile(attempt.id()); + handled++; } - handled++; } } } return handled; } - private void requeue(RecipientDeliveryId recipientDeliveryId) { - transactions.inWrite( - () -> - deliveries.transition( - recipientDeliveryId, - RecipientDeliveryState.READY_TO_DISPATCH, - java.util.Optional.of(clock.instant()))); + /** Requeues the abandoned dispatch; 1 when it was still abandoned and moved, 0 otherwise. */ + private int requeue(RecipientLease abandoned) { + return transactions + .inWrite(() -> deliveries.requeueAbandoned(abandoned, clock.instant())) + .isPresent() + ? 1 + : 0; } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java index 101c9f35..c82fa5aa 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java @@ -49,4 +49,28 @@ public interface RecipientDeliveryStorePort { RecipientDeliveryState state, Optional nextDispatchAt, RecipientLease lease); + + /** + * Returns a job a dead worker left dispatching to the queue, only while it is still the abandoned + * dispatch recovery found. + * + *

One conditional write on the abandoned lease — same owner, same fence, still {@code + * DISPATCHING}, lease not extended past what recovery saw — rather than a read followed by an + * unconditional transition. Recovery runs on every instance, and between finding the job and + * writing it another instance may have requeued it and a new worker claimed it, or the original + * holder may have finished; an unconditional write put such a job back to {@code + * READY_TO_DISPATCH} and it was dispatched again. + * + *

The target state is the operation's meaning, as {@code EXPIRED} is {@code + * RecipientLeaseStorePort.expireOverdue}'s, so it is not an argument. What recovery decides is + * which operation applies: this one when no attempt may have reached the provider, the + * reconciliation path otherwise. + * + * @param abandoned the lease as {@link RecipientLeaseStorePort#expiredDispatching} returned it + * @param nextDispatchAt when the job becomes due again + * @return the requeued job, or empty when it is no longer that abandoned dispatch and nothing was + * written + */ + Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java index 0b9962a3..5660e4d3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java @@ -16,6 +16,10 @@ import java.util.Objects; * that no longer exists and updates nothing. It travels with the lease rather than being re-read, * because re-reading is the race. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param recipientDeliveryId the job this lease is for * @param owner the holder, including the process incarnation that claimed it * @param fence the claim generation; a lower fence has been superseded @@ -29,11 +33,11 @@ public record RecipientLease( Objects.requireNonNull(owner, "owner"); Objects.requireNonNull(leaseUntil, "leaseUntil"); if (owner.isBlank()) { - throw new IllegalArgumentException("owner"); + throw new IllegalStateException("a lease needs an owner"); } if (fence < 1) { // A zero fence is the value an unclaimed row holds; a lease can never legitimately carry it. - throw new IllegalArgumentException("fence"); + throw new IllegalStateException("fence must be >= 1, was " + fence); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java index 634fabd7..da251a5d 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java @@ -1,6 +1,5 @@ package dev.caskeleton.application.notification.platform.dispatch; -import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import java.time.Duration; import java.util.List; import java.util.Optional; @@ -25,6 +24,9 @@ public interface RecipientLeaseStorePort { /** * Extend a lease that is still being worked on. * + *

The template's dispatcher never calls it: a dispatch runs inside the lease it claimed. It is + * here for a deployment whose own dispatch outlasts one lease. + * * @return the extended lease, or empty when this holder no longer owns the job */ Optional renew(RecipientLease lease, Duration leaseDuration); @@ -40,8 +42,24 @@ public interface RecipientLeaseStorePort { /** Whether this lease is still the current one, for checking before an external side effect. */ boolean stillHeld(RecipientLease lease); - /** Jobs left {@code DISPATCHING} by a worker that died, for reconciliation. */ - List expiredDispatching(int limit, Duration olderThan); + /** + * Jobs left {@code DISPATCHING} by a worker that died, for recovery. + * + *

Each comes back as the lease it was abandoned under — owner, fence and expiry as they stood + * when it was found. Recovery decides what to do from attempts it reads afterwards, and the lease + * is what lets its write say "only if this is still that abandoned dispatch": another instance's + * recovery, a new claim or the holder itself may have moved the job in between. + * + *

Only jobs a claim leased: an owner and a fence of at least one. A {@code DISPATCHING} job + * without them is no lease anyone holds — one left before the fence existed, or one moved there + * without a claim — and is not returned; offered, it would be a lease nobody can build, and the + * pass would fail on it every time. + * + * @param limit how many to return + * @param olderThan how long a lease must have been expired before the job counts as abandoned + * @return the abandoned leases, oldest expiry first + */ + List expiredDispatching(int limit, Duration olderThan); /** * Retire deliveries whose window closed before anyone claimed them. diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java index c8545b62..e0d3e401 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java @@ -24,6 +24,10 @@ import java.util.UUID; * the store's writes after the claim match on it, so an instance that held the job past its lease * cannot reschedule or remove it once another instance has claimed it since. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param id the job identity * @param attemptId the attempt whose outcome is unknown * @param providerProfileId the profile that must answer, since capability is per profile @@ -55,10 +59,10 @@ public record ReconciliationJob( Objects.requireNonNull(createdAt, "createdAt"); Objects.requireNonNull(updatedAt, "updatedAt"); if (attempts < 0) { - throw new IllegalArgumentException("attempts must not be negative"); + throw new IllegalStateException("attempts must not be negative"); } if (claimFence < 0) { - throw new IllegalArgumentException("claimFence must not be negative"); + throw new IllegalStateException("claimFence must not be negative"); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java index ce48cf07..7ddece3f 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java @@ -12,7 +12,7 @@ import java.util.Objects; * being applied reported exactly the same numbers as an idle healthy one. Readiness that cannot * distinguish those two states is not readiness. * - * @param backlogDepth deliveries due now and not yet claimed + * @param backlogDepth deliveries a claim would take now: due, not expired, not under a live lease * @param oldestDueAge how long the oldest of them has been due * @param stuckLeases leases whose holder has not finished and whose expiry has passed * @param pendingProjections provider events stored and not yet applied diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java index 79a817aa..f6ae4f15 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java @@ -6,7 +6,25 @@ public interface NotificationServingStatePort { /** * Measure the backlog, the stuck leases and the projection and reconciliation lag. * + *

Several questions, for the health snapshot. A caller that needs only the queue depth asks + * {@link #backlogDepth()} instead: the scheduler reads the depth on every poll, and asking all of + * them that often ran four statements a poll, one of them over the whole provider-event table + * (round9 review 2026-11-12, RR9-01-14). + * * @return the current state */ NotificationServingState currentState(); + + /** + * How many deliveries a claim would take now — due, not expired, and not under a live lease: the + * {@link NotificationServingState#backlogDepth()} of {@link #currentState()}, measured alone. + * + *

The default takes it from the full snapshot, so an implementation that can only measure + * everything at once still answers; a store that can count the backlog by itself overrides it. + * + * @return the backlog depth + */ + default long backlogDepth() { + return currentState().backlogDepth(); + } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java index 7ff29358..ed508150 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java @@ -186,6 +186,40 @@ class LeaseRecoveryServiceTest { assertThat(recovery.recoverOnce()).isEqualTo(3); } + @Test + void aDeliveryItsHolderFinishedAfterRecoveryFoundItIsNotRequeued() { + // given + RecipientDeliveryId id = jobOf(NOW, Optional.empty()); + RecipientLease lease = queue.claim("worker-a", 10, Duration.ofSeconds(30)).getFirst(); + workerDied(); + // The holder completes the job between recovery finding it abandoned and recovery writing it + // — the window another instance or a slow worker uses in production. + attempts.beforeAttemptsOf( + delivery -> { + var unused = + queue.transitionHeldBy( + delivery, RecipientDeliveryState.COMPLETED, Optional.empty(), lease); + }); + LeaseRecoveryService racing = + new LeaseRecoveryService( + queue, + attempts, + queue, + reconciled::add, + new DirectTransactionPort(), + CLOCK, + Duration.ofMinutes(1), + 10); + + // when + racing.recoverOnce(); + + // then + assertThat(stateOf(id)) + .as("the holder was only slow; recovery's requeue must not reopen what it finished") + .isEqualTo(RecipientDeliveryState.COMPLETED); + } + private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence notStarted() { return dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..dd33283f --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java @@ -0,0 +1,76 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.admin.AdminOperationClaim; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.api.ProviderProfileId; +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * The leases, jobs and claims a notification store builds from its rows refuse a bad value as each + * record says. + */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + RecipientDeliveryId delivery = new RecipientDeliveryId(UUID.randomUUID()); + return Stream.of( + Arguments.of( + "owner 없는 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, " ", 1, NOW), + "a lease needs an owner"), + Arguments.of( + "fence 0인 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, "worker-1", 0, NOW), + "fence must be >= 1, was 0"), + Arguments.of( + "시도 수가 음수인 맞추기 job", + (ThrowingCallable) () -> job(-1, 0), + "attempts must not be negative"), + Arguments.of( + "fence가 음수인 맞추기 job", + (ThrowingCallable) () -> job(0, -1), + "claimFence must not be negative"), + Arguments.of( + "결과 없는 운영자 재생", + (ThrowingCallable) + () -> new AdminOperationClaim(AdminOperationClaim.Outcome.REPLAY, Optional.empty()), + "a replay must carry the result it is replaying")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } + + private static ReconciliationJob job(int attempts, long claimFence) { + return new ReconciliationJob( + UUID.randomUUID(), + new DeliveryAttemptId(UUID.randomUUID()), + new ProviderProfileId("email-main"), + NOW, + attempts, + Optional.empty(), + NOW, + NOW, + claimFence); + } +} diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java index ae257085..65bc4d4d 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; /** * The fake of the attempt store and the two read models built on it, held to {@link @@ -47,6 +48,8 @@ public final class InMemoryDeliveryAttemptStore /** Stored attempts, in insertion order, as their rows read back. */ private final Map rows = new LinkedHashMap<>(); + private volatile Consumer beforeAttemptsOf = id -> {}; + private final Map requestIdHashes = new LinkedHashMap<>(); private final Map engagement = new LinkedHashMap<>(); private final Map suppression = new LinkedHashMap<>(); @@ -140,7 +143,23 @@ public final class InMemoryDeliveryAttemptStore } @Override - public synchronized List attemptsOf( + public List attemptsOf(RecipientDeliveryId recipientDeliveryId) { + // Outside the monitor: the hook writes to the recipient queue, as another worker would. + beforeAttemptsOf.accept(recipientDeliveryId); + return attemptsFor(recipientDeliveryId); + } + + /** + * Runs {@code hook} before every {@link #attemptsOf} read, for a test that needs another worker + * to act between a caller finding a delivery and reading its attempts. + * + * @param hook what happens, given the delivery being read + */ + public void beforeAttemptsOf(Consumer hook) { + beforeAttemptsOf = Objects.requireNonNull(hook, "hook"); + } + + private synchronized List attemptsFor( RecipientDeliveryId recipientDeliveryId) { return rows.values().stream() .filter(row -> row.recipientDeliveryId().equals(recipientDeliveryId)) @@ -155,7 +174,7 @@ public final class InMemoryDeliveryAttemptStore @Override public synchronized int nextAttemptNo(RecipientDeliveryId recipientDeliveryId) { - return attemptsOf(recipientDeliveryId).stream() + return attemptsFor(recipientDeliveryId).stream() .mapToInt(DeliveryAttemptRecord::attemptNo) .max() .orElse(0) diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java index 89517a47..7714a522 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java @@ -160,7 +160,7 @@ public final class InMemoryRecipientQueue } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); @@ -170,9 +170,18 @@ public final class InMemoryRecipientQueue return jobs() .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) .filter(job -> job.leaseUntil().filter(until -> until.isBefore(threshold)).isPresent()) + // Only what a claim wrote: an owner and a fence. The JPA store skips the rest the same + // way. + .filter(job -> job.leaseOwner().isPresent() && fences.getOrDefault(job.id(), 0L) > 0) .sorted(Comparator.comparing((RecipientDeliveryRecord job) -> job.leaseUntil().get())) .limit(limit) - .map(RecipientDeliveryRecord::id) + .map( + job -> + new RecipientLease( + job.id(), + job.leaseOwner().orElseThrow(), + fences.getOrDefault(job.id(), 0L), + job.leaseUntil().orElseThrow())) .toList(); } } @@ -294,6 +303,35 @@ public final class InMemoryRecipientQueue } } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + synchronized (requests) { + return heldBy(abandoned) + .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) + .filter( + job -> + job.leaseUntil() + .filter(until -> !until.isAfter(abandoned.leaseUntil())) + .isPresent()) + .map( + job -> { + RecipientDeliveryRecord moved = + copy( + job, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(nextDispatchAt), + job.leaseOwner(), + job.leaseUntil(), + clock.instant()); + requests.replace(moved); + return moved; + }); + } + } + // ---- rows ------------------------------------------------------------------------------------- private Stream jobs() { diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java index 7ceefe82..5ec3dead 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java @@ -244,6 +244,84 @@ public abstract class RecipientDeliveryStoreContract { List.of(Optional.empty(), Optional.empty()), RecipientDeliveryState.DISPATCHING, 0); } + @Test + void anAbandonedDispatchIsRequeued() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat( + requeued.map(stored -> List.of(stored.id(), stored.state(), stored.nextDispatchAt()))) + .contains( + List.of( + job.id(), + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW.plus(LEASE).plusSeconds(1)))); + } + + @Test + void requeueOfAJobItsHolderFinishedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease lease = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction( + () -> + deliveries() + .transitionHeldBy( + job.id(), RecipientDeliveryState.COMPLETED, Optional.empty(), lease)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(lease, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.COMPLETED); + } + + @Test + void requeueOfAJobAnotherClaimTookWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + claim(); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .as("the second recovery found the same abandoned lease; the new claim raised the fence") + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + + @Test + void requeueOfAJobWhoseLeaseWasRenewedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease found = claim(); + clock.moveTo(NOW.plus(LEASE).minusSeconds(1)); + inTransaction(() -> leases().renew(found, LEASE)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(found, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + private RecipientDeliveryRecord givenAQueuedJob() { NotificationRequestRecord request = NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), NOW); diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java index 0a97a7c1..4ddfb077 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java @@ -278,25 +278,77 @@ public abstract class RecipientLeaseStoreContract { } @Test - @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만 복구 대상으로 나온다") + @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만, 버려질 때의 lease로 복구 대상에 나온다") void onlyJobsWhoseLeaseEndedLongEnoughAgoAreAbandoned() { // given - RecipientDeliveryId abandoned = given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); - claim("worker-a"); + given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); + RecipientLease abandoned = claim("worker-a").getFirst(); clock.moveTo(NOW.plus(LEASE).plusSeconds(30)); given(RecipientDeliveryState.READY_TO_DISPATCH, NOW.plus(LEASE), null); claim("worker-b"); clock.moveTo(NOW.plus(LEASE).plus(Duration.ofMinutes(2))); // when - List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); // then assertThat(expired) - .as("a job whose lease is still running belongs to a live worker") + .as( + "a job whose lease is still running belongs to a live worker; the abandoned one comes" + + " back as the lease it was abandoned under") .containsExactly(abandoned); } + @Test + @DisplayName("claim이 쓰지 않은 DISPATCHING 작업(owner는 있고 fence가 없음)은 복구 대상에 나오지 않는다") + void aDispatchingJobNoClaimWroteIsNotOfferedForRecovery() { + // given + // The shape a row left dispatching before the fence existed has: an owner and a lease expiry + // written by hand, and the fence column's default. + NotificationRequestRecord request = + NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), clock.instant()); + RecipientDeliveryRecord job = + NotificationRows.job( + request, + RecipientDeliveryState.DISPATCHING, + Optional.of(NOW), + Optional.empty(), + clock.instant()); + RecipientDeliveryRecord leasedByHand = + new RecipientDeliveryRecord( + job.id(), + job.notificationId(), + job.tenantId(), + job.recipientRef(), + job.locale(), + job.timeZone(), + job.routingPlan(), + job.routeCursor(), + job.state(), + job.submissionOutcome(), + job.deliveryOutcome(), + job.evidenceLevel(), + job.ambiguousAttemptExists(), + job.duplicateRisk(), + job.nextDispatchAt(), + job.expiresAt(), + Optional.of("legacy-worker"), + Optional.of(NOW.minus(Duration.ofHours(1))), + job.attemptCount(), + job.lastFailureCategory(), + job.createdAt(), + job.updatedAt()); + inTransaction(() -> requests().insert(request, List.of(leasedByHand))); + + // when + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + + // then + assertThat(expired) + .as("a lease the application cannot build would fail every recovery pass after it") + .isEmpty(); + } + private RecipientDeliveryId given( RecipientDeliveryState state, Instant dueAt, Instant expiresAt) { NotificationRequestRecord request = diff --git a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml index 959da41d..c48f1228 100644 --- a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml +++ b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml @@ -43,8 +43,9 @@ app: default-locale: ${APP_NOTIFICATION_PLATFORM_DEFAULT_LOCALE:ko-KR} dispatch: claim-batch-size: ${APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE:50} - # The lease must outlast a provider call plus its timeout, or a live worker's delivery is - # claimed by a second one while the first is still waiting on the provider. + # The lease must outlast a provider call, or a live worker's delivery is claimed by a second + # one while the first is still waiting on the provider. Startup refuses a lease that is not + # longer than every enabled provider profile's timeout. lease-duration: ${APP_NOTIFICATION_PLATFORM_LEASE_DURATION:2m} poll-interval: ${APP_NOTIFICATION_PLATFORM_POLL_INTERVAL:1s} max-global-concurrency: ${APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY:64} @@ -117,7 +118,7 @@ app: primary-for-channel: ${APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY:true} environment: ${APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT:local} credential-profile: ${APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE:default} - timeout: ${APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT:10s} + timeout: ${APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT:30s} max-concurrency: ${APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY:4} rate-per-second: ${APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND:10} ses: