diff --git a/.env.example b/.env.example index 1c68b40e..09bb6b74 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,7 @@ APP_CACHE_REDIS_COMMAND_TIMEOUT=200ms APP_CACHE_REDIS_POSITIVE_SOFT_TTL=30s APP_CACHE_REDIS_MINIMUM_HARD_TTL=1s APP_CACHE_REDIS_SEMANTIC_REGION=default +APP_CACHE_REDIS_MAXIMUM_VALUE_BYTES=1048576 APP_MESSAGING_BROKER=kafka APP_MESSAGING_PRODUCER_ID= APP_MESSAGING_KAFKA_BROKERS= @@ -293,7 +294,7 @@ APP_NOTIFICATION_PLATFORM_SMTP_ENABLED=false APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY=true APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT=local APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE=default -APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=10s +APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT=30s APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY=4 APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND=10 APP_NOTIFICATION_PLATFORM_SMTP_SENDER_IDENTITY=no-reply@example.invalid diff --git a/config/spotbugs/exclude.xml b/config/spotbugs/exclude.xml index 2ee07473..05aee37c 100644 --- a/config/spotbugs/exclude.xml +++ b/config/spotbugs/exclude.xml @@ -208,12 +208,19 @@ Annotation matcher covers every class annotated at class level, including ones added later. A class that annotates only some methods is proxied all the same but is not matched here; name it when one appears. A @Configuration class is CGLIB-subclassed for its @Bean - methods for the same reason. --> + methods for the same reason. Every use case carries @UseCaseCapability and is not final by + the use-case contract (src/application/core/AGENTS.md), @SystemInvoked and query use cases + included, so the annotation matches them too; together with the *UseCase name, so a class + that is not a use case does not hide a finding behind the annotation. --> + + + + @@ -359,6 +366,15 @@ + + + + + + + } and followed by a {@code sql} * fence. The lanes that run these blocks against real PostgreSQL read them from the document rather diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java index 98739e4d..a483b2fe 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/autoconfigure/JpaLiveEventReplayActivationTest.java @@ -1,18 +1,34 @@ package dev.caskeleton.adapter.outbound.persistence.autoconfigure; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import dev.caskeleton.adapter.outbound.persistence.liveevent.JpaLiveEventReplay; import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventJpaRepository; +import dev.caskeleton.adapter.outbound.persistence.liveevent.LiveEventReaper; import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.boot.context.properties.bind.UnboundConfigurationPropertiesException; import org.springframework.boot.convert.ApplicationConversionService; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.scheduling.config.FixedDelayTask; +import org.springframework.scheduling.config.ScheduledTask; +import org.springframework.scheduling.config.ScheduledTaskHolder; class JpaLiveEventReplayActivationTest { @@ -34,6 +50,110 @@ class JpaLiveEventReplayActivationTest { }); } + @Test + void theReaperIsAssembledWithTheReplayStore() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H") + .run( + context -> + assertThat(context) + .as("without the reaper nothing removes a row past its retention") + .hasSingleBean(LiveEventReaper.class)); + } + + @Test + void noReaperIsAssembledWithoutTheReplayStore() { + runner.run(context -> assertThat(context).doesNotHaveBean(LiveEventReaper.class)); + } + + @Test + void aZeroReaperIntervalFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-interval=PT0S") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .hasMessageContaining("app.realtime.replay.reaper-interval must be positive")); + } + + @Test + void aMisspeltReplayKeyFailsStartup() { + runner + .withPropertyValues( + "app.realtime.replay.enabled=true", + "app.realtime.replay.retention=PT1H", + "app.realtime.replay.reaper-intervall=PT1M") + .run( + context -> + assertThat(context.getStartupFailure()) + .rootCause() + .isInstanceOf(UnboundConfigurationPropertiesException.class) + .hasMessageContaining("reaper-intervall")); + } + + @ParameterizedTest(name = "[{index}] reaper-interval {0} → {1}") + @CsvSource( + value = {"UNSET, PT10M", "PT3M, PT3M"}, + nullValues = "UNSET") + @DisplayName("reaper는 묶인 reaper-interval(기본 PT10M)마다 돌고, 첫 회도 그 간격 뒤다") + void theReaperRunsAtTheBoundInterval(String configured, Duration expected) { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + if (configured != null) { + replay = replay.withPropertyValues("app.realtime.replay.reaper-interval=" + configured); + } + + // when + List> schedules = new ArrayList<>(); + replay.run(context -> schedules.addAll(fixedDelaySchedules(context))); + + // then + assertThat(schedules) + .as("the schedule is the bound settings' interval, its default included") + .containsExactly(List.of(expected, expected)); + } + + @Test + void theScheduledTaskIsTheReaper() { + // given + ApplicationContextRunner replay = + runner.withPropertyValues( + "app.realtime.replay.enabled=true", "app.realtime.replay.retention=PT1H"); + + // when + List reapedThrough = new ArrayList<>(); + replay.run( + context -> { + fixedDelayTasks(context).forEach(task -> task.getRunnable().run()); + reapedThrough.add(context.getBean(LiveEventJpaRepository.class)); + }); + + // then + verify(reapedThrough.getFirst()).deleteExpiredBatch(any(Instant.class), anyInt()); + } + + /** Interval and initial delay of every fixed-delay task the context scheduled. */ + private static List> fixedDelaySchedules(ApplicationContext context) { + return fixedDelayTasks(context).stream() + .map(task -> List.of(task.getIntervalDuration(), task.getInitialDelayDuration())) + .toList(); + } + + private static List fixedDelayTasks(ApplicationContext context) { + return context.getBean(ScheduledTaskHolder.class).getScheduledTasks().stream() + .map(ScheduledTask::getTask) + .filter(FixedDelayTask.class::isInstance) + .map(FixedDelayTask.class::cast) + .toList(); + } + @Test void enabledReplayCapabilityRequiresExplicitRetention() { runner @@ -42,7 +162,8 @@ class JpaLiveEventReplayActivationTest { context -> { assertThat(context).hasFailed(); assertThat(context.getStartupFailure()) - .hasMessageContaining("app.realtime.replay.retention"); + .rootCause() + .hasMessageContaining("app.realtime.replay.retention must be set"); }); } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java index a560adf4..74694d1f 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/experimental/ExperimentalFailClosedTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.adapter.outbound.persistence.experimental; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.persistence.experimental.database.TenantPoolBudget; import dev.caskeleton.adapter.outbound.persistence.experimental.replica.ConsistencyAwareDataSourceRouter; @@ -103,14 +104,28 @@ class ExperimentalFailClosedTest { void tenantPoolBudgetIsBounded() { var budget = new TenantPoolBudget(2, 20); - assertThat(budget.hasCapacity(1, 10)).isTrue(); - assertThatThrownBy(() -> budget.requireCapacity(2, 10)) + assertThat(budget.hasCapacity(1, 10, 10)).isTrue(); + assertThatThrownBy(() -> budget.requireCapacity(2, 10, 1)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("tenant pool budget"); - assertThatThrownBy(() -> budget.requireCapacity(1, 20)) + assertThatThrownBy(() -> budget.requireCapacity(1, 20, 1)) .isInstanceOf(IllegalStateException.class); } + @Test + void aNewPoolLargerThanTheConnectionsLeftIsRefused() { + // given + var budget = new TenantPoolBudget(4, 20); + + // when + Throwable refused = catchThrowable(() -> budget.requireCapacity(1, 15, 10)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("tenant pool budget exhausted"); + } + /** A lag monitor with a fixed answer, including "I do not know". */ private record FixedLagMonitor(Optional replayedThrough, Optional lag) implements ReplicaLagMonitor {} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java new file mode 100644 index 00000000..1fb7f801 --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java @@ -0,0 +1,45 @@ +package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.Mockito.mock; + +import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction; +import java.time.Duration; +import java.util.Set; +import org.hibernate.SessionFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The stateless runner's transaction bound is the write bound every other transaction here has. + * + *

The runner restates the value instead of reading it, because the {@code hibernate} package may + * not depend on {@code transaction} (JpaModuleBoundaryTest). A test may depend on both, so it is + * the place the two copies are held to each other (round9 review 2026-11-12, RR9-01-25). + */ +class StatelessRunnerTimeoutTest { + + @Test + void theRunnersDefaultIsSpringTransactionsWriteDefault() { + assertThat(HibernateStatelessSessionRunner.DEFAULT_TIMEOUT) + .isEqualTo(SpringTransaction.Timeouts.DEFAULT.write()); + } + + @ParameterizedTest + @ValueSource(strings = {"PT0S", "PT-1S", "PT1.5S", "PT2147483648S"}) + void aTimeoutThatIsNotAWholePositiveNumberOfSecondsIsTheAssemblysFault(String timeout) { + // when + Throwable refused = + catchThrowable( + () -> + new HibernateStatelessSessionRunner( + mock(SessionFactory.class), Set.of(), Duration.parse(timeout))); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("must be a positive whole number of seconds"); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java index 72235d67..e80b313d 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java @@ -2,23 +2,61 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.time.Clock; -import java.time.Instant; -import java.time.ZoneOffset; import org.junit.jupiter.api.Test; +/** + * Unit tests for {@link IdempotencyReaper}: the delete runs in bounded batches until one comes back + * short, as the outbox reaper's does. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement itself — + * expiry judged by the database clock — runs against PostgreSQL in {@code + * IdempotencyReaperIntegrationTest}. + */ class IdempotencyReaperTest { - private static final Instant NOW = Instant.parse("2026-06-09T12:00:00Z"); + private static final int BATCH = IdempotencyReaper.BATCH_SIZE; + + private final IdempotencyRecordJpaRepository repository = + mock(IdempotencyRecordJpaRepository.class); + private final IdempotencyReaper reaper = new IdempotencyReaper(repository); @Test - void reapDeletesRecordsExpiredAsOfNow() { - IdempotencyRecordJpaRepository repo = mock(IdempotencyRecordJpaRepository.class); - when(repo.deleteExpired(NOW)).thenReturn(3); - IdempotencyReaper reaper = new IdempotencyReaper(repo, Clock.fixed(NOW, ZoneOffset.UTC)); + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); - assertThat(reaper.reap()).isEqualTo(3); + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(2 * BATCH + 3); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(3)).deleteExpiredBatch(BATCH); + } + + @Test + void reapReturnsZeroWhenNothingHasExpired() { + // given + when(repository.deleteExpiredBatch(BATCH)).thenReturn(0); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isZero(); } } diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java new file mode 100644 index 00000000..b43a316e --- /dev/null +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java @@ -0,0 +1,53 @@ +package dev.caskeleton.adapter.outbound.persistence.liveevent; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LiveEventReaper}: rows retained no later than the clock's now go, in + * bounded batches until one comes back short. + * + *

The repository is a Spring Data boundary, so Mockito stands in for it. The statement runs + * against PostgreSQL in {@code JpaLiveEventReplayContractTest}. + */ +class LiveEventReaperTest { + + private static final Instant NOW = Instant.parse("2026-11-13T00:00:00Z"); + private static final int BATCH = LiveEventReaper.BATCH_SIZE; + + private final LiveEventJpaRepository repository = mock(LiveEventJpaRepository.class); + private final LiveEventReaper reaper = + new LiveEventReaper(repository, Clock.fixed(NOW, ZoneOffset.UTC)); + + @Test + void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + int purged = reaper.reap(); + + // then + assertThat(purged).isEqualTo(BATCH + 4); + } + + @Test + void eachBatchIsItsOwnDeleteStatement() { + // given + when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4); + + // when + var unused = reaper.reap(); + + // then + verify(repository, times(2)).deleteExpiredBatch(NOW, BATCH); + } +} diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java index 0d647c57..b6ed1ca0 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java +++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java @@ -78,6 +78,20 @@ class DistributedLockPersistenceConfigTest { }); } + @Test + @DisplayName("읽는 곳이 없던 app.lock.wait-time은 모르는 키로 기동을 실패시킨다") + void theRemovedWaitTimeKeyFailsStartup() { + // when + contextRunner + .withPropertyValues("app.lock.wait-time=3s") + .run( + context -> { + // then + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("wait-time"); + }); + } + @Test void multiInstanceBuildsTheJdbcProviderAndNotTheInProcessProvider() { contextRunner diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 index 1eb57d14..93a51fe4 100644 --- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 +++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 @@ -14,6 +14,7 @@ be51fabd07220f9ef885534ba36ada5f1491baf24cdfb54d25d93a19ddc29926 jpa/notification-platform/V11__notification_reconciliation_claim_lease.sql 43f9429f91c07528154ec01b721649b2c25fa98857ac154c7c08f4655634657f jpa/notification-platform/V12__notification_suppression_target_unique.sql d01aaf59f66834358abf18246faa9a9eb8dfa9f42097ca9d968c44edff564630 jpa/notification-platform/V13__notification_request_acceptance.sql +8a7dc3604d44f1438b39dfec3aa688f792e7363cd3c1b74bf0c799a6b53f9f0d jpa/notification-platform/V14__notification_provider_event_open_index.sql 1ac7cc1c2509868eb0ce202e1917bb36846f98f17ad4922abe3fac0ef128199b jpa/notification-platform/V1__notification_platform_core.sql 0add67aa23580e51712f84fabe9b09e08401ef708cfa1b3292b120fdb5c12fa2 jpa/notification-platform/V2__notification_platform_contact_template_policy.sql f5b98598a59cc4376f5d233b410f8a5ae9f3cd94156291ebdf2d0da5a66f34ef jpa/notification-platform/V3__notification_platform_inbox_admin.sql @@ -42,6 +43,7 @@ a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19 25db296e55444636c8a017d9b736eecf4a839a9caf1733c0c4a5f37bae5a3161 postgresql/V20__outbox_event_deferred_attempt_count.sql 21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql 20da5649a29bf2305f0e04ab14003b21b7f120d867439fb01f1658c98179dba0 postgresql/V22__outbox_event_last_failure_code.sql +9492ace50ab828144792b77c72d92fce50bec29a028ebc917f7e55facf58d85a postgresql/V23__durable_operation_lease_fence.sql 8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql 6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java index ef80fdb1..a435e5b5 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java @@ -7,9 +7,14 @@ import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedExc import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; /** * Bounds how many tenant clients exist at once (advanced plan Task 11). @@ -18,6 +23,13 @@ import java.util.Set; * connection pool and its own monitoring threads, so a thousand tenants is a thousand pools. The * registry caps the number and evicts idle ones, and refuses rather than exceeding the cap — an * unbounded registry fails later, as connection exhaustion on an unrelated request. + * + *

{@link #acquire} hands back a {@link Lease}; the client is in use until every lease on it is + * closed, and only a client nobody holds is idle. Acquire used to return nothing, so idle meant + * "acquired longest ago" and eviction could close a client a long-running caller was still using. + * Closing an evicted client is this registry's own clean-up: if it fails, the tenant whose acquire + * triggered the eviction is not told about another tenant's client — the failure is counted in + * {@link #evictionCloseFailures()} (round9 review 2026-11-12, RR9-01-27). */ @MongoAdvancedEntryPoint(MongoCapability.DATABASE_PER_TENANT) public final class MongoTenantClientRegistry { @@ -26,11 +38,21 @@ public final class MongoTenantClientRegistry { private final Duration idleEviction; - private final Map lastUsed = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map lastUsed = new ConcurrentHashMap<>(); - private final Map clients = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map clients = new ConcurrentHashMap<>(); - private final java.util.function.Function clientFactory; + private final Map leases = new HashMap<>(); + + /** + * Guards the cap, the lease counts and the eviction as one decision. A private monitor rather + * than the registry itself, so code outside cannot take it and stall every acquire. + */ + private final Object lock = new Object(); + + private final AtomicLong evictionCloseFailures = new AtomicLong(); + + private final Function clientFactory; private final Clock clock; @@ -47,15 +69,15 @@ public final class MongoTenantClientRegistry { MongoAdvancedCapabilityGuard guard, int maximumActiveClients, Duration idleEviction, - java.util.function.Function clientFactory) { + Function clientFactory) { this(guard, maximumActiveClients, idleEviction, clientFactory, Clock.systemUTC()); } /** * A registry that owns real clients. * - *

Idle eviction reads {@code clock}; {@link #acquire(String)} used to read {@code - * Instant.now()} beside it. + *

Acquiring, closing a lease and idle eviction all read {@code clock}; {@link + * #acquire(String)} used to read {@code Instant.now()} beside it. * *

The factory is called at most once per tenant, inside the map's own computation, so two * threads asking for the same tenant get one client rather than two — a per-tenant client carries @@ -65,7 +87,7 @@ public final class MongoTenantClientRegistry { MongoAdvancedCapabilityGuard guard, int maximumActiveClients, Duration idleEviction, - java.util.function.Function clientFactory, + Function clientFactory, Clock clock) { Objects.requireNonNull(guard, "guard").require(MongoTenantClientRegistry.class); this.clientFactory = Objects.requireNonNull(clientFactory, "clientFactory"); @@ -80,41 +102,128 @@ public final class MongoTenantClientRegistry { /** * Acquires a client for a tenant. * + * @return the lease to close when the work with the client is done * @throws MongoOperationRejectedException when the cap is reached and nothing can be evicted */ - public void acquire(String tenantKey) { - acquire(tenantKey, clock.instant()); + public Lease acquire(String tenantKey) { + Objects.requireNonNull(tenantKey, "tenantKey"); + synchronized (lock) { + return acquireHeld(tenantKey); + } } - /** Acquires a client for a tenant at an explicit instant, so eviction is testable. */ - public synchronized void acquire(String tenantKey, Instant now) { - Objects.requireNonNull(tenantKey, "tenantKey"); - Objects.requireNonNull(now, "now"); + private Lease acquireHeld(String tenantKey) { + // One time source for the acquire, the close and the eviction: an overload that took the + // instant as an argument let a caller acquire in one timeline and close in the clock's. + Instant now = clock.instant(); // Synchronized rather than a bare ConcurrentHashMap: the cap, the eviction and the create are // one decision. Checking the size on one thread while another inserts is how a cap of ten ends // up holding twelve connection pools. - if (lastUsed.containsKey(tenantKey)) { - lastUsed.put(tenantKey, now); - return; + if (!lastUsed.containsKey(tenantKey)) { + evictIdle(now); + if (lastUsed.size() >= maximumActiveClients) { + throw MongoOperationRejectedException.of( + "tenancy.client", + "the tenant client cap of " + + maximumActiveClients + + " is reached and no client is idle; each tenant client carries its own" + + " connection pool and monitoring threads, so the cap is what stops one process" + + " from exhausting the cluster's connections"); + } + clients.computeIfAbsent(tenantKey, clientFactory); } - evictIdle(now); - if (lastUsed.size() >= maximumActiveClients) { - throw MongoOperationRejectedException.of( - "tenancy.client", - "the tenant client cap of " - + maximumActiveClients - + " is reached and no client is idle; each tenant client carries its own connection " - + "pool and monitoring threads, so the cap is what stops one process from exhausting " - + "the cluster's connections"); - } - clients.computeIfAbsent(tenantKey, clientFactory); lastUsed.put(tenantKey, now); + leases.merge(tenantKey, 1, Integer::sum); + return new HeldLease(tenantKey, clients.get(tenantKey)); } - /** Releases a tenant's client and closes it. */ - public synchronized void release(String tenantKey) { - lastUsed.remove(Objects.requireNonNull(tenantKey, "tenantKey")); - close(tenantKey); + /** + * Releases a tenant's client and closes it. + * + * @throws IllegalStateException when a lease on the client is still open, or the close fails + */ + public void release(String tenantKey) { + Objects.requireNonNull(tenantKey, "tenantKey"); + synchronized (lock) { + if (leases.containsKey(tenantKey)) { + throw new IllegalStateException( + "the client for tenant '" + tenantKey + "' is still in use and cannot be released"); + } + lastUsed.remove(tenantKey); + close(tenantKey); + } + } + + /** + * How many evicted clients failed to close. + * + *

The registry has dropped each of them and freed its slot; the count is how an operator + * learns that clients — and their pools — may still be open behind it. + */ + public long evictionCloseFailures() { + return evictionCloseFailures.get(); + } + + /** + * One use of a tenant client; close it when the work with that client is done. + * + *

An interface so the registry's own implementation stays private: only the registry makes a + * lease, and only through {@link #acquire}. + */ + public interface Lease extends AutoCloseable { + + /** The tenant's client, valid until this lease is closed. */ + AutoCloseable client(); + + /** Ends this use. Closing twice is the same as closing once. */ + @Override + void close(); + } + + private final class HeldLease implements Lease { + + private final String tenantKey; + + private final AutoCloseable client; + + private boolean closed; + + private HeldLease(String tenantKey, AutoCloseable client) { + this.tenantKey = tenantKey; + this.client = client; + } + + @Override + public AutoCloseable client() { + return client; + } + + @Override + public void close() { + synchronized (lock) { + if (closed) { + return; + } + closed = true; + leases.computeIfPresent(tenantKey, (tenant, open) -> open == 1 ? null : open - 1); + // Idle from the end of the last use, not its start: a client held for half an hour was in + // use all that time, and measuring from the acquire evicted it the moment it was let go. + Instant released = clock.instant(); + lastUsed.computeIfPresent( + tenantKey, (tenant, used) -> released.isAfter(used) ? released : used); + } + } + } + + /** + * How many leases are open now, across all tenants. + * + *

A lease nobody closed keeps its client's slot for good; this is how that becomes visible. + */ + public int openLeases() { + synchronized (lock) { + return leases.values().stream().mapToInt(Integer::intValue).sum(); + } } /** The tenants with an active client. */ @@ -128,8 +237,9 @@ public final class MongoTenantClientRegistry { } private void evictIdle(Instant now) { - java.util.List idle = + List idle = lastUsed.entrySet().stream() + .filter(entry -> !leases.containsKey(entry.getKey())) .filter(entry -> entry.getValue().plus(idleEviction).isBefore(now)) .map(Map.Entry::getKey) .toList(); @@ -138,7 +248,11 @@ public final class MongoTenantClientRegistry { lastUsed.remove(tenant); // Evicting the bookkeeping entry without closing the client frees the slot and keeps the // connection pool, which is the exhaustion this cap exists to prevent, delayed. - close(tenant); + try { + close(tenant); + } catch (IllegalStateException closeFailed) { + evictionCloseFailures.incrementAndGet(); + } }); } diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java index 8ed663b7..0f0a91d4 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java @@ -6,20 +6,30 @@ import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer; import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry; import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget; import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.stream.Stream; +import org.bson.Document; +import org.bson.RawBsonDocument; +import org.bson.codecs.Codec; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.AggregationOperation; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; -import org.springframework.data.mongodb.core.aggregation.AggregationResults; /** * Runs an aggregation only after its stages, budget and lookups have been checked (design §17). * *

Every guard runs before the first stage reaches the server, because an aggregation that is - * going to be refused should cost nothing. The one guard that runs afterwards is the result-count - * check: {@code $limit} bounds what the pipeline emits, but a pipeline whose own shape produced - * more than the budget allows has still told the caller something worth failing over. + * going to be refused should cost nothing. Two run while the results arrive: the result count and + * the result bytes. The pipeline gets a final {@code $limit} of one document past the count budget, + * so the server never produces more than that, and the results are read from the cursor one + * document at a time, the encoded size of each added up as it arrives. They used to be read whole — + * every document in memory before anything was counted — and the byte budget was declared and never + * checked (round9 review 2026-11-12, RR9-01-20; {@code docs/mongodb/query-aggregation-guide.md} + * 「집계에 예산을 둔다」). */ public final class PolicyAwareMongoAggregationExecutor { @@ -53,6 +63,14 @@ public final class PolicyAwareMongoAggregationExecutor { /** * Executes a typed aggregation. * + *

Results are read as {@link Document}, sized, then mapped with the template's converter. Two + * costs follow, both deliberate. Each document is encoded once more to be measured: the template + * hands back decoded documents, so the bytes the server sent are not available, and the budget + * has to be in bytes. And the mapping is the converter's alone: the template's typed aggregate + * would also publish {@code AfterLoadEvent}/{@code AfterConvertEvent} and run entity callbacks, + * and those do not run for an aggregation result read here. A listener that needs them maps from + * the returned type itself. + * * @throws MongoOperationRejectedException when a stage, lookup, budget or result size is not * permitted */ @@ -71,19 +89,38 @@ public final class PolicyAwareMongoAggregationExecutor { MongoOperationBudget effective = enforcer.narrow(budget, budget.narrowedTo(budgetFor(context, budget))); + List bounded = new ArrayList<>(plan.operations()); + // One past the budget: enough to know the budget was exceeded, never more than that. + bounded.add(Aggregation.limit(effective.maxResults() + 1L)); Aggregation aggregation = - Aggregation.newAggregation(plan.operations()).withOptions(optionsFor(profile, effective)); - AggregationResults results = operations.aggregate(aggregation, collection, outputType); - List mapped = results.getMappedResults(); - if (mapped.size() > effective.maxResults()) { - throw MongoOperationRejectedException.of( - "aggregation.result", - "the aggregation returned " - + mapped.size() - + " documents, above the budget of " - + effective.maxResults()); + Aggregation.newAggregation(bounded).withOptions(optionsFor(profile, effective)); + Codec sizing = operations.getConverter().getCodecRegistry().get(Document.class); + List mapped = new ArrayList<>(); + long bytes = 0; + try (Stream documents = + operations.aggregateStream(aggregation, collection, Document.class)) { + Iterator cursor = documents.iterator(); + while (cursor.hasNext()) { + Document document = cursor.next(); + if (mapped.size() == effective.maxResults()) { + throw MongoOperationRejectedException.of( + "aggregation.result", + "the aggregation returned more than the budget of " + + effective.maxResults() + + " documents"); + } + bytes += new RawBsonDocument(document, sizing).getByteBuffer().remaining(); + if (bytes > effective.maxResultBytes()) { + throw MongoOperationRejectedException.of( + "aggregation.result", + "the aggregation returned more than the budget of " + + effective.maxResultBytes() + + " bytes"); + } + mapped.add(operations.getConverter().read(outputType, document)); + } } - return mapped; + return List.copyOf(mapped); } private static MongoOperationBudget budgetFor( diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java index 0466ca15..855db8ec 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java @@ -36,20 +36,46 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge private static final String MIGRATION_ID = "migrationId"; + private static final String FENCE = "fence"; + private final MongoCollection ledger; private final MongoCollection checkpoints; + private final MongoCollection locks; + + /** + * A ledger over the default collections of {@code database}, the lock's included. + * + *

The lock collection is {@link MongoCollectionMigrationLock#DEFAULT_COLLECTION}. A deployment + * that keeps its runner lease in a collection of another name uses the three-collection + * constructor with that collection; with this one the ledger would read a lock that is never + * taken and refuse nothing on its account. + * + * @param database the database being migrated + */ public MongoCollectionMigrationLedger(MongoDatabase database) { this( Objects.requireNonNull(database, "database").getCollection(DEFAULT_COLLECTION), - database.getCollection(DEFAULT_CHECKPOINT_COLLECTION)); + database.getCollection(DEFAULT_CHECKPOINT_COLLECTION), + database.getCollection(MongoCollectionMigrationLock.DEFAULT_COLLECTION)); } + /** + * A ledger over explicit collections. + * + * @param ledger the applied migrations + * @param checkpoints the running migrations' positions + * @param locks the collection {@link MongoCollectionMigrationLock} keeps the runner lease in, + * read to refuse a replaced runner's ledger entry + */ public MongoCollectionMigrationLedger( - MongoCollection ledger, MongoCollection checkpoints) { + MongoCollection ledger, + MongoCollection checkpoints, + MongoCollection locks) { this.ledger = Objects.requireNonNull(ledger, "ledger"); this.checkpoints = Objects.requireNonNull(checkpoints, "checkpoints"); + this.locks = Objects.requireNonNull(locks, "locks"); } /** @@ -90,13 +116,34 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge Objects.requireNonNull(checksum, "checksum"); Objects.requireNonNull(operator, "operator"); Objects.requireNonNull(appliedAt, "appliedAt"); - requireCurrentFence(fence, "ledger entry for " + migrationId.value()); + requireFenced(fence, "ledger entry for " + migrationId.value()); + // A superseded runner that wakes up after its replacement took the lease must not record the + // migration as applied under it (round9 review 2026-11-12, RR9-01-28). Two pieces of evidence: + // the lock's newest acquisition, which a replacement leaves the moment it takes the lease, and + // a checkpoint a newer runner wrote. + // + // A narrowing, not a guarantee: the reads and the insert are separate operations, so a runner + // replaced between them still records. What stops a superseded runner first is the lock's + // refresh, which refuses it; this check is the second line for a runner that skipped a + // refresh. Making it atomic needs the lock and the ledger in one transaction, which a + // standalone deployment does not have. + if (MongoCollectionMigrationLock.newestFence(locks) > fence + || checkpoints + .find( + Filters.and( + Filters.eq(MIGRATION_ID, migrationId.value()), Filters.gt(FENCE, fence))) + .first() + != null) { + throw MongoOperationRejectedException.of( + "migration.fence", + "a newer migration runner owns the lease; this runner's ledger entry was refused"); + } ledger.insertOne( new Document(MIGRATION_ID, migrationId.value()) .append("checksum", checksum.value()) .append("operator", operator) .append("appliedAt", Date.from(appliedAt)) - .append("fence", fence)); + .append(FENCE, fence)); } @Override @@ -117,7 +164,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge @Override public void saveCheckpoint(MongoMigrationCheckpoint checkpoint, long fence) { Objects.requireNonNull(checkpoint, "checkpoint"); - requireCurrentFence(fence, "checkpoint for " + checkpoint.migrationId().value()); + requireFenced(fence, "checkpoint for " + checkpoint.migrationId().value()); // Conditioned on the stored fence as well as matched by id: a superseded runner that wakes up // mid-batch must not move the checkpoint of the runner that replaced it, which would send the // live migration back over rows it has already done or forward past rows it has not. @@ -126,7 +173,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge .append("resumePosition", checkpoint.resumePosition()) .append("processedCount", checkpoint.processedCount()) .append("updatedAt", Date.from(checkpoint.updatedAt())) - .append("fence", fence); + .append(FENCE, fence); // Replace-then-insert rather than one upsert, and the two cases this separates are the whole // behaviour of the fence. @@ -147,7 +194,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge .replaceOne( Filters.and( Filters.eq(MIGRATION_ID, checkpoint.migrationId().value()), - Filters.or(Filters.exists("fence", false), Filters.lte("fence", fence))), + Filters.or(Filters.exists(FENCE, false), Filters.lte(FENCE, fence))), document, new ReplaceOptions().upsert(false)) .getMatchedCount(); @@ -172,9 +219,13 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge * Refuses a write from an unfenced lease. * *

An unfenced lease cannot prove the writer is still the owner, and the whole point of the - * ledger is that it says what actually happened. + * ledger is that it says what actually happened. Whether a fenced writer is still the current one + * is decided in two places, not here: the runner refreshes its lease, conditioned on the fence, + * right before it writes, and each write is conditioned on the newest fence the ledger has seen + * for the migration. The method was called {@code requireCurrentFence} and checked neither + * (round9 review 2026-11-12, RR9-01-28). */ - private static void requireCurrentFence(long fence, String what) { + private static void requireFenced(long fence, String what) { if (fence == MongoMigrationLock.UNFENCED) { throw MongoOperationRejectedException.of( "migration.fence", diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java index cca219af..94c4ac26 100644 --- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java +++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java @@ -6,6 +6,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; +import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -125,6 +126,19 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock { return fence; } + /** + * The fence of the newest acquisition of the lock in {@code locks}, or {@link #UNFENCED} when the + * lock was never taken there. + * + *

For the ledger: a write whose fence is older than this one comes from a runner that has been + * replaced, whether or not its replacement has written anything yet. + */ + static long newestFence(MongoCollection locks) { + Document current = + Objects.requireNonNull(locks, "locks").find(Filters.eq("_id", LOCK_ID)).first(); + return current == null ? UNFENCED : fenceOf(current); + } + private static long fenceOf(Document lock) { Object stored = lock.get("fence"); return stored instanceof Number number ? number.longValue() : 1L; @@ -165,7 +179,11 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock { .getMatchedCount(); if (stillOwned != 1) { held = false; - throw new IllegalStateException( + // The refusal the heartbeat promises (MongoMigrationHeartbeat#beat), so a migration that + // stops on it stops for the same reason the heartbeat's own ownership check gives. It was an + // IllegalStateException, which read as a server defect (round9 review 2026-11-12, RR9-01-28). + throw MongoOperationRejectedException.of( + "migration.lock", "the migration lease was lost before it could be refreshed; another runner may have " + "taken it after this one appeared to stall"); } diff --git a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java index abbcad9c..edb28966 100644 --- a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java +++ b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java @@ -224,6 +224,95 @@ class MongoMigrationLaneTest { assertThat(ledger.findCheckpoint(id).orElseThrow().processedCount()).isEqualTo(200); } + @Test + void aSupersededRunnerCannotRecordTheMigrationAppliedUnderItsReplacement() { + // given + MongoDatabase database = freshDatabase(); + MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database); + ledger.ensureIndexes(); + MongoMigrationId id = new MongoMigrationId("20260811-011"); + ledger.saveCheckpoint( + MongoMigrationCheckpoint.start(id, FIXED.instant()) + .advancedTo("o-300", 300, FIXED.instant()), + 2L); + + // when + Throwable refused = + catchThrowable( + () -> + ledger.recordApplied( + id, new MongoMigrationChecksum("abc"), "operator", FIXED.instant(), 1L)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("a newer migration runner owns the lease"); + } + + @Test + void aReplacedRunnerCannotRecordTheMigrationBeforeItsReplacementWroteAnything() { + // given + MongoDatabase database = freshDatabase(); + MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database); + ledger.ensureIndexes(); + MongoMigrationId id = new MongoMigrationId("20260811-012"); + Instant early = Instant.parse("2026-08-11T00:00:00Z"); + MongoMigrationLock stalled = + MongoCollectionMigrationLock.tryAcquire( + database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC)) + .orElseThrow(); + var replacement = + MongoCollectionMigrationLock.tryAcquire( + database, + "next-runner", + LEASE, + Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC)) + .orElseThrow(); + + // when + Throwable refused = + catchThrowable( + () -> + ledger.recordApplied( + id, + new MongoMigrationChecksum("abc"), + "operator", + FIXED.instant(), + stalled.fence())); + + // then + assertThat(refused) + .as("%s holds the lease with a newer fence and has not checkpointed yet", replacement) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("a newer migration runner owns the lease"); + } + + @Test + void aLeaseLostBeforeItsRefreshIsTheRefusalTheHeartbeatPromises() { + // given + MongoDatabase database = freshDatabase(); + Instant early = Instant.parse("2026-08-11T00:00:00Z"); + MongoMigrationLock stalled = + MongoCollectionMigrationLock.tryAcquire( + database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC)) + .orElseThrow(); + var unused = + MongoCollectionMigrationLock.tryAcquire( + database, + "next-runner", + LEASE, + Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC)) + .orElseThrow(); + + // when + Throwable refused = catchThrowable(() -> stalled.refresh(LEASE)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("the migration lease was lost"); + } + @Test void theLedgerRefusesADoubleApplyOfTheSameId() { MongoDatabase database = freshDatabase(); diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java index 97309f29..06f1316e 100644 --- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java +++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java @@ -3,6 +3,7 @@ package dev.caskeleton.adapter.outbound.mongo.advanced.tenancy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityFlags; import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityGuard; @@ -19,10 +20,25 @@ import dev.caskeleton.adapter.outbound.mongo.imperative.atomic.AtomicFilter; import dev.caskeleton.adapter.outbound.mongo.migration.MongoMigrationCheckpoint; import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoCollectionManifest; import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoIndexManifest; +import java.io.IOException; +import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.IntStream; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -37,6 +53,8 @@ class MongoTenancyContractTest { MongoAdvancedCapabilityFlags.allDisabled() .withEnabled(MongoCapability.DATABASE_PER_TENANT)); + private static final Instant START = Instant.parse("2026-08-13T00:00:00Z"); + @Test void missingTenantContextFailsClosed() { assertThatThrownBy(() -> injector.apply(Optional.empty(), AtomicFilter.id("o-1"))) @@ -116,29 +134,99 @@ class MongoTenancyContractTest { @Test void registryEnforcesMaximumActiveClients() { + // given MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2); - Instant now = Instant.parse("2026-08-13T00:00:00Z"); + registry.acquire("t1"); + registry.acquire("t2"); - registry.acquire("t1", now); - registry.acquire("t2", now); + // when + Throwable refused = catchThrowable(() -> registry.acquire("t3")); - assertThatThrownBy(() -> registry.acquire("t3", now)) - .isInstanceOf(MongoOperationRejectedException.class); - assertThat(registry.activeCount()).isEqualTo(2); + // then + assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.activeCount())) + .containsExactly(true, 2); } @Test void anIdleClientIsEvictedToMakeRoom() { - MongoTenantClientRegistry registry = - new MongoTenantClientRegistry(TENANCY, 1, Duration.ofMinutes(5)); - Instant start = Instant.parse("2026-08-13T00:00:00Z"); + // given + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock); + registry.acquire("t1").close(); + clock.advance(Duration.ofMinutes(10)); - registry.acquire("t1", start); - registry.acquire("t2", start.plus(Duration.ofMinutes(10))); + // when + registry.acquire("t2"); + // then assertThat(registry.activeTenants()).containsExactly("t2"); } + @Test + void aClientIsIdleFromTheEndOfItsLastUseNotItsStart() { + // given + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock); + MongoTenantClientRegistry.Lease held = registry.acquire("t1"); + clock.advance(Duration.ofMinutes(30)); + held.close(); + clock.advance(Duration.ofMinutes(1)); + + // when + Throwable refused = catchThrowable(() -> registry.acquire("t2")); + + // then + assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.openLeases())) + .as("held for thirty minutes, it was in use all that time and is not idle yet") + .containsExactly(true, 0); + } + + @Test + void anUnclosedLeaseIsCounted() { + // given + MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2); + + // when + var unused = registry.acquire("t1"); + + // then + assertThat(registry.openLeases()).isEqualTo(1); + } + + private static MongoTenantClientRegistry registry( + int cap, Function clients, Clock clock) { + return new MongoTenantClientRegistry(TENANCY, cap, Duration.ofMinutes(5), clients, clock); + } + + /** A clock a test moves by hand: one time source for acquiring, closing and eviction. */ + private static final class MovingClock extends Clock { + + private final AtomicReference now; + + private MovingClock(Instant start) { + this.now = new AtomicReference<>(start); + } + + private void advance(Duration by) { + now.updateAndGet(instant -> instant.plus(by)); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return now.get(); + } + } + @Test void aTenantDatabaseCannotServeBeforeItsSchemaIsValidated() { MongoTenantLifecyclePolicy policy = MongoTenantLifecyclePolicy.standard(); @@ -172,100 +260,112 @@ class MongoTenancyContractTest { } @Test - @org.junit.jupiter.api.DisplayName("the same tenant is created once under concurrency") - void theSameTenantIsCreatedOnce() throws Exception { - java.util.concurrent.atomic.AtomicInteger created = - new java.util.concurrent.atomic.AtomicInteger(); + void theSameTenantIsCreatedOnceUnderConcurrency() throws Exception { + // given + AtomicInteger created = new AtomicInteger(); MongoTenantClientRegistry registry = - new MongoTenantClientRegistry( - TENANCY, + registry( 8, - Duration.ofMinutes(10), tenant -> { created.incrementAndGet(); return () -> {}; - }); - java.util.concurrent.ExecutorService pool = - java.util.concurrent.Executors.newFixedThreadPool(8); + }, + new MovingClock(START)); + List> work = + IntStream.range(0, 50) + .>mapToObj( + index -> + () -> { + registry.acquire("t1").close(); + return null; + }) + .toList(); + ExecutorService pool = Executors.newFixedThreadPool(8); + + // when try { - java.util.List> work = - java.util.stream.IntStream.range(0, 50) - .>mapToObj( - index -> - () -> { - registry.acquire("t1", Instant.parse("2026-08-13T00:00:00Z")); - return null; - }) - .toList(); - for (java.util.concurrent.Future future : pool.invokeAll(work)) { + for (Future future : pool.invokeAll(work)) { future.get(); } } finally { pool.shutdownNow(); } - assertThat(created) + // then + assertThat(List.of(created.get(), registry.activeCount())) .as("a per-tenant client carries a pool and monitoring threads; a second one doubles both") - .hasValue(1); - assertThat(registry.activeCount()).isEqualTo(1); + .containsExactly(1, 1); } @Test - @org.junit.jupiter.api.DisplayName("acquire(tenant)는 주입된 clock으로 idle 여부를 판단한다") - void anAcquireWithoutAnInstantReadsTheInjectedClock() { + void aClientStillHeldIsNotEvictedHoweverLongAgoItWasAcquired() { // given - java.util.List closed = new java.util.ArrayList<>(); - java.util.concurrent.atomic.AtomicReference now = - new java.util.concurrent.atomic.AtomicReference<>(Instant.parse("2026-08-13T00:00:00Z")); - java.time.Clock clock = - new java.time.Clock() { - @Override - public java.time.ZoneId getZone() { - return java.time.ZoneOffset.UTC; - } + List closed = new ArrayList<>(); + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock); + MongoTenantClientRegistry.Lease longRunning = registry.acquire("t1"); + clock.advance(Duration.ofMinutes(10)); - @Override - public java.time.Clock withZone(java.time.ZoneId zone) { - return this; - } + // when + Throwable refused = catchThrowable(() -> registry.acquire("t2")); - @Override - public Instant instant() { - return now.get(); - } - }; + // then + assertThat(List.of(refused instanceof MongoOperationRejectedException, closed)) + .as( + "t1 is in use by %s; closing it would pull the pool out from under that work", + longRunning) + .containsExactly(true, List.of()); + } + + @Test + void anotherTenantsClientThatFailsToCloseDoesNotFailThisAcquire() { + // given + MovingClock clock = new MovingClock(START); MongoTenantClientRegistry registry = - new MongoTenantClientRegistry( - TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant), clock); - registry.acquire("t1"); - now.set(now.get().plus(Duration.ofMinutes(10))); + registry( + 1, + tenant -> + () -> { + if (tenant.equals("t1")) { + throw new IOException("t1's pool would not shut down"); + } + }, + clock); + registry.acquire("t1").close(); + clock.advance(Duration.ofMinutes(10)); + + // when + MongoTenantClientRegistry.Lease admitted = registry.acquire("t2"); + + // then + assertThat( + List.of( + admitted.client() != null, + registry.activeTenants(), + registry.evictionCloseFailures())) + .containsExactly(true, Set.of("t2"), 1L); + } + + @Test + void evictionClosesTheClientItEvicts() { + // given + List closed = new ArrayList<>(); + MovingClock clock = new MovingClock(START); + MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock); + registry.acquire("t1").close(); + clock.advance(Duration.ofMinutes(10)); // when registry.acquire("t2"); // then - assertThat(closed).containsExactly("t1"); - } - - @Test - @org.junit.jupiter.api.DisplayName("eviction closes the client it evicts") - void evictionClosesTheClient() { - java.util.List closed = new java.util.ArrayList<>(); - MongoTenantClientRegistry registry = - new MongoTenantClientRegistry( - TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant)); - Instant start = Instant.parse("2026-08-13T00:00:00Z"); - - registry.acquire("t1", start); - registry.acquire("t2", start.plus(Duration.ofMinutes(10))); - assertThat(closed) .as("freeing the slot without closing the client keeps the connection pool it counted") .containsExactly("t1"); } @Test - @org.junit.jupiter.api.DisplayName("a coordinator resumes from its durable store after a restart") + @DisplayName("a coordinator resumes from its durable store after a restart") void aCoordinatorResumesFromItsDurableStore() { RecordingCheckpointStore store = new RecordingCheckpointStore(); MongoTenantMigrationCoordinator first = diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java index 89fd5ff0..bcf90f54 100644 --- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java +++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java @@ -3,11 +3,31 @@ package dev.caskeleton.adapter.outbound.mongo.aggregation; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.mongodb.MongoClientSettings; +import dev.caskeleton.adapter.outbound.mongo.api.MongoOperationContext; +import dev.caskeleton.adapter.outbound.mongo.api.consistency.MongoConsistencyProfile; import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException; +import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer; +import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry; +import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget; +import java.time.Duration; +import java.util.List; import java.util.Set; +import org.bson.Document; +import org.bson.conversions.Bson; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.data.mongodb.core.MongoOperations; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.convert.MongoConverter; +import org.springframework.data.mongodb.core.query.Criteria; /** Design §17 — write stages are D4, and stage cost is graded rather than assumed. */ @Tag("mongodb-contract") @@ -105,4 +125,141 @@ class PolicyAwareMongoAggregationExecutorTest { assertThat(MongoAggregationProfile.stableRead().allowDiskUse()).isFalse(); assertThat(MongoAggregationProfile.stableRead().strictMapping()).isTrue(); } + + private static final MongoOperationContext CONTEXT = + MongoOperationContext.of( + "orders.summary", + "primary", + "orders", + MongoConsistencyProfile.PRIMARY_MAJORITY, + Duration.ofSeconds(2)); + + private static final MongoAggregationPlan OPEN_ORDERS = + MongoAggregationPlan.builder() + .stage("$match", Aggregation.match(Criteria.where("status").is("OPEN"))) + .build(); + + @Test + void thePipelineEndsWithALimitOnePastTheCountBudget() { + // given + ArgumentCaptor sent = ArgumentCaptor.forClass(Aggregation.class); + MongoOperations operations = returning(List.of(), sent); + + // when + var unused = + executorOver(operations, budget(3, 1_048_576L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class); + + // then + List pipeline = sent.getValue().toPipeline(Aggregation.DEFAULT_CONTEXT); + assertThat(pipeline.get(pipeline.size() - 1)).isEqualTo(new Document("$limit", 4L)); + } + + @Test + void moreDocumentsThanTheCountBudgetAreRefused() { + // given + MongoOperations operations = + returning( + List.of(order(1), order(2), order(3), order(4)), + ArgumentCaptor.forClass(Aggregation.class)); + + // when + Throwable refused = + catchThrowable( + () -> + executorOver(operations, budget(3, 1_048_576L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("more than the budget of 3 documents"); + } + + @Test + void documentsLargerTogetherThanTheByteBudgetAreRefused() { + // given + // Each order encodes to well over 40 bytes: two of them pass any 80-byte budget. + MongoOperations operations = + returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class)); + + // when + Throwable refused = + catchThrowable( + () -> + executorOver(operations, budget(10, 80L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class)); + + // then + assertThat(refused) + .isInstanceOf(MongoOperationRejectedException.class) + .hasMessageContaining("more than the budget of 80 bytes"); + } + + @Test + void resultsInsideBothBudgetsAreReturnedInCursorOrder() { + // given + MongoOperations operations = + returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class)); + + // when + List results = + executorOver(operations, budget(10, 1_048_576L)) + .execute( + CONTEXT, + MongoAggregationProfile.stableRead(), + OPEN_ORDERS, + "orders", + Document.class); + + // then + assertThat(results).containsExactly(order(1), order(2)); + } + + private static MongoOperationBudget budget(int maxResults, long maxResultBytes) { + return new MongoOperationBudget(maxResults, maxResultBytes, 500L, 50); + } + + private static PolicyAwareMongoAggregationExecutor executorOver( + MongoOperations operations, MongoOperationBudget budget) { + return new PolicyAwareMongoAggregationExecutor( + operations, + MongoBudgetPolicyRegistry.builder().register("orders.summary", budget).build(), + new MongoBudgetEnforcer()); + } + + /** The driver boundary: a cursor over {@code documents}, mapped as the converter would. */ + private static MongoOperations returning( + List documents, ArgumentCaptor sent) { + MongoOperations operations = mock(MongoOperations.class); + MongoConverter converter = mock(MongoConverter.class); + when(operations.getConverter()).thenReturn(converter); + when(converter.getCodecRegistry()).thenReturn(MongoClientSettings.getDefaultCodecRegistry()); + when(converter.read(eq(Document.class), any(Bson.class))) + .thenAnswer(invocation -> invocation.getArgument(1)); + when(operations.aggregateStream(sent.capture(), eq("orders"), eq(Document.class))) + .thenAnswer(invocation -> documents.stream()); + return operations; + } + + private static Document order(int number) { + return new Document("_id", "order-" + number) + .append("status", "OPEN") + .append("customer", "customer-" + number); + } } diff --git a/src/adapter/outbound/persistence-notification/AGENTS.md b/src/adapter/outbound/persistence-notification/AGENTS.md index 94fa03ad..cb9ae8b1 100644 --- a/src/adapter/outbound/persistence-notification/AGENTS.md +++ b/src/adapter/outbound/persistence-notification/AGENTS.md @@ -8,7 +8,8 @@ - 패키지 루트: `dev.caskeleton.adapter.outbound.persistence.notification`. 패키지 규칙은 `persistence-fileserver`와 같아서 두 capability leaf를 같은 방식으로 읽는다: - `platform.entity` — JPA 행만. - - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). + - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). `platform`의 store도 + 이 SQL 조각을 쓴다. - `platform` — store 어댑터(`Jpa*`, `Jdbc*`, inbox의 `JpaNotificationInbox`와 그 outbox 도우미), `NotificationRecordMapper`, payload codec. - `configuration` — store를 bean으로 등록하는 자동 구성 창구. leaf 루트에는 scan 설정과 schema stream이 있다. diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java index 5a81e94b..72a52a8e 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java @@ -1,13 +1,17 @@ package dev.caskeleton.adapter.outbound.persistence.notification.platform; +import dev.caskeleton.adapter.outbound.persistence.notification.platform.repository.RecipientClaimSql; import dev.caskeleton.application.notification.platform.observation.NotificationServingState; import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Map; import java.util.Objects; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; /** * Measures the platform's serving state from the durable rows. @@ -17,34 +21,44 @@ import org.springframework.jdbc.core.JdbcOperations; * identical. These are the questions an operator actually asks during an incident: how far behind * are we, how long has the oldest thing waited, and is anything stuck. * - *

One statement per question rather than one joined statement. They are read independently, a - * slow one does not delay the others, and each index the queries rely on is the same index the - * corresponding worker already uses to claim. + *

One statement per question rather than one joined statement. They are read independently and a + * slow one does not delay the others. Every statement is bounded by an index: the backlog and + * stuck-lease counts by the predicates the claim uses, the reconciliation count by the job's due + * time, and the projection lag by {@code ix_notification_provider_event_open} (V14), which holds + * only the events still waiting for something — the provider-event table itself is never purged, so + * the lag query used to read all of it (round9 review 2026-11-12, RR9-01-14). + * + *

The scheduler reads only {@link #backlogDepth()}, on every poll; the full snapshot is for the + * health endpoint. */ public final class JdbcNotificationServingState implements NotificationServingStatePort { /** - * Deliveries that are due and unclaimed. + * Deliveries the claim would take now. * - *

The lease predicate matches the claim's: a row whose lease has expired is available again, - * so it is backlog. Counting it as in-flight is how a stuck worker hides a growing queue. + *

The {@code WHERE} is the claim's own fragment ({@link RecipientClaimSql#CLAIMABLE_AT_NOW}): + * the same states, due, not expired, and a lease that is absent or has run out. A row whose lease + * has expired is available again, so it is backlog; counting it as in-flight is how a stuck + * worker hides a growing queue. An expired delivery is not backlog: the expiry sweep retires it + * unsent. The fragment's parameter is named, so this statement is too. */ private static final String BACKLOG = """ SELECT count(*) AS depth, - COALESCE(EXTRACT(EPOCH FROM (? - min(next_dispatch_at))), 0) AS oldest_seconds + COALESCE(EXTRACT(EPOCH FROM (:now - min(next_dispatch_at))), 0) AS oldest_seconds FROM notification_recipient_delivery - WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING') - AND next_dispatch_at IS NOT NULL - AND next_dispatch_at <= ? - AND (lease_until IS NULL OR lease_until <= ?) - """; + WHERE\s""" + + RecipientClaimSql.CLAIMABLE_AT_NOW; /** * Leases whose holder never came back. * - *

Distinct from backlog: these rows are claimable again, but the fact that they had to expire - * means a worker died mid-dispatch, and a provider call may have happened without a record. + *

Distinct from backlog: these rows are still dispatching, which the claim never takes; lease + * recovery requeues or reconciles them one lease after they ran out. A row left from before + * notification V4 (an owner but fence 0) is counted here and not recovered; {@code + * docs/upgrade/2026-11-round9-claim-fence.md}, row 5, says how to hand it to recovery. That a + * lease had to run out means a worker died or stalled mid-dispatch, and a provider call may have + * happened without a record. */ private static final String STUCK_LEASES = """ @@ -56,7 +70,21 @@ public final class JdbcNotificationServingState implements NotificationServingSt AND delivery_state = 'DISPATCHING' """; - /** Provider events stored and not yet applied to a delivery. */ + /** + * The provider events still waiting for something: the lag query's {@code WHERE}, and the + * predicate of the partial index {@code V14} builds for it. One string, held to the migration by + * {@code JdbcNotificationServingStateOpenEventsTest}: a query the index does not cover reads the + * whole table again, which is what the index was made to stop. + */ + static final String OPEN_EVENTS = + "projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL"; + + /** + * Provider events stored and not yet applied to a delivery. + * + *

The {@code WHERE} is exactly the predicate of {@code ix_notification_provider_event_open}: + * every row any of the four counts can see, and none of the applied, matched history. + */ private static final String PROJECTION_LAG = """ SELECT @@ -66,7 +94,9 @@ public final class JdbcNotificationServingState implements NotificationServingSt COALESCE(EXTRACT(EPOCH FROM (? - min(received_at) FILTER (WHERE projection_status = 'PENDING'))), 0) AS oldest_seconds FROM notification_provider_event - """; + WHERE + """ + + OPEN_EVENTS; /** Questions to providers that are due and unanswered. */ private static final String RECONCILIATION_LAG = @@ -78,26 +108,36 @@ public final class JdbcNotificationServingState implements NotificationServingSt """; private final JdbcOperations jdbc; + private final NamedParameterJdbcOperations named; private final Clock clock; public JdbcNotificationServingState(JdbcOperations jdbc, Clock clock) { this.jdbc = Objects.requireNonNull(jdbc, "jdbc"); + this.named = new NamedParameterJdbcTemplate(jdbc); this.clock = Objects.requireNonNull(clock, "clock"); } + @Override + public long backlogDepth() { + Measurement backlog = measureBacklog(Timestamp.from(clock.instant())); + return backlog == null ? 0 : backlog.count(); + } + + /** The backlog query alone: the one statement the scheduler's queue-depth gauge needs. */ + private Measurement measureBacklog(Timestamp at) { + return named.queryForObject( + BACKLOG, + Map.of("now", at), + (rows, rowNumber) -> + new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds"))); + } + @Override public NotificationServingState currentState() { Instant now = clock.instant(); Timestamp at = Timestamp.from(now); - Measurement backlog = - jdbc.queryForObject( - BACKLOG, - (rows, rowNumber) -> - new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")), - at, - at, - at); + Measurement backlog = measureBacklog(at); Long stuck = jdbc.queryForObject(STUCK_LEASES, Long.class, at); ProjectionMeasurement projection = jdbc.queryForObject( diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java index 74728e9e..61b49e2c 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java @@ -33,10 +33,10 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort { @Override public dev.caskeleton.application.notification.platform.admin.AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action) { + String operationId, String requestFingerprint, AdminActor actor, String action) { CallerTransaction.require("admin operation claim"); Objects.requireNonNull(operationId, "operationId"); - Objects.requireNonNull(commandFingerprint, "commandFingerprint"); + Objects.requireNonNull(requestFingerprint, "requestFingerprint"); Objects.requireNonNull(actor, "actor"); Objects.requireNonNull(action, "action"); @@ -46,7 +46,7 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort { operationId, action, actor.actorRef(), - commandFingerprint, + requestFingerprint, clock.instant()); if (claimed == 1) { return dev.caskeleton.application.notification.platform.admin.AdminOperationClaim.claimed(); @@ -61,7 +61,7 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort { } String phase = String.valueOf(existing.get(0)[0]); String fingerprint = existing.get(0)[1] == null ? null : String.valueOf(existing.get(0)[1]); - if (fingerprint != null && !fingerprint.equals(commandFingerprint)) { + if (fingerprint != null && !fingerprint.equals(requestFingerprint)) { // The same id, a different command. Returning the earlier result would execute neither. return dev.caskeleton.application.notification.platform.admin.AdminOperationClaim.conflict(); } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java index 23fdb2dc..a2da23c2 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java @@ -99,6 +99,23 @@ public final class JpaRecipientDeliveryStore implements RecipientDeliveryStorePo return reread(written, record.id()); } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + CallerTransaction.require("recipient delivery requeueAbandoned"); + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + int written = + recipients.requeueAbandoned( + abandoned.recipientDeliveryId().value(), + abandoned.owner(), + abandoned.fence(), + abandoned.leaseUntil(), + nextDispatchAt, + clock.instant()); + return reread(written, abandoned.recipientDeliveryId()); + } + @Override public Optional transitionHeldBy( RecipientDeliveryId id, diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java index 65784bac..220d4acb 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java @@ -5,9 +5,11 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import dev.caskeleton.application.notification.platform.dispatch.RecipientLease; import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import java.sql.Timestamp; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.OffsetDateTime; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -37,9 +39,10 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort { } /** - * The one write here without {@code CallerTransaction.require}: the scheduler claims before any - * use case runs, and the claim is a single CTE statement that selects, locks and fences its rows - * by itself — there is no second write it has to be atomic with. + * The one write here without {@code CallerTransaction.require}: the claim is a single CTE + * statement that selects, locks and fences its rows by itself — there is no second write it has + * to be atomic with. {@code ClaimDueDeliveriesUseCase} opens a write transaction around it + * anyway, as every use case that writes does. */ @Override public List claim(String workerId, int limit, Duration leaseDuration) { @@ -109,13 +112,38 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort { } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); } return recipients.selectExpiredDispatching(clock.instant().minus(olderThan), limit).stream() - .map(RecipientDeliveryId::new) + .map( + row -> + new RecipientLease( + new RecipientDeliveryId((UUID) row[0]), + (String) row[1], + ((Number) row[2]).longValue(), + instantOf(row[3]))) .toList(); } + + /** + * A {@code timestamptz} column of a native row. Hibernate hands it back as an {@link + * OffsetDateTime} or a {@link Timestamp} depending on its JDBC type mapping, never null here + * because the select only returns rows that have a lease. + */ + private static Instant instantOf(Object column) { + if (column instanceof Instant instant) { + return instant; + } + if (column instanceof OffsetDateTime offset) { + return offset.toInstant(); + } + if (column instanceof Timestamp timestamp) { + return timestamp.toInstant(); + } + throw new IllegalStateException( + "lease_until came back as " + (column == null ? "null" : column.getClass().getName())); + } } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java index 15428027..e7ccfb32 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java @@ -60,18 +60,16 @@ public interface InboxItemJpaRepository extends Repository{@code read_at IS NULL} in the predicate is what makes the update idempotent under * concurrency: a second caller updates zero rows instead of overwriting the first caller's * timestamp. - */ - /** - * Marks everything up to and including a cursor position as read. * *

The cutoff is a tuple, matching the list's own {@code (created_at DESC, id DESC)} ordering. * It used to be a timestamp alone, and the list's order is not a total order on {@code @@ -95,14 +93,23 @@ public interface InboxItemJpaRepository extends RepositoryThe lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code * PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and * the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The - * named parameters are Spring Data's. + * named parameters are bound by Spring Data for the repository statements, and by a {@code + * NamedParameterJdbcTemplate} where the serving state reads {@link #CLAIMABLE_AT_NOW}. */ public final class RecipientClaimSql { + /** + * The states a delivery waits in to be sent, which the claim takes from, the expiry retires from + * and the serving state counts as backlog. One list so the statements cannot disagree about what + * is waiting. + */ + public static final String CLAIMABLE_STATES = "('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"; + + /** + * Which deliveries the claim takes at {@code :now}: due, in a waiting state, not expired, and + * with no lease or one that has run out. The serving state counts its backlog with this same + * fragment, so the backlog is exactly what a claim would take. + */ + public static final String CLAIMABLE_AT_NOW = + "next_dispatch_at <= :now" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND (expires_at IS NULL OR expires_at > :now)" + + " AND (lease_until IS NULL OR lease_until < :now)"; + /** * Claims due deliveries and returns their fences. * @@ -28,10 +48,8 @@ public final class RecipientClaimSql { public static final String CLAIM_BATCH = "WITH claimable AS (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE next_dispatch_at <= :now" - + " AND delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" - + " AND (expires_at IS NULL OR expires_at > :now)" - + " AND (lease_until IS NULL OR lease_until < :now)" + + " WHERE " + + CLAIMABLE_AT_NOW + " ORDER BY next_dispatch_at, id" + " FOR UPDATE SKIP LOCKED" + " LIMIT :batchSize" @@ -51,7 +69,18 @@ public final class RecipientClaimSql { + " WHERE d.id = claimable.id" + " RETURNING d.id, d.lease_fence"; - /** Retires deliveries whose window closed before anyone claimed them. */ + /** + * Retires deliveries whose window closed before anyone claimed them. + * + *

A delivery a worker claimed while this statement ran must not be retired to {@code EXPIRED} + * under the worker dispatching it, which is what the unlocked subquery did (round9 review + * 2026-11-12, RR9-01-13). Two things stop it. The subquery locks the rows it picks and skips rows + * another transaction holds, so a delivery being claimed is not picked. The state and expiry + * conditions are also on the {@code UPDATE} itself: under READ COMMITTED PostgreSQL re-checks + * only the outer {@code WHERE} against a row it had to wait for, so if the lock clause is ever + * lost the claimed row is still left alone. A delivery being claimed right now is not expiring + * either: the claim refuses expired rows in the same statement. + */ public static final String EXPIRE_OVERDUE = "UPDATE notification_recipient_delivery SET" + " delivery_state = 'EXPIRED'," @@ -61,11 +90,17 @@ public final class RecipientClaimSql { + " updated_at = :now" + " WHERE id IN (" + " SELECT id FROM notification_recipient_delivery" - + " WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')" + + " WHERE delivery_state IN " + + CLAIMABLE_STATES + " AND expires_at IS NOT NULL" + " AND expires_at <= :now" + " ORDER BY expires_at" - + " LIMIT :batchSize)"; + + " FOR UPDATE SKIP LOCKED" + + " LIMIT :batchSize)" + + " AND delivery_state IN " + + CLAIMABLE_STATES + + " AND expires_at IS NOT NULL" + + " AND expires_at <= :now"; private RecipientClaimSql() {} } diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java index b5f1ee18..1240cf0f 100644 --- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java +++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java @@ -183,13 +183,48 @@ public interface RecipientDeliveryJpaRepository extends RepositoryOnly rows a claim wrote: an owner and a fence of at least one. The requeue matches on both, + * and a row without them — one left dispatching before notification V4 added the fence — is no + * lease anyone holds. Read here it would be a lease the application refuses to build, and the one + * row would fail every recovery pass after it. + * + * @return one row per delivery: id, lease_owner, lease_fence, lease_until + */ @Query( value = - "SELECT id FROM notification_recipient_delivery " + "SELECT id, lease_owner, lease_fence, lease_until FROM notification_recipient_delivery " + "WHERE delivery_state = 'DISPATCHING' AND lease_until < :threshold " + + "AND lease_owner IS NOT NULL AND lease_fence > 0 " + "ORDER BY lease_until LIMIT :batchSize", nativeQuery = true) - List selectExpiredDispatching( + List selectExpiredDispatching( @Param("threshold") Instant threshold, @Param("batchSize") int batchSize); + + /** + * Returns an abandoned dispatch to the queue while it is still the one recovery found. + * + *

Owner, fence, state and expiry are all conditions. A new claim raises the fence, a finished + * holder moves the state, another instance's recovery moves the state, and a holder that renewed + * in time moves the expiry past what recovery saw; in each case the job is no longer abandoned + * and this matches nothing. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query( + value = + "UPDATE notification_recipient_delivery " + + "SET delivery_state = 'READY_TO_DISPATCH', next_dispatch_at = :nextDispatchAt, " + + "version = version + 1, updated_at = :now " + + "WHERE id = :id AND lease_owner = :owner AND lease_fence = :fence " + + "AND delivery_state = 'DISPATCHING' AND lease_until <= :leaseUntil", + nativeQuery = true) + int requeueAbandoned( + @Param("id") UUID id, + @Param("owner") String owner, + @Param("fence") long fence, + @Param("leaseUntil") Instant leaseUntil, + @Param("nextDispatchAt") Instant nextDispatchAt, + @Param("now") Instant now); } diff --git a/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql new file mode 100644 index 00000000..2d2416a2 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql @@ -0,0 +1,26 @@ +-- An index over the provider events still waiting for something. +-- +-- The serving-state projection lag counts pending, failed and unmatched events. It had no WHERE and +-- read the whole notification_provider_event table, which nothing ever purges, and the scheduler +-- asked for it on every poll (round9 review 2026-11-12, RR9-01-14). The query now names exactly +-- this predicate, so it reads only these rows; applied, matched events — the history that grows — +-- are outside the index. +-- +-- Built inside this migration's transaction, not CONCURRENTLY, as V21 of the core stream builds its +-- index: a transactional file fails whole and leaves nothing to repair, where a failed concurrent +-- build leaves an INVALID index and a failed history row an operator must clear by hand +-- (docs/runbooks/migration-failed.md). The cost is that the build holds SHARE on +-- notification_provider_event: provider callbacks wait for it, and providers retry a callback that +-- does not get an answer. On a table already large in production, create the same index +-- CONCURRENTLY by hand before deploying; IF NOT EXISTS then makes this migration a no-op. + +-- Bound the wait for the lock (SHARE on notification_provider_event, held until commit). Unbounded, +-- it queues behind any open transaction that touched the table, and every callback insert queues +-- behind the migration. Five seconds, then the migration fails, rolls back whole and the stream +-- stays at V13: deploy again once that transaction has ended (docs/jpa/migration-guide.md "Bound +-- the lock wait"). +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX IF NOT EXISTS ix_notification_provider_event_open + ON notification_provider_event (received_at) + WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL; diff --git a/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java new file mode 100644 index 00000000..0805a0d6 --- /dev/null +++ b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java @@ -0,0 +1,42 @@ +package dev.caskeleton.adapter.outbound.persistence.notification.platform; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import org.junit.jupiter.api.Test; + +/** + * The projection-lag query and the partial index built for it name the same events. + * + *

The index is what keeps the lag query, which the scheduler runs on every poll, from reading + * the whole provider-event history. A predicate that drifted on either side would leave the query + * outside the index and scanning the table again, with nothing failing. + */ +class JdbcNotificationServingStateOpenEventsTest { + + private static final String V14 = + "db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql"; + + @Test + void theLagQueryAndItsPartialIndexNameTheSameEvents() throws IOException { + // given + String migration; + try (InputStream sql = getClass().getClassLoader().getResourceAsStream(V14)) { + migration = new String(Objects.requireNonNull(sql, V14).readAllBytes(), UTF_8); + } + int index = migration.indexOf("CREATE INDEX"); + + // when + String indexPredicate = + migration + .substring( + migration.indexOf("WHERE ", index) + "WHERE ".length(), migration.lastIndexOf(';')) + .trim(); + + // then + assertThat(indexPredicate).isEqualTo(JdbcNotificationServingState.OPEN_EVENTS); + } +} diff --git a/src/application/core/AGENTS.md b/src/application/core/AGENTS.md index 9d5e5859..de330cad 100644 --- a/src/application/core/AGENTS.md +++ b/src/application/core/AGENTS.md @@ -73,11 +73,12 @@ architectureTest)가 강제한다. 나머지는 리뷰 규칙이고, 리뷰가 |---|---|---|---| | 유스케이스(inbound 포트 구현) | `UseCase`, ArchUnit이 강제 | 기능의 `application` 패키지 | `*ApplicationUseCase`(application-fileserver, application-notification), `*Service`(notification, fileserver) | | command·query record | `*Command` / `*Query` | 그 유스케이스 옆 | — | +| 요청 값 record: command가 담는 호출자 요청 내용(호출자가 누구인지는 뺀 값), 또는 포트 인자 묶음 | `*Request` | 그것을 담는 command나 받는 포트 옆 | `CreateUploadRequest`·`FinalizeUploadRequest`(application-fileserver `upload`, command는 `port.in`), `VerificationRequest`(application-fileserver `api.security`, 포트는 `upload`), application-objectstorage의 `request` 패키지(포트는 `port`, leaf가 값 타입 패키지로 정했다) | | aggregate 영속(aggregate 전체를 저장하고 돌려준다) | `Repository` | aggregate의 **domain** 패키지, 도메인이 가진다 | — | | 애플리케이션이 가진 기술 저장소(outbox, 멱등성, operation 행) | `StorePort` | application 패키지 | `*WritePort`(옛 onboarding fixture, 지움) | | aggregate가 아니라 projection을 돌려주는 읽기 모델 | `QueryPort` | application 패키지. projection만 돌려준다(ArchUnit) | — | | 그 밖의 outbound 포트 | `Port` | application 패키지 | — | -| 서버가 정하는 id 생성 | `IdFactory extends IdFactory<Id>` | domain 패키지. id record는 `implements ResourceId` | `DurableOperationIdFactory`(durable operation id는 aggregate의 resource id가 아니라 `IdFactory`를 따르지 않는다) | +| 서버가 정하는 id 생성 | `IdFactory extends IdFactory<Id>` | domain 패키지. id record는 `implements ResourceId` | `DurableOperationIdFactory`(durable operation id는 aggregate의 resource id가 아니라 `IdFactory`를 따르지 않는다), `NotificationIdGeneratorPort`(application-notification), `UploadIdentifierPort`(application-fileserver). 셋 다 사용자 전송이 쥐면 `USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS`가 막는다 | | 설정 바인딩 | `*Settings`(운영) 또는 `*Policy`(인가·도메인 정책), ArchUnit이 강제 | 그것을 바인딩하는 leaf | — | | Throwable | `*Exception`, ArchUnit이 강제 | — | — | | 포트의 계약 시험 키트 | 포트 소유자의 testFixtures에 둔 추상 `Contract`. fake의 시험과 각 어댑터의 시험이 상속한다 | testFixtures | 값 타입 시험의 이름이 한때 `*ContractTest`였다. 지금은 `*ValueTypesTest`다 | diff --git a/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java b/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java index aeff40dc..35885ba0 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java @@ -13,6 +13,10 @@ import java.util.regex.Pattern; * until the record expired (round6 RR6-02-01, {@code IdempotencyStoreContract}). One attempt is one * sender at a time: a handle from before a takeover is stale, but two calls sending the same * attempt at once are not told apart — retry after the call has returned, not beside it. + * + *

The owner token is the store's: 32 secure-random bytes as 64 lower-case hex characters, which + * both stores compare as a fixed-width value. It is not a lease owner token (Base64URL, chosen by + * the lease caller); the transaction {@code OperationId} Javadoc compares the identities. */ public record IdempotencyClaimAttempt(String ownerToken, OperationId operationId) { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java b/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java index 4618c613..6d32894d 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java @@ -11,6 +11,11 @@ final class LeaseValidation { private LeaseValidation() {} + /** + * A lease's operation id or owner token: 16..128 Base64URL characters, so it fits a header and a + * Redis key without escaping and is long enough to be unguessable. Narrower than the transaction + * {@code OperationId}, whose Javadoc compares the operation-shaped ids and why they differ. + */ static String opaqueToken(String value, String field) { if (value == null || value.length() < 16 diff --git a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java index 535a146d..8c4a6de9 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java @@ -19,10 +19,15 @@ public interface IntegrationEventPublishPort { CompletionStage publish(ValidatedIntegrationEvent event); /** - * The longest one publish can take before the returned stage completes, when the publisher knows - * it. The outbox relay sizes its batch against it (see {@code + * The longest one publish can take, from the call to {@code publish} until the returned stage + * completes, when the publisher knows it. The outbox relay sizes its batch against it (see {@code * OutboxMessagePublishPort#longestAttempt}). * + *

A bound, not a hint: a caller may stop waiting once it has passed and treat the attempt as + * ambiguous, so a publisher that reports less than it can take turns slow answers into ambiguous + * ones. Positive when present. Empty means unknown, and then the stage's own completion is the + * only bound. + * * @return the longest attempt, or empty when unknown */ default Optional longestAttempt() { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java index adc509c3..b4958ab6 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java @@ -1,5 +1,9 @@ package dev.caskeleton.application.messaging.event; +import dev.caskeleton.shared.digest.Sha256; +import java.time.Instant; +import java.util.Objects; +import java.util.UUID; import java.util.regex.Pattern; /** @@ -73,6 +77,35 @@ public final class IntegrationIdentity { return value != null && UUID_V7.matcher(value).matches(); } + /** + * A canonical lower-case UUIDv7 derived from {@code value}: the same value and instant always + * give the same id. + * + *

For an event whose id must follow from something that is not itself a UUIDv7 — an operation + * id a client chose, say — so a retry drafts the same event. The time field is {@code at}, so the + * id orders like a minted one; the other 74 bits are the SHA-256 of {@code value}. Kept here, + * with the grammar it must satisfy, rather than in each use case that needs one: the derivation + * is messaging identity, not the use case's own rule. + * + * @param value what the id is derived from + * @param at the instant its time field carries + * @return a canonical lower-case UUIDv7 + */ + public static String uuidV7Derived(String value, Instant at) { + Objects.requireNonNull(value, "value"); + Objects.requireNonNull(at, "at"); + byte[] digest = Sha256.ofUtf8(value).bytes(); + long millis = at.toEpochMilli() & 0xFFFF_FFFF_FFFFL; + long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL); + long mostSignificant = (millis << 16) | (0x7L << 12) | randA; + long randB = 0; + for (int i = 2; i < 10; i++) { + randB = (randB << 8) | (digest[i] & 0xFFL); + } + long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L; + return new UUID(mostSignificant, leastSignificant).toString(); + } + /** Refuses a correlation or causation id outside the grammar or the bound. */ static void requireCorrelation(String field, String value) { if (!isCanonical(value, CORRELATION_MAX_LENGTH)) { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java index aa9fc88e..6b4e0a6b 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java @@ -1,19 +1,23 @@ package dev.caskeleton.application.operation; import dev.caskeleton.application.command.Command; +import dev.caskeleton.application.tenant.TenantId; import java.util.Objects; +import java.util.Optional; /** * A caller asking to cancel one operation. * * @param operationId the identity to cancel * @param principal the authenticated caller, or null when the caller is not authenticated - * @param tenantId the caller's tenant, or null when the request is not tenant scoped + * @param tenant the caller's tenant, empty when the request is not tenant scoped */ public record CancelDurableOperationCommand( - DurableOperationId operationId, String principal, String tenantId) implements Command { + DurableOperationId operationId, String principal, Optional tenant) + implements Command { public CancelDurableOperationCommand { Objects.requireNonNull(operationId, "operationId"); + Objects.requireNonNull(tenant, "tenant"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java index 6c3c2449..9aa35d16 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java @@ -63,7 +63,7 @@ public class CancelDurableOperationUseCase .filter( operation -> DurableOperationAccessPolicy.mayAccess( - operation, command.principal(), command.tenantId())) + operation, command.principal(), command.tenant())) .map(visible -> operations.cancel(command.operationId(), command.principal(), now)); }); } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java index 6674f707..3e799bf6 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java @@ -1,6 +1,8 @@ package dev.caskeleton.application.operation; +import dev.caskeleton.application.tenant.TenantId; import java.util.Objects; +import java.util.Optional; /** * Who may read or cancel an operation: the principal that submitted it, in the tenant it was @@ -20,10 +22,21 @@ final class DurableOperationAccessPolicy { * * @param operation the stored operation * @param principal the authenticated caller, or null when the caller is not authenticated - * @param tenantId the caller's tenant, or null when the request is not tenant scoped + * @param tenant the caller's tenant, empty when the request is not tenant scoped */ - static boolean mayAccess(DurableOperation operation, String principal, String tenantId) { + static boolean mayAccess( + DurableOperation operation, String principal, Optional tenant) { return operation.principal().equals(principal) - && Objects.equals(operation.tenantId(), tenantId); + && Objects.equals(operation.tenantId(), storedForm(tenant)); + } + + /** + * The tenant as the operation row holds it: its value, or null when not tenant scoped. + * + * @param tenant the caller's tenant + * @return the stored form + */ + static String storedForm(Optional tenant) { + return tenant.map(TenantId::value).orElse(null); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java new file mode 100644 index 00000000..80bdbe0c --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java @@ -0,0 +1,45 @@ +package dev.caskeleton.application.operation; + +import java.util.Objects; + +/** + * One worker's claim on one durable operation, as returned by {@link + * DurableOperationStorePort#claimNext}. + * + *

The {@code fence} is what tells one run from the next. Every claim increments it, and a + * heartbeat, progress report or completion names the fence it holds, so a run that stalled past its + * lease finds nothing left to change instead of overwriting the run that took over. The worker id + * alone cannot do this: a worker that restarts with the same id, or one process running two + * executions under one id, would otherwise have its earlier run accepted as the current one. + * + *

Kept apart from {@link DurableOperationSubmission} on purpose, as the outbox keeps its lease + * apart from the claimed event: the work to execute is the submission, and the claim is store + * bookkeeping the work has no use for. + * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * + * @param submission the claimed work + * @param workerId the worker that holds the claim + * @param fence the fencing token this claim was granted; at least 1 + */ +public record DurableOperationClaim( + DurableOperationSubmission submission, String workerId, long fence) { + + public DurableOperationClaim { + Objects.requireNonNull(submission, "submission"); + if (workerId == null || workerId.isBlank()) { + throw new IllegalStateException( + "a claim needs an owner; an anonymous one cannot be verified when the work completes"); + } + if (fence < 1) { + throw new IllegalStateException("fence must be >= 1, was " + fence); + } + } + + /** The claimed operation. */ + public DurableOperationId operationId() { + return submission.operationId(); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java index 573e0135..f8c85981 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java @@ -10,9 +10,10 @@ import java.util.regex.Pattern; */ public record DurableOperationId(String value) { - // Bounded and free of anything that needs escaping. The identifier travels to whatever transport - // published the operation and is handed back to callers, so a value that one encoding mangles is - // a defect that only appears for some clients. + // Bounded and free of anything that needs escaping: RFC 3986 unreserved characters, because the + // identifier is a URL path segment (/operations/{id}) and is handed back to callers, so a value + // that one encoding mangles is a defect that only appears for some clients. Narrower than the + // transaction OperationId for that reason (its Javadoc compares the operation-shaped ids). private static final Pattern GRAMMAR = Pattern.compile("[A-Za-z0-9._~-]{1,128}"); public DurableOperationId { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java index 440d3fb2..887f5c72 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java @@ -11,9 +11,12 @@ import java.util.Optional; *

A port because the application owns what an operation is and nothing about how it is stored. * The JPA adapter behind it is one implementation; an in-memory one serves tests. * - *

Every method that changes a running operation takes the {@code workerId} and verifies it. A - * worker whose lease lapsed may still be alive and still finishing its work, and letting it report - * a result after another worker took over is how an operation ends up with two answers. + *

Every method that changes a running operation takes the {@link DurableOperationClaim} that + * {@link #claimNext} returned and verifies both its worker and its fence. A worker whose lease + * lapsed may still be alive and still finishing its work, and letting it report a result after + * another run took over is how an operation ends up with two answers. The worker id alone does not + * tell the runs apart: the same id may claim the operation again after a restart, so the fence — + * incremented by every claim — is what the store compares. */ public interface DurableOperationStorePort { @@ -24,8 +27,8 @@ public interface DurableOperationStorePort { * rather than a second one is what makes a resubmission safe while the first is still running. * * @param submission what was accepted - * @return the operation, and whether this call created it; a retry under the same operation id is - * not a creation + * @return the operation, and whether this call created it; a resubmission that finds the + * operation an earlier call created, under its own fresh id, is not a creation */ DurableOperationSubmitResult submit(DurableOperationSubmission submission); @@ -43,48 +46,43 @@ public interface DurableOperationStorePort { * *

Runnable means PENDING, or RUNNING with a lapsed lease. The second case is the recovery * path: without it, an operation claimed by a worker that then died is never picked up again. + * Every claim gets a fence higher than any earlier claim of the same operation, whoever made it. * * @param workerId who is claiming * @param leaseDuration how long the claim holds without a heartbeat * @param now the current instant + * @return the claim the worker names on every later write, or empty when nothing is runnable */ - Optional claimNext( - String workerId, Duration leaseDuration, Instant now); + Optional claimNext(String workerId, Duration leaseDuration, Instant now); /** - * Extends a lease the worker still holds. + * Extends a lease the claim still holds. * - * @return false when the lease is no longer this worker's, which means it must stop + * @return false when the lease lapsed or another claim (by any worker, this one included) took + * the operation over, which means this run must stop */ - boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now); + boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now); /** - * Publishes progress for an operation this worker holds. + * Publishes progress for an operation the claim holds. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over or it is no longer running */ - boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress); + boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress); /** * Records success. * - * @return false when the lease is no longer this worker's, so the result is not recorded + * @return false when another claim took the operation over, so the result is not recorded */ - boolean succeed( - DurableOperationId operationId, String workerId, String resultReference, Instant completedAt); + boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt); /** * Records failure. * - * @return false when the lease is no longer this worker's + * @return false when another claim took the operation over, so the failure is not recorded */ - boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt); + boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt); /** * Requests cancellation. diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java index 1d43820a..2c0edd5d 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java @@ -5,10 +5,10 @@ import java.util.Objects; /** * What a store did with a submission. * - *

{@code created} comes from the store's own write, not from comparing identifiers afterwards. A - * retry under the same operation id finds the operation it created itself, and an identifier - * comparison cannot tell that retry from the first call — which is how a retry appended its outbox - * event a second time. + *

{@code created} comes from the store's own write, not from comparing identifiers afterwards. + * Every call mints a fresh operation id, and a resubmission is found by scope and fingerprint as + * the operation the first call created; only the store's write tells that resubmission from the + * first call, which is what keeps it from appending its outbox event a second time. * * @param operation the operation holding the submission's scope * @param created whether this call inserted it diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java index e6ce2ae6..afb85f8f 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java @@ -1,19 +1,22 @@ package dev.caskeleton.application.operation; import dev.caskeleton.application.query.Query; +import dev.caskeleton.application.tenant.TenantId; import java.util.Objects; +import java.util.Optional; /** * A caller asking to read one operation. * * @param operationId the identity being polled * @param principal the authenticated caller, or null when the caller is not authenticated - * @param tenantId the caller's tenant, or null when the request is not tenant scoped + * @param tenant the caller's tenant, empty when the request is not tenant scoped */ public record GetDurableOperationQuery( - DurableOperationId operationId, String principal, String tenantId) implements Query { + DurableOperationId operationId, String principal, Optional tenant) implements Query { public GetDurableOperationQuery { Objects.requireNonNull(operationId, "operationId"); + Objects.requireNonNull(tenant, "tenant"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java index 3fb1bfed..4a66ef81 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java @@ -15,7 +15,7 @@ import java.util.Optional; transactionMode = TransactionMode.READ_ONLY, idempotency = Idempotency.IDEMPOTENT, repositoryAccess = RepositoryAccess.READ_REPOSITORY) -public final class GetDurableOperationUseCase +public class GetDurableOperationUseCase implements QueryUseCase> { private final DurableOperationStorePort operations; @@ -47,6 +47,6 @@ public final class GetDurableOperationUseCase .filter( operation -> DurableOperationAccessPolicy.mayAccess( - operation, query.principal(), query.tenantId())); + operation, query.principal(), query.tenant())); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java index 0d401654..d56f22bb 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java @@ -2,34 +2,34 @@ package dev.caskeleton.application.operation; import dev.caskeleton.application.command.Command; import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.tenant.TenantId; import java.time.Duration; import java.util.Objects; +import java.util.Optional; /** * A request to run work asynchronously. * - *

The caller supplies the operation id rather than the store minting one. That is what lets the - * transport hand back a {@code Location} it already knows, and it makes the whole submission one - * value the caller can retry: a generated id would differ on every attempt, so a retry after an - * ambiguous failure would create a second operation. + *

No operation id: {@link SubmitDurableOperationUseCase} mints one with {@link + * DurableOperationIdFactory}, and a resubmission of the same work by the same caller is recognised + * by its scope and payload fingerprint, not by an id the caller kept. The transport used to mint it + * and hand it in (round9 review 2026-11-12, RR9-05-13). * *

There is no event type to choose. Every accepted operation announces itself with the one * {@link DurableOperationSubmittedContract} event, which only wakes a worker; letting a caller name * the type would let it publish any event the broker carries. * - * @param operationId the identity to use * @param operationName what work this is * @param principal who is asking - * @param tenantId the tenant, or null when the operation is not tenant scoped + * @param tenant the tenant, empty when the operation is not tenant scoped * @param fingerprint the semantic fingerprint of the request * @param payload the serialized request a worker will execute * @param timeToLive how long the record and its result stay available */ public record OperationCommand( - DurableOperationId operationId, String operationName, String principal, - String tenantId, + Optional tenant, RequestFingerprint fingerprint, String payload, Duration timeToLive) @@ -39,8 +39,8 @@ public record OperationCommand( public static final Duration DEFAULT_TIME_TO_LIVE = Duration.ofDays(1); public OperationCommand { - Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(operationName, "operationName"); + Objects.requireNonNull(tenant, "tenant"); Objects.requireNonNull(fingerprint, "fingerprint"); Objects.requireNonNull(payload, "payload"); Objects.requireNonNull(timeToLive, "timeToLive"); diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java index 86328d84..618f23c1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java @@ -11,6 +11,10 @@ import java.util.Objects; * ever with no worker behind it — the failure that makes a queue stop draining while every * dashboard reports it busy. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param workerId who holds it * @param acquiredAt when it was taken * @param expiresAt when another worker may take it @@ -22,11 +26,11 @@ public record OperationLease(String workerId, Instant acquiredAt, Instant expire Objects.requireNonNull(acquiredAt, "acquiredAt"); Objects.requireNonNull(expiresAt, "expiresAt"); if (workerId.isBlank()) { - throw new IllegalArgumentException( + throw new IllegalStateException( "a lease needs an owner; an anonymous one cannot be verified when the work completes"); } if (!expiresAt.isAfter(acquiredAt)) { - throw new IllegalArgumentException("a lease that expires when taken is not a lease"); + throw new IllegalStateException("a lease that expires when taken is not a lease"); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java index e5d8719c..45471cef 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java @@ -15,12 +15,10 @@ import dev.caskeleton.application.security.RequiresPermission; import dev.caskeleton.application.transaction.TransactionMode; import dev.caskeleton.application.transaction.TransactionPort; import dev.caskeleton.application.usecase.CommandUseCase; -import dev.caskeleton.shared.digest.Sha256; import java.time.Clock; import java.time.Instant; import java.util.Objects; import java.util.Optional; -import java.util.UUID; /** * Accepts long-running work: one transaction that writes the operation row and the outbox row. @@ -49,8 +47,9 @@ import java.util.UUID; @RequiresPermission("operation:submit") @UseCaseCapability( transactionMode = TransactionMode.WRITE, - // KEYED, not IDEMPOTENT. Re-running this use case is only safe because the caller supplies the - // operation id and the fingerprint scope; without a key it would accept the same work twice. + // KEYED, not IDEMPOTENT. Re-running this use case is only safe because the store recognises the + // submission scope and the payload fingerprint; without that key it would accept the same work + // twice. idempotency = Idempotency.KEYED, repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, // False, and load-bearing. Nothing here reaches the broker: the outbox row is the publish @@ -72,6 +71,7 @@ public class SubmitDurableOperationUseCase private final OutboxAppendPort outbox; private final IntegrationEventEncoderPort encoder; private final TransactionPort transactions; + private final DurableOperationIdFactory operationIds; private final Clock clock; /** @@ -82,6 +82,7 @@ public class SubmitDurableOperationUseCase * @param encoder turns the submission event into the validated event the outbox stores; it must * know {@link DurableOperationSubmittedContract} * @param transactions opens the write transaction both share + * @param operationIds mints the id of a new submission * @param clock the submission clock */ public SubmitDurableOperationUseCase( @@ -89,8 +90,10 @@ public class SubmitDurableOperationUseCase OutboxAppendPort outbox, IntegrationEventEncoderPort encoder, TransactionPort transactions, + DurableOperationIdFactory operationIds, Clock clock) { this.operations = Objects.requireNonNull(operations, "operations"); + this.operationIds = Objects.requireNonNull(operationIds, "operationIds"); this.outbox = Objects.requireNonNull(outbox, "outbox"); this.encoder = Objects.requireNonNull(encoder, "encoder"); this.transactions = Objects.requireNonNull(transactions, "transactions"); @@ -110,18 +113,20 @@ public class SubmitDurableOperationUseCase () -> { DurableOperationSubmission submission = new DurableOperationSubmission( - command.operationId(), + // Minted per call. A resubmission's id is never stored: the store answers with + // the operation the first submission created. + operationIds.newOperationId(), command.operationName(), command.principal(), - command.tenantId(), + DurableOperationAccessPolicy.storedForm(command.tenant()), command.fingerprint(), command.payload(), clock.instant(), command.timeToLive()); DurableOperationSubmitResult submitted = operations.submit(submission); DurableOperation operation = submitted.operation(); - // The store's answer, not an id comparison: a retry under the same operation id finds - // the operation it created itself, and the ids match. + // The store's answer, not an id comparison: every call mints a fresh id, and a retry of + // the same submission is found by the store as the operation the first call created. boolean alreadyAccepted = !submitted.created(); if (!alreadyAccepted) { @@ -147,8 +152,8 @@ public class SubmitDurableOperationUseCase // Single-tenant scope even when the command names a tenant: the event only wakes a worker, the // tenant travels in the stored operation, and mapping a TenantId into the lower-case scope // grammar is the messaging platform's decision, not this use case's. - // No request correlation: the event correlates to the operation itself, so a retry under the - // same operation id produces the same event. + // No request correlation: the event correlates to the operation itself, so a retry that the + // store resolves to the same operation produces the same event. return IntegrationEventDrafts.draft( CONTRACT, new EventId(identity), @@ -160,15 +165,15 @@ public class SubmitDurableOperationUseCase } /** - * The operation id itself when it is a canonical lower-case UUIDv7 — every id the HTTP adapter - * mints is — and otherwise a UUIDv7 derived from it. + * The operation id itself when it is a canonical lower-case UUIDv7 — every id the shipped {@code + * DurableOperationIdFactory} mints is — and otherwise a UUIDv7 derived from it. * *

The event id is the messaging platform's message id, which is a canonical lower-case UUIDv7; - * the draft refuses anything else, so an operation id a client chose ({@code ~report}, a v4, an - * upper-case spelling) cannot be the event id. The derived id keeps the event deterministic for - * the operation — the same id and submission time always give the same event id — and the payload - * carries the operation id unchanged for the worker. Its time field is the submission time, so it - * orders like a minted one; its other 74 bits are the SHA-256 of the operation id. + * the draft refuses anything else, so an id a deployment's own factory mints ({@code ~report}, a + * v4, an upper-case spelling) cannot be the event id. The derived id keeps the event + * deterministic for the operation — the same id and submission time always give the same event id + * — and the payload carries the operation id unchanged for the worker. The derivation is {@link + * IntegrationIdentity#uuidV7Derived}, with the submission time as its time field. * *

It replaced an {@code operation-} digest, which fit the stored identity grammar and * passed the append, and which the bridge then refused, parking the committed event (review @@ -179,15 +184,6 @@ public class SubmitDurableOperationUseCase if (IntegrationIdentity.isCanonicalUuidV7(value)) { return value; } - byte[] digest = Sha256.ofUtf8(value).bytes(); - long millis = submittedAt.toEpochMilli() & 0xFFFF_FFFF_FFFFL; - long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL); - long mostSignificant = (millis << 16) | (0x7L << 12) | randA; - long randB = 0; - for (int i = 2; i < 10; i++) { - randB = (randB << 8) | (digest[i] & 0xFFL); - } - long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L; - return new UUID(mostSignificant, leastSignificant).toString(); + return IntegrationIdentity.uuidV7Derived(value, submittedAt); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java index bafa1b5e..05c0062e 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java @@ -12,6 +12,10 @@ import java.util.Objects; * publishes the bytes that were validated rather than a re-rendering of them. The accessors below * name the few identities the relay and its diagnostics read without reaching into the envelope. * + *

A store builds it from a row, never from a request, so a count it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the appended event * @param status the row's lifecycle status; {@link OutboxEventStatus#IN_FLIGHT} right after a claim * @param attemptCount claims made, counting the claim that returned this row; the retry delay grows @@ -30,10 +34,10 @@ public record ClaimedOutboxEvent( Objects.requireNonNull(event, "event must not be null"); Objects.requireNonNull(status, "status must not be null"); if (attemptCount < 0) { - throw new IllegalArgumentException("attemptCount must be >= 0, was " + attemptCount); + throw new IllegalStateException("attemptCount must be >= 0, was " + attemptCount); } if (deferredAttemptCount < 0 || deferredAttemptCount > attemptCount) { - throw new IllegalArgumentException( + throw new IllegalStateException( "deferredAttemptCount must be between 0 and attemptCount (" + attemptCount + "), was " diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java index eed397ab..ae783bc1 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java @@ -13,6 +13,10 @@ import java.util.Objects; *

Kept apart from {@link ClaimedOutboxEvent} on purpose: the broker adapter receives the event, * and the claim is store bookkeeping it has no use for. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param event the claimed row * @param owner the relay instance that holds the claim * @param token the fencing token this claim was granted @@ -22,10 +26,10 @@ public record OutboxLease(ClaimedOutboxEvent event, String owner, long token) { public OutboxLease { Objects.requireNonNull(event, "event must not be null"); if (owner == null || owner.isBlank()) { - throw new IllegalArgumentException("owner must not be blank"); + throw new IllegalStateException("owner must not be blank"); } if (token < 1) { - throw new IllegalArgumentException("token must be >= 1, was " + token); + throw new IllegalStateException("token must be >= 1, was " + token); } } diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java index 68ae05cf..0425030a 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java @@ -40,7 +40,7 @@ import java.util.function.Supplier; idempotency = Idempotency.IDEMPOTENT, repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, externalOutboundAllowed = true) -public final class PublishPendingOutboxEventsUseCase +public class PublishPendingOutboxEventsUseCase implements CommandUseCase { /** The failure code of a thrown failure whose type has no simple name. */ diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java new file mode 100644 index 00000000..14879d5a --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java @@ -0,0 +1,16 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.query.Query; +import java.util.Objects; + +/** + * A reconnecting client's stream, asked what it still holds. + * + * @param streamId which stream + */ +public record GetReplayWindowQuery(String streamId) implements Query { + + public GetReplayWindowQuery { + Objects.requireNonNull(streamId, "streamId"); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java new file mode 100644 index 00000000..6d586a46 --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java @@ -0,0 +1,41 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.QueryUseCase; +import java.util.Objects; + +/** + * What a stream still holds, so a transport can tell "your cursor is too old" from "nothing new". + * + *

Every read goes through a query use case: a transport adapter that held {@link + * LiveEventReplayQueryPort} itself read the event log with no capability declared and no read + * boundary (round9 review 2026-11-12, RR9-05-10). + */ +@UseCaseCapability( + transactionMode = TransactionMode.READ_ONLY, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.READ_REPOSITORY) +public class GetReplayWindowUseCase implements QueryUseCase { + + private final LiveEventReplayQueryPort replay; + private final TransactionPort transactions; + + /** + * @param replay the retained event history + * @param transactions opens the read transaction + */ + public GetReplayWindowUseCase(LiveEventReplayQueryPort replay, TransactionPort transactions) { + this.replay = Objects.requireNonNull(replay, "replay"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + @Override + public ReplayWindow handle(GetReplayWindowQuery query) { + Objects.requireNonNull(query, "query"); + return transactions.inRead(() -> replay.window(query.streamId())); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java similarity index 59% rename from src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java rename to src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java index 741b0fa6..8d4cdf87 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java @@ -5,15 +5,18 @@ import java.util.List; /** * Retained event history, for a client that reconnects and wants to continue. * - *

The history lives wherever the messaging platform keeps it, and this port is the only way a - * transport adapter reaches it. That boundary is the point: a web or websocket module that kept its - * own copy would have its own retention, its own eviction and its own opinion about ordering, and - * the two would diverge silently because both would look plausible. + *

The history lives wherever the messaging platform keeps it, and this port is the only way to + * it — through {@code GetReplayWindowUseCase} and {@code ReplayLiveEventsUseCase}, never from a + * transport directly. A read model that returns projections, so a {@code *QueryPort}: named {@code + * LiveEventReplayPort} until 2026-11-13, when a web bridge that held it read the log with no use + * case in between (round9 review 2026-11-12, RR9-05-10). That boundary is the point: a web or + * websocket module that kept its own copy would have its own retention, its own eviction and its + * own opinion about ordering, and the two would diverge silently because both would look plausible. * *

{@link #window} exists so a caller can distinguish "your cursor is too old" from "there is * nothing new", which {@link #replayAfter} returning an empty list cannot. */ -public interface LiveEventReplayPort { +public interface LiveEventReplayQueryPort { /** What the stream still holds. */ ReplayWindow window(String streamId); diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java new file mode 100644 index 00000000..23b091b4 --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.query.Query; +import java.util.Objects; + +/** + * A reconnecting client's request for what it missed. + * + * @param streamId which stream + * @param afterPosition the last position the client already has; 0 for the beginning + * @param limit the most events to return + */ +public record ReplayLiveEventsQuery(String streamId, long afterPosition, int limit) + implements Query { + + public ReplayLiveEventsQuery { + Objects.requireNonNull(streamId, "streamId"); + if (afterPosition < 0 || limit < 1) { + throw new IllegalArgumentException("afterPosition must be >= 0 and limit >= 1"); + } + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java new file mode 100644 index 00000000..2d72a1f0 --- /dev/null +++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java @@ -0,0 +1,45 @@ +package dev.caskeleton.application.realtime; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.QueryUseCase; +import java.util.List; +import java.util.Objects; + +/** + * The events a reconnecting client missed, a bounded page at a time. + * + * @see GetReplayWindowUseCase + */ +@UseCaseCapability( + transactionMode = TransactionMode.READ_ONLY, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.READ_REPOSITORY) +public class ReplayLiveEventsUseCase + implements QueryUseCase> { + + private final LiveEventReplayQueryPort replay; + private final TransactionPort transactions; + + /** + * @param replay the retained event history + * @param transactions opens the read transaction + */ + public ReplayLiveEventsUseCase(LiveEventReplayQueryPort replay, TransactionPort transactions) { + this.replay = Objects.requireNonNull(replay, "replay"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + /** + * @throws ReplayCursorUnavailableException when the position is older than what is retained + */ + @Override + public List handle(ReplayLiveEventsQuery query) { + Objects.requireNonNull(query, "query"); + return transactions.inRead( + () -> replay.replayAfter(query.streamId(), query.afterPosition(), query.limit())); + } +} diff --git a/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java b/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java index f9704f17..6e230d94 100644 --- a/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java +++ b/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java @@ -7,6 +7,15 @@ import java.util.Objects; * *

The value is intentionally opaque. Adapters may use it for reconciliation, but must not invent * a replacement identity after an uncertain commit. + * + *

The widest of the operation-shaped identities, on purpose: the caller chooses it and it is + * only ever compared, never put in a URL or a key, so any printable ASCII is allowed. The others + * are narrower because of where they travel — a {@link + * dev.caskeleton.application.operation.DurableOperationId} is a URL path segment, a lease's + * operation id and owner token are Base64URL so they fit a header and a Redis key unescaped, and an + * idempotency owner token is the 32 random bytes the store mints, as 64 hex characters. One value + * type for all four would either let a path segment carry {@code /} or refuse caller ids that are + * valid today (round9 review 2026-11-12, RR9-05-11). */ public record OperationId(String value) { diff --git a/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java b/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java deleted file mode 100644 index 23912c34..00000000 --- a/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java +++ /dev/null @@ -1,9 +0,0 @@ -package dev.caskeleton.application.transaction; - -/** Application-owned read consistency vocabulary. */ -public enum ReadConsistency { - STRONG, - READ_YOUR_WRITES, - EVENTUAL, - BOUNDED_STALENESS -} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java index 628ee3ef..69219b0a 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore; +import dev.caskeleton.application.tenant.TenantId; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Clock; import java.time.Duration; @@ -28,7 +29,8 @@ class CancelDurableOperationUseCaseTest { operations.submit(submission("t-1")); // when - useCase.handle(new CancelDurableOperationCommand(ID, "alice", "t-1")); + useCase.handle( + new CancelDurableOperationCommand(ID, "alice", Optional.of(new TenantId("t-1")))); // then assertThat(operations.find(ID, "alice", NOW).orElseThrow().state()) @@ -39,12 +41,13 @@ class CancelDurableOperationUseCaseTest { void cancelOfAFinishedOperationReportsNoChange() { // given operations.submit(submission(null)); - operations.claimNext("worker-a", Duration.ofMinutes(1), NOW); - operations.succeed(ID, "worker-a", "/results/1", NOW); + DurableOperationClaim claim = + operations.claimNext("worker-a", Duration.ofMinutes(1), NOW).orElseThrow(); + operations.succeed(claim, "/results/1", NOW); // when Optional canceled = - useCase.handle(new CancelDurableOperationCommand(ID, "alice", null)); + useCase.handle(new CancelDurableOperationCommand(ID, "alice", Optional.empty())); // then assertThat(canceled).contains(false); @@ -57,7 +60,8 @@ class CancelDurableOperationUseCaseTest { // when Optional canceled = - useCase.handle(new CancelDurableOperationCommand(ID, "alice", "t-2")); + useCase.handle( + new CancelDurableOperationCommand(ID, "alice", Optional.of(new TenantId("t-2")))); // then assertThat(canceled).isEmpty(); @@ -69,7 +73,8 @@ class CancelDurableOperationUseCaseTest { operations.submit(submission(null)); // when - Optional canceled = useCase.handle(new CancelDurableOperationCommand(ID, null, null)); + Optional canceled = + useCase.handle(new CancelDurableOperationCommand(ID, null, Optional.empty())); // then assertThat(canceled).isEmpty(); diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java index 44c66954..7be18cc6 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java @@ -2,6 +2,7 @@ package dev.caskeleton.application.operation; import static org.assertj.core.api.Assertions.assertThat; +import dev.caskeleton.application.tenant.TenantId; import java.time.Instant; import java.util.Optional; import org.junit.jupiter.params.ParameterizedTest; @@ -33,7 +34,7 @@ class DurableOperationAccessPolicyTest { String owner, String ownerTenant, String caller, String callerTenant, boolean expected) { assertThat( DurableOperationAccessPolicy.mayAccess( - operation(owner, ownerTenant), caller, callerTenant)) + operation(owner, ownerTenant), caller, TenantId.optional(callerTenant))) .isEqualTo(expected); } diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java index 292cce01..3cb19a13 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore; +import dev.caskeleton.application.tenant.TenantId; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Clock; import java.time.Duration; @@ -29,7 +30,7 @@ class GetDurableOperationUseCaseTest { // when Optional read = - useCase.handle(new GetDurableOperationQuery(ID, "alice", "t-1")); + useCase.handle(new GetDurableOperationQuery(ID, "alice", Optional.of(new TenantId("t-1")))); // then assertThat(read).contains(submitted); @@ -42,7 +43,7 @@ class GetDurableOperationUseCaseTest { // when Optional read = - useCase.handle(new GetDurableOperationQuery(ID, "alice", "t-2")); + useCase.handle(new GetDurableOperationQuery(ID, "alice", Optional.of(new TenantId("t-2")))); // then assertThat(read).isEmpty(); @@ -54,7 +55,8 @@ class GetDurableOperationUseCaseTest { operations.submit(submission(null)); // when - Optional read = useCase.handle(new GetDurableOperationQuery(ID, null, null)); + Optional read = + useCase.handle(new GetDurableOperationQuery(ID, null, Optional.empty())); // then assertThat(read).isEmpty(); diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..13f6efaf --- /dev/null +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java @@ -0,0 +1,80 @@ +package dev.caskeleton.application.operation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.idempotency.RequestFingerprint; +import dev.caskeleton.application.outbox.ClaimedOutboxEvent; +import dev.caskeleton.application.outbox.OutboxEventStatus; +import dev.caskeleton.application.outbox.OutboxLease; +import dev.caskeleton.application.outbox.testkit.OutboxEvents; +import java.time.Duration; +import java.time.Instant; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** The claims and leases a store builds from its rows refuse a bad value as each record says. */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + DurableOperationSubmission submission = + new DurableOperationSubmission( + new DurableOperationId("op-1"), + "reports.generate", + "alice", + "tenant-a", + new RequestFingerprint("a".repeat(64)), + "{}", + NOW, + Duration.ofHours(1)); + return Stream.of( + Arguments.of( + "owner 없는 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, " ", 1), + "a claim needs an owner"), + Arguments.of( + "fence 0인 claim", + (ThrowingCallable) () -> new DurableOperationClaim(submission, "worker-1", 0), + "fence must be >= 1, was 0"), + Arguments.of( + "owner 없는 lease", + (ThrowingCallable) () -> new OperationLease(" ", NOW, NOW.plusSeconds(30)), + "a lease needs an owner"), + Arguments.of( + "잡는 순간 끝나는 lease", + (ThrowingCallable) () -> new OperationLease("worker-1", NOW, NOW), + "a lease that expires when taken is not a lease"), + Arguments.of( + "owner 없는 outbox lease", + (ThrowingCallable) () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), " ", 1), + "owner must not be blank"), + Arguments.of( + "token 0인 outbox lease", + (ThrowingCallable) + () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), "relay-1", 0), + "token must be >= 1, was 0"), + Arguments.of( + "미룬 claim이 claim보다 많은 outbox 행", + (ThrowingCallable) + () -> + new ClaimedOutboxEvent( + OutboxEvents.event("evt-2", NOW), OutboxEventStatus.IN_FLIGHT, 1, 2), + "deferredAttemptCount must be between 0 and attemptCount (1), was 2")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } +} diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java index ac469e2d..9053cfab 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java @@ -16,7 +16,10 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -34,7 +37,7 @@ class SubmitDurableOperationUseCaseTest { private static final Instant NOW = Instant.parse("2026-08-25T10:00:00Z"); - /** An id the HTTP adapter mints: a canonical lower-case UUIDv7, used as the event id itself. */ + /** An id the factory mints: a canonical lower-case UUIDv7, used as the event id itself. */ private static final String OPERATION_UUID = "01994e11-4d88-7000-8000-0000000000a1"; private static final DurableOperationId ID = new DurableOperationId(OPERATION_UUID); @@ -48,16 +51,25 @@ class SubmitDurableOperationUseCaseTest { private final FailingOutbox outbox = new FailingOutbox(transactions); private final FakeIntegrationEventEncoder encoder = new FakeIntegrationEventEncoder(new DurableOperationSubmittedContract()); + + /** What the id factory mints, in order: one id per submission, kept or not. */ + private final Deque mintable = + new ArrayDeque<>(List.of(ID, new DurableOperationId("op-2"), new DurableOperationId("op-3"))); + private final CommandUseCase useCase = new SubmitDurableOperationUseCase( - operations, outbox, encoder, transactions, Clock.fixed(NOW, ZoneOffset.UTC)); + operations, + outbox, + encoder, + transactions, + mintable::removeFirst, + Clock.fixed(NOW, ZoneOffset.UTC)); - private static OperationCommand command(DurableOperationId id, String hash) { + private static OperationCommand command(String hash) { return new OperationCommand( - id, "reports.generate", "alice", - null, + Optional.empty(), new RequestFingerprint(hash), "{\"range\":\"2026-08\"}", Duration.ofHours(6)); @@ -66,7 +78,7 @@ class SubmitDurableOperationUseCaseTest { @Test @DisplayName("an accepted submission writes both the operation and the outbox event") void submissionWritesBoth() { - OperationAcceptance acceptance = useCase.handle(command(ID, "a".repeat(64))); + OperationAcceptance acceptance = useCase.handle(command("a".repeat(64))); assertThat(acceptance.operationId()).isEqualTo(ID); assertThat(acceptance.alreadyAccepted()).isFalse(); @@ -74,10 +86,20 @@ class SubmitDurableOperationUseCaseTest { assertThat(outbox.appended()).hasSize(1); } + @Test + @DisplayName("operation id는 호출자가 아니라 use case가 id factory로 만든다") + void theOperationIdIsMintedByTheUseCase() { + // when + OperationAcceptance acceptance = useCase.handle(command("d".repeat(64))); + + // then + assertThat(List.of(acceptance.operationId(), mintable.size())).containsExactly(ID, 2); + } + @Test @DisplayName("both writes happen inside one transaction") void bothWritesShareOneTransaction() { - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // Not an implementation detail. Outside one transaction the two writes can commit // independently, and every failure mode this use case exists to prevent comes back. The @@ -92,7 +114,7 @@ class SubmitDurableOperationUseCaseTest { void rollbackLeavesNothing() { outbox.failNext = true; - assertThatThrownBy(() -> useCase.handle(command(ID, "a".repeat(64)))) + assertThatThrownBy(() -> useCase.handle(command("a".repeat(64)))) .isInstanceOf(IllegalStateException.class); assertThat(operations.size()).isZero(); @@ -102,10 +124,9 @@ class SubmitDurableOperationUseCaseTest { @Test @DisplayName("an identical resubmission returns the first operation and publishes nothing new") void resubmissionReusesTheFirstOperation() { - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); - OperationAcceptance repeat = - useCase.handle(command(new DurableOperationId("op-2"), "a".repeat(64))); + OperationAcceptance repeat = useCase.handle(command("a".repeat(64))); // The client is pointed at the work that is actually running. Starting a second operation // would run the same report twice and hand back an id for the copy. @@ -116,12 +137,12 @@ class SubmitDurableOperationUseCaseTest { } @Test - void retryUnderTheSameOperationIdAppendsTheOutboxEventOnlyOnce() { + void aRetriedSubmissionAppendsTheOutboxEventOnlyOnce() { // given - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // when - OperationAcceptance retry = useCase.handle(command(ID, "a".repeat(64))); + OperationAcceptance retry = useCase.handle(command("a".repeat(64))); // then assertThat(List.of(retry.alreadyAccepted(), outbox.appended().size())).containsExactly(true, 1); @@ -130,10 +151,9 @@ class SubmitDurableOperationUseCaseTest { @Test @DisplayName("a different request under the same principal is a different operation") void differentRequestsAreDifferentOperations() { - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); - OperationAcceptance second = - useCase.handle(command(new DurableOperationId("op-2"), "b".repeat(64))); + OperationAcceptance second = useCase.handle(command("b".repeat(64))); assertThat(second.alreadyAccepted()).isFalse(); assertThat(operations.size()).isEqualTo(2); @@ -143,7 +163,7 @@ class SubmitDurableOperationUseCaseTest { @Test void theOutboxEventIsTheSubmittedContractAnchoredToTheOperation() { // given - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // when ValidatedIntegrationEvent event = outbox.appended().get(0); @@ -167,7 +187,7 @@ class SubmitDurableOperationUseCaseTest { @Test void theSubmittedEventNamesTheOperationAndNotTheRequest() { // when - useCase.handle(command(ID, "a".repeat(64))); + useCase.handle(command("a".repeat(64))); // then // The worker reads what to run from the submission it claims; the request never travels. @@ -200,13 +220,13 @@ class SubmitDurableOperationUseCaseTest { } @Test - @DisplayName("UUIDv7이 아닌 operation id로 받은 작업도 outbox에 발행할 수 있는 event로 남는다") - void aSubmissionUnderAClientChosenIdStillAppendsAPublishableEvent() { + @DisplayName("id factory가 UUIDv7이 아닌 id를 만들어도 outbox에 발행할 수 있는 event로 남는다") + void aSubmissionUnderANonUuidV7IdStillAppendsAPublishableEvent() { // given - DurableOperationId clientChosen = new DurableOperationId("~report"); + mintable.addFirst(new DurableOperationId("~report")); // when - useCase.handle(command(clientChosen, "c".repeat(64))); + useCase.handle(command("c".repeat(64))); // then assertThat(outbox.appended()) diff --git a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java index a5a6a3fa..b22ab546 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java @@ -49,7 +49,8 @@ class ClaimedOutboxEventContractTest { // then assertThat(thrown) - .isInstanceOf(IllegalArgumentException.class) + .as("a store builds the claim from its row, so a bad count is the store's fault") + .isInstanceOf(IllegalStateException.class) .hasMessage("attemptCount must be >= 0, was -1"); } } diff --git a/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java index 75963702..b7f98131 100644 --- a/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java +++ b/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java @@ -1,17 +1,17 @@ package dev.caskeleton.application.realtime.testkit; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import java.time.Duration; /** The in-memory history keeps the replay contract the transport tests run over. */ -class InMemoryLiveEventReplayContractTest extends LiveEventReplayContract { +class InMemoryLiveEventReplayContractTest extends LiveEventReplayQueryContract { private static final Duration RETENTION = Duration.ofHours(1); private final InMemoryLiveEventReplay replay = new InMemoryLiveEventReplay(clock(), RETENTION); @Override - protected LiveEventReplayPort replay() { + protected LiveEventReplayQueryPort replay() { return replay; } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java index b7fe8a5c..a913ffb4 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java @@ -4,11 +4,13 @@ import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort; import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent; import dev.caskeleton.application.outbox.OutboxPublishAttempt; import dev.caskeleton.application.outbox.OutboxPublishOutcome; -import java.util.ArrayList; +import java.time.Duration; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; /** * The {@link IntegrationEventPublishPort} fake: records every event it is handed and answers with @@ -17,8 +19,55 @@ import java.util.concurrent.CompletionStage; */ public final class FakeIntegrationEventPublisher implements IntegrationEventPublishPort { - private final List published = new ArrayList<>(); + // A relay thread publishes while the test thread reads. + private final List published = new CopyOnWriteArrayList<>(); private OutboxPublishAttempt answer = OutboxPublishAttempt.confirmed(); + private boolean silent; + private Duration blocksFor = Duration.ZERO; + private RuntimeException fails; + private Optional longestAttempt = Optional.empty(); + + /** + * Never completes the stage of any later publish, and reports {@code longestAttempt} as the + * longest a publish takes: a broker that does not answer, behind a publisher that promised it + * would within that bound. + * + * @param longestAttempt what {@link #longestAttempt()} reports + * @return this fake + */ + public FakeIntegrationEventPublisher silentPast(Duration longestAttempt) { + this.longestAttempt = Optional.of(Objects.requireNonNull(longestAttempt, "longestAttempt")); + this.silent = true; + return this; + } + + /** + * Holds the caller for {@code blocking} inside every later {@code publish}, before the stage is + * handed back: a send that blocks its caller, as a producer waiting on cluster metadata does. + * + * @param blocking how long {@code publish} blocks + * @return this fake + */ + public FakeIntegrationEventPublisher blockingFor(Duration blocking) { + this.blocksFor = Objects.requireNonNull(blocking, "blocking"); + return this; + } + + /** + * Completes every later publish's stage exceptionally with {@code failure}. + * + * @param failure what the stage fails with + * @return this fake + */ + public FakeIntegrationEventPublisher failingWith(RuntimeException failure) { + this.fails = Objects.requireNonNull(failure, "failure"); + return this; + } + + @Override + public Optional longestAttempt() { + return longestAttempt; + } /** * Answers every later publish with {@code outcome}, and for anything but a confirmation the @@ -45,7 +94,17 @@ public final class FakeIntegrationEventPublisher implements IntegrationEventPubl @Override public CompletionStage publish(ValidatedIntegrationEvent event) { published.add(Objects.requireNonNull(event, "event must not be null")); - return CompletableFuture.completedFuture(answer); + if (!blocksFor.isZero()) { + try { + Thread.sleep(blocksFor); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + if (fails != null) { + return CompletableFuture.failedFuture(fails); + } + return silent ? new CompletableFuture<>() : CompletableFuture.completedFuture(answer); } /** Every event handed to this fake, in order. */ diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java index 4be958d2..38f896aa 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.caskeleton.application.idempotency.RequestFingerprint; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -25,6 +26,10 @@ import org.junit.jupiter.api.Test; * *

Every call goes through {@link #inTransaction}, because the real store refuses writes without * a caller transaction and because a resubmission must leave that transaction usable. + * + *

A lease write names the {@link DurableOperationClaim} it holds. The fence cases are why: the + * same worker id claiming an operation again must fence out its own earlier run, which a check on + * the worker id alone lets through. */ public abstract class DurableOperationStoreContract { @@ -33,6 +38,9 @@ public abstract class DurableOperationStoreContract { private static final Duration TTL = Duration.ofDays(1); private static final String HASH_A = "a".repeat(64); private static final String HASH_B = "b".repeat(64); + private static final String HASH_C = "c".repeat(64); + private static final OperationFailure TIMED_OUT = + new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); /** The store under test, empty at the start of each test. */ protected abstract DurableOperationStorePort store(); @@ -182,10 +190,36 @@ public abstract class DurableOperationStoreContract { submit(submission); // when - Optional claimed = claim("worker-a", NOW); + Optional claimed = claim("worker-a", NOW); // then - assertThat(claimed).contains(submission); + assertThat(claimed).contains(new DurableOperationClaim(submission, "worker-a", 1)); + } + + @Test + void everyClaimOfAnOperationGetsAHigherFence() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim first = claim("worker-a", NOW).orElseThrow(); + + // when + DurableOperationClaim second = claim("worker-a", NOW.plus(LEASE).plusSeconds(1)).orElseThrow(); + + // then + assertThat(second.fence()).isGreaterThan(first.fence()); + } + + @Test + void canceledOperationIsNotHandedToAWorker() { + // given + submit(submission("op-1", null, HASH_A)); + cancel("op-1", "alice", NOW.plusSeconds(5)); + + // when + Optional claimed = claim("worker-a", NOW.plusSeconds(6)); + + // then + assertThat(claimed).isEmpty(); } @Test @@ -195,7 +229,7 @@ public abstract class DurableOperationStoreContract { claim("worker-a", NOW); // when - Optional claimed = claim("worker-b", NOW.plusSeconds(30)); + Optional claimed = claim("worker-b", NOW.plusSeconds(30)); // then assertThat(claimed).isEmpty(); @@ -224,11 +258,11 @@ public abstract class DurableOperationStoreContract { void heartbeatFromTheLeaseHolderExtendsTheLease() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant beat = NOW.plusSeconds(60); // when - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, beat)); + inTransaction(() -> store().heartbeat(held, LEASE, beat)); // then assertThat(find("op-1", "alice", beat).orElseThrow().lease()) @@ -239,13 +273,12 @@ public abstract class DurableOperationStoreContract { void heartbeatFromAWorkerThatLostTheLeaseIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when - boolean extended = - inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, later.plusSeconds(1))); + boolean extended = inTransaction(() -> store().heartbeat(stale, LEASE, later.plusSeconds(1))); // then assertThat(extended).isFalse(); @@ -255,12 +288,12 @@ public abstract class DurableOperationStoreContract { void progressFromTheLeaseHolderIsPublished() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationProgressSnapshot progress = new OperationProgressSnapshot(3, Optional.of(10L), Optional.of("extracting")); // when - inTransaction(() -> store().reportProgress(id("op-1"), "worker-a", progress)); + inTransaction(() -> store().reportProgress(held, progress)); // then assertThat(find("op-1", "alice", NOW).orElseThrow().progress()).contains(progress); @@ -270,12 +303,13 @@ public abstract class DurableOperationStoreContract { void progressFromAnotherWorkerIsRefused() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + DurableOperationClaim impostor = + new DurableOperationClaim(held.submission(), "worker-b", held.fence()); // when boolean published = - inTransaction( - () -> store().reportProgress(id("op-1"), "worker-b", OperationProgressSnapshot.of(1))); + inTransaction(() -> store().reportProgress(impostor, OperationProgressSnapshot.of(1))); // then assertThat(published).isFalse(); @@ -285,11 +319,11 @@ public abstract class DurableOperationStoreContract { void successFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); Instant done = NOW.plusSeconds(10); // when - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", done)); + inTransaction(() -> store().succeed(held, "/results/a", done)); // then assertThat(find("op-1", "alice", done).orElseThrow()) @@ -309,28 +343,106 @@ public abstract class DurableOperationStoreContract { void staleWorkerCannotRecordAResultOverItsSuccessor() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow(); Instant later = NOW.plus(LEASE).plusSeconds(1); claim("worker-b", later); // when boolean recorded = - inTransaction( - () -> store().succeed(id("op-1"), "worker-a", "/results/a", later.plusSeconds(1))); + inTransaction(() -> store().succeed(stale, "/results/a", later.plusSeconds(1))); // then assertThat(recorded).isFalse(); } + @Test + void heartbeatOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().heartbeat(earlier, LEASE, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void progressOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().reportProgress(earlier, OperationProgressSnapshot.of(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void successOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void failureOfAnEarlierRunOfTheSameWorkerIsRefused() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + claim("worker-a", later); + + // when + boolean accepted = inTransaction(() -> store().fail(earlier, TIMED_OUT, later.plusSeconds(1))); + + // then + assertThat(accepted).isFalse(); + } + + @Test + void resultOfTheRunThatTookOverSurvivesItsSameWorkerPredecessor() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow(); + Instant later = NOW.plus(LEASE).plusSeconds(1); + DurableOperationClaim current = claim("worker-a", later).orElseThrow(); + inTransaction(() -> store().succeed(current, "/results/current", later.plusSeconds(1))); + + // when + inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(2))); + + // then + assertThat(find("op-1", "alice", later.plusSeconds(2)).orElseThrow().resultReference()) + .contains("/results/current"); + } + @Test void failureFromTheLeaseHolderIsRecorded() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); OperationFailure failure = new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true); // when - inTransaction(() -> store().fail(id("op-1"), "worker-a", failure, NOW.plusSeconds(10))); + inTransaction(() -> store().fail(held, failure, NOW.plusSeconds(10))); // then assertThat(find("op-1", "alice", NOW.plusSeconds(10)).orElseThrow()) @@ -369,8 +481,8 @@ public abstract class DurableOperationStoreContract { void cancelNeverReversesASuccess() { // given submit(submission("op-1", null, HASH_A)); - claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); // when cancel("op-1", "alice", NOW.plusSeconds(20)); @@ -407,19 +519,35 @@ public abstract class DurableOperationStoreContract { } @Test - void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + void reclaimLeavesALiveLeaseAlone() { // given submit(submission("op-1", null, HASH_A)); claim("worker-a", NOW); - inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10))); + + // when + List reclaimed = + inTransaction(() -> store().reclaimExpiredLeases(NOW.plusSeconds(30))); + + // then + assertThat(reclaimed).isEmpty(); + } + + @Test + void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() { + // given + submit(submission("op-1", null, HASH_A)); + DurableOperationClaim held = claim("worker-a", NOW).orElseThrow(); + inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10))); submit(submission("op-2", null, HASH_B)); + submit(submission("op-3", null, HASH_C)); + claim("worker-b", NOW.plusSeconds(20)); // when List expired = inTransaction(() -> store().expireStaleOperations(NOW.plus(TTL).plus(TTL))); // then - assertThat(expired).containsExactly(id("op-2")); + assertThat(expired).containsExactlyInAnyOrder(id("op-2"), id("op-3")); } private DurableOperationSubmitResult submit(DurableOperationSubmission submission) { @@ -430,7 +558,7 @@ public abstract class DurableOperationStoreContract { return inTransaction(() -> store().find(id(operationId), principal, now)); } - private Optional claim(String workerId, Instant now) { + private Optional claim(String workerId, Instant now) { return inTransaction(() -> store().claimNext(workerId, LEASE, now)); } diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java index 019193c3..3c76c99a 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java @@ -1,6 +1,7 @@ package dev.caskeleton.application.operation.testkit; import dev.caskeleton.application.operation.DurableOperation; +import dev.caskeleton.application.operation.DurableOperationClaim; import dev.caskeleton.application.operation.DurableOperationId; import dev.caskeleton.application.operation.DurableOperationState; import dev.caskeleton.application.operation.DurableOperationStorePort; @@ -26,8 +27,10 @@ import java.util.concurrent.ConcurrentHashMap; *

Written to the same rules rather than to whatever makes the tests pass: a submission scope * that already exists returns the existing operation, a lease is checked on every write, and a * terminal operation is never reopened. A fake that was laxer than the adapter would let the use - * case tests certify behaviour the real store refuses. {@link DurableOperationStoreContract} holds - * this fake and the adapter to the same cases. + * case tests certify behaviour the real store refuses. Every claim increments the operation's + * fence, and a write is accepted only from the claim holding the current fence, as the adapter's + * {@code lease_fence} column does. {@link DurableOperationStoreContract} holds this fake and the + * adapter to the same cases. * *

Built over a {@link DirectTransactionPort}, every write made inside one of its transactions is * undone when that transaction rolls back, as the database undoes it; built without one, writes @@ -41,6 +44,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor private final Map byScope = new ConcurrentHashMap<>(); private final Map submissions = new ConcurrentHashMap<>(); + private final Map fences = new ConcurrentHashMap<>(); private final DirectTransactionPort transactions; /** A store whose writes stay as they are made. */ @@ -103,7 +107,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public Optional claimNext( + public Optional claimNext( String workerId, Duration leaseDuration, Instant now) { return byId.values().stream() .filter( @@ -126,19 +130,22 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty(), Optional.of(new OperationLease(workerId, now, now.plus(leaseDuration)))); write(claimed.operationId(), claimed); - return submissions.get(claimed.operationId()); + long previous = fences.getOrDefault(claimed.operationId(), 0L); + fences.put(claimed.operationId(), previous + 1); + onRollback(() -> fences.put(claimed.operationId(), previous)); + return new DurableOperationClaim( + submissions.get(claimed.operationId()), workerId, previous + 1); }); } @Override - public boolean heartbeat( - DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) { - DurableOperation operation = byId.get(operationId); - if (!holdsLease(operation, workerId, now)) { + public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) { + DurableOperation operation = byId.get(claim.operationId()); + if (!holdsLease(operation, claim, now)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -152,14 +159,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean reportProgress( - DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, operation.state(), @@ -173,17 +179,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean succeed( - DurableOperationId operationId, - String workerId, - String resultReference, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.SUCCEEDED, @@ -197,17 +199,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor } @Override - public boolean fail( - DurableOperationId operationId, - String workerId, - OperationFailure failure, - Instant completedAt) { - DurableOperation operation = byId.get(operationId); - if (operation == null || !ownedBy(operation, workerId)) { + public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) { + DurableOperation operation = byId.get(claim.operationId()); + if (!heldBy(operation, claim)) { return false; } write( - operationId, + claim.operationId(), replace( operation, DurableOperationState.FAILED, @@ -316,6 +314,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor byId.clear(); byScope.clear(); submissions.clear(); + fences.clear(); } // Same as the adapter: a poll after the TTL sees EXPIRED even before the sweeper runs. @@ -334,13 +333,15 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor Optional.empty()); } - private static boolean holdsLease(DurableOperation operation, String workerId, Instant now) { - return operation != null && ownedBy(operation, workerId) && !operation.leaseExpiredAt(now); + private boolean holdsLease(DurableOperation operation, DurableOperationClaim claim, Instant now) { + return heldBy(operation, claim) && !operation.leaseExpiredAt(now); } - private static boolean ownedBy(DurableOperation operation, String workerId) { - return operation.state() == DurableOperationState.RUNNING - && operation.lease().map(lease -> lease.workerId().equals(workerId)).orElse(false); + private boolean heldBy(DurableOperation operation, DurableOperationClaim claim) { + return operation != null + && operation.state() == DurableOperationState.RUNNING + && operation.lease().map(lease -> lease.workerId().equals(claim.workerId())).orElse(false) + && fences.getOrDefault(claim.operationId(), 0L) == claim.fence(); } private static DurableOperation replace( diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java index f6dc06d2..462220e4 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java @@ -1,6 +1,6 @@ package dev.caskeleton.application.realtime.testkit; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; import dev.caskeleton.application.realtime.ReplayWindow; import dev.caskeleton.application.realtime.ReplayedEvent; @@ -15,15 +15,15 @@ import java.util.Objects; import java.util.Optional; /** - * In-memory {@link LiveEventReplayPort} with the rules of the JPA history: positions per stream - * from 1 and never reused, an event replayable until {@code appendedAt + retention} on the given - * clock, a cursor older than what is retained refused with {@link - * ReplayCursorUnavailableException}. {@link LiveEventReplayContract} holds it and the JPA store to - * the same cases. + * In-memory {@link LiveEventReplayQueryPort} with the rules of the JPA history: positions per + * stream from 1 and never reused, an event replayable until {@code appendedAt + retention} on the + * given clock, a cursor older than what is retained refused with {@link + * ReplayCursorUnavailableException}. {@link LiveEventReplayQueryContract} holds it and the JPA + * store to the same cases. * *

Single-threaded: one instance per test. */ -public final class InMemoryLiveEventReplay implements LiveEventReplayPort { +public final class InMemoryLiveEventReplay implements LiveEventReplayQueryPort { private record Stored(ReplayedEvent event, Instant retainedUntil) {} diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java similarity index 94% rename from src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java rename to src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java index 4c7e5b1f..81a3c335 100644 --- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java +++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java @@ -3,7 +3,7 @@ package dev.caskeleton.application.realtime.testkit; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; -import dev.caskeleton.application.realtime.LiveEventReplayPort; +import dev.caskeleton.application.realtime.LiveEventReplayQueryPort; import dev.caskeleton.application.realtime.ReplayCursorUnavailableException; import dev.caskeleton.application.realtime.ReplayWindow; import dev.caskeleton.application.realtime.ReplayedEvent; @@ -18,14 +18,14 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; /** - * What every {@link LiveEventReplayPort} promises a reconnecting client, run against the in-memory - * fake and the JPA history on real PostgreSQL. + * What every {@link LiveEventReplayQueryPort} promises a reconnecting client, run against the + * in-memory fake and the JPA history on real PostgreSQL. * *

Writing history is not on the port (a transport reads it and never writes it), so each * implementation supplies its own {@link #append}. Retention is measured on {@link #clock()}, which * the implementation is built with and the contract moves. */ -public abstract class LiveEventReplayContract { +public abstract class LiveEventReplayQueryContract { private final ContractClock clock = new ContractClock(Instant.parse("2026-11-03T00:00:00Z")); @@ -43,7 +43,7 @@ public abstract class LiveEventReplayContract { * * @return the port */ - protected abstract LiveEventReplayPort replay(); + protected abstract LiveEventReplayQueryPort replay(); /** * Appends an event the way the store's writer does. diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java index 1b5e7b0d..fa5ffde3 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java @@ -10,14 +10,36 @@ import java.util.Optional; * Authoritative file metadata port. * *

Every transition is conditional on both the expected version and the expected state, so two - * writers racing on the same record produce exactly one winner and one optimistic conflict. + * writers racing on the same record produce exactly one winner and one optimistic conflict. How + * each write fails is part of the contract, because the use cases decide from the failure whether + * to retry, report a conflict or treat the file as gone. A record that does not exist fails as a + * lost precondition, like one that moved on. */ public interface FileMetadataPort { + /** + * Records a new file in {@code CREATED}. + * + * @return the stored record, version 0 + * @throws RuntimeException the store's own refusal when a record already exists under {@code + * draft.fileId()}; file ids are server-minted, so that is a defect, not a race to recover + * from + */ FileRecord insert(FileRecordDraft draft); Optional find(FileId fileId); + /** + * Moves a record from {@code expectedState} at {@code expectedVersion} to {@code targetState}. + * + * @return the record as stored after the move + * @throws IllegalStateException when {@code expectedState} may never move to {@code targetState} + * — a caller defect, refused before anything is written + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the record is no longer at that version and state + * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code + * FILE_NOT_FOUND} when the record disappeared during the move + */ FileRecord transition( FileId fileId, long expectedVersion, @@ -25,6 +47,16 @@ public interface FileMetadataPort { FileState targetState, FileRecordMutation mutation); + /** + * Moves a record at {@code expectedVersion} in a deletable state to {@code DELETING}. + * + * @return the record as stored after the move + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the record is no longer at that version or not in a deletable + * state + * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code + * FILE_NOT_FOUND} when the record disappeared during the move + */ FileRecord markDeleting(FileId fileId, long expectedVersion); /** @@ -33,6 +65,12 @@ public interface FileMetadataPort { *

A namespace is a metadata grouping, so this changes one column and never touches the * immutable physical object. Copying gigabytes to express an ownership change would also break * every strong validator already handed to clients. + * + * @return the record as stored after the move + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the record is no longer {@code READY} at that version + * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code + * FILE_NOT_FOUND} when the record disappeared during the move */ FileRecord relocate(FileId fileId, long expectedVersion, StorageNamespace targetNamespace); diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java index 9116e18f..42062eeb 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java @@ -18,6 +18,14 @@ public interface UploadSessionPort { Optional find(UploadId uploadId); + /** + * Takes the writer lease on an upload that holds none, or whose lease has lapsed. + * + * @return the new lease, with a fresh token and the session version after the claim + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when another owner holds an unexpired lease, the session is no + * longer at {@code expectedVersion}, or the upload has expired (then not retryable) + */ WriterLease acquireLease( UploadId uploadId, String owner, Instant now, Duration leaseDuration, long expectedVersion); @@ -34,6 +42,15 @@ public interface UploadSessionPort { */ WriterLease renewLease(WriterLease lease, Instant now, Duration leaseDuration); + /** + * Advances the committed offset from {@code expectedOffset} to {@code committedOffset} for the + * holder of {@code lease}. + * + * @return the session as stored after the commit + * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code + * CONCURRENT_MODIFICATION} when the lease token no longer holds the session or the stored + * offset is not {@code expectedOffset}; nothing is committed + */ UploadSession commitOffset( UploadId uploadId, WriterLease lease, long expectedOffset, long committedOffset); diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java index 4577fec0..ad90e67d 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java @@ -44,6 +44,16 @@ public interface CleanupQueuePort { /** Records a failed attempt and when to try again, if {@code item}'s claim is still current. */ void markFailed(CleanupItem item, String reasonCode, Instant nextAttemptAt); + /** + * Gives a claimed item back unattempted, if {@code item}'s claim is still current. + * + *

For an item the batch claimed and then did not start — its byte budget or its claim's lease + * ran out first. The claim ends and the item is claimable again at {@code nextAttemptAt}; the + * attempt counter does not move, because nothing was tried. Recorded as a failure instead, every + * such release spent an attempt, and a busy queue's tail was abandoned without ever being tried. + */ + void release(CleanupItem item, Instant nextAttemptAt); + /** * Parks an item for good, if {@code item}'s claim is still current. * diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java index d28a7d40..71033678 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java @@ -49,6 +49,20 @@ public final class DefaultCleanupService implements CleanupService { */ public static final Duration CLAIM_LEASE = Duration.ofMinutes(10); + /** + * No claimed item is started once less than this is left of the batch's claim lease. + * + *

A batch claims up to {@code maxItems} under one lease and works them one after another, so + * the last item starts long after the claim. Past the lease, the reaper may already have handed + * that item to another worker, and both delete it and settle its quota. The outbox relay checks + * its batch against its lease when it is assembled and the notification scheduler claims only + * what it can start; cleanup has no bound on one delete to check against, so it stops starting + * items instead, and releases the rest for the next batch the way the byte budget does (round9 + * review 2026-11-12, RR9-01-23). Two minutes is the time one delete against slow storage is + * allowed to finish inside its lease. + */ + static final Duration LAST_START_BEFORE_LEASE_ENDS = Duration.ofMinutes(2); + private final CleanupQueuePort queue; private final CleanupContentPort contentGateway; private final FileMetadataPort metadataStore; @@ -95,6 +109,7 @@ public final class DefaultCleanupService implements CleanupService { // claiming means the recovered items are eligible in this same pass. transactions.inWrite(() -> queue.reclaimExpiredClaims(now, maxItems)); List due = transactions.inWrite(() -> queue.claimDue(now, maxItems, CLAIM_LEASE)); + Instant lastStart = now.plus(CLAIM_LEASE).minus(LAST_START_BEFORE_LEASE_ENDS); int deleted = 0; int skippedActiveLease = 0; @@ -113,10 +128,16 @@ public final class DefaultCleanupService implements CleanupService { failed++; continue; } + if (!clock.instant().isBefore(lastStart)) { + // The claim's lease is nearly spent; an item started now could still be running when the + // reaper hands it to another worker. Given back untried, like the byte budget's. + release(item, now); + continue; + } if (reclaimed >= maxBytes) { - // The byte budget is spent; the remaining claims are released for the next batch rather - // than being marked done, so nothing is silently dropped. - markFailed(item, "BATCH_BYTE_BUDGET_EXHAUSTED", now); + // The byte budget is spent; the remaining claims are given back untried rather than + // being marked done, so nothing is silently dropped and no attempt is spent on them. + release(item, now); continue; } Outcome outcome = process(item, now); @@ -330,6 +351,11 @@ public final class DefaultCleanupService implements CleanupService { }); } + /** Gives a claimed item back untried, in a write transaction of its own. */ + private void release(CleanupItem item, Instant nextAttemptAt) { + transactions.inWrite(() -> queue.release(item, nextAttemptAt)); + } + /** * Reschedules an item, or parks it once this failure spends its retry budget. * diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java new file mode 100644 index 00000000..4b4f8612 --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java @@ -0,0 +1,18 @@ +package dev.caskeleton.application.fileserver.cleanup; + +import dev.caskeleton.application.command.Command; + +/** + * The bounds of one unattended cleanup batch. + * + * @param maxItems most queued items the batch claims + * @param maxBytes most bytes the batch reclaims before it stops starting items + */ +public record RunScheduledCleanupCommand(int maxItems, long maxBytes) implements Command { + + public RunScheduledCleanupCommand { + if (maxItems < 1 || maxBytes < 1) { + throw new IllegalStateException("cleanup batch bounds must be positive"); + } + } +} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java new file mode 100644 index 00000000..9eed3d9e --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java @@ -0,0 +1,48 @@ +package dev.caskeleton.application.fileserver.cleanup; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.Objects; + +/** + * One unattended cleanup batch: reclaim what the queue holds, within the batch bounds. + * + *

The same work an operator starts with {@code CleanupUploadsUseCase}, which declares {@code + * file:admin}. A write both a user and the system start is two use cases: this one is the + * scheduler's, on a thread with no {@code Authentication}, so it carries no permission and no user + * transport may reach it. + * + *

{@code RepositoryAccess.NONE}: the batch is {@link CleanupService}'s, which opens a boundary + * per claimed item. One around the whole batch would hold a connection across every deletion. + * + *

Not a bean. The Fileserver composition's cleanup worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the Fileserver cleanup worker runs it on the scheduler thread, which has no" + + " Authentication; an operator's run is CleanupUploadsUseCase with file:admin") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // The cleanup service deletes staged and orphaned objects in the object store. + externalOutboundAllowed = true) +public class RunScheduledCleanupUseCase + implements CommandUseCase { + + private final CleanupService cleanup; + + public RunScheduledCleanupUseCase(CleanupService cleanup) { + this.cleanup = Objects.requireNonNull(cleanup, "cleanup"); + } + + @Override + public CleanupBatchResult handle(RunScheduledCleanupCommand command) { + Objects.requireNonNull(command, "command"); + return cleanup.runBatch(command.maxItems(), command.maxBytes()); + } +} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java deleted file mode 100644 index ab6fd5df..00000000 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java +++ /dev/null @@ -1,28 +0,0 @@ -package dev.caskeleton.application.fileserver.observability; - -/** - * The exact span names the design fixes. - * - *

Naming them once means a dashboard or an alert built on one of these keeps working when the - * code that emits it moves. A span name invented at a call site would silently break both. - */ -public final class FileserverSpans { - - public static final String UPLOAD_CREATE = "upload.create"; - public static final String UPLOAD_APPEND = "upload.append"; - public static final String UPLOAD_FINALIZE = "upload.finalize"; - public static final String VERIFY_DIGEST = "verify.digest"; - public static final String VERIFY_MEDIA_TYPE = "verify.media-type"; - public static final String VERIFY_MALWARE = "verify.malware"; - public static final String STORAGE_PUBLISH = "storage.publish"; - public static final String STORAGE_STAT = "storage.stat"; - public static final String METADATA_TRANSITION = "metadata.transition"; - public static final String DOWNLOAD_AUTHORIZE = "download.authorize"; - public static final String DOWNLOAD_RESOLVE_RANGE = "download.resolve-range"; - public static final String DOWNLOAD_OPEN = "download.open"; - public static final String DOWNLOAD_DELEGATE = "download.delegate"; - public static final String CLEANUP_ITEM = "cleanup.item"; - public static final String RECONCILE_FILE = "reconcile.file"; - - private FileserverSpans() {} -} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java index a1f15e75..5e2f0fb3 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in; import dev.caskeleton.application.command.Command; import dev.caskeleton.application.fileserver.api.security.RequestContext; import java.util.Objects; +import java.util.Optional; /** * Run one cleanup batch. @@ -14,7 +15,28 @@ import java.util.Objects; public record CleanupUploadsCommand(int maxItems, long maxBytes, RequestContext context) implements Command { + /** How many queued items a request that names no bound processes. */ + public static final int DEFAULT_MAX_ITEMS = 100; + + /** The byte budget of a request that names none: 1 GiB. */ + public static final long DEFAULT_MAX_BYTES = 1L << 30; + public CleanupUploadsCommand { Objects.requireNonNull(context, "context"); } + + /** + * The batch a request asks for, with absent bounds at the use case's defaults rather than at + * whatever a transport chose (round9 review 2026-11-12, RR9-05-19). + * + * @param maxItems absent means {@link #DEFAULT_MAX_ITEMS} + * @param maxBytes absent means {@link #DEFAULT_MAX_BYTES} + * @param context the operator + * @return the command + */ + public static CleanupUploadsCommand requested( + Optional maxItems, Optional maxBytes, RequestContext context) { + return new CleanupUploadsCommand( + maxItems.orElse(DEFAULT_MAX_ITEMS), maxBytes.orElse(DEFAULT_MAX_BYTES), context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java index 3a8960e4..f3176cc0 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in; import dev.caskeleton.application.fileserver.api.security.RequestContext; import dev.caskeleton.application.query.Query; import java.util.Objects; +import java.util.Optional; /** * List incomplete uploads. @@ -12,7 +13,16 @@ import java.util.Objects; */ public record ListIncompleteUploadsQuery(int limit, RequestContext context) implements Query { + /** How many a request that names no limit lists. */ + public static final int DEFAULT_LIMIT = 100; + public ListIncompleteUploadsQuery { Objects.requireNonNull(context, "context"); } + + /** The query a request asks for; an absent limit is {@link #DEFAULT_LIMIT}. */ + public static ListIncompleteUploadsQuery requested( + Optional limit, RequestContext context) { + return new ListIncompleteUploadsQuery(limit.orElse(DEFAULT_LIMIT), context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java index 6456d0a9..a6d71fae 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in; import dev.caskeleton.application.fileserver.api.security.RequestContext; import dev.caskeleton.application.query.Query; import java.util.Objects; +import java.util.Optional; /** * List orphaned objects. @@ -12,7 +13,15 @@ import java.util.Objects; */ public record ListOrphansQuery(int limit, RequestContext context) implements Query { + /** How many a request that names no limit lists. */ + public static final int DEFAULT_LIMIT = 100; + public ListOrphansQuery { Objects.requireNonNull(context, "context"); } + + /** The query a request asks for; an absent limit is {@link #DEFAULT_LIMIT}. */ + public static ListOrphansQuery requested(Optional limit, RequestContext context) { + return new ListOrphansQuery(limit.orElse(DEFAULT_LIMIT), context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java index 52d826fb..9982be14 100644 --- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java @@ -4,6 +4,7 @@ import dev.caskeleton.application.command.Command; import dev.caskeleton.application.fileserver.api.security.RequestContext; import java.util.List; import java.util.Objects; +import java.util.Optional; /** * Request to reconcile orphaned physical objects. @@ -45,9 +46,55 @@ public record OrphanReconcileCommand( expectedFingerprints = List.copyOf(expectedFingerprints); } + /** How many candidates a request that names no limit considers. */ + public static final int DEFAULT_LIMIT = 100; + + /** The byte budget of an apply that names none: 1 GiB. */ + public static final long DEFAULT_APPLY_MAX_BYTES = 1L << 30; + + /** The audit reason of an apply that names none. */ + public static final String DEFAULT_APPLY_REASON = "ORPHAN_APPLY"; + /** Bounded dry run, the default and the only shape a caller can reach without opting in. */ public static OrphanReconcileCommand dryRun(int limit, RequestContext context) { return new OrphanReconcileCommand( true, limit, Long.MAX_VALUE, List.of(), "ORPHAN_SCAN", context); } + + /** + * The command a request asks for, with every absent field at its safe value: a dry run, the + * default page, no fingerprints. + * + *

The defaults are the use case's, not the transport's, so every transport that offers the + * operation reaches the same ones; they used to be written into the HTTP controller (round9 + * review 2026-11-12, RR9-05-19). Nothing here can be omitted into a destructive default: an apply + * still has to name its fingerprints. + * + * @param dryRun list only; absent means true + * @param limit the most candidates; absent means {@link #DEFAULT_LIMIT} + * @param maxBytes an apply's byte budget; absent means {@link #DEFAULT_APPLY_MAX_BYTES} + * @param expectedFingerprints the objects an apply may remove; absent means none + * @param reasonCode an apply's audit reason; absent means {@link #DEFAULT_APPLY_REASON} + * @param context the operator + * @return the command + */ + public static OrphanReconcileCommand requested( + Optional dryRun, + Optional limit, + Optional maxBytes, + Optional> expectedFingerprints, + Optional reasonCode, + RequestContext context) { + int page = limit.orElse(DEFAULT_LIMIT); + if (dryRun.orElse(true)) { + return dryRun(page, context); + } + return new OrphanReconcileCommand( + false, + page, + maxBytes.orElse(DEFAULT_APPLY_MAX_BYTES), + expectedFingerprints.orElse(List.of()), + reasonCode.orElse(DEFAULT_APPLY_REASON), + context); + } } diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java new file mode 100644 index 00000000..8d611047 --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java @@ -0,0 +1,17 @@ +package dev.caskeleton.application.fileserver.recovery; + +import dev.caskeleton.application.command.Command; + +/** + * The bound of one unattended reconciliation pass. + * + * @param limit most open recovery items the pass settles + */ +public record ReconcileRecoverableFilesCommand(int limit) implements Command { + + public ReconcileRecoverableFilesCommand { + if (limit < 1) { + throw new IllegalStateException("reconciliation limit must be positive"); + } + } +} diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java new file mode 100644 index 00000000..a0347a2e --- /dev/null +++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java @@ -0,0 +1,48 @@ +package dev.caskeleton.application.fileserver.recovery; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.List; +import java.util.Objects; + +/** + * One unattended reconciliation pass: settle the files an ambiguous publish or an unconfirmed + * commit left open, a bounded number at a time. + * + *

The scheduler's, on a thread with no {@code Authentication}, so it carries no permission and + * no user transport may reach it. + * + *

{@code RepositoryAccess.NONE}: each file is {@link FileReconciliationService}'s to settle, in + * a boundary of its own around each write. + * + *

Not a bean. The Fileserver composition's cleanup worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the Fileserver cleanup worker runs it on the scheduler thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // The reconciliation service reads the object store to decide each file. + externalOutboundAllowed = true) +public class ReconcileRecoverableFilesUseCase + implements CommandUseCase> { + + private final FileReconciliationService reconciliation; + + public ReconcileRecoverableFilesUseCase(FileReconciliationService reconciliation) { + this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation"); + } + + @Override + public List handle(ReconcileRecoverableFilesCommand command) { + Objects.requireNonNull(command, "command"); + return reconciliation.reconcileRecoverable(command.limit()); + } +} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java index 05357b59..0bd7a4c9 100644 Binary files a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java and b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java differ diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java index dd0bd88d..ead9afb2 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java @@ -26,7 +26,10 @@ import dev.caskeleton.application.fileserver.testkit.ReadyFileFixture; import dev.caskeleton.application.fileserver.testkit.RecordingQuotaReclaimGateway; import java.time.Clock; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.List; import java.util.OptionalLong; import java.util.UUID; import org.junit.jupiter.api.DisplayName; @@ -321,7 +324,91 @@ class CleanupServiceTest { CleanupBatchResult result = cleanup.runBatch(100, 1); assertThat(result.deleted()).isEqualTo(1); - assertThat(queue.failed().values()).contains("BATCH_BYTE_BUDGET_EXHAUSTED"); + assertThat(List.of(queue.released().size(), queue.failed().size())) + .as("the deferred item is given back untried; recorded as failed it spent an attempt") + .containsExactly(1, 0); + } + + @Test + void itemsTheClaimLeaseNoLongerCoversAreReleasedNotStarted() { + // given + // Each staging delete takes just over half the time an item may still start in (the lease less + // its last two minutes): the first two items run, the third is handed back to the queue. + MovableClock slowStorage = new MovableClock(FileserverFixtures.NOW); + Duration perDelete = + DefaultCleanupService.CLAIM_LEASE + .minus(DefaultCleanupService.LAST_START_BEFORE_LEASE_ENDS) + .dividedBy(2) + .plusSeconds(1); + content.afterEachStagingDiscard(() -> slowStorage.advance(perDelete)); + CleanupService timed = + new DefaultCleanupService( + queue, + content, + metadata, + sessions, + quota, + Duration.ofMinutes(5), + FileserverMetricsPort.noop(), + transactions, + slowStorage); + for (int item = 0; item < 3; item++) { + cancelledStagingItem(); + } + + // when + CleanupBatchResult result = timed.runBatch(100, 1L << 30); + + // then + assertThat(List.of(result.deleted(), queue.released().size(), queue.failed().size())) + .as( + "the third item comes back untried rather than failed; that a release leaves the" + + " attempt counter where it was is the queue contract's") + .containsExactly(2, 1, 0); + } + + private void cancelledStagingItem() { + UploadId uploadId = UploadId.of(UUID.randomUUID()); + FileId fileId = FileId.of(UUID.randomUUID()); + sessions.create( + new UploadSessionDraft( + uploadId, + fileId, + UploadMode.SINGLE_SHOT, + OptionalLong.of(SIZE), + FileserverFixtures.NOW.plusSeconds(3600))); + sessions.terminate(uploadId); + content.storeStaging(uploadId); + queue.enqueue(CleanupRequest.forStaging(CleanupType.CANCELLED_STAGING, fileId, uploadId)); + } + + /** A clock the slow-storage gateway moves forward. */ + private static final class MovableClock extends Clock { + + private Instant now; + + private MovableClock(Instant start) { + this.now = start; + } + + private void advance(Duration duration) { + now = now.plus(duration); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return now; + } } @Test diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java new file mode 100644 index 00000000..ee95821c --- /dev/null +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java @@ -0,0 +1,60 @@ +package dev.caskeleton.application.fileserver.cleanup; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.security.SystemInvoked; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's cleanup run is a use case of its own, apart from the operator's (round9 review + * 2026-11-12, RR9-05-06). + */ +class RunScheduledCleanupUseCaseTest { + + @Test + @DisplayName("예약 cleanup은 command의 항목·byte 상한으로 batch 하나를 돌린다") + void aScheduledRunReclaimsOneBatchWithinTheCommandsBounds() { + // given + List> batches = new ArrayList<>(); + RunScheduledCleanupUseCase useCase = + new RunScheduledCleanupUseCase( + (maxItems, maxBytes) -> { + batches.add(List.of((long) maxItems, maxBytes)); + return CleanupBatchResult.empty(); + }); + + // when + useCase.handle(new RunScheduledCleanupCommand(25, 4_096)); + + // then + assertThat(batches).containsExactly(List.of(25L, 4_096L)); + } + + @Test + @DisplayName("시스템만 부르는 쓰기라 권한을 선언하지 않고 @SystemInvoked로 그 이유를 적는다") + void itIsMarkedAsTheSystemsAndDeclaresNoPermission() { + assertThat( + List.of( + RunScheduledCleanupUseCase.class.isAnnotationPresent(SystemInvoked.class), + RunScheduledCleanupUseCase.class.isAnnotationPresent(RequiresPermission.class))) + .containsExactly(true, false); + } + + @Test + @DisplayName("상한이 0인 batch는 만들 수 없다") + void aBatchWithNoRoomIsRefused() { + // when + Throwable refused = catchThrowable(() -> new RunScheduledCleanupCommand(0, 4_096)); + + // then + assertThat(refused) + .as("the worker's settings build the batch, so a bad bound is the assembly's fault") + .isInstanceOf(IllegalStateException.class) + .hasMessage("cleanup batch bounds must be positive"); + } +} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java index affc9182..6a240a63 100644 --- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java @@ -76,30 +76,6 @@ class FileserverObservabilityTest { } } - @Test - void everyDesignSpanNameIsDeclaredExactlyOnce() { - List spans = - List.of( - FileserverSpans.UPLOAD_CREATE, - FileserverSpans.UPLOAD_APPEND, - FileserverSpans.UPLOAD_FINALIZE, - FileserverSpans.VERIFY_DIGEST, - FileserverSpans.VERIFY_MEDIA_TYPE, - FileserverSpans.VERIFY_MALWARE, - FileserverSpans.STORAGE_PUBLISH, - FileserverSpans.STORAGE_STAT, - FileserverSpans.METADATA_TRANSITION, - FileserverSpans.DOWNLOAD_AUTHORIZE, - FileserverSpans.DOWNLOAD_RESOLVE_RANGE, - FileserverSpans.DOWNLOAD_OPEN, - FileserverSpans.DOWNLOAD_DELEGATE, - FileserverSpans.CLEANUP_ITEM, - FileserverSpans.RECONCILE_FILE); - - assertThat(spans).doesNotHaveDuplicates().hasSize(15); - assertThat(spans).allSatisfy(name -> assertThat(name).matches("[a-z]+\\.[a-z-]+")); - } - @Test void anAuditEventCarriesFingerprintsRatherThanNames() { SafeFileFingerprint fingerprint = new SafeFileFingerprint(KEY); diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java new file mode 100644 index 00000000..f90e8e26 --- /dev/null +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java @@ -0,0 +1,116 @@ +package dev.caskeleton.application.fileserver.port.in; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.security.FileAccessSubject; +import dev.caskeleton.application.fileserver.api.security.RequestContext; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * What an operator's admin request means when it leaves a field out. + * + *

The defaults are the use case's, not the transport's: the controller used to fill them in + * itself, so a second transport would have had to copy them (round9 review 2026-11-12, RR9-05-19). + */ +class AdminRequestDefaultsTest { + + private static final RequestContext OPERATOR = + new RequestContext(FileAccessSubject.of("operator-1", Set.of("admin")), "trace-1", "node-a"); + + @Test + @DisplayName("값을 하나도 주지 않은 orphan 정리 요청은 dry run이고 아무것도 지우지 않는다") + void anOrphanRequestWithNothingSetIsADryRun() { + // when + OrphanReconcileCommand command = + OrphanReconcileCommand.requested( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + OPERATOR); + + // then + assertThat(command) + .extracting( + OrphanReconcileCommand::dryRun, + OrphanReconcileCommand::limit, + OrphanReconcileCommand::expectedFingerprints) + .containsExactly(true, OrphanReconcileCommand.DEFAULT_LIMIT, List.of()); + } + + @Test + @DisplayName("apply 요청은 byte 상한 1GiB와 기본 이유 코드를 받는다") + void anApplyRequestGetsTheDefaultByteCeilingAndReason() { + // when + OrphanReconcileCommand command = + OrphanReconcileCommand.requested( + Optional.of(false), + Optional.empty(), + Optional.empty(), + Optional.of(List.of("f".repeat(64))), + Optional.empty(), + OPERATOR); + + // then + assertThat(command) + .extracting( + OrphanReconcileCommand::dryRun, + OrphanReconcileCommand::maxBytes, + OrphanReconcileCommand::reasonCode) + .containsExactly(false, 1L << 30, "ORPHAN_APPLY"); + } + + @Test + @DisplayName("지울 fingerprint를 대지 않은 apply 요청은 거절한다") + void anApplyRequestThatNamesNoFingerprintIsRefused() { + // when + Throwable refused = + catchThrowable( + () -> + OrphanReconcileCommand.requested( + Optional.of(false), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + OPERATOR)); + + // then + assertThat(refused) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("an apply must name the fingerprints it intends to remove"); + } + + @Test + @DisplayName("cleanup 요청은 항목 100개·1GiB 상한을 기본으로 받는다") + void aCleanupRequestGetsTheDefaultBatchBounds() { + // when + CleanupUploadsCommand command = + CleanupUploadsCommand.requested(Optional.empty(), Optional.empty(), OPERATOR); + + // then + assertThat(command) + .extracting(CleanupUploadsCommand::maxItems, CleanupUploadsCommand::maxBytes) + .containsExactly(100, 1L << 30); + } + + @Test + @DisplayName("목록 조회는 쪽 크기 100을 기본으로 받고, 준 값은 그대로 쓴다") + void listingsDefaultToAPageOfOneHundredAndKeepAGivenLimit() { + // when + List limits = + List.of( + ListOrphansQuery.requested(Optional.empty(), OPERATOR).limit(), + ListIncompleteUploadsQuery.requested(Optional.empty(), OPERATOR).limit(), + ListOrphansQuery.requested(Optional.of(7), OPERATOR).limit()); + + // then + assertThat(limits).containsExactly(100, 100, 7); + } +} diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java new file mode 100644 index 00000000..a2e1e2c4 --- /dev/null +++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java @@ -0,0 +1,68 @@ +package dev.caskeleton.application.fileserver.recovery; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.fileserver.api.FileId; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.security.SystemInvoked; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's reconciliation pass is a {@code @SystemInvoked} use case (round9 review + * 2026-11-12, RR9-05-06). + */ +class ReconcileRecoverableFilesUseCaseTest { + + @Test + @DisplayName("예약 reconcile은 command의 상한만큼 열린 복구 항목을 정리한다") + void aScheduledPassSettlesAsManyOpenItemsAsTheCommandAllows() { + // given + List limits = new ArrayList<>(); + ReconcileRecoverableFilesUseCase useCase = + new ReconcileRecoverableFilesUseCase( + new FileReconciliationService() { + @Override + public ReconciliationResult reconcile(FileId fileId) { + throw new UnsupportedOperationException("the pass reconciles in batches only"); + } + + @Override + public List reconcileRecoverable(int limit) { + limits.add(limit); + return List.of(); + } + }); + + // when + useCase.handle(new ReconcileRecoverableFilesCommand(40)); + + // then + assertThat(limits).containsExactly(40); + } + + @Test + void itIsMarkedAsTheSystemsAndDeclaresNoPermission() { + assertThat( + List.of( + ReconcileRecoverableFilesUseCase.class.isAnnotationPresent(SystemInvoked.class), + ReconcileRecoverableFilesUseCase.class.isAnnotationPresent( + RequiresPermission.class))) + .containsExactly(true, false); + } + + @Test + @DisplayName("상한이 0인 pass는 만들 수 없고, 조립의 잘못으로 실패한다") + void aPassWithNoRoomIsRefused() { + // when + Throwable refused = catchThrowable(() -> new ReconcileRecoverableFilesCommand(0)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessage("reconciliation limit must be positive"); + } +} diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java index 0db36011..b9cc169b 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java @@ -119,6 +119,40 @@ public abstract class CleanupQueuePortContract { }); } + @Test + void aReleasedItemIsClaimableAgainWithItsAttemptUnchanged() { + // given + enqueue(contentCleanup()); + CleanupItem item = claim(due(), 10).get(0); + settle(() -> queue().release(item, due())); + + // when + List again = claim(due(), 10); + + // then + assertThat(again) + .singleElement() + .extracting(CleanupItem::cleanupId, CleanupItem::attempt) + .containsExactly(item.cleanupId(), 0); + } + + @Test + void aReleaseUnderAReclaimedTokenChangesNothing() { + // given + enqueue(contentCleanup()); + CleanupItem stale = claim(due(), 10).get(0); + settle(() -> queue().release(stale, due())); + var unused = claim(due(), 10); + + // when + settle(() -> queue().release(stale, due())); + + // then + assertThat(claim(due(), 10)) + .as("the stale release would hand the current claim's item to a second worker") + .isEmpty(); + } + @Test @DisplayName("완료된 항목은 다시 claim되지 않는다") void aDoneItemIsNeverClaimedAgain() { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java index 8a485a3c..3b842068 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java @@ -7,6 +7,7 @@ import dev.caskeleton.application.fileserver.api.content.DeleteResult; import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; /** Tracks which physical objects still exist so a test can prove nothing live was removed. */ @@ -16,6 +17,7 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { private final Set stagingObjects = new LinkedHashSet<>(); private final Set quarantined = new LinkedHashSet<>(); private final List preconditions = new java.util.ArrayList<>(); + private Runnable afterStagingDiscard = () -> {}; public void store(ContentKey key) { objects.add(key); @@ -46,9 +48,20 @@ public final class FakeCleanupContentGateway implements CleanupContentPort { return DeleteResult.removed(precondition.expectedSize().orElse(0)); } + /** + * Runs {@code hook} after every staging discard, for a test that needs the storage to be slow — + * advancing its clock — or to fail mid-batch. + * + * @param hook what a discard does after removing the staging object + */ + public void afterEachStagingDiscard(Runnable hook) { + afterStagingDiscard = Objects.requireNonNull(hook, "hook"); + } + @Override public void discardStaging(UploadId uploadId) { stagingObjects.remove(uploadId); + afterStagingDiscard.run(); } public void quarantine(ContentKey key) { diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java index 50edd600..0dc81d84 100644 --- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java +++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java @@ -35,6 +35,7 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { private final Map rows = new LinkedHashMap<>(); private final List done = new ArrayList<>(); private final Map failed = new LinkedHashMap<>(); + private final List released = new ArrayList<>(); private final Map abandoned = new LinkedHashMap<>(); private final List reclaimSweeps = new ArrayList<>(); private final List claimSweeps = new ArrayList<>(); @@ -108,6 +109,15 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { } } + @Override + public void release(CleanupItem item, Instant nextAttemptAt) { + Row row = rows.get(item.cleanupId()); + if (row != null && item.claimToken().equals(row.claimToken)) { + row.release(nextAttemptAt); + released.add(item); + } + } + @Override public void markAbandoned(CleanupItem item, String reasonCode) { if (settle(item, Status.ABANDONED, clock.instant())) { @@ -151,6 +161,11 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { return Map.copyOf(failed); } + /** Items the caller gave back untried, in order. */ + public List released() { + return List.copyOf(released); + } + /** Items the caller parked for good, with the reason given. */ public Map abandoned() { return Map.copyOf(abandoned); @@ -180,6 +195,13 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort { this.nextAttemptAt = nextAttemptAt; } + private void release(Instant nextAttemptAt) { + this.status = attempt == 0 ? Status.PENDING : Status.FAILED; + this.nextAttemptAt = nextAttemptAt; + this.claimToken = null; + this.leaseUntil = null; + } + private void settle(Status status, Instant nextAttemptAt) { this.status = status; this.attempt++; diff --git a/src/application/notification/AGENTS.md b/src/application/notification/AGENTS.md index 9c163f4d..cb807a1a 100644 --- a/src/application/notification/AGENTS.md +++ b/src/application/notification/AGENTS.md @@ -12,7 +12,7 @@ ## 책임 - 알림 플랫폼의 업무 흐름: 제출·예약·취소·조회, template 게시, provider callback 수집, 발송 판정과 재시도 정책. -- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`BatchNotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. +- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. - testFixtures(흐름 패키지의 `testkit`): 저장 포트의 in-memory fake와 공유 계약(`*Contract`), provider adapter 계약(`ProviderAdapterContract`). 모든 포트에 있지는 않다. adapter의 시험이 같은 계약을 상속한다. ## 규칙 @@ -20,6 +20,8 @@ - framework 의존이 없다(`ca.java-library`). Spring·JPA·HTTP 타입을 쓰지 않는다. 허용 의존은 registry의 `application-core`·`shared-contract`뿐이다. - 전송 DTO(HTTP 요청·응답 record)를 여기에 두지 않는다. 전송 모양은 `adapter:inbound:web-notification`의 것이다. - 발송 판정(마케팅 억제, category 필수)은 여기서 한다. 전송이나 설정에서 기본값을 골라 넣지 않는다. +- 시스템이 부르는 쓰기는 `@SystemInvoked` 유스케이스다: 발송 claim·발송(`ClaimDueDeliveriesUseCase`·`DispatchClaimedDeliveryUseCase`), lease 회복, provider 결과 맞추기, provider event 다시 투영. `composition:notification-platform`의 worker가 `new`로 만들어 쥐고, bean으로 두지 않는다. 그 유스케이스가 감싸는 서비스(`LeaseRecoveryService`·`ReconciliationService` 등)는 bean이지만 inbound가 쥐면 빌드가 실패한다(`INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES`). 언제 다시 묻고 언제 포기하는지는 이 유스케이스가 정한다. worker는 스레드와 일정만 가진다. +- 운영자 연산(redrive·reconcile·suppress·provider 상태)은 연산마다 `platform.admin`의 유스케이스 클래스이고(`RedriveDeliveryUseCase` 등, command는 그 옆), `notification-admin:*` permission을 선언한다. `platform.port.in`에 두지 않는다: command가 `AdminActor` 같은 운영자 어휘를 실어 `port.in`과 `admin`이 서로를 보게 된다. 운영자 authority와 tenant 범위는 command의 `AdminActor`로 한 번 더 본다. - 공개 타입이 늘면 `docs/notification/api-surface-snapshot.txt`가 바뀐다(`adapter:outbound:notification`의 `verifyNotificationApiSurface`가 이 leaf까지 본다). ## 테스트 diff --git a/src/application/notification/README.md b/src/application/notification/README.md index 682a0357..b1372fc9 100644 --- a/src/application/notification/README.md +++ b/src/application/notification/README.md @@ -5,7 +5,7 @@ ## 책임과 소유하지 않는 것 - 알림 플랫폼의 업무 흐름: 제출·예약·취소·조회, template 게시, provider callback 수집, 발송 판정과 재시도 정책. -- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`BatchNotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. +- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다. - testFixtures(흐름 패키지의 `testkit`): 저장 포트의 in-memory fake와 공유 계약(`*Contract`), provider adapter 계약(`ProviderAdapterContract`). 모든 포트에 있지는 않다. adapter의 시험이 같은 계약을 상속한다. - 소유하지 않는 것: HTTP 모양(`adapter:inbound:web-notification`), provider 호출·설정(`adapter:outbound:notification`), 저장(`adapter:outbound:persistence-notification`), 조립(`composition:notification-platform`). diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java index bd902930..40006855 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java @@ -11,6 +11,10 @@ import java.util.Optional; * the work is done; and the same id with a different command is a mistake worth reporting rather * than an idempotent repeat. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param outcome what the caller may do * @param existing the earlier result, when there is one */ @@ -37,7 +41,7 @@ public record AdminOperationClaim(Outcome outcome, OptionalThe claim and the operation run inside one write transaction. PostgreSQL's unique claim row - * serializes a concurrent caller on the operation id; the loser observes the committed result - * instead of reaching the side effect. + *

The use case opens that transaction and calls this in it, so the claim and the operation + * commit or roll back together: the boundary is the use case's to declare, and the architecture + * rule that matches a capability to its {@code TransactionPort} call reads it there. PostgreSQL's + * unique claim row serializes a concurrent caller on the operation id; the loser observes the + * committed result instead of reaching the side effect. */ - AdminOperationResult execute( + AdminOperationResult runClaimed( String operationId, - String commandFingerprint, + String requestFingerprint, AdminActor actor, String action, Supplier operation) { Objects.requireNonNull(operation, "operation"); - return transactions.inWrite( - () -> { - AdminOperationClaim claim = - operations.claim(operationId, commandFingerprint, actor, action); - return claim.mayProceed() ? operation.get() : withoutRunning(claim, operationId); - }); + AdminOperationClaim claim = operations.claim(operationId, requestFingerprint, actor, action); + return claim.mayProceed() ? operation.get() : withoutRunning(claim, operationId); } /** * Claims the operation id in its own transaction and runs the operation with none open. * - *

For an operation that calls a provider over the network. Inside {@link #execute} the call + *

For an operation that calls a provider over the network. Inside {@link #runClaimed} the call * held a write transaction — and a pooled connection — for as long as the provider took, once per * attempt in the batch. The operation opens its own boundaries for what it writes, including * saving its result, which completes the claim. @@ -53,14 +52,14 @@ final class AdminOperationExecutor { */ AdminOperationResult executeOutsideTransaction( String operationId, - String commandFingerprint, + String requestFingerprint, AdminActor actor, String action, Supplier operation) { Objects.requireNonNull(operation, "operation"); AdminOperationClaim claim = transactions.inWrite( - () -> operations.claim(operationId, commandFingerprint, actor, action)); + () -> operations.claim(operationId, requestFingerprint, actor, action)); if (!claim.mayProceed()) { return withoutRunning(claim, operationId); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java similarity index 50% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java index a2f1510d..9239c52f 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java @@ -4,55 +4,55 @@ import dev.caskeleton.shared.digest.Sha256; import java.util.Objects; /** - * Stable SHA-256 identity of an operator command, excluding the operation id itself. + * Stable SHA-256 identity of an operator request, excluding the operation id itself. * *

The operation id is the idempotency key. The fingerprint answers the separate question: - * whether a repeated key names the same command. Every field is length-framed so user supplied + * whether a repeated key names the same request. Every field is length-framed so user supplied * reason text cannot imitate a field boundary. */ -final class AdminOperationCommandFingerprint { +final class AdminOperationRequestFingerprint { - String of(RedriveCommand command) { - Objects.requireNonNull(command, "command"); + String of(RedriveRequest request) { + Objects.requireNonNull(request, "request"); StringBuilder canonical = action("REDRIVE"); - field(canonical, "attemptId", command.attemptId().value().toString()); - field(canonical, "reason", command.reason()); - field(canonical, "approveDuplicateRisk", Boolean.toString(command.approveDuplicateRisk())); + field(canonical, "attemptId", request.attemptId().value().toString()); + field(canonical, "reason", request.reason()); + field(canonical, "approveDuplicateRisk", Boolean.toString(request.approveDuplicateRisk())); return digest(canonical); } - String of(ReconcileCommand command) { - Objects.requireNonNull(command, "command"); + String of(ReconcileRequest request) { + Objects.requireNonNull(request, "request"); StringBuilder canonical = action("RECONCILE"); - command.attemptIds().stream() + request.attemptIds().stream() .map(id -> id.value().toString()) .sorted() .forEach(value -> field(canonical, "attemptId", value)); - field(canonical, "reason", command.reason()); + field(canonical, "reason", request.reason()); return digest(canonical); } - String of(SuppressCommand command) { - Objects.requireNonNull(command, "command"); - StringBuilder canonical = action(command.remove() ? "SUPPRESS_REMOVE" : "SUPPRESS_ADD"); - field(canonical, "tenantId", command.tenantId().value()); - field(canonical, "scope", command.scope().name()); - field(canonical, "reason", command.reason().name()); - field(canonical, "targetFingerprint", command.targetFingerprint()); + String of(SuppressionRequest request) { + Objects.requireNonNull(request, "request"); + StringBuilder canonical = action(request.remove() ? "SUPPRESS_REMOVE" : "SUPPRESS_ADD"); + field(canonical, "tenantId", request.tenantId().value()); + field(canonical, "scope", request.scope().name()); + field(canonical, "reason", request.reason().name()); + field(canonical, "targetFingerprint", request.targetFingerprint()); field( canonical, "expiresAt", - command.expiresAt().map(value -> Long.toString(value.toEpochMilli())).orElse("-")); - field(canonical, "reasonText", command.reasonText()); + request.expiresAt().map(value -> Long.toString(value.toEpochMilli())).orElse("-")); + field(canonical, "reasonText", request.reasonText()); return digest(canonical); } - String of(SetProviderStateCommand command) { - Objects.requireNonNull(command, "command"); + String of(ProviderStateRequest request) { + Objects.requireNonNull(request, "request"); StringBuilder canonical = action("PROVIDER_STATE"); - field(canonical, "profileId", command.profileId().value()); - field(canonical, "desiredState", command.desiredState().name()); - field(canonical, "reason", command.reason()); + field(canonical, "profileId", request.profileId().value()); + field(canonical, "desiredState", request.desiredState().name()); + field(canonical, "reason", request.reason()); return digest(canonical); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java index 88f9b373..0c526edc 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java @@ -19,15 +19,17 @@ public interface AdminOperationStorePort { * was checked but never held, so it prevented a repeat and not a race. A destructive operation * run twice concurrently is the exact failure the id exists to stop. * - *

The fingerprint is part of the claim: the same id with a different command is not a replay, - * it is a conflict, and answering it with the earlier result would silently execute neither. + *

The fingerprint is part of the claim: the same id with a different request is not a replay, + * it is a conflict, and answering it with the earlier result would silently execute neither. It + * digests the operator's request and leaves the actor out; the store keeps it in the column V8 + * named {@code command_fingerprint}. * * @param operationId the caller's idempotency key - * @param commandFingerprint a digest of the command this id was presented with + * @param requestFingerprint a digest of the request this id was presented with * @param actor who is asking * @param action what they are asking for * @return whether this caller may proceed, and what the earlier one did if not */ AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action); + String operationId, String requestFingerprint, AdminActor actor, String action); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java new file mode 100644 index 00000000..d6626f7f --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's change to one provider profile's runtime state. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record ChangeProviderStateCommand(ProviderStateRequest request, AdminActor actor) + implements Command { + + public ChangeProviderStateCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java new file mode 100644 index 00000000..4dc51de9 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java @@ -0,0 +1,130 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.error.FailureCategory; +import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode; +import dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor; +import dev.caskeleton.application.notification.platform.api.error.ProviderConfigurationException; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Enables, disables or drains a provider profile as an operator. + * + *

A provider profile is shared by every tenant, so only an operator with no tenant scope may + * change it. The operation id is claimed before the runtime is asked, in the same transaction. + */ +@RequiresPermission(NotificationAdminPermissions.PROVIDER_CONTROL) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // The provider runtime and the audit port are the notification adapter's. + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class ChangeProviderStateUseCase + implements CommandUseCase { + + private static final String ACTION = "ADMIN_PROVIDER_STATE"; + + private final AdminAuthorizationGuard authorization; + private final ProviderRuntimeControlPort runtimes; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final Clock clock; + + public ChangeProviderStateUseCase( + AdminAuthorizationGuard authorization, + ProviderRuntimeControlPort runtimes, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.runtimes = Objects.requireNonNull(runtimes, "runtimes"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(ChangeProviderStateCommand change) { + Objects.requireNonNull(change, "change"); + ProviderStateRequest request = change.request(); + AdminActor actor = change.actor(); + authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL); + authorization.requireGlobal(actor, NotificationAdminAuthority.PROVIDER_CONTROL); + + if (request.dryRun()) { + return new AdminOperationResult( + request.operationId(), + true, + 1, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of("DRY_RUN")); + } + + return transactions.inWrite( + () -> + operationExecutor.runClaimed( + request.operationId(), + requestFingerprints.of(request), + actor, + ACTION, + () -> change(request, actor))); + } + + private AdminOperationResult change(ProviderStateRequest request, AdminActor actor) { + boolean applied = + runtimes.setState(request.profileId(), request.desiredState(), request.reason()); + if (!applied) { + // The transition was refused — an authentication failure is not cleared by declaring the + // provider healthy, and a healthy runtime is not degraded on request. Recording success for a + // state change that did not happen tells an operator the provider is serving when it is not. + throw new ProviderConfigurationException( + NotificationFailureDescriptor.preDispatch( + NotificationFailureCode.PROVIDER_CONFIGURATION_INVALID, + FailureCategory.AUTHORIZATION)); + } + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + 1, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(request.reason())); + audit.record( + new NotificationAuditEvent( + ACTION, + actor.actorRef(), + Optional.of(request.reason()), + Optional.of(request.operationId()), + clock.instant(), + Map.of( + "providerProfile", request.profileId().value(), + "status", request.desiredState().name()))); + return operations.save(result, actor, ACTION); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java new file mode 100644 index 00000000..8a493bac --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's change to one suppression entry. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record ChangeSuppressionCommand(SuppressionRequest request, AdminActor actor) + implements Command { + + public ChangeSuppressionCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java new file mode 100644 index 00000000..4351472c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java @@ -0,0 +1,149 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.notification.platform.policy.SuppressionEntry; +import dev.caskeleton.application.notification.platform.policy.SuppressionId; +import dev.caskeleton.application.notification.platform.policy.SuppressionSource; +import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Adds or lifts a suppression entry as an operator. + * + *

Lifting is by fingerprint match rather than by id, because an operator lifting a suppression + * knows the target, not the row identifier the platform assigned. + */ +@RequiresPermission(NotificationAdminPermissions.SUPPRESS) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // The audit port and the id generator are the notification adapter's. + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class ChangeSuppressionUseCase + implements CommandUseCase { + + private final AdminAuthorizationGuard authorization; + private final SuppressionStorePort suppressions; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final NotificationIdGeneratorPort ids; + private final Clock clock; + + public ChangeSuppressionUseCase( + AdminAuthorizationGuard authorization, + SuppressionStorePort suppressions, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + NotificationIdGeneratorPort ids, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.suppressions = Objects.requireNonNull(suppressions, "suppressions"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.ids = Objects.requireNonNull(ids, "ids"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(ChangeSuppressionCommand change) { + Objects.requireNonNull(change, "change"); + SuppressionRequest request = change.request(); + AdminActor actor = change.actor(); + authorization.require(actor, NotificationAdminAuthority.SUPPRESS); + authorization.requireTenant(actor, request.tenantId(), NotificationAdminAuthority.SUPPRESS); + + if (request.dryRun()) { + return new AdminOperationResult( + request.operationId(), + true, + 1, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of("DRY_RUN")); + } + + String action = request.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD"; + return transactions.inWrite( + () -> + operationExecutor.runClaimed( + request.operationId(), + requestFingerprints.of(request), + actor, + action, + () -> change(request, actor, action))); + } + + private AdminOperationResult change(SuppressionRequest request, AdminActor actor, String action) { + int affected = request.remove() ? lift(request) : add(request); + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + affected, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(request.reasonText())); + audit.record( + new NotificationAuditEvent( + request.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED", + actor.actorRef(), + Optional.of(request.reason().name()), + Optional.of(request.operationId()), + clock.instant(), + Map.of())); + return operations.save(result, actor, action); + } + + private int lift(SuppressionRequest request) { + long lifted = + suppressions + .activeFor(request.tenantId(), request.targetFingerprint(), clock.instant()) + .stream() + .map(entry -> suppressions.remove(request.tenantId(), entry.id())) + .filter(Optional::isPresent) + .count(); + return lifted > 0 ? 1 : 0; + } + + private int add(SuppressionRequest request) { + suppressions.upsert( + new SuppressionEntry( + // ids.nextId(), not UUID.randomUUID(): a random v4 id is unordered, so the suppression + // table's index degrades the way the platform's UUIDv7 policy exists to prevent. + new SuppressionId(ids.nextId()), + request.tenantId(), + request.scope(), + request.reason(), + request.targetFingerprint(), + Optional.empty(), + clock.instant(), + request.expiresAt(), + SuppressionSource.ADMIN)); + return 1; + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java deleted file mode 100644 index 684bad6e..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java +++ /dev/null @@ -1,362 +0,0 @@ -package dev.caskeleton.application.notification.platform.admin; - -import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; -import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; -import dev.caskeleton.application.notification.platform.api.error.FailureCategory; -import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode; -import dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor; -import dev.caskeleton.application.notification.platform.api.error.ProviderConfigurationException; -import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot; -import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; -import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; -import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; -import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; -import dev.caskeleton.application.notification.platform.policy.SuppressionEntry; -import dev.caskeleton.application.notification.platform.policy.SuppressionId; -import dev.caskeleton.application.notification.platform.policy.SuppressionSource; -import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; -import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; -import dev.caskeleton.application.transaction.TransactionPort; -import java.time.Clock; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -/** - * N4 operator plane. - * - *

It lived in the outbound notification adapter, where it performed operator authorization, - * tenant scoping, duplicate-risk approval, idempotency and transaction orchestration — every one of - * them a decision about what the product allows, in the layer whose job is speaking provider - * protocols. What remains out there is {@link ProviderRuntimeControlPort}: two operations that turn - * an application request into a runtime transition. - * - *

Four properties hold for every operation: a separate authority, an idempotent operation id, a - * recorded reason, and an audit row. The idempotency matters more than it looks — an operator - * retrying a redrive after a timeout must not send the message twice, which is exactly the failure - * the operation is trying to repair. - * - *

A dry run reads and reports but writes nothing, so an operator can see the blast radius of a - * bulk action before committing to it. - */ -public final class NotificationAdminApplicationService implements NotificationAdminService { - - private final AdminAuthorizationGuard authorization; - private final DuplicateRiskGuard duplicateRiskGuard; - private final DeliveryAttemptStorePort attempts; - private final RecipientDeliveryStorePort recipients; - private final ReconciliationService reconciliation; - private final SuppressionStorePort suppressions; - private final ProviderRuntimeControlPort runtimes; - private final AdminOperationStorePort operations; - private final AdminOperationExecutor operationExecutor; - private final AdminOperationCommandFingerprint commandFingerprints; - private final TransactionPort transactions; - private final NotificationAuditPort audit; - private final dev.caskeleton.application.notification.platform.dispatch - .NotificationIdGeneratorPort - ids; - private final Clock clock; - - public NotificationAdminApplicationService( - AdminAuthorizationGuard authorization, - DuplicateRiskGuard duplicateRiskGuard, - DeliveryAttemptStorePort attempts, - RecipientDeliveryStorePort recipients, - ReconciliationService reconciliation, - SuppressionStorePort suppressions, - ProviderRuntimeControlPort runtimes, - AdminOperationStorePort operations, - NotificationAuditPort audit, - TransactionPort transactions, - dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort ids, - Clock clock) { - this.authorization = Objects.requireNonNull(authorization, "authorization"); - this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard"); - this.attempts = Objects.requireNonNull(attempts, "attempts"); - this.recipients = Objects.requireNonNull(recipients, "recipients"); - this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation"); - this.suppressions = Objects.requireNonNull(suppressions, "suppressions"); - this.runtimes = Objects.requireNonNull(runtimes, "runtimes"); - this.operations = Objects.requireNonNull(operations, "operations"); - this.transactions = Objects.requireNonNull(transactions, "transactions"); - this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); - this.commandFingerprints = new AdminOperationCommandFingerprint(); - this.audit = Objects.requireNonNull(audit, "audit"); - this.ids = Objects.requireNonNull(ids, "ids"); - this.clock = Objects.requireNonNull(clock, "clock"); - } - - @Override - public AdminOperationResult redrive(RedriveCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.REDRIVE); - - DeliveryAttemptSnapshot original = - attempts - .snapshot(command.attemptId()) - .orElseThrow(() -> new IllegalStateException("delivery attempt is not available")); - authorization.requireTenant(actor, original.tenantId(), NotificationAdminAuthority.REDRIVE); - duplicateRiskGuard.verify(original, command.approveDuplicateRisk()); - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - 1, - Optional.of(original.notificationId()), - Optional.of(original.recipientDeliveryId()), - Optional.empty(), - List.of("DRY_RUN")); - } - - return operationExecutor.execute( - command.operationId(), - commandFingerprints.of(command), - actor, - "ADMIN_REDRIVE", - () -> { - // The logical identities are preserved and only the attempt is new, so the history stays - // one story rather than becoming two unrelated notifications. - recipients.transition( - original.recipientDeliveryId(), - RecipientDeliveryState.READY_TO_DISPATCH, - Optional.of(clock.instant())); - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - 1, - Optional.of(original.notificationId()), - Optional.of(original.recipientDeliveryId()), - Optional.empty(), - List.of(command.reason())); - audit.record( - new NotificationAuditEvent( - "ADMIN_REDRIVE", - actor.actorRef(), - Optional.of(command.reason()), - Optional.of(command.operationId()), - clock.instant(), - Map.of( - "provider", original.providerId().value(), - "channel", original.channel().name()))); - return operations.save(result, actor, "ADMIN_REDRIVE"); - }); - } - - @Override - public AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.RECONCILE); - // Every attempt is checked before any is touched, so a batch with one foreign id changes none. - // An id with no attempt behind it is left to reconciliation, which does nothing with it. - for (DeliveryAttemptId attemptId : command.attemptIds()) { - attempts - .snapshot(attemptId) - .ifPresent( - attempt -> - authorization.requireTenant( - actor, attempt.tenantId(), NotificationAdminAuthority.RECONCILE)); - } - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - command.attemptIds().size(), - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of("DRY_RUN")); - } - - // Outside a transaction: each reconcile asks a provider over the network and opens its own - // boundaries for what it writes. - return operationExecutor.executeOutsideTransaction( - command.operationId(), - commandFingerprints.of(command), - actor, - "ADMIN_RECONCILE", - () -> { - List reasons = new ArrayList<>(); - int reconciled = 0; - for (DeliveryAttemptId attemptId : command.attemptIds()) { - reconciliation.reconcile(attemptId); - reconciled++; - } - reasons.add(command.reason()); - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - reconciled, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.copyOf(reasons)); - return transactions.inWrite( - () -> { - audit.record( - new NotificationAuditEvent( - "ADMIN_RECONCILE", - actor.actorRef(), - Optional.of(command.reason()), - Optional.of(command.operationId()), - clock.instant(), - Map.of())); - return operations.save(result, actor, "ADMIN_RECONCILE"); - }); - }); - } - - @Override - public AdminOperationResult suppress(SuppressCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.SUPPRESS); - authorization.requireTenant(actor, command.tenantId(), NotificationAdminAuthority.SUPPRESS); - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - 1, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of("DRY_RUN")); - } - - String action = command.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD"; - return operationExecutor.execute( - command.operationId(), - commandFingerprints.of(command), - actor, - action, - () -> { - int affected; - if (command.remove()) { - // Removal is by fingerprint match rather than by id, because an operator lifting a - // suppression knows the target, not the row identifier the platform assigned. - affected = - suppressions - .activeFor( - command.tenantId(), command.targetFingerprint(), clock.instant()) - .stream() - .map(entry -> suppressions.remove(command.tenantId(), entry.id())) - .filter(Optional::isPresent) - .count() - > 0 - ? 1 - : 0; - } else { - suppressions.upsert( - new SuppressionEntry( - // ids.nextId(), not UUID.randomUUID(). The identifier contract requires the - // generator port, and while this class lived in the outbound adapter the rule - // that says so did not apply to it — a random v4 id here is unordered, so the - // suppression table's index degrades the way the platform's own UUIDv7 policy - // exists to prevent. - new SuppressionId(ids.nextId()), - command.tenantId(), - command.scope(), - command.reason(), - command.targetFingerprint(), - Optional.empty(), - clock.instant(), - command.expiresAt(), - SuppressionSource.ADMIN)); - affected = 1; - } - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - affected, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of(command.reasonText())); - audit.record( - new NotificationAuditEvent( - command.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED", - actor.actorRef(), - Optional.of(command.reason().name()), - Optional.of(command.operationId()), - clock.instant(), - Map.of())); - return operations.save(result, actor, action); - }); - } - - @Override - public AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor) { - Objects.requireNonNull(command, "command"); - authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL); - authorization.requireGlobal(actor, NotificationAdminAuthority.PROVIDER_CONTROL); - - if (command.dryRun()) { - return new AdminOperationResult( - command.operationId(), - true, - 1, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of("DRY_RUN")); - } - - return operationExecutor.execute( - command.operationId(), - commandFingerprints.of(command), - actor, - "ADMIN_PROVIDER_STATE", - () -> { - boolean applied = - runtimes.setState(command.profileId(), command.desiredState(), command.reason()); - if (!applied) { - // The transition was refused — an authentication failure is not cleared by declaring - // the provider healthy, and a healthy runtime is not degraded on request. Recording - // success for a state change that did not happen tells an operator the provider is - // serving when it is not. - throw new ProviderConfigurationException( - NotificationFailureDescriptor.preDispatch( - NotificationFailureCode.PROVIDER_CONFIGURATION_INVALID, - FailureCategory.AUTHORIZATION)); - } - - AdminOperationResult result = - new AdminOperationResult( - command.operationId(), - false, - 1, - Optional.empty(), - Optional.empty(), - Optional.empty(), - List.of(command.reason())); - audit.record( - new NotificationAuditEvent( - "ADMIN_PROVIDER_STATE", - actor.actorRef(), - Optional.of(command.reason()), - Optional.of(command.operationId()), - clock.instant(), - Map.of( - "providerProfile", command.profileId().value(), - "status", command.desiredState().name()))); - return operations.save(result, actor, "ADMIN_PROVIDER_STATE"); - }); - } - - /** Current state of a provider runtime, for the health endpoint. */ - public ProviderRuntimeState providerState( - dev.caskeleton.application.notification.platform.api.ProviderProfileId profileId) { - return runtimes.state(profileId); - } -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java new file mode 100644 index 00000000..be4175d1 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java @@ -0,0 +1,27 @@ +package dev.caskeleton.application.notification.platform.admin; + +/** + * The permissions the four operator-plane use cases declare. + * + *

One per operation, like the {@link NotificationAdminAuthority} the operator also has to hold: + * the permission is what the transport's caller may invoke at all, the authority and tenant scope + * on the {@code AdminActor} are what the operation then checks against the delivery it touches. + * Which roles hold them by default is the notification module defaults' to say, not this type's: + * {@code docs/notification/operations.md} 「운영자 연산」. + */ +public final class NotificationAdminPermissions { + + /** Re-run one delivery. */ + public static final String REDRIVE = "notification-admin:redrive"; + + /** Ask providers about a batch of attempts. */ + public static final String RECONCILE = "notification-admin:reconcile"; + + /** Add or lift a suppression entry. */ + public static final String SUPPRESS = "notification-admin:suppress"; + + /** Change a provider profile's runtime state. */ + public static final String PROVIDER_CONTROL = "notification-admin:provider-control"; + + private NotificationAdminPermissions() {} +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java deleted file mode 100644 index 5b003cc8..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java +++ /dev/null @@ -1,22 +0,0 @@ -package dev.caskeleton.application.notification.platform.admin; - -/** - * N4 operator plane. - * - *

Every operation requires a separate authority, an idempotent operation id, a reason and an - * audit record. It is deliberately not reachable from the application-facing APIs. - */ -public interface NotificationAdminService { - - /** Re-run a delivery while preserving its logical identity. */ - AdminOperationResult redrive(RedriveCommand command, AdminActor actor); - - /** Force reconciliation of a bounded batch. */ - AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor); - - /** Add or remove a suppression entry. */ - AdminOperationResult suppress(SuppressCommand command, AdminActor actor); - - /** Change a provider runtime state. */ - AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor); -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java similarity index 91% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java index c942f21f..033884a4 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java @@ -5,14 +5,14 @@ import dev.caskeleton.application.notification.platform.provider.ProviderRuntime import java.util.Objects; /** Enable, disable or drain a provider profile. */ -public record SetProviderStateCommand( +public record ProviderStateRequest( String operationId, ProviderProfileId profileId, ProviderRuntimeState desiredState, String reason, boolean dryRun) { - public SetProviderStateCommand { + public ProviderStateRequest { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(profileId, "profileId"); Objects.requireNonNull(desiredState, "desiredState"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java new file mode 100644 index 00000000..8f23b5aa --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java @@ -0,0 +1,22 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's reconciliation of a bounded batch of attempts. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record ReconcileDeliveriesCommand(ReconcileRequest request, AdminActor actor) + implements Command { + + public ReconcileDeliveriesCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java new file mode 100644 index 00000000..aba76aca --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java @@ -0,0 +1,143 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Asks providers, on an operator's request, what happened to a bounded batch of attempts. + * + *

Every attempt is checked against the operator's tenant scope before any is touched, so a batch + * with one foreign id changes none. An id with no attempt behind it is left to reconciliation, + * which does nothing with it. + * + *

The provider calls run with no transaction open: each one asks over the network, and inside a + * write transaction it held a pooled connection for as long as the provider took, once per attempt. + * The operation id is claimed in its own transaction first, and the audit row and result are + * written in one at the end. + */ +@RequiresPermission(NotificationAdminPermissions.RECONCILE) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class ReconcileDeliveriesUseCase + implements CommandUseCase { + + private static final String ACTION = "ADMIN_RECONCILE"; + + private final AdminAuthorizationGuard authorization; + private final DeliveryAttemptStorePort attempts; + private final ReconciliationService reconciliation; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final Clock clock; + + public ReconcileDeliveriesUseCase( + AdminAuthorizationGuard authorization, + DeliveryAttemptStorePort attempts, + ReconciliationService reconciliation, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.attempts = Objects.requireNonNull(attempts, "attempts"); + this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(ReconcileDeliveriesCommand batch) { + Objects.requireNonNull(batch, "batch"); + ReconcileRequest request = batch.request(); + AdminActor actor = batch.actor(); + authorization.require(actor, NotificationAdminAuthority.RECONCILE); + // Read in a read transaction of its own: the providers are asked with none open, so the tenant + // check cannot share the boundary the answers are recorded in. + transactions.inRead( + () -> { + for (DeliveryAttemptId attemptId : request.attemptIds()) { + attempts + .snapshot(attemptId) + .ifPresent( + attempt -> + authorization.requireTenant( + actor, attempt.tenantId(), NotificationAdminAuthority.RECONCILE)); + } + return null; + }); + + if (request.dryRun()) { + return new AdminOperationResult( + request.operationId(), + true, + request.attemptIds().size(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of("DRY_RUN")); + } + + return operationExecutor.executeOutsideTransaction( + request.operationId(), + requestFingerprints.of(request), + actor, + ACTION, + () -> reconcile(request, actor)); + } + + private AdminOperationResult reconcile(ReconcileRequest request, AdminActor actor) { + int reconciled = 0; + for (DeliveryAttemptId attemptId : request.attemptIds()) { + reconciliation.reconcile(attemptId); + reconciled++; + } + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + reconciled, + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(request.reason())); + return transactions.inWrite( + () -> { + audit.record( + new NotificationAuditEvent( + ACTION, + actor.actorRef(), + Optional.of(request.reason()), + Optional.of(request.operationId()), + clock.instant(), + Map.of())); + return operations.save(result, actor, ACTION); + }); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java similarity index 92% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java index b62d5b7d..491b8526 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java @@ -5,12 +5,12 @@ import java.util.List; import java.util.Objects; /** Operator-triggered reconciliation over a bounded batch of attempts. */ -public record ReconcileCommand( +public record ReconcileRequest( String operationId, List attemptIds, String reason, boolean dryRun) { public static final int MAX_BATCH = 200; - public ReconcileCommand { + public ReconcileRequest { Objects.requireNonNull(operationId, "operationId"); attemptIds = List.copyOf(Objects.requireNonNull(attemptIds, "attemptIds")); Objects.requireNonNull(reason, "reason"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java new file mode 100644 index 00000000..ab395b61 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java @@ -0,0 +1,21 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * An operator's redrive of one delivery attempt. + * + *

The request and the operator are apart because the operation id's fingerprint covers the + * request only: the same operator retrying the same request replays it. + * + * @param request what to do + * @param actor who asks, with the operator authorities and tenant scope the transport resolved + */ +public record RedriveDeliveryCommand(RedriveRequest request, AdminActor actor) implements Command { + + public RedriveDeliveryCommand { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(actor, "actor"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java new file mode 100644 index 00000000..3b96e42c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java @@ -0,0 +1,149 @@ +package dev.caskeleton.application.notification.platform.admin; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; +import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot; +import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; +import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.security.RequiresPermission; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Re-runs one delivery while preserving its logical identity. + * + *

The logical identities are kept and only the next attempt is new, so the history stays one + * story rather than becoming two unrelated notifications. An operator retrying after a timeout must + * not send the message twice — exactly the failure a redrive is trying to repair — so the operation + * id is claimed before the delivery is touched, in the same transaction. + * + *

A dry run reads and reports but writes nothing. + */ +@RequiresPermission(NotificationAdminPermissions.REDRIVE) +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // The audit port is the notification adapter's. + externalOutboundAllowed = true, + crossTenantAdmin = true) +// Not final: @RequiresPermission is applied through a CGLIB proxy. +public class RedriveDeliveryUseCase + implements CommandUseCase { + + private static final String ACTION = "ADMIN_REDRIVE"; + + private final AdminAuthorizationGuard authorization; + private final DuplicateRiskGuard duplicateRiskGuard; + private final DeliveryAttemptStorePort attempts; + private final RecipientDeliveryStorePort recipients; + private final AdminOperationStorePort operations; + private final AdminOperationExecutor operationExecutor; + private final AdminOperationRequestFingerprint requestFingerprints = + new AdminOperationRequestFingerprint(); + private final NotificationAuditPort audit; + private final TransactionPort transactions; + private final Clock clock; + + public RedriveDeliveryUseCase( + AdminAuthorizationGuard authorization, + DuplicateRiskGuard duplicateRiskGuard, + DeliveryAttemptStorePort attempts, + RecipientDeliveryStorePort recipients, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + this.authorization = Objects.requireNonNull(authorization, "authorization"); + this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard"); + this.attempts = Objects.requireNonNull(attempts, "attempts"); + this.recipients = Objects.requireNonNull(recipients, "recipients"); + this.operations = Objects.requireNonNull(operations, "operations"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions); + this.audit = Objects.requireNonNull(audit, "audit"); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public AdminOperationResult handle(RedriveDeliveryCommand delivery) { + Objects.requireNonNull(delivery, "delivery"); + RedriveRequest request = delivery.request(); + AdminActor actor = delivery.actor(); + authorization.require(actor, NotificationAdminAuthority.REDRIVE); + + if (request.dryRun()) { + DeliveryAttemptSnapshot original = transactions.inRead(() -> checkedOriginal(request, actor)); + return new AdminOperationResult( + request.operationId(), + true, + 1, + Optional.of(original.notificationId()), + Optional.of(original.recipientDeliveryId()), + Optional.empty(), + List.of("DRY_RUN")); + } + + // The checks read the attempt in the transaction that claims the operation and writes, so the + // attempt they approved is the one redriven. + return transactions.inWrite( + () -> { + DeliveryAttemptSnapshot original = checkedOriginal(request, actor); + return operationExecutor.runClaimed( + request.operationId(), + requestFingerprints.of(request), + actor, + ACTION, + () -> redrive(request, actor, original)); + }); + } + + private DeliveryAttemptSnapshot checkedOriginal(RedriveRequest request, AdminActor actor) { + DeliveryAttemptSnapshot original = + attempts + .snapshot(request.attemptId()) + .orElseThrow(() -> new IllegalStateException("delivery attempt is not available")); + authorization.requireTenant(actor, original.tenantId(), NotificationAdminAuthority.REDRIVE); + duplicateRiskGuard.verify(original, request.approveDuplicateRisk()); + return original; + } + + private AdminOperationResult redrive( + RedriveRequest request, AdminActor actor, DeliveryAttemptSnapshot original) { + recipients.transition( + original.recipientDeliveryId(), + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(clock.instant())); + + AdminOperationResult result = + new AdminOperationResult( + request.operationId(), + false, + 1, + Optional.of(original.notificationId()), + Optional.of(original.recipientDeliveryId()), + Optional.empty(), + List.of(request.reason())); + audit.record( + new NotificationAuditEvent( + ACTION, + actor.actorRef(), + Optional.of(request.reason()), + Optional.of(request.operationId()), + clock.instant(), + Map.of( + "provider", original.providerId().value(), + "channel", original.channel().name()))); + return operations.save(result, actor, ACTION); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java similarity index 93% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java index fb9f031c..fe973b96 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java @@ -9,14 +9,14 @@ import java.util.Objects; *

A redrive reuses the pinned template version and rendered digest: it re-executes the original * notification rather than sending a new, possibly different message under an old identity. */ -public record RedriveCommand( +public record RedriveRequest( String operationId, DeliveryAttemptId attemptId, String reason, boolean approveDuplicateRisk, boolean dryRun) { - public RedriveCommand { + public RedriveRequest { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(attemptId, "attemptId"); Objects.requireNonNull(reason, "reason"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java similarity index 94% rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java index 1c4a3764..891d9698 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java @@ -8,7 +8,7 @@ import java.util.Objects; import java.util.Optional; /** Add or remove a suppression entry as an operator. */ -public record SuppressCommand( +public record SuppressionRequest( String operationId, TenantId tenantId, SuppressionScope scope, @@ -19,7 +19,7 @@ public record SuppressCommand( String reasonText, boolean dryRun) { - public SuppressCommand { + public SuppressionRequest { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(tenantId, "tenantId"); Objects.requireNonNull(scope, "scope"); diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java deleted file mode 100644 index 56924a78..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java +++ /dev/null @@ -1,19 +0,0 @@ -package dev.caskeleton.application.notification.platform.api; - -/** - * Provider-assigned event identity. Absent for providers that do not emit one; the ledger then - * falls back to a deterministic fingerprint. - */ -public record ProviderEventId(String value) { - - private static final int MAX_LENGTH = 300; - - public ProviderEventId { - if (value == null || value.isBlank()) { - throw new IllegalArgumentException("value"); - } - if (value.length() > MAX_LENGTH) { - throw new IllegalArgumentException("value exceeds " + MAX_LENGTH + " characters"); - } - } -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java deleted file mode 100644 index ba926e17..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java +++ /dev/null @@ -1,11 +0,0 @@ -package dev.caskeleton.application.notification.platform.api.error; - -/** The delivery expiry passed before a new attempt could start. */ -public class NotificationExpiredException extends NotificationException { - - private static final long serialVersionUID = 1L; - - public NotificationExpiredException(NotificationFailureDescriptor descriptor) { - super(descriptor); - } -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java index 7db02dfb..e24d819c 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java @@ -3,6 +3,7 @@ package dev.caskeleton.application.notification.platform.callback; import dev.caskeleton.application.notification.platform.observation.NotificationMetricName; import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; import dev.caskeleton.application.transaction.TransactionPort; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -72,6 +73,26 @@ public final class ProviderEventProjectionService { transactions.inWrite(() -> ledger.markFailed(event.id(), reasonCode)); } + /** + * Stored events that are not matched to an attempt yet, read in a read transaction of its own. + * + * @param limit the most events to return + * @return the oldest unmatched events + */ + public List unmatched(int limit) { + return transactions.inRead(() -> ledger.unmatched(limit)); + } + + /** + * Stored events whose projection has not run or failed, read in a read transaction of its own. + * + * @param limit the most events to return + * @return the oldest events awaiting projection + */ + public List pendingProjection(int limit) { + return transactions.inRead(() -> ledger.pendingProjection(limit)); + } + private Optional projectInTransaction(ProviderEventRecord event) { Optional attempt = resolve(event); if (attempt.isEmpty()) { diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java new file mode 100644 index 00000000..b9cb86c8 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.callback; + +import dev.caskeleton.application.command.Command; + +/** One replay pass over the stored, unprojected provider events; its bound is the use case's. */ +public record ReplayProviderEventsCommand() implements Command {} diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java similarity index 50% rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java index 50292159..833e4d2a 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java @@ -1,8 +1,11 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.application.notification.platform.callback; -import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger; -import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; -import dev.caskeleton.application.notification.platform.callback.ProviderEventRecord; +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; import java.util.List; import java.util.Objects; @@ -19,26 +22,41 @@ import java.util.Objects; * ingestion path deliberately stores such an event as {@code PENDING} rather than dropping it. With * nothing replaying it, "we will match it later" was true only in the comment: the delivery stayed * unconfirmed forever and the callback that would have confirmed it sat in a table. + * + *

A use case, not a worker of the notification adapter, which it used to be (round9 review + * 2026-11-12, RR9-05-06, RR9-05-08). It holds no transaction and touches no repository itself: + * {@link ProviderEventProjectionService} reads each list in a read transaction and projects, or + * marks failed, each event in a write transaction of its own, so one event's failure does not roll + * back the others. + * + *

Not a bean. The notification composition's background workers construct it and keep it. */ -public final class ProviderEventReplayWorker { +@SystemInvoked( + reason = + "the notification background workers run it on their scheduler thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + // Every read and write is the projection service's, each in a boundary of that service's. + repositoryAccess = RepositoryAccess.NONE) +public class ReplayProviderEventsUseCase + implements CommandUseCase { - private final ProviderEventLedger ledger; private final ProviderEventProjectionService projection; private final int batchSize; /** - * Creates the replay worker. + * Creates the use case. * - * @param ledger the stored events - * @param projection the projection the events feed + * @param projection reads the stored events and projects them * @param batchSize how many events one pass handles + * @throws IllegalStateException if the batch size is not positive, which is the assembly's fault */ - public ProviderEventReplayWorker( - ProviderEventLedger ledger, ProviderEventProjectionService projection, int batchSize) { - this.ledger = Objects.requireNonNull(ledger, "ledger"); + public ReplayProviderEventsUseCase(ProviderEventProjectionService projection, int batchSize) { this.projection = Objects.requireNonNull(projection, "projection"); if (batchSize < 1) { - throw new IllegalArgumentException("batchSize"); + throw new IllegalStateException("batchSize must be positive"); } this.batchSize = batchSize; } @@ -52,10 +70,12 @@ public final class ProviderEventReplayWorker { * * @return how many events projected successfully */ - public int replayOnce() { + @Override + public Integer handle(ReplayProviderEventsCommand command) { + Objects.requireNonNull(command, "command"); int projected = 0; - projected += replay(ledger.unmatched(batchSize)); - projected += replay(ledger.pendingProjection(batchSize)); + projected += replay(projection.unmatched(batchSize)); + projected += replay(projection.pendingProjection(batchSize)); return projected; } @@ -69,8 +89,8 @@ public final class ProviderEventReplayWorker { projected++; } } catch (RuntimeException failure) { - // Through the projection service, which opens the transaction: this worker has none, and - // marking the event is a modifying query that throws without one. + // Through the projection service, which opens the transaction: this use case holds none + // here, and marking the event is a modifying query that throws without one. projection.markFailed(event, failure.getClass().getSimpleName()); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java index d8ed77b3..3107e0ac 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java @@ -137,6 +137,9 @@ public class AcceptNotificationApplicationUseCase implements AcceptNotificationU command.templateVersion().orElse(NotificationTemplateVersion.FIRST_VERSION), command.locale().orElse(defaultLocale)); CorrelationId correlationId = new CorrelationId(ids.nextId().toString()); + // Not the HTTP Idempotency-Key the executor replays: a business key that makes the + // notification itself one per tenant and lives as long as it does, derived from the plan when + // the caller gave none (manual 05 「executor를 쓰지 않는 중복 판정 둘」). IdempotencyKey idempotencyKey = command .idempotencyKey() diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java index 9ca2b512..0204482e 100644 Binary files a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java and b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java differ diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java new file mode 100644 index 00000000..8a7e0b00 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java @@ -0,0 +1,24 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; +import java.time.Duration; +import java.util.Objects; + +/** + * One claim of due deliveries for a scheduler worker. + * + * @param workerId the claiming worker, including its process incarnation + * @param limit the most deliveries to claim; only as many as can start now + * @param leaseDuration how long the claimed deliveries are the worker's + */ +public record ClaimDueDeliveriesCommand(String workerId, int limit, Duration leaseDuration) + implements Command { + + public ClaimDueDeliveriesCommand { + Objects.requireNonNull(workerId, "workerId"); + Objects.requireNonNull(leaseDuration, "leaseDuration"); + if (limit < 1) { + throw new IllegalStateException("limit must be positive"); + } + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java new file mode 100644 index 00000000..941fb393 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java @@ -0,0 +1,47 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.List; +import java.util.Objects; + +/** + * Claims the deliveries that are due, each under a fresh lease and fence. + * + *

The first of the scheduler's two steps; {@link DispatchClaimedDeliveryUseCase} is the second, + * once per claimed delivery. The worker that drives them owns only threads and concurrency permits: + * what is claimed, and under which lease, is decided here (round9 review 2026-11-12, RR9-05-06). + * + *

Not a bean. The notification composition's scheduler worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the notification scheduler worker claims on its polling thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.NOT_IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY) +public class ClaimDueDeliveriesUseCase + implements CommandUseCase> { + + private final RecipientLeaseStorePort leases; + private final TransactionPort transactions; + + public ClaimDueDeliveriesUseCase(RecipientLeaseStorePort leases, TransactionPort transactions) { + this.leases = Objects.requireNonNull(leases, "leases"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + @Override + public List handle(ClaimDueDeliveriesCommand command) { + Objects.requireNonNull(command, "command"); + return transactions.inWrite( + () -> leases.claim(command.workerId(), command.limit(), command.leaseDuration())); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java new file mode 100644 index 00000000..ab060270 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java @@ -0,0 +1,16 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; +import java.util.Objects; + +/** + * One claimed delivery to send. + * + * @param lease the claim the delivery is held under + */ +public record DispatchClaimedDeliveryCommand(RecipientLease lease) implements Command { + + public DispatchClaimedDeliveryCommand { + Objects.requireNonNull(lease, "lease"); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java new file mode 100644 index 00000000..68e33fde --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java @@ -0,0 +1,46 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.Objects; + +/** + * Sends one claimed delivery, under its lease. + * + *

{@code RepositoryAccess.NONE}: the delivery is {@link NotificationDispatchService}'s, which + * commits the attempt row before the provider call and records the outcome after it, each in a + * boundary of its own. One around the whole dispatch would hold a connection across the provider + * call. + * + *

Not a bean. The notification composition's scheduler worker constructs it and keeps it. + */ +@SystemInvoked( + reason = + "the notification scheduler worker dispatches on its executor threads, which have no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.NOT_IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // The dispatch reaches the provider. + externalOutboundAllowed = true) +public class DispatchClaimedDeliveryUseCase + implements CommandUseCase { + + private final NotificationDispatchService dispatcher; + + public DispatchClaimedDeliveryUseCase(NotificationDispatchService dispatcher) { + this.dispatcher = Objects.requireNonNull(dispatcher, "dispatcher"); + } + + @Override + public Void handle(DispatchClaimedDeliveryCommand command) { + Objects.requireNonNull(command, "command"); + dispatcher.dispatch(command.lease()); + return null; + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java index bb5f12fd..901fd901 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java @@ -2,7 +2,6 @@ package dev.caskeleton.application.notification.platform.dispatch; import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; -import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; import dev.caskeleton.application.transaction.TransactionPort; import java.time.Duration; import java.util.List; @@ -27,6 +26,13 @@ import java.util.Objects; * nothing to iterate. Those rows stayed {@code DISPATCHING} forever, holding a delivery nobody had * even tried to send. * + *

A requeue is one write conditional on the abandoned lease recovery found (owner, fence, state, + * expiry), not a read followed by an unconditional transition. The pass runs on every instance, and + * between finding a job and writing it another instance's recovery may have requeued it and a new + * worker claimed it, or the holder may have finished; the unconditional write put such a job back + * to {@code READY_TO_DISPATCH} and it was dispatched again (round9 review 2026-11-12, RR9-01-12). A + * requeue that matches nothing is not counted as handled. + * *

Every write opens its own transaction; the pass itself runs on a bare scheduler thread. The * stores behind these ports are modifying JPA queries, which throw outside a transaction — so the * pass used to fail on its first statement, every time, and the scheduler only logged it. It lives @@ -110,14 +116,14 @@ public final class LeaseRecoveryService { // Retiring expired deliveries rides along with recovery because both answer the same operator // question — "why is this row still here?" — and neither needs a scheduler of its own. int handled = transactions.inWrite(() -> leases.expireOverdue(batchSize)); - List abandoned = leases.expiredDispatching(batchSize, staleAfter); - for (var recipientDeliveryId : abandoned) { + List abandoned = leases.expiredDispatching(batchSize, staleAfter); + for (var lease : abandoned) { + RecipientDeliveryId recipientDeliveryId = lease.recipientDeliveryId(); var attemptsOfDelivery = attempts.attemptsOf(recipientDeliveryId); if (attemptsOfDelivery.isEmpty()) { // Crash before the attempt row: no provider call can have happened, so this is the one // safe requeue. Leaving it DISPATCHING stranded the delivery permanently. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } if (attemptsOfDelivery.stream().allMatch(attempt -> attempt.completedAt().isPresent())) { @@ -127,8 +133,7 @@ public final class LeaseRecoveryService { // the delivery, and dispatch derives the next step from exactly those facts — an accepted // submission stops as COMPLETED, an ambiguous one goes to reconciliation — so handing it // back to dispatch cannot send a second copy of something the provider accepted. - requeue(recipientDeliveryId); - handled++; + handled += requeue(lease); continue; } for (var attempt : attemptsOfDelivery) { @@ -141,25 +146,25 @@ public final class LeaseRecoveryService { // identically, so every abandoned attempt had to be reconciled. The pre-call row the // dispatcher writes never lands here: it records the start as unknown, because it is // still the stored state while the provider call runs. - requeue(recipientDeliveryId); + handled += requeue(lease); } else { // Outside any transaction: reconciliation asks the provider over the network and opens // its own boundaries for what it writes. reconciliation.reconcile(attempt.id()); + handled++; } - handled++; } } } return handled; } - private void requeue(RecipientDeliveryId recipientDeliveryId) { - transactions.inWrite( - () -> - deliveries.transition( - recipientDeliveryId, - RecipientDeliveryState.READY_TO_DISPATCH, - java.util.Optional.of(clock.instant()))); + /** Requeues the abandoned dispatch; 1 when it was still abandoned and moved, 0 otherwise. */ + private int requeue(RecipientLease abandoned) { + return transactions + .inWrite(() -> deliveries.requeueAbandoned(abandoned, clock.instant())) + .isPresent() + ? 1 + : 0; } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java index 101c9f35..c82fa5aa 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java @@ -49,4 +49,28 @@ public interface RecipientDeliveryStorePort { RecipientDeliveryState state, Optional nextDispatchAt, RecipientLease lease); + + /** + * Returns a job a dead worker left dispatching to the queue, only while it is still the abandoned + * dispatch recovery found. + * + *

One conditional write on the abandoned lease — same owner, same fence, still {@code + * DISPATCHING}, lease not extended past what recovery saw — rather than a read followed by an + * unconditional transition. Recovery runs on every instance, and between finding the job and + * writing it another instance may have requeued it and a new worker claimed it, or the original + * holder may have finished; an unconditional write put such a job back to {@code + * READY_TO_DISPATCH} and it was dispatched again. + * + *

The target state is the operation's meaning, as {@code EXPIRED} is {@code + * RecipientLeaseStorePort.expireOverdue}'s, so it is not an argument. What recovery decides is + * which operation applies: this one when no attempt may have reached the provider, the + * reconciliation path otherwise. + * + * @param abandoned the lease as {@link RecipientLeaseStorePort#expiredDispatching} returned it + * @param nextDispatchAt when the job becomes due again + * @return the requeued job, or empty when it is no longer that abandoned dispatch and nothing was + * written + */ + Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt); } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java index 0b9962a3..5660e4d3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java @@ -16,6 +16,10 @@ import java.util.Objects; * that no longer exists and updates nothing. It travels with the lease rather than being re-read, * because re-reading is the race. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param recipientDeliveryId the job this lease is for * @param owner the holder, including the process incarnation that claimed it * @param fence the claim generation; a lower fence has been superseded @@ -29,11 +33,11 @@ public record RecipientLease( Objects.requireNonNull(owner, "owner"); Objects.requireNonNull(leaseUntil, "leaseUntil"); if (owner.isBlank()) { - throw new IllegalArgumentException("owner"); + throw new IllegalStateException("a lease needs an owner"); } if (fence < 1) { // A zero fence is the value an unclaimed row holds; a lease can never legitimately carry it. - throw new IllegalArgumentException("fence"); + throw new IllegalStateException("fence must be >= 1, was " + fence); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java index 634fabd7..da251a5d 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java @@ -1,6 +1,5 @@ package dev.caskeleton.application.notification.platform.dispatch; -import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; import java.time.Duration; import java.util.List; import java.util.Optional; @@ -25,6 +24,9 @@ public interface RecipientLeaseStorePort { /** * Extend a lease that is still being worked on. * + *

The template's dispatcher never calls it: a dispatch runs inside the lease it claimed. It is + * here for a deployment whose own dispatch outlasts one lease. + * * @return the extended lease, or empty when this holder no longer owns the job */ Optional renew(RecipientLease lease, Duration leaseDuration); @@ -40,8 +42,24 @@ public interface RecipientLeaseStorePort { /** Whether this lease is still the current one, for checking before an external side effect. */ boolean stillHeld(RecipientLease lease); - /** Jobs left {@code DISPATCHING} by a worker that died, for reconciliation. */ - List expiredDispatching(int limit, Duration olderThan); + /** + * Jobs left {@code DISPATCHING} by a worker that died, for recovery. + * + *

Each comes back as the lease it was abandoned under — owner, fence and expiry as they stood + * when it was found. Recovery decides what to do from attempts it reads afterwards, and the lease + * is what lets its write say "only if this is still that abandoned dispatch": another instance's + * recovery, a new claim or the holder itself may have moved the job in between. + * + *

Only jobs a claim leased: an owner and a fence of at least one. A {@code DISPATCHING} job + * without them is no lease anyone holds — one left before the fence existed, or one moved there + * without a claim — and is not returned; offered, it would be a lease nobody can build, and the + * pass would fail on it every time. + * + * @param limit how many to return + * @param olderThan how long a lease must have been expired before the job counts as abandoned + * @return the abandoned leases, oldest expiry first + */ + List expiredDispatching(int limit, Duration olderThan); /** * Retire deliveries whose window closed before anyone claimed them. diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java new file mode 100644 index 00000000..2308ac3f --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; + +/** One reconciliation pass over the due jobs; its bounds are the use case's. */ +public record ReconcileOutstandingAttemptsCommand() implements Command {} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java new file mode 100644 index 00000000..4edf380d --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java @@ -0,0 +1,185 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +/** + * Asks the provider what happened to attempts whose outcome is unknown. + * + *

Reconciliation only ever ran when a lease expired and recovery walked past an incomplete + * attempt. An ambiguous submission whose worker exited cleanly — the common case, because a worker + * that records AMBIGUOUS and then finishes its shift has not crashed — was never asked about again. + * The delivery sat {@code RECONCILIATION_REQUIRED} indefinitely, which reads as a queue that + * stopped rather than as an outcome nobody knows. + * + *

An unsupported provider is not retried in a loop. Without a status-query capability the answer + * will not change, so the job is completed and the attempt stays visibly ambiguous for an operator + * — a busy loop against a capability that does not exist is how a background worker burns a + * connection pool while achieving nothing. + * + *

Every instance runs this worker, so the claim is a lease: the jobs a pass takes are this + * instance's until {@code claimLease} has passed, and no transaction is held while the provider is + * asked. A pass that outlives its lease may find a job taken over by another instance; the store + * then ignores this pass's write for it, so the job is asked about twice but recorded once. + * + *

A use case, not a worker of the notification adapter, which it used to be: when to ask again, + * when to give up and what each answer means are the platform's decisions, and the adapter's job is + * speaking provider protocols. The store writes each run in a write transaction of their own; the + * provider is asked with none open (round9 review 2026-11-12, RR9-05-06, RR9-05-08). + * + *

Not a bean. The notification composition's background workers construct it and keep it. + */ +@SystemInvoked( + reason = + "the notification background workers run it on their scheduler thread, which has no" + + " Authentication; an operator's run is ReconcileDeliveriesUseCase") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.WRITE_REPOSITORY, + // Each job asks a provider what happened. + externalOutboundAllowed = true) +public class ReconcileOutstandingAttemptsUseCase + implements CommandUseCase { + + private final ReconciliationJobStorePort jobs; + private final Function reconciler; + private final TransactionPort transactions; + private final Clock clock; + private final Duration retryBackoff; + private final Duration claimLease; + private final int batchSize; + private final int maxAttempts; + + /** + * Creates the use case. + * + *

The timing is derived here rather than in the assembly, because when to ask again and when + * to give up are this use case's decisions. A failed or crashed question waits one delivery lease + * before it is asked again. The claim lease reuses the delivery lease rather than adding a + * setting: one pass asks up to {@code batchSize} providers in turn, so a pass slower than the + * lease lets another instance re-ask its remaining jobs, and the claim fence keeps those answers + * recorded once. A job may ask once plus the delivery's additional attempts, the same budget a + * delivery has for being sent. + * + * @param jobs the outstanding questions + * @param reconciler asks one attempt's provider; it opens its own boundaries for what it writes + * @param transactions opens the write transaction around each job-store write + * @param clock the clock + * @param deliveryLease the delivery lease: the retry delay and how long one pass owns its jobs + * @param batchSize how many jobs one pass handles + * @param maxAdditionalAttempts how many more times a job may ask after its first question + * @throws IllegalStateException if a bound is not positive, which is the assembly's fault + */ + public ReconcileOutstandingAttemptsUseCase( + ReconciliationJobStorePort jobs, + Function reconciler, + TransactionPort transactions, + Clock clock, + Duration deliveryLease, + int batchSize, + int maxAdditionalAttempts) { + this.jobs = Objects.requireNonNull(jobs, "jobs"); + this.reconciler = Objects.requireNonNull(reconciler, "reconciler"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + this.clock = Objects.requireNonNull(clock, "clock"); + Objects.requireNonNull(deliveryLease, "deliveryLease"); + if (deliveryLease.isNegative() || deliveryLease.isZero()) { + throw new IllegalStateException("deliveryLease must be positive"); + } + if (batchSize < 1) { + throw new IllegalStateException("batchSize must be positive"); + } + if (maxAdditionalAttempts < 0) { + throw new IllegalStateException("maxAdditionalAttempts must not be negative"); + } + this.retryBackoff = deliveryLease; + this.claimLease = deliveryLease; + this.batchSize = batchSize; + this.maxAttempts = maxAdditionalAttempts + 1; + } + + /** + * Runs one pass. + * + * @return how many jobs reached a terminal decision + */ + @Override + public Integer handle(ReconcileOutstandingAttemptsCommand command) { + Objects.requireNonNull(command, "command"); + List due = + transactions.inWrite(() -> jobs.claimDue(batchSize, clock.instant(), claimLease)); + int settled = 0; + for (ReconciliationJob job : due) { + // One job's failure is not the pass's: a provider that is refusing connections would + // otherwise stop every other provider's jobs behind it. + try { + if (handle(job)) { + settled++; + } + } catch (RuntimeException failure) { + reschedule(job, failure.getClass().getSimpleName(), clock.instant().plus(retryBackoff)); + } + } + return settled; + } + + private boolean handle(ReconciliationJob job) { + if (job.attempts() >= maxAttempts) { + // Asked enough times. Completing the job leaves the attempt ambiguous and visible rather + // than asking forever; an outcome that has not arrived after this many tries is an + // operator's decision, not a scheduler's. + transactions.inWrite( + () -> { + jobs.reschedule(job, "MAX_ATTEMPTS", clock.instant().plus(retryBackoff)); + jobs.complete(job); + }); + return true; + } + ReconciliationResult result = reconciler.apply(job.attemptId()); + return switch (result) { + case ReconciliationResult.Confirmed ignored -> { + complete(job); + yield true; + } + case ReconciliationResult.Unsupported ignored -> { + // The capability does not exist; asking again cannot change that. + complete(job); + yield true; + } + case ReconciliationResult.StillUnknown stillUnknown -> { + reschedule(job, "STILL_UNKNOWN", stillUnknown.nextCheckAt()); + yield false; + } + case ReconciliationResult.Failed failed -> { + if (failed.retryable()) { + reschedule(job, failed.code(), clock.instant().plus(retryBackoff)); + yield false; + } + complete(job); + yield true; + } + }; + } + + private void complete(ReconciliationJob job) { + transactions.inWrite(() -> jobs.complete(job)); + } + + private void reschedule(ReconciliationJob job, String result, Instant nextCheckAt) { + transactions.inWrite(() -> jobs.reschedule(job, result, nextCheckAt)); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java index c8545b62..e0d3e401 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java @@ -24,6 +24,10 @@ import java.util.UUID; * the store's writes after the claim match on it, so an instance that held the job past its lease * cannot reschedule or remove it once another instance has claimed it since. * + *

A store builds it from a row, never from a request, so a value it refuses is a server fault: + * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would + * answer with a 400 (manual 06). + * * @param id the job identity * @param attemptId the attempt whose outcome is unknown * @param providerProfileId the profile that must answer, since capability is per profile @@ -55,10 +59,10 @@ public record ReconciliationJob( Objects.requireNonNull(createdAt, "createdAt"); Objects.requireNonNull(updatedAt, "updatedAt"); if (attempts < 0) { - throw new IllegalArgumentException("attempts must not be negative"); + throw new IllegalStateException("attempts must not be negative"); } if (claimFence < 0) { - throw new IllegalArgumentException("claimFence must not be negative"); + throw new IllegalStateException("claimFence must not be negative"); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java new file mode 100644 index 00000000..6ed19414 --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.command.Command; + +/** One lease-recovery pass; its bounds are the recovery service's. */ +public record RecoverAbandonedDeliveriesCommand() implements Command {} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java new file mode 100644 index 00000000..5d51446a --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java @@ -0,0 +1,47 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.security.SystemInvoked; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; +import java.util.Objects; + +/** + * Returns deliveries a dead worker left dispatching to the queue, or to reconciliation when an + * attempt may have reached the provider. + * + *

{@code RepositoryAccess.NONE}: the pass is {@link LeaseRecoveryService}'s, which opens a + * boundary per delivery it settles and none around the provider question. + * + *

Not a bean. The notification composition's background workers construct it and keep it. + */ +@SystemInvoked( + reason = + "the notification background workers run it on their scheduler thread, which has no" + + " Authentication") +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE, + // Recovery may ask a provider what happened to an attempt. + externalOutboundAllowed = true) +public class RecoverAbandonedDeliveriesUseCase + implements CommandUseCase { + + private final LeaseRecoveryService recovery; + + public RecoverAbandonedDeliveriesUseCase(LeaseRecoveryService recovery) { + this.recovery = Objects.requireNonNull(recovery, "recovery"); + } + + /** + * @return how many deliveries the pass settled + */ + @Override + public Integer handle(RecoverAbandonedDeliveriesCommand command) { + Objects.requireNonNull(command, "command"); + return recovery.recoverOnce(); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java new file mode 100644 index 00000000..b46a378e --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java @@ -0,0 +1,6 @@ +package dev.caskeleton.application.notification.platform.observation; + +import dev.caskeleton.application.query.Query; + +/** Asks how many deliveries are due and not yet sent; it has no parameters. */ +public record MeasureDispatchBacklogQuery() implements Query {} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java new file mode 100644 index 00000000..efce336c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java @@ -0,0 +1,52 @@ +package dev.caskeleton.application.notification.platform.observation; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.transaction.TransactionPort; +import dev.caskeleton.application.usecase.QueryUseCase; +import java.util.Objects; + +/** + * Measures the dispatch backlog the scheduler publishes as its queue-depth gauge on every poll. + * + *

The depth alone, one statement in a read transaction of its own: the full serving snapshot is + * several statements and is the health endpoint's. The scheduler read the serving-state port + * directly, with no use case and no read boundary in between (round9 review 2026-11-12, application + * structure). The notification composition constructs it for the scheduler; nothing else needs it. + */ +@UseCaseCapability( + transactionMode = TransactionMode.READ_ONLY, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.READ_REPOSITORY) +public class MeasureDispatchBacklogUseCase + implements QueryUseCase { + + private final NotificationServingStatePort servingState; + private final TransactionPort transactions; + + /** + * Creates the use case. + * + * @param servingState the serving-state read model + * @param transactions opens the read transaction around the measurement + */ + public MeasureDispatchBacklogUseCase( + NotificationServingStatePort servingState, TransactionPort transactions) { + this.servingState = Objects.requireNonNull(servingState, "servingState"); + this.transactions = Objects.requireNonNull(transactions, "transactions"); + } + + /** + * Measures the backlog. + * + * @param query the measurement request + * @return how many deliveries are due and not yet sent + */ + @Override + public Long handle(MeasureDispatchBacklogQuery query) { + Objects.requireNonNull(query, "query"); + return transactions.inRead(servingState::backlogDepth); + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java index ce48cf07..7ddece3f 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java @@ -12,7 +12,7 @@ import java.util.Objects; * being applied reported exactly the same numbers as an idle healthy one. Readiness that cannot * distinguish those two states is not readiness. * - * @param backlogDepth deliveries due now and not yet claimed + * @param backlogDepth deliveries a claim would take now: due, not expired, not under a live lease * @param oldestDueAge how long the oldest of them has been due * @param stuckLeases leases whose holder has not finished and whose expiry has passed * @param pendingProjections provider events stored and not yet applied diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java index 79a817aa..f6ae4f15 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java @@ -6,7 +6,25 @@ public interface NotificationServingStatePort { /** * Measure the backlog, the stuck leases and the projection and reconciliation lag. * + *

Several questions, for the health snapshot. A caller that needs only the queue depth asks + * {@link #backlogDepth()} instead: the scheduler reads the depth on every poll, and asking all of + * them that often ran four statements a poll, one of them over the whole provider-event table + * (round9 review 2026-11-12, RR9-01-14). + * * @return the current state */ NotificationServingState currentState(); + + /** + * How many deliveries a claim would take now — due, not expired, and not under a live lease: the + * {@link NotificationServingState#backlogDepth()} of {@link #currentState()}, measured alone. + * + *

The default takes it from the full snapshot, so an implementation that can only measure + * everything at once still answers; a store that can count the backlog by itself overrides it. + * + * @return the backlog depth + */ + default long backlogDepth() { + return currentState().backlogDepth(); + } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java index 9953dca2..56088cdf 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java @@ -19,14 +19,15 @@ public final class DefaultNotificationRetryPolicy implements NotificationRetryPo private final JitterSource jitter; private final Duration reconcileDelay; - public DefaultNotificationRetryPolicy( - RetryBackoff backoff, JitterSource jitter, Duration reconcileDelay) { - this.backoff = Objects.requireNonNull(backoff, "backoff"); + /** + * @param timing the backoff and the reconcile delay, already checked by {@link RetryTiming} + * @param jitter the randomness the backoff draws from + */ + public DefaultNotificationRetryPolicy(RetryTiming timing, JitterSource jitter) { + Objects.requireNonNull(timing, "timing"); + this.backoff = timing.backoff(); this.jitter = Objects.requireNonNull(jitter, "jitter"); - this.reconcileDelay = Objects.requireNonNull(reconcileDelay, "reconcileDelay"); - if (reconcileDelay.isNegative()) { - throw new IllegalArgumentException("reconcileDelay"); - } + this.reconcileDelay = timing.reconcileDelay(); } @Override diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java index 2e478427..fb8aa0b3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java @@ -9,7 +9,7 @@ public record EligibilityResult(boolean allowed, List reasonCodes) { public EligibilityResult { reasonCodes = List.copyOf(Objects.requireNonNull(reasonCodes, "reasonCodes")); if (!allowed && reasonCodes.isEmpty()) { - throw new IllegalArgumentException("a block must carry at least one reason code"); + throw new IllegalStateException("a block must carry at least one reason code"); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java index f6ba8ab3..ab282fbb 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java @@ -30,10 +30,11 @@ public record RetryBackoff(Duration base, Duration max) { Objects.requireNonNull(base, "base"); Objects.requireNonNull(max, "max"); if (base.isNegative() || base.isZero()) { - throw new IllegalArgumentException("base"); + throw new IllegalStateException("the backoff base must be positive, was " + base); } if (max.compareTo(base) < 0) { - throw new IllegalArgumentException("max must not be smaller than base"); + throw new IllegalStateException( + "the backoff max (" + max + ") must not be shorter than its base (" + base + ")"); } } @@ -42,7 +43,7 @@ public record RetryBackoff(Duration base, Duration max) { Objects.requireNonNull(retryAfter, "retryAfter"); Objects.requireNonNull(jitter, "jitter"); if (attemptNumber < 1) { - throw new IllegalArgumentException("attemptNumber"); + throw new IllegalStateException("attemptNumber must be >= 1, was " + attemptNumber); } Duration jittered = new Backoff(base, max).fullJitter(attemptNumber, asRandom(jitter)); Duration computed = jittered.compareTo(base) < 0 ? base : jittered; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java index 796b56da..88f519b6 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java @@ -10,7 +10,8 @@ public record RetryBudget(int maxAdditionalAttempts, int consumed) { public RetryBudget { if (maxAdditionalAttempts < 0 || consumed < 0) { - throw new IllegalArgumentException("budget values must not be negative"); + throw new IllegalStateException( + "budget values must not be negative, were " + maxAdditionalAttempts + " and " + consumed); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java index 500992bf..f2b87992 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java @@ -47,7 +47,7 @@ public record RetryContext( Objects.requireNonNull(estimatedDispatchDuration, "estimatedDispatchDuration"); Objects.requireNonNull(now, "now"); if (attemptNumber < 1) { - throw new IllegalArgumentException("attemptNumber"); + throw new IllegalStateException("attemptNumber must be >= 1, was " + attemptNumber); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java index f38b9657..4bd55a18 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java @@ -12,7 +12,7 @@ public sealed interface RetryDecision { public RetryAfter { Objects.requireNonNull(delay, "delay"); if (delay.isNegative()) { - throw new IllegalArgumentException("delay"); + throw new IllegalStateException("delay must not be negative, was " + delay); } } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java new file mode 100644 index 00000000..4f6769ac --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java @@ -0,0 +1,31 @@ +package dev.caskeleton.application.notification.platform.policy; + +import java.time.Duration; +import java.util.Objects; + +/** + * When a failed delivery is tried again: the backoff between attempts, and how long an ambiguous + * submission waits before the provider is asked what happened. + * + *

One value so its rules have one home. The retry settings build it from their keys and the + * policy reads it; neither restates what a valid timing is. + * + * @param backoff the delay between attempts + * @param reconcileDelay how long after an ambiguous submission reconciliation first asks; zero asks + * at once + */ +public record RetryTiming(RetryBackoff backoff, Duration reconcileDelay) { + + /** The shipped timing: the default backoff, reconciliation after a minute. */ + public static final RetryTiming DEFAULT = + new RetryTiming(RetryBackoff.DEFAULT, Duration.ofMinutes(1)); + + public RetryTiming { + Objects.requireNonNull(backoff, "backoff"); + Objects.requireNonNull(reconcileDelay, "reconcileDelay"); + if (reconcileDelay.isNegative()) { + throw new IllegalStateException( + "the reconcile delay must not be negative, was " + reconcileDelay); + } + } +} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java index a55a3ee6..8773a4c3 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java @@ -19,7 +19,7 @@ public record RouteCandidate( Objects.requireNonNull(contactPointId, "contactPointId"); Objects.requireNonNull(providerProfileId, "providerProfileId"); if (routeIndex < 0) { - throw new IllegalArgumentException("routeIndex"); + throw new IllegalStateException("routeIndex must be >= 0, was " + routeIndex); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java index 3185d341..7b5b98f1 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java @@ -27,7 +27,7 @@ public record RoutingContext( Objects.requireNonNull(expiresAt, "expiresAt"); Objects.requireNonNull(now, "now"); if (routeCursor < 0) { - throw new IllegalArgumentException("routeCursor"); + throw new IllegalStateException("routeCursor must be >= 0, was " + routeCursor); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java index 218f6cd4..bbee04ea 100644 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java @@ -16,10 +16,10 @@ public record RoutingDecision( Objects.requireNonNull(selected, "selected"); Objects.requireNonNull(reasonCode, "reasonCode"); if (reasonCode.isBlank()) { - throw new IllegalArgumentException("reasonCode"); + throw new IllegalStateException("a routing decision needs a reason code"); } if (reconciliationRequired && (retryAllowed || fallbackAllowed)) { - throw new IllegalArgumentException( + throw new IllegalStateException( "an attempt awaiting reconciliation may not also be retried or fallen back"); } } diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java new file mode 100644 index 00000000..a21e000c --- /dev/null +++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java @@ -0,0 +1,16 @@ +/** + * The notification platform's delivery policy: who may be sent to (eligibility: consent, + * preference, suppression), what counts as the same request twice (deduplication), and which route + * to try and when to retry, reroute or stop. + * + *

The values the platform, or a policy it calls, assembles ({@link RetryTiming}, {@link + * RetryBackoff}, {@link RetryBudget}, {@link RetryContext}, {@link RetryDecision.RetryAfter}, + * {@link RouteCandidate}, {@link RoutingContext}, {@link RoutingDecision}, {@link + * EligibilityResult}) are built from settings, the shipped defaults, its own counters and stored + * delivery state: what they refuse never comes from a request. A refusal is a server fault, {@link + * IllegalStateException}, not the {@code IllegalArgumentException} the web layer would answer with + * a 400 (manual 06). The records that carry what a request may send ({@link ConsentRecord}, {@link + * PreferenceRecord}, {@link RecipientIdentity}, {@link SuppressionEntry}) refuse with {@code + * IllegalArgumentException}. + */ +package dev.caskeleton.application.notification.platform.policy; diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java deleted file mode 100644 index 43fb9527..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java +++ /dev/null @@ -1,16 +0,0 @@ -package dev.caskeleton.application.notification.platform.provider; - -import java.util.List; -import java.util.concurrent.CompletionStage; - -/** - * Optional batching extension. - * - *

One transport call may carry many submissions, but the result list is positional: every - * recipient keeps its own attempt and its own outcome. - */ -public interface BatchNotificationProviderAdapter extends NotificationProviderAdapter { - - /** Submit a batch and return one result per input, in input order. */ - CompletionStage> submitBatch(List submissions); -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java deleted file mode 100644 index a5ba086c..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java +++ /dev/null @@ -1,10 +0,0 @@ -package dev.caskeleton.application.notification.platform.provider; - -import dev.caskeleton.application.notification.platform.api.CollapseSpec; - -/** Maps a platform collapse hint onto a provider-native key, validating provider limits first. */ -public interface CollapseCapability { - - /** Translate the spec, or fail when the key violates a provider limit. */ - ProviderCollapseMapping map(CollapseSpec spec); -} diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java deleted file mode 100644 index 121c940a..00000000 --- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java +++ /dev/null @@ -1,25 +0,0 @@ -package dev.caskeleton.application.notification.platform.provider; - -import java.util.Objects; - -/** - * Provider-native collapse key. - * - *

{@code guaranteesSingleUserNotification} is always false: collapse replaces an undelivered - * message inside the provider, it cannot recall one the device already showed. - */ -public record ProviderCollapseMapping(String nativeKey, String headerOrFieldName) { - - public ProviderCollapseMapping { - Objects.requireNonNull(nativeKey, "nativeKey"); - Objects.requireNonNull(headerOrFieldName, "headerOrFieldName"); - if (nativeKey.isBlank() || headerOrFieldName.isBlank()) { - throw new IllegalArgumentException("collapse mapping must not be blank"); - } - } - - /** Collapse never promises the user sees exactly one notification. */ - public boolean guaranteesSingleUserNotification() { - return false; - } -} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java index 80b5960d..fec63e1a 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java @@ -22,7 +22,7 @@ class AdminOperationExecutorTest { private final InMemoryAdminOperationStore store = new InMemoryAdminOperationStore(); @Test - @DisplayName("처음 claim한 operation만 claim을 쥔 채 한 쓰기 트랜잭션 안에서 부수 효과를 실행한다") + @DisplayName("처음 claim한 operation만 claim을 쥔 채 부수 효과를 실행하고, 트랜잭션은 열지 않는다") void claimedOperationIsTheOnlyOutcomeThatRunsTheSideEffect() { // given DirectTransactionPort transactions = new DirectTransactionPort(); @@ -31,7 +31,7 @@ class AdminOperationExecutorTest { // when AdminOperationResult result = - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE", sideEffect(sideEffects)); // then @@ -41,12 +41,10 @@ class AdminOperationExecutorTest { sideEffects.get(), store.claim("op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE"), transactions.committed())) - .as("the operation saves its own result; until then the id answers in-progress") - .containsExactly( - "op-1", - 1, - AdminOperationClaim.inProgress(), - List.of(DirectTransactionPort.Boundary.WRITE)); + .as( + "the operation saves its own result; until then the id answers in-progress. The" + + " use case owns the write transaction the claim runs in") + .containsExactly("op-1", 1, AdminOperationClaim.inProgress(), List.of()); } @Test @@ -62,7 +60,7 @@ class AdminOperationExecutorTest { // when AdminOperationResult replayed = - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE", sideEffect(sideEffects)); // then @@ -82,7 +80,7 @@ class AdminOperationExecutorTest { Throwable thrown = catchThrowable( () -> - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_REDRIVE", sideEffect(sideEffects))); // then @@ -103,7 +101,7 @@ class AdminOperationExecutorTest { Throwable thrown = catchThrowable( () -> - executor.execute( + executor.runClaimed( "op-1", FINGERPRINT, ACTOR, "ADMIN_REDRIVE", sideEffect(sideEffects))); // then diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java similarity index 65% rename from src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java index d5f48645..1d33fa71 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java @@ -7,17 +7,17 @@ import java.util.List; import java.util.UUID; import org.junit.jupiter.api.Test; -class AdminOperationCommandFingerprintTest { +class AdminOperationRequestFingerprintTest { - private final AdminOperationCommandFingerprint fingerprints = - new AdminOperationCommandFingerprint(); + private final AdminOperationRequestFingerprint fingerprints = + new AdminOperationRequestFingerprint(); @Test - void fingerprintIsStableSha256AndChangesWithCommandMeaning() { + void fingerprintIsStableSha256AndChangesWithRequestMeaning() { DeliveryAttemptId attempt = new DeliveryAttemptId(UUID.fromString("00000000-0000-0000-0000-1")); - RedriveCommand first = new RedriveCommand("op-1", attempt, "retry", true, false); - RedriveCommand same = new RedriveCommand("another-id", attempt, "retry", true, false); - RedriveCommand changed = new RedriveCommand("op-1", attempt, "different", true, false); + RedriveRequest first = new RedriveRequest("op-1", attempt, "retry", true, false); + RedriveRequest same = new RedriveRequest("another-id", attempt, "retry", true, false); + RedriveRequest changed = new RedriveRequest("op-1", attempt, "different", true, false); assertThat(fingerprints.of(first)).matches("[0-9a-f]{64}"); assertThat(fingerprints.of(same)).isEqualTo(fingerprints.of(first)); @@ -31,10 +31,10 @@ class AdminOperationCommandFingerprintTest { DeliveryAttemptId second = new DeliveryAttemptId(UUID.fromString("00000000-0000-0000-0000-000000000002")); - ReconcileCommand ordered = - new ReconcileCommand("op-1", List.of(first, second), "reconcile", false); - ReconcileCommand reversed = - new ReconcileCommand("op-2", List.of(second, first), "reconcile", false); + ReconcileRequest ordered = + new ReconcileRequest("op-1", List.of(first, second), "reconcile", false); + ReconcileRequest reversed = + new ReconcileRequest("op-2", List.of(second, first), "reconcile", false); assertThat(fingerprints.of(ordered)).isEqualTo(fingerprints.of(reversed)); } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java similarity index 56% rename from src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java index 2a7e002b..23681873 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java @@ -27,9 +27,9 @@ import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemor import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent; import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; -import dev.caskeleton.application.notification.platform.policy.SuppressionEntry; -import dev.caskeleton.application.notification.platform.policy.SuppressionId; -import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.notification.platform.policy.SuppressionReason; +import dev.caskeleton.application.notification.platform.policy.SuppressionScope; +import dev.caskeleton.application.notification.platform.policy.testkit.InMemorySuppressionStore; import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; @@ -42,10 +42,14 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -class NotificationAdminApplicationServiceTest { +/** + * The four operator-plane use cases, each its own entrypoint with its own permission (round9 review + * 2026-11-12, RR9-05-07: one service implemented all four behind an interface no use-case rule + * saw). + */ +class NotificationAdminUseCasesTest { private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-09-20T00:00:00Z"), ZoneOffset.UTC); @@ -56,15 +60,16 @@ class NotificationAdminApplicationServiceTest { @Test void providerStateMutationClaimsBeforeCallingTheRuntime() { Fixture fixture = new Fixture(); - SetProviderStateCommand command = - new SetProviderStateCommand( + ProviderStateRequest request = + new ProviderStateRequest( "op-1", new ProviderProfileId("ses-primary"), ProviderRuntimeState.DISABLED, "provider maintenance", false); - AdminOperationResult result = fixture.service.setProviderState(command, ACTOR); + AdminOperationResult result = + fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR)); assertThat(result.operationId()).isEqualTo("op-1"); assertThat(fixture.operations.claims).isEqualTo(1); @@ -77,18 +82,40 @@ class NotificationAdminApplicationServiceTest { .containsExactly("ADMIN_PROVIDER_STATE"); } + @Test + void providerStateClaimAndChangeCommitInTheUseCasesOneWriteTransaction() { + // given + Fixture fixture = new Fixture(); + ProviderStateRequest request = + new ProviderStateRequest( + "op-tx", + new ProviderProfileId("ses-primary"), + ProviderRuntimeState.DRAINING, + "provider maintenance", + false); + + // when + fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR)); + + // then + assertThat(List.of(fixture.transactions.committed(), fixture.operations.claimsInTransaction)) + .as("a claim outside the transaction would commit while the change rolled back") + .containsExactly(List.of(DirectTransactionPort.Boundary.WRITE), List.of(true)); + } + @Test void providerStateDryRunDoesNotClaimOrMutateRuntime() { Fixture fixture = new Fixture(); - SetProviderStateCommand command = - new SetProviderStateCommand( + ProviderStateRequest request = + new ProviderStateRequest( "op-dry", new ProviderProfileId("ses-primary"), ProviderRuntimeState.DRAINING, "inspect only", true); - AdminOperationResult result = fixture.service.setProviderState(command, ACTOR); + AdminOperationResult result = + fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR)); assertThat(result.dryRun()).isTrue(); assertThat(fixture.operations.claims).isZero(); @@ -97,7 +124,6 @@ class NotificationAdminApplicationServiceTest { } @Test - @DisplayName("tenant 범위 운영자는 다른 tenant의 attempt를 reconcile할 수 없다") void aTenantScopedOperatorCannotReconcileAnotherTenantsAttempt() { // given Fixture fixture = new Fixture(); @@ -109,11 +135,13 @@ class NotificationAdminApplicationServiceTest { "operator-b", Set.of(NotificationAdminAuthority.RECONCILE), Optional.of(new TenantId("tenant-b"))); - ReconcileCommand command = - new ReconcileCommand("op-r", List.of(foreign.attemptId()), "stuck attempt", false); + ReconcileRequest request = + new ReconcileRequest("op-r", List.of(foreign.attemptId()), "stuck attempt", false); // when - Throwable failure = catchThrowable(() -> fixture.service.reconcile(command, tenantB)); + Throwable failure = + catchThrowable( + () -> fixture.reconcile.handle(new ReconcileDeliveriesCommand(request, tenantB))); // then assertThat(failure) @@ -124,7 +152,6 @@ class NotificationAdminApplicationServiceTest { } @Test - @DisplayName("tenant 범위 운영자는 모든 tenant가 공유하는 provider 상태를 바꿀 수 없다") void aTenantScopedOperatorCannotChangeAGlobalProviderState() { // given Fixture fixture = new Fixture(); @@ -133,8 +160,8 @@ class NotificationAdminApplicationServiceTest { "operator-b", Set.of(NotificationAdminAuthority.PROVIDER_CONTROL), Optional.of(new TenantId("tenant-b"))); - SetProviderStateCommand command = - new SetProviderStateCommand( + ProviderStateRequest request = + new ProviderStateRequest( "op-p", new ProviderProfileId("ses-primary"), ProviderRuntimeState.DISABLED, @@ -142,7 +169,9 @@ class NotificationAdminApplicationServiceTest { false); // when - Throwable failure = catchThrowable(() -> fixture.service.setProviderState(command, tenantB)); + Throwable failure = + catchThrowable( + () -> fixture.providerState.handle(new ChangeProviderStateCommand(request, tenantB))); // then assertThat(failure) @@ -154,7 +183,6 @@ class NotificationAdminApplicationServiceTest { } @Test - @DisplayName("운영자 reconcile은 DB 트랜잭션을 연 채로 provider에 묻지 않는다") void reconcileAsksTheProviderWithNoTransactionOpen() { // given List transactionOpenDuringProviderCall = new ArrayList<>(); @@ -182,9 +210,10 @@ class NotificationAdminApplicationServiceTest { "operator-1", Set.of(NotificationAdminAuthority.RECONCILE), Optional.empty()); // when - fixture.service.reconcile( - new ReconcileCommand("op-rc", List.of(attempt.attemptId()), "stuck attempt", false), - operator); + fixture.reconcile.handle( + new ReconcileDeliveriesCommand( + new ReconcileRequest("op-rc", List.of(attempt.attemptId()), "stuck attempt", false), + operator)); // then assertThat(transactionOpenDuringProviderCall) @@ -192,6 +221,132 @@ class NotificationAdminApplicationServiceTest { .containsExactly(false); } + @Test + void redriveReturnsTheDeliveryToTheQueueUnderItsClaim() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED); + AdminActor operator = + new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty()); + RedriveRequest request = + new RedriveRequest("op-rd", attempt.attemptId(), "provider outage", true, false); + + // when + fixture.redrive.handle(new RedriveDeliveryCommand(request, operator)); + + // then + assertThat( + List.of( + fixture.recipients.find(attempt.recipientDeliveryId()).orElseThrow().state(), + fixture.operations.claimsInTransaction, + fixture.audit.events.stream().map(NotificationAuditEvent::action).toList())) + .containsExactly( + RecipientDeliveryState.READY_TO_DISPATCH, List.of(true), List.of("ADMIN_REDRIVE")); + } + + @Test + void aRedriveReadsItsAttemptInsideTheWriteTransaction() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED); + List openWhileReading = new ArrayList<>(); + fixture.attempts.beforeSnapshot(id -> openWhileReading.add(fixture.transactions.active())); + AdminActor operator = + new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty()); + + // when + fixture.redrive.handle( + new RedriveDeliveryCommand( + new RedriveRequest("op-rd-tx", attempt.attemptId(), "provider outage", true, false), + operator)); + + // then + assertThat(List.of(openWhileReading, fixture.transactions.committed())) + .as("the checks read the attempt the claim then redrives, in one transaction") + .containsExactly(List.of(true), List.of(DirectTransactionPort.Boundary.WRITE)); + } + + @Test + void aReconcileDryRunChecksItsTenantsInsideAReadTransaction() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.DISPATCHING); + List openWhileReading = new ArrayList<>(); + fixture.attempts.beforeSnapshot(id -> openWhileReading.add(fixture.transactions.active())); + AdminActor operator = + new AdminActor( + "operator-1", Set.of(NotificationAdminAuthority.RECONCILE), Optional.empty()); + + // when + fixture.reconcile.handle( + new ReconcileDeliveriesCommand( + new ReconcileRequest("op-rc-tx", List.of(attempt.attemptId()), "stuck attempt", true), + operator)); + + // then + assertThat(List.of(openWhileReading, fixture.transactions.committed())) + .containsExactly(List.of(true), List.of(DirectTransactionPort.Boundary.READ)); + } + + @Test + void aRedriveDryRunReadsInOneReadTransactionAndClaimsNothing() { + // given + Fixture fixture = new Fixture(); + DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a")); + AttemptSnapshots.store( + fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED); + AdminActor operator = + new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty()); + RedriveRequest request = + new RedriveRequest("op-rd-dry", attempt.attemptId(), "provider outage", true, true); + + // when + fixture.redrive.handle(new RedriveDeliveryCommand(request, operator)); + + // then + assertThat(List.of(fixture.transactions.committed(), fixture.operations.claimsInTransaction)) + .containsExactly(List.of(DirectTransactionPort.Boundary.READ), List.of()); + } + + @Test + void anOperatorSuppressionIsStoredUnderItsClaim() { + // given + Fixture fixture = new Fixture(); + TenantId tenant = new TenantId("tenant-a"); + AdminActor operator = + new AdminActor( + "operator-1", Set.of(NotificationAdminAuthority.SUPPRESS), Optional.of(tenant)); + SuppressionRequest request = + new SuppressionRequest( + "op-s", + tenant, + SuppressionScope.CONTACT_POINT, + SuppressionReason.ADMIN_BLOCK, + "c".repeat(64), + Optional.empty(), + false, + "reported abuse", + false); + + // when + AdminOperationResult result = + fixture.suppression.handle(new ChangeSuppressionCommand(request, operator)); + + // then + assertThat( + List.of( + result.affected(), + fixture.suppressions.activeFor(tenant, "c".repeat(64), CLOCK.instant()).size(), + fixture.operations.claimsInTransaction)) + .containsExactly(1, 1, List.of(true)); + } + private static DeliveryAttemptSnapshot attemptOf(TenantId tenantId) { return new DeliveryAttemptSnapshot( new DeliveryAttemptId(UUID.fromString("00000000-0000-7000-8000-000000000011")), @@ -225,7 +380,11 @@ class NotificationAdminApplicationServiceTest { private final RecordingOperations operations = new RecordingOperations(); private final RecordingAudit audit = new RecordingAudit(); private final DirectTransactionPort transactions = new DirectTransactionPort(); - private final NotificationAdminApplicationService service; + private final InMemorySuppressionStore suppressions = new InMemorySuppressionStore(); + private final ChangeProviderStateUseCase providerState; + private final ReconcileDeliveriesUseCase reconcile; + private final RedriveDeliveryUseCase redrive; + private final ChangeSuppressionUseCase suppression; private Fixture() { this(null); @@ -235,24 +394,42 @@ class NotificationAdminApplicationServiceTest { * @param gateway the provider the reconciliation asks, or null for one that supports nothing */ private Fixture(ReconciliationGatewayPort gateway) { + operations.transactions = transactions; ReconciliationService reconciliation = gateway == null ? PlatformCollaborators.reconciliation(attempts, recipients, CLOCK) : PlatformCollaborators.reconciliation(attempts, recipients, gateway, CLOCK); - service = - new NotificationAdminApplicationService( + providerState = + new ChangeProviderStateUseCase( + new AdminAuthorizationGuard(), runtime, operations, audit, transactions, CLOCK); + redrive = + new RedriveDeliveryUseCase( new AdminAuthorizationGuard(), new DuplicateRiskGuard(), attempts, recipients, - reconciliation, - new UnusedSuppressionStore(), - runtime, + operations, + audit, + transactions, + CLOCK); + suppression = + new ChangeSuppressionUseCase( + new AdminAuthorizationGuard(), + suppressions, operations, audit, transactions, new PlatformFakes.SequentialIds(), CLOCK); + reconcile = + new ReconcileDeliveriesUseCase( + new AdminAuthorizationGuard(), + attempts, + reconciliation, + operations, + audit, + transactions, + CLOCK); } } @@ -277,6 +454,8 @@ class NotificationAdminApplicationServiceTest { } private static final class RecordingOperations implements AdminOperationStorePort { + private final List claimsInTransaction = new ArrayList<>(); + private DirectTransactionPort transactions; private int claims; private int saved; @@ -293,9 +472,10 @@ class NotificationAdminApplicationServiceTest { @Override public AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action) { + String operationId, String requestFingerprint, AdminActor actor, String action) { claims++; - assertThat(commandFingerprint).matches("[0-9a-f]{64}"); + claimsInTransaction.add(transactions.active()); + assertThat(requestFingerprint).matches("[0-9a-f]{64}"); return AdminOperationClaim.claimed(); } } @@ -308,25 +488,4 @@ class NotificationAdminApplicationServiceTest { events.add(event); } } - - private static final class UnusedSuppressionStore implements SuppressionStorePort { - @Override - public List activeFor( - dev.caskeleton.application.notification.platform.api.TenantId tenantId, - String targetFingerprint, - Instant at) { - return List.of(); - } - - @Override - public SuppressionEntry upsert(SuppressionEntry entry) { - throw new UnsupportedOperationException(); - } - - @Override - public Optional remove( - dev.caskeleton.application.notification.platform.api.TenantId tenantId, SuppressionId id) { - return Optional.empty(); - } - } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java new file mode 100644 index 00000000..9014a229 --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java @@ -0,0 +1,90 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.api.TenantId; +import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; +import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's claim, run through its {@code @SystemInvoked} use case. + * + *

The claim used to run with no transaction of the scheduler's own: the store's statement was + * its whole boundary. It now commits in the use case's write transaction. + */ +class ClaimDueDeliveriesUseCaseTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + private static final Duration LEASE = Duration.ofMinutes(2); + + private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore(); + private final InMemoryRecipientQueue queue = + new InMemoryRecipientQueue(requests, Clock.fixed(NOW, ZoneOffset.UTC)); + private final DirectTransactionPort transactions = new DirectTransactionPort(); + private final ClaimDueDeliveriesUseCase useCase = + new ClaimDueDeliveriesUseCase(queue, transactions); + + @Test + @DisplayName("due 발송은 쓰기 트랜잭션 하나에서 이 worker의 lease로 claim된다") + void aDueDeliveryIsClaimedForTheWorkerInOneWriteTransaction() { + // given + RecipientDeliveryId due = dueJob(); + + // when + List claimed = + useCase.handle(new ClaimDueDeliveriesCommand("worker-1", 10, LEASE)); + + // then + assertThat( + List.of( + claimed.stream().map(RecipientLease::recipientDeliveryId).toList(), + claimed.stream().map(RecipientLease::owner).toList(), + claimed.stream().map(RecipientLease::leaseUntil).toList(), + transactions.committed())) + .containsExactly( + List.of(due), + List.of("worker-1"), + List.of(NOW.plus(LEASE)), + List.of(DirectTransactionPort.Boundary.WRITE)); + } + + @Test + @DisplayName("한 번에 0개를 claim하라는 command는 조립의 잘못으로 실패한다") + void aClaimOfNothingIsTheAssemblysFault() { + // when + Throwable refused = catchThrowable(() -> new ClaimDueDeliveriesCommand("worker-1", 0, LEASE)); + + // then + assertThat(refused) + .isInstanceOf(IllegalStateException.class) + .hasMessage("limit must be positive"); + } + + private RecipientDeliveryId dueJob() { + var request = + NotificationRows.request(new TenantId("tenant-a"), "idem-" + UUID.randomUUID(), NOW); + var job = + NotificationRows.job( + request, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW.minusSeconds(1)), + Optional.empty(), + NOW); + requests.insert(request, List.of(job)); + return job.id(); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java new file mode 100644 index 00000000..47d87614 --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java @@ -0,0 +1,78 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryContactPointStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryDeliveryAttemptStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; +import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy; +import dev.caskeleton.application.notification.platform.policy.RetryDecision; +import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * The scheduler's dispatch of one claimed delivery, run through its {@code @SystemInvoked} use + * case. The use case holds no boundary of its own; the dispatch service opens each one. + */ +class DispatchClaimedDeliveryUseCaseTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); + + private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore(); + private final InMemoryRecipientQueue recipients = new InMemoryRecipientQueue(requests, CLOCK); + private final InMemoryDeliveryAttemptStore attempts = new InMemoryDeliveryAttemptStore(requests); + private final DirectTransactionPort transactions = new DirectTransactionPort(); + private final PlatformFakes.UnreachableGateway gateway = new PlatformFakes.UnreachableGateway(); + + private final DispatchClaimedDeliveryUseCase useCase = + new DispatchClaimedDeliveryUseCase( + new NotificationDispatchService( + requests, + recipients, + attempts, + new DispatchPipeline( + new InMemoryContactPointStore(CLOCK), + NotificationEligibilityPolicy.permitAll(), + new PlatformFakes.RecordingMetrics()), + new RoutingDecisionEngine(), + context -> new RetryDecision.Stop("unused"), + new DeliveryAttemptFactory(new PlatformFakes.SequentialIds()), + PlatformCollaborators.outcomeRecorder(attempts, recipients), + gateway, + command -> { + throw new AssertionError("a lease whose delivery is gone renders nothing"); + }, + recipients, + transactions, + CLOCK, + 3, + Duration.ofSeconds(5))); + + @Test + @DisplayName("delivery가 사라진 lease는 dispatch 서비스의 읽기·쓰기 경계에서 풀리고 provider에 닿지 않는다") + void aLeaseWhoseDeliveryIsGoneIsReleasedWithoutReachingTheProvider() { + // given + RecipientLease gone = + new RecipientLease( + new RecipientDeliveryId(UUID.randomUUID()), "worker-a", 1L, NOW.plusSeconds(30)); + + // when + useCase.handle(new DispatchClaimedDeliveryCommand(gone)); + + // then + assertThat(List.of(transactions.committed(), gateway.calls())) + .containsExactly( + List.of(DirectTransactionPort.Boundary.READ, DirectTransactionPort.Boundary.WRITE), 0); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java index 7ff29358..ed508150 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java @@ -186,6 +186,40 @@ class LeaseRecoveryServiceTest { assertThat(recovery.recoverOnce()).isEqualTo(3); } + @Test + void aDeliveryItsHolderFinishedAfterRecoveryFoundItIsNotRequeued() { + // given + RecipientDeliveryId id = jobOf(NOW, Optional.empty()); + RecipientLease lease = queue.claim("worker-a", 10, Duration.ofSeconds(30)).getFirst(); + workerDied(); + // The holder completes the job between recovery finding it abandoned and recovery writing it + // — the window another instance or a slow worker uses in production. + attempts.beforeAttemptsOf( + delivery -> { + var unused = + queue.transitionHeldBy( + delivery, RecipientDeliveryState.COMPLETED, Optional.empty(), lease); + }); + LeaseRecoveryService racing = + new LeaseRecoveryService( + queue, + attempts, + queue, + reconciled::add, + new DirectTransactionPort(), + CLOCK, + Duration.ofMinutes(1), + 10); + + // when + racing.recoverOnce(); + + // then + assertThat(stateOf(id)) + .as("the holder was only slow; recovery's requeue must not reopen what it finished") + .isEqualTo(RecipientDeliveryState.COMPLETED); + } + private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence notStarted() { return dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java index e4e13f18..b5ab61a3 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java @@ -32,8 +32,8 @@ import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemor import dev.caskeleton.application.notification.platform.policy.DefaultNotificationRetryPolicy; import dev.caskeleton.application.notification.platform.policy.JitterSource; import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy; -import dev.caskeleton.application.notification.platform.policy.RetryBackoff; import dev.caskeleton.application.notification.platform.policy.RetryDecision; +import dev.caskeleton.application.notification.platform.policy.RetryTiming; import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine; import dev.caskeleton.application.notification.platform.port.in.SubmitNotificationCommand; import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities; @@ -70,7 +70,7 @@ class NotificationDispatchServiceTest { new InMemoryRecipientQueue(requests, Clock.fixed(NOW, ZoneOffset.UTC)); InMemoryDeliveryAttemptStore attempts = new InMemoryDeliveryAttemptStore(requests); DirectTransactionPort transactions = new DirectTransactionPort(); - FailingGateway gateway = new FailingGateway(); + PlatformFakes.UnreachableGateway gateway = new PlatformFakes.UnreachableGateway(); NotificationDispatchService service = new NotificationDispatchService( requests, @@ -104,7 +104,7 @@ class NotificationDispatchServiceTest { // The WRITE boundary after the read is the release; the queue has no row to show it on. assertThat(transactions.committed()).containsExactly(Boundary.READ, Boundary.WRITE); - assertThat(gateway.calls).isZero(); + assertThat(gateway.calls()).isZero(); } @Test @@ -500,8 +500,7 @@ class NotificationDispatchServiceTest { NotificationEligibilityPolicy.permitAll(), new PlatformFakes.RecordingMetrics()), new RoutingDecisionEngine(), - new DefaultNotificationRetryPolicy( - RetryBackoff.DEFAULT, JitterSource.none(), Duration.ofMinutes(1)), + new DefaultNotificationRetryPolicy(RetryTiming.DEFAULT, JitterSource.none()), new DeliveryAttemptFactory(new PlatformFakes.SequentialIds()), PlatformCollaborators.outcomeRecorder(attempts, recipients), gateway, @@ -631,26 +630,4 @@ class NotificationDispatchServiceTest { throw new AssertionError("nothing may be submitted through a profile that is gone"); } } - - private static final class FailingGateway implements ProviderDispatchGatewayPort { - private int calls; - - @Override - public ProviderProfileSnapshot profile(ProviderProfileId profileId) { - calls++; - throw new AssertionError("gateway profile must not be reached"); - } - - @Override - public ProviderRuntimeState state(ProviderProfileId profileId) { - calls++; - throw new AssertionError("gateway state must not be reached"); - } - - @Override - public ProviderSubmissionResult submit(ProviderSubmission submission) { - calls++; - throw new AssertionError("provider submit must not be reached"); - } - } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java index a04d1631..7fc4750f 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java @@ -14,6 +14,10 @@ import dev.caskeleton.application.notification.platform.api.routing.DeliveryStra import dev.caskeleton.application.notification.platform.api.routing.ExplicitChannel; import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; import dev.caskeleton.application.notification.platform.policy.RouteCandidate; +import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot; +import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; +import dev.caskeleton.application.notification.platform.provider.ProviderSubmission; +import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -35,6 +39,35 @@ public final class PlatformFakes { private PlatformFakes() {} + /** A provider gateway the dispatch under test must not reach; it counts what reached it. */ + public static final class UnreachableGateway implements ProviderDispatchGatewayPort { + + private int calls; + + /** How many calls reached the gateway. */ + public int calls() { + return calls; + } + + @Override + public ProviderProfileSnapshot profile(ProviderProfileId profileId) { + calls++; + throw new AssertionError("gateway profile must not be reached"); + } + + @Override + public ProviderRuntimeState state(ProviderProfileId profileId) { + calls++; + throw new AssertionError("gateway state must not be reached"); + } + + @Override + public ProviderSubmissionResult submit(ProviderSubmission submission) { + calls++; + throw new AssertionError("provider submit must not be reached"); + } + } + /** Metrics sink that keeps what it was told, so tag discipline stays assertable. */ public static final class RecordingMetrics implements NotificationMetricsPort { diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java similarity index 50% rename from src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java index ad25e7bd..2cbe1abb 100644 --- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java @@ -1,4 +1,4 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.application.notification.platform.dispatch; import static org.assertj.core.api.Assertions.assertThat; @@ -6,14 +6,14 @@ import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; import dev.caskeleton.application.notification.platform.api.NormalizedEventType; import dev.caskeleton.application.notification.platform.api.NormalizedProviderEvent; import dev.caskeleton.application.notification.platform.api.ProviderProfileId; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob; -import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort; import dev.caskeleton.application.notification.platform.provider.ReconciliationResult; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -29,117 +29,190 @@ import org.junit.jupiter.api.Test; * in {@code RECONCILIATION_REQUIRED} permanently. The job table for exactly this has existed since * V3 with no entity, no store and no worker. */ -class ReconciliationJobWorkerTest { +class ReconcileOutstandingAttemptsUseCaseTest { private static final Instant NOW = Instant.parse("2026-08-15T09:00:00Z"); private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); - private static final Duration CLAIM_LEASE = Duration.ofMinutes(2); + private static final Duration DELIVERY_LEASE = Duration.ofMinutes(2); + private static final int MAX_ADDITIONAL_ATTEMPTS = 2; private final RecordingJobs jobs = new RecordingJobs(); - private final Map answers = new java.util.HashMap<>(); + private final Map answers = new HashMap<>(); + private final List asked = new ArrayList<>(); + private final DirectTransactionPort transactions = new DirectTransactionPort(); - private final ReconciliationJobWorker worker = - new ReconciliationJobWorker( - jobs, answers::get, CLOCK, Duration.ofMinutes(5), CLAIM_LEASE, 10, 3); + private final ReconcileOutstandingAttemptsUseCase useCase = + new ReconcileOutstandingAttemptsUseCase( + jobs, + attemptId -> { + asked.add(attemptId); + return answers.get(attemptId); + }, + transactions, + CLOCK, + DELIVERY_LEASE, + 10, + MAX_ADDITIONAL_ATTEMPTS); @Test - @DisplayName("a confirmed outcome completes the job") void aConfirmedOutcomeCompletesTheJob() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Confirmed(event())); - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed).containsExactly(job.id()); + // when + int settled = useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(settled, jobs.completed)).containsExactly(1, List.of(job.id())); } @Test - @DisplayName("an unsupported provider is not asked in a loop") - void anUnsupportedProviderIsNotAskedInALoop() { + @DisplayName("상태 조회가 없는 provider는 다시 묻지 않고 job을 끝낸다. 답이 바뀌지 않는다") + void anUnsupportedProviderIsCompletedRatherThanAskedAgain() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Unsupported()); - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed) - .as( - "without a status-query capability the answer will not change; the attempt stays" - + " visibly ambiguous for an operator instead") - .containsExactly(job.id()); - assertThat(jobs.rescheduled).isEmpty(); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(jobs.completed, jobs.rescheduled)) + .as("the attempt stays visibly ambiguous for an operator instead") + .containsExactly(List.of(job.id()), List.of()); } @Test - @DisplayName("a still-unknown outcome is asked again at the provider's own next time") - void aStillUnknownOutcomeIsAskedAgain() { + void aStillUnknownOutcomeIsAskedAgainAtTheProvidersOwnNextTime() { + // given ReconciliationJob job = due(0); Instant providerSuggested = NOW.plus(Duration.ofHours(2)); answers.put(job.attemptId(), new ReconciliationResult.StillUnknown(providerSuggested)); - assertThat(worker.reconcileOnce()).isZero(); - assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), providerSuggested)); - assertThat(jobs.completed).isEmpty(); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(jobs.rescheduled, jobs.completed)) + .containsExactly(List.of(Map.entry(job.id(), providerSuggested)), List.of()); } @Test - @DisplayName("a retryable failure is asked again after the backoff") - void aRetryableFailureIsAskedAgain() { + void aRetryableFailureIsAskedAgainOneDeliveryLeaseLater() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Failed("PROVIDER_TIMEOUT", true)); - assertThat(worker.reconcileOnce()).isZero(); - assertThat(jobs.rescheduled) - .containsExactly(Map.entry(job.id(), NOW.plus(Duration.ofMinutes(5)))); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), NOW.plus(DELIVERY_LEASE))); } @Test - @DisplayName("a non-retryable failure completes the job") void aNonRetryableFailureCompletesTheJob() { + // given ReconciliationJob job = due(0); answers.put(job.attemptId(), new ReconciliationResult.Failed("ATTEMPT_UNKNOWN", false)); - assertThat(worker.reconcileOnce()).isEqualTo(1); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then assertThat(jobs.completed).containsExactly(job.id()); } @Test - @DisplayName("a job that has asked enough times stops asking") - void aJobThatHasAskedEnoughTimesStopsAsking() { - ReconciliationJob job = due(3); + @DisplayName("첫 질문과 추가 시도 2번, 모두 3번을 물은 job은 더 묻지 않고 끝난다") + void aJobThatAskedOnceMoreThanItsAdditionalAttemptsStopsAsking() { + // given + ReconciliationJob job = due(MAX_ADDITIONAL_ATTEMPTS + 1); - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed) + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(asked, jobs.completed)) .as("an outcome that has not arrived after this many tries is an operator's decision") - .containsExactly(job.id()); - assertThat(answers).isEmpty(); + .containsExactly(List.of(), List.of(job.id())); } @Test - @DisplayName("one job throwing does not stop the rest of the pass") - void oneJobThrowingDoesNotStopTheRest() { + @DisplayName("2번 물은 job은 추가 시도가 남아 한 번 더 묻는다") + void aJobWithAnAdditionalAttemptLeftIsAskedAgain() { + // given + ReconciliationJob job = due(MAX_ADDITIONAL_ATTEMPTS); + answers.put(job.attemptId(), new ReconciliationResult.Unsupported()); + + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(asked).containsExactly(job.attemptId()); + } + + @Test + void oneJobThrowingDoesNotStopTheRestOfThePass() { + // given + // The failing job has no answer registered, so the reconciler returns null and the switch + // throws — which is exactly the shape of a provider refusing connections. ReconciliationJob failing = due(0); ReconciliationJob healthy = due(0); answers.put(healthy.attemptId(), new ReconciliationResult.Confirmed(event())); - // The failing job has no answer registered, so the reconciler returns null and the switch - // throws — which is exactly the shape of a provider refusing connections. - assertThat(worker.reconcileOnce()).isEqualTo(1); - assertThat(jobs.completed).containsExactly(healthy.id()); - assertThat(jobs.rescheduled).hasSize(1); - assertThat(jobs.rescheduled.get(0).getKey()).isEqualTo(failing.id()); + // when + useCase.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(jobs.completed, jobs.rescheduled.stream().map(Map.Entry::getKey).toList())) + .containsExactly(List.of(healthy.id()), List.of(failing.id())); } @Test - @DisplayName("한 번의 pass는 설정된 lease 동안 소유하도록 job을 claim한다") - void aPassClaimsItsJobsForTheConfiguredLease() { + @DisplayName("한 번의 pass는 delivery lease 동안 소유하도록 job을 claim한다") + void aPassClaimsItsJobsForOneDeliveryLease() { // given due(0); // when - worker.reconcileOnce(); + useCase.handle(new ReconcileOutstandingAttemptsCommand()); // then assertThat(jobs.claimLeases) .as("without a lease the claim ends with the statement and every instance takes the job") - .containsExactly(CLAIM_LEASE); + .containsExactly(DELIVERY_LEASE); + } + + @Test + @DisplayName("provider에는 트랜잭션 없이 묻고, job 기록은 그 앞뒤의 쓰기 트랜잭션에서 한다") + void theProviderIsAskedWithNoTransactionOpenAndTheJobIsRecordedInOne() { + // given + due(0); + List openWhileAsking = new ArrayList<>(); + ReconcileOutstandingAttemptsUseCase observing = + new ReconcileOutstandingAttemptsUseCase( + jobs, + attemptId -> { + openWhileAsking.add(transactions.active()); + return new ReconciliationResult.Unsupported(); + }, + transactions, + CLOCK, + DELIVERY_LEASE, + 10, + MAX_ADDITIONAL_ATTEMPTS); + + // when + observing.handle(new ReconcileOutstandingAttemptsCommand()); + + // then + assertThat(List.of(openWhileAsking, transactions.committed())) + .as("the claim and the completion each commit; the provider call holds no connection") + .containsExactly( + List.of(false), + List.of(DirectTransactionPort.Boundary.WRITE, DirectTransactionPort.Boundary.WRITE)); } private ReconciliationJob due(int attempts) { diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java new file mode 100644 index 00000000..0065ffec --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java @@ -0,0 +1,77 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.api.TenantId; +import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState; +import dev.caskeleton.application.notification.platform.dispatch.testkit.ContractClock; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryDeliveryAttemptStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore; +import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue; +import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows; +import dev.caskeleton.application.transaction.testkit.DirectTransactionPort; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** The background workers' lease recovery, run through its {@code @SystemInvoked} use case. */ +class RecoverAbandonedDeliveriesUseCaseTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore(); + private final ContractClock queueClock = new ContractClock(NOW); + private final InMemoryRecipientQueue queue = new InMemoryRecipientQueue(requests, queueClock); + private final List reconciled = new ArrayList<>(); + + private final RecoverAbandonedDeliveriesUseCase useCase = + new RecoverAbandonedDeliveriesUseCase( + new LeaseRecoveryService( + queue, + new InMemoryDeliveryAttemptStore(requests), + queue, + reconciled::add, + new DirectTransactionPort(), + Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofMinutes(1), + 10)); + + @Test + @DisplayName("worker가 attempt를 남기기 전에 죽은 delivery는 한 pass에서 다시 보낼 차례로 돌아간다") + void aDeliveryAbandonedBeforeItsAttemptIsRequeuedInOnePass() { + // given + RecipientDeliveryId abandoned = abandonedJob(); + queueClock.moveTo(NOW.plus(Duration.ofMinutes(5))); + + // when + int recovered = useCase.handle(new RecoverAbandonedDeliveriesCommand()); + + // then + assertThat(List.of(recovered, queue.find(abandoned).orElseThrow().state(), reconciled)) + .containsExactly(1, RecipientDeliveryState.READY_TO_DISPATCH, List.of()); + } + + private RecipientDeliveryId abandonedJob() { + var request = + NotificationRows.request(new TenantId("tenant-a"), "idem-" + UUID.randomUUID(), NOW); + var job = + NotificationRows.job( + request, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW), + Optional.empty(), + NOW); + requests.insert(request, List.of(job)); + queue.claim("worker-a", 10, Duration.ofSeconds(30)); + return job.id(); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java new file mode 100644 index 00000000..dd33283f --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java @@ -0,0 +1,76 @@ +package dev.caskeleton.application.notification.platform.dispatch; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.admin.AdminOperationClaim; +import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId; +import dev.caskeleton.application.notification.platform.api.ProviderProfileId; +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * The leases, jobs and claims a notification store builds from its rows refuse a bad value as each + * record says. + */ +class StoreBuiltRecordsTest { + + private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z"); + + static Stream refusedValues() { + RecipientDeliveryId delivery = new RecipientDeliveryId(UUID.randomUUID()); + return Stream.of( + Arguments.of( + "owner 없는 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, " ", 1, NOW), + "a lease needs an owner"), + Arguments.of( + "fence 0인 수신자 lease", + (ThrowingCallable) () -> new RecipientLease(delivery, "worker-1", 0, NOW), + "fence must be >= 1, was 0"), + Arguments.of( + "시도 수가 음수인 맞추기 job", + (ThrowingCallable) () -> job(-1, 0), + "attempts must not be negative"), + Arguments.of( + "fence가 음수인 맞추기 job", + (ThrowingCallable) () -> job(0, -1), + "claimFence must not be negative"), + Arguments.of( + "결과 없는 운영자 재생", + (ThrowingCallable) + () -> new AdminOperationClaim(AdminOperationClaim.Outcome.REPLAY, Optional.empty()), + "a replay must carry the result it is replaying")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueAStoreCannotHaveBuiltIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message); + } + + private static ReconciliationJob job(int attempts, long claimFence) { + return new ReconciliationJob( + UUID.randomUUID(), + new DeliveryAttemptId(UUID.randomUUID()), + new ProviderProfileId("email-main"), + NOW, + attempts, + Optional.empty(), + NOW, + NOW, + claimFence); + } +} diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java index ecc03283..904b0a0a 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/DeduplicationAndCollapseTest.java @@ -2,14 +2,10 @@ package dev.caskeleton.application.notification.platform.policy; import static org.assertj.core.api.Assertions.assertThat; -import dev.caskeleton.application.notification.platform.api.CollapseScope; -import dev.caskeleton.application.notification.platform.api.CollapseSpec; import dev.caskeleton.application.notification.platform.api.DeduplicationAction; import dev.caskeleton.application.notification.platform.api.DeduplicationSpec; import dev.caskeleton.application.notification.platform.api.NotificationId; import dev.caskeleton.application.notification.platform.policy.testkit.InMemoryDeduplicationStore; -import dev.caskeleton.application.notification.platform.provider.CollapseCapability; -import dev.caskeleton.application.notification.platform.provider.ProviderCollapseMapping; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -55,14 +51,4 @@ class DeduplicationAndCollapseTest { assertThat(other.duplicate()).isFalse(); } - - @Test - void collapseDoesNotReportExactlyOnceOrDelivered() { - CollapseCapability capability = - spec -> new ProviderCollapseMapping(spec.key(), "apns-collapse-id"); - var mapped = capability.map(new CollapseSpec("feed:user-1", CollapseScope.RECIPIENT)); - - assertThat(mapped.guaranteesSingleUserNotification()).isFalse(); - assertThat(mapped.nativeKey()).isEqualTo("feed:user-1"); - } } diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java index d757f954..d774edc0 100644 --- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/NotificationRetryPolicyTest.java @@ -17,7 +17,7 @@ class NotificationRetryPolicyTest { private static final Instant NOW = Instant.parse("2026-08-10T00:00:00Z"); private final NotificationRetryPolicy policy = - new DefaultNotificationRetryPolicy(RetryBackoff.DEFAULT, bound -> 0L, Duration.ofMinutes(1)); + new DefaultNotificationRetryPolicy(RetryTiming.DEFAULT, bound -> 0L); @Test void authenticationFailureStopsInsteadOfRetryingEveryMessage() { diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java new file mode 100644 index 00000000..dffa456b --- /dev/null +++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/policy/PolicyValuesTest.java @@ -0,0 +1,137 @@ +package dev.caskeleton.application.notification.platform.policy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId; +import dev.caskeleton.application.notification.platform.api.delivery.AttemptConfirmation; +import dev.caskeleton.application.notification.platform.api.delivery.SubmissionOutcome; +import dev.caskeleton.application.notification.platform.api.error.FailureCategory; +import dev.caskeleton.application.notification.platform.api.routing.Channel; +import dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence; +import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * The retry and routing values refuse what no settings or stored state could hold, as the package + * says. + */ +class PolicyValuesTest { + + static Stream refusedValues() { + return Stream.of( + Arguments.of( + "backoff 하한 0", + (ThrowingCallable) + () -> + new RetryTiming( + new RetryBackoff(Duration.ZERO, Duration.ofMinutes(5)), + Duration.ofMinutes(1)), + "the backoff base must be positive, was PT0S"), + Arguments.of( + "backoff 상한이 하한보다 짧음", + (ThrowingCallable) + () -> + new RetryTiming( + new RetryBackoff(Duration.ofMinutes(1), Duration.ofSeconds(30)), + Duration.ofMinutes(1)), + "the backoff max (PT30S) must not be shorter than its base (PT1M)"), + Arguments.of( + "음수 reconcile 지연", + (ThrowingCallable) () -> new RetryTiming(RetryBackoff.DEFAULT, Duration.ofSeconds(-1)), + "the reconcile delay must not be negative, was PT-1S"), + Arguments.of( + "attempt 0의 delay", + (ThrowingCallable) + () -> RetryBackoff.DEFAULT.delay(0, Optional.empty(), JitterSource.none()), + "attemptNumber must be >= 1, was 0"), + Arguments.of( + "attempt 0의 retry context", + (ThrowingCallable) () -> contextForAttempt(0), + "attemptNumber must be >= 1, was 0"), + Arguments.of( + "소비가 음수인 budget", + (ThrowingCallable) () -> new RetryBudget(2, -1), + "budget values must not be negative, were 2 and -1"), + Arguments.of( + "음수 delay의 재시도 결정", + (ThrowingCallable) () -> new RetryDecision.RetryAfter(Duration.ofSeconds(-1)), + "delay must not be negative, was PT-1S"), + Arguments.of( + "음수 위치의 route 후보", + (ThrowingCallable) () -> PolicyFixtures.route(-1, Channel.PUSH, true), + "routeIndex must be >= 0, was -1"), + Arguments.of( + "음수 cursor의 routing context", + (ThrowingCallable) + () -> + new RoutingContext( + new RecipientDeliveryId(UUID.randomUUID()), + PolicyFixtures.pushThenSms(), + -1, + SubmissionOutcome.NOT_SUBMITTED, + Optional.empty(), + false, + Optional.empty(), + Instant.parse("2026-11-13T09:00:00Z")), + "routeCursor must be >= 0, was -1"), + Arguments.of( + "사유 코드 없는 routing 결정", + (ThrowingCallable) () -> RoutingDecision.reconcile(" ", false), + "a routing decision needs a reason code"), + Arguments.of( + "맞추기와 재시도를 함께 고른 routing 결정", + (ThrowingCallable) + () -> new RoutingDecision(Optional.empty(), true, false, true, false, "BOTH"), + "an attempt awaiting reconciliation may not also be retried or fallen back"), + Arguments.of( + "맞추기와 fallback을 함께 고른 routing 결정", + (ThrowingCallable) + () -> new RoutingDecision(Optional.empty(), false, true, true, false, "BOTH"), + "an attempt awaiting reconciliation may not also be retried or fallen back"), + Arguments.of( + "사유 없는 수신 자격 차단", + (ThrowingCallable) () -> EligibilityResult.blocked(), + "a block must carry at least one reason code")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("refusedValues") + void aValueNoSettingsOrStoredStateCouldHoldIsAServerFault( + String name, ThrowingCallable build, String message) { + // when + Throwable refused = catchThrowable(build); + + // then + assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessage(message); + } + + private static RetryContext contextForAttempt(int attemptNumber) { + Instant now = Instant.parse("2026-11-13T09:00:00Z"); + return new RetryContext( + FailureCategory.TRANSIENT_PROVIDER, + AttemptConfirmation.REJECTED, + SubmissionOutcome.NOT_SUBMITTED, + ProviderExecutionEvidence.notStarted(), + false, + false, + true, + false, + false, + Optional.of(now.plus(Duration.ofHours(1))), + Optional.empty(), + attemptNumber, + new RetryBudget(2, 0), + ProviderRuntimeState.HEALTHY, + Duration.ofSeconds(1), + now); + } +} diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java index 8c222dcf..a2a6e80d 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/admin/testkit/InMemoryAdminOperationStore.java @@ -49,17 +49,17 @@ public final class InMemoryAdminOperationStore implements AdminOperationStorePor @Override public synchronized AdminOperationClaim claim( - String operationId, String commandFingerprint, AdminActor actor, String action) { + String operationId, String requestFingerprint, AdminActor actor, String action) { Objects.requireNonNull(operationId, "operationId"); - Objects.requireNonNull(commandFingerprint, "commandFingerprint"); + Objects.requireNonNull(requestFingerprint, "requestFingerprint"); Objects.requireNonNull(actor, "actor"); Objects.requireNonNull(action, "action"); Row existing = rows.get(operationId); if (existing == null) { - rows.put(operationId, new Row(commandFingerprint, actor.actorRef(), action)); + rows.put(operationId, new Row(requestFingerprint, actor.actorRef(), action)); return AdminOperationClaim.claimed(); } - if (!existing.fingerprint.equals(commandFingerprint)) { + if (!existing.fingerprint.equals(requestFingerprint)) { return AdminOperationClaim.conflict(); } return existing diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java index ae257085..748f00fb 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryDeliveryAttemptStore.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; /** * The fake of the attempt store and the two read models built on it, held to {@link @@ -47,6 +48,10 @@ public final class InMemoryDeliveryAttemptStore /** Stored attempts, in insertion order, as their rows read back. */ private final Map rows = new LinkedHashMap<>(); + private volatile Consumer beforeAttemptsOf = id -> {}; + + private volatile Consumer beforeSnapshot = id -> {}; + private final Map requestIdHashes = new LinkedHashMap<>(); private final Map engagement = new LinkedHashMap<>(); private final Map suppression = new LinkedHashMap<>(); @@ -140,7 +145,23 @@ public final class InMemoryDeliveryAttemptStore } @Override - public synchronized List attemptsOf( + public List attemptsOf(RecipientDeliveryId recipientDeliveryId) { + // Outside the monitor: the hook writes to the recipient queue, as another worker would. + beforeAttemptsOf.accept(recipientDeliveryId); + return attemptsFor(recipientDeliveryId); + } + + /** + * Runs {@code hook} before every {@link #attemptsOf} read, for a test that needs another worker + * to act between a caller finding a delivery and reading its attempts. + * + * @param hook what happens, given the delivery being read + */ + public void beforeAttemptsOf(Consumer hook) { + beforeAttemptsOf = Objects.requireNonNull(hook, "hook"); + } + + private synchronized List attemptsFor( RecipientDeliveryId recipientDeliveryId) { return rows.values().stream() .filter(row -> row.recipientDeliveryId().equals(recipientDeliveryId)) @@ -150,12 +171,23 @@ public final class InMemoryDeliveryAttemptStore @Override public synchronized Optional snapshot(DeliveryAttemptId id) { + beforeSnapshot.accept(id); return find(id).flatMap(this::toSnapshot); } + /** + * Runs {@code hook} before every {@link #snapshot} read, for a test that needs to see what was + * open when a caller read an attempt. + * + * @param hook what to run, given the attempt being read + */ + public void beforeSnapshot(Consumer hook) { + beforeSnapshot = Objects.requireNonNull(hook, "hook"); + } + @Override public synchronized int nextAttemptNo(RecipientDeliveryId recipientDeliveryId) { - return attemptsOf(recipientDeliveryId).stream() + return attemptsFor(recipientDeliveryId).stream() .mapToInt(DeliveryAttemptRecord::attemptNo) .max() .orElse(0) diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java index 89517a47..7714a522 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/InMemoryRecipientQueue.java @@ -160,7 +160,7 @@ public final class InMemoryRecipientQueue } @Override - public List expiredDispatching(int limit, Duration olderThan) { + public List expiredDispatching(int limit, Duration olderThan) { Objects.requireNonNull(olderThan, "olderThan"); if (limit < 1) { throw new IllegalArgumentException("limit"); @@ -170,9 +170,18 @@ public final class InMemoryRecipientQueue return jobs() .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) .filter(job -> job.leaseUntil().filter(until -> until.isBefore(threshold)).isPresent()) + // Only what a claim wrote: an owner and a fence. The JPA store skips the rest the same + // way. + .filter(job -> job.leaseOwner().isPresent() && fences.getOrDefault(job.id(), 0L) > 0) .sorted(Comparator.comparing((RecipientDeliveryRecord job) -> job.leaseUntil().get())) .limit(limit) - .map(RecipientDeliveryRecord::id) + .map( + job -> + new RecipientLease( + job.id(), + job.leaseOwner().orElseThrow(), + fences.getOrDefault(job.id(), 0L), + job.leaseUntil().orElseThrow())) .toList(); } } @@ -294,6 +303,35 @@ public final class InMemoryRecipientQueue } } + @Override + public Optional requeueAbandoned( + RecipientLease abandoned, Instant nextDispatchAt) { + Objects.requireNonNull(abandoned, "abandoned"); + Objects.requireNonNull(nextDispatchAt, "nextDispatchAt"); + synchronized (requests) { + return heldBy(abandoned) + .filter(job -> job.state() == RecipientDeliveryState.DISPATCHING) + .filter( + job -> + job.leaseUntil() + .filter(until -> !until.isAfter(abandoned.leaseUntil())) + .isPresent()) + .map( + job -> { + RecipientDeliveryRecord moved = + copy( + job, + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(nextDispatchAt), + job.leaseOwner(), + job.leaseUntil(), + clock.instant()); + requests.replace(moved); + return moved; + }); + } + } + // ---- rows ------------------------------------------------------------------------------------- private Stream jobs() { diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java index 7ceefe82..5ec3dead 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientDeliveryStoreContract.java @@ -244,6 +244,84 @@ public abstract class RecipientDeliveryStoreContract { List.of(Optional.empty(), Optional.empty()), RecipientDeliveryState.DISPATCHING, 0); } + @Test + void anAbandonedDispatchIsRequeued() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat( + requeued.map(stored -> List.of(stored.id(), stored.state(), stored.nextDispatchAt()))) + .contains( + List.of( + job.id(), + RecipientDeliveryState.READY_TO_DISPATCH, + Optional.of(NOW.plus(LEASE).plusSeconds(1)))); + } + + @Test + void requeueOfAJobItsHolderFinishedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease lease = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction( + () -> + deliveries() + .transitionHeldBy( + job.id(), RecipientDeliveryState.COMPLETED, Optional.empty(), lease)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(lease, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.COMPLETED); + } + + @Test + void requeueOfAJobAnotherClaimTookWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease abandoned = claim(); + clock.moveTo(NOW.plus(LEASE).plusSeconds(1)); + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + claim(); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(abandoned, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .as("the second recovery found the same abandoned lease; the new claim raised the fence") + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + + @Test + void requeueOfAJobWhoseLeaseWasRenewedWritesNothing() { + // given + RecipientDeliveryRecord job = givenAQueuedJob(); + RecipientLease found = claim(); + clock.moveTo(NOW.plus(LEASE).minusSeconds(1)); + inTransaction(() -> leases().renew(found, LEASE)); + + // when + Optional requeued = + inTransaction(() -> deliveries().requeueAbandoned(found, clock.instant())); + + // then + assertThat(List.of(requeued, deliveries().find(job.id()).orElseThrow().state())) + .containsExactly(Optional.empty(), RecipientDeliveryState.DISPATCHING); + } + private RecipientDeliveryRecord givenAQueuedJob() { NotificationRequestRecord request = NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), NOW); diff --git a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java index 0a97a7c1..4ddfb077 100644 --- a/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java +++ b/src/application/notification/src/testFixtures/java/dev/caskeleton/application/notification/platform/dispatch/testkit/RecipientLeaseStoreContract.java @@ -278,25 +278,77 @@ public abstract class RecipientLeaseStoreContract { } @Test - @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만 복구 대상으로 나온다") + @DisplayName("lease가 기준보다 오래 전에 끝난 DISPATCHING 작업만, 버려질 때의 lease로 복구 대상에 나온다") void onlyJobsWhoseLeaseEndedLongEnoughAgoAreAbandoned() { // given - RecipientDeliveryId abandoned = given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); - claim("worker-a"); + given(RecipientDeliveryState.READY_TO_DISPATCH, NOW, null); + RecipientLease abandoned = claim("worker-a").getFirst(); clock.moveTo(NOW.plus(LEASE).plusSeconds(30)); given(RecipientDeliveryState.READY_TO_DISPATCH, NOW.plus(LEASE), null); claim("worker-b"); clock.moveTo(NOW.plus(LEASE).plus(Duration.ofMinutes(2))); // when - List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); // then assertThat(expired) - .as("a job whose lease is still running belongs to a live worker") + .as( + "a job whose lease is still running belongs to a live worker; the abandoned one comes" + + " back as the lease it was abandoned under") .containsExactly(abandoned); } + @Test + @DisplayName("claim이 쓰지 않은 DISPATCHING 작업(owner는 있고 fence가 없음)은 복구 대상에 나오지 않는다") + void aDispatchingJobNoClaimWroteIsNotOfferedForRecovery() { + // given + // The shape a row left dispatching before the fence existed has: an owner and a lease expiry + // written by hand, and the fence column's default. + NotificationRequestRecord request = + NotificationRows.request(TENANT, "idem-" + UUID.randomUUID(), clock.instant()); + RecipientDeliveryRecord job = + NotificationRows.job( + request, + RecipientDeliveryState.DISPATCHING, + Optional.of(NOW), + Optional.empty(), + clock.instant()); + RecipientDeliveryRecord leasedByHand = + new RecipientDeliveryRecord( + job.id(), + job.notificationId(), + job.tenantId(), + job.recipientRef(), + job.locale(), + job.timeZone(), + job.routingPlan(), + job.routeCursor(), + job.state(), + job.submissionOutcome(), + job.deliveryOutcome(), + job.evidenceLevel(), + job.ambiguousAttemptExists(), + job.duplicateRisk(), + job.nextDispatchAt(), + job.expiresAt(), + Optional.of("legacy-worker"), + Optional.of(NOW.minus(Duration.ofHours(1))), + job.attemptCount(), + job.lastFailureCategory(), + job.createdAt(), + job.updatedAt()); + inTransaction(() -> requests().insert(request, List.of(leasedByHand))); + + // when + List expired = leases().expiredDispatching(10, Duration.ofMinutes(1)); + + // then + assertThat(expired) + .as("a lease the application cannot build would fail every recovery pass after it") + .isEmpty(); + } + private RecipientDeliveryId given( RecipientDeliveryState state, Instant dueAt, Instant expiresAt) { NotificationRequestRecord request = diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java index 17516da9..613d3910 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/AdapterBoundaryArchitectureTest.java @@ -16,6 +16,7 @@ import com.tngtech.archunit.lang.ArchCondition; import com.tngtech.archunit.lang.ArchRule; import com.tngtech.archunit.lang.ConditionEvents; import com.tngtech.archunit.lang.SimpleConditionEvent; +import dev.caskeleton.application.usecase.UseCase; import java.util.Map; import java.util.Set; @@ -36,6 +37,27 @@ import java.util.Set; @AnalyzeClasses(packages = "dev.caskeleton", importOptions = ProductionClassImportOption.class) class AdapterBoundaryArchitectureTest { + /** + * The application types behind a use case: services, orchestrators and coordinators that are not + * use cases themselves, interfaces included. Interfaces were left out until 2026-11-13, so a + * controller injecting the fileserver's {@code CleanupService} — the service the cleanup worker's + * {@code @SystemInvoked} use case wraps — passed (round9 review 2026-11-12, application + * structure). + */ + static final DescribedPredicate ARE_APPLICATION_SERVICES = + JavaClass.Predicates.resideInAPackage("..application..") + .and(DescribedPredicate.not(JavaClass.Predicates.resideInAPackage("..adapter.."))) + .and( + DescribedPredicate.describe( + "named *Service, *Orchestrator or *Coordinator and not a use case", + type -> + type.getName().startsWith("dev.caskeleton.") + && !type.isAssignableTo(UseCase.class) + && (type.getSimpleName().endsWith("Service") + || type.getSimpleName().endsWith("Orchestrator") + || type.getSimpleName().endsWith("Coordinator")))) + .as("application services, orchestrators and coordinators that are not use cases"); + /** * A transport must depend on an inbound port, never on the class behind it. * @@ -50,26 +72,15 @@ class AdapterBoundaryArchitectureTest { static final ArchRule INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES = noClasses() .that() - .resideInAPackage("dev.caskeleton.adapter.inbound..") + .resideInAPackage("..adapter.inbound..") .should() - .dependOnClassesThat( - new com.tngtech.archunit.base.DescribedPredicate<>( - "are concrete application orchestration services") { - @Override - public boolean test(com.tngtech.archunit.core.domain.JavaClass candidate) { - return candidate.getPackageName().startsWith("dev.caskeleton.application.") - && !candidate.isInterface() - && !candidate - .getModifiers() - .contains(com.tngtech.archunit.core.domain.JavaModifier.ABSTRACT) - && (candidate.getSimpleName().endsWith("Service") - || candidate.getSimpleName().endsWith("Orchestrator")); - } - }) + .dependOnClassesThat(ARE_APPLICATION_SERVICES) .because( "INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES: an entrypoint reached " - + "through a concrete class bypasses the use-case fitness gate, which is what " - + "declares transaction mode, repository access and outbound permission") + + "through a service, orchestrator or coordinator — class or interface — bypasses " + + "the use-case fitness gate, which is what declares transaction mode, repository " + + "access, outbound permission and the required permission. A service a " + + "@SystemInvoked use case wraps writes with no permission check at all") .allowEmptyShould(true); /** @@ -451,6 +462,57 @@ class AdapterBoundaryArchitectureTest { + " or a query port (AGENTS.md HARD-STOP 2)") .allowEmptyShould(true); + /** + * The server-assigned id ports: minting an id is part of a use case, so the transport that hands + * a submission to one does not mint the id the use case will store. {@code *IdFactory} is the + * name new code uses; {@code *IdGeneratorPort} and {@code *IdentifierPort} are the names the + * notification and fileserver platforms already had (application-core AGENTS 이름 규칙). + */ + static final DescribedPredicate ARE_ID_FACTORIES = + JavaClass.Predicates.resideInAnyPackage("..domain..", "..application..") + .and( + DescribedPredicate.describe( + "named *IdFactory, *IdGeneratorPort or *IdentifierPort", + type -> + type.getName().startsWith("dev.caskeleton.") + && (type.getSimpleName().endsWith("IdFactory") + || type.getSimpleName().endsWith("IdGeneratorPort") + || type.getSimpleName().endsWith("IdentifierPort")))) + .as("id factories"); + + /** + * The packages a user request comes in through: every inbound package but the system-driven ones + * ({@code ApplicationBoundaryArchitectureTest.SYSTEM_DRIVEN_INBOUND_PACKAGES}), so a transport + * added later is covered without being listed here. + */ + static final DescribedPredicate ARE_IN_USER_TRANSPORT_PACKAGES = + JavaClass.Predicates.resideInAPackage("..adapter.inbound..") + .and( + DescribedPredicate.not( + JavaClass.Predicates.resideInAnyPackage( + ApplicationBoundaryArchitectureTest.SYSTEM_DRIVEN_INBOUND_PACKAGES))) + .as("classes in a user-transport package (an inbound package that is not system-driven)"); + + /** + * The rule above for every class of a user transport, not only its endpoints: a stream bridge, a + * data fetcher helper or a filter that holds a store, query or id port reaches persistence with + * no use case, no capability and no read boundary in between. A web streaming bridge read the + * live event log this way, and the durable operation controller minted operation ids (round9 + * review 2026-11-12, RR9-05-10, RR9-05-13). + */ + @ArchTest + static final ArchRule USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS = + noClasses() + .that(ARE_IN_USER_TRANSPORT_PACKAGES) + .should() + .dependOnClassesThat(ARE_PERSISTENCE_PORTS.or(ARE_ID_FACTORIES)) + .as( + "USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS: a user transport reads and" + + " writes through a use case; it holds no domain repository port, application" + + " store or query port, and no id factory or id generator port (AGENTS.md HARD-STOP 2," + + " application-core AGENTS 「읽기·query 경로」)") + .allowEmptyShould(true); + /** * A controller is an inbound adapter, so it lives in one. One filed in the composition root or a * composition module is reached by no transport rule that names inbound packages (app {@code diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java index 2bd3a769..04037b92 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ApplicationBoundaryArchitectureTest.java @@ -454,7 +454,7 @@ class ApplicationBoundaryArchitectureTest { * inbound package — web, GraphQL, gRPC, WebSocket, and any transport added later — is a way a * user request comes in. A new system-driven inbound family joins this list, nowhere else. */ - private static final String[] SYSTEM_DRIVEN_INBOUND_PACKAGES = { + static final String[] SYSTEM_DRIVEN_INBOUND_PACKAGES = { "..adapter.inbound.messaging..", "..adapter.inbound.scheduler.." }; diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java index 3c5fc54f..e321b0e3 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchRuleDiscoveryContractTest.java @@ -100,6 +100,7 @@ class ArchRuleDiscoveryContractTest { "CONTROLLER_REQUEST_MAPPINGS_FOLLOW_AIP122", "DOMAIN_CONTEXT_PROPAGATION_PRIMITIVES_STAY_UNSHIPPED", "ENDPOINTS_DO_NOT_HOLD_PERSISTENCE_PORTS", + "USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS", "DOMAIN_ENTITIES_DO_NOT_CARRY_AUDIT_FIELDS", "DOMAIN_EVENTS_ARE_RECORDS", "DOMAIN_EVENTS_ARE_TRANSPORT_FREE", diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java index de48d4ea..262030d3 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/ArchitectureViolationFixtureTest.java @@ -7,13 +7,17 @@ import com.tngtech.archunit.core.domain.JavaClasses; import com.tngtech.archunit.core.importer.ClassFileImporter; import com.tngtech.archunit.lang.EvaluationResult; import dev.caskeleton.application.architecture.violations.ApplicationDiagnosticFrameworkViolation; +import dev.caskeleton.application.realtime.GetReplayWindowUseCase; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.LibraryMessageHandlerSystemCallerFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.RecordSystemCallerFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.SystemCallerHandlerFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.SystemCallerHandlerRegistrationFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.messaging.SystemCallerWithNestedValueTypesFixture; import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web.PingControllerFixture; +import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web.UseCaseNamedServiceHoldingControllerFixture; +import dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web.UseCaseReadingStreamBridgeFixture; import dev.caskeleton.bootstrap.architecture.allowed.application.CleanProjectionQueryPort; +import dev.caskeleton.bootstrap.architecture.allowed.application.ReportExportService; import dev.caskeleton.bootstrap.architecture.allowed.configuration.FrameCountingInterceptorFixture; import dev.caskeleton.bootstrap.architecture.allowed.configuration.FrameInterceptorConfigurationFixture; import dev.caskeleton.bootstrap.architecture.allowed.configuration.OrderHandlerFixture; @@ -36,14 +40,20 @@ import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.schedule import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.scheduler.ScheduledWorkFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.scheduler.SystemCallerWithNestedEndpointFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.scheduler.SystemJobLauncherFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.CoordinatorHoldingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.DomainRepositoryHoldingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.MessageHandlerInjectingControllerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.ServiceInterfaceHoldingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemCallerControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemCallerInvokingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemCallerRelayingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemInvokedCallingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.SystemJobInterfaceInvokingControllerFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller.UnmarkedRelayCallingControllerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.IdGeneratingReceiptHandlerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.IdMintingReceiptHandlerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.IdentifierHoldingReceiptHandlerFixture; +import dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream.QueryPortHoldingStreamBridgeFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.outbound.RawExternalResponseFixture; import dev.caskeleton.bootstrap.architecture.violations.adapter.outbound.messaging.GenericTypeLeakingAdapterFixture; import dev.caskeleton.bootstrap.architecture.violations.application.BulkWriteWithoutWriteAccessUseCase; @@ -56,6 +66,12 @@ import dev.caskeleton.bootstrap.architecture.violations.application.OutboundWith import dev.caskeleton.bootstrap.architecture.violations.application.RawLeakQueryPort; import dev.caskeleton.bootstrap.architecture.violations.application.ReadOnlyRepositoryWriteUseCase; import dev.caskeleton.bootstrap.architecture.violations.application.SystemInvokedFixtureUseCase; +import dev.caskeleton.bootstrap.architecture.violations.application.history.EventHistoryQueryPort; +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdFactory; +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdGeneratorPort; +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdentifierPort; +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.MaintenanceService; +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.WriterSlotCoordinator; import dev.caskeleton.bootstrap.architecture.violations.configuration.RoutesBesideASystemCallerConfigurationFixture; import dev.caskeleton.bootstrap.architecture.violations.configuration.RunNowHandlerFunctionConfigurationFixture; import dev.caskeleton.bootstrap.architecture.violations.configuration.RunNowRouteConfigurationFixture; @@ -73,9 +89,12 @@ import dev.caskeleton.bootstrap.architecture.violations.persistence.flyway.Refer import dev.caskeleton.bootstrap.architecture.violations.security.AuthorizationManagerReachingSystemCallerFixture; import dev.caskeleton.bootstrap.architecture.violations.streaming.JakartaWebSocketEndpointFixture; import dev.caskeleton.bootstrap.architecture.violations.streaming.SpringWebSocketHandlerFixture; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; /** @@ -1811,6 +1830,187 @@ class ArchitectureViolationFixtureTest { .contains(OrderRepository.class.getName())); } + @Test + @DisplayName("controller가 아닌 web stream bridge도 query 포트를 직접 들면 사용자 전송 포트 규칙에 걸린다") + void aWebStreamBridgeHoldingAQueryPortIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(QueryPortHoldingStreamBridgeFixture.class, EventHistoryQueryPort.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(QueryPortHoldingStreamBridgeFixture.class.getName()) + .contains(EventHistoryQueryPort.class.getName())); + } + + @Test + @DisplayName("domain repository 포트를 든 controller는 사용자 전송 포트 규칙에도 걸린다") + void aControllerHoldingADomainRepositoryPortIsCaughtByTheTransportRuleToo() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(DomainRepositoryHoldingControllerFixture.class, OrderRepository.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(DomainRepositoryHoldingControllerFixture.class.getName())); + } + + @Test + @DisplayName("web handler가 id factory를 들고 id를 직접 만들면 사용자 전송 포트 규칙에 걸린다") + void aWebHandlerMintingIdsIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(IdMintingReceiptHandlerFixture.class, ReceiptIdFactory.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(IdMintingReceiptHandlerFixture.class.getName()) + .contains(ReceiptIdFactory.class.getName())); + } + + static Stream idPortsNamedOtherThanFactory() { + return Stream.of( + Arguments.of(IdGeneratingReceiptHandlerFixture.class, ReceiptIdGeneratorPort.class), + Arguments.of(IdentifierHoldingReceiptHandlerFixture.class, ReceiptIdentifierPort.class)); + } + + @ParameterizedTest(name = "{1}") + @MethodSource("idPortsNamedOtherThanFactory") + void aWebHandlerHoldingAnIdPortOfAnotherNameIsCaught(Class handler, Class port) { + // given + JavaClasses fixtureOnly = new ClassFileImporter().importClasses(handler, port); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> assertThat(detail).contains(handler.getName()).contains(port.getName())); + } + + @Test + void aStreamBridgeReadingThroughAQueryUseCasePassesTheTransportPortRule() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(UseCaseReadingStreamBridgeFixture.class, GetReplayWindowUseCase.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()).isEmpty(); + } + + @Test + @DisplayName("controller가 시스템 유스케이스가 감싸는 서비스 interface를 들면 inbound 서비스 규칙에 걸린다") + void aControllerHoldingAServiceInterfaceIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses( + ServiceInterfaceHoldingControllerFixture.class, MaintenanceService.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(ServiceInterfaceHoldingControllerFixture.class.getName()) + .contains(MaintenanceService.class.getName())); + } + + @Test + void aControllerHoldingACoordinatorIsCaught() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(CoordinatorHoldingControllerFixture.class, WriterSlotCoordinator.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()) + .anySatisfy( + detail -> + assertThat(detail) + .contains(CoordinatorHoldingControllerFixture.class.getName()) + .contains(WriterSlotCoordinator.class.getName())); + } + + @Test + void aControllerHoldingAUseCaseNamedServicePassesTheInboundServiceRule() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses( + UseCaseNamedServiceHoldingControllerFixture.class, ReportExportService.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()).isEmpty(); + } + + @Test + void aStreamBridgeReadingThroughAQueryUseCasePassesTheInboundServiceRule() { + // given + JavaClasses fixtureOnly = + new ClassFileImporter() + .importClasses(UseCaseReadingStreamBridgeFixture.class, GetReplayWindowUseCase.class); + + // when + EvaluationResult result = + AdapterBoundaryArchitectureTest.INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES + .evaluate(fixtureOnly); + + // then + assertThat(result.getFailureReport().getDetails()).isEmpty(); + } + @Test @DisplayName("포트를 들지 않고 inbound 패키지에 있는 controller는 endpoint 포트 규칙과 배치 규칙을 통과한다") void anInboundControllerHoldingNoPersistencePortPasses() { diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java index 4170e12f..1bfd3980 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/FileserverUseCaseContractDeviationTest.java @@ -8,7 +8,6 @@ import com.tngtech.archunit.core.importer.ClassFileImporter; import dev.caskeleton.application.usecase.UseCase; import java.util.List; import java.util.Set; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; /** @@ -21,12 +20,12 @@ import org.junit.jupiter.api.Test; * *

Every operation the transport reaches is now a use case (owner decision 32): one port per * operation under {@code application.fileserver.port.in}, each implementation declaring its - * capability and, for the writes, a {@code file:*} permission. What remains here is what no - * transport calls. The cleanup and reconciliation services are driven by background workers that - * have no caller identity; a mutating use case must declare a permission - * (MUTATING_USE_CASES_DECLARE_REQUIRED_PERMISSION), and a permission a scheduler cannot hold would - * refuse every run. The writer-lease coordinator is a collaborator the append use case holds, not - * an entrypoint. + * capability and, for the writes, a {@code file:*} permission. What remains here are collaborators, + * not entrypoints: the cleanup and reconciliation services, interface and implementation, are held + * by the operator's use cases and by the cleanup worker's two {@code @SystemInvoked} use cases + * ({@code RunScheduledCleanupUseCase}, {@code ReconcileRecoverableFilesUseCase}, round9 review + * 2026-11-12 RR9-05-06), and the writer-lease coordinator by the append use case. No transport and + * no worker reaches any of them without a use case in between. * *

A new Fileserver application service still fails this test, so the exemption cannot quietly * grow back. @@ -40,68 +39,46 @@ class FileserverUseCaseContractDeviationTest { * a pattern would silently absorb the next service, which is the behaviour being prevented. */ private static final Set KNOWN_DEVIATIONS = - Set.of("DefaultCleanupService", "DefaultFileReconciliationService", "WriterLeaseCoordinator"); + Set.of( + "CleanupService", + "DefaultCleanupService", + "DefaultFileReconciliationService", + "FileReconciliationService", + "WriterLeaseCoordinator"); @Test - @DisplayName("no new Fileserver application service may bypass the use-case contract") - void theExemptionIsFrozenAndMayOnlyShrink() { + void theServicesOutsideTheUseCaseContractAreExactlyTheFrozenList() { + // given JavaClasses imported = new ClassFileImporter() .withImportOption(new ProductionClassImportOption()) .importPackages("dev.caskeleton.application.fileserver"); + // when List bypassing = imported.stream() .filter(JavaClass::isTopLevelClass) - .filter(type -> !type.isInterface() && !type.isEnum()) + .filter(type -> !type.isEnum()) .filter( type -> type.getModifiers() .contains(com.tngtech.archunit.core.domain.JavaModifier.PUBLIC)) - .filter(FileserverUseCaseContractDeviationTest::looksLikeAnInboundService) - .filter(type -> !type.isAssignableTo(UseCase.class)) + // The inbound rule's own definition, interfaces and coordinators included, so a type + // that rule refuses a transport can only be one this list names. + .filter(AdapterBoundaryArchitectureTest.ARE_APPLICATION_SERVICES) .map(JavaClass::getSimpleName) .sorted() .toList(); + // then assertThat(bypassing) .as( "a Fileserver application service that is not a use case declares none of the " + "transaction mode, idempotency or repository access the contract requires, and " + "the ArchUnit rules cannot see it. Make it a use case behind a port.in " - + "interface (docs/fileserver/design-deviations.md §5).") - .allSatisfy(name -> assertThat(KNOWN_DEVIATIONS).contains(name)); - } - - @Test - @DisplayName("the exemption list names only services that still exist") - void theExemptionListDoesNotOutliveItsEntries() { - JavaClasses imported = - new ClassFileImporter() - .withImportOption(new ProductionClassImportOption()) - .importPackages("dev.caskeleton.application.fileserver"); - - Set present = - imported.stream() - .map(JavaClass::getSimpleName) - .collect(java.util.stream.Collectors.toSet()); - - assertThat(KNOWN_DEVIATIONS) - .as( - "an entry for a service that no longer exists hides the fact that the deviation " - + "shrank, which is the only progress signal this list carries") - .allSatisfy(name -> assertThat(present).contains(name)); - } - - /** - * An orchestrating service, as opposed to a value, a policy or a port. - * - *

Name-based because that is what the repository's own naming convention makes reliable here: - * these types are reached from a controller and coordinate ports, which is precisely the shape - * the use-case contract governs. - */ - private static boolean looksLikeAnInboundService(JavaClass type) { - String name = type.getSimpleName(); - return name.endsWith("Service") || name.endsWith("Coordinator"); + + "interface (docs/fileserver/design-deviations.md §5). An entry whose service " + + "became a use case or went away is struck from the list, the only progress " + + "signal it carries.") + .containsExactlyInAnyOrderElementsOf(KNOWN_DEVIATIONS); } } diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java new file mode 100644 index 00000000..f84281a0 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/NotificationUseCaseContractDeviationTest.java @@ -0,0 +1,81 @@ +package dev.caskeleton.bootstrap.architecture; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaClasses; +import com.tngtech.archunit.core.domain.JavaModifier; +import com.tngtech.archunit.core.importer.ClassFileImporter; +import dev.caskeleton.application.usecase.UseCase; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** + * Freezes the notification services that are not use cases, as {@link + * FileserverUseCaseContractDeviationTest} does for the Fileserver. + * + *

The ArchUnit rules that check capability, transaction boundary and permission only match types + * that implement {@link UseCase}; a service that never does is invisible to them. Every entrypoint + * of the platform is now a use case: the transport's through {@code platform.port.in}, the + * operator's four, and the system's — claim and dispatch, lease recovery, reconciliation, + * provider-event replay — as {@code @SystemInvoked} use cases the composition's workers hold + * (round9 review 2026-11-12, RR9-05-06). What remains below are the collaborators those use cases + * share: each opens its own boundaries around each write, which is why it is not one use case with + * one declared boundary. + * + *

A new notification service that is not a use case fails this test, so the list cannot quietly + * grow back. + */ +class NotificationUseCaseContractDeviationTest { + + /** + * The collaborators that predate the rule. + * + *

May shrink and must not grow. Spelled out rather than derived: a pattern would absorb the + * next service, which is the behaviour being prevented. + */ + private static final Set KNOWN_COLLABORATORS = + Set.of( + "DeduplicationService", + "LeaseRecoveryService", + "NotificationDispatchService", + "NotificationSubmissionService", + "ProviderEventProjectionService", + "ReconciliationService"); + + @Test + void theServicesOutsideTheUseCaseContractAreExactlyTheFrozenList() { + // given + JavaClasses imported = notificationClasses(); + + // when + List bypassing = + imported.stream() + .filter(JavaClass::isTopLevelClass) + .filter(type -> !type.isEnum()) + .filter(type -> type.getModifiers().contains(JavaModifier.PUBLIC)) + // The inbound rule's own definition, interfaces and orchestrators included, so a type + // that rule refuses a transport can only be one this list names. + .filter(AdapterBoundaryArchitectureTest.ARE_APPLICATION_SERVICES) + .map(JavaClass::getSimpleName) + .sorted() + .toList(); + + // then + assertThat(bypassing) + .as( + "a new notification service that is not a use case declares no capability, boundary" + + " or permission, and the ArchUnit rules cannot see it: make the entrypoint a use" + + " case and keep the service a collaborator it holds. An entry whose service" + + " became a use case or went away is struck from the list, the only progress" + + " signal it carries") + .containsExactlyInAnyOrderElementsOf(KNOWN_COLLABORATORS); + } + + private static JavaClasses notificationClasses() { + return new ClassFileImporter() + .withImportOption(new ProductionClassImportOption()) + .importPackages("dev.caskeleton.application.notification"); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java new file mode 100644 index 00000000..6c950da1 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseNamedServiceHoldingControllerFixture.java @@ -0,0 +1,25 @@ +package dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web; + +import dev.caskeleton.bootstrap.architecture.allowed.application.ReportExportService; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Positive fixture for {@code INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a + * controller handed a use case that happens to be named {@code *Service}. The rule refuses services + * that are not use cases, so this one passes. + */ +@RestController +public class UseCaseNamedServiceHoldingControllerFixture { + + private final ReportExportService exports; + + public UseCaseNamedServiceHoldingControllerFixture(ReportExportService exports) { + this.exports = exports; + } + + @PostMapping("/exports") + public int export() { + return exports.handle(new ReportExportService.Export()); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java new file mode 100644 index 00000000..186972dc --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/UseCaseReadingStreamBridgeFixture.java @@ -0,0 +1,22 @@ +package dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web; + +import dev.caskeleton.application.realtime.GetReplayWindowQuery; +import dev.caskeleton.application.realtime.GetReplayWindowUseCase; + +/** + * Positive fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS} and {@code + * INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a web stream bridge that reads + * through a query use case, which both rules let through. + */ +public final class UseCaseReadingStreamBridgeFixture { + + private final GetReplayWindowUseCase window; + + public UseCaseReadingStreamBridgeFixture(GetReplayWindowUseCase window) { + this.window = window; + } + + public boolean resumable(String stream, long position) { + return window.handle(new GetReplayWindowQuery(stream)).canResumeAfter(position); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java index 2ed3ad1c..3cf8408f 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/adapter/inbound/web/package-info.java @@ -2,5 +2,8 @@ * "Allowed-as-data" web endpoint for the system-caller rules: a controller in a user-transport * package that reaches no system caller. A configuration may register it beside a system caller * (review 2026-11-04 RR4-01-02). + * + *

Also the inbound adapters the transport and inbound service rules let through: a controller + * holding a use case named {@code *Service}, and a stream bridge reading through a query use case. */ package dev.caskeleton.bootstrap.architecture.allowed.adapter.inbound.web; diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java new file mode 100644 index 00000000..2d21e091 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/ReportExportService.java @@ -0,0 +1,28 @@ +package dev.caskeleton.bootstrap.architecture.allowed.application; + +import dev.caskeleton.application.capability.Idempotency; +import dev.caskeleton.application.capability.RepositoryAccess; +import dev.caskeleton.application.capability.UseCaseCapability; +import dev.caskeleton.application.command.Command; +import dev.caskeleton.application.transaction.TransactionMode; +import dev.caskeleton.application.usecase.CommandUseCase; + +/** + * A use case whose name ends in {@code Service}, as older application code still has: the target of + * {@code UseCaseNamedServiceHoldingControllerFixture}, which the inbound service rule lets through + * because the type is a use case. + */ +@UseCaseCapability( + transactionMode = TransactionMode.WRITE, + idempotency = Idempotency.IDEMPOTENT, + repositoryAccess = RepositoryAccess.NONE) +public class ReportExportService implements CommandUseCase { + + @Override + public Integer handle(Export command) { + return 0; + } + + /** The export request. */ + public record Export() implements Command {} +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java index b3d4464f..f30a8790 100644 --- a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/allowed/application/package-info.java @@ -4,5 +4,8 @@ * QUERY_PORTS_DO_NOT_LEAK_DOMAIN_JPA_OR_WEB_TYPES} rule must NOT flag — loaded in isolation by * {@link dev.caskeleton.bootstrap.architecture.ArchitectureViolationFixtureTest} to prove the rule * does not over-block clean projection reads. + * + *

Also the targets of the inbound service rule's positive fixtures: a use case whose name ends + * in {@code Service} ({@code ReportExportService}) is a port, not a wrapped service. */ package dev.caskeleton.bootstrap.architecture.allowed.application; diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java new file mode 100644 index 00000000..63d0f205 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/CoordinatorHoldingControllerFixture.java @@ -0,0 +1,24 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller; + +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.WriterSlotCoordinator; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Negative fixture for {@code INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a + * controller handed an application coordinator instead of the use case that holds it. + */ +@RestController +public class CoordinatorHoldingControllerFixture { + + private final WriterSlotCoordinator writers; + + public CoordinatorHoldingControllerFixture(WriterSlotCoordinator writers) { + this.writers = writers; + } + + @PostMapping("/writers") + public boolean take() { + return writers.take("key"); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java new file mode 100644 index 00000000..4a033bba --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/controller/ServiceInterfaceHoldingControllerFixture.java @@ -0,0 +1,25 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.controller; + +import dev.caskeleton.bootstrap.architecture.violations.application.maintenance.MaintenanceService; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Negative fixture for {@code INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES}: a + * controller handed the service interface a {@code @SystemInvoked} use case wraps, which runs the + * write with no permission check. The rule once skipped interfaces (round9 review 2026-11-12). + */ +@RestController +public class ServiceInterfaceHoldingControllerFixture { + + private final MaintenanceService maintenance; + + public ServiceInterfaceHoldingControllerFixture(MaintenanceService maintenance) { + this.maintenance = maintenance; + } + + @PostMapping("/maintenance") + public int run() { + return maintenance.runOnce(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java new file mode 100644 index 00000000..58e2fd26 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdGeneratingReceiptHandlerFixture.java @@ -0,0 +1,21 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdGeneratorPort; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web handler + * minting ids through a port named {@code *IdGeneratorPort}, as the notification platform's is, + * rather than {@code *IdFactory}. + */ +public final class IdGeneratingReceiptHandlerFixture { + + private final ReceiptIdGeneratorPort ids; + + public IdGeneratingReceiptHandlerFixture(ReceiptIdGeneratorPort ids) { + this.ids = ids; + } + + public String accept() { + return ids.nextId(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java new file mode 100644 index 00000000..d947c15d --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdMintingReceiptHandlerFixture.java @@ -0,0 +1,21 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdFactory; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web handler + * that mints the id the use case will store instead of leaving it to the use case, as the durable + * operation controller did (round9 review 2026-11-12). + */ +public final class IdMintingReceiptHandlerFixture { + + private final ReceiptIdFactory ids; + + public IdMintingReceiptHandlerFixture(ReceiptIdFactory ids) { + this.ids = ids; + } + + public String accept() { + return ids.next(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java new file mode 100644 index 00000000..792acdd2 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/IdentifierHoldingReceiptHandlerFixture.java @@ -0,0 +1,20 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.ReceiptIdentifierPort; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web handler + * minting ids through a port named {@code *IdentifierPort}, as the fileserver's upload id port is. + */ +public final class IdentifierHoldingReceiptHandlerFixture { + + private final ReceiptIdentifierPort ids; + + public IdentifierHoldingReceiptHandlerFixture(ReceiptIdentifierPort ids) { + this.ids = ids; + } + + public String accept() { + return ids.nextId(); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java new file mode 100644 index 00000000..6bef9ce2 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/adapter/inbound/web/stream/QueryPortHoldingStreamBridgeFixture.java @@ -0,0 +1,21 @@ +package dev.caskeleton.bootstrap.architecture.violations.adapter.inbound.web.stream; + +import dev.caskeleton.bootstrap.architecture.violations.application.history.EventHistoryQueryPort; + +/** + * Negative fixture for {@code USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS}: a web stream + * bridge — no controller — that reads a query port itself instead of through a query use case + * (round9 review 2026-11-12). + */ +public final class QueryPortHoldingStreamBridgeFixture { + + private final EventHistoryQueryPort history; + + public QueryPortHoldingStreamBridgeFixture(EventHistoryQueryPort history) { + this.history = history; + } + + public int replayed(String stream) { + return history.count(stream); + } +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java new file mode 100644 index 00000000..40e74c4b --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/EventHistoryQueryPort.java @@ -0,0 +1,8 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** A read-model query port, the target of {@code QueryPortHoldingStreamBridgeFixture}. */ +public interface EventHistoryQueryPort { + + /** How many events the stream holds. */ + int count(String stream); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java new file mode 100644 index 00000000..105e97bb --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdFactory.java @@ -0,0 +1,8 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** A server-assigned id factory, the target of {@code IdMintingReceiptHandlerFixture}. */ +public interface ReceiptIdFactory { + + /** The next receipt id. */ + String next(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java new file mode 100644 index 00000000..024978b7 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdGeneratorPort.java @@ -0,0 +1,8 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** An id port named the older way, the target of {@code IdGeneratingReceiptHandlerFixture}. */ +public interface ReceiptIdGeneratorPort { + + /** The next receipt id. */ + String nextId(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java new file mode 100644 index 00000000..09106f54 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/history/ReceiptIdentifierPort.java @@ -0,0 +1,11 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.history; + +/** + * An id port named {@code *IdentifierPort}, as the fileserver's upload id port is: the target of + * {@code IdentifierHoldingReceiptHandlerFixture}. + */ +public interface ReceiptIdentifierPort { + + /** The next receipt id. */ + String nextId(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java new file mode 100644 index 00000000..3433f2a5 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/MaintenanceService.java @@ -0,0 +1,11 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.maintenance; + +/** + * An application service interface that a system use case wraps, the target of {@code + * ServiceInterfaceHoldingControllerFixture}. + */ +public interface MaintenanceService { + + /** Runs one maintenance pass with no permission check of its own. */ + int runOnce(); +} diff --git a/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java new file mode 100644 index 00000000..f2e64e56 --- /dev/null +++ b/src/bootstrap/app/src/architectureTest/java/dev/caskeleton/bootstrap/architecture/violations/application/maintenance/WriterSlotCoordinator.java @@ -0,0 +1,10 @@ +package dev.caskeleton.bootstrap.architecture.violations.application.maintenance; + +/** An application coordinator, the target of {@code CoordinatorHoldingControllerFixture}. */ +public final class WriterSlotCoordinator { + + /** Takes the writer slot for a key. */ + public boolean take(String key) { + return !key.isEmpty(); + } +} diff --git a/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java b/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java index cab63f22..c1133358 100644 --- a/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java +++ b/src/bootstrap/durable-operation/src/main/java/dev/caskeleton/composition/operation/DurableOperationAutoConfiguration.java @@ -71,8 +71,10 @@ public class DurableOperationAutoConfiguration { OutboxAppendPort outbox, IntegrationEventEncoderPort encoder, TransactionPort transactions, + DurableOperationIdFactory operationIds, Clock clock) { - return new SubmitDurableOperationUseCase(operations, outbox, encoder, transactions, clock); + return new SubmitDurableOperationUseCase( + operations, outbox, encoder, transactions, operationIds, clock); } /** Reads one operation for the caller that submitted it. */ diff --git a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java index 7098d385..06fceacb 100644 --- a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java +++ b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverCleanupWorker.java @@ -4,7 +4,11 @@ import static net.logstash.logback.argument.StructuredArguments.kv; import dev.caskeleton.application.fileserver.cleanup.CleanupBatchResult; import dev.caskeleton.application.fileserver.cleanup.CleanupService; +import dev.caskeleton.application.fileserver.cleanup.RunScheduledCleanupCommand; +import dev.caskeleton.application.fileserver.cleanup.RunScheduledCleanupUseCase; import dev.caskeleton.application.fileserver.recovery.FileReconciliationService; +import dev.caskeleton.application.fileserver.recovery.ReconcileRecoverableFilesCommand; +import dev.caskeleton.application.fileserver.recovery.ReconcileRecoverableFilesUseCase; import dev.caskeleton.application.fileserver.recovery.ReconciliationResult; import dev.caskeleton.shared.error.ApiErrorCarrier; import dev.caskeleton.shared.error.ApiErrorCode; @@ -33,28 +37,30 @@ import org.slf4j.LoggerFactory; * interval rather than declared here with a property placeholder. Two declarations of the same * interval — one typed and validated, one a raw string with its own default — is one declaration * too many: they drift, and the one that wins is whichever the scheduler happened to read. + * + *

Each step is a {@code @SystemInvoked} use case the worker constructs from the services it is + * given and keeps to itself, the way the outbox relay's scheduler holds its use case: a write the + * system starts is a use case of its own, apart from the one an operator starts with a permission + * (manual 07 「시스템이 부르는 쓰기」). Neither is a bean. */ public final class FileserverCleanupWorker { private static final Logger LOG = LoggerFactory.getLogger(FileserverCleanupWorker.class); - private final CleanupService cleanupService; - private final FileReconciliationService reconciliationService; - private final int maxItems; - private final long maxBytes; + private final RunScheduledCleanupUseCase cleanup; + private final ReconcileRecoverableFilesUseCase reconciliation; + private final RunScheduledCleanupCommand batch; + private final ReconcileRecoverableFilesCommand pass; public FileserverCleanupWorker( CleanupService cleanupService, FileReconciliationService reconciliationService, int maxItems, long maxBytes) { - if (maxItems < 1 || maxBytes < 1) { - throw new IllegalArgumentException("cleanup batch bounds must be positive"); - } - this.cleanupService = cleanupService; - this.reconciliationService = reconciliationService; - this.maxItems = maxItems; - this.maxBytes = maxBytes; + this.cleanup = new RunScheduledCleanupUseCase(cleanupService); + this.reconciliation = new ReconcileRecoverableFilesUseCase(reconciliationService); + this.batch = new RunScheduledCleanupCommand(maxItems, maxBytes); + this.pass = new ReconcileRecoverableFilesCommand(maxItems); } /** @@ -71,7 +77,7 @@ public final class FileserverCleanupWorker { private void runReconciliation() { try { - List results = reconciliationService.reconcileRecoverable(maxItems); + List results = reconciliation.handle(pass); if (!results.isEmpty()) { LOG.info( "fileserver.reconcile examined={}", @@ -88,7 +94,7 @@ public final class FileserverCleanupWorker { private void runCleanup() { try { - CleanupBatchResult result = cleanupService.runBatch(maxItems, maxBytes); + CleanupBatchResult result = cleanup.handle(batch); if (result.deleted() > 0 || result.failed() > 0) { LOG.info( "fileserver.cleanup deleted={} skippedActiveLease={} skippedStateChanged={} " diff --git a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverMaintenanceServiceConfiguration.java b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverMaintenanceServiceConfiguration.java similarity index 79% rename from src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverMaintenanceServiceConfiguration.java rename to src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverMaintenanceServiceConfiguration.java index 6cdc398f..5b3bf455 100644 --- a/src/adapter/outbound/fileserver/src/main/java/dev/caskeleton/adapter/outbound/fileserver/platform/autoconfigure/FileserverMaintenanceServiceConfiguration.java +++ b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverMaintenanceServiceConfiguration.java @@ -1,5 +1,7 @@ -package dev.caskeleton.adapter.outbound.fileserver.platform.autoconfigure; +package dev.caskeleton.composition.fileserver; +import dev.caskeleton.adapter.outbound.fileserver.platform.autoconfigure.FileserverMetricsSupport; +import dev.caskeleton.adapter.outbound.fileserver.platform.autoconfigure.FileserverPlatformSettings; import dev.caskeleton.application.fileserver.api.metadata.FileMetadataPort; import dev.caskeleton.application.fileserver.api.metadata.UploadSessionPort; import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort; @@ -21,6 +23,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +/** + * The cleanup and reconciliation services: the operator's cleanup use case and the cleanup worker + * both hold them. + * + *

Here rather than in {@code adapter:outbound:fileserver}, where they were: each one decides + * what may be deleted or restored and opens its own transactions around each write, and an outbound + * adapter assembling application services is the adapter deciding where a unit of work begins + * (round9 review 2026-11-12, RR9-05-08). + */ @Configuration(proxyBeanMethods = false) class FileserverMaintenanceServiceConfiguration { diff --git a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java index 6b093f55..51104259 100644 --- a/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java +++ b/src/bootstrap/fileserver-platform/src/main/java/dev/caskeleton/composition/fileserver/FileserverPlatformAutoConfiguration.java @@ -45,6 +45,7 @@ import org.springframework.context.annotation.Import; FileserverFileUseCaseConfiguration.class, FileserverAdminReadUseCaseConfiguration.class, FileserverAdminActionUseCaseConfiguration.class, + FileserverMaintenanceServiceConfiguration.class, FileserverCleanupConfiguration.class, FileserverCleanupDisabledConfiguration.class, // The HTTP surface itself. Its controllers used to be found by the composition root's component diff --git a/src/bootstrap/notification-platform/README.md b/src/bootstrap/notification-platform/README.md index 962b47b1..3396d220 100644 --- a/src/bootstrap/notification-platform/README.md +++ b/src/bootstrap/notification-platform/README.md @@ -13,8 +13,10 @@ composition:notification-platform ├─ NotificationApplicationCompositionAutoConfiguration (use case 조립, `enabled` 조건) │ ├─ NotificationSubmissionConfiguration 접수·예약·취소·조회·템플릿 발행 use case - │ ├─ NotificationDeliveryConfiguration 발송·재조정·lease 회수·ledger 투영·운영자 plane - │ └─ NotificationCallbackConfiguration provider 콜백 수집 + ProviderCallbackIngressBridge + │ ├─ NotificationDeliveryConfiguration 발송·재조정·lease 회수·ledger 투영 서비스 + │ ├─ NotificationAdminConfiguration 운영자 use case 넷(redrive·reconcile·suppress·provider 상태) + │ ├─ NotificationCallbackConfiguration provider 콜백 수집 + ProviderCallbackIngressBridge + │ └─ NotificationWorkerConfiguration (SERVING만) 발송 scheduler와 회복·replay·맞추기 worker ├─ config/module-defaults/notification.yml (설정 기본값, support 로더가 읽음) ├─ adapter:outbound:notification provider(SMTP·SES), 저장 외 런타임, NotificationPlatformAutoConfiguration ├─ adapter:outbound:persistence-notification (runtimeOnly) 알림 entity·저장소·Flyway stream @@ -25,6 +27,11 @@ composition:notification-platform - 조립 순서: 이 모듈의 root는 `NotificationPlatformAutoConfiguration`(outbound leaf) 뒤에 온다 (`afterName`). +- worker: 두 worker(`NotificationSchedulerWorker`, `NotificationBackgroundWorkers`)는 스레드와 일정만 가진다. 하는 일은 + `@SystemInvoked` use case(claim·발송·lease 회복·provider event replay·맞추기)와 backlog 측정 query use case이고, worker + 설정이 `new`로 만들어 넘긴다. + bean이 아니어서 어떤 전송도 닿지 못한다(manual 07 「시스템이 부르는 쓰기」). `mode=INGEST_ONLY`면 worker 설정 자체가 + 실리지 않는다. - 설정: `config/module-defaults/notification.yml`을 support의 `ModuleDefaultsEnvironmentPostProcessor`가 읽는다. 우선순위는 가장 낮다. 그래서 `application*.yml`, 환경 변수, 시스템 속성이 항상 이긴다. app-bootstrap은 이 파일을 모른다. - 저장소: 요청·발송·ledger 테이블과 migration은 `adapter:outbound:persistence-notification`에 있고, diff --git a/src/bootstrap/notification-platform/build.gradle b/src/bootstrap/notification-platform/build.gradle index 06dedbf6..bdb28475 100644 --- a/src/bootstrap/notification-platform/build.gradle +++ b/src/bootstrap/notification-platform/build.gradle @@ -27,6 +27,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-autoconfigure' implementation 'org.springframework:spring-context' + // The two worker threads log a failed pass instead of letting it cancel the schedule. + implementation 'org.slf4j:slf4j-api' testImplementation 'org.springframework.boot:spring-boot-starter-test' // NotificationModuleDefaultsTest reads the shipped file through the loader that serves it at diff --git a/src/bootstrap/notification-platform/gradle.lockfile b/src/bootstrap/notification-platform/gradle.lockfile index a981ae24..03e15727 100644 --- a/src/bootstrap/notification-platform/gradle.lockfile +++ b/src/bootstrap/notification-platform/gradle.lockfile @@ -151,7 +151,7 @@ org.reactivestreams:reactive-streams:1.0.4=runtimeClasspath,testRuntimeClasspath org.reflections:reflections:0.10.2=checkstyle org.skyscreamer:jsonassert:1.5.3=testCompileClasspath,testRuntimeClasspath org.slf4j:jul-to-slf4j:2.0.20=runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.20=runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.20=compileClasspath,runtimeClasspath,spotbugs,spotbugsSlf4j,testCompileClasspath,testRuntimeClasspath org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j org.slf4j:slf4j-simple:2.0.20=checkstyle org.springdoc:springdoc-openapi-starter-common:3.0.0=runtimeClasspath,testRuntimeClasspath diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java new file mode 100644 index 00000000..92ff423f --- /dev/null +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationAdminConfiguration.java @@ -0,0 +1,92 @@ +package dev.caskeleton.composition.notification; + +import dev.caskeleton.application.notification.platform.admin.AdminAuthorizationGuard; +import dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort; +import dev.caskeleton.application.notification.platform.admin.ChangeProviderStateUseCase; +import dev.caskeleton.application.notification.platform.admin.ChangeSuppressionUseCase; +import dev.caskeleton.application.notification.platform.admin.DuplicateRiskGuard; +import dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort; +import dev.caskeleton.application.notification.platform.admin.ReconcileDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.admin.RedriveDeliveryUseCase; +import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort; +import dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort; +import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; +import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort; +import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort; +import dev.caskeleton.application.transaction.TransactionPort; +import java.time.Clock; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * The operator plane: redrive, reconcile, suppress and provider control, one use case each. + * + *

Beans, so the method-security proxy applies each one's {@code @RequiresPermission}. The + * template ships no transport that calls them; a deployment that exposes the operator plane adds + * one. Which roles hold the {@code notification-admin:*} permissions by default is this module's + * {@code config/module-defaults/notification.yml}. + */ +@Configuration(proxyBeanMethods = false) +class NotificationAdminConfiguration { + + @Bean + RedriveDeliveryUseCase notificationRedriveDeliveryUseCase( + DeliveryAttemptStorePort attempts, + RecipientDeliveryStorePort recipients, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + return new RedriveDeliveryUseCase( + new AdminAuthorizationGuard(), + new DuplicateRiskGuard(), + attempts, + recipients, + operations, + audit, + transactions, + clock); + } + + @Bean + ReconcileDeliveriesUseCase notificationReconcileDeliveriesUseCase( + DeliveryAttemptStorePort attempts, + ReconciliationService reconciliation, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + return new ReconcileDeliveriesUseCase( + new AdminAuthorizationGuard(), + attempts, + reconciliation, + operations, + audit, + transactions, + clock); + } + + @Bean + ChangeSuppressionUseCase notificationChangeSuppressionUseCase( + SuppressionStorePort suppressions, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + NotificationIdGeneratorPort ids, + Clock clock) { + return new ChangeSuppressionUseCase( + new AdminAuthorizationGuard(), suppressions, operations, audit, transactions, ids, clock); + } + + @Bean + ChangeProviderStateUseCase notificationChangeProviderStateUseCase( + ProviderRuntimeControlPort runtimes, + AdminOperationStorePort operations, + NotificationAuditPort audit, + TransactionPort transactions, + Clock clock) { + return new ChangeProviderStateUseCase( + new AdminAuthorizationGuard(), runtimes, operations, audit, transactions, clock); + } +} diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java index bfb43440..afbcef41 100644 --- a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationApplicationCompositionAutoConfiguration.java @@ -7,9 +7,10 @@ import org.springframework.context.annotation.Import; /** * Application-owned Notification orchestration and transaction-bound use-case assembly. * - *

The root holds the capability switch and imports one configuration per concern. It orders - * itself after the outbound leaf's root, whose stores, providers and policies every use case here - * consumes. + *

The root holds the capability switch and imports one configuration per concern, the workers + * included: nothing else loads {@link NotificationWorkerConfiguration}, which carries the mode + * condition. It orders itself after the outbound leaf's root, whose stores, providers and policies + * every use case here consumes. */ @AutoConfiguration( afterName = @@ -22,6 +23,8 @@ import org.springframework.context.annotation.Import; @Import({ NotificationSubmissionConfiguration.class, NotificationDeliveryConfiguration.class, - NotificationCallbackConfiguration.class + NotificationAdminConfiguration.class, + NotificationCallbackConfiguration.class, + NotificationWorkerConfiguration.class }) public class NotificationApplicationCompositionAutoConfiguration {} diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationBackgroundWorkers.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationBackgroundWorkers.java similarity index 55% rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationBackgroundWorkers.java rename to src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationBackgroundWorkers.java index b4585d25..2e8b2ea7 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationBackgroundWorkers.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationBackgroundWorkers.java @@ -1,42 +1,58 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.composition.notification; -import dev.caskeleton.application.notification.platform.dispatch.LeaseRecoveryService; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsCommand; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsCommand; +import dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesCommand; +import dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesUseCase; import java.time.Duration; +import java.util.List; import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.IntSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Runs the recovery and replay passes, and stops them on shutdown. + * Runs the lease recovery, provider-event replay and reconciliation passes, and stops them on + * shutdown. * - *

Both existed as classes with no caller. {@code LeaseRecoveryService} was written to recover - * deliveries a dead worker left in flight and was never scheduled; the ledger's replay queries were - * implemented in persistence and never read. The dispatch scheduler's own error path assumed - * recovery existed — it leaves a lease to expire rather than releasing it optimistically, on the - * grounds that recovery will decide — so the absence turned a deliberate design into a leak. + *

The first two existed as classes with no caller. {@code LeaseRecoveryService} was written to + * recover deliveries a dead worker left in flight and was never scheduled; the ledger's replay + * queries were implemented in persistence and never read. The dispatch scheduler's own error path + * assumed recovery existed — it leaves a lease to expire rather than releasing it optimistically, + * on the grounds that recovery will decide — so the absence turned a deliberate design into a leak. * - *

One executor for both, because they are cheap, periodic, and must stop together. A failure in - * one pass is logged and the schedule continues: an exception escaping a scheduled task cancels it - * silently, which is how a background worker stops running without anybody being told. + *

One executor for the three, because they are cheap, periodic, and must stop together. A + * failure in one pass is logged and the schedule continues: an exception escaping a scheduled task + * cancels it silently, which is how a background worker stops running without anybody being told. + * + *

Each pass is a {@code @SystemInvoked} use case this composition constructs and hands in, the + * way the outbox relay's scheduler holds its use case. The class owns the thread and the schedule + * and nothing else; it lived in the notification adapter, which owned the reconciliation and replay + * decisions too (round9 review 2026-11-12, RR9-05-06, RR9-05-08). */ +// Public, unlike the scheduler worker: its bean is @ConditionalOnMissingBean, so a deployment may +// replace it with one of its own, built from the same use cases. public final class NotificationBackgroundWorkers implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(NotificationBackgroundWorkers.class); - private final LeaseRecoveryService recovery; - private final ProviderEventReplayWorker replay; - private final ReconciliationJobWorker reconciliation; + private final RecoverAbandonedDeliveriesUseCase recovery; + private final ReplayProviderEventsUseCase replay; + private final ReconcileOutstandingAttemptsUseCase reconciliation; private final Duration interval; private final Duration shutdownGrace; private final ScheduledExecutorService scheduler; private final AtomicBoolean started = new AtomicBoolean(); - private final java.util.List> passes = - new java.util.concurrent.CopyOnWriteArrayList<>(); + private final List> passes = new CopyOnWriteArrayList<>(); /** * Creates the workers. @@ -48,9 +64,9 @@ public final class NotificationBackgroundWorkers implements AutoCloseable { * @param shutdownGrace how long shutdown waits for a pass in flight */ public NotificationBackgroundWorkers( - LeaseRecoveryService recovery, - ProviderEventReplayWorker replay, - ReconciliationJobWorker reconciliation, + RecoverAbandonedDeliveriesUseCase recovery, + ReplayProviderEventsUseCase replay, + ReconcileOutstandingAttemptsUseCase reconciliation, Duration interval, Duration shutdownGrace) { this.recovery = Objects.requireNonNull(recovery, "recovery"); @@ -69,21 +85,22 @@ public final class NotificationBackgroundWorkers implements AutoCloseable { }); } - /** Starts both passes. Idempotent. */ + /** Starts the three passes. Idempotent. */ public void start() { if (!started.compareAndSet(false, true)) { return; } - schedule("recovery", recovery::recoverOnce); - schedule("provider-event-replay", replay::replayOnce); - schedule("reconciliation", reconciliation::reconcileOnce); + schedule("recovery", () -> recovery.handle(new RecoverAbandonedDeliveriesCommand())); + schedule("provider-event-replay", () -> replay.handle(new ReplayProviderEventsCommand())); + schedule( + "reconciliation", () -> reconciliation.handle(new ReconcileOutstandingAttemptsCommand())); } - private void schedule(String name, java.util.function.IntSupplier pass) { + private void schedule(String name, IntSupplier pass) { // The handle is kept so cancellation is possible and so the ignored-future check has an // answer: the task swallows its own exceptions, so the future never completes exceptionally // and there is nothing for a caller to observe on it. - java.util.concurrent.ScheduledFuture scheduled = + ScheduledFuture scheduled = scheduler.scheduleWithFixedDelay( () -> { try { @@ -135,7 +152,7 @@ public final class NotificationBackgroundWorkers implements AutoCloseable { private static Duration requirePositive(Duration value, String name) { Objects.requireNonNull(value, name); if (value.isNegative() || value.isZero()) { - throw new IllegalArgumentException(name + " must be positive and finite"); + throw new IllegalStateException(name + " must be positive and finite"); } return value; } diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java index 44156285..8fa0728f 100644 --- a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationDeliveryConfiguration.java @@ -1,12 +1,6 @@ package dev.caskeleton.composition.notification; import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; -import dev.caskeleton.application.notification.platform.admin.AdminAuthorizationGuard; -import dev.caskeleton.application.notification.platform.admin.AdminOperationStorePort; -import dev.caskeleton.application.notification.platform.admin.DuplicateRiskGuard; -import dev.caskeleton.application.notification.platform.admin.NotificationAdminApplicationService; -import dev.caskeleton.application.notification.platform.admin.NotificationAdminService; -import dev.caskeleton.application.notification.platform.admin.ProviderRuntimeControlPort; import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptResolverPort; import dev.caskeleton.application.notification.platform.callback.DeliveryProjectionStorePort; import dev.caskeleton.application.notification.platform.callback.NotificationSideEffectPort; @@ -42,8 +36,9 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** - * Doing the work and correcting it: dispatch, reconciliation, lease recovery, ledger projection and - * the operator plane that redrives and suppresses deliveries. + * Doing the work and correcting it: dispatch, reconciliation, lease recovery and ledger projection. + * The operator plane that redrives and suppresses deliveries is {@link + * NotificationAdminConfiguration}. * *

Here rather than in the notification adapter because each of these opens its own transactions * around each write, and the adapter must not decide where a unit of work begins (NTF-020). @@ -163,34 +158,4 @@ class NotificationDeliveryConfiguration { return new ProviderEventProjectionService( ledger, projector, resolver, projections, sideEffects, transactions, metrics); } - - /** The N4 operator plane. */ - @Bean - NotificationAdminService notificationAdminService( - AdminAuthorizationGuard authorization, - DuplicateRiskGuard duplicateRiskGuard, - DeliveryAttemptStorePort attempts, - RecipientDeliveryStorePort recipients, - ReconciliationService reconciliation, - SuppressionStorePort suppressions, - ProviderRuntimeControlPort runtimes, - AdminOperationStorePort operations, - NotificationAuditPort audit, - TransactionPort transactions, - NotificationIdGeneratorPort ids, - Clock clock) { - return new NotificationAdminApplicationService( - authorization, - duplicateRiskGuard, - attempts, - recipients, - reconciliation, - suppressions, - runtimes, - operations, - audit, - transactions, - ids, - clock); - } } diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationSchedulerWorker.java similarity index 80% rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java rename to src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationSchedulerWorker.java index d761c923..da822066 100644 --- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/NotificationSchedulerWorker.java +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationSchedulerWorker.java @@ -1,11 +1,15 @@ -package dev.caskeleton.adapter.outbound.notification.platform.dispatch; +package dev.caskeleton.composition.notification; -import dev.caskeleton.application.notification.platform.dispatch.NotificationDispatchService; +import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; +import dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesCommand; +import dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryCommand; +import dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryUseCase; import dev.caskeleton.application.notification.platform.dispatch.RecipientLease; -import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogQuery; +import dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogUseCase; import dev.caskeleton.application.notification.platform.observation.NotificationMetricName; import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; -import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; import java.util.List; import java.util.Map; import java.util.Objects; @@ -27,16 +31,21 @@ import org.slf4j.LoggerFactory; *

Claiming is bounded twice over: by the claim batch size and by a global concurrency permit. * The second bound matters because a slow provider would otherwise let the queue depth become the * thread count. + * + *

What is claimed and how one delivery is sent are two {@code @SystemInvoked} use cases this + * composition constructs and hands in, and the backlog it publishes is a query use case; the worker + * owns the polling thread, the executor and the permits. It lived in the notification adapter + * (round9 review 2026-11-12, RR9-05-06, RR9-05-08). */ -public final class NotificationSchedulerWorker implements AutoCloseable { +final class NotificationSchedulerWorker implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(NotificationSchedulerWorker.class); - private final RecipientLeaseStorePort leases; - private final NotificationDispatchService dispatcher; + private final ClaimDueDeliveriesUseCase claim; + private final DispatchClaimedDeliveryUseCase dispatch; private final NotificationMetricsPort metrics; private final NotificationDispatchProperties properties; - private final NotificationServingStatePort servingState; + private final MeasureDispatchBacklogUseCase backlog; private final String workerId; private final ExecutorService dispatchExecutor; private final Semaphore globalConcurrency; @@ -46,18 +55,18 @@ public final class NotificationSchedulerWorker implements AutoCloseable { /** The polling thread, kept so shutdown can actually stop it. */ private volatile Thread pollingThread; - public NotificationSchedulerWorker( - RecipientLeaseStorePort leases, - NotificationDispatchService dispatcher, + NotificationSchedulerWorker( + ClaimDueDeliveriesUseCase claim, + DispatchClaimedDeliveryUseCase dispatch, NotificationMetricsPort metrics, NotificationDispatchProperties properties, - NotificationServingStatePort servingState, + MeasureDispatchBacklogUseCase backlog, String workerId) { - this.leases = Objects.requireNonNull(leases, "leases"); - this.dispatcher = Objects.requireNonNull(dispatcher, "dispatcher"); + this.claim = Objects.requireNonNull(claim, "claim"); + this.dispatch = Objects.requireNonNull(dispatch, "dispatch"); this.metrics = Objects.requireNonNull(metrics, "metrics"); this.properties = Objects.requireNonNull(properties, "properties"); - this.servingState = Objects.requireNonNull(servingState, "servingState"); + this.backlog = Objects.requireNonNull(backlog, "backlog"); this.workerId = Objects.requireNonNull(workerId, "workerId"); this.dispatchExecutor = Executors.newVirtualThreadPerTaskExecutor(); this.globalConcurrency = new Semaphore(properties.maxGlobalConcurrency()); @@ -76,7 +85,9 @@ public final class NotificationSchedulerWorker implements AutoCloseable { if (executable < 1) { return 0; } - List claimed = leases.claim(workerId, executable, properties.leaseDuration()); + List claimed = + claim.handle( + new ClaimDueDeliveriesCommand(workerId, executable, properties.leaseDuration())); // The backlog, not the batch. This gauge used to report claimed.size(), which is bounded above // by the claim batch size — so a queue of ten and a queue of ten million published the same // number, and the one metric named "queue depth" was the one that could not show a queue @@ -84,14 +95,14 @@ public final class NotificationSchedulerWorker implements AutoCloseable { metrics.gauge( NotificationMetricName.QUEUE_DEPTH, Map.of(), - (double) servingState.currentState().backlogDepth()); + (double) backlog.handle(new MeasureDispatchBacklogQuery())); for (RecipientLease lease : claimed) { globalConcurrency.acquireUninterruptibly(); dispatchExecutor.execute( () -> { try { - dispatcher.dispatch(lease); + dispatch.handle(new DispatchClaimedDeliveryCommand(lease)); } catch (RuntimeException failure) { // The lease is left to expire rather than being released optimistically: a worker // that diff --git a/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java new file mode 100644 index 00000000..5aff20af --- /dev/null +++ b/src/bootstrap/notification-platform/src/main/java/dev/caskeleton/composition/notification/NotificationWorkerConfiguration.java @@ -0,0 +1,129 @@ +package dev.caskeleton.composition.notification; + +import dev.caskeleton.adapter.outbound.notification.platform.dispatch.NotificationDispatchProperties; +import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService; +import dev.caskeleton.application.notification.platform.callback.ReplayProviderEventsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.ClaimDueDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.dispatch.DispatchClaimedDeliveryUseCase; +import dev.caskeleton.application.notification.platform.dispatch.LeaseRecoveryService; +import dev.caskeleton.application.notification.platform.dispatch.NotificationDispatchService; +import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconcileOutstandingAttemptsUseCase; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort; +import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService; +import dev.caskeleton.application.notification.platform.dispatch.RecoverAbandonedDeliveriesUseCase; +import dev.caskeleton.application.notification.platform.observation.MeasureDispatchBacklogUseCase; +import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort; +import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort; +import dev.caskeleton.application.transaction.TransactionPort; +import java.time.Clock; +import java.util.UUID; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Everything that runs on a schedule rather than on a request, and therefore everything {@code + * INGEST_ONLY} must not have. + * + *

The system's writes are {@code @SystemInvoked} use cases constructed here with {@code new} and + * handed to the two thread-owning workers, the way the outbox relay's composition hands its use + * case to its scheduler (manual 07 「시스템이 부르는 쓰기」). None of them is a bean: a bean is what a + * transport could reach, and these write with no permission check. The workers and this + * configuration lived in the notification adapter, which decided when to reconcile and replay there + * (round9 review 2026-11-12, RR9-05-06, RR9-05-08). + * + *

Gated on the mode as one configuration rather than a condition per bean. Both beans call + * {@code start()}, so a deployment in {@code INGEST_ONLY} — the mode whose whole purpose is to + * accept and store without delivering — would otherwise start the dispatch scheduler, lease + * recovery, provider-event replay and reconciliation. A per-bean condition is one a future bean can + * forget; a configuration that is not loaded cannot be. {@code matchIfMissing = true} because + * {@code SERVING} is the shipped default: an absent mode must still deliver. + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty( + prefix = "app.notification.platform", + name = "mode", + havingValue = "SERVING", + matchIfMissing = true) +class NotificationWorkerConfiguration { + + /** + * Schedules lease recovery, provider-event replay and reconciliation, and stops them on shutdown. + * + * @param recovery the lease recovery service + * @param projection the projection the events feed + * @param jobs the outstanding reconciliation questions + * @param reconciliation asks one attempt's provider + * @param transactions opens the use cases' own boundaries + * @param clock the clock + * @param properties the dispatch bounds + * @return the started background workers + */ + @Bean(destroyMethod = "close") + @ConditionalOnMissingBean + NotificationBackgroundWorkers notificationBackgroundWorkers( + LeaseRecoveryService recovery, + ProviderEventProjectionService projection, + ReconciliationJobStorePort jobs, + ReconciliationService reconciliation, + TransactionPort transactions, + Clock clock, + NotificationDispatchProperties properties) { + NotificationBackgroundWorkers workers = + new NotificationBackgroundWorkers( + new RecoverAbandonedDeliveriesUseCase(recovery), + new ReplayProviderEventsUseCase(projection, properties.claimBatchSize()), + new ReconcileOutstandingAttemptsUseCase( + jobs, + reconciliation::reconcile, + transactions, + clock, + properties.leaseDuration(), + properties.claimBatchSize(), + properties.maxAdditionalAttempts()), + properties.leaseDuration(), + properties.shutdownGrace()); + workers.start(); + return workers; + } + + /** + * The dispatch scheduler. + * + *

The owner carries a per-process nonce, not just a configured name. Every instance used to + * call itself {@code notification-worker-1}, so a restarted process and the one it replaced were + * the same owner as far as the database was concerned — and a stale process's release could clear + * the lease its successor was working under. The nonce makes each incarnation distinguishable; + * the lease fence handles the rest. + * + * @param leases the recipient lease store + * @param dispatcher the dispatch service + * @param transactions opens the claim's write transaction and the backlog's read + * @param metrics the metrics port + * @param properties the dispatch bounds + * @param servingState the serving-state port + * @return the started scheduler worker + */ + @Bean(destroyMethod = "close") + NotificationSchedulerWorker notificationSchedulerWorker( + RecipientLeaseStorePort leases, + NotificationDispatchService dispatcher, + TransactionPort transactions, + NotificationMetricsPort metrics, + NotificationDispatchProperties properties, + NotificationServingStatePort servingState) { + String owner = "notification-worker-1:" + UUID.randomUUID(); + NotificationSchedulerWorker worker = + new NotificationSchedulerWorker( + new ClaimDueDeliveriesUseCase(leases, transactions), + new DispatchClaimedDeliveryUseCase(dispatcher), + metrics, + properties, + new MeasureDispatchBacklogUseCase(servingState, transactions), + owner); + worker.start(); + return worker; + } +} diff --git a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml index 959da41d..d12a9bde 100644 --- a/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml +++ b/src/bootstrap/notification-platform/src/main/resources/config/module-defaults/notification.yml @@ -21,16 +21,20 @@ info: enabled: ${app.notification.platform.enabled:true} app: - # The notification platform's own two permissions, granted to two roles rather than one. - # Publishing a template changes what every future submission renders, so a service that may send - # notifications is not thereby a service that may rewrite them; a deployment that wants both - # grants both roles. Here rather than in app-bootstrap's security.yml (07-08): the mapping goes - # when this module's line is removed. Map entries from every source merge, so a deployment's own - # app.authz.role-permissions adds roles beside these and can override either key. + # The notification platform's permissions, granted to three roles. Sending and publishing a + # template are two roles rather than one: publishing changes what every future submission + # renders, so a service that may send notifications is not thereby a service that may rewrite + # them; a deployment that wants both grants both roles. The operator plane's four permissions go + # to one operator role; the operator's authorities and tenant scope are checked again inside each + # use case, on the AdminActor the transport resolves. Here rather than in app-bootstrap's + # security.yml (07-08): the mapping goes when this module's line is removed. Map entries from + # every source merge, so a deployment's own app.authz.role-permissions adds roles beside these + # and can override any of the three keys. authz: role-permissions: notification-submit: notification:submit notification-template-publish: notification-template:publish + notification-admin: notification-admin:redrive,notification-admin:reconcile,notification-admin:suppress,notification-admin:provider-control notification: platform: # The capability switch. False binds nothing at all. @@ -43,8 +47,9 @@ app: default-locale: ${APP_NOTIFICATION_PLATFORM_DEFAULT_LOCALE:ko-KR} dispatch: claim-batch-size: ${APP_NOTIFICATION_PLATFORM_CLAIM_BATCH_SIZE:50} - # The lease must outlast a provider call plus its timeout, or a live worker's delivery is - # claimed by a second one while the first is still waiting on the provider. + # The lease must outlast a provider call, or a live worker's delivery is claimed by a second + # one while the first is still waiting on the provider. Startup refuses a lease that is not + # longer than every enabled provider profile's timeout. lease-duration: ${APP_NOTIFICATION_PLATFORM_LEASE_DURATION:2m} poll-interval: ${APP_NOTIFICATION_PLATFORM_POLL_INTERVAL:1s} max-global-concurrency: ${APP_NOTIFICATION_PLATFORM_MAX_CONCURRENCY:64} @@ -56,6 +61,15 @@ app: # An ambiguous attempt means the request reached the provider and the outcome could not be # read. Falling back would risk sending the same notification twice, so it is off. allow-ambiguous-fallback: ${APP_NOTIFICATION_PLATFORM_ALLOW_AMBIGUOUS_FALLBACK:false} + # How long a failed delivery waits before its next attempt (full jitter between base and max; a + # provider's Retry-After counts up to max), and how long after an ambiguous submission + # reconciliation first asks the provider. Tune them against the provider's own rate limits. + # Left unset here: the shipped timing (1s, 5m, 1m) lives in NotificationPlatformSettings.Retry + # alone. Override with + # retry: + # backoff-base: 1s + # backoff-max: 5m + # reconcile-delay: 1m # Readiness is deployment-specific: throughput and operational SLOs determine what backlog # is acceptable. These defaults preserve the original behavior but are no longer hard-coded. serving-thresholds: @@ -117,7 +131,7 @@ app: primary-for-channel: ${APP_NOTIFICATION_PLATFORM_SMTP_PRIMARY:true} environment: ${APP_NOTIFICATION_PLATFORM_SMTP_ENVIRONMENT:local} credential-profile: ${APP_NOTIFICATION_PLATFORM_SMTP_CREDENTIAL_PROFILE:default} - timeout: ${APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT:10s} + timeout: ${APP_NOTIFICATION_PLATFORM_SMTP_TIMEOUT:30s} max-concurrency: ${APP_NOTIFICATION_PLATFORM_SMTP_MAX_CONCURRENCY:4} rate-per-second: ${APP_NOTIFICATION_PLATFORM_SMTP_RATE_PER_SECOND:10} ses: diff --git a/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java index a4aa0d25..87d7f94d 100644 --- a/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java +++ b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationModuleDefaultsTest.java @@ -6,6 +6,7 @@ import dev.caskeleton.adapter.inbound.web.notification.platform.callback.Notific import dev.caskeleton.adapter.outbound.notification.platform.provider.ses.SesCallbackAdapter; import dev.caskeleton.adapter.outbound.notification.platform.security.AesGcmCallbackPayloadProtection; import dev.caskeleton.adapter.outbound.support.startup.ModuleDefaultsEnvironmentPostProcessor; +import dev.caskeleton.application.notification.platform.admin.NotificationAdminPermissions; import java.time.Duration; import java.util.List; import java.util.Map; @@ -113,11 +114,20 @@ class NotificationModuleDefaultsTest { List permissions = List.of( environment.getProperty("app.authz.role-permissions.notification-submit", ""), - environment.getProperty( - "app.authz.role-permissions.notification-template-publish", "")); + environment.getProperty("app.authz.role-permissions.notification-template-publish", ""), + environment.getProperty("app.authz.role-permissions.notification-admin", "")); // then - assertThat(permissions).containsExactly("notification:submit", "notification-template:publish"); + assertThat(permissions) + .containsExactly( + "notification:submit", + "notification-template:publish", + String.join( + ",", + NotificationAdminPermissions.REDRIVE, + NotificationAdminPermissions.RECONCILE, + NotificationAdminPermissions.SUPPRESS, + NotificationAdminPermissions.PROVIDER_CONTROL)); } private static StandardEnvironment processed(Map deployment) { diff --git a/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java new file mode 100644 index 00000000..90de9d87 --- /dev/null +++ b/src/bootstrap/notification-platform/src/test/java/dev/caskeleton/composition/notification/NotificationWorkerLifecycleTest.java @@ -0,0 +1,111 @@ +package dev.caskeleton.composition.notification; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.application.security.SystemInvoked; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; + +/** + * {@code INGEST_ONLY} starts no dispatch, recovery or reconciliation thread (NTF-INT-003). + * + *

The worker configuration once carried the master switch and nothing else, and both of its + * worker beans called {@code start()} in the factory method. So a deployment in {@code INGEST_ONLY} + * — the mode whose entire purpose is to accept and store without delivering — started the dispatch + * scheduler, the lease recovery pass, the provider-event replay worker and the reconciliation job. + * + *

The gate is structural rather than a condition repeated per bean: the two started beans live + * in one configuration that carries the mode condition. Asserted on the configuration's own + * annotations rather than by booting the platform: a full notification context needs a database, + * provider credentials and a secrets resolver. The live thread count is asserted by the {@code + * local-notification-ingest} Compose lane. + */ +class NotificationWorkerLifecycleTest { + + /** The bean methods that start something, and therefore may not exist outside SERVING. */ + private static final List THREAD_STARTING_BEANS = + List.of("notificationBackgroundWorkers", "notificationSchedulerWorker"); + + @Test + @DisplayName("스레드를 시작하는 worker bean은 모두 SERVING 모드 조건이 걸린 configuration에 있다") + void everyThreadStartingBeanIsServingOnly() { + // given + ConditionalOnProperty condition = + NotificationWorkerConfiguration.class.getAnnotation(ConditionalOnProperty.class); + + // when + List declared = + Arrays.stream(NotificationWorkerConfiguration.class.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(Bean.class)) + .map(Method::getName) + .sorted() + .toList(); + + // then + assertThat( + List.of( + declared, + condition.prefix(), + List.of(condition.name()), + condition.havingValue(), + condition.matchIfMissing())) + .as( + "SERVING is the shipped default, so an absent mode must still deliver; a worker bean " + + "outside this configuration would run in INGEST_ONLY too") + .containsExactly( + THREAD_STARTING_BEANS.stream().sorted().toList(), + "app.notification.platform", + List.of("mode"), + "SERVING", + true); + } + + @Test + @DisplayName("worker configuration 밖의 조립은 SERVING 조건이 없어 INGEST_ONLY에서도 받는다") + void theIngestSideStaysAvailableInBothModes() { + // given + Import imports = + NotificationApplicationCompositionAutoConfiguration.class.getAnnotation(Import.class); + + // when + List gatedOnTheMode = + Arrays.stream(imports.value()) + .filter(type -> type.isAnnotationPresent(ConditionalOnProperty.class)) + .filter( + type -> + Arrays.asList(type.getAnnotation(ConditionalOnProperty.class).name()) + .contains("mode")) + .map(Class::getSimpleName) + .toList(); + + // then + assertThat(gatedOnTheMode) + .as( + "only the workers stop in INGEST_ONLY; submission, delivery services, callbacks and " + + "the operator plane are what an ingest-only deployment still serves") + .containsExactly("NotificationWorkerConfiguration"); + } + + @Test + @DisplayName("worker가 쥐는 시스템 쓰기 use case는 bean으로 내놓지 않는다") + void noSystemInvokedUseCaseIsABeanOfTheWorkerConfiguration() { + // when + List systemInvokedBeans = + Arrays.stream(NotificationWorkerConfiguration.class.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(Bean.class)) + .filter(method -> method.getReturnType().isAnnotationPresent(SystemInvoked.class)) + .map(Method::getName) + .toList(); + + // then + assertThat(systemInvokedBeans) + .as("a bean is what a transport could reach, and these write with no permission check") + .isEmpty(); + } +}