lag)
implements ReplicaLagMonitor {}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java
new file mode 100644
index 00000000..1fb7f801
--- /dev/null
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/hibernate/stateless/StatelessRunnerTimeoutTest.java
@@ -0,0 +1,45 @@
+package dev.caskeleton.adapter.outbound.persistence.hibernate.stateless;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.Mockito.mock;
+
+import dev.caskeleton.adapter.outbound.persistence.transaction.SpringTransaction;
+import java.time.Duration;
+import java.util.Set;
+import org.hibernate.SessionFactory;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/**
+ * The stateless runner's transaction bound is the write bound every other transaction here has.
+ *
+ * The runner restates the value instead of reading it, because the {@code hibernate} package may
+ * not depend on {@code transaction} (JpaModuleBoundaryTest). A test may depend on both, so it is
+ * the place the two copies are held to each other (round9 review 2026-11-12, RR9-01-25).
+ */
+class StatelessRunnerTimeoutTest {
+
+ @Test
+ void theRunnersDefaultIsSpringTransactionsWriteDefault() {
+ assertThat(HibernateStatelessSessionRunner.DEFAULT_TIMEOUT)
+ .isEqualTo(SpringTransaction.Timeouts.DEFAULT.write());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"PT0S", "PT-1S", "PT1.5S", "PT2147483648S"})
+ void aTimeoutThatIsNotAWholePositiveNumberOfSecondsIsTheAssemblysFault(String timeout) {
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ new HibernateStatelessSessionRunner(
+ mock(SessionFactory.class), Set.of(), Duration.parse(timeout)));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("must be a positive whole number of seconds");
+ }
+}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java
index 72235d67..e80b313d 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/idempotency/jpa/IdempotencyReaperTest.java
@@ -2,23 +2,61 @@ package dev.caskeleton.adapter.outbound.persistence.idempotency.jpa;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.time.Clock;
-import java.time.Instant;
-import java.time.ZoneOffset;
import org.junit.jupiter.api.Test;
+/**
+ * Unit tests for {@link IdempotencyReaper}: the delete runs in bounded batches until one comes back
+ * short, as the outbox reaper's does.
+ *
+ *
The repository is a Spring Data boundary, so Mockito stands in for it. The statement itself —
+ * expiry judged by the database clock — runs against PostgreSQL in {@code
+ * IdempotencyReaperIntegrationTest}.
+ */
class IdempotencyReaperTest {
- private static final Instant NOW = Instant.parse("2026-06-09T12:00:00Z");
+ private static final int BATCH = IdempotencyReaper.BATCH_SIZE;
+
+ private final IdempotencyRecordJpaRepository repository =
+ mock(IdempotencyRecordJpaRepository.class);
+ private final IdempotencyReaper reaper = new IdempotencyReaper(repository);
@Test
- void reapDeletesRecordsExpiredAsOfNow() {
- IdempotencyRecordJpaRepository repo = mock(IdempotencyRecordJpaRepository.class);
- when(repo.deleteExpired(NOW)).thenReturn(3);
- IdempotencyReaper reaper = new IdempotencyReaper(repo, Clock.fixed(NOW, ZoneOffset.UTC));
+ void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() {
+ // given
+ when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3);
- assertThat(reaper.reap()).isEqualTo(3);
+ // when
+ int purged = reaper.reap();
+
+ // then
+ assertThat(purged).isEqualTo(2 * BATCH + 3);
+ }
+
+ @Test
+ void eachBatchIsItsOwnDeleteStatement() {
+ // given
+ when(repository.deleteExpiredBatch(BATCH)).thenReturn(BATCH, BATCH, 3);
+
+ // when
+ var unused = reaper.reap();
+
+ // then
+ verify(repository, times(3)).deleteExpiredBatch(BATCH);
+ }
+
+ @Test
+ void reapReturnsZeroWhenNothingHasExpired() {
+ // given
+ when(repository.deleteExpiredBatch(BATCH)).thenReturn(0);
+
+ // when
+ int purged = reaper.reap();
+
+ // then
+ assertThat(purged).isZero();
}
}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java
new file mode 100644
index 00000000..b43a316e
--- /dev/null
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/liveevent/LiveEventReaperTest.java
@@ -0,0 +1,53 @@
+package dev.caskeleton.adapter.outbound.persistence.liveevent;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link LiveEventReaper}: rows retained no later than the clock's now go, in
+ * bounded batches until one comes back short.
+ *
+ *
The repository is a Spring Data boundary, so Mockito stands in for it. The statement runs
+ * against PostgreSQL in {@code JpaLiveEventReplayContractTest}.
+ */
+class LiveEventReaperTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T00:00:00Z");
+ private static final int BATCH = LiveEventReaper.BATCH_SIZE;
+
+ private final LiveEventJpaRepository repository = mock(LiveEventJpaRepository.class);
+ private final LiveEventReaper reaper =
+ new LiveEventReaper(repository, Clock.fixed(NOW, ZoneOffset.UTC));
+
+ @Test
+ void reapKeepsDeletingBatchesUntilOneComesBackShortAndReturnsTheTotal() {
+ // given
+ when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4);
+
+ // when
+ int purged = reaper.reap();
+
+ // then
+ assertThat(purged).isEqualTo(BATCH + 4);
+ }
+
+ @Test
+ void eachBatchIsItsOwnDeleteStatement() {
+ // given
+ when(repository.deleteExpiredBatch(NOW, BATCH)).thenReturn(BATCH, 4);
+
+ // when
+ var unused = reaper.reap();
+
+ // then
+ verify(repository, times(2)).deleteExpiredBatch(NOW, BATCH);
+ }
+}
diff --git a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java
index 0d647c57..b6ed1ca0 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java
+++ b/src/adapter/outbound/persistence-jpa/src/test/java/dev/caskeleton/adapter/outbound/persistence/lock/DistributedLockPersistenceConfigTest.java
@@ -78,6 +78,20 @@ class DistributedLockPersistenceConfigTest {
});
}
+ @Test
+ @DisplayName("읽는 곳이 없던 app.lock.wait-time은 모르는 키로 기동을 실패시킨다")
+ void theRemovedWaitTimeKeyFailsStartup() {
+ // when
+ contextRunner
+ .withPropertyValues("app.lock.wait-time=3s")
+ .run(
+ context -> {
+ // then
+ assertThat(context).hasFailed();
+ assertThat(context.getStartupFailure()).hasStackTraceContaining("wait-time");
+ });
+ }
+
@Test
void multiInstanceBuildsTheJdbcProviderAndNotTheInProcessProvider() {
contextRunner
diff --git a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256 b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256
index 1eb57d14..93a51fe4 100644
--- a/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256
+++ b/src/adapter/outbound/persistence-jpa/src/test/resources/db/applied-migrations.sha256
@@ -14,6 +14,7 @@
be51fabd07220f9ef885534ba36ada5f1491baf24cdfb54d25d93a19ddc29926 jpa/notification-platform/V11__notification_reconciliation_claim_lease.sql
43f9429f91c07528154ec01b721649b2c25fa98857ac154c7c08f4655634657f jpa/notification-platform/V12__notification_suppression_target_unique.sql
d01aaf59f66834358abf18246faa9a9eb8dfa9f42097ca9d968c44edff564630 jpa/notification-platform/V13__notification_request_acceptance.sql
+8a7dc3604d44f1438b39dfec3aa688f792e7363cd3c1b74bf0c799a6b53f9f0d jpa/notification-platform/V14__notification_provider_event_open_index.sql
1ac7cc1c2509868eb0ce202e1917bb36846f98f17ad4922abe3fac0ef128199b jpa/notification-platform/V1__notification_platform_core.sql
0add67aa23580e51712f84fabe9b09e08401ef708cfa1b3292b120fdb5c12fa2 jpa/notification-platform/V2__notification_platform_contact_template_policy.sql
f5b98598a59cc4376f5d233b410f8a5ae9f3cd94156291ebdf2d0da5a66f34ef jpa/notification-platform/V3__notification_platform_inbox_admin.sql
@@ -42,6 +43,7 @@ a0306e5b0fbcaf43984d256362dc4244aec57a0ce5031e4f5263be3762fe1caf postgresql/V19
25db296e55444636c8a017d9b736eecf4a839a9caf1733c0c4a5f37bae5a3161 postgresql/V20__outbox_event_deferred_attempt_count.sql
21744ea1aab2b29fbb3417b9a5c4389522199413048cc7035aa649ffcdd852cf postgresql/V21__outbox_aggregate_sequence.sql
20da5649a29bf2305f0e04ab14003b21b7f120d867439fb01f1658c98179dba0 postgresql/V22__outbox_event_last_failure_code.sql
+9492ace50ab828144792b77c72d92fce50bec29a028ebc917f7e55facf58d85a postgresql/V23__durable_operation_lease_fence.sql
8ab106731d4d04d3719da0692069947736dc490ca46aa5a3ea9bdf65a5d956dc postgresql/V1__idempotency_record.sql
6b62a01a4eeb792b81015b4810ecc6f3f021e0a17dea50c871493f7694d4f0b9 postgresql/V3__outbox_event.sql
c57ab8a35a10079190ad8a563be58fae4147c9406c5bb752debb6998324ccdf6 postgresql/V4__int_lock.sql
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java
index ef80fdb1..a435e5b5 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/database/MongoTenantClientRegistry.java
@@ -7,9 +7,14 @@ import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedExc
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
/**
* Bounds how many tenant clients exist at once (advanced plan Task 11).
@@ -18,6 +23,13 @@ import java.util.Set;
* connection pool and its own monitoring threads, so a thousand tenants is a thousand pools. The
* registry caps the number and evicts idle ones, and refuses rather than exceeding the cap — an
* unbounded registry fails later, as connection exhaustion on an unrelated request.
+ *
+ *
{@link #acquire} hands back a {@link Lease}; the client is in use until every lease on it is
+ * closed, and only a client nobody holds is idle. Acquire used to return nothing, so idle meant
+ * "acquired longest ago" and eviction could close a client a long-running caller was still using.
+ * Closing an evicted client is this registry's own clean-up: if it fails, the tenant whose acquire
+ * triggered the eviction is not told about another tenant's client — the failure is counted in
+ * {@link #evictionCloseFailures()} (round9 review 2026-11-12, RR9-01-27).
*/
@MongoAdvancedEntryPoint(MongoCapability.DATABASE_PER_TENANT)
public final class MongoTenantClientRegistry {
@@ -26,11 +38,21 @@ public final class MongoTenantClientRegistry {
private final Duration idleEviction;
- private final Map lastUsed = new java.util.concurrent.ConcurrentHashMap<>();
+ private final Map lastUsed = new ConcurrentHashMap<>();
- private final Map clients = new java.util.concurrent.ConcurrentHashMap<>();
+ private final Map clients = new ConcurrentHashMap<>();
- private final java.util.function.Function clientFactory;
+ private final Map leases = new HashMap<>();
+
+ /**
+ * Guards the cap, the lease counts and the eviction as one decision. A private monitor rather
+ * than the registry itself, so code outside cannot take it and stall every acquire.
+ */
+ private final Object lock = new Object();
+
+ private final AtomicLong evictionCloseFailures = new AtomicLong();
+
+ private final Function clientFactory;
private final Clock clock;
@@ -47,15 +69,15 @@ public final class MongoTenantClientRegistry {
MongoAdvancedCapabilityGuard guard,
int maximumActiveClients,
Duration idleEviction,
- java.util.function.Function clientFactory) {
+ Function clientFactory) {
this(guard, maximumActiveClients, idleEviction, clientFactory, Clock.systemUTC());
}
/**
* A registry that owns real clients.
*
- * Idle eviction reads {@code clock}; {@link #acquire(String)} used to read {@code
- * Instant.now()} beside it.
+ *
Acquiring, closing a lease and idle eviction all read {@code clock}; {@link
+ * #acquire(String)} used to read {@code Instant.now()} beside it.
*
*
The factory is called at most once per tenant, inside the map's own computation, so two
* threads asking for the same tenant get one client rather than two — a per-tenant client carries
@@ -65,7 +87,7 @@ public final class MongoTenantClientRegistry {
MongoAdvancedCapabilityGuard guard,
int maximumActiveClients,
Duration idleEviction,
- java.util.function.Function clientFactory,
+ Function clientFactory,
Clock clock) {
Objects.requireNonNull(guard, "guard").require(MongoTenantClientRegistry.class);
this.clientFactory = Objects.requireNonNull(clientFactory, "clientFactory");
@@ -80,41 +102,128 @@ public final class MongoTenantClientRegistry {
/**
* Acquires a client for a tenant.
*
+ * @return the lease to close when the work with the client is done
* @throws MongoOperationRejectedException when the cap is reached and nothing can be evicted
*/
- public void acquire(String tenantKey) {
- acquire(tenantKey, clock.instant());
+ public Lease acquire(String tenantKey) {
+ Objects.requireNonNull(tenantKey, "tenantKey");
+ synchronized (lock) {
+ return acquireHeld(tenantKey);
+ }
}
- /** Acquires a client for a tenant at an explicit instant, so eviction is testable. */
- public synchronized void acquire(String tenantKey, Instant now) {
- Objects.requireNonNull(tenantKey, "tenantKey");
- Objects.requireNonNull(now, "now");
+ private Lease acquireHeld(String tenantKey) {
+ // One time source for the acquire, the close and the eviction: an overload that took the
+ // instant as an argument let a caller acquire in one timeline and close in the clock's.
+ Instant now = clock.instant();
// Synchronized rather than a bare ConcurrentHashMap: the cap, the eviction and the create are
// one decision. Checking the size on one thread while another inserts is how a cap of ten ends
// up holding twelve connection pools.
- if (lastUsed.containsKey(tenantKey)) {
- lastUsed.put(tenantKey, now);
- return;
+ if (!lastUsed.containsKey(tenantKey)) {
+ evictIdle(now);
+ if (lastUsed.size() >= maximumActiveClients) {
+ throw MongoOperationRejectedException.of(
+ "tenancy.client",
+ "the tenant client cap of "
+ + maximumActiveClients
+ + " is reached and no client is idle; each tenant client carries its own"
+ + " connection pool and monitoring threads, so the cap is what stops one process"
+ + " from exhausting the cluster's connections");
+ }
+ clients.computeIfAbsent(tenantKey, clientFactory);
}
- evictIdle(now);
- if (lastUsed.size() >= maximumActiveClients) {
- throw MongoOperationRejectedException.of(
- "tenancy.client",
- "the tenant client cap of "
- + maximumActiveClients
- + " is reached and no client is idle; each tenant client carries its own connection "
- + "pool and monitoring threads, so the cap is what stops one process from exhausting "
- + "the cluster's connections");
- }
- clients.computeIfAbsent(tenantKey, clientFactory);
lastUsed.put(tenantKey, now);
+ leases.merge(tenantKey, 1, Integer::sum);
+ return new HeldLease(tenantKey, clients.get(tenantKey));
}
- /** Releases a tenant's client and closes it. */
- public synchronized void release(String tenantKey) {
- lastUsed.remove(Objects.requireNonNull(tenantKey, "tenantKey"));
- close(tenantKey);
+ /**
+ * Releases a tenant's client and closes it.
+ *
+ * @throws IllegalStateException when a lease on the client is still open, or the close fails
+ */
+ public void release(String tenantKey) {
+ Objects.requireNonNull(tenantKey, "tenantKey");
+ synchronized (lock) {
+ if (leases.containsKey(tenantKey)) {
+ throw new IllegalStateException(
+ "the client for tenant '" + tenantKey + "' is still in use and cannot be released");
+ }
+ lastUsed.remove(tenantKey);
+ close(tenantKey);
+ }
+ }
+
+ /**
+ * How many evicted clients failed to close.
+ *
+ * The registry has dropped each of them and freed its slot; the count is how an operator
+ * learns that clients — and their pools — may still be open behind it.
+ */
+ public long evictionCloseFailures() {
+ return evictionCloseFailures.get();
+ }
+
+ /**
+ * One use of a tenant client; close it when the work with that client is done.
+ *
+ *
An interface so the registry's own implementation stays private: only the registry makes a
+ * lease, and only through {@link #acquire}.
+ */
+ public interface Lease extends AutoCloseable {
+
+ /** The tenant's client, valid until this lease is closed. */
+ AutoCloseable client();
+
+ /** Ends this use. Closing twice is the same as closing once. */
+ @Override
+ void close();
+ }
+
+ private final class HeldLease implements Lease {
+
+ private final String tenantKey;
+
+ private final AutoCloseable client;
+
+ private boolean closed;
+
+ private HeldLease(String tenantKey, AutoCloseable client) {
+ this.tenantKey = tenantKey;
+ this.client = client;
+ }
+
+ @Override
+ public AutoCloseable client() {
+ return client;
+ }
+
+ @Override
+ public void close() {
+ synchronized (lock) {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ leases.computeIfPresent(tenantKey, (tenant, open) -> open == 1 ? null : open - 1);
+ // Idle from the end of the last use, not its start: a client held for half an hour was in
+ // use all that time, and measuring from the acquire evicted it the moment it was let go.
+ Instant released = clock.instant();
+ lastUsed.computeIfPresent(
+ tenantKey, (tenant, used) -> released.isAfter(used) ? released : used);
+ }
+ }
+ }
+
+ /**
+ * How many leases are open now, across all tenants.
+ *
+ *
A lease nobody closed keeps its client's slot for good; this is how that becomes visible.
+ */
+ public int openLeases() {
+ synchronized (lock) {
+ return leases.values().stream().mapToInt(Integer::intValue).sum();
+ }
}
/** The tenants with an active client. */
@@ -128,8 +237,9 @@ public final class MongoTenantClientRegistry {
}
private void evictIdle(Instant now) {
- java.util.List idle =
+ List idle =
lastUsed.entrySet().stream()
+ .filter(entry -> !leases.containsKey(entry.getKey()))
.filter(entry -> entry.getValue().plus(idleEviction).isBefore(now))
.map(Map.Entry::getKey)
.toList();
@@ -138,7 +248,11 @@ public final class MongoTenantClientRegistry {
lastUsed.remove(tenant);
// Evicting the bookkeeping entry without closing the client frees the slot and keeps the
// connection pool, which is the exhaustion this cap exists to prevent, delayed.
- close(tenant);
+ try {
+ close(tenant);
+ } catch (IllegalStateException closeFailed) {
+ evictionCloseFailures.incrementAndGet();
+ }
});
}
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java
index 8ed663b7..0f0a91d4 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutor.java
@@ -6,20 +6,30 @@ import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer;
import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry;
import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget;
import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
+import java.util.stream.Stream;
+import org.bson.Document;
+import org.bson.RawBsonDocument;
+import org.bson.codecs.Codec;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
-import org.springframework.data.mongodb.core.aggregation.AggregationResults;
/**
* Runs an aggregation only after its stages, budget and lookups have been checked (design §17).
*
* Every guard runs before the first stage reaches the server, because an aggregation that is
- * going to be refused should cost nothing. The one guard that runs afterwards is the result-count
- * check: {@code $limit} bounds what the pipeline emits, but a pipeline whose own shape produced
- * more than the budget allows has still told the caller something worth failing over.
+ * going to be refused should cost nothing. Two run while the results arrive: the result count and
+ * the result bytes. The pipeline gets a final {@code $limit} of one document past the count budget,
+ * so the server never produces more than that, and the results are read from the cursor one
+ * document at a time, the encoded size of each added up as it arrives. They used to be read whole —
+ * every document in memory before anything was counted — and the byte budget was declared and never
+ * checked (round9 review 2026-11-12, RR9-01-20; {@code docs/mongodb/query-aggregation-guide.md}
+ * 「집계에 예산을 둔다」).
*/
public final class PolicyAwareMongoAggregationExecutor {
@@ -53,6 +63,14 @@ public final class PolicyAwareMongoAggregationExecutor {
/**
* Executes a typed aggregation.
*
+ *
Results are read as {@link Document}, sized, then mapped with the template's converter. Two
+ * costs follow, both deliberate. Each document is encoded once more to be measured: the template
+ * hands back decoded documents, so the bytes the server sent are not available, and the budget
+ * has to be in bytes. And the mapping is the converter's alone: the template's typed aggregate
+ * would also publish {@code AfterLoadEvent}/{@code AfterConvertEvent} and run entity callbacks,
+ * and those do not run for an aggregation result read here. A listener that needs them maps from
+ * the returned type itself.
+ *
* @throws MongoOperationRejectedException when a stage, lookup, budget or result size is not
* permitted
*/
@@ -71,19 +89,38 @@ public final class PolicyAwareMongoAggregationExecutor {
MongoOperationBudget effective =
enforcer.narrow(budget, budget.narrowedTo(budgetFor(context, budget)));
+ List bounded = new ArrayList<>(plan.operations());
+ // One past the budget: enough to know the budget was exceeded, never more than that.
+ bounded.add(Aggregation.limit(effective.maxResults() + 1L));
Aggregation aggregation =
- Aggregation.newAggregation(plan.operations()).withOptions(optionsFor(profile, effective));
- AggregationResults results = operations.aggregate(aggregation, collection, outputType);
- List mapped = results.getMappedResults();
- if (mapped.size() > effective.maxResults()) {
- throw MongoOperationRejectedException.of(
- "aggregation.result",
- "the aggregation returned "
- + mapped.size()
- + " documents, above the budget of "
- + effective.maxResults());
+ Aggregation.newAggregation(bounded).withOptions(optionsFor(profile, effective));
+ Codec sizing = operations.getConverter().getCodecRegistry().get(Document.class);
+ List mapped = new ArrayList<>();
+ long bytes = 0;
+ try (Stream documents =
+ operations.aggregateStream(aggregation, collection, Document.class)) {
+ Iterator cursor = documents.iterator();
+ while (cursor.hasNext()) {
+ Document document = cursor.next();
+ if (mapped.size() == effective.maxResults()) {
+ throw MongoOperationRejectedException.of(
+ "aggregation.result",
+ "the aggregation returned more than the budget of "
+ + effective.maxResults()
+ + " documents");
+ }
+ bytes += new RawBsonDocument(document, sizing).getByteBuffer().remaining();
+ if (bytes > effective.maxResultBytes()) {
+ throw MongoOperationRejectedException.of(
+ "aggregation.result",
+ "the aggregation returned more than the budget of "
+ + effective.maxResultBytes()
+ + " bytes");
+ }
+ mapped.add(operations.getConverter().read(outputType, document));
+ }
}
- return mapped;
+ return List.copyOf(mapped);
}
private static MongoOperationBudget budgetFor(
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java
index 0466ca15..855db8ec 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLedger.java
@@ -36,20 +36,46 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
private static final String MIGRATION_ID = "migrationId";
+ private static final String FENCE = "fence";
+
private final MongoCollection ledger;
private final MongoCollection checkpoints;
+ private final MongoCollection locks;
+
+ /**
+ * A ledger over the default collections of {@code database}, the lock's included.
+ *
+ * The lock collection is {@link MongoCollectionMigrationLock#DEFAULT_COLLECTION}. A deployment
+ * that keeps its runner lease in a collection of another name uses the three-collection
+ * constructor with that collection; with this one the ledger would read a lock that is never
+ * taken and refuse nothing on its account.
+ *
+ * @param database the database being migrated
+ */
public MongoCollectionMigrationLedger(MongoDatabase database) {
this(
Objects.requireNonNull(database, "database").getCollection(DEFAULT_COLLECTION),
- database.getCollection(DEFAULT_CHECKPOINT_COLLECTION));
+ database.getCollection(DEFAULT_CHECKPOINT_COLLECTION),
+ database.getCollection(MongoCollectionMigrationLock.DEFAULT_COLLECTION));
}
+ /**
+ * A ledger over explicit collections.
+ *
+ * @param ledger the applied migrations
+ * @param checkpoints the running migrations' positions
+ * @param locks the collection {@link MongoCollectionMigrationLock} keeps the runner lease in,
+ * read to refuse a replaced runner's ledger entry
+ */
public MongoCollectionMigrationLedger(
- MongoCollection ledger, MongoCollection checkpoints) {
+ MongoCollection ledger,
+ MongoCollection checkpoints,
+ MongoCollection locks) {
this.ledger = Objects.requireNonNull(ledger, "ledger");
this.checkpoints = Objects.requireNonNull(checkpoints, "checkpoints");
+ this.locks = Objects.requireNonNull(locks, "locks");
}
/**
@@ -90,13 +116,34 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
Objects.requireNonNull(checksum, "checksum");
Objects.requireNonNull(operator, "operator");
Objects.requireNonNull(appliedAt, "appliedAt");
- requireCurrentFence(fence, "ledger entry for " + migrationId.value());
+ requireFenced(fence, "ledger entry for " + migrationId.value());
+ // A superseded runner that wakes up after its replacement took the lease must not record the
+ // migration as applied under it (round9 review 2026-11-12, RR9-01-28). Two pieces of evidence:
+ // the lock's newest acquisition, which a replacement leaves the moment it takes the lease, and
+ // a checkpoint a newer runner wrote.
+ //
+ // A narrowing, not a guarantee: the reads and the insert are separate operations, so a runner
+ // replaced between them still records. What stops a superseded runner first is the lock's
+ // refresh, which refuses it; this check is the second line for a runner that skipped a
+ // refresh. Making it atomic needs the lock and the ledger in one transaction, which a
+ // standalone deployment does not have.
+ if (MongoCollectionMigrationLock.newestFence(locks) > fence
+ || checkpoints
+ .find(
+ Filters.and(
+ Filters.eq(MIGRATION_ID, migrationId.value()), Filters.gt(FENCE, fence)))
+ .first()
+ != null) {
+ throw MongoOperationRejectedException.of(
+ "migration.fence",
+ "a newer migration runner owns the lease; this runner's ledger entry was refused");
+ }
ledger.insertOne(
new Document(MIGRATION_ID, migrationId.value())
.append("checksum", checksum.value())
.append("operator", operator)
.append("appliedAt", Date.from(appliedAt))
- .append("fence", fence));
+ .append(FENCE, fence));
}
@Override
@@ -117,7 +164,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
@Override
public void saveCheckpoint(MongoMigrationCheckpoint checkpoint, long fence) {
Objects.requireNonNull(checkpoint, "checkpoint");
- requireCurrentFence(fence, "checkpoint for " + checkpoint.migrationId().value());
+ requireFenced(fence, "checkpoint for " + checkpoint.migrationId().value());
// Conditioned on the stored fence as well as matched by id: a superseded runner that wakes up
// mid-batch must not move the checkpoint of the runner that replaced it, which would send the
// live migration back over rows it has already done or forward past rows it has not.
@@ -126,7 +173,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
.append("resumePosition", checkpoint.resumePosition())
.append("processedCount", checkpoint.processedCount())
.append("updatedAt", Date.from(checkpoint.updatedAt()))
- .append("fence", fence);
+ .append(FENCE, fence);
// Replace-then-insert rather than one upsert, and the two cases this separates are the whole
// behaviour of the fence.
@@ -147,7 +194,7 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
.replaceOne(
Filters.and(
Filters.eq(MIGRATION_ID, checkpoint.migrationId().value()),
- Filters.or(Filters.exists("fence", false), Filters.lte("fence", fence))),
+ Filters.or(Filters.exists(FENCE, false), Filters.lte(FENCE, fence))),
document,
new ReplaceOptions().upsert(false))
.getMatchedCount();
@@ -172,9 +219,13 @@ public final class MongoCollectionMigrationLedger implements MongoMigrationLedge
* Refuses a write from an unfenced lease.
*
* An unfenced lease cannot prove the writer is still the owner, and the whole point of the
- * ledger is that it says what actually happened.
+ * ledger is that it says what actually happened. Whether a fenced writer is still the current one
+ * is decided in two places, not here: the runner refreshes its lease, conditioned on the fence,
+ * right before it writes, and each write is conditioned on the newest fence the ledger has seen
+ * for the migration. The method was called {@code requireCurrentFence} and checked neither
+ * (round9 review 2026-11-12, RR9-01-28).
*/
- private static void requireCurrentFence(long fence, String what) {
+ private static void requireFenced(long fence, String what) {
if (fence == MongoMigrationLock.UNFENCED) {
throw MongoOperationRejectedException.of(
"migration.fence",
diff --git a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java
index cca219af..94c4ac26 100644
--- a/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java
+++ b/src/adapter/outbound/persistence-mongo/src/main/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoCollectionMigrationLock.java
@@ -6,6 +6,7 @@ import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Updates;
+import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
@@ -125,6 +126,19 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock {
return fence;
}
+ /**
+ * The fence of the newest acquisition of the lock in {@code locks}, or {@link #UNFENCED} when the
+ * lock was never taken there.
+ *
+ *
For the ledger: a write whose fence is older than this one comes from a runner that has been
+ * replaced, whether or not its replacement has written anything yet.
+ */
+ static long newestFence(MongoCollection locks) {
+ Document current =
+ Objects.requireNonNull(locks, "locks").find(Filters.eq("_id", LOCK_ID)).first();
+ return current == null ? UNFENCED : fenceOf(current);
+ }
+
private static long fenceOf(Document lock) {
Object stored = lock.get("fence");
return stored instanceof Number number ? number.longValue() : 1L;
@@ -165,7 +179,11 @@ public final class MongoCollectionMigrationLock implements MongoMigrationLock {
.getMatchedCount();
if (stillOwned != 1) {
held = false;
- throw new IllegalStateException(
+ // The refusal the heartbeat promises (MongoMigrationHeartbeat#beat), so a migration that
+ // stops on it stops for the same reason the heartbeat's own ownership check gives. It was an
+ // IllegalStateException, which read as a server defect (round9 review 2026-11-12, RR9-01-28).
+ throw MongoOperationRejectedException.of(
+ "migration.lock",
"the migration lease was lost before it could be refreshed; another runner may have "
+ "taken it after this one appeared to stall");
}
diff --git a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java
index abbcad9c..edb28966 100644
--- a/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java
+++ b/src/adapter/outbound/persistence-mongo/src/mongoIntegrationTest/java/dev/caskeleton/adapter/outbound/mongo/migration/MongoMigrationLaneTest.java
@@ -224,6 +224,95 @@ class MongoMigrationLaneTest {
assertThat(ledger.findCheckpoint(id).orElseThrow().processedCount()).isEqualTo(200);
}
+ @Test
+ void aSupersededRunnerCannotRecordTheMigrationAppliedUnderItsReplacement() {
+ // given
+ MongoDatabase database = freshDatabase();
+ MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database);
+ ledger.ensureIndexes();
+ MongoMigrationId id = new MongoMigrationId("20260811-011");
+ ledger.saveCheckpoint(
+ MongoMigrationCheckpoint.start(id, FIXED.instant())
+ .advancedTo("o-300", 300, FIXED.instant()),
+ 2L);
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ ledger.recordApplied(
+ id, new MongoMigrationChecksum("abc"), "operator", FIXED.instant(), 1L));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("a newer migration runner owns the lease");
+ }
+
+ @Test
+ void aReplacedRunnerCannotRecordTheMigrationBeforeItsReplacementWroteAnything() {
+ // given
+ MongoDatabase database = freshDatabase();
+ MongoCollectionMigrationLedger ledger = new MongoCollectionMigrationLedger(database);
+ ledger.ensureIndexes();
+ MongoMigrationId id = new MongoMigrationId("20260811-012");
+ Instant early = Instant.parse("2026-08-11T00:00:00Z");
+ MongoMigrationLock stalled =
+ MongoCollectionMigrationLock.tryAcquire(
+ database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC))
+ .orElseThrow();
+ var replacement =
+ MongoCollectionMigrationLock.tryAcquire(
+ database,
+ "next-runner",
+ LEASE,
+ Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC))
+ .orElseThrow();
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ ledger.recordApplied(
+ id,
+ new MongoMigrationChecksum("abc"),
+ "operator",
+ FIXED.instant(),
+ stalled.fence()));
+
+ // then
+ assertThat(refused)
+ .as("%s holds the lease with a newer fence and has not checkpointed yet", replacement)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("a newer migration runner owns the lease");
+ }
+
+ @Test
+ void aLeaseLostBeforeItsRefreshIsTheRefusalTheHeartbeatPromises() {
+ // given
+ MongoDatabase database = freshDatabase();
+ Instant early = Instant.parse("2026-08-11T00:00:00Z");
+ MongoMigrationLock stalled =
+ MongoCollectionMigrationLock.tryAcquire(
+ database, "stalled-runner", LEASE, Clock.fixed(early, ZoneOffset.UTC))
+ .orElseThrow();
+ var unused =
+ MongoCollectionMigrationLock.tryAcquire(
+ database,
+ "next-runner",
+ LEASE,
+ Clock.fixed(early.plus(LEASE).plusSeconds(1), ZoneOffset.UTC))
+ .orElseThrow();
+
+ // when
+ Throwable refused = catchThrowable(() -> stalled.refresh(LEASE));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("the migration lease was lost");
+ }
+
@Test
void theLedgerRefusesADoubleApplyOfTheSameId() {
MongoDatabase database = freshDatabase();
diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java
index 97309f29..06f1316e 100644
--- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java
+++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/advanced/tenancy/MongoTenancyContractTest.java
@@ -3,6 +3,7 @@ package dev.caskeleton.adapter.outbound.mongo.advanced.tenancy;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityFlags;
import dev.caskeleton.adapter.outbound.mongo.advanced.MongoAdvancedCapabilityGuard;
@@ -19,10 +20,25 @@ import dev.caskeleton.adapter.outbound.mongo.imperative.atomic.AtomicFilter;
import dev.caskeleton.adapter.outbound.mongo.migration.MongoMigrationCheckpoint;
import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoCollectionManifest;
import dev.caskeleton.adapter.outbound.mongo.schema.manifest.MongoIndexManifest;
+import java.io.IOException;
+import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -37,6 +53,8 @@ class MongoTenancyContractTest {
MongoAdvancedCapabilityFlags.allDisabled()
.withEnabled(MongoCapability.DATABASE_PER_TENANT));
+ private static final Instant START = Instant.parse("2026-08-13T00:00:00Z");
+
@Test
void missingTenantContextFailsClosed() {
assertThatThrownBy(() -> injector.apply(Optional.empty(), AtomicFilter.id("o-1")))
@@ -116,29 +134,99 @@ class MongoTenancyContractTest {
@Test
void registryEnforcesMaximumActiveClients() {
+ // given
MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2);
- Instant now = Instant.parse("2026-08-13T00:00:00Z");
+ registry.acquire("t1");
+ registry.acquire("t2");
- registry.acquire("t1", now);
- registry.acquire("t2", now);
+ // when
+ Throwable refused = catchThrowable(() -> registry.acquire("t3"));
- assertThatThrownBy(() -> registry.acquire("t3", now))
- .isInstanceOf(MongoOperationRejectedException.class);
- assertThat(registry.activeCount()).isEqualTo(2);
+ // then
+ assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.activeCount()))
+ .containsExactly(true, 2);
}
@Test
void anIdleClientIsEvictedToMakeRoom() {
- MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(TENANCY, 1, Duration.ofMinutes(5));
- Instant start = Instant.parse("2026-08-13T00:00:00Z");
+ // given
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock);
+ registry.acquire("t1").close();
+ clock.advance(Duration.ofMinutes(10));
- registry.acquire("t1", start);
- registry.acquire("t2", start.plus(Duration.ofMinutes(10)));
+ // when
+ registry.acquire("t2");
+ // then
assertThat(registry.activeTenants()).containsExactly("t2");
}
+ @Test
+ void aClientIsIdleFromTheEndOfItsLastUseNotItsStart() {
+ // given
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> {}, clock);
+ MongoTenantClientRegistry.Lease held = registry.acquire("t1");
+ clock.advance(Duration.ofMinutes(30));
+ held.close();
+ clock.advance(Duration.ofMinutes(1));
+
+ // when
+ Throwable refused = catchThrowable(() -> registry.acquire("t2"));
+
+ // then
+ assertThat(List.of(refused instanceof MongoOperationRejectedException, registry.openLeases()))
+ .as("held for thirty minutes, it was in use all that time and is not idle yet")
+ .containsExactly(true, 0);
+ }
+
+ @Test
+ void anUnclosedLeaseIsCounted() {
+ // given
+ MongoTenantClientRegistry registry = new MongoTenantClientRegistry(TENANCY, 2);
+
+ // when
+ var unused = registry.acquire("t1");
+
+ // then
+ assertThat(registry.openLeases()).isEqualTo(1);
+ }
+
+ private static MongoTenantClientRegistry registry(
+ int cap, Function clients, Clock clock) {
+ return new MongoTenantClientRegistry(TENANCY, cap, Duration.ofMinutes(5), clients, clock);
+ }
+
+ /** A clock a test moves by hand: one time source for acquiring, closing and eviction. */
+ private static final class MovingClock extends Clock {
+
+ private final AtomicReference now;
+
+ private MovingClock(Instant start) {
+ this.now = new AtomicReference<>(start);
+ }
+
+ private void advance(Duration by) {
+ now.updateAndGet(instant -> instant.plus(by));
+ }
+
+ @Override
+ public ZoneId getZone() {
+ return ZoneOffset.UTC;
+ }
+
+ @Override
+ public Clock withZone(ZoneId zone) {
+ return this;
+ }
+
+ @Override
+ public Instant instant() {
+ return now.get();
+ }
+ }
+
@Test
void aTenantDatabaseCannotServeBeforeItsSchemaIsValidated() {
MongoTenantLifecyclePolicy policy = MongoTenantLifecyclePolicy.standard();
@@ -172,100 +260,112 @@ class MongoTenancyContractTest {
}
@Test
- @org.junit.jupiter.api.DisplayName("the same tenant is created once under concurrency")
- void theSameTenantIsCreatedOnce() throws Exception {
- java.util.concurrent.atomic.AtomicInteger created =
- new java.util.concurrent.atomic.AtomicInteger();
+ void theSameTenantIsCreatedOnceUnderConcurrency() throws Exception {
+ // given
+ AtomicInteger created = new AtomicInteger();
MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(
- TENANCY,
+ registry(
8,
- Duration.ofMinutes(10),
tenant -> {
created.incrementAndGet();
return () -> {};
- });
- java.util.concurrent.ExecutorService pool =
- java.util.concurrent.Executors.newFixedThreadPool(8);
+ },
+ new MovingClock(START));
+ List> work =
+ IntStream.range(0, 50)
+ .>mapToObj(
+ index ->
+ () -> {
+ registry.acquire("t1").close();
+ return null;
+ })
+ .toList();
+ ExecutorService pool = Executors.newFixedThreadPool(8);
+
+ // when
try {
- java.util.List> work =
- java.util.stream.IntStream.range(0, 50)
- .>mapToObj(
- index ->
- () -> {
- registry.acquire("t1", Instant.parse("2026-08-13T00:00:00Z"));
- return null;
- })
- .toList();
- for (java.util.concurrent.Future future : pool.invokeAll(work)) {
+ for (Future future : pool.invokeAll(work)) {
future.get();
}
} finally {
pool.shutdownNow();
}
- assertThat(created)
+ // then
+ assertThat(List.of(created.get(), registry.activeCount()))
.as("a per-tenant client carries a pool and monitoring threads; a second one doubles both")
- .hasValue(1);
- assertThat(registry.activeCount()).isEqualTo(1);
+ .containsExactly(1, 1);
}
@Test
- @org.junit.jupiter.api.DisplayName("acquire(tenant)는 주입된 clock으로 idle 여부를 판단한다")
- void anAcquireWithoutAnInstantReadsTheInjectedClock() {
+ void aClientStillHeldIsNotEvictedHoweverLongAgoItWasAcquired() {
// given
- java.util.List closed = new java.util.ArrayList<>();
- java.util.concurrent.atomic.AtomicReference now =
- new java.util.concurrent.atomic.AtomicReference<>(Instant.parse("2026-08-13T00:00:00Z"));
- java.time.Clock clock =
- new java.time.Clock() {
- @Override
- public java.time.ZoneId getZone() {
- return java.time.ZoneOffset.UTC;
- }
+ List closed = new ArrayList<>();
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock);
+ MongoTenantClientRegistry.Lease longRunning = registry.acquire("t1");
+ clock.advance(Duration.ofMinutes(10));
- @Override
- public java.time.Clock withZone(java.time.ZoneId zone) {
- return this;
- }
+ // when
+ Throwable refused = catchThrowable(() -> registry.acquire("t2"));
- @Override
- public Instant instant() {
- return now.get();
- }
- };
+ // then
+ assertThat(List.of(refused instanceof MongoOperationRejectedException, closed))
+ .as(
+ "t1 is in use by %s; closing it would pull the pool out from under that work",
+ longRunning)
+ .containsExactly(true, List.of());
+ }
+
+ @Test
+ void anotherTenantsClientThatFailsToCloseDoesNotFailThisAcquire() {
+ // given
+ MovingClock clock = new MovingClock(START);
MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(
- TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant), clock);
- registry.acquire("t1");
- now.set(now.get().plus(Duration.ofMinutes(10)));
+ registry(
+ 1,
+ tenant ->
+ () -> {
+ if (tenant.equals("t1")) {
+ throw new IOException("t1's pool would not shut down");
+ }
+ },
+ clock);
+ registry.acquire("t1").close();
+ clock.advance(Duration.ofMinutes(10));
+
+ // when
+ MongoTenantClientRegistry.Lease admitted = registry.acquire("t2");
+
+ // then
+ assertThat(
+ List.of(
+ admitted.client() != null,
+ registry.activeTenants(),
+ registry.evictionCloseFailures()))
+ .containsExactly(true, Set.of("t2"), 1L);
+ }
+
+ @Test
+ void evictionClosesTheClientItEvicts() {
+ // given
+ List closed = new ArrayList<>();
+ MovingClock clock = new MovingClock(START);
+ MongoTenantClientRegistry registry = registry(1, tenant -> () -> closed.add(tenant), clock);
+ registry.acquire("t1").close();
+ clock.advance(Duration.ofMinutes(10));
// when
registry.acquire("t2");
// then
- assertThat(closed).containsExactly("t1");
- }
-
- @Test
- @org.junit.jupiter.api.DisplayName("eviction closes the client it evicts")
- void evictionClosesTheClient() {
- java.util.List closed = new java.util.ArrayList<>();
- MongoTenantClientRegistry registry =
- new MongoTenantClientRegistry(
- TENANCY, 1, Duration.ofMinutes(5), tenant -> () -> closed.add(tenant));
- Instant start = Instant.parse("2026-08-13T00:00:00Z");
-
- registry.acquire("t1", start);
- registry.acquire("t2", start.plus(Duration.ofMinutes(10)));
-
assertThat(closed)
.as("freeing the slot without closing the client keeps the connection pool it counted")
.containsExactly("t1");
}
@Test
- @org.junit.jupiter.api.DisplayName("a coordinator resumes from its durable store after a restart")
+ @DisplayName("a coordinator resumes from its durable store after a restart")
void aCoordinatorResumesFromItsDurableStore() {
RecordingCheckpointStore store = new RecordingCheckpointStore();
MongoTenantMigrationCoordinator first =
diff --git a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java
index 89fd5ff0..bcf90f54 100644
--- a/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java
+++ b/src/adapter/outbound/persistence-mongo/src/test/java/dev/caskeleton/adapter/outbound/mongo/aggregation/PolicyAwareMongoAggregationExecutorTest.java
@@ -3,11 +3,31 @@ package dev.caskeleton.adapter.outbound.mongo.aggregation;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import com.mongodb.MongoClientSettings;
+import dev.caskeleton.adapter.outbound.mongo.api.MongoOperationContext;
+import dev.caskeleton.adapter.outbound.mongo.api.consistency.MongoConsistencyProfile;
import dev.caskeleton.adapter.outbound.mongo.api.error.MongoOperationRejectedException;
+import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetEnforcer;
+import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoBudgetPolicyRegistry;
+import dev.caskeleton.adapter.outbound.mongo.query.budget.MongoOperationBudget;
+import java.time.Duration;
+import java.util.List;
import java.util.Set;
+import org.bson.Document;
+import org.bson.conversions.Bson;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.data.mongodb.core.MongoOperations;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.convert.MongoConverter;
+import org.springframework.data.mongodb.core.query.Criteria;
/** Design §17 — write stages are D4, and stage cost is graded rather than assumed. */
@Tag("mongodb-contract")
@@ -105,4 +125,141 @@ class PolicyAwareMongoAggregationExecutorTest {
assertThat(MongoAggregationProfile.stableRead().allowDiskUse()).isFalse();
assertThat(MongoAggregationProfile.stableRead().strictMapping()).isTrue();
}
+
+ private static final MongoOperationContext CONTEXT =
+ MongoOperationContext.of(
+ "orders.summary",
+ "primary",
+ "orders",
+ MongoConsistencyProfile.PRIMARY_MAJORITY,
+ Duration.ofSeconds(2));
+
+ private static final MongoAggregationPlan OPEN_ORDERS =
+ MongoAggregationPlan.builder()
+ .stage("$match", Aggregation.match(Criteria.where("status").is("OPEN")))
+ .build();
+
+ @Test
+ void thePipelineEndsWithALimitOnePastTheCountBudget() {
+ // given
+ ArgumentCaptor sent = ArgumentCaptor.forClass(Aggregation.class);
+ MongoOperations operations = returning(List.of(), sent);
+
+ // when
+ var unused =
+ executorOver(operations, budget(3, 1_048_576L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class);
+
+ // then
+ List pipeline = sent.getValue().toPipeline(Aggregation.DEFAULT_CONTEXT);
+ assertThat(pipeline.get(pipeline.size() - 1)).isEqualTo(new Document("$limit", 4L));
+ }
+
+ @Test
+ void moreDocumentsThanTheCountBudgetAreRefused() {
+ // given
+ MongoOperations operations =
+ returning(
+ List.of(order(1), order(2), order(3), order(4)),
+ ArgumentCaptor.forClass(Aggregation.class));
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ executorOver(operations, budget(3, 1_048_576L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("more than the budget of 3 documents");
+ }
+
+ @Test
+ void documentsLargerTogetherThanTheByteBudgetAreRefused() {
+ // given
+ // Each order encodes to well over 40 bytes: two of them pass any 80-byte budget.
+ MongoOperations operations =
+ returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class));
+
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ executorOver(operations, budget(10, 80L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(MongoOperationRejectedException.class)
+ .hasMessageContaining("more than the budget of 80 bytes");
+ }
+
+ @Test
+ void resultsInsideBothBudgetsAreReturnedInCursorOrder() {
+ // given
+ MongoOperations operations =
+ returning(List.of(order(1), order(2)), ArgumentCaptor.forClass(Aggregation.class));
+
+ // when
+ List results =
+ executorOver(operations, budget(10, 1_048_576L))
+ .execute(
+ CONTEXT,
+ MongoAggregationProfile.stableRead(),
+ OPEN_ORDERS,
+ "orders",
+ Document.class);
+
+ // then
+ assertThat(results).containsExactly(order(1), order(2));
+ }
+
+ private static MongoOperationBudget budget(int maxResults, long maxResultBytes) {
+ return new MongoOperationBudget(maxResults, maxResultBytes, 500L, 50);
+ }
+
+ private static PolicyAwareMongoAggregationExecutor executorOver(
+ MongoOperations operations, MongoOperationBudget budget) {
+ return new PolicyAwareMongoAggregationExecutor(
+ operations,
+ MongoBudgetPolicyRegistry.builder().register("orders.summary", budget).build(),
+ new MongoBudgetEnforcer());
+ }
+
+ /** The driver boundary: a cursor over {@code documents}, mapped as the converter would. */
+ private static MongoOperations returning(
+ List documents, ArgumentCaptor sent) {
+ MongoOperations operations = mock(MongoOperations.class);
+ MongoConverter converter = mock(MongoConverter.class);
+ when(operations.getConverter()).thenReturn(converter);
+ when(converter.getCodecRegistry()).thenReturn(MongoClientSettings.getDefaultCodecRegistry());
+ when(converter.read(eq(Document.class), any(Bson.class)))
+ .thenAnswer(invocation -> invocation.getArgument(1));
+ when(operations.aggregateStream(sent.capture(), eq("orders"), eq(Document.class)))
+ .thenAnswer(invocation -> documents.stream());
+ return operations;
+ }
+
+ private static Document order(int number) {
+ return new Document("_id", "order-" + number)
+ .append("status", "OPEN")
+ .append("customer", "customer-" + number);
+ }
}
diff --git a/src/adapter/outbound/persistence-notification/AGENTS.md b/src/adapter/outbound/persistence-notification/AGENTS.md
index 94fa03ad..cb9ae8b1 100644
--- a/src/adapter/outbound/persistence-notification/AGENTS.md
+++ b/src/adapter/outbound/persistence-notification/AGENTS.md
@@ -8,7 +8,8 @@
- 패키지 루트: `dev.caskeleton.adapter.outbound.persistence.notification`. 패키지 규칙은
`persistence-fileserver`와 같아서 두 capability leaf를 같은 방식으로 읽는다:
- `platform.entity` — JPA 행만.
- - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`).
+ - `platform.repository` — Spring Data 인터페이스와 그것이 돌리는 SQL 글(`RecipientClaimSql`). `platform`의 store도
+ 이 SQL 조각을 쓴다.
- `platform` — store 어댑터(`Jpa*`, `Jdbc*`, inbox의 `JpaNotificationInbox`와 그 outbox 도우미),
`NotificationRecordMapper`, payload codec.
- `configuration` — store를 bean으로 등록하는 자동 구성 창구. leaf 루트에는 scan 설정과 schema stream이 있다.
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java
index 5a81e94b..72a52a8e 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingState.java
@@ -1,13 +1,17 @@
package dev.caskeleton.adapter.outbound.persistence.notification.platform;
+import dev.caskeleton.adapter.outbound.persistence.notification.platform.repository.RecipientClaimSql;
import dev.caskeleton.application.notification.platform.observation.NotificationServingState;
import dev.caskeleton.application.notification.platform.observation.NotificationServingStatePort;
import java.sql.Timestamp;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.Map;
import java.util.Objects;
import org.springframework.jdbc.core.JdbcOperations;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
/**
* Measures the platform's serving state from the durable rows.
@@ -17,34 +21,44 @@ import org.springframework.jdbc.core.JdbcOperations;
* identical. These are the questions an operator actually asks during an incident: how far behind
* are we, how long has the oldest thing waited, and is anything stuck.
*
- * One statement per question rather than one joined statement. They are read independently, a
- * slow one does not delay the others, and each index the queries rely on is the same index the
- * corresponding worker already uses to claim.
+ *
One statement per question rather than one joined statement. They are read independently and a
+ * slow one does not delay the others. Every statement is bounded by an index: the backlog and
+ * stuck-lease counts by the predicates the claim uses, the reconciliation count by the job's due
+ * time, and the projection lag by {@code ix_notification_provider_event_open} (V14), which holds
+ * only the events still waiting for something — the provider-event table itself is never purged, so
+ * the lag query used to read all of it (round9 review 2026-11-12, RR9-01-14).
+ *
+ *
The scheduler reads only {@link #backlogDepth()}, on every poll; the full snapshot is for the
+ * health endpoint.
*/
public final class JdbcNotificationServingState implements NotificationServingStatePort {
/**
- * Deliveries that are due and unclaimed.
+ * Deliveries the claim would take now.
*
- *
The lease predicate matches the claim's: a row whose lease has expired is available again,
- * so it is backlog. Counting it as in-flight is how a stuck worker hides a growing queue.
+ *
The {@code WHERE} is the claim's own fragment ({@link RecipientClaimSql#CLAIMABLE_AT_NOW}):
+ * the same states, due, not expired, and a lease that is absent or has run out. A row whose lease
+ * has expired is available again, so it is backlog; counting it as in-flight is how a stuck
+ * worker hides a growing queue. An expired delivery is not backlog: the expiry sweep retires it
+ * unsent. The fragment's parameter is named, so this statement is too.
*/
private static final String BACKLOG =
"""
SELECT count(*) AS depth,
- COALESCE(EXTRACT(EPOCH FROM (? - min(next_dispatch_at))), 0) AS oldest_seconds
+ COALESCE(EXTRACT(EPOCH FROM (:now - min(next_dispatch_at))), 0) AS oldest_seconds
FROM notification_recipient_delivery
- WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')
- AND next_dispatch_at IS NOT NULL
- AND next_dispatch_at <= ?
- AND (lease_until IS NULL OR lease_until <= ?)
- """;
+ WHERE\s"""
+ + RecipientClaimSql.CLAIMABLE_AT_NOW;
/**
* Leases whose holder never came back.
*
- *
Distinct from backlog: these rows are claimable again, but the fact that they had to expire
- * means a worker died mid-dispatch, and a provider call may have happened without a record.
+ *
Distinct from backlog: these rows are still dispatching, which the claim never takes; lease
+ * recovery requeues or reconciles them one lease after they ran out. A row left from before
+ * notification V4 (an owner but fence 0) is counted here and not recovered; {@code
+ * docs/upgrade/2026-11-round9-claim-fence.md}, row 5, says how to hand it to recovery. That a
+ * lease had to run out means a worker died or stalled mid-dispatch, and a provider call may have
+ * happened without a record.
*/
private static final String STUCK_LEASES =
"""
@@ -56,7 +70,21 @@ public final class JdbcNotificationServingState implements NotificationServingSt
AND delivery_state = 'DISPATCHING'
""";
- /** Provider events stored and not yet applied to a delivery. */
+ /**
+ * The provider events still waiting for something: the lag query's {@code WHERE}, and the
+ * predicate of the partial index {@code V14} builds for it. One string, held to the migration by
+ * {@code JdbcNotificationServingStateOpenEventsTest}: a query the index does not cover reads the
+ * whole table again, which is what the index was made to stop.
+ */
+ static final String OPEN_EVENTS =
+ "projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL";
+
+ /**
+ * Provider events stored and not yet applied to a delivery.
+ *
+ *
The {@code WHERE} is exactly the predicate of {@code ix_notification_provider_event_open}:
+ * every row any of the four counts can see, and none of the applied, matched history.
+ */
private static final String PROJECTION_LAG =
"""
SELECT
@@ -66,7 +94,9 @@ public final class JdbcNotificationServingState implements NotificationServingSt
COALESCE(EXTRACT(EPOCH FROM (? - min(received_at)
FILTER (WHERE projection_status = 'PENDING'))), 0) AS oldest_seconds
FROM notification_provider_event
- """;
+ WHERE
+ """
+ + OPEN_EVENTS;
/** Questions to providers that are due and unanswered. */
private static final String RECONCILIATION_LAG =
@@ -78,26 +108,36 @@ public final class JdbcNotificationServingState implements NotificationServingSt
""";
private final JdbcOperations jdbc;
+ private final NamedParameterJdbcOperations named;
private final Clock clock;
public JdbcNotificationServingState(JdbcOperations jdbc, Clock clock) {
this.jdbc = Objects.requireNonNull(jdbc, "jdbc");
+ this.named = new NamedParameterJdbcTemplate(jdbc);
this.clock = Objects.requireNonNull(clock, "clock");
}
+ @Override
+ public long backlogDepth() {
+ Measurement backlog = measureBacklog(Timestamp.from(clock.instant()));
+ return backlog == null ? 0 : backlog.count();
+ }
+
+ /** The backlog query alone: the one statement the scheduler's queue-depth gauge needs. */
+ private Measurement measureBacklog(Timestamp at) {
+ return named.queryForObject(
+ BACKLOG,
+ Map.of("now", at),
+ (rows, rowNumber) ->
+ new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")));
+ }
+
@Override
public NotificationServingState currentState() {
Instant now = clock.instant();
Timestamp at = Timestamp.from(now);
- Measurement backlog =
- jdbc.queryForObject(
- BACKLOG,
- (rows, rowNumber) ->
- new Measurement(rows.getLong("depth"), rows.getDouble("oldest_seconds")),
- at,
- at,
- at);
+ Measurement backlog = measureBacklog(at);
Long stuck = jdbc.queryForObject(STUCK_LEASES, Long.class, at);
ProjectionMeasurement projection =
jdbc.queryForObject(
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java
index 74728e9e..61b49e2c 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaAdminOperationStore.java
@@ -33,10 +33,10 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort {
@Override
public dev.caskeleton.application.notification.platform.admin.AdminOperationClaim claim(
- String operationId, String commandFingerprint, AdminActor actor, String action) {
+ String operationId, String requestFingerprint, AdminActor actor, String action) {
CallerTransaction.require("admin operation claim");
Objects.requireNonNull(operationId, "operationId");
- Objects.requireNonNull(commandFingerprint, "commandFingerprint");
+ Objects.requireNonNull(requestFingerprint, "requestFingerprint");
Objects.requireNonNull(actor, "actor");
Objects.requireNonNull(action, "action");
@@ -46,7 +46,7 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort {
operationId,
action,
actor.actorRef(),
- commandFingerprint,
+ requestFingerprint,
clock.instant());
if (claimed == 1) {
return dev.caskeleton.application.notification.platform.admin.AdminOperationClaim.claimed();
@@ -61,7 +61,7 @@ public final class JpaAdminOperationStore implements AdminOperationStorePort {
}
String phase = String.valueOf(existing.get(0)[0]);
String fingerprint = existing.get(0)[1] == null ? null : String.valueOf(existing.get(0)[1]);
- if (fingerprint != null && !fingerprint.equals(commandFingerprint)) {
+ if (fingerprint != null && !fingerprint.equals(requestFingerprint)) {
// The same id, a different command. Returning the earlier result would execute neither.
return dev.caskeleton.application.notification.platform.admin.AdminOperationClaim.conflict();
}
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java
index 23fdb2dc..a2da23c2 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientDeliveryStore.java
@@ -99,6 +99,23 @@ public final class JpaRecipientDeliveryStore implements RecipientDeliveryStorePo
return reread(written, record.id());
}
+ @Override
+ public Optional requeueAbandoned(
+ RecipientLease abandoned, Instant nextDispatchAt) {
+ CallerTransaction.require("recipient delivery requeueAbandoned");
+ Objects.requireNonNull(abandoned, "abandoned");
+ Objects.requireNonNull(nextDispatchAt, "nextDispatchAt");
+ int written =
+ recipients.requeueAbandoned(
+ abandoned.recipientDeliveryId().value(),
+ abandoned.owner(),
+ abandoned.fence(),
+ abandoned.leaseUntil(),
+ nextDispatchAt,
+ clock.instant());
+ return reread(written, abandoned.recipientDeliveryId());
+ }
+
@Override
public Optional transitionHeldBy(
RecipientDeliveryId id,
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java
index 65784bac..220d4acb 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JpaRecipientLeaseStore.java
@@ -5,9 +5,11 @@ import dev.caskeleton.adapter.outbound.persistence.transaction.CallerTransaction
import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
import dev.caskeleton.application.notification.platform.dispatch.RecipientLease;
import dev.caskeleton.application.notification.platform.dispatch.RecipientLeaseStorePort;
+import java.sql.Timestamp;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.time.OffsetDateTime;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -37,9 +39,10 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort {
}
/**
- * The one write here without {@code CallerTransaction.require}: the scheduler claims before any
- * use case runs, and the claim is a single CTE statement that selects, locks and fences its rows
- * by itself — there is no second write it has to be atomic with.
+ * The one write here without {@code CallerTransaction.require}: the claim is a single CTE
+ * statement that selects, locks and fences its rows by itself — there is no second write it has
+ * to be atomic with. {@code ClaimDueDeliveriesUseCase} opens a write transaction around it
+ * anyway, as every use case that writes does.
*/
@Override
public List claim(String workerId, int limit, Duration leaseDuration) {
@@ -109,13 +112,38 @@ public final class JpaRecipientLeaseStore implements RecipientLeaseStorePort {
}
@Override
- public List expiredDispatching(int limit, Duration olderThan) {
+ public List expiredDispatching(int limit, Duration olderThan) {
Objects.requireNonNull(olderThan, "olderThan");
if (limit < 1) {
throw new IllegalArgumentException("limit");
}
return recipients.selectExpiredDispatching(clock.instant().minus(olderThan), limit).stream()
- .map(RecipientDeliveryId::new)
+ .map(
+ row ->
+ new RecipientLease(
+ new RecipientDeliveryId((UUID) row[0]),
+ (String) row[1],
+ ((Number) row[2]).longValue(),
+ instantOf(row[3])))
.toList();
}
+
+ /**
+ * A {@code timestamptz} column of a native row. Hibernate hands it back as an {@link
+ * OffsetDateTime} or a {@link Timestamp} depending on its JDBC type mapping, never null here
+ * because the select only returns rows that have a lease.
+ */
+ private static Instant instantOf(Object column) {
+ if (column instanceof Instant instant) {
+ return instant;
+ }
+ if (column instanceof OffsetDateTime offset) {
+ return offset.toInstant();
+ }
+ if (column instanceof Timestamp timestamp) {
+ return timestamp.toInstant();
+ }
+ throw new IllegalStateException(
+ "lease_until came back as " + (column == null ? "null" : column.getClass().getName()));
+ }
}
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java
index 15428027..e7ccfb32 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/InboxItemJpaRepository.java
@@ -60,18 +60,16 @@ public interface InboxItemJpaRepository extends Repository{@code read_at IS NULL} in the predicate is what makes the update idempotent under
* concurrency: a second caller updates zero rows instead of overwriting the first caller's
* timestamp.
- */
- /**
- * Marks everything up to and including a cursor position as read.
*
* The cutoff is a tuple, matching the list's own {@code (created_at DESC, id DESC)} ordering.
* It used to be a timestamp alone, and the list's order is not a total order on {@code
@@ -95,14 +93,23 @@ public interface InboxItemJpaRepository extends RepositoryThe lease store contract ({@code RecipientLeaseStoreContract}, run on PostgreSQL by {@code
* PostgreSqlRecipientLeaseStoreContractTest}) now drives this statement through the repository and
* the store, so no test retypes it or runs it beside the adapter (review 2026-11-01 04-14). The
- * named parameters are Spring Data's.
+ * named parameters are bound by Spring Data for the repository statements, and by a {@code
+ * NamedParameterJdbcTemplate} where the serving state reads {@link #CLAIMABLE_AT_NOW}.
*/
public final class RecipientClaimSql {
+ /**
+ * The states a delivery waits in to be sent, which the claim takes from, the expiry retires from
+ * and the serving state counts as backlog. One list so the statements cannot disagree about what
+ * is waiting.
+ */
+ public static final String CLAIMABLE_STATES = "('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')";
+
+ /**
+ * Which deliveries the claim takes at {@code :now}: due, in a waiting state, not expired, and
+ * with no lease or one that has run out. The serving state counts its backlog with this same
+ * fragment, so the backlog is exactly what a claim would take.
+ */
+ public static final String CLAIMABLE_AT_NOW =
+ "next_dispatch_at <= :now"
+ + " AND delivery_state IN "
+ + CLAIMABLE_STATES
+ + " AND (expires_at IS NULL OR expires_at > :now)"
+ + " AND (lease_until IS NULL OR lease_until < :now)";
+
/**
* Claims due deliveries and returns their fences.
*
@@ -28,10 +48,8 @@ public final class RecipientClaimSql {
public static final String CLAIM_BATCH =
"WITH claimable AS ("
+ " SELECT id FROM notification_recipient_delivery"
- + " WHERE next_dispatch_at <= :now"
- + " AND delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"
- + " AND (expires_at IS NULL OR expires_at > :now)"
- + " AND (lease_until IS NULL OR lease_until < :now)"
+ + " WHERE "
+ + CLAIMABLE_AT_NOW
+ " ORDER BY next_dispatch_at, id"
+ " FOR UPDATE SKIP LOCKED"
+ " LIMIT :batchSize"
@@ -51,7 +69,18 @@ public final class RecipientClaimSql {
+ " WHERE d.id = claimable.id"
+ " RETURNING d.id, d.lease_fence";
- /** Retires deliveries whose window closed before anyone claimed them. */
+ /**
+ * Retires deliveries whose window closed before anyone claimed them.
+ *
+ * A delivery a worker claimed while this statement ran must not be retired to {@code EXPIRED}
+ * under the worker dispatching it, which is what the unlocked subquery did (round9 review
+ * 2026-11-12, RR9-01-13). Two things stop it. The subquery locks the rows it picks and skips rows
+ * another transaction holds, so a delivery being claimed is not picked. The state and expiry
+ * conditions are also on the {@code UPDATE} itself: under READ COMMITTED PostgreSQL re-checks
+ * only the outer {@code WHERE} against a row it had to wait for, so if the lock clause is ever
+ * lost the claimed row is still left alone. A delivery being claimed right now is not expiring
+ * either: the claim refuses expired rows in the same statement.
+ */
public static final String EXPIRE_OVERDUE =
"UPDATE notification_recipient_delivery SET"
+ " delivery_state = 'EXPIRED',"
@@ -61,11 +90,17 @@ public final class RecipientClaimSql {
+ " updated_at = :now"
+ " WHERE id IN ("
+ " SELECT id FROM notification_recipient_delivery"
- + " WHERE delivery_state IN ('PENDING', 'READY_TO_DISPATCH', 'RETRY_WAITING')"
+ + " WHERE delivery_state IN "
+ + CLAIMABLE_STATES
+ " AND expires_at IS NOT NULL"
+ " AND expires_at <= :now"
+ " ORDER BY expires_at"
- + " LIMIT :batchSize)";
+ + " FOR UPDATE SKIP LOCKED"
+ + " LIMIT :batchSize)"
+ + " AND delivery_state IN "
+ + CLAIMABLE_STATES
+ + " AND expires_at IS NOT NULL"
+ + " AND expires_at <= :now";
private RecipientClaimSql() {}
}
diff --git a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java
index b5f1ee18..1240cf0f 100644
--- a/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java
+++ b/src/adapter/outbound/persistence-notification/src/main/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/repository/RecipientDeliveryJpaRepository.java
@@ -183,13 +183,48 @@ public interface RecipientDeliveryJpaRepository extends RepositoryOnly rows a claim wrote: an owner and a fence of at least one. The requeue matches on both,
+ * and a row without them — one left dispatching before notification V4 added the fence — is no
+ * lease anyone holds. Read here it would be a lease the application refuses to build, and the one
+ * row would fail every recovery pass after it.
+ *
+ * @return one row per delivery: id, lease_owner, lease_fence, lease_until
+ */
@Query(
value =
- "SELECT id FROM notification_recipient_delivery "
+ "SELECT id, lease_owner, lease_fence, lease_until FROM notification_recipient_delivery "
+ "WHERE delivery_state = 'DISPATCHING' AND lease_until < :threshold "
+ + "AND lease_owner IS NOT NULL AND lease_fence > 0 "
+ "ORDER BY lease_until LIMIT :batchSize",
nativeQuery = true)
- List selectExpiredDispatching(
+ List selectExpiredDispatching(
@Param("threshold") Instant threshold, @Param("batchSize") int batchSize);
+
+ /**
+ * Returns an abandoned dispatch to the queue while it is still the one recovery found.
+ *
+ * Owner, fence, state and expiry are all conditions. A new claim raises the fence, a finished
+ * holder moves the state, another instance's recovery moves the state, and a holder that renewed
+ * in time moves the expiry past what recovery saw; in each case the job is no longer abandoned
+ * and this matches nothing.
+ */
+ @Modifying(clearAutomatically = true, flushAutomatically = true)
+ @Query(
+ value =
+ "UPDATE notification_recipient_delivery "
+ + "SET delivery_state = 'READY_TO_DISPATCH', next_dispatch_at = :nextDispatchAt, "
+ + "version = version + 1, updated_at = :now "
+ + "WHERE id = :id AND lease_owner = :owner AND lease_fence = :fence "
+ + "AND delivery_state = 'DISPATCHING' AND lease_until <= :leaseUntil",
+ nativeQuery = true)
+ int requeueAbandoned(
+ @Param("id") UUID id,
+ @Param("owner") String owner,
+ @Param("fence") long fence,
+ @Param("leaseUntil") Instant leaseUntil,
+ @Param("nextDispatchAt") Instant nextDispatchAt,
+ @Param("now") Instant now);
}
diff --git a/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql
new file mode 100644
index 00000000..2d2416a2
--- /dev/null
+++ b/src/adapter/outbound/persistence-notification/src/main/resources/db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql
@@ -0,0 +1,26 @@
+-- An index over the provider events still waiting for something.
+--
+-- The serving-state projection lag counts pending, failed and unmatched events. It had no WHERE and
+-- read the whole notification_provider_event table, which nothing ever purges, and the scheduler
+-- asked for it on every poll (round9 review 2026-11-12, RR9-01-14). The query now names exactly
+-- this predicate, so it reads only these rows; applied, matched events — the history that grows —
+-- are outside the index.
+--
+-- Built inside this migration's transaction, not CONCURRENTLY, as V21 of the core stream builds its
+-- index: a transactional file fails whole and leaves nothing to repair, where a failed concurrent
+-- build leaves an INVALID index and a failed history row an operator must clear by hand
+-- (docs/runbooks/migration-failed.md). The cost is that the build holds SHARE on
+-- notification_provider_event: provider callbacks wait for it, and providers retry a callback that
+-- does not get an answer. On a table already large in production, create the same index
+-- CONCURRENTLY by hand before deploying; IF NOT EXISTS then makes this migration a no-op.
+
+-- Bound the wait for the lock (SHARE on notification_provider_event, held until commit). Unbounded,
+-- it queues behind any open transaction that touched the table, and every callback insert queues
+-- behind the migration. Five seconds, then the migration fails, rolls back whole and the stream
+-- stays at V13: deploy again once that transaction has ended (docs/jpa/migration-guide.md "Bound
+-- the lock wait").
+SET LOCAL lock_timeout = '5s';
+
+CREATE INDEX IF NOT EXISTS ix_notification_provider_event_open
+ ON notification_provider_event (received_at)
+ WHERE projection_status IN ('PENDING', 'FAILED') OR attempt_id IS NULL;
diff --git a/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java
new file mode 100644
index 00000000..0805a0d6
--- /dev/null
+++ b/src/adapter/outbound/persistence-notification/src/test/java/dev/caskeleton/adapter/outbound/persistence/notification/platform/JdbcNotificationServingStateOpenEventsTest.java
@@ -0,0 +1,42 @@
+package dev.caskeleton.adapter.outbound.persistence.notification.platform;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Objects;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The projection-lag query and the partial index built for it name the same events.
+ *
+ *
The index is what keeps the lag query, which the scheduler runs on every poll, from reading
+ * the whole provider-event history. A predicate that drifted on either side would leave the query
+ * outside the index and scanning the table again, with nothing failing.
+ */
+class JdbcNotificationServingStateOpenEventsTest {
+
+ private static final String V14 =
+ "db/migration/jpa/notification-platform/V14__notification_provider_event_open_index.sql";
+
+ @Test
+ void theLagQueryAndItsPartialIndexNameTheSameEvents() throws IOException {
+ // given
+ String migration;
+ try (InputStream sql = getClass().getClassLoader().getResourceAsStream(V14)) {
+ migration = new String(Objects.requireNonNull(sql, V14).readAllBytes(), UTF_8);
+ }
+ int index = migration.indexOf("CREATE INDEX");
+
+ // when
+ String indexPredicate =
+ migration
+ .substring(
+ migration.indexOf("WHERE ", index) + "WHERE ".length(), migration.lastIndexOf(';'))
+ .trim();
+
+ // then
+ assertThat(indexPredicate).isEqualTo(JdbcNotificationServingState.OPEN_EVENTS);
+ }
+}
diff --git a/src/application/core/AGENTS.md b/src/application/core/AGENTS.md
index 9d5e5859..de330cad 100644
--- a/src/application/core/AGENTS.md
+++ b/src/application/core/AGENTS.md
@@ -73,11 +73,12 @@ architectureTest)가 강제한다. 나머지는 리뷰 규칙이고, 리뷰가
|---|---|---|---|
| 유스케이스(inbound 포트 구현) | `UseCase`, ArchUnit이 강제 | 기능의 `application` 패키지 | `*ApplicationUseCase`(application-fileserver, application-notification), `*Service`(notification, fileserver) |
| command·query record | `*Command` / `*Query` | 그 유스케이스 옆 | — |
+| 요청 값 record: command가 담는 호출자 요청 내용(호출자가 누구인지는 뺀 값), 또는 포트 인자 묶음 | `*Request` | 그것을 담는 command나 받는 포트 옆 | `CreateUploadRequest`·`FinalizeUploadRequest`(application-fileserver `upload`, command는 `port.in`), `VerificationRequest`(application-fileserver `api.security`, 포트는 `upload`), application-objectstorage의 `request` 패키지(포트는 `port`, leaf가 값 타입 패키지로 정했다) |
| aggregate 영속(aggregate 전체를 저장하고 돌려준다) | `Repository` | aggregate의 **domain** 패키지, 도메인이 가진다 | — |
| 애플리케이션이 가진 기술 저장소(outbox, 멱등성, operation 행) | `StorePort` | application 패키지 | `*WritePort`(옛 onboarding fixture, 지움) |
| aggregate가 아니라 projection을 돌려주는 읽기 모델 | `QueryPort` | application 패키지. projection만 돌려준다(ArchUnit) | — |
| 그 밖의 outbound 포트 | `Port` | application 패키지 | — |
-| 서버가 정하는 id 생성 | `IdFactory extends IdFactory<Id>` | domain 패키지. id record는 `implements ResourceId` | `DurableOperationIdFactory`(durable operation id는 aggregate의 resource id가 아니라 `IdFactory`를 따르지 않는다) |
+| 서버가 정하는 id 생성 | `IdFactory extends IdFactory<Id>` | domain 패키지. id record는 `implements ResourceId` | `DurableOperationIdFactory`(durable operation id는 aggregate의 resource id가 아니라 `IdFactory`를 따르지 않는다), `NotificationIdGeneratorPort`(application-notification), `UploadIdentifierPort`(application-fileserver). 셋 다 사용자 전송이 쥐면 `USER_TRANSPORTS_DO_NOT_HOLD_PERSISTENCE_OR_ID_PORTS`가 막는다 |
| 설정 바인딩 | `*Settings`(운영) 또는 `*Policy`(인가·도메인 정책), ArchUnit이 강제 | 그것을 바인딩하는 leaf | — |
| Throwable | `*Exception`, ArchUnit이 강제 | — | — |
| 포트의 계약 시험 키트 | 포트 소유자의 testFixtures에 둔 추상 `Contract`. fake의 시험과 각 어댑터의 시험이 상속한다 | testFixtures | 값 타입 시험의 이름이 한때 `*ContractTest`였다. 지금은 `*ValueTypesTest`다 |
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java b/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java
index aeff40dc..35885ba0 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/idempotency/IdempotencyClaimAttempt.java
@@ -13,6 +13,10 @@ import java.util.regex.Pattern;
* until the record expired (round6 RR6-02-01, {@code IdempotencyStoreContract}). One attempt is one
* sender at a time: a handle from before a takeover is stale, but two calls sending the same
* attempt at once are not told apart — retry after the call has returned, not beside it.
+ *
+ * The owner token is the store's: 32 secure-random bytes as 64 lower-case hex characters, which
+ * both stores compare as a fixed-width value. It is not a lease owner token (Base64URL, chosen by
+ * the lease caller); the transaction {@code OperationId} Javadoc compares the identities.
*/
public record IdempotencyClaimAttempt(String ownerToken, OperationId operationId) {
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java b/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java
index 4618c613..6d32894d 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/lease/LeaseValidation.java
@@ -11,6 +11,11 @@ final class LeaseValidation {
private LeaseValidation() {}
+ /**
+ * A lease's operation id or owner token: 16..128 Base64URL characters, so it fits a header and a
+ * Redis key without escaping and is long enough to be unguessable. Narrower than the transaction
+ * {@code OperationId}, whose Javadoc compares the operation-shaped ids and why they differ.
+ */
static String opaqueToken(String value, String field) {
if (value == null
|| value.length() < 16
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java
index 535a146d..8c4a6de9 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationEventPublishPort.java
@@ -19,10 +19,15 @@ public interface IntegrationEventPublishPort {
CompletionStage publish(ValidatedIntegrationEvent event);
/**
- * The longest one publish can take before the returned stage completes, when the publisher knows
- * it. The outbox relay sizes its batch against it (see {@code
+ * The longest one publish can take, from the call to {@code publish} until the returned stage
+ * completes, when the publisher knows it. The outbox relay sizes its batch against it (see {@code
* OutboxMessagePublishPort#longestAttempt}).
*
+ * A bound, not a hint: a caller may stop waiting once it has passed and treat the attempt as
+ * ambiguous, so a publisher that reports less than it can take turns slow answers into ambiguous
+ * ones. Positive when present. Empty means unknown, and then the stage's own completion is the
+ * only bound.
+ *
* @return the longest attempt, or empty when unknown
*/
default Optional longestAttempt() {
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java
index adc509c3..b4958ab6 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/messaging/event/IntegrationIdentity.java
@@ -1,5 +1,9 @@
package dev.caskeleton.application.messaging.event;
+import dev.caskeleton.shared.digest.Sha256;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.UUID;
import java.util.regex.Pattern;
/**
@@ -73,6 +77,35 @@ public final class IntegrationIdentity {
return value != null && UUID_V7.matcher(value).matches();
}
+ /**
+ * A canonical lower-case UUIDv7 derived from {@code value}: the same value and instant always
+ * give the same id.
+ *
+ * For an event whose id must follow from something that is not itself a UUIDv7 — an operation
+ * id a client chose, say — so a retry drafts the same event. The time field is {@code at}, so the
+ * id orders like a minted one; the other 74 bits are the SHA-256 of {@code value}. Kept here,
+ * with the grammar it must satisfy, rather than in each use case that needs one: the derivation
+ * is messaging identity, not the use case's own rule.
+ *
+ * @param value what the id is derived from
+ * @param at the instant its time field carries
+ * @return a canonical lower-case UUIDv7
+ */
+ public static String uuidV7Derived(String value, Instant at) {
+ Objects.requireNonNull(value, "value");
+ Objects.requireNonNull(at, "at");
+ byte[] digest = Sha256.ofUtf8(value).bytes();
+ long millis = at.toEpochMilli() & 0xFFFF_FFFF_FFFFL;
+ long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL);
+ long mostSignificant = (millis << 16) | (0x7L << 12) | randA;
+ long randB = 0;
+ for (int i = 2; i < 10; i++) {
+ randB = (randB << 8) | (digest[i] & 0xFFL);
+ }
+ long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L;
+ return new UUID(mostSignificant, leastSignificant).toString();
+ }
+
/** Refuses a correlation or causation id outside the grammar or the bound. */
static void requireCorrelation(String field, String value) {
if (!isCanonical(value, CORRELATION_MAX_LENGTH)) {
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java
index aa9fc88e..6b4e0a6b 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationCommand.java
@@ -1,19 +1,23 @@
package dev.caskeleton.application.operation;
import dev.caskeleton.application.command.Command;
+import dev.caskeleton.application.tenant.TenantId;
import java.util.Objects;
+import java.util.Optional;
/**
* A caller asking to cancel one operation.
*
* @param operationId the identity to cancel
* @param principal the authenticated caller, or null when the caller is not authenticated
- * @param tenantId the caller's tenant, or null when the request is not tenant scoped
+ * @param tenant the caller's tenant, empty when the request is not tenant scoped
*/
public record CancelDurableOperationCommand(
- DurableOperationId operationId, String principal, String tenantId) implements Command {
+ DurableOperationId operationId, String principal, Optional tenant)
+ implements Command {
public CancelDurableOperationCommand {
Objects.requireNonNull(operationId, "operationId");
+ Objects.requireNonNull(tenant, "tenant");
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java
index 6c3c2449..9aa35d16 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/CancelDurableOperationUseCase.java
@@ -63,7 +63,7 @@ public class CancelDurableOperationUseCase
.filter(
operation ->
DurableOperationAccessPolicy.mayAccess(
- operation, command.principal(), command.tenantId()))
+ operation, command.principal(), command.tenant()))
.map(visible -> operations.cancel(command.operationId(), command.principal(), now));
});
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java
index 6674f707..3e799bf6 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationAccessPolicy.java
@@ -1,6 +1,8 @@
package dev.caskeleton.application.operation;
+import dev.caskeleton.application.tenant.TenantId;
import java.util.Objects;
+import java.util.Optional;
/**
* Who may read or cancel an operation: the principal that submitted it, in the tenant it was
@@ -20,10 +22,21 @@ final class DurableOperationAccessPolicy {
*
* @param operation the stored operation
* @param principal the authenticated caller, or null when the caller is not authenticated
- * @param tenantId the caller's tenant, or null when the request is not tenant scoped
+ * @param tenant the caller's tenant, empty when the request is not tenant scoped
*/
- static boolean mayAccess(DurableOperation operation, String principal, String tenantId) {
+ static boolean mayAccess(
+ DurableOperation operation, String principal, Optional tenant) {
return operation.principal().equals(principal)
- && Objects.equals(operation.tenantId(), tenantId);
+ && Objects.equals(operation.tenantId(), storedForm(tenant));
+ }
+
+ /**
+ * The tenant as the operation row holds it: its value, or null when not tenant scoped.
+ *
+ * @param tenant the caller's tenant
+ * @return the stored form
+ */
+ static String storedForm(Optional tenant) {
+ return tenant.map(TenantId::value).orElse(null);
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java
new file mode 100644
index 00000000..80bdbe0c
--- /dev/null
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationClaim.java
@@ -0,0 +1,45 @@
+package dev.caskeleton.application.operation;
+
+import java.util.Objects;
+
+/**
+ * One worker's claim on one durable operation, as returned by {@link
+ * DurableOperationStorePort#claimNext}.
+ *
+ * The {@code fence} is what tells one run from the next. Every claim increments it, and a
+ * heartbeat, progress report or completion names the fence it holds, so a run that stalled past its
+ * lease finds nothing left to change instead of overwriting the run that took over. The worker id
+ * alone cannot do this: a worker that restarts with the same id, or one process running two
+ * executions under one id, would otherwise have its earlier run accepted as the current one.
+ *
+ *
Kept apart from {@link DurableOperationSubmission} on purpose, as the outbox keeps its lease
+ * apart from the claimed event: the work to execute is the submission, and the claim is store
+ * bookkeeping the work has no use for.
+ *
+ *
A store builds it from a row, never from a request, so a value it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
+ * @param submission the claimed work
+ * @param workerId the worker that holds the claim
+ * @param fence the fencing token this claim was granted; at least 1
+ */
+public record DurableOperationClaim(
+ DurableOperationSubmission submission, String workerId, long fence) {
+
+ public DurableOperationClaim {
+ Objects.requireNonNull(submission, "submission");
+ if (workerId == null || workerId.isBlank()) {
+ throw new IllegalStateException(
+ "a claim needs an owner; an anonymous one cannot be verified when the work completes");
+ }
+ if (fence < 1) {
+ throw new IllegalStateException("fence must be >= 1, was " + fence);
+ }
+ }
+
+ /** The claimed operation. */
+ public DurableOperationId operationId() {
+ return submission.operationId();
+ }
+}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java
index 573e0135..f8c85981 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationId.java
@@ -10,9 +10,10 @@ import java.util.regex.Pattern;
*/
public record DurableOperationId(String value) {
- // Bounded and free of anything that needs escaping. The identifier travels to whatever transport
- // published the operation and is handed back to callers, so a value that one encoding mangles is
- // a defect that only appears for some clients.
+ // Bounded and free of anything that needs escaping: RFC 3986 unreserved characters, because the
+ // identifier is a URL path segment (/operations/{id}) and is handed back to callers, so a value
+ // that one encoding mangles is a defect that only appears for some clients. Narrower than the
+ // transaction OperationId for that reason (its Javadoc compares the operation-shaped ids).
private static final Pattern GRAMMAR = Pattern.compile("[A-Za-z0-9._~-]{1,128}");
public DurableOperationId {
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java
index 440d3fb2..887f5c72 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationStorePort.java
@@ -11,9 +11,12 @@ import java.util.Optional;
*
A port because the application owns what an operation is and nothing about how it is stored.
* The JPA adapter behind it is one implementation; an in-memory one serves tests.
*
- *
Every method that changes a running operation takes the {@code workerId} and verifies it. A
- * worker whose lease lapsed may still be alive and still finishing its work, and letting it report
- * a result after another worker took over is how an operation ends up with two answers.
+ *
Every method that changes a running operation takes the {@link DurableOperationClaim} that
+ * {@link #claimNext} returned and verifies both its worker and its fence. A worker whose lease
+ * lapsed may still be alive and still finishing its work, and letting it report a result after
+ * another run took over is how an operation ends up with two answers. The worker id alone does not
+ * tell the runs apart: the same id may claim the operation again after a restart, so the fence —
+ * incremented by every claim — is what the store compares.
*/
public interface DurableOperationStorePort {
@@ -24,8 +27,8 @@ public interface DurableOperationStorePort {
* rather than a second one is what makes a resubmission safe while the first is still running.
*
* @param submission what was accepted
- * @return the operation, and whether this call created it; a retry under the same operation id is
- * not a creation
+ * @return the operation, and whether this call created it; a resubmission that finds the
+ * operation an earlier call created, under its own fresh id, is not a creation
*/
DurableOperationSubmitResult submit(DurableOperationSubmission submission);
@@ -43,48 +46,43 @@ public interface DurableOperationStorePort {
*
*
Runnable means PENDING, or RUNNING with a lapsed lease. The second case is the recovery
* path: without it, an operation claimed by a worker that then died is never picked up again.
+ * Every claim gets a fence higher than any earlier claim of the same operation, whoever made it.
*
* @param workerId who is claiming
* @param leaseDuration how long the claim holds without a heartbeat
* @param now the current instant
+ * @return the claim the worker names on every later write, or empty when nothing is runnable
*/
- Optional claimNext(
- String workerId, Duration leaseDuration, Instant now);
+ Optional claimNext(String workerId, Duration leaseDuration, Instant now);
/**
- * Extends a lease the worker still holds.
+ * Extends a lease the claim still holds.
*
- * @return false when the lease is no longer this worker's, which means it must stop
+ * @return false when the lease lapsed or another claim (by any worker, this one included) took
+ * the operation over, which means this run must stop
*/
- boolean heartbeat(
- DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now);
+ boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now);
/**
- * Publishes progress for an operation this worker holds.
+ * Publishes progress for an operation the claim holds.
*
- * @return false when the lease is no longer this worker's
+ * @return false when another claim took the operation over or it is no longer running
*/
- boolean reportProgress(
- DurableOperationId operationId, String workerId, OperationProgressSnapshot progress);
+ boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress);
/**
* Records success.
*
- * @return false when the lease is no longer this worker's, so the result is not recorded
+ * @return false when another claim took the operation over, so the result is not recorded
*/
- boolean succeed(
- DurableOperationId operationId, String workerId, String resultReference, Instant completedAt);
+ boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt);
/**
* Records failure.
*
- * @return false when the lease is no longer this worker's
+ * @return false when another claim took the operation over, so the failure is not recorded
*/
- boolean fail(
- DurableOperationId operationId,
- String workerId,
- OperationFailure failure,
- Instant completedAt);
+ boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt);
/**
* Requests cancellation.
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java
index 1d43820a..2c0edd5d 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/DurableOperationSubmitResult.java
@@ -5,10 +5,10 @@ import java.util.Objects;
/**
* What a store did with a submission.
*
- * {@code created} comes from the store's own write, not from comparing identifiers afterwards. A
- * retry under the same operation id finds the operation it created itself, and an identifier
- * comparison cannot tell that retry from the first call — which is how a retry appended its outbox
- * event a second time.
+ *
{@code created} comes from the store's own write, not from comparing identifiers afterwards.
+ * Every call mints a fresh operation id, and a resubmission is found by scope and fingerprint as
+ * the operation the first call created; only the store's write tells that resubmission from the
+ * first call, which is what keeps it from appending its outbox event a second time.
*
* @param operation the operation holding the submission's scope
* @param created whether this call inserted it
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java
index e6ce2ae6..afb85f8f 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationQuery.java
@@ -1,19 +1,22 @@
package dev.caskeleton.application.operation;
import dev.caskeleton.application.query.Query;
+import dev.caskeleton.application.tenant.TenantId;
import java.util.Objects;
+import java.util.Optional;
/**
* A caller asking to read one operation.
*
* @param operationId the identity being polled
* @param principal the authenticated caller, or null when the caller is not authenticated
- * @param tenantId the caller's tenant, or null when the request is not tenant scoped
+ * @param tenant the caller's tenant, empty when the request is not tenant scoped
*/
public record GetDurableOperationQuery(
- DurableOperationId operationId, String principal, String tenantId) implements Query {
+ DurableOperationId operationId, String principal, Optional tenant) implements Query {
public GetDurableOperationQuery {
Objects.requireNonNull(operationId, "operationId");
+ Objects.requireNonNull(tenant, "tenant");
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java
index 3fb1bfed..4a66ef81 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/GetDurableOperationUseCase.java
@@ -15,7 +15,7 @@ import java.util.Optional;
transactionMode = TransactionMode.READ_ONLY,
idempotency = Idempotency.IDEMPOTENT,
repositoryAccess = RepositoryAccess.READ_REPOSITORY)
-public final class GetDurableOperationUseCase
+public class GetDurableOperationUseCase
implements QueryUseCase> {
private final DurableOperationStorePort operations;
@@ -47,6 +47,6 @@ public final class GetDurableOperationUseCase
.filter(
operation ->
DurableOperationAccessPolicy.mayAccess(
- operation, query.principal(), query.tenantId()));
+ operation, query.principal(), query.tenant()));
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java
index 0d401654..d56f22bb 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationCommand.java
@@ -2,34 +2,34 @@ package dev.caskeleton.application.operation;
import dev.caskeleton.application.command.Command;
import dev.caskeleton.application.idempotency.RequestFingerprint;
+import dev.caskeleton.application.tenant.TenantId;
import java.time.Duration;
import java.util.Objects;
+import java.util.Optional;
/**
* A request to run work asynchronously.
*
- * The caller supplies the operation id rather than the store minting one. That is what lets the
- * transport hand back a {@code Location} it already knows, and it makes the whole submission one
- * value the caller can retry: a generated id would differ on every attempt, so a retry after an
- * ambiguous failure would create a second operation.
+ *
No operation id: {@link SubmitDurableOperationUseCase} mints one with {@link
+ * DurableOperationIdFactory}, and a resubmission of the same work by the same caller is recognised
+ * by its scope and payload fingerprint, not by an id the caller kept. The transport used to mint it
+ * and hand it in (round9 review 2026-11-12, RR9-05-13).
*
*
There is no event type to choose. Every accepted operation announces itself with the one
* {@link DurableOperationSubmittedContract} event, which only wakes a worker; letting a caller name
* the type would let it publish any event the broker carries.
*
- * @param operationId the identity to use
* @param operationName what work this is
* @param principal who is asking
- * @param tenantId the tenant, or null when the operation is not tenant scoped
+ * @param tenant the tenant, empty when the operation is not tenant scoped
* @param fingerprint the semantic fingerprint of the request
* @param payload the serialized request a worker will execute
* @param timeToLive how long the record and its result stay available
*/
public record OperationCommand(
- DurableOperationId operationId,
String operationName,
String principal,
- String tenantId,
+ Optional tenant,
RequestFingerprint fingerprint,
String payload,
Duration timeToLive)
@@ -39,8 +39,8 @@ public record OperationCommand(
public static final Duration DEFAULT_TIME_TO_LIVE = Duration.ofDays(1);
public OperationCommand {
- Objects.requireNonNull(operationId, "operationId");
Objects.requireNonNull(operationName, "operationName");
+ Objects.requireNonNull(tenant, "tenant");
Objects.requireNonNull(fingerprint, "fingerprint");
Objects.requireNonNull(payload, "payload");
Objects.requireNonNull(timeToLive, "timeToLive");
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java
index 86328d84..618f23c1 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/OperationLease.java
@@ -11,6 +11,10 @@ import java.util.Objects;
* ever with no worker behind it — the failure that makes a queue stop draining while every
* dashboard reports it busy.
*
+ * A store builds it from a row, never from a request, so a value it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
* @param workerId who holds it
* @param acquiredAt when it was taken
* @param expiresAt when another worker may take it
@@ -22,11 +26,11 @@ public record OperationLease(String workerId, Instant acquiredAt, Instant expire
Objects.requireNonNull(acquiredAt, "acquiredAt");
Objects.requireNonNull(expiresAt, "expiresAt");
if (workerId.isBlank()) {
- throw new IllegalArgumentException(
+ throw new IllegalStateException(
"a lease needs an owner; an anonymous one cannot be verified when the work completes");
}
if (!expiresAt.isAfter(acquiredAt)) {
- throw new IllegalArgumentException("a lease that expires when taken is not a lease");
+ throw new IllegalStateException("a lease that expires when taken is not a lease");
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java
index e5d8719c..45471cef 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCase.java
@@ -15,12 +15,10 @@ import dev.caskeleton.application.security.RequiresPermission;
import dev.caskeleton.application.transaction.TransactionMode;
import dev.caskeleton.application.transaction.TransactionPort;
import dev.caskeleton.application.usecase.CommandUseCase;
-import dev.caskeleton.shared.digest.Sha256;
import java.time.Clock;
import java.time.Instant;
import java.util.Objects;
import java.util.Optional;
-import java.util.UUID;
/**
* Accepts long-running work: one transaction that writes the operation row and the outbox row.
@@ -49,8 +47,9 @@ import java.util.UUID;
@RequiresPermission("operation:submit")
@UseCaseCapability(
transactionMode = TransactionMode.WRITE,
- // KEYED, not IDEMPOTENT. Re-running this use case is only safe because the caller supplies the
- // operation id and the fingerprint scope; without a key it would accept the same work twice.
+ // KEYED, not IDEMPOTENT. Re-running this use case is only safe because the store recognises the
+ // submission scope and the payload fingerprint; without that key it would accept the same work
+ // twice.
idempotency = Idempotency.KEYED,
repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
// False, and load-bearing. Nothing here reaches the broker: the outbox row is the publish
@@ -72,6 +71,7 @@ public class SubmitDurableOperationUseCase
private final OutboxAppendPort outbox;
private final IntegrationEventEncoderPort encoder;
private final TransactionPort transactions;
+ private final DurableOperationIdFactory operationIds;
private final Clock clock;
/**
@@ -82,6 +82,7 @@ public class SubmitDurableOperationUseCase
* @param encoder turns the submission event into the validated event the outbox stores; it must
* know {@link DurableOperationSubmittedContract}
* @param transactions opens the write transaction both share
+ * @param operationIds mints the id of a new submission
* @param clock the submission clock
*/
public SubmitDurableOperationUseCase(
@@ -89,8 +90,10 @@ public class SubmitDurableOperationUseCase
OutboxAppendPort outbox,
IntegrationEventEncoderPort encoder,
TransactionPort transactions,
+ DurableOperationIdFactory operationIds,
Clock clock) {
this.operations = Objects.requireNonNull(operations, "operations");
+ this.operationIds = Objects.requireNonNull(operationIds, "operationIds");
this.outbox = Objects.requireNonNull(outbox, "outbox");
this.encoder = Objects.requireNonNull(encoder, "encoder");
this.transactions = Objects.requireNonNull(transactions, "transactions");
@@ -110,18 +113,20 @@ public class SubmitDurableOperationUseCase
() -> {
DurableOperationSubmission submission =
new DurableOperationSubmission(
- command.operationId(),
+ // Minted per call. A resubmission's id is never stored: the store answers with
+ // the operation the first submission created.
+ operationIds.newOperationId(),
command.operationName(),
command.principal(),
- command.tenantId(),
+ DurableOperationAccessPolicy.storedForm(command.tenant()),
command.fingerprint(),
command.payload(),
clock.instant(),
command.timeToLive());
DurableOperationSubmitResult submitted = operations.submit(submission);
DurableOperation operation = submitted.operation();
- // The store's answer, not an id comparison: a retry under the same operation id finds
- // the operation it created itself, and the ids match.
+ // The store's answer, not an id comparison: every call mints a fresh id, and a retry of
+ // the same submission is found by the store as the operation the first call created.
boolean alreadyAccepted = !submitted.created();
if (!alreadyAccepted) {
@@ -147,8 +152,8 @@ public class SubmitDurableOperationUseCase
// Single-tenant scope even when the command names a tenant: the event only wakes a worker, the
// tenant travels in the stored operation, and mapping a TenantId into the lower-case scope
// grammar is the messaging platform's decision, not this use case's.
- // No request correlation: the event correlates to the operation itself, so a retry under the
- // same operation id produces the same event.
+ // No request correlation: the event correlates to the operation itself, so a retry that the
+ // store resolves to the same operation produces the same event.
return IntegrationEventDrafts.draft(
CONTRACT,
new EventId(identity),
@@ -160,15 +165,15 @@ public class SubmitDurableOperationUseCase
}
/**
- * The operation id itself when it is a canonical lower-case UUIDv7 — every id the HTTP adapter
- * mints is — and otherwise a UUIDv7 derived from it.
+ * The operation id itself when it is a canonical lower-case UUIDv7 — every id the shipped {@code
+ * DurableOperationIdFactory} mints is — and otherwise a UUIDv7 derived from it.
*
*
The event id is the messaging platform's message id, which is a canonical lower-case UUIDv7;
- * the draft refuses anything else, so an operation id a client chose ({@code ~report}, a v4, an
- * upper-case spelling) cannot be the event id. The derived id keeps the event deterministic for
- * the operation — the same id and submission time always give the same event id — and the payload
- * carries the operation id unchanged for the worker. Its time field is the submission time, so it
- * orders like a minted one; its other 74 bits are the SHA-256 of the operation id.
+ * the draft refuses anything else, so an id a deployment's own factory mints ({@code ~report}, a
+ * v4, an upper-case spelling) cannot be the event id. The derived id keeps the event
+ * deterministic for the operation — the same id and submission time always give the same event id
+ * — and the payload carries the operation id unchanged for the worker. The derivation is {@link
+ * IntegrationIdentity#uuidV7Derived}, with the submission time as its time field.
*
*
It replaced an {@code operation-} digest, which fit the stored identity grammar and
* passed the append, and which the bridge then refused, parking the committed event (review
@@ -179,15 +184,6 @@ public class SubmitDurableOperationUseCase
if (IntegrationIdentity.isCanonicalUuidV7(value)) {
return value;
}
- byte[] digest = Sha256.ofUtf8(value).bytes();
- long millis = submittedAt.toEpochMilli() & 0xFFFF_FFFF_FFFFL;
- long randA = ((digest[0] & 0x0FL) << 8) | (digest[1] & 0xFFL);
- long mostSignificant = (millis << 16) | (0x7L << 12) | randA;
- long randB = 0;
- for (int i = 2; i < 10; i++) {
- randB = (randB << 8) | (digest[i] & 0xFFL);
- }
- long leastSignificant = (randB & 0x3FFF_FFFF_FFFF_FFFFL) | 0x8000_0000_0000_0000L;
- return new UUID(mostSignificant, leastSignificant).toString();
+ return IntegrationIdentity.uuidV7Derived(value, submittedAt);
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java
index bafa1b5e..05c0062e 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/ClaimedOutboxEvent.java
@@ -12,6 +12,10 @@ import java.util.Objects;
* publishes the bytes that were validated rather than a re-rendering of them. The accessors below
* name the few identities the relay and its diagnostics read without reaching into the envelope.
*
+ * A store builds it from a row, never from a request, so a count it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
* @param event the appended event
* @param status the row's lifecycle status; {@link OutboxEventStatus#IN_FLIGHT} right after a claim
* @param attemptCount claims made, counting the claim that returned this row; the retry delay grows
@@ -30,10 +34,10 @@ public record ClaimedOutboxEvent(
Objects.requireNonNull(event, "event must not be null");
Objects.requireNonNull(status, "status must not be null");
if (attemptCount < 0) {
- throw new IllegalArgumentException("attemptCount must be >= 0, was " + attemptCount);
+ throw new IllegalStateException("attemptCount must be >= 0, was " + attemptCount);
}
if (deferredAttemptCount < 0 || deferredAttemptCount > attemptCount) {
- throw new IllegalArgumentException(
+ throw new IllegalStateException(
"deferredAttemptCount must be between 0 and attemptCount ("
+ attemptCount
+ "), was "
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java
index eed397ab..ae783bc1 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/OutboxLease.java
@@ -13,6 +13,10 @@ import java.util.Objects;
*
Kept apart from {@link ClaimedOutboxEvent} on purpose: the broker adapter receives the event,
* and the claim is store bookkeeping it has no use for.
*
+ *
A store builds it from a row, never from a request, so a value it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
* @param event the claimed row
* @param owner the relay instance that holds the claim
* @param token the fencing token this claim was granted
@@ -22,10 +26,10 @@ public record OutboxLease(ClaimedOutboxEvent event, String owner, long token) {
public OutboxLease {
Objects.requireNonNull(event, "event must not be null");
if (owner == null || owner.isBlank()) {
- throw new IllegalArgumentException("owner must not be blank");
+ throw new IllegalStateException("owner must not be blank");
}
if (token < 1) {
- throw new IllegalArgumentException("token must be >= 1, was " + token);
+ throw new IllegalStateException("token must be >= 1, was " + token);
}
}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java
index 68ae05cf..0425030a 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/outbox/PublishPendingOutboxEventsUseCase.java
@@ -40,7 +40,7 @@ import java.util.function.Supplier;
idempotency = Idempotency.IDEMPOTENT,
repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
externalOutboundAllowed = true)
-public final class PublishPendingOutboxEventsUseCase
+public class PublishPendingOutboxEventsUseCase
implements CommandUseCase {
/** The failure code of a thrown failure whose type has no simple name. */
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java
new file mode 100644
index 00000000..14879d5a
--- /dev/null
+++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowQuery.java
@@ -0,0 +1,16 @@
+package dev.caskeleton.application.realtime;
+
+import dev.caskeleton.application.query.Query;
+import java.util.Objects;
+
+/**
+ * A reconnecting client's stream, asked what it still holds.
+ *
+ * @param streamId which stream
+ */
+public record GetReplayWindowQuery(String streamId) implements Query {
+
+ public GetReplayWindowQuery {
+ Objects.requireNonNull(streamId, "streamId");
+ }
+}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java
new file mode 100644
index 00000000..6d586a46
--- /dev/null
+++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/GetReplayWindowUseCase.java
@@ -0,0 +1,41 @@
+package dev.caskeleton.application.realtime;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.QueryUseCase;
+import java.util.Objects;
+
+/**
+ * What a stream still holds, so a transport can tell "your cursor is too old" from "nothing new".
+ *
+ * Every read goes through a query use case: a transport adapter that held {@link
+ * LiveEventReplayQueryPort} itself read the event log with no capability declared and no read
+ * boundary (round9 review 2026-11-12, RR9-05-10).
+ */
+@UseCaseCapability(
+ transactionMode = TransactionMode.READ_ONLY,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.READ_REPOSITORY)
+public class GetReplayWindowUseCase implements QueryUseCase {
+
+ private final LiveEventReplayQueryPort replay;
+ private final TransactionPort transactions;
+
+ /**
+ * @param replay the retained event history
+ * @param transactions opens the read transaction
+ */
+ public GetReplayWindowUseCase(LiveEventReplayQueryPort replay, TransactionPort transactions) {
+ this.replay = Objects.requireNonNull(replay, "replay");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ }
+
+ @Override
+ public ReplayWindow handle(GetReplayWindowQuery query) {
+ Objects.requireNonNull(query, "query");
+ return transactions.inRead(() -> replay.window(query.streamId()));
+ }
+}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java
similarity index 59%
rename from src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java
rename to src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java
index 741b0fa6..8d4cdf87 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayPort.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/LiveEventReplayQueryPort.java
@@ -5,15 +5,18 @@ import java.util.List;
/**
* Retained event history, for a client that reconnects and wants to continue.
*
- * The history lives wherever the messaging platform keeps it, and this port is the only way a
- * transport adapter reaches it. That boundary is the point: a web or websocket module that kept its
- * own copy would have its own retention, its own eviction and its own opinion about ordering, and
- * the two would diverge silently because both would look plausible.
+ *
The history lives wherever the messaging platform keeps it, and this port is the only way to
+ * it — through {@code GetReplayWindowUseCase} and {@code ReplayLiveEventsUseCase}, never from a
+ * transport directly. A read model that returns projections, so a {@code *QueryPort}: named {@code
+ * LiveEventReplayPort} until 2026-11-13, when a web bridge that held it read the log with no use
+ * case in between (round9 review 2026-11-12, RR9-05-10). That boundary is the point: a web or
+ * websocket module that kept its own copy would have its own retention, its own eviction and its
+ * own opinion about ordering, and the two would diverge silently because both would look plausible.
*
*
{@link #window} exists so a caller can distinguish "your cursor is too old" from "there is
* nothing new", which {@link #replayAfter} returning an empty list cannot.
*/
-public interface LiveEventReplayPort {
+public interface LiveEventReplayQueryPort {
/** What the stream still holds. */
ReplayWindow window(String streamId);
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java
new file mode 100644
index 00000000..23b091b4
--- /dev/null
+++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsQuery.java
@@ -0,0 +1,22 @@
+package dev.caskeleton.application.realtime;
+
+import dev.caskeleton.application.query.Query;
+import java.util.Objects;
+
+/**
+ * A reconnecting client's request for what it missed.
+ *
+ * @param streamId which stream
+ * @param afterPosition the last position the client already has; 0 for the beginning
+ * @param limit the most events to return
+ */
+public record ReplayLiveEventsQuery(String streamId, long afterPosition, int limit)
+ implements Query {
+
+ public ReplayLiveEventsQuery {
+ Objects.requireNonNull(streamId, "streamId");
+ if (afterPosition < 0 || limit < 1) {
+ throw new IllegalArgumentException("afterPosition must be >= 0 and limit >= 1");
+ }
+ }
+}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java
new file mode 100644
index 00000000..2d72a1f0
--- /dev/null
+++ b/src/application/core/src/main/java/dev/caskeleton/application/realtime/ReplayLiveEventsUseCase.java
@@ -0,0 +1,45 @@
+package dev.caskeleton.application.realtime;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.QueryUseCase;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * The events a reconnecting client missed, a bounded page at a time.
+ *
+ * @see GetReplayWindowUseCase
+ */
+@UseCaseCapability(
+ transactionMode = TransactionMode.READ_ONLY,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.READ_REPOSITORY)
+public class ReplayLiveEventsUseCase
+ implements QueryUseCase> {
+
+ private final LiveEventReplayQueryPort replay;
+ private final TransactionPort transactions;
+
+ /**
+ * @param replay the retained event history
+ * @param transactions opens the read transaction
+ */
+ public ReplayLiveEventsUseCase(LiveEventReplayQueryPort replay, TransactionPort transactions) {
+ this.replay = Objects.requireNonNull(replay, "replay");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ }
+
+ /**
+ * @throws ReplayCursorUnavailableException when the position is older than what is retained
+ */
+ @Override
+ public List handle(ReplayLiveEventsQuery query) {
+ Objects.requireNonNull(query, "query");
+ return transactions.inRead(
+ () -> replay.replayAfter(query.streamId(), query.afterPosition(), query.limit()));
+ }
+}
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java b/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java
index f9704f17..6e230d94 100644
--- a/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java
+++ b/src/application/core/src/main/java/dev/caskeleton/application/transaction/OperationId.java
@@ -7,6 +7,15 @@ import java.util.Objects;
*
* The value is intentionally opaque. Adapters may use it for reconciliation, but must not invent
* a replacement identity after an uncertain commit.
+ *
+ *
The widest of the operation-shaped identities, on purpose: the caller chooses it and it is
+ * only ever compared, never put in a URL or a key, so any printable ASCII is allowed. The others
+ * are narrower because of where they travel — a {@link
+ * dev.caskeleton.application.operation.DurableOperationId} is a URL path segment, a lease's
+ * operation id and owner token are Base64URL so they fit a header and a Redis key unescaped, and an
+ * idempotency owner token is the 32 random bytes the store mints, as 64 hex characters. One value
+ * type for all four would either let a path segment carry {@code /} or refuse caller ids that are
+ * valid today (round9 review 2026-11-12, RR9-05-11).
*/
public record OperationId(String value) {
diff --git a/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java b/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java
deleted file mode 100644
index 23912c34..00000000
--- a/src/application/core/src/main/java/dev/caskeleton/application/transaction/ReadConsistency.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package dev.caskeleton.application.transaction;
-
-/** Application-owned read consistency vocabulary. */
-public enum ReadConsistency {
- STRONG,
- READ_YOUR_WRITES,
- EVENTUAL,
- BOUNDED_STALENESS
-}
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java
index 628ee3ef..69219b0a 100644
--- a/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java
+++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/CancelDurableOperationUseCaseTest.java
@@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import dev.caskeleton.application.idempotency.RequestFingerprint;
import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore;
+import dev.caskeleton.application.tenant.TenantId;
import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
import java.time.Clock;
import java.time.Duration;
@@ -28,7 +29,8 @@ class CancelDurableOperationUseCaseTest {
operations.submit(submission("t-1"));
// when
- useCase.handle(new CancelDurableOperationCommand(ID, "alice", "t-1"));
+ useCase.handle(
+ new CancelDurableOperationCommand(ID, "alice", Optional.of(new TenantId("t-1"))));
// then
assertThat(operations.find(ID, "alice", NOW).orElseThrow().state())
@@ -39,12 +41,13 @@ class CancelDurableOperationUseCaseTest {
void cancelOfAFinishedOperationReportsNoChange() {
// given
operations.submit(submission(null));
- operations.claimNext("worker-a", Duration.ofMinutes(1), NOW);
- operations.succeed(ID, "worker-a", "/results/1", NOW);
+ DurableOperationClaim claim =
+ operations.claimNext("worker-a", Duration.ofMinutes(1), NOW).orElseThrow();
+ operations.succeed(claim, "/results/1", NOW);
// when
Optional canceled =
- useCase.handle(new CancelDurableOperationCommand(ID, "alice", null));
+ useCase.handle(new CancelDurableOperationCommand(ID, "alice", Optional.empty()));
// then
assertThat(canceled).contains(false);
@@ -57,7 +60,8 @@ class CancelDurableOperationUseCaseTest {
// when
Optional canceled =
- useCase.handle(new CancelDurableOperationCommand(ID, "alice", "t-2"));
+ useCase.handle(
+ new CancelDurableOperationCommand(ID, "alice", Optional.of(new TenantId("t-2"))));
// then
assertThat(canceled).isEmpty();
@@ -69,7 +73,8 @@ class CancelDurableOperationUseCaseTest {
operations.submit(submission(null));
// when
- Optional canceled = useCase.handle(new CancelDurableOperationCommand(ID, null, null));
+ Optional canceled =
+ useCase.handle(new CancelDurableOperationCommand(ID, null, Optional.empty()));
// then
assertThat(canceled).isEmpty();
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java
index 44c66954..7be18cc6 100644
--- a/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java
+++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/DurableOperationAccessPolicyTest.java
@@ -2,6 +2,7 @@ package dev.caskeleton.application.operation;
import static org.assertj.core.api.Assertions.assertThat;
+import dev.caskeleton.application.tenant.TenantId;
import java.time.Instant;
import java.util.Optional;
import org.junit.jupiter.params.ParameterizedTest;
@@ -33,7 +34,7 @@ class DurableOperationAccessPolicyTest {
String owner, String ownerTenant, String caller, String callerTenant, boolean expected) {
assertThat(
DurableOperationAccessPolicy.mayAccess(
- operation(owner, ownerTenant), caller, callerTenant))
+ operation(owner, ownerTenant), caller, TenantId.optional(callerTenant)))
.isEqualTo(expected);
}
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java
index 292cce01..3cb19a13 100644
--- a/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java
+++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/GetDurableOperationUseCaseTest.java
@@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import dev.caskeleton.application.idempotency.RequestFingerprint;
import dev.caskeleton.application.operation.testkit.InMemoryDurableOperationStore;
+import dev.caskeleton.application.tenant.TenantId;
import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
import java.time.Clock;
import java.time.Duration;
@@ -29,7 +30,7 @@ class GetDurableOperationUseCaseTest {
// when
Optional read =
- useCase.handle(new GetDurableOperationQuery(ID, "alice", "t-1"));
+ useCase.handle(new GetDurableOperationQuery(ID, "alice", Optional.of(new TenantId("t-1"))));
// then
assertThat(read).contains(submitted);
@@ -42,7 +43,7 @@ class GetDurableOperationUseCaseTest {
// when
Optional read =
- useCase.handle(new GetDurableOperationQuery(ID, "alice", "t-2"));
+ useCase.handle(new GetDurableOperationQuery(ID, "alice", Optional.of(new TenantId("t-2"))));
// then
assertThat(read).isEmpty();
@@ -54,7 +55,8 @@ class GetDurableOperationUseCaseTest {
operations.submit(submission(null));
// when
- Optional read = useCase.handle(new GetDurableOperationQuery(ID, null, null));
+ Optional read =
+ useCase.handle(new GetDurableOperationQuery(ID, null, Optional.empty()));
// then
assertThat(read).isEmpty();
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java
new file mode 100644
index 00000000..13f6efaf
--- /dev/null
+++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/StoreBuiltRecordsTest.java
@@ -0,0 +1,80 @@
+package dev.caskeleton.application.operation;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+import dev.caskeleton.application.idempotency.RequestFingerprint;
+import dev.caskeleton.application.outbox.ClaimedOutboxEvent;
+import dev.caskeleton.application.outbox.OutboxEventStatus;
+import dev.caskeleton.application.outbox.OutboxLease;
+import dev.caskeleton.application.outbox.testkit.OutboxEvents;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.stream.Stream;
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+/** The claims and leases a store builds from its rows refuse a bad value as each record says. */
+class StoreBuiltRecordsTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z");
+
+ static Stream refusedValues() {
+ DurableOperationSubmission submission =
+ new DurableOperationSubmission(
+ new DurableOperationId("op-1"),
+ "reports.generate",
+ "alice",
+ "tenant-a",
+ new RequestFingerprint("a".repeat(64)),
+ "{}",
+ NOW,
+ Duration.ofHours(1));
+ return Stream.of(
+ Arguments.of(
+ "owner 없는 claim",
+ (ThrowingCallable) () -> new DurableOperationClaim(submission, " ", 1),
+ "a claim needs an owner"),
+ Arguments.of(
+ "fence 0인 claim",
+ (ThrowingCallable) () -> new DurableOperationClaim(submission, "worker-1", 0),
+ "fence must be >= 1, was 0"),
+ Arguments.of(
+ "owner 없는 lease",
+ (ThrowingCallable) () -> new OperationLease(" ", NOW, NOW.plusSeconds(30)),
+ "a lease needs an owner"),
+ Arguments.of(
+ "잡는 순간 끝나는 lease",
+ (ThrowingCallable) () -> new OperationLease("worker-1", NOW, NOW),
+ "a lease that expires when taken is not a lease"),
+ Arguments.of(
+ "owner 없는 outbox lease",
+ (ThrowingCallable) () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), " ", 1),
+ "owner must not be blank"),
+ Arguments.of(
+ "token 0인 outbox lease",
+ (ThrowingCallable)
+ () -> new OutboxLease(OutboxEvents.claimed("evt-1", NOW, 1), "relay-1", 0),
+ "token must be >= 1, was 0"),
+ Arguments.of(
+ "미룬 claim이 claim보다 많은 outbox 행",
+ (ThrowingCallable)
+ () ->
+ new ClaimedOutboxEvent(
+ OutboxEvents.event("evt-2", NOW), OutboxEventStatus.IN_FLIGHT, 1, 2),
+ "deferredAttemptCount must be between 0 and attemptCount (1), was 2"));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("refusedValues")
+ void aValueAStoreCannotHaveBuiltIsAServerFault(
+ String name, ThrowingCallable build, String message) {
+ // when
+ Throwable refused = catchThrowable(build);
+
+ // then
+ assertThat(refused).isInstanceOf(IllegalStateException.class).hasMessageContaining(message);
+ }
+}
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java b/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java
index ac469e2d..9053cfab 100644
--- a/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java
+++ b/src/application/core/src/test/java/dev/caskeleton/application/operation/SubmitDurableOperationUseCaseTest.java
@@ -16,7 +16,10 @@ import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
+import java.util.ArrayDeque;
+import java.util.Deque;
import java.util.List;
+import java.util.Optional;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -34,7 +37,7 @@ class SubmitDurableOperationUseCaseTest {
private static final Instant NOW = Instant.parse("2026-08-25T10:00:00Z");
- /** An id the HTTP adapter mints: a canonical lower-case UUIDv7, used as the event id itself. */
+ /** An id the factory mints: a canonical lower-case UUIDv7, used as the event id itself. */
private static final String OPERATION_UUID = "01994e11-4d88-7000-8000-0000000000a1";
private static final DurableOperationId ID = new DurableOperationId(OPERATION_UUID);
@@ -48,16 +51,25 @@ class SubmitDurableOperationUseCaseTest {
private final FailingOutbox outbox = new FailingOutbox(transactions);
private final FakeIntegrationEventEncoder encoder =
new FakeIntegrationEventEncoder(new DurableOperationSubmittedContract());
+
+ /** What the id factory mints, in order: one id per submission, kept or not. */
+ private final Deque mintable =
+ new ArrayDeque<>(List.of(ID, new DurableOperationId("op-2"), new DurableOperationId("op-3")));
+
private final CommandUseCase useCase =
new SubmitDurableOperationUseCase(
- operations, outbox, encoder, transactions, Clock.fixed(NOW, ZoneOffset.UTC));
+ operations,
+ outbox,
+ encoder,
+ transactions,
+ mintable::removeFirst,
+ Clock.fixed(NOW, ZoneOffset.UTC));
- private static OperationCommand command(DurableOperationId id, String hash) {
+ private static OperationCommand command(String hash) {
return new OperationCommand(
- id,
"reports.generate",
"alice",
- null,
+ Optional.empty(),
new RequestFingerprint(hash),
"{\"range\":\"2026-08\"}",
Duration.ofHours(6));
@@ -66,7 +78,7 @@ class SubmitDurableOperationUseCaseTest {
@Test
@DisplayName("an accepted submission writes both the operation and the outbox event")
void submissionWritesBoth() {
- OperationAcceptance acceptance = useCase.handle(command(ID, "a".repeat(64)));
+ OperationAcceptance acceptance = useCase.handle(command("a".repeat(64)));
assertThat(acceptance.operationId()).isEqualTo(ID);
assertThat(acceptance.alreadyAccepted()).isFalse();
@@ -74,10 +86,20 @@ class SubmitDurableOperationUseCaseTest {
assertThat(outbox.appended()).hasSize(1);
}
+ @Test
+ @DisplayName("operation id는 호출자가 아니라 use case가 id factory로 만든다")
+ void theOperationIdIsMintedByTheUseCase() {
+ // when
+ OperationAcceptance acceptance = useCase.handle(command("d".repeat(64)));
+
+ // then
+ assertThat(List.of(acceptance.operationId(), mintable.size())).containsExactly(ID, 2);
+ }
+
@Test
@DisplayName("both writes happen inside one transaction")
void bothWritesShareOneTransaction() {
- useCase.handle(command(ID, "a".repeat(64)));
+ useCase.handle(command("a".repeat(64)));
// Not an implementation detail. Outside one transaction the two writes can commit
// independently, and every failure mode this use case exists to prevent comes back. The
@@ -92,7 +114,7 @@ class SubmitDurableOperationUseCaseTest {
void rollbackLeavesNothing() {
outbox.failNext = true;
- assertThatThrownBy(() -> useCase.handle(command(ID, "a".repeat(64))))
+ assertThatThrownBy(() -> useCase.handle(command("a".repeat(64))))
.isInstanceOf(IllegalStateException.class);
assertThat(operations.size()).isZero();
@@ -102,10 +124,9 @@ class SubmitDurableOperationUseCaseTest {
@Test
@DisplayName("an identical resubmission returns the first operation and publishes nothing new")
void resubmissionReusesTheFirstOperation() {
- useCase.handle(command(ID, "a".repeat(64)));
+ useCase.handle(command("a".repeat(64)));
- OperationAcceptance repeat =
- useCase.handle(command(new DurableOperationId("op-2"), "a".repeat(64)));
+ OperationAcceptance repeat = useCase.handle(command("a".repeat(64)));
// The client is pointed at the work that is actually running. Starting a second operation
// would run the same report twice and hand back an id for the copy.
@@ -116,12 +137,12 @@ class SubmitDurableOperationUseCaseTest {
}
@Test
- void retryUnderTheSameOperationIdAppendsTheOutboxEventOnlyOnce() {
+ void aRetriedSubmissionAppendsTheOutboxEventOnlyOnce() {
// given
- useCase.handle(command(ID, "a".repeat(64)));
+ useCase.handle(command("a".repeat(64)));
// when
- OperationAcceptance retry = useCase.handle(command(ID, "a".repeat(64)));
+ OperationAcceptance retry = useCase.handle(command("a".repeat(64)));
// then
assertThat(List.of(retry.alreadyAccepted(), outbox.appended().size())).containsExactly(true, 1);
@@ -130,10 +151,9 @@ class SubmitDurableOperationUseCaseTest {
@Test
@DisplayName("a different request under the same principal is a different operation")
void differentRequestsAreDifferentOperations() {
- useCase.handle(command(ID, "a".repeat(64)));
+ useCase.handle(command("a".repeat(64)));
- OperationAcceptance second =
- useCase.handle(command(new DurableOperationId("op-2"), "b".repeat(64)));
+ OperationAcceptance second = useCase.handle(command("b".repeat(64)));
assertThat(second.alreadyAccepted()).isFalse();
assertThat(operations.size()).isEqualTo(2);
@@ -143,7 +163,7 @@ class SubmitDurableOperationUseCaseTest {
@Test
void theOutboxEventIsTheSubmittedContractAnchoredToTheOperation() {
// given
- useCase.handle(command(ID, "a".repeat(64)));
+ useCase.handle(command("a".repeat(64)));
// when
ValidatedIntegrationEvent event = outbox.appended().get(0);
@@ -167,7 +187,7 @@ class SubmitDurableOperationUseCaseTest {
@Test
void theSubmittedEventNamesTheOperationAndNotTheRequest() {
// when
- useCase.handle(command(ID, "a".repeat(64)));
+ useCase.handle(command("a".repeat(64)));
// then
// The worker reads what to run from the submission it claims; the request never travels.
@@ -200,13 +220,13 @@ class SubmitDurableOperationUseCaseTest {
}
@Test
- @DisplayName("UUIDv7이 아닌 operation id로 받은 작업도 outbox에 발행할 수 있는 event로 남는다")
- void aSubmissionUnderAClientChosenIdStillAppendsAPublishableEvent() {
+ @DisplayName("id factory가 UUIDv7이 아닌 id를 만들어도 outbox에 발행할 수 있는 event로 남는다")
+ void aSubmissionUnderANonUuidV7IdStillAppendsAPublishableEvent() {
// given
- DurableOperationId clientChosen = new DurableOperationId("~report");
+ mintable.addFirst(new DurableOperationId("~report"));
// when
- useCase.handle(command(clientChosen, "c".repeat(64)));
+ useCase.handle(command("c".repeat(64)));
// then
assertThat(outbox.appended())
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java
index a5a6a3fa..b22ab546 100644
--- a/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java
+++ b/src/application/core/src/test/java/dev/caskeleton/application/outbox/ClaimedOutboxEventContractTest.java
@@ -49,7 +49,8 @@ class ClaimedOutboxEventContractTest {
// then
assertThat(thrown)
- .isInstanceOf(IllegalArgumentException.class)
+ .as("a store builds the claim from its row, so a bad count is the store's fault")
+ .isInstanceOf(IllegalStateException.class)
.hasMessage("attemptCount must be >= 0, was -1");
}
}
diff --git a/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java b/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java
index 75963702..b7f98131 100644
--- a/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java
+++ b/src/application/core/src/test/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplayContractTest.java
@@ -1,17 +1,17 @@
package dev.caskeleton.application.realtime.testkit;
-import dev.caskeleton.application.realtime.LiveEventReplayPort;
+import dev.caskeleton.application.realtime.LiveEventReplayQueryPort;
import java.time.Duration;
/** The in-memory history keeps the replay contract the transport tests run over. */
-class InMemoryLiveEventReplayContractTest extends LiveEventReplayContract {
+class InMemoryLiveEventReplayContractTest extends LiveEventReplayQueryContract {
private static final Duration RETENTION = Duration.ofHours(1);
private final InMemoryLiveEventReplay replay = new InMemoryLiveEventReplay(clock(), RETENTION);
@Override
- protected LiveEventReplayPort replay() {
+ protected LiveEventReplayQueryPort replay() {
return replay;
}
diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java
index b7fe8a5c..a913ffb4 100644
--- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java
+++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/messaging/event/testkit/FakeIntegrationEventPublisher.java
@@ -4,11 +4,13 @@ import dev.caskeleton.application.messaging.event.IntegrationEventPublishPort;
import dev.caskeleton.application.messaging.event.ValidatedIntegrationEvent;
import dev.caskeleton.application.outbox.OutboxPublishAttempt;
import dev.caskeleton.application.outbox.OutboxPublishOutcome;
-import java.util.ArrayList;
+import java.time.Duration;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
+import java.util.concurrent.CopyOnWriteArrayList;
/**
* The {@link IntegrationEventPublishPort} fake: records every event it is handed and answers with
@@ -17,8 +19,55 @@ import java.util.concurrent.CompletionStage;
*/
public final class FakeIntegrationEventPublisher implements IntegrationEventPublishPort {
- private final List published = new ArrayList<>();
+ // A relay thread publishes while the test thread reads.
+ private final List published = new CopyOnWriteArrayList<>();
private OutboxPublishAttempt answer = OutboxPublishAttempt.confirmed();
+ private boolean silent;
+ private Duration blocksFor = Duration.ZERO;
+ private RuntimeException fails;
+ private Optional longestAttempt = Optional.empty();
+
+ /**
+ * Never completes the stage of any later publish, and reports {@code longestAttempt} as the
+ * longest a publish takes: a broker that does not answer, behind a publisher that promised it
+ * would within that bound.
+ *
+ * @param longestAttempt what {@link #longestAttempt()} reports
+ * @return this fake
+ */
+ public FakeIntegrationEventPublisher silentPast(Duration longestAttempt) {
+ this.longestAttempt = Optional.of(Objects.requireNonNull(longestAttempt, "longestAttempt"));
+ this.silent = true;
+ return this;
+ }
+
+ /**
+ * Holds the caller for {@code blocking} inside every later {@code publish}, before the stage is
+ * handed back: a send that blocks its caller, as a producer waiting on cluster metadata does.
+ *
+ * @param blocking how long {@code publish} blocks
+ * @return this fake
+ */
+ public FakeIntegrationEventPublisher blockingFor(Duration blocking) {
+ this.blocksFor = Objects.requireNonNull(blocking, "blocking");
+ return this;
+ }
+
+ /**
+ * Completes every later publish's stage exceptionally with {@code failure}.
+ *
+ * @param failure what the stage fails with
+ * @return this fake
+ */
+ public FakeIntegrationEventPublisher failingWith(RuntimeException failure) {
+ this.fails = Objects.requireNonNull(failure, "failure");
+ return this;
+ }
+
+ @Override
+ public Optional longestAttempt() {
+ return longestAttempt;
+ }
/**
* Answers every later publish with {@code outcome}, and for anything but a confirmation the
@@ -45,7 +94,17 @@ public final class FakeIntegrationEventPublisher implements IntegrationEventPubl
@Override
public CompletionStage publish(ValidatedIntegrationEvent event) {
published.add(Objects.requireNonNull(event, "event must not be null"));
- return CompletableFuture.completedFuture(answer);
+ if (!blocksFor.isZero()) {
+ try {
+ Thread.sleep(blocksFor);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (fails != null) {
+ return CompletableFuture.failedFuture(fails);
+ }
+ return silent ? new CompletableFuture<>() : CompletableFuture.completedFuture(answer);
}
/** Every event handed to this fake, in order. */
diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java
index 4be958d2..38f896aa 100644
--- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java
+++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/DurableOperationStoreContract.java
@@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import dev.caskeleton.application.idempotency.RequestFingerprint;
import dev.caskeleton.application.operation.DurableOperation;
+import dev.caskeleton.application.operation.DurableOperationClaim;
import dev.caskeleton.application.operation.DurableOperationId;
import dev.caskeleton.application.operation.DurableOperationState;
import dev.caskeleton.application.operation.DurableOperationStorePort;
@@ -25,6 +26,10 @@ import org.junit.jupiter.api.Test;
*
* Every call goes through {@link #inTransaction}, because the real store refuses writes without
* a caller transaction and because a resubmission must leave that transaction usable.
+ *
+ *
A lease write names the {@link DurableOperationClaim} it holds. The fence cases are why: the
+ * same worker id claiming an operation again must fence out its own earlier run, which a check on
+ * the worker id alone lets through.
*/
public abstract class DurableOperationStoreContract {
@@ -33,6 +38,9 @@ public abstract class DurableOperationStoreContract {
private static final Duration TTL = Duration.ofDays(1);
private static final String HASH_A = "a".repeat(64);
private static final String HASH_B = "b".repeat(64);
+ private static final String HASH_C = "c".repeat(64);
+ private static final OperationFailure TIMED_OUT =
+ new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true);
/** The store under test, empty at the start of each test. */
protected abstract DurableOperationStorePort store();
@@ -182,10 +190,36 @@ public abstract class DurableOperationStoreContract {
submit(submission);
// when
- Optional claimed = claim("worker-a", NOW);
+ Optional claimed = claim("worker-a", NOW);
// then
- assertThat(claimed).contains(submission);
+ assertThat(claimed).contains(new DurableOperationClaim(submission, "worker-a", 1));
+ }
+
+ @Test
+ void everyClaimOfAnOperationGetsAHigherFence() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim first = claim("worker-a", NOW).orElseThrow();
+
+ // when
+ DurableOperationClaim second = claim("worker-a", NOW.plus(LEASE).plusSeconds(1)).orElseThrow();
+
+ // then
+ assertThat(second.fence()).isGreaterThan(first.fence());
+ }
+
+ @Test
+ void canceledOperationIsNotHandedToAWorker() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ cancel("op-1", "alice", NOW.plusSeconds(5));
+
+ // when
+ Optional claimed = claim("worker-a", NOW.plusSeconds(6));
+
+ // then
+ assertThat(claimed).isEmpty();
}
@Test
@@ -195,7 +229,7 @@ public abstract class DurableOperationStoreContract {
claim("worker-a", NOW);
// when
- Optional claimed = claim("worker-b", NOW.plusSeconds(30));
+ Optional claimed = claim("worker-b", NOW.plusSeconds(30));
// then
assertThat(claimed).isEmpty();
@@ -224,11 +258,11 @@ public abstract class DurableOperationStoreContract {
void heartbeatFromTheLeaseHolderExtendsTheLease() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
Instant beat = NOW.plusSeconds(60);
// when
- inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, beat));
+ inTransaction(() -> store().heartbeat(held, LEASE, beat));
// then
assertThat(find("op-1", "alice", beat).orElseThrow().lease())
@@ -239,13 +273,12 @@ public abstract class DurableOperationStoreContract {
void heartbeatFromAWorkerThatLostTheLeaseIsRefused() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow();
Instant later = NOW.plus(LEASE).plusSeconds(1);
claim("worker-b", later);
// when
- boolean extended =
- inTransaction(() -> store().heartbeat(id("op-1"), "worker-a", LEASE, later.plusSeconds(1)));
+ boolean extended = inTransaction(() -> store().heartbeat(stale, LEASE, later.plusSeconds(1)));
// then
assertThat(extended).isFalse();
@@ -255,12 +288,12 @@ public abstract class DurableOperationStoreContract {
void progressFromTheLeaseHolderIsPublished() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
OperationProgressSnapshot progress =
new OperationProgressSnapshot(3, Optional.of(10L), Optional.of("extracting"));
// when
- inTransaction(() -> store().reportProgress(id("op-1"), "worker-a", progress));
+ inTransaction(() -> store().reportProgress(held, progress));
// then
assertThat(find("op-1", "alice", NOW).orElseThrow().progress()).contains(progress);
@@ -270,12 +303,13 @@ public abstract class DurableOperationStoreContract {
void progressFromAnotherWorkerIsRefused() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
+ DurableOperationClaim impostor =
+ new DurableOperationClaim(held.submission(), "worker-b", held.fence());
// when
boolean published =
- inTransaction(
- () -> store().reportProgress(id("op-1"), "worker-b", OperationProgressSnapshot.of(1)));
+ inTransaction(() -> store().reportProgress(impostor, OperationProgressSnapshot.of(1)));
// then
assertThat(published).isFalse();
@@ -285,11 +319,11 @@ public abstract class DurableOperationStoreContract {
void successFromTheLeaseHolderIsRecorded() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
Instant done = NOW.plusSeconds(10);
// when
- inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", done));
+ inTransaction(() -> store().succeed(held, "/results/a", done));
// then
assertThat(find("op-1", "alice", done).orElseThrow())
@@ -309,28 +343,106 @@ public abstract class DurableOperationStoreContract {
void staleWorkerCannotRecordAResultOverItsSuccessor() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim stale = claim("worker-a", NOW).orElseThrow();
Instant later = NOW.plus(LEASE).plusSeconds(1);
claim("worker-b", later);
// when
boolean recorded =
- inTransaction(
- () -> store().succeed(id("op-1"), "worker-a", "/results/a", later.plusSeconds(1)));
+ inTransaction(() -> store().succeed(stale, "/results/a", later.plusSeconds(1)));
// then
assertThat(recorded).isFalse();
}
+ @Test
+ void heartbeatOfAnEarlierRunOfTheSameWorkerIsRefused() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow();
+ Instant later = NOW.plus(LEASE).plusSeconds(1);
+ claim("worker-a", later);
+
+ // when
+ boolean accepted = inTransaction(() -> store().heartbeat(earlier, LEASE, later.plusSeconds(1)));
+
+ // then
+ assertThat(accepted).isFalse();
+ }
+
+ @Test
+ void progressOfAnEarlierRunOfTheSameWorkerIsRefused() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow();
+ Instant later = NOW.plus(LEASE).plusSeconds(1);
+ claim("worker-a", later);
+
+ // when
+ boolean accepted =
+ inTransaction(() -> store().reportProgress(earlier, OperationProgressSnapshot.of(1)));
+
+ // then
+ assertThat(accepted).isFalse();
+ }
+
+ @Test
+ void successOfAnEarlierRunOfTheSameWorkerIsRefused() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow();
+ Instant later = NOW.plus(LEASE).plusSeconds(1);
+ claim("worker-a", later);
+
+ // when
+ boolean accepted =
+ inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(1)));
+
+ // then
+ assertThat(accepted).isFalse();
+ }
+
+ @Test
+ void failureOfAnEarlierRunOfTheSameWorkerIsRefused() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow();
+ Instant later = NOW.plus(LEASE).plusSeconds(1);
+ claim("worker-a", later);
+
+ // when
+ boolean accepted = inTransaction(() -> store().fail(earlier, TIMED_OUT, later.plusSeconds(1)));
+
+ // then
+ assertThat(accepted).isFalse();
+ }
+
+ @Test
+ void resultOfTheRunThatTookOverSurvivesItsSameWorkerPredecessor() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim earlier = claim("worker-a", NOW).orElseThrow();
+ Instant later = NOW.plus(LEASE).plusSeconds(1);
+ DurableOperationClaim current = claim("worker-a", later).orElseThrow();
+ inTransaction(() -> store().succeed(current, "/results/current", later.plusSeconds(1)));
+
+ // when
+ inTransaction(() -> store().succeed(earlier, "/results/earlier", later.plusSeconds(2)));
+
+ // then
+ assertThat(find("op-1", "alice", later.plusSeconds(2)).orElseThrow().resultReference())
+ .contains("/results/current");
+ }
+
@Test
void failureFromTheLeaseHolderIsRecorded() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
OperationFailure failure = new OperationFailure("DEPENDENCY_TIMEOUT", "timed out", true);
// when
- inTransaction(() -> store().fail(id("op-1"), "worker-a", failure, NOW.plusSeconds(10)));
+ inTransaction(() -> store().fail(held, failure, NOW.plusSeconds(10)));
// then
assertThat(find("op-1", "alice", NOW.plusSeconds(10)).orElseThrow())
@@ -369,8 +481,8 @@ public abstract class DurableOperationStoreContract {
void cancelNeverReversesASuccess() {
// given
submit(submission("op-1", null, HASH_A));
- claim("worker-a", NOW);
- inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10)));
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
+ inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10)));
// when
cancel("op-1", "alice", NOW.plusSeconds(20));
@@ -407,19 +519,35 @@ public abstract class DurableOperationStoreContract {
}
@Test
- void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() {
+ void reclaimLeavesALiveLeaseAlone() {
// given
submit(submission("op-1", null, HASH_A));
claim("worker-a", NOW);
- inTransaction(() -> store().succeed(id("op-1"), "worker-a", "/results/a", NOW.plusSeconds(10)));
+
+ // when
+ List reclaimed =
+ inTransaction(() -> store().reclaimExpiredLeases(NOW.plusSeconds(30)));
+
+ // then
+ assertThat(reclaimed).isEmpty();
+ }
+
+ @Test
+ void expirySweepExpiresAgedOutWorkAndLeavesFinishedWorkAlone() {
+ // given
+ submit(submission("op-1", null, HASH_A));
+ DurableOperationClaim held = claim("worker-a", NOW).orElseThrow();
+ inTransaction(() -> store().succeed(held, "/results/a", NOW.plusSeconds(10)));
submit(submission("op-2", null, HASH_B));
+ submit(submission("op-3", null, HASH_C));
+ claim("worker-b", NOW.plusSeconds(20));
// when
List expired =
inTransaction(() -> store().expireStaleOperations(NOW.plus(TTL).plus(TTL)));
// then
- assertThat(expired).containsExactly(id("op-2"));
+ assertThat(expired).containsExactlyInAnyOrder(id("op-2"), id("op-3"));
}
private DurableOperationSubmitResult submit(DurableOperationSubmission submission) {
@@ -430,7 +558,7 @@ public abstract class DurableOperationStoreContract {
return inTransaction(() -> store().find(id(operationId), principal, now));
}
- private Optional claim(String workerId, Instant now) {
+ private Optional claim(String workerId, Instant now) {
return inTransaction(() -> store().claimNext(workerId, LEASE, now));
}
diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java
index 019193c3..3c76c99a 100644
--- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java
+++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/operation/testkit/InMemoryDurableOperationStore.java
@@ -1,6 +1,7 @@
package dev.caskeleton.application.operation.testkit;
import dev.caskeleton.application.operation.DurableOperation;
+import dev.caskeleton.application.operation.DurableOperationClaim;
import dev.caskeleton.application.operation.DurableOperationId;
import dev.caskeleton.application.operation.DurableOperationState;
import dev.caskeleton.application.operation.DurableOperationStorePort;
@@ -26,8 +27,10 @@ import java.util.concurrent.ConcurrentHashMap;
* Written to the same rules rather than to whatever makes the tests pass: a submission scope
* that already exists returns the existing operation, a lease is checked on every write, and a
* terminal operation is never reopened. A fake that was laxer than the adapter would let the use
- * case tests certify behaviour the real store refuses. {@link DurableOperationStoreContract} holds
- * this fake and the adapter to the same cases.
+ * case tests certify behaviour the real store refuses. Every claim increments the operation's
+ * fence, and a write is accepted only from the claim holding the current fence, as the adapter's
+ * {@code lease_fence} column does. {@link DurableOperationStoreContract} holds this fake and the
+ * adapter to the same cases.
*
*
Built over a {@link DirectTransactionPort}, every write made inside one of its transactions is
* undone when that transaction rolls back, as the database undoes it; built without one, writes
@@ -41,6 +44,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
private final Map byScope = new ConcurrentHashMap<>();
private final Map submissions =
new ConcurrentHashMap<>();
+ private final Map fences = new ConcurrentHashMap<>();
private final DirectTransactionPort transactions;
/** A store whose writes stay as they are made. */
@@ -103,7 +107,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
}
@Override
- public Optional claimNext(
+ public Optional claimNext(
String workerId, Duration leaseDuration, Instant now) {
return byId.values().stream()
.filter(
@@ -126,19 +130,22 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
Optional.empty(),
Optional.of(new OperationLease(workerId, now, now.plus(leaseDuration))));
write(claimed.operationId(), claimed);
- return submissions.get(claimed.operationId());
+ long previous = fences.getOrDefault(claimed.operationId(), 0L);
+ fences.put(claimed.operationId(), previous + 1);
+ onRollback(() -> fences.put(claimed.operationId(), previous));
+ return new DurableOperationClaim(
+ submissions.get(claimed.operationId()), workerId, previous + 1);
});
}
@Override
- public boolean heartbeat(
- DurableOperationId operationId, String workerId, Duration leaseDuration, Instant now) {
- DurableOperation operation = byId.get(operationId);
- if (!holdsLease(operation, workerId, now)) {
+ public boolean heartbeat(DurableOperationClaim claim, Duration leaseDuration, Instant now) {
+ DurableOperation operation = byId.get(claim.operationId());
+ if (!holdsLease(operation, claim, now)) {
return false;
}
write(
- operationId,
+ claim.operationId(),
replace(
operation,
operation.state(),
@@ -152,14 +159,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
}
@Override
- public boolean reportProgress(
- DurableOperationId operationId, String workerId, OperationProgressSnapshot progress) {
- DurableOperation operation = byId.get(operationId);
- if (operation == null || !ownedBy(operation, workerId)) {
+ public boolean reportProgress(DurableOperationClaim claim, OperationProgressSnapshot progress) {
+ DurableOperation operation = byId.get(claim.operationId());
+ if (!heldBy(operation, claim)) {
return false;
}
write(
- operationId,
+ claim.operationId(),
replace(
operation,
operation.state(),
@@ -173,17 +179,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
}
@Override
- public boolean succeed(
- DurableOperationId operationId,
- String workerId,
- String resultReference,
- Instant completedAt) {
- DurableOperation operation = byId.get(operationId);
- if (operation == null || !ownedBy(operation, workerId)) {
+ public boolean succeed(DurableOperationClaim claim, String resultReference, Instant completedAt) {
+ DurableOperation operation = byId.get(claim.operationId());
+ if (!heldBy(operation, claim)) {
return false;
}
write(
- operationId,
+ claim.operationId(),
replace(
operation,
DurableOperationState.SUCCEEDED,
@@ -197,17 +199,13 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
}
@Override
- public boolean fail(
- DurableOperationId operationId,
- String workerId,
- OperationFailure failure,
- Instant completedAt) {
- DurableOperation operation = byId.get(operationId);
- if (operation == null || !ownedBy(operation, workerId)) {
+ public boolean fail(DurableOperationClaim claim, OperationFailure failure, Instant completedAt) {
+ DurableOperation operation = byId.get(claim.operationId());
+ if (!heldBy(operation, claim)) {
return false;
}
write(
- operationId,
+ claim.operationId(),
replace(
operation,
DurableOperationState.FAILED,
@@ -316,6 +314,7 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
byId.clear();
byScope.clear();
submissions.clear();
+ fences.clear();
}
// Same as the adapter: a poll after the TTL sees EXPIRED even before the sweeper runs.
@@ -334,13 +333,15 @@ public final class InMemoryDurableOperationStore implements DurableOperationStor
Optional.empty());
}
- private static boolean holdsLease(DurableOperation operation, String workerId, Instant now) {
- return operation != null && ownedBy(operation, workerId) && !operation.leaseExpiredAt(now);
+ private boolean holdsLease(DurableOperation operation, DurableOperationClaim claim, Instant now) {
+ return heldBy(operation, claim) && !operation.leaseExpiredAt(now);
}
- private static boolean ownedBy(DurableOperation operation, String workerId) {
- return operation.state() == DurableOperationState.RUNNING
- && operation.lease().map(lease -> lease.workerId().equals(workerId)).orElse(false);
+ private boolean heldBy(DurableOperation operation, DurableOperationClaim claim) {
+ return operation != null
+ && operation.state() == DurableOperationState.RUNNING
+ && operation.lease().map(lease -> lease.workerId().equals(claim.workerId())).orElse(false)
+ && fences.getOrDefault(claim.operationId(), 0L) == claim.fence();
}
private static DurableOperation replace(
diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java
index f6dc06d2..462220e4 100644
--- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java
+++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/InMemoryLiveEventReplay.java
@@ -1,6 +1,6 @@
package dev.caskeleton.application.realtime.testkit;
-import dev.caskeleton.application.realtime.LiveEventReplayPort;
+import dev.caskeleton.application.realtime.LiveEventReplayQueryPort;
import dev.caskeleton.application.realtime.ReplayCursorUnavailableException;
import dev.caskeleton.application.realtime.ReplayWindow;
import dev.caskeleton.application.realtime.ReplayedEvent;
@@ -15,15 +15,15 @@ import java.util.Objects;
import java.util.Optional;
/**
- * In-memory {@link LiveEventReplayPort} with the rules of the JPA history: positions per stream
- * from 1 and never reused, an event replayable until {@code appendedAt + retention} on the given
- * clock, a cursor older than what is retained refused with {@link
- * ReplayCursorUnavailableException}. {@link LiveEventReplayContract} holds it and the JPA store to
- * the same cases.
+ * In-memory {@link LiveEventReplayQueryPort} with the rules of the JPA history: positions per
+ * stream from 1 and never reused, an event replayable until {@code appendedAt + retention} on the
+ * given clock, a cursor older than what is retained refused with {@link
+ * ReplayCursorUnavailableException}. {@link LiveEventReplayQueryContract} holds it and the JPA
+ * store to the same cases.
*
* Single-threaded: one instance per test.
*/
-public final class InMemoryLiveEventReplay implements LiveEventReplayPort {
+public final class InMemoryLiveEventReplay implements LiveEventReplayQueryPort {
private record Stored(ReplayedEvent event, Instant retainedUntil) {}
diff --git a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java
similarity index 94%
rename from src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java
rename to src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java
index 4c7e5b1f..81a3c335 100644
--- a/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayContract.java
+++ b/src/application/core/src/testFixtures/java/dev/caskeleton/application/realtime/testkit/LiveEventReplayQueryContract.java
@@ -3,7 +3,7 @@ package dev.caskeleton.application.realtime.testkit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.catchThrowable;
-import dev.caskeleton.application.realtime.LiveEventReplayPort;
+import dev.caskeleton.application.realtime.LiveEventReplayQueryPort;
import dev.caskeleton.application.realtime.ReplayCursorUnavailableException;
import dev.caskeleton.application.realtime.ReplayWindow;
import dev.caskeleton.application.realtime.ReplayedEvent;
@@ -18,14 +18,14 @@ import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
- * What every {@link LiveEventReplayPort} promises a reconnecting client, run against the in-memory
- * fake and the JPA history on real PostgreSQL.
+ * What every {@link LiveEventReplayQueryPort} promises a reconnecting client, run against the
+ * in-memory fake and the JPA history on real PostgreSQL.
*
*
Writing history is not on the port (a transport reads it and never writes it), so each
* implementation supplies its own {@link #append}. Retention is measured on {@link #clock()}, which
* the implementation is built with and the contract moves.
*/
-public abstract class LiveEventReplayContract {
+public abstract class LiveEventReplayQueryContract {
private final ContractClock clock = new ContractClock(Instant.parse("2026-11-03T00:00:00Z"));
@@ -43,7 +43,7 @@ public abstract class LiveEventReplayContract {
*
* @return the port
*/
- protected abstract LiveEventReplayPort replay();
+ protected abstract LiveEventReplayQueryPort replay();
/**
* Appends an event the way the store's writer does.
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java
index 1b5e7b0d..fa5ffde3 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/FileMetadataPort.java
@@ -10,14 +10,36 @@ import java.util.Optional;
* Authoritative file metadata port.
*
*
Every transition is conditional on both the expected version and the expected state, so two
- * writers racing on the same record produce exactly one winner and one optimistic conflict.
+ * writers racing on the same record produce exactly one winner and one optimistic conflict. How
+ * each write fails is part of the contract, because the use cases decide from the failure whether
+ * to retry, report a conflict or treat the file as gone. A record that does not exist fails as a
+ * lost precondition, like one that moved on.
*/
public interface FileMetadataPort {
+ /**
+ * Records a new file in {@code CREATED}.
+ *
+ * @return the stored record, version 0
+ * @throws RuntimeException the store's own refusal when a record already exists under {@code
+ * draft.fileId()}; file ids are server-minted, so that is a defect, not a race to recover
+ * from
+ */
FileRecord insert(FileRecordDraft draft);
Optional find(FileId fileId);
+ /**
+ * Moves a record from {@code expectedState} at {@code expectedVersion} to {@code targetState}.
+ *
+ * @return the record as stored after the move
+ * @throws IllegalStateException when {@code expectedState} may never move to {@code targetState}
+ * — a caller defect, refused before anything is written
+ * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code
+ * CONCURRENT_MODIFICATION} when the record is no longer at that version and state
+ * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code
+ * FILE_NOT_FOUND} when the record disappeared during the move
+ */
FileRecord transition(
FileId fileId,
long expectedVersion,
@@ -25,6 +47,16 @@ public interface FileMetadataPort {
FileState targetState,
FileRecordMutation mutation);
+ /**
+ * Moves a record at {@code expectedVersion} in a deletable state to {@code DELETING}.
+ *
+ * @return the record as stored after the move
+ * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code
+ * CONCURRENT_MODIFICATION} when the record is no longer at that version or not in a deletable
+ * state
+ * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code
+ * FILE_NOT_FOUND} when the record disappeared during the move
+ */
FileRecord markDeleting(FileId fileId, long expectedVersion);
/**
@@ -33,6 +65,12 @@ public interface FileMetadataPort {
* A namespace is a metadata grouping, so this changes one column and never touches the
* immutable physical object. Copying gigabytes to express an ownership change would also break
* every strong validator already handed to clients.
+ *
+ * @return the record as stored after the move
+ * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code
+ * CONCURRENT_MODIFICATION} when the record is no longer {@code READY} at that version
+ * @throws dev.caskeleton.application.fileserver.api.error.FileNotFoundException with {@code
+ * FILE_NOT_FOUND} when the record disappeared during the move
*/
FileRecord relocate(FileId fileId, long expectedVersion, StorageNamespace targetNamespace);
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java
index 9116e18f..42062eeb 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/api/metadata/UploadSessionPort.java
@@ -18,6 +18,14 @@ public interface UploadSessionPort {
Optional find(UploadId uploadId);
+ /**
+ * Takes the writer lease on an upload that holds none, or whose lease has lapsed.
+ *
+ * @return the new lease, with a fresh token and the session version after the claim
+ * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code
+ * CONCURRENT_MODIFICATION} when another owner holds an unexpired lease, the session is no
+ * longer at {@code expectedVersion}, or the upload has expired (then not retryable)
+ */
WriterLease acquireLease(
UploadId uploadId, String owner, Instant now, Duration leaseDuration, long expectedVersion);
@@ -34,6 +42,15 @@ public interface UploadSessionPort {
*/
WriterLease renewLease(WriterLease lease, Instant now, Duration leaseDuration);
+ /**
+ * Advances the committed offset from {@code expectedOffset} to {@code committedOffset} for the
+ * holder of {@code lease}.
+ *
+ * @return the session as stored after the commit
+ * @throws dev.caskeleton.application.fileserver.api.error.FileserverException with {@code
+ * CONCURRENT_MODIFICATION} when the lease token no longer holds the session or the stored
+ * offset is not {@code expectedOffset}; nothing is committed
+ */
UploadSession commitOffset(
UploadId uploadId, WriterLease lease, long expectedOffset, long committedOffset);
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java
index 4577fec0..ad90e67d 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/CleanupQueuePort.java
@@ -44,6 +44,16 @@ public interface CleanupQueuePort {
/** Records a failed attempt and when to try again, if {@code item}'s claim is still current. */
void markFailed(CleanupItem item, String reasonCode, Instant nextAttemptAt);
+ /**
+ * Gives a claimed item back unattempted, if {@code item}'s claim is still current.
+ *
+ * For an item the batch claimed and then did not start — its byte budget or its claim's lease
+ * ran out first. The claim ends and the item is claimable again at {@code nextAttemptAt}; the
+ * attempt counter does not move, because nothing was tried. Recorded as a failure instead, every
+ * such release spent an attempt, and a busy queue's tail was abandoned without ever being tried.
+ */
+ void release(CleanupItem item, Instant nextAttemptAt);
+
/**
* Parks an item for good, if {@code item}'s claim is still current.
*
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java
index d28a7d40..71033678 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/DefaultCleanupService.java
@@ -49,6 +49,20 @@ public final class DefaultCleanupService implements CleanupService {
*/
public static final Duration CLAIM_LEASE = Duration.ofMinutes(10);
+ /**
+ * No claimed item is started once less than this is left of the batch's claim lease.
+ *
+ *
A batch claims up to {@code maxItems} under one lease and works them one after another, so
+ * the last item starts long after the claim. Past the lease, the reaper may already have handed
+ * that item to another worker, and both delete it and settle its quota. The outbox relay checks
+ * its batch against its lease when it is assembled and the notification scheduler claims only
+ * what it can start; cleanup has no bound on one delete to check against, so it stops starting
+ * items instead, and releases the rest for the next batch the way the byte budget does (round9
+ * review 2026-11-12, RR9-01-23). Two minutes is the time one delete against slow storage is
+ * allowed to finish inside its lease.
+ */
+ static final Duration LAST_START_BEFORE_LEASE_ENDS = Duration.ofMinutes(2);
+
private final CleanupQueuePort queue;
private final CleanupContentPort contentGateway;
private final FileMetadataPort metadataStore;
@@ -95,6 +109,7 @@ public final class DefaultCleanupService implements CleanupService {
// claiming means the recovered items are eligible in this same pass.
transactions.inWrite(() -> queue.reclaimExpiredClaims(now, maxItems));
List due = transactions.inWrite(() -> queue.claimDue(now, maxItems, CLAIM_LEASE));
+ Instant lastStart = now.plus(CLAIM_LEASE).minus(LAST_START_BEFORE_LEASE_ENDS);
int deleted = 0;
int skippedActiveLease = 0;
@@ -113,10 +128,16 @@ public final class DefaultCleanupService implements CleanupService {
failed++;
continue;
}
+ if (!clock.instant().isBefore(lastStart)) {
+ // The claim's lease is nearly spent; an item started now could still be running when the
+ // reaper hands it to another worker. Given back untried, like the byte budget's.
+ release(item, now);
+ continue;
+ }
if (reclaimed >= maxBytes) {
- // The byte budget is spent; the remaining claims are released for the next batch rather
- // than being marked done, so nothing is silently dropped.
- markFailed(item, "BATCH_BYTE_BUDGET_EXHAUSTED", now);
+ // The byte budget is spent; the remaining claims are given back untried rather than
+ // being marked done, so nothing is silently dropped and no attempt is spent on them.
+ release(item, now);
continue;
}
Outcome outcome = process(item, now);
@@ -330,6 +351,11 @@ public final class DefaultCleanupService implements CleanupService {
});
}
+ /** Gives a claimed item back untried, in a write transaction of its own. */
+ private void release(CleanupItem item, Instant nextAttemptAt) {
+ transactions.inWrite(() -> queue.release(item, nextAttemptAt));
+ }
+
/**
* Reschedules an item, or parks it once this failure spends its retry budget.
*
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java
new file mode 100644
index 00000000..4b4f8612
--- /dev/null
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupCommand.java
@@ -0,0 +1,18 @@
+package dev.caskeleton.application.fileserver.cleanup;
+
+import dev.caskeleton.application.command.Command;
+
+/**
+ * The bounds of one unattended cleanup batch.
+ *
+ * @param maxItems most queued items the batch claims
+ * @param maxBytes most bytes the batch reclaims before it stops starting items
+ */
+public record RunScheduledCleanupCommand(int maxItems, long maxBytes) implements Command {
+
+ public RunScheduledCleanupCommand {
+ if (maxItems < 1 || maxBytes < 1) {
+ throw new IllegalStateException("cleanup batch bounds must be positive");
+ }
+ }
+}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java
new file mode 100644
index 00000000..9eed3d9e
--- /dev/null
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCase.java
@@ -0,0 +1,48 @@
+package dev.caskeleton.application.fileserver.cleanup;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.util.Objects;
+
+/**
+ * One unattended cleanup batch: reclaim what the queue holds, within the batch bounds.
+ *
+ * The same work an operator starts with {@code CleanupUploadsUseCase}, which declares {@code
+ * file:admin}. A write both a user and the system start is two use cases: this one is the
+ * scheduler's, on a thread with no {@code Authentication}, so it carries no permission and no user
+ * transport may reach it.
+ *
+ *
{@code RepositoryAccess.NONE}: the batch is {@link CleanupService}'s, which opens a boundary
+ * per claimed item. One around the whole batch would hold a connection across every deletion.
+ *
+ *
Not a bean. The Fileserver composition's cleanup worker constructs it and keeps it.
+ */
+@SystemInvoked(
+ reason =
+ "the Fileserver cleanup worker runs it on the scheduler thread, which has no"
+ + " Authentication; an operator's run is CleanupUploadsUseCase with file:admin")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.NONE,
+ // The cleanup service deletes staged and orphaned objects in the object store.
+ externalOutboundAllowed = true)
+public class RunScheduledCleanupUseCase
+ implements CommandUseCase {
+
+ private final CleanupService cleanup;
+
+ public RunScheduledCleanupUseCase(CleanupService cleanup) {
+ this.cleanup = Objects.requireNonNull(cleanup, "cleanup");
+ }
+
+ @Override
+ public CleanupBatchResult handle(RunScheduledCleanupCommand command) {
+ Objects.requireNonNull(command, "command");
+ return cleanup.runBatch(command.maxItems(), command.maxBytes());
+ }
+}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java
deleted file mode 100644
index ab6fd5df..00000000
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/observability/FileserverSpans.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package dev.caskeleton.application.fileserver.observability;
-
-/**
- * The exact span names the design fixes.
- *
- * Naming them once means a dashboard or an alert built on one of these keeps working when the
- * code that emits it moves. A span name invented at a call site would silently break both.
- */
-public final class FileserverSpans {
-
- public static final String UPLOAD_CREATE = "upload.create";
- public static final String UPLOAD_APPEND = "upload.append";
- public static final String UPLOAD_FINALIZE = "upload.finalize";
- public static final String VERIFY_DIGEST = "verify.digest";
- public static final String VERIFY_MEDIA_TYPE = "verify.media-type";
- public static final String VERIFY_MALWARE = "verify.malware";
- public static final String STORAGE_PUBLISH = "storage.publish";
- public static final String STORAGE_STAT = "storage.stat";
- public static final String METADATA_TRANSITION = "metadata.transition";
- public static final String DOWNLOAD_AUTHORIZE = "download.authorize";
- public static final String DOWNLOAD_RESOLVE_RANGE = "download.resolve-range";
- public static final String DOWNLOAD_OPEN = "download.open";
- public static final String DOWNLOAD_DELEGATE = "download.delegate";
- public static final String CLEANUP_ITEM = "cleanup.item";
- public static final String RECONCILE_FILE = "reconcile.file";
-
- private FileserverSpans() {}
-}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java
index a1f15e75..5e2f0fb3 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/CleanupUploadsCommand.java
@@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in;
import dev.caskeleton.application.command.Command;
import dev.caskeleton.application.fileserver.api.security.RequestContext;
import java.util.Objects;
+import java.util.Optional;
/**
* Run one cleanup batch.
@@ -14,7 +15,28 @@ import java.util.Objects;
public record CleanupUploadsCommand(int maxItems, long maxBytes, RequestContext context)
implements Command {
+ /** How many queued items a request that names no bound processes. */
+ public static final int DEFAULT_MAX_ITEMS = 100;
+
+ /** The byte budget of a request that names none: 1 GiB. */
+ public static final long DEFAULT_MAX_BYTES = 1L << 30;
+
public CleanupUploadsCommand {
Objects.requireNonNull(context, "context");
}
+
+ /**
+ * The batch a request asks for, with absent bounds at the use case's defaults rather than at
+ * whatever a transport chose (round9 review 2026-11-12, RR9-05-19).
+ *
+ * @param maxItems absent means {@link #DEFAULT_MAX_ITEMS}
+ * @param maxBytes absent means {@link #DEFAULT_MAX_BYTES}
+ * @param context the operator
+ * @return the command
+ */
+ public static CleanupUploadsCommand requested(
+ Optional maxItems, Optional maxBytes, RequestContext context) {
+ return new CleanupUploadsCommand(
+ maxItems.orElse(DEFAULT_MAX_ITEMS), maxBytes.orElse(DEFAULT_MAX_BYTES), context);
+ }
}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java
index 3a8960e4..f3176cc0 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListIncompleteUploadsQuery.java
@@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in;
import dev.caskeleton.application.fileserver.api.security.RequestContext;
import dev.caskeleton.application.query.Query;
import java.util.Objects;
+import java.util.Optional;
/**
* List incomplete uploads.
@@ -12,7 +13,16 @@ import java.util.Objects;
*/
public record ListIncompleteUploadsQuery(int limit, RequestContext context) implements Query {
+ /** How many a request that names no limit lists. */
+ public static final int DEFAULT_LIMIT = 100;
+
public ListIncompleteUploadsQuery {
Objects.requireNonNull(context, "context");
}
+
+ /** The query a request asks for; an absent limit is {@link #DEFAULT_LIMIT}. */
+ public static ListIncompleteUploadsQuery requested(
+ Optional limit, RequestContext context) {
+ return new ListIncompleteUploadsQuery(limit.orElse(DEFAULT_LIMIT), context);
+ }
}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java
index 6456d0a9..a6d71fae 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/ListOrphansQuery.java
@@ -3,6 +3,7 @@ package dev.caskeleton.application.fileserver.port.in;
import dev.caskeleton.application.fileserver.api.security.RequestContext;
import dev.caskeleton.application.query.Query;
import java.util.Objects;
+import java.util.Optional;
/**
* List orphaned objects.
@@ -12,7 +13,15 @@ import java.util.Objects;
*/
public record ListOrphansQuery(int limit, RequestContext context) implements Query {
+ /** How many a request that names no limit lists. */
+ public static final int DEFAULT_LIMIT = 100;
+
public ListOrphansQuery {
Objects.requireNonNull(context, "context");
}
+
+ /** The query a request asks for; an absent limit is {@link #DEFAULT_LIMIT}. */
+ public static ListOrphansQuery requested(Optional limit, RequestContext context) {
+ return new ListOrphansQuery(limit.orElse(DEFAULT_LIMIT), context);
+ }
}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java
index 52d826fb..9982be14 100644
--- a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/port/in/OrphanReconcileCommand.java
@@ -4,6 +4,7 @@ import dev.caskeleton.application.command.Command;
import dev.caskeleton.application.fileserver.api.security.RequestContext;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
/**
* Request to reconcile orphaned physical objects.
@@ -45,9 +46,55 @@ public record OrphanReconcileCommand(
expectedFingerprints = List.copyOf(expectedFingerprints);
}
+ /** How many candidates a request that names no limit considers. */
+ public static final int DEFAULT_LIMIT = 100;
+
+ /** The byte budget of an apply that names none: 1 GiB. */
+ public static final long DEFAULT_APPLY_MAX_BYTES = 1L << 30;
+
+ /** The audit reason of an apply that names none. */
+ public static final String DEFAULT_APPLY_REASON = "ORPHAN_APPLY";
+
/** Bounded dry run, the default and the only shape a caller can reach without opting in. */
public static OrphanReconcileCommand dryRun(int limit, RequestContext context) {
return new OrphanReconcileCommand(
true, limit, Long.MAX_VALUE, List.of(), "ORPHAN_SCAN", context);
}
+
+ /**
+ * The command a request asks for, with every absent field at its safe value: a dry run, the
+ * default page, no fingerprints.
+ *
+ * The defaults are the use case's, not the transport's, so every transport that offers the
+ * operation reaches the same ones; they used to be written into the HTTP controller (round9
+ * review 2026-11-12, RR9-05-19). Nothing here can be omitted into a destructive default: an apply
+ * still has to name its fingerprints.
+ *
+ * @param dryRun list only; absent means true
+ * @param limit the most candidates; absent means {@link #DEFAULT_LIMIT}
+ * @param maxBytes an apply's byte budget; absent means {@link #DEFAULT_APPLY_MAX_BYTES}
+ * @param expectedFingerprints the objects an apply may remove; absent means none
+ * @param reasonCode an apply's audit reason; absent means {@link #DEFAULT_APPLY_REASON}
+ * @param context the operator
+ * @return the command
+ */
+ public static OrphanReconcileCommand requested(
+ Optional dryRun,
+ Optional limit,
+ Optional maxBytes,
+ Optional> expectedFingerprints,
+ Optional reasonCode,
+ RequestContext context) {
+ int page = limit.orElse(DEFAULT_LIMIT);
+ if (dryRun.orElse(true)) {
+ return dryRun(page, context);
+ }
+ return new OrphanReconcileCommand(
+ false,
+ page,
+ maxBytes.orElse(DEFAULT_APPLY_MAX_BYTES),
+ expectedFingerprints.orElse(List.of()),
+ reasonCode.orElse(DEFAULT_APPLY_REASON),
+ context);
+ }
}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java
new file mode 100644
index 00000000..8d611047
--- /dev/null
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesCommand.java
@@ -0,0 +1,17 @@
+package dev.caskeleton.application.fileserver.recovery;
+
+import dev.caskeleton.application.command.Command;
+
+/**
+ * The bound of one unattended reconciliation pass.
+ *
+ * @param limit most open recovery items the pass settles
+ */
+public record ReconcileRecoverableFilesCommand(int limit) implements Command {
+
+ public ReconcileRecoverableFilesCommand {
+ if (limit < 1) {
+ throw new IllegalStateException("reconciliation limit must be positive");
+ }
+ }
+}
diff --git a/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java
new file mode 100644
index 00000000..a0347a2e
--- /dev/null
+++ b/src/application/fileserver/src/main/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCase.java
@@ -0,0 +1,48 @@
+package dev.caskeleton.application.fileserver.recovery;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * One unattended reconciliation pass: settle the files an ambiguous publish or an unconfirmed
+ * commit left open, a bounded number at a time.
+ *
+ * The scheduler's, on a thread with no {@code Authentication}, so it carries no permission and
+ * no user transport may reach it.
+ *
+ *
{@code RepositoryAccess.NONE}: each file is {@link FileReconciliationService}'s to settle, in
+ * a boundary of its own around each write.
+ *
+ *
Not a bean. The Fileserver composition's cleanup worker constructs it and keeps it.
+ */
+@SystemInvoked(
+ reason =
+ "the Fileserver cleanup worker runs it on the scheduler thread, which has no"
+ + " Authentication")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.NONE,
+ // The reconciliation service reads the object store to decide each file.
+ externalOutboundAllowed = true)
+public class ReconcileRecoverableFilesUseCase
+ implements CommandUseCase> {
+
+ private final FileReconciliationService reconciliation;
+
+ public ReconcileRecoverableFilesUseCase(FileReconciliationService reconciliation) {
+ this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation");
+ }
+
+ @Override
+ public List handle(ReconcileRecoverableFilesCommand command) {
+ Objects.requireNonNull(command, "command");
+ return reconciliation.reconcileRecoverable(command.limit());
+ }
+}
diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java
index 05357b59..0bd7a4c9 100644
Binary files a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java and b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/api/security/OriginalFilenamePolicyTest.java differ
diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java
index dd0bd88d..ead9afb2 100644
--- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java
+++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/CleanupServiceTest.java
@@ -26,7 +26,10 @@ import dev.caskeleton.application.fileserver.testkit.ReadyFileFixture;
import dev.caskeleton.application.fileserver.testkit.RecordingQuotaReclaimGateway;
import java.time.Clock;
import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
import java.time.ZoneOffset;
+import java.util.List;
import java.util.OptionalLong;
import java.util.UUID;
import org.junit.jupiter.api.DisplayName;
@@ -321,7 +324,91 @@ class CleanupServiceTest {
CleanupBatchResult result = cleanup.runBatch(100, 1);
assertThat(result.deleted()).isEqualTo(1);
- assertThat(queue.failed().values()).contains("BATCH_BYTE_BUDGET_EXHAUSTED");
+ assertThat(List.of(queue.released().size(), queue.failed().size()))
+ .as("the deferred item is given back untried; recorded as failed it spent an attempt")
+ .containsExactly(1, 0);
+ }
+
+ @Test
+ void itemsTheClaimLeaseNoLongerCoversAreReleasedNotStarted() {
+ // given
+ // Each staging delete takes just over half the time an item may still start in (the lease less
+ // its last two minutes): the first two items run, the third is handed back to the queue.
+ MovableClock slowStorage = new MovableClock(FileserverFixtures.NOW);
+ Duration perDelete =
+ DefaultCleanupService.CLAIM_LEASE
+ .minus(DefaultCleanupService.LAST_START_BEFORE_LEASE_ENDS)
+ .dividedBy(2)
+ .plusSeconds(1);
+ content.afterEachStagingDiscard(() -> slowStorage.advance(perDelete));
+ CleanupService timed =
+ new DefaultCleanupService(
+ queue,
+ content,
+ metadata,
+ sessions,
+ quota,
+ Duration.ofMinutes(5),
+ FileserverMetricsPort.noop(),
+ transactions,
+ slowStorage);
+ for (int item = 0; item < 3; item++) {
+ cancelledStagingItem();
+ }
+
+ // when
+ CleanupBatchResult result = timed.runBatch(100, 1L << 30);
+
+ // then
+ assertThat(List.of(result.deleted(), queue.released().size(), queue.failed().size()))
+ .as(
+ "the third item comes back untried rather than failed; that a release leaves the"
+ + " attempt counter where it was is the queue contract's")
+ .containsExactly(2, 1, 0);
+ }
+
+ private void cancelledStagingItem() {
+ UploadId uploadId = UploadId.of(UUID.randomUUID());
+ FileId fileId = FileId.of(UUID.randomUUID());
+ sessions.create(
+ new UploadSessionDraft(
+ uploadId,
+ fileId,
+ UploadMode.SINGLE_SHOT,
+ OptionalLong.of(SIZE),
+ FileserverFixtures.NOW.plusSeconds(3600)));
+ sessions.terminate(uploadId);
+ content.storeStaging(uploadId);
+ queue.enqueue(CleanupRequest.forStaging(CleanupType.CANCELLED_STAGING, fileId, uploadId));
+ }
+
+ /** A clock the slow-storage gateway moves forward. */
+ private static final class MovableClock extends Clock {
+
+ private Instant now;
+
+ private MovableClock(Instant start) {
+ this.now = start;
+ }
+
+ private void advance(Duration duration) {
+ now = now.plus(duration);
+ }
+
+ @Override
+ public ZoneId getZone() {
+ return ZoneOffset.UTC;
+ }
+
+ @Override
+ public Clock withZone(ZoneId zone) {
+ return this;
+ }
+
+ @Override
+ public Instant instant() {
+ return now;
+ }
}
@Test
diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java
new file mode 100644
index 00000000..ee95821c
--- /dev/null
+++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/cleanup/RunScheduledCleanupUseCaseTest.java
@@ -0,0 +1,60 @@
+package dev.caskeleton.application.fileserver.cleanup;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+import dev.caskeleton.application.security.RequiresPermission;
+import dev.caskeleton.application.security.SystemInvoked;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The scheduler's cleanup run is a use case of its own, apart from the operator's (round9 review
+ * 2026-11-12, RR9-05-06).
+ */
+class RunScheduledCleanupUseCaseTest {
+
+ @Test
+ @DisplayName("예약 cleanup은 command의 항목·byte 상한으로 batch 하나를 돌린다")
+ void aScheduledRunReclaimsOneBatchWithinTheCommandsBounds() {
+ // given
+ List> batches = new ArrayList<>();
+ RunScheduledCleanupUseCase useCase =
+ new RunScheduledCleanupUseCase(
+ (maxItems, maxBytes) -> {
+ batches.add(List.of((long) maxItems, maxBytes));
+ return CleanupBatchResult.empty();
+ });
+
+ // when
+ useCase.handle(new RunScheduledCleanupCommand(25, 4_096));
+
+ // then
+ assertThat(batches).containsExactly(List.of(25L, 4_096L));
+ }
+
+ @Test
+ @DisplayName("시스템만 부르는 쓰기라 권한을 선언하지 않고 @SystemInvoked로 그 이유를 적는다")
+ void itIsMarkedAsTheSystemsAndDeclaresNoPermission() {
+ assertThat(
+ List.of(
+ RunScheduledCleanupUseCase.class.isAnnotationPresent(SystemInvoked.class),
+ RunScheduledCleanupUseCase.class.isAnnotationPresent(RequiresPermission.class)))
+ .containsExactly(true, false);
+ }
+
+ @Test
+ @DisplayName("상한이 0인 batch는 만들 수 없다")
+ void aBatchWithNoRoomIsRefused() {
+ // when
+ Throwable refused = catchThrowable(() -> new RunScheduledCleanupCommand(0, 4_096));
+
+ // then
+ assertThat(refused)
+ .as("the worker's settings build the batch, so a bad bound is the assembly's fault")
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("cleanup batch bounds must be positive");
+ }
+}
diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java
index affc9182..6a240a63 100644
--- a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java
+++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/observability/FileserverObservabilityTest.java
@@ -76,30 +76,6 @@ class FileserverObservabilityTest {
}
}
- @Test
- void everyDesignSpanNameIsDeclaredExactlyOnce() {
- List spans =
- List.of(
- FileserverSpans.UPLOAD_CREATE,
- FileserverSpans.UPLOAD_APPEND,
- FileserverSpans.UPLOAD_FINALIZE,
- FileserverSpans.VERIFY_DIGEST,
- FileserverSpans.VERIFY_MEDIA_TYPE,
- FileserverSpans.VERIFY_MALWARE,
- FileserverSpans.STORAGE_PUBLISH,
- FileserverSpans.STORAGE_STAT,
- FileserverSpans.METADATA_TRANSITION,
- FileserverSpans.DOWNLOAD_AUTHORIZE,
- FileserverSpans.DOWNLOAD_RESOLVE_RANGE,
- FileserverSpans.DOWNLOAD_OPEN,
- FileserverSpans.DOWNLOAD_DELEGATE,
- FileserverSpans.CLEANUP_ITEM,
- FileserverSpans.RECONCILE_FILE);
-
- assertThat(spans).doesNotHaveDuplicates().hasSize(15);
- assertThat(spans).allSatisfy(name -> assertThat(name).matches("[a-z]+\\.[a-z-]+"));
- }
-
@Test
void anAuditEventCarriesFingerprintsRatherThanNames() {
SafeFileFingerprint fingerprint = new SafeFileFingerprint(KEY);
diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java
new file mode 100644
index 00000000..f90e8e26
--- /dev/null
+++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/port/in/AdminRequestDefaultsTest.java
@@ -0,0 +1,116 @@
+package dev.caskeleton.application.fileserver.port.in;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+import dev.caskeleton.application.fileserver.api.security.FileAccessSubject;
+import dev.caskeleton.application.fileserver.api.security.RequestContext;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * What an operator's admin request means when it leaves a field out.
+ *
+ * The defaults are the use case's, not the transport's: the controller used to fill them in
+ * itself, so a second transport would have had to copy them (round9 review 2026-11-12, RR9-05-19).
+ */
+class AdminRequestDefaultsTest {
+
+ private static final RequestContext OPERATOR =
+ new RequestContext(FileAccessSubject.of("operator-1", Set.of("admin")), "trace-1", "node-a");
+
+ @Test
+ @DisplayName("값을 하나도 주지 않은 orphan 정리 요청은 dry run이고 아무것도 지우지 않는다")
+ void anOrphanRequestWithNothingSetIsADryRun() {
+ // when
+ OrphanReconcileCommand command =
+ OrphanReconcileCommand.requested(
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ OPERATOR);
+
+ // then
+ assertThat(command)
+ .extracting(
+ OrphanReconcileCommand::dryRun,
+ OrphanReconcileCommand::limit,
+ OrphanReconcileCommand::expectedFingerprints)
+ .containsExactly(true, OrphanReconcileCommand.DEFAULT_LIMIT, List.of());
+ }
+
+ @Test
+ @DisplayName("apply 요청은 byte 상한 1GiB와 기본 이유 코드를 받는다")
+ void anApplyRequestGetsTheDefaultByteCeilingAndReason() {
+ // when
+ OrphanReconcileCommand command =
+ OrphanReconcileCommand.requested(
+ Optional.of(false),
+ Optional.empty(),
+ Optional.empty(),
+ Optional.of(List.of("f".repeat(64))),
+ Optional.empty(),
+ OPERATOR);
+
+ // then
+ assertThat(command)
+ .extracting(
+ OrphanReconcileCommand::dryRun,
+ OrphanReconcileCommand::maxBytes,
+ OrphanReconcileCommand::reasonCode)
+ .containsExactly(false, 1L << 30, "ORPHAN_APPLY");
+ }
+
+ @Test
+ @DisplayName("지울 fingerprint를 대지 않은 apply 요청은 거절한다")
+ void anApplyRequestThatNamesNoFingerprintIsRefused() {
+ // when
+ Throwable refused =
+ catchThrowable(
+ () ->
+ OrphanReconcileCommand.requested(
+ Optional.of(false),
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ OPERATOR));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("an apply must name the fingerprints it intends to remove");
+ }
+
+ @Test
+ @DisplayName("cleanup 요청은 항목 100개·1GiB 상한을 기본으로 받는다")
+ void aCleanupRequestGetsTheDefaultBatchBounds() {
+ // when
+ CleanupUploadsCommand command =
+ CleanupUploadsCommand.requested(Optional.empty(), Optional.empty(), OPERATOR);
+
+ // then
+ assertThat(command)
+ .extracting(CleanupUploadsCommand::maxItems, CleanupUploadsCommand::maxBytes)
+ .containsExactly(100, 1L << 30);
+ }
+
+ @Test
+ @DisplayName("목록 조회는 쪽 크기 100을 기본으로 받고, 준 값은 그대로 쓴다")
+ void listingsDefaultToAPageOfOneHundredAndKeepAGivenLimit() {
+ // when
+ List limits =
+ List.of(
+ ListOrphansQuery.requested(Optional.empty(), OPERATOR).limit(),
+ ListIncompleteUploadsQuery.requested(Optional.empty(), OPERATOR).limit(),
+ ListOrphansQuery.requested(Optional.of(7), OPERATOR).limit());
+
+ // then
+ assertThat(limits).containsExactly(100, 100, 7);
+ }
+}
diff --git a/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java
new file mode 100644
index 00000000..a2e1e2c4
--- /dev/null
+++ b/src/application/fileserver/src/test/java/dev/caskeleton/application/fileserver/recovery/ReconcileRecoverableFilesUseCaseTest.java
@@ -0,0 +1,68 @@
+package dev.caskeleton.application.fileserver.recovery;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+import dev.caskeleton.application.fileserver.api.FileId;
+import dev.caskeleton.application.security.RequiresPermission;
+import dev.caskeleton.application.security.SystemInvoked;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The scheduler's reconciliation pass is a {@code @SystemInvoked} use case (round9 review
+ * 2026-11-12, RR9-05-06).
+ */
+class ReconcileRecoverableFilesUseCaseTest {
+
+ @Test
+ @DisplayName("예약 reconcile은 command의 상한만큼 열린 복구 항목을 정리한다")
+ void aScheduledPassSettlesAsManyOpenItemsAsTheCommandAllows() {
+ // given
+ List limits = new ArrayList<>();
+ ReconcileRecoverableFilesUseCase useCase =
+ new ReconcileRecoverableFilesUseCase(
+ new FileReconciliationService() {
+ @Override
+ public ReconciliationResult reconcile(FileId fileId) {
+ throw new UnsupportedOperationException("the pass reconciles in batches only");
+ }
+
+ @Override
+ public List reconcileRecoverable(int limit) {
+ limits.add(limit);
+ return List.of();
+ }
+ });
+
+ // when
+ useCase.handle(new ReconcileRecoverableFilesCommand(40));
+
+ // then
+ assertThat(limits).containsExactly(40);
+ }
+
+ @Test
+ void itIsMarkedAsTheSystemsAndDeclaresNoPermission() {
+ assertThat(
+ List.of(
+ ReconcileRecoverableFilesUseCase.class.isAnnotationPresent(SystemInvoked.class),
+ ReconcileRecoverableFilesUseCase.class.isAnnotationPresent(
+ RequiresPermission.class)))
+ .containsExactly(true, false);
+ }
+
+ @Test
+ @DisplayName("상한이 0인 pass는 만들 수 없고, 조립의 잘못으로 실패한다")
+ void aPassWithNoRoomIsRefused() {
+ // when
+ Throwable refused = catchThrowable(() -> new ReconcileRecoverableFilesCommand(0));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("reconciliation limit must be positive");
+ }
+}
diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java
index 0db36011..b9cc169b 100644
--- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java
+++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/CleanupQueuePortContract.java
@@ -119,6 +119,40 @@ public abstract class CleanupQueuePortContract {
});
}
+ @Test
+ void aReleasedItemIsClaimableAgainWithItsAttemptUnchanged() {
+ // given
+ enqueue(contentCleanup());
+ CleanupItem item = claim(due(), 10).get(0);
+ settle(() -> queue().release(item, due()));
+
+ // when
+ List again = claim(due(), 10);
+
+ // then
+ assertThat(again)
+ .singleElement()
+ .extracting(CleanupItem::cleanupId, CleanupItem::attempt)
+ .containsExactly(item.cleanupId(), 0);
+ }
+
+ @Test
+ void aReleaseUnderAReclaimedTokenChangesNothing() {
+ // given
+ enqueue(contentCleanup());
+ CleanupItem stale = claim(due(), 10).get(0);
+ settle(() -> queue().release(stale, due()));
+ var unused = claim(due(), 10);
+
+ // when
+ settle(() -> queue().release(stale, due()));
+
+ // then
+ assertThat(claim(due(), 10))
+ .as("the stale release would hand the current claim's item to a second worker")
+ .isEmpty();
+ }
+
@Test
@DisplayName("완료된 항목은 다시 claim되지 않는다")
void aDoneItemIsNeverClaimedAgain() {
diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java
index 8a485a3c..3b842068 100644
--- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java
+++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/FakeCleanupContentGateway.java
@@ -7,6 +7,7 @@ import dev.caskeleton.application.fileserver.api.content.DeleteResult;
import dev.caskeleton.application.fileserver.cleanup.CleanupContentPort;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
/** Tracks which physical objects still exist so a test can prove nothing live was removed. */
@@ -16,6 +17,7 @@ public final class FakeCleanupContentGateway implements CleanupContentPort {
private final Set stagingObjects = new LinkedHashSet<>();
private final Set quarantined = new LinkedHashSet<>();
private final List preconditions = new java.util.ArrayList<>();
+ private Runnable afterStagingDiscard = () -> {};
public void store(ContentKey key) {
objects.add(key);
@@ -46,9 +48,20 @@ public final class FakeCleanupContentGateway implements CleanupContentPort {
return DeleteResult.removed(precondition.expectedSize().orElse(0));
}
+ /**
+ * Runs {@code hook} after every staging discard, for a test that needs the storage to be slow —
+ * advancing its clock — or to fail mid-batch.
+ *
+ * @param hook what a discard does after removing the staging object
+ */
+ public void afterEachStagingDiscard(Runnable hook) {
+ afterStagingDiscard = Objects.requireNonNull(hook, "hook");
+ }
+
@Override
public void discardStaging(UploadId uploadId) {
stagingObjects.remove(uploadId);
+ afterStagingDiscard.run();
}
public void quarantine(ContentKey key) {
diff --git a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java
index 50edd600..0dc81d84 100644
--- a/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java
+++ b/src/application/fileserver/src/testFixtures/java/dev/caskeleton/application/fileserver/testkit/InMemoryCleanupQueue.java
@@ -35,6 +35,7 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort {
private final Map rows = new LinkedHashMap<>();
private final List done = new ArrayList<>();
private final Map failed = new LinkedHashMap<>();
+ private final List released = new ArrayList<>();
private final Map abandoned = new LinkedHashMap<>();
private final List reclaimSweeps = new ArrayList<>();
private final List claimSweeps = new ArrayList<>();
@@ -108,6 +109,15 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort {
}
}
+ @Override
+ public void release(CleanupItem item, Instant nextAttemptAt) {
+ Row row = rows.get(item.cleanupId());
+ if (row != null && item.claimToken().equals(row.claimToken)) {
+ row.release(nextAttemptAt);
+ released.add(item);
+ }
+ }
+
@Override
public void markAbandoned(CleanupItem item, String reasonCode) {
if (settle(item, Status.ABANDONED, clock.instant())) {
@@ -151,6 +161,11 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort {
return Map.copyOf(failed);
}
+ /** Items the caller gave back untried, in order. */
+ public List released() {
+ return List.copyOf(released);
+ }
+
/** Items the caller parked for good, with the reason given. */
public Map abandoned() {
return Map.copyOf(abandoned);
@@ -180,6 +195,13 @@ public final class InMemoryCleanupQueue implements CleanupQueuePort {
this.nextAttemptAt = nextAttemptAt;
}
+ private void release(Instant nextAttemptAt) {
+ this.status = attempt == 0 ? Status.PENDING : Status.FAILED;
+ this.nextAttemptAt = nextAttemptAt;
+ this.claimToken = null;
+ this.leaseUntil = null;
+ }
+
private void settle(Status status, Instant nextAttemptAt) {
this.status = status;
this.attempt++;
diff --git a/src/application/notification/AGENTS.md b/src/application/notification/AGENTS.md
index 9c163f4d..cb807a1a 100644
--- a/src/application/notification/AGENTS.md
+++ b/src/application/notification/AGENTS.md
@@ -12,7 +12,7 @@
## 책임
- 알림 플랫폼의 업무 흐름: 제출·예약·취소·조회, template 게시, provider callback 수집, 발송 판정과 재시도 정책.
-- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`BatchNotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다.
+- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다.
- testFixtures(흐름 패키지의 `testkit`): 저장 포트의 in-memory fake와 공유 계약(`*Contract`), provider adapter 계약(`ProviderAdapterContract`). 모든 포트에 있지는 않다. adapter의 시험이 같은 계약을 상속한다.
## 규칙
@@ -20,6 +20,8 @@
- framework 의존이 없다(`ca.java-library`). Spring·JPA·HTTP 타입을 쓰지 않는다. 허용 의존은 registry의 `application-core`·`shared-contract`뿐이다.
- 전송 DTO(HTTP 요청·응답 record)를 여기에 두지 않는다. 전송 모양은 `adapter:inbound:web-notification`의 것이다.
- 발송 판정(마케팅 억제, category 필수)은 여기서 한다. 전송이나 설정에서 기본값을 골라 넣지 않는다.
+- 시스템이 부르는 쓰기는 `@SystemInvoked` 유스케이스다: 발송 claim·발송(`ClaimDueDeliveriesUseCase`·`DispatchClaimedDeliveryUseCase`), lease 회복, provider 결과 맞추기, provider event 다시 투영. `composition:notification-platform`의 worker가 `new`로 만들어 쥐고, bean으로 두지 않는다. 그 유스케이스가 감싸는 서비스(`LeaseRecoveryService`·`ReconciliationService` 등)는 bean이지만 inbound가 쥐면 빌드가 실패한다(`INBOUND_ADAPTERS_DEPEND_ON_PORTS_NOT_APPLICATION_SERVICES`). 언제 다시 묻고 언제 포기하는지는 이 유스케이스가 정한다. worker는 스레드와 일정만 가진다.
+- 운영자 연산(redrive·reconcile·suppress·provider 상태)은 연산마다 `platform.admin`의 유스케이스 클래스이고(`RedriveDeliveryUseCase` 등, command는 그 옆), `notification-admin:*` permission을 선언한다. `platform.port.in`에 두지 않는다: command가 `AdminActor` 같은 운영자 어휘를 실어 `port.in`과 `admin`이 서로를 보게 된다. 운영자 authority와 tenant 범위는 command의 `AdminActor`로 한 번 더 본다.
- 공개 타입이 늘면 `docs/notification/api-surface-snapshot.txt`가 바뀐다(`adapter:outbound:notification`의 `verifyNotificationApiSurface`가 이 leaf까지 본다).
## 테스트
diff --git a/src/application/notification/README.md b/src/application/notification/README.md
index 682a0357..b1372fc9 100644
--- a/src/application/notification/README.md
+++ b/src/application/notification/README.md
@@ -5,7 +5,7 @@
## 책임과 소유하지 않는 것
- 알림 플랫폼의 업무 흐름: 제출·예약·취소·조회, template 게시, provider callback 수집, 발송 판정과 재시도 정책.
-- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`BatchNotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다.
+- 들어오는 포트(`platform.port.in`의 `*UseCase`·`*Command`·`*Query`), 나가는 포트(쓰는 흐름의 패키지에 있다: `platform.dispatch`·`platform.callback`·`platform.policy`·`platform.observation`·`platform.admin`·`platform.inbox`·`platform.template`·`platform.provider`·`platform.security`), 오류 어휘(`platform.api.error`). `port.out` 패키지는 따로 없다. 나가는 포트 이름은 대개 `*Port`지만 아닌 것도 있다: `provider.NotificationProviderAdapter`·`AttachmentResolver`, `security.SecretMaterialProvider`·`ContactPointProtector`, `template.TemplateRegistry`, `callback.ProviderEventLedger`. 새 포트를 만들기 전에 이 이름들도 찾는다.
- testFixtures(흐름 패키지의 `testkit`): 저장 포트의 in-memory fake와 공유 계약(`*Contract`), provider adapter 계약(`ProviderAdapterContract`). 모든 포트에 있지는 않다. adapter의 시험이 같은 계약을 상속한다.
- 소유하지 않는 것: HTTP 모양(`adapter:inbound:web-notification`), provider 호출·설정(`adapter:outbound:notification`), 저장(`adapter:outbound:persistence-notification`), 조립(`composition:notification-platform`).
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java
index bd902930..40006855 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationClaim.java
@@ -11,6 +11,10 @@ import java.util.Optional;
* the work is done; and the same id with a different command is a mistake worth reporting rather
* than an idempotent repeat.
*
+ * A store builds it from a row, never from a request, so a value it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
* @param outcome what the caller may do
* @param existing the earlier result, when there is one
*/
@@ -37,7 +41,7 @@ public record AdminOperationClaim(Outcome outcome, OptionalThe claim and the operation run inside one write transaction. PostgreSQL's unique claim row
- * serializes a concurrent caller on the operation id; the loser observes the committed result
- * instead of reaching the side effect.
+ * The use case opens that transaction and calls this in it, so the claim and the operation
+ * commit or roll back together: the boundary is the use case's to declare, and the architecture
+ * rule that matches a capability to its {@code TransactionPort} call reads it there. PostgreSQL's
+ * unique claim row serializes a concurrent caller on the operation id; the loser observes the
+ * committed result instead of reaching the side effect.
*/
- AdminOperationResult execute(
+ AdminOperationResult runClaimed(
String operationId,
- String commandFingerprint,
+ String requestFingerprint,
AdminActor actor,
String action,
Supplier operation) {
Objects.requireNonNull(operation, "operation");
- return transactions.inWrite(
- () -> {
- AdminOperationClaim claim =
- operations.claim(operationId, commandFingerprint, actor, action);
- return claim.mayProceed() ? operation.get() : withoutRunning(claim, operationId);
- });
+ AdminOperationClaim claim = operations.claim(operationId, requestFingerprint, actor, action);
+ return claim.mayProceed() ? operation.get() : withoutRunning(claim, operationId);
}
/**
* Claims the operation id in its own transaction and runs the operation with none open.
*
- * For an operation that calls a provider over the network. Inside {@link #execute} the call
+ *
For an operation that calls a provider over the network. Inside {@link #runClaimed} the call
* held a write transaction — and a pooled connection — for as long as the provider took, once per
* attempt in the batch. The operation opens its own boundaries for what it writes, including
* saving its result, which completes the claim.
@@ -53,14 +52,14 @@ final class AdminOperationExecutor {
*/
AdminOperationResult executeOutsideTransaction(
String operationId,
- String commandFingerprint,
+ String requestFingerprint,
AdminActor actor,
String action,
Supplier operation) {
Objects.requireNonNull(operation, "operation");
AdminOperationClaim claim =
transactions.inWrite(
- () -> operations.claim(operationId, commandFingerprint, actor, action));
+ () -> operations.claim(operationId, requestFingerprint, actor, action));
if (!claim.mayProceed()) {
return withoutRunning(claim, operationId);
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java
similarity index 50%
rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java
rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java
index a2f1510d..9239c52f 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprint.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprint.java
@@ -4,55 +4,55 @@ import dev.caskeleton.shared.digest.Sha256;
import java.util.Objects;
/**
- * Stable SHA-256 identity of an operator command, excluding the operation id itself.
+ * Stable SHA-256 identity of an operator request, excluding the operation id itself.
*
* The operation id is the idempotency key. The fingerprint answers the separate question:
- * whether a repeated key names the same command. Every field is length-framed so user supplied
+ * whether a repeated key names the same request. Every field is length-framed so user supplied
* reason text cannot imitate a field boundary.
*/
-final class AdminOperationCommandFingerprint {
+final class AdminOperationRequestFingerprint {
- String of(RedriveCommand command) {
- Objects.requireNonNull(command, "command");
+ String of(RedriveRequest request) {
+ Objects.requireNonNull(request, "request");
StringBuilder canonical = action("REDRIVE");
- field(canonical, "attemptId", command.attemptId().value().toString());
- field(canonical, "reason", command.reason());
- field(canonical, "approveDuplicateRisk", Boolean.toString(command.approveDuplicateRisk()));
+ field(canonical, "attemptId", request.attemptId().value().toString());
+ field(canonical, "reason", request.reason());
+ field(canonical, "approveDuplicateRisk", Boolean.toString(request.approveDuplicateRisk()));
return digest(canonical);
}
- String of(ReconcileCommand command) {
- Objects.requireNonNull(command, "command");
+ String of(ReconcileRequest request) {
+ Objects.requireNonNull(request, "request");
StringBuilder canonical = action("RECONCILE");
- command.attemptIds().stream()
+ request.attemptIds().stream()
.map(id -> id.value().toString())
.sorted()
.forEach(value -> field(canonical, "attemptId", value));
- field(canonical, "reason", command.reason());
+ field(canonical, "reason", request.reason());
return digest(canonical);
}
- String of(SuppressCommand command) {
- Objects.requireNonNull(command, "command");
- StringBuilder canonical = action(command.remove() ? "SUPPRESS_REMOVE" : "SUPPRESS_ADD");
- field(canonical, "tenantId", command.tenantId().value());
- field(canonical, "scope", command.scope().name());
- field(canonical, "reason", command.reason().name());
- field(canonical, "targetFingerprint", command.targetFingerprint());
+ String of(SuppressionRequest request) {
+ Objects.requireNonNull(request, "request");
+ StringBuilder canonical = action(request.remove() ? "SUPPRESS_REMOVE" : "SUPPRESS_ADD");
+ field(canonical, "tenantId", request.tenantId().value());
+ field(canonical, "scope", request.scope().name());
+ field(canonical, "reason", request.reason().name());
+ field(canonical, "targetFingerprint", request.targetFingerprint());
field(
canonical,
"expiresAt",
- command.expiresAt().map(value -> Long.toString(value.toEpochMilli())).orElse("-"));
- field(canonical, "reasonText", command.reasonText());
+ request.expiresAt().map(value -> Long.toString(value.toEpochMilli())).orElse("-"));
+ field(canonical, "reasonText", request.reasonText());
return digest(canonical);
}
- String of(SetProviderStateCommand command) {
- Objects.requireNonNull(command, "command");
+ String of(ProviderStateRequest request) {
+ Objects.requireNonNull(request, "request");
StringBuilder canonical = action("PROVIDER_STATE");
- field(canonical, "profileId", command.profileId().value());
- field(canonical, "desiredState", command.desiredState().name());
- field(canonical, "reason", command.reason());
+ field(canonical, "profileId", request.profileId().value());
+ field(canonical, "desiredState", request.desiredState().name());
+ field(canonical, "reason", request.reason());
return digest(canonical);
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java
index 88f9b373..0c526edc 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/AdminOperationStorePort.java
@@ -19,15 +19,17 @@ public interface AdminOperationStorePort {
* was checked but never held, so it prevented a repeat and not a race. A destructive operation
* run twice concurrently is the exact failure the id exists to stop.
*
- *
The fingerprint is part of the claim: the same id with a different command is not a replay,
- * it is a conflict, and answering it with the earlier result would silently execute neither.
+ *
The fingerprint is part of the claim: the same id with a different request is not a replay,
+ * it is a conflict, and answering it with the earlier result would silently execute neither. It
+ * digests the operator's request and leaves the actor out; the store keeps it in the column V8
+ * named {@code command_fingerprint}.
*
* @param operationId the caller's idempotency key
- * @param commandFingerprint a digest of the command this id was presented with
+ * @param requestFingerprint a digest of the request this id was presented with
* @param actor who is asking
* @param action what they are asking for
* @return whether this caller may proceed, and what the earlier one did if not
*/
AdminOperationClaim claim(
- String operationId, String commandFingerprint, AdminActor actor, String action);
+ String operationId, String requestFingerprint, AdminActor actor, String action);
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java
new file mode 100644
index 00000000..d6626f7f
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateCommand.java
@@ -0,0 +1,22 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.command.Command;
+import java.util.Objects;
+
+/**
+ * An operator's change to one provider profile's runtime state.
+ *
+ *
The request and the operator are apart because the operation id's fingerprint covers the
+ * request only: the same operator retrying the same request replays it.
+ *
+ * @param request what to do
+ * @param actor who asks, with the operator authorities and tenant scope the transport resolved
+ */
+public record ChangeProviderStateCommand(ProviderStateRequest request, AdminActor actor)
+ implements Command {
+
+ public ChangeProviderStateCommand {
+ Objects.requireNonNull(request, "request");
+ Objects.requireNonNull(actor, "actor");
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java
new file mode 100644
index 00000000..4dc51de9
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeProviderStateUseCase.java
@@ -0,0 +1,130 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.notification.platform.api.error.FailureCategory;
+import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode;
+import dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor;
+import dev.caskeleton.application.notification.platform.api.error.ProviderConfigurationException;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
+import dev.caskeleton.application.security.RequiresPermission;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.time.Clock;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Enables, disables or drains a provider profile as an operator.
+ *
+ *
A provider profile is shared by every tenant, so only an operator with no tenant scope may
+ * change it. The operation id is claimed before the runtime is asked, in the same transaction.
+ */
+@RequiresPermission(NotificationAdminPermissions.PROVIDER_CONTROL)
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
+ // The provider runtime and the audit port are the notification adapter's.
+ externalOutboundAllowed = true,
+ crossTenantAdmin = true)
+// Not final: @RequiresPermission is applied through a CGLIB proxy.
+public class ChangeProviderStateUseCase
+ implements CommandUseCase {
+
+ private static final String ACTION = "ADMIN_PROVIDER_STATE";
+
+ private final AdminAuthorizationGuard authorization;
+ private final ProviderRuntimeControlPort runtimes;
+ private final AdminOperationStorePort operations;
+ private final AdminOperationExecutor operationExecutor;
+ private final AdminOperationRequestFingerprint requestFingerprints =
+ new AdminOperationRequestFingerprint();
+ private final NotificationAuditPort audit;
+ private final TransactionPort transactions;
+ private final Clock clock;
+
+ public ChangeProviderStateUseCase(
+ AdminAuthorizationGuard authorization,
+ ProviderRuntimeControlPort runtimes,
+ AdminOperationStorePort operations,
+ NotificationAuditPort audit,
+ TransactionPort transactions,
+ Clock clock) {
+ this.authorization = Objects.requireNonNull(authorization, "authorization");
+ this.runtimes = Objects.requireNonNull(runtimes, "runtimes");
+ this.operations = Objects.requireNonNull(operations, "operations");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions);
+ this.audit = Objects.requireNonNull(audit, "audit");
+ this.clock = Objects.requireNonNull(clock, "clock");
+ }
+
+ @Override
+ public AdminOperationResult handle(ChangeProviderStateCommand change) {
+ Objects.requireNonNull(change, "change");
+ ProviderStateRequest request = change.request();
+ AdminActor actor = change.actor();
+ authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL);
+ authorization.requireGlobal(actor, NotificationAdminAuthority.PROVIDER_CONTROL);
+
+ if (request.dryRun()) {
+ return new AdminOperationResult(
+ request.operationId(),
+ true,
+ 1,
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ List.of("DRY_RUN"));
+ }
+
+ return transactions.inWrite(
+ () ->
+ operationExecutor.runClaimed(
+ request.operationId(),
+ requestFingerprints.of(request),
+ actor,
+ ACTION,
+ () -> change(request, actor)));
+ }
+
+ private AdminOperationResult change(ProviderStateRequest request, AdminActor actor) {
+ boolean applied =
+ runtimes.setState(request.profileId(), request.desiredState(), request.reason());
+ if (!applied) {
+ // The transition was refused — an authentication failure is not cleared by declaring the
+ // provider healthy, and a healthy runtime is not degraded on request. Recording success for a
+ // state change that did not happen tells an operator the provider is serving when it is not.
+ throw new ProviderConfigurationException(
+ NotificationFailureDescriptor.preDispatch(
+ NotificationFailureCode.PROVIDER_CONFIGURATION_INVALID,
+ FailureCategory.AUTHORIZATION));
+ }
+ AdminOperationResult result =
+ new AdminOperationResult(
+ request.operationId(),
+ false,
+ 1,
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ List.of(request.reason()));
+ audit.record(
+ new NotificationAuditEvent(
+ ACTION,
+ actor.actorRef(),
+ Optional.of(request.reason()),
+ Optional.of(request.operationId()),
+ clock.instant(),
+ Map.of(
+ "providerProfile", request.profileId().value(),
+ "status", request.desiredState().name())));
+ return operations.save(result, actor, ACTION);
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java
new file mode 100644
index 00000000..8a493bac
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionCommand.java
@@ -0,0 +1,22 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.command.Command;
+import java.util.Objects;
+
+/**
+ * An operator's change to one suppression entry.
+ *
+ * The request and the operator are apart because the operation id's fingerprint covers the
+ * request only: the same operator retrying the same request replays it.
+ *
+ * @param request what to do
+ * @param actor who asks, with the operator authorities and tenant scope the transport resolved
+ */
+public record ChangeSuppressionCommand(SuppressionRequest request, AdminActor actor)
+ implements Command {
+
+ public ChangeSuppressionCommand {
+ Objects.requireNonNull(request, "request");
+ Objects.requireNonNull(actor, "actor");
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java
new file mode 100644
index 00000000..4351472c
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ChangeSuppressionUseCase.java
@@ -0,0 +1,149 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
+import dev.caskeleton.application.notification.platform.policy.SuppressionEntry;
+import dev.caskeleton.application.notification.platform.policy.SuppressionId;
+import dev.caskeleton.application.notification.platform.policy.SuppressionSource;
+import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort;
+import dev.caskeleton.application.security.RequiresPermission;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.time.Clock;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Adds or lifts a suppression entry as an operator.
+ *
+ *
Lifting is by fingerprint match rather than by id, because an operator lifting a suppression
+ * knows the target, not the row identifier the platform assigned.
+ */
+@RequiresPermission(NotificationAdminPermissions.SUPPRESS)
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
+ // The audit port and the id generator are the notification adapter's.
+ externalOutboundAllowed = true,
+ crossTenantAdmin = true)
+// Not final: @RequiresPermission is applied through a CGLIB proxy.
+public class ChangeSuppressionUseCase
+ implements CommandUseCase {
+
+ private final AdminAuthorizationGuard authorization;
+ private final SuppressionStorePort suppressions;
+ private final AdminOperationStorePort operations;
+ private final AdminOperationExecutor operationExecutor;
+ private final AdminOperationRequestFingerprint requestFingerprints =
+ new AdminOperationRequestFingerprint();
+ private final NotificationAuditPort audit;
+ private final TransactionPort transactions;
+ private final NotificationIdGeneratorPort ids;
+ private final Clock clock;
+
+ public ChangeSuppressionUseCase(
+ AdminAuthorizationGuard authorization,
+ SuppressionStorePort suppressions,
+ AdminOperationStorePort operations,
+ NotificationAuditPort audit,
+ TransactionPort transactions,
+ NotificationIdGeneratorPort ids,
+ Clock clock) {
+ this.authorization = Objects.requireNonNull(authorization, "authorization");
+ this.suppressions = Objects.requireNonNull(suppressions, "suppressions");
+ this.operations = Objects.requireNonNull(operations, "operations");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions);
+ this.audit = Objects.requireNonNull(audit, "audit");
+ this.ids = Objects.requireNonNull(ids, "ids");
+ this.clock = Objects.requireNonNull(clock, "clock");
+ }
+
+ @Override
+ public AdminOperationResult handle(ChangeSuppressionCommand change) {
+ Objects.requireNonNull(change, "change");
+ SuppressionRequest request = change.request();
+ AdminActor actor = change.actor();
+ authorization.require(actor, NotificationAdminAuthority.SUPPRESS);
+ authorization.requireTenant(actor, request.tenantId(), NotificationAdminAuthority.SUPPRESS);
+
+ if (request.dryRun()) {
+ return new AdminOperationResult(
+ request.operationId(),
+ true,
+ 1,
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ List.of("DRY_RUN"));
+ }
+
+ String action = request.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD";
+ return transactions.inWrite(
+ () ->
+ operationExecutor.runClaimed(
+ request.operationId(),
+ requestFingerprints.of(request),
+ actor,
+ action,
+ () -> change(request, actor, action)));
+ }
+
+ private AdminOperationResult change(SuppressionRequest request, AdminActor actor, String action) {
+ int affected = request.remove() ? lift(request) : add(request);
+ AdminOperationResult result =
+ new AdminOperationResult(
+ request.operationId(),
+ false,
+ affected,
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ List.of(request.reasonText()));
+ audit.record(
+ new NotificationAuditEvent(
+ request.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED",
+ actor.actorRef(),
+ Optional.of(request.reason().name()),
+ Optional.of(request.operationId()),
+ clock.instant(),
+ Map.of()));
+ return operations.save(result, actor, action);
+ }
+
+ private int lift(SuppressionRequest request) {
+ long lifted =
+ suppressions
+ .activeFor(request.tenantId(), request.targetFingerprint(), clock.instant())
+ .stream()
+ .map(entry -> suppressions.remove(request.tenantId(), entry.id()))
+ .filter(Optional::isPresent)
+ .count();
+ return lifted > 0 ? 1 : 0;
+ }
+
+ private int add(SuppressionRequest request) {
+ suppressions.upsert(
+ new SuppressionEntry(
+ // ids.nextId(), not UUID.randomUUID(): a random v4 id is unordered, so the suppression
+ // table's index degrades the way the platform's UUIDv7 policy exists to prevent.
+ new SuppressionId(ids.nextId()),
+ request.tenantId(),
+ request.scope(),
+ request.reason(),
+ request.targetFingerprint(),
+ Optional.empty(),
+ clock.instant(),
+ request.expiresAt(),
+ SuppressionSource.ADMIN));
+ return 1;
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java
deleted file mode 100644
index 684bad6e..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationService.java
+++ /dev/null
@@ -1,362 +0,0 @@
-package dev.caskeleton.application.notification.platform.admin;
-
-import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
-import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
-import dev.caskeleton.application.notification.platform.api.error.FailureCategory;
-import dev.caskeleton.application.notification.platform.api.error.NotificationFailureCode;
-import dev.caskeleton.application.notification.platform.api.error.NotificationFailureDescriptor;
-import dev.caskeleton.application.notification.platform.api.error.ProviderConfigurationException;
-import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot;
-import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort;
-import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort;
-import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService;
-import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
-import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
-import dev.caskeleton.application.notification.platform.policy.SuppressionEntry;
-import dev.caskeleton.application.notification.platform.policy.SuppressionId;
-import dev.caskeleton.application.notification.platform.policy.SuppressionSource;
-import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort;
-import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
-import dev.caskeleton.application.transaction.TransactionPort;
-import java.time.Clock;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-
-/**
- * N4 operator plane.
- *
- * It lived in the outbound notification adapter, where it performed operator authorization,
- * tenant scoping, duplicate-risk approval, idempotency and transaction orchestration — every one of
- * them a decision about what the product allows, in the layer whose job is speaking provider
- * protocols. What remains out there is {@link ProviderRuntimeControlPort}: two operations that turn
- * an application request into a runtime transition.
- *
- *
Four properties hold for every operation: a separate authority, an idempotent operation id, a
- * recorded reason, and an audit row. The idempotency matters more than it looks — an operator
- * retrying a redrive after a timeout must not send the message twice, which is exactly the failure
- * the operation is trying to repair.
- *
- *
A dry run reads and reports but writes nothing, so an operator can see the blast radius of a
- * bulk action before committing to it.
- */
-public final class NotificationAdminApplicationService implements NotificationAdminService {
-
- private final AdminAuthorizationGuard authorization;
- private final DuplicateRiskGuard duplicateRiskGuard;
- private final DeliveryAttemptStorePort attempts;
- private final RecipientDeliveryStorePort recipients;
- private final ReconciliationService reconciliation;
- private final SuppressionStorePort suppressions;
- private final ProviderRuntimeControlPort runtimes;
- private final AdminOperationStorePort operations;
- private final AdminOperationExecutor operationExecutor;
- private final AdminOperationCommandFingerprint commandFingerprints;
- private final TransactionPort transactions;
- private final NotificationAuditPort audit;
- private final dev.caskeleton.application.notification.platform.dispatch
- .NotificationIdGeneratorPort
- ids;
- private final Clock clock;
-
- public NotificationAdminApplicationService(
- AdminAuthorizationGuard authorization,
- DuplicateRiskGuard duplicateRiskGuard,
- DeliveryAttemptStorePort attempts,
- RecipientDeliveryStorePort recipients,
- ReconciliationService reconciliation,
- SuppressionStorePort suppressions,
- ProviderRuntimeControlPort runtimes,
- AdminOperationStorePort operations,
- NotificationAuditPort audit,
- TransactionPort transactions,
- dev.caskeleton.application.notification.platform.dispatch.NotificationIdGeneratorPort ids,
- Clock clock) {
- this.authorization = Objects.requireNonNull(authorization, "authorization");
- this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard");
- this.attempts = Objects.requireNonNull(attempts, "attempts");
- this.recipients = Objects.requireNonNull(recipients, "recipients");
- this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation");
- this.suppressions = Objects.requireNonNull(suppressions, "suppressions");
- this.runtimes = Objects.requireNonNull(runtimes, "runtimes");
- this.operations = Objects.requireNonNull(operations, "operations");
- this.transactions = Objects.requireNonNull(transactions, "transactions");
- this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions);
- this.commandFingerprints = new AdminOperationCommandFingerprint();
- this.audit = Objects.requireNonNull(audit, "audit");
- this.ids = Objects.requireNonNull(ids, "ids");
- this.clock = Objects.requireNonNull(clock, "clock");
- }
-
- @Override
- public AdminOperationResult redrive(RedriveCommand command, AdminActor actor) {
- Objects.requireNonNull(command, "command");
- authorization.require(actor, NotificationAdminAuthority.REDRIVE);
-
- DeliveryAttemptSnapshot original =
- attempts
- .snapshot(command.attemptId())
- .orElseThrow(() -> new IllegalStateException("delivery attempt is not available"));
- authorization.requireTenant(actor, original.tenantId(), NotificationAdminAuthority.REDRIVE);
- duplicateRiskGuard.verify(original, command.approveDuplicateRisk());
-
- if (command.dryRun()) {
- return new AdminOperationResult(
- command.operationId(),
- true,
- 1,
- Optional.of(original.notificationId()),
- Optional.of(original.recipientDeliveryId()),
- Optional.empty(),
- List.of("DRY_RUN"));
- }
-
- return operationExecutor.execute(
- command.operationId(),
- commandFingerprints.of(command),
- actor,
- "ADMIN_REDRIVE",
- () -> {
- // The logical identities are preserved and only the attempt is new, so the history stays
- // one story rather than becoming two unrelated notifications.
- recipients.transition(
- original.recipientDeliveryId(),
- RecipientDeliveryState.READY_TO_DISPATCH,
- Optional.of(clock.instant()));
-
- AdminOperationResult result =
- new AdminOperationResult(
- command.operationId(),
- false,
- 1,
- Optional.of(original.notificationId()),
- Optional.of(original.recipientDeliveryId()),
- Optional.empty(),
- List.of(command.reason()));
- audit.record(
- new NotificationAuditEvent(
- "ADMIN_REDRIVE",
- actor.actorRef(),
- Optional.of(command.reason()),
- Optional.of(command.operationId()),
- clock.instant(),
- Map.of(
- "provider", original.providerId().value(),
- "channel", original.channel().name())));
- return operations.save(result, actor, "ADMIN_REDRIVE");
- });
- }
-
- @Override
- public AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor) {
- Objects.requireNonNull(command, "command");
- authorization.require(actor, NotificationAdminAuthority.RECONCILE);
- // Every attempt is checked before any is touched, so a batch with one foreign id changes none.
- // An id with no attempt behind it is left to reconciliation, which does nothing with it.
- for (DeliveryAttemptId attemptId : command.attemptIds()) {
- attempts
- .snapshot(attemptId)
- .ifPresent(
- attempt ->
- authorization.requireTenant(
- actor, attempt.tenantId(), NotificationAdminAuthority.RECONCILE));
- }
-
- if (command.dryRun()) {
- return new AdminOperationResult(
- command.operationId(),
- true,
- command.attemptIds().size(),
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- List.of("DRY_RUN"));
- }
-
- // Outside a transaction: each reconcile asks a provider over the network and opens its own
- // boundaries for what it writes.
- return operationExecutor.executeOutsideTransaction(
- command.operationId(),
- commandFingerprints.of(command),
- actor,
- "ADMIN_RECONCILE",
- () -> {
- List reasons = new ArrayList<>();
- int reconciled = 0;
- for (DeliveryAttemptId attemptId : command.attemptIds()) {
- reconciliation.reconcile(attemptId);
- reconciled++;
- }
- reasons.add(command.reason());
-
- AdminOperationResult result =
- new AdminOperationResult(
- command.operationId(),
- false,
- reconciled,
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- List.copyOf(reasons));
- return transactions.inWrite(
- () -> {
- audit.record(
- new NotificationAuditEvent(
- "ADMIN_RECONCILE",
- actor.actorRef(),
- Optional.of(command.reason()),
- Optional.of(command.operationId()),
- clock.instant(),
- Map.of()));
- return operations.save(result, actor, "ADMIN_RECONCILE");
- });
- });
- }
-
- @Override
- public AdminOperationResult suppress(SuppressCommand command, AdminActor actor) {
- Objects.requireNonNull(command, "command");
- authorization.require(actor, NotificationAdminAuthority.SUPPRESS);
- authorization.requireTenant(actor, command.tenantId(), NotificationAdminAuthority.SUPPRESS);
-
- if (command.dryRun()) {
- return new AdminOperationResult(
- command.operationId(),
- true,
- 1,
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- List.of("DRY_RUN"));
- }
-
- String action = command.remove() ? "ADMIN_SUPPRESS_REMOVE" : "ADMIN_SUPPRESS_ADD";
- return operationExecutor.execute(
- command.operationId(),
- commandFingerprints.of(command),
- actor,
- action,
- () -> {
- int affected;
- if (command.remove()) {
- // Removal is by fingerprint match rather than by id, because an operator lifting a
- // suppression knows the target, not the row identifier the platform assigned.
- affected =
- suppressions
- .activeFor(
- command.tenantId(), command.targetFingerprint(), clock.instant())
- .stream()
- .map(entry -> suppressions.remove(command.tenantId(), entry.id()))
- .filter(Optional::isPresent)
- .count()
- > 0
- ? 1
- : 0;
- } else {
- suppressions.upsert(
- new SuppressionEntry(
- // ids.nextId(), not UUID.randomUUID(). The identifier contract requires the
- // generator port, and while this class lived in the outbound adapter the rule
- // that says so did not apply to it — a random v4 id here is unordered, so the
- // suppression table's index degrades the way the platform's own UUIDv7 policy
- // exists to prevent.
- new SuppressionId(ids.nextId()),
- command.tenantId(),
- command.scope(),
- command.reason(),
- command.targetFingerprint(),
- Optional.empty(),
- clock.instant(),
- command.expiresAt(),
- SuppressionSource.ADMIN));
- affected = 1;
- }
-
- AdminOperationResult result =
- new AdminOperationResult(
- command.operationId(),
- false,
- affected,
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- List.of(command.reasonText()));
- audit.record(
- new NotificationAuditEvent(
- command.remove() ? "ADMIN_SUPPRESSION_REMOVED" : "ADMIN_SUPPRESSION_ADDED",
- actor.actorRef(),
- Optional.of(command.reason().name()),
- Optional.of(command.operationId()),
- clock.instant(),
- Map.of()));
- return operations.save(result, actor, action);
- });
- }
-
- @Override
- public AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor) {
- Objects.requireNonNull(command, "command");
- authorization.require(actor, NotificationAdminAuthority.PROVIDER_CONTROL);
- authorization.requireGlobal(actor, NotificationAdminAuthority.PROVIDER_CONTROL);
-
- if (command.dryRun()) {
- return new AdminOperationResult(
- command.operationId(),
- true,
- 1,
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- List.of("DRY_RUN"));
- }
-
- return operationExecutor.execute(
- command.operationId(),
- commandFingerprints.of(command),
- actor,
- "ADMIN_PROVIDER_STATE",
- () -> {
- boolean applied =
- runtimes.setState(command.profileId(), command.desiredState(), command.reason());
- if (!applied) {
- // The transition was refused — an authentication failure is not cleared by declaring
- // the provider healthy, and a healthy runtime is not degraded on request. Recording
- // success for a state change that did not happen tells an operator the provider is
- // serving when it is not.
- throw new ProviderConfigurationException(
- NotificationFailureDescriptor.preDispatch(
- NotificationFailureCode.PROVIDER_CONFIGURATION_INVALID,
- FailureCategory.AUTHORIZATION));
- }
-
- AdminOperationResult result =
- new AdminOperationResult(
- command.operationId(),
- false,
- 1,
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- List.of(command.reason()));
- audit.record(
- new NotificationAuditEvent(
- "ADMIN_PROVIDER_STATE",
- actor.actorRef(),
- Optional.of(command.reason()),
- Optional.of(command.operationId()),
- clock.instant(),
- Map.of(
- "providerProfile", command.profileId().value(),
- "status", command.desiredState().name())));
- return operations.save(result, actor, "ADMIN_PROVIDER_STATE");
- });
- }
-
- /** Current state of a provider runtime, for the health endpoint. */
- public ProviderRuntimeState providerState(
- dev.caskeleton.application.notification.platform.api.ProviderProfileId profileId) {
- return runtimes.state(profileId);
- }
-}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java
new file mode 100644
index 00000000..be4175d1
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminPermissions.java
@@ -0,0 +1,27 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+/**
+ * The permissions the four operator-plane use cases declare.
+ *
+ * One per operation, like the {@link NotificationAdminAuthority} the operator also has to hold:
+ * the permission is what the transport's caller may invoke at all, the authority and tenant scope
+ * on the {@code AdminActor} are what the operation then checks against the delivery it touches.
+ * Which roles hold them by default is the notification module defaults' to say, not this type's:
+ * {@code docs/notification/operations.md} 「운영자 연산」.
+ */
+public final class NotificationAdminPermissions {
+
+ /** Re-run one delivery. */
+ public static final String REDRIVE = "notification-admin:redrive";
+
+ /** Ask providers about a batch of attempts. */
+ public static final String RECONCILE = "notification-admin:reconcile";
+
+ /** Add or lift a suppression entry. */
+ public static final String SUPPRESS = "notification-admin:suppress";
+
+ /** Change a provider profile's runtime state. */
+ public static final String PROVIDER_CONTROL = "notification-admin:provider-control";
+
+ private NotificationAdminPermissions() {}
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java
deleted file mode 100644
index 5b003cc8..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminService.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package dev.caskeleton.application.notification.platform.admin;
-
-/**
- * N4 operator plane.
- *
- *
Every operation requires a separate authority, an idempotent operation id, a reason and an
- * audit record. It is deliberately not reachable from the application-facing APIs.
- */
-public interface NotificationAdminService {
-
- /** Re-run a delivery while preserving its logical identity. */
- AdminOperationResult redrive(RedriveCommand command, AdminActor actor);
-
- /** Force reconciliation of a bounded batch. */
- AdminOperationResult reconcile(ReconcileCommand command, AdminActor actor);
-
- /** Add or remove a suppression entry. */
- AdminOperationResult suppress(SuppressCommand command, AdminActor actor);
-
- /** Change a provider runtime state. */
- AdminOperationResult setProviderState(SetProviderStateCommand command, AdminActor actor);
-}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java
similarity index 91%
rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java
rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java
index c942f21f..033884a4 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SetProviderStateCommand.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ProviderStateRequest.java
@@ -5,14 +5,14 @@ import dev.caskeleton.application.notification.platform.provider.ProviderRuntime
import java.util.Objects;
/** Enable, disable or drain a provider profile. */
-public record SetProviderStateCommand(
+public record ProviderStateRequest(
String operationId,
ProviderProfileId profileId,
ProviderRuntimeState desiredState,
String reason,
boolean dryRun) {
- public SetProviderStateCommand {
+ public ProviderStateRequest {
Objects.requireNonNull(operationId, "operationId");
Objects.requireNonNull(profileId, "profileId");
Objects.requireNonNull(desiredState, "desiredState");
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java
new file mode 100644
index 00000000..8f23b5aa
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesCommand.java
@@ -0,0 +1,22 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.command.Command;
+import java.util.Objects;
+
+/**
+ * An operator's reconciliation of a bounded batch of attempts.
+ *
+ *
The request and the operator are apart because the operation id's fingerprint covers the
+ * request only: the same operator retrying the same request replays it.
+ *
+ * @param request what to do
+ * @param actor who asks, with the operator authorities and tenant scope the transport resolved
+ */
+public record ReconcileDeliveriesCommand(ReconcileRequest request, AdminActor actor)
+ implements Command {
+
+ public ReconcileDeliveriesCommand {
+ Objects.requireNonNull(request, "request");
+ Objects.requireNonNull(actor, "actor");
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java
new file mode 100644
index 00000000..aba76aca
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileDeliveriesUseCase.java
@@ -0,0 +1,143 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
+import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort;
+import dev.caskeleton.application.notification.platform.dispatch.ReconciliationService;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
+import dev.caskeleton.application.security.RequiresPermission;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.time.Clock;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Asks providers, on an operator's request, what happened to a bounded batch of attempts.
+ *
+ *
Every attempt is checked against the operator's tenant scope before any is touched, so a batch
+ * with one foreign id changes none. An id with no attempt behind it is left to reconciliation,
+ * which does nothing with it.
+ *
+ *
The provider calls run with no transaction open: each one asks over the network, and inside a
+ * write transaction it held a pooled connection for as long as the provider took, once per attempt.
+ * The operation id is claimed in its own transaction first, and the audit row and result are
+ * written in one at the end.
+ */
+@RequiresPermission(NotificationAdminPermissions.RECONCILE)
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
+ externalOutboundAllowed = true,
+ crossTenantAdmin = true)
+// Not final: @RequiresPermission is applied through a CGLIB proxy.
+public class ReconcileDeliveriesUseCase
+ implements CommandUseCase {
+
+ private static final String ACTION = "ADMIN_RECONCILE";
+
+ private final AdminAuthorizationGuard authorization;
+ private final DeliveryAttemptStorePort attempts;
+ private final ReconciliationService reconciliation;
+ private final AdminOperationStorePort operations;
+ private final AdminOperationExecutor operationExecutor;
+ private final AdminOperationRequestFingerprint requestFingerprints =
+ new AdminOperationRequestFingerprint();
+ private final NotificationAuditPort audit;
+ private final TransactionPort transactions;
+ private final Clock clock;
+
+ public ReconcileDeliveriesUseCase(
+ AdminAuthorizationGuard authorization,
+ DeliveryAttemptStorePort attempts,
+ ReconciliationService reconciliation,
+ AdminOperationStorePort operations,
+ NotificationAuditPort audit,
+ TransactionPort transactions,
+ Clock clock) {
+ this.authorization = Objects.requireNonNull(authorization, "authorization");
+ this.attempts = Objects.requireNonNull(attempts, "attempts");
+ this.reconciliation = Objects.requireNonNull(reconciliation, "reconciliation");
+ this.operations = Objects.requireNonNull(operations, "operations");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions);
+ this.audit = Objects.requireNonNull(audit, "audit");
+ this.clock = Objects.requireNonNull(clock, "clock");
+ }
+
+ @Override
+ public AdminOperationResult handle(ReconcileDeliveriesCommand batch) {
+ Objects.requireNonNull(batch, "batch");
+ ReconcileRequest request = batch.request();
+ AdminActor actor = batch.actor();
+ authorization.require(actor, NotificationAdminAuthority.RECONCILE);
+ // Read in a read transaction of its own: the providers are asked with none open, so the tenant
+ // check cannot share the boundary the answers are recorded in.
+ transactions.inRead(
+ () -> {
+ for (DeliveryAttemptId attemptId : request.attemptIds()) {
+ attempts
+ .snapshot(attemptId)
+ .ifPresent(
+ attempt ->
+ authorization.requireTenant(
+ actor, attempt.tenantId(), NotificationAdminAuthority.RECONCILE));
+ }
+ return null;
+ });
+
+ if (request.dryRun()) {
+ return new AdminOperationResult(
+ request.operationId(),
+ true,
+ request.attemptIds().size(),
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ List.of("DRY_RUN"));
+ }
+
+ return operationExecutor.executeOutsideTransaction(
+ request.operationId(),
+ requestFingerprints.of(request),
+ actor,
+ ACTION,
+ () -> reconcile(request, actor));
+ }
+
+ private AdminOperationResult reconcile(ReconcileRequest request, AdminActor actor) {
+ int reconciled = 0;
+ for (DeliveryAttemptId attemptId : request.attemptIds()) {
+ reconciliation.reconcile(attemptId);
+ reconciled++;
+ }
+ AdminOperationResult result =
+ new AdminOperationResult(
+ request.operationId(),
+ false,
+ reconciled,
+ Optional.empty(),
+ Optional.empty(),
+ Optional.empty(),
+ List.of(request.reason()));
+ return transactions.inWrite(
+ () -> {
+ audit.record(
+ new NotificationAuditEvent(
+ ACTION,
+ actor.actorRef(),
+ Optional.of(request.reason()),
+ Optional.of(request.operationId()),
+ clock.instant(),
+ Map.of()));
+ return operations.save(result, actor, ACTION);
+ });
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java
similarity index 92%
rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java
rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java
index b62d5b7d..491b8526 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileCommand.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/ReconcileRequest.java
@@ -5,12 +5,12 @@ import java.util.List;
import java.util.Objects;
/** Operator-triggered reconciliation over a bounded batch of attempts. */
-public record ReconcileCommand(
+public record ReconcileRequest(
String operationId, List attemptIds, String reason, boolean dryRun) {
public static final int MAX_BATCH = 200;
- public ReconcileCommand {
+ public ReconcileRequest {
Objects.requireNonNull(operationId, "operationId");
attemptIds = List.copyOf(Objects.requireNonNull(attemptIds, "attemptIds"));
Objects.requireNonNull(reason, "reason");
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java
new file mode 100644
index 00000000..ab395b61
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryCommand.java
@@ -0,0 +1,21 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.command.Command;
+import java.util.Objects;
+
+/**
+ * An operator's redrive of one delivery attempt.
+ *
+ * The request and the operator are apart because the operation id's fingerprint covers the
+ * request only: the same operator retrying the same request replays it.
+ *
+ * @param request what to do
+ * @param actor who asks, with the operator authorities and tenant scope the transport resolved
+ */
+public record RedriveDeliveryCommand(RedriveRequest request, AdminActor actor) implements Command {
+
+ public RedriveDeliveryCommand {
+ Objects.requireNonNull(request, "request");
+ Objects.requireNonNull(actor, "actor");
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java
new file mode 100644
index 00000000..3b96e42c
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveDeliveryUseCase.java
@@ -0,0 +1,149 @@
+package dev.caskeleton.application.notification.platform.admin;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
+import dev.caskeleton.application.notification.platform.callback.DeliveryAttemptSnapshot;
+import dev.caskeleton.application.notification.platform.dispatch.DeliveryAttemptStorePort;
+import dev.caskeleton.application.notification.platform.dispatch.RecipientDeliveryStorePort;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
+import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
+import dev.caskeleton.application.security.RequiresPermission;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.time.Clock;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Re-runs one delivery while preserving its logical identity.
+ *
+ *
The logical identities are kept and only the next attempt is new, so the history stays one
+ * story rather than becoming two unrelated notifications. An operator retrying after a timeout must
+ * not send the message twice — exactly the failure a redrive is trying to repair — so the operation
+ * id is claimed before the delivery is touched, in the same transaction.
+ *
+ *
A dry run reads and reports but writes nothing.
+ */
+@RequiresPermission(NotificationAdminPermissions.REDRIVE)
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
+ // The audit port is the notification adapter's.
+ externalOutboundAllowed = true,
+ crossTenantAdmin = true)
+// Not final: @RequiresPermission is applied through a CGLIB proxy.
+public class RedriveDeliveryUseCase
+ implements CommandUseCase {
+
+ private static final String ACTION = "ADMIN_REDRIVE";
+
+ private final AdminAuthorizationGuard authorization;
+ private final DuplicateRiskGuard duplicateRiskGuard;
+ private final DeliveryAttemptStorePort attempts;
+ private final RecipientDeliveryStorePort recipients;
+ private final AdminOperationStorePort operations;
+ private final AdminOperationExecutor operationExecutor;
+ private final AdminOperationRequestFingerprint requestFingerprints =
+ new AdminOperationRequestFingerprint();
+ private final NotificationAuditPort audit;
+ private final TransactionPort transactions;
+ private final Clock clock;
+
+ public RedriveDeliveryUseCase(
+ AdminAuthorizationGuard authorization,
+ DuplicateRiskGuard duplicateRiskGuard,
+ DeliveryAttemptStorePort attempts,
+ RecipientDeliveryStorePort recipients,
+ AdminOperationStorePort operations,
+ NotificationAuditPort audit,
+ TransactionPort transactions,
+ Clock clock) {
+ this.authorization = Objects.requireNonNull(authorization, "authorization");
+ this.duplicateRiskGuard = Objects.requireNonNull(duplicateRiskGuard, "duplicateRiskGuard");
+ this.attempts = Objects.requireNonNull(attempts, "attempts");
+ this.recipients = Objects.requireNonNull(recipients, "recipients");
+ this.operations = Objects.requireNonNull(operations, "operations");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ this.operationExecutor = new AdminOperationExecutor(this.operations, this.transactions);
+ this.audit = Objects.requireNonNull(audit, "audit");
+ this.clock = Objects.requireNonNull(clock, "clock");
+ }
+
+ @Override
+ public AdminOperationResult handle(RedriveDeliveryCommand delivery) {
+ Objects.requireNonNull(delivery, "delivery");
+ RedriveRequest request = delivery.request();
+ AdminActor actor = delivery.actor();
+ authorization.require(actor, NotificationAdminAuthority.REDRIVE);
+
+ if (request.dryRun()) {
+ DeliveryAttemptSnapshot original = transactions.inRead(() -> checkedOriginal(request, actor));
+ return new AdminOperationResult(
+ request.operationId(),
+ true,
+ 1,
+ Optional.of(original.notificationId()),
+ Optional.of(original.recipientDeliveryId()),
+ Optional.empty(),
+ List.of("DRY_RUN"));
+ }
+
+ // The checks read the attempt in the transaction that claims the operation and writes, so the
+ // attempt they approved is the one redriven.
+ return transactions.inWrite(
+ () -> {
+ DeliveryAttemptSnapshot original = checkedOriginal(request, actor);
+ return operationExecutor.runClaimed(
+ request.operationId(),
+ requestFingerprints.of(request),
+ actor,
+ ACTION,
+ () -> redrive(request, actor, original));
+ });
+ }
+
+ private DeliveryAttemptSnapshot checkedOriginal(RedriveRequest request, AdminActor actor) {
+ DeliveryAttemptSnapshot original =
+ attempts
+ .snapshot(request.attemptId())
+ .orElseThrow(() -> new IllegalStateException("delivery attempt is not available"));
+ authorization.requireTenant(actor, original.tenantId(), NotificationAdminAuthority.REDRIVE);
+ duplicateRiskGuard.verify(original, request.approveDuplicateRisk());
+ return original;
+ }
+
+ private AdminOperationResult redrive(
+ RedriveRequest request, AdminActor actor, DeliveryAttemptSnapshot original) {
+ recipients.transition(
+ original.recipientDeliveryId(),
+ RecipientDeliveryState.READY_TO_DISPATCH,
+ Optional.of(clock.instant()));
+
+ AdminOperationResult result =
+ new AdminOperationResult(
+ request.operationId(),
+ false,
+ 1,
+ Optional.of(original.notificationId()),
+ Optional.of(original.recipientDeliveryId()),
+ Optional.empty(),
+ List.of(request.reason()));
+ audit.record(
+ new NotificationAuditEvent(
+ ACTION,
+ actor.actorRef(),
+ Optional.of(request.reason()),
+ Optional.of(request.operationId()),
+ clock.instant(),
+ Map.of(
+ "provider", original.providerId().value(),
+ "channel", original.channel().name())));
+ return operations.save(result, actor, ACTION);
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java
similarity index 93%
rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java
rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java
index fb9f031c..fe973b96 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveCommand.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/RedriveRequest.java
@@ -9,14 +9,14 @@ import java.util.Objects;
* A redrive reuses the pinned template version and rendered digest: it re-executes the original
* notification rather than sending a new, possibly different message under an old identity.
*/
-public record RedriveCommand(
+public record RedriveRequest(
String operationId,
DeliveryAttemptId attemptId,
String reason,
boolean approveDuplicateRisk,
boolean dryRun) {
- public RedriveCommand {
+ public RedriveRequest {
Objects.requireNonNull(operationId, "operationId");
Objects.requireNonNull(attemptId, "attemptId");
Objects.requireNonNull(reason, "reason");
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java
similarity index 94%
rename from src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java
rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java
index 1c4a3764..891d9698 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressCommand.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/admin/SuppressionRequest.java
@@ -8,7 +8,7 @@ import java.util.Objects;
import java.util.Optional;
/** Add or remove a suppression entry as an operator. */
-public record SuppressCommand(
+public record SuppressionRequest(
String operationId,
TenantId tenantId,
SuppressionScope scope,
@@ -19,7 +19,7 @@ public record SuppressCommand(
String reasonText,
boolean dryRun) {
- public SuppressCommand {
+ public SuppressionRequest {
Objects.requireNonNull(operationId, "operationId");
Objects.requireNonNull(tenantId, "tenantId");
Objects.requireNonNull(scope, "scope");
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java
deleted file mode 100644
index 56924a78..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/ProviderEventId.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package dev.caskeleton.application.notification.platform.api;
-
-/**
- * Provider-assigned event identity. Absent for providers that do not emit one; the ledger then
- * falls back to a deterministic fingerprint.
- */
-public record ProviderEventId(String value) {
-
- private static final int MAX_LENGTH = 300;
-
- public ProviderEventId {
- if (value == null || value.isBlank()) {
- throw new IllegalArgumentException("value");
- }
- if (value.length() > MAX_LENGTH) {
- throw new IllegalArgumentException("value exceeds " + MAX_LENGTH + " characters");
- }
- }
-}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java
deleted file mode 100644
index ba926e17..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/api/error/NotificationExpiredException.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package dev.caskeleton.application.notification.platform.api.error;
-
-/** The delivery expiry passed before a new attempt could start. */
-public class NotificationExpiredException extends NotificationException {
-
- private static final long serialVersionUID = 1L;
-
- public NotificationExpiredException(NotificationFailureDescriptor descriptor) {
- super(descriptor);
- }
-}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java
index 7db02dfb..e24d819c 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ProviderEventProjectionService.java
@@ -3,6 +3,7 @@ package dev.caskeleton.application.notification.platform.callback;
import dev.caskeleton.application.notification.platform.observation.NotificationMetricName;
import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort;
import dev.caskeleton.application.transaction.TransactionPort;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -72,6 +73,26 @@ public final class ProviderEventProjectionService {
transactions.inWrite(() -> ledger.markFailed(event.id(), reasonCode));
}
+ /**
+ * Stored events that are not matched to an attempt yet, read in a read transaction of its own.
+ *
+ * @param limit the most events to return
+ * @return the oldest unmatched events
+ */
+ public List unmatched(int limit) {
+ return transactions.inRead(() -> ledger.unmatched(limit));
+ }
+
+ /**
+ * Stored events whose projection has not run or failed, read in a read transaction of its own.
+ *
+ * @param limit the most events to return
+ * @return the oldest events awaiting projection
+ */
+ public List pendingProjection(int limit) {
+ return transactions.inRead(() -> ledger.pendingProjection(limit));
+ }
+
private Optional projectInTransaction(ProviderEventRecord event) {
Optional attempt = resolve(event);
if (attempt.isEmpty()) {
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java
new file mode 100644
index 00000000..b9cb86c8
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsCommand.java
@@ -0,0 +1,6 @@
+package dev.caskeleton.application.notification.platform.callback;
+
+import dev.caskeleton.application.command.Command;
+
+/** One replay pass over the stored, unprojected provider events; its bound is the use case's. */
+public record ReplayProviderEventsCommand() implements Command {}
diff --git a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java
similarity index 50%
rename from src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java
rename to src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java
index 50292159..833e4d2a 100644
--- a/src/adapter/outbound/notification/src/main/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ProviderEventReplayWorker.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/callback/ReplayProviderEventsUseCase.java
@@ -1,8 +1,11 @@
-package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
+package dev.caskeleton.application.notification.platform.callback;
-import dev.caskeleton.application.notification.platform.callback.ProviderEventLedger;
-import dev.caskeleton.application.notification.platform.callback.ProviderEventProjectionService;
-import dev.caskeleton.application.notification.platform.callback.ProviderEventRecord;
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.usecase.CommandUseCase;
import java.util.List;
import java.util.Objects;
@@ -19,26 +22,41 @@ import java.util.Objects;
* ingestion path deliberately stores such an event as {@code PENDING} rather than dropping it. With
* nothing replaying it, "we will match it later" was true only in the comment: the delivery stayed
* unconfirmed forever and the callback that would have confirmed it sat in a table.
+ *
+ * A use case, not a worker of the notification adapter, which it used to be (round9 review
+ * 2026-11-12, RR9-05-06, RR9-05-08). It holds no transaction and touches no repository itself:
+ * {@link ProviderEventProjectionService} reads each list in a read transaction and projects, or
+ * marks failed, each event in a write transaction of its own, so one event's failure does not roll
+ * back the others.
+ *
+ *
Not a bean. The notification composition's background workers construct it and keep it.
*/
-public final class ProviderEventReplayWorker {
+@SystemInvoked(
+ reason =
+ "the notification background workers run it on their scheduler thread, which has no"
+ + " Authentication")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ // Every read and write is the projection service's, each in a boundary of that service's.
+ repositoryAccess = RepositoryAccess.NONE)
+public class ReplayProviderEventsUseCase
+ implements CommandUseCase {
- private final ProviderEventLedger ledger;
private final ProviderEventProjectionService projection;
private final int batchSize;
/**
- * Creates the replay worker.
+ * Creates the use case.
*
- * @param ledger the stored events
- * @param projection the projection the events feed
+ * @param projection reads the stored events and projects them
* @param batchSize how many events one pass handles
+ * @throws IllegalStateException if the batch size is not positive, which is the assembly's fault
*/
- public ProviderEventReplayWorker(
- ProviderEventLedger ledger, ProviderEventProjectionService projection, int batchSize) {
- this.ledger = Objects.requireNonNull(ledger, "ledger");
+ public ReplayProviderEventsUseCase(ProviderEventProjectionService projection, int batchSize) {
this.projection = Objects.requireNonNull(projection, "projection");
if (batchSize < 1) {
- throw new IllegalArgumentException("batchSize");
+ throw new IllegalStateException("batchSize must be positive");
}
this.batchSize = batchSize;
}
@@ -52,10 +70,12 @@ public final class ProviderEventReplayWorker {
*
* @return how many events projected successfully
*/
- public int replayOnce() {
+ @Override
+ public Integer handle(ReplayProviderEventsCommand command) {
+ Objects.requireNonNull(command, "command");
int projected = 0;
- projected += replay(ledger.unmatched(batchSize));
- projected += replay(ledger.pendingProjection(batchSize));
+ projected += replay(projection.unmatched(batchSize));
+ projected += replay(projection.pendingProjection(batchSize));
return projected;
}
@@ -69,8 +89,8 @@ public final class ProviderEventReplayWorker {
projected++;
}
} catch (RuntimeException failure) {
- // Through the projection service, which opens the transaction: this worker has none, and
- // marking the event is a modifying query that throws without one.
+ // Through the projection service, which opens the transaction: this use case holds none
+ // here, and marking the event is a modifying query that throws without one.
projection.markFailed(event, failure.getClass().getSimpleName());
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java
index d8ed77b3..3107e0ac 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/AcceptNotificationApplicationUseCase.java
@@ -137,6 +137,9 @@ public class AcceptNotificationApplicationUseCase implements AcceptNotificationU
command.templateVersion().orElse(NotificationTemplateVersion.FIRST_VERSION),
command.locale().orElse(defaultLocale));
CorrelationId correlationId = new CorrelationId(ids.nextId().toString());
+ // Not the HTTP Idempotency-Key the executor replays: a business key that makes the
+ // notification itself one per tenant and lives as long as it does, derived from the plan when
+ // the caller gave none (manual 05 「executor를 쓰지 않는 중복 판정 둘」).
IdempotencyKey idempotencyKey =
command
.idempotencyKey()
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java
index 9ca2b512..0204482e 100644
Binary files a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java and b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/CanonicalNotificationPlanEncoder.java differ
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java
new file mode 100644
index 00000000..8a7e0b00
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesCommand.java
@@ -0,0 +1,24 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.command.Command;
+import java.time.Duration;
+import java.util.Objects;
+
+/**
+ * One claim of due deliveries for a scheduler worker.
+ *
+ * @param workerId the claiming worker, including its process incarnation
+ * @param limit the most deliveries to claim; only as many as can start now
+ * @param leaseDuration how long the claimed deliveries are the worker's
+ */
+public record ClaimDueDeliveriesCommand(String workerId, int limit, Duration leaseDuration)
+ implements Command {
+
+ public ClaimDueDeliveriesCommand {
+ Objects.requireNonNull(workerId, "workerId");
+ Objects.requireNonNull(leaseDuration, "leaseDuration");
+ if (limit < 1) {
+ throw new IllegalStateException("limit must be positive");
+ }
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java
new file mode 100644
index 00000000..941fb393
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCase.java
@@ -0,0 +1,47 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Claims the deliveries that are due, each under a fresh lease and fence.
+ *
+ * The first of the scheduler's two steps; {@link DispatchClaimedDeliveryUseCase} is the second,
+ * once per claimed delivery. The worker that drives them owns only threads and concurrency permits:
+ * what is claimed, and under which lease, is decided here (round9 review 2026-11-12, RR9-05-06).
+ *
+ *
Not a bean. The notification composition's scheduler worker constructs it and keeps it.
+ */
+@SystemInvoked(
+ reason =
+ "the notification scheduler worker claims on its polling thread, which has no"
+ + " Authentication")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.NOT_IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.WRITE_REPOSITORY)
+public class ClaimDueDeliveriesUseCase
+ implements CommandUseCase> {
+
+ private final RecipientLeaseStorePort leases;
+ private final TransactionPort transactions;
+
+ public ClaimDueDeliveriesUseCase(RecipientLeaseStorePort leases, TransactionPort transactions) {
+ this.leases = Objects.requireNonNull(leases, "leases");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ }
+
+ @Override
+ public List handle(ClaimDueDeliveriesCommand command) {
+ Objects.requireNonNull(command, "command");
+ return transactions.inWrite(
+ () -> leases.claim(command.workerId(), command.limit(), command.leaseDuration()));
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java
new file mode 100644
index 00000000..ab060270
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryCommand.java
@@ -0,0 +1,16 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.command.Command;
+import java.util.Objects;
+
+/**
+ * One claimed delivery to send.
+ *
+ * @param lease the claim the delivery is held under
+ */
+public record DispatchClaimedDeliveryCommand(RecipientLease lease) implements Command {
+
+ public DispatchClaimedDeliveryCommand {
+ Objects.requireNonNull(lease, "lease");
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java
new file mode 100644
index 00000000..68e33fde
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCase.java
@@ -0,0 +1,46 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.util.Objects;
+
+/**
+ * Sends one claimed delivery, under its lease.
+ *
+ * {@code RepositoryAccess.NONE}: the delivery is {@link NotificationDispatchService}'s, which
+ * commits the attempt row before the provider call and records the outcome after it, each in a
+ * boundary of its own. One around the whole dispatch would hold a connection across the provider
+ * call.
+ *
+ *
Not a bean. The notification composition's scheduler worker constructs it and keeps it.
+ */
+@SystemInvoked(
+ reason =
+ "the notification scheduler worker dispatches on its executor threads, which have no"
+ + " Authentication")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.NOT_IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.NONE,
+ // The dispatch reaches the provider.
+ externalOutboundAllowed = true)
+public class DispatchClaimedDeliveryUseCase
+ implements CommandUseCase {
+
+ private final NotificationDispatchService dispatcher;
+
+ public DispatchClaimedDeliveryUseCase(NotificationDispatchService dispatcher) {
+ this.dispatcher = Objects.requireNonNull(dispatcher, "dispatcher");
+ }
+
+ @Override
+ public Void handle(DispatchClaimedDeliveryCommand command) {
+ Objects.requireNonNull(command, "command");
+ dispatcher.dispatch(command.lease());
+ return null;
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java
index bb5f12fd..901fd901 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryService.java
@@ -2,7 +2,6 @@ package dev.caskeleton.application.notification.platform.dispatch;
import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
-import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
import dev.caskeleton.application.transaction.TransactionPort;
import java.time.Duration;
import java.util.List;
@@ -27,6 +26,13 @@ import java.util.Objects;
* nothing to iterate. Those rows stayed {@code DISPATCHING} forever, holding a delivery nobody had
* even tried to send.
*
+ * A requeue is one write conditional on the abandoned lease recovery found (owner, fence, state,
+ * expiry), not a read followed by an unconditional transition. The pass runs on every instance, and
+ * between finding a job and writing it another instance's recovery may have requeued it and a new
+ * worker claimed it, or the holder may have finished; the unconditional write put such a job back
+ * to {@code READY_TO_DISPATCH} and it was dispatched again (round9 review 2026-11-12, RR9-01-12). A
+ * requeue that matches nothing is not counted as handled.
+ *
*
Every write opens its own transaction; the pass itself runs on a bare scheduler thread. The
* stores behind these ports are modifying JPA queries, which throw outside a transaction — so the
* pass used to fail on its first statement, every time, and the scheduler only logged it. It lives
@@ -110,14 +116,14 @@ public final class LeaseRecoveryService {
// Retiring expired deliveries rides along with recovery because both answer the same operator
// question — "why is this row still here?" — and neither needs a scheduler of its own.
int handled = transactions.inWrite(() -> leases.expireOverdue(batchSize));
- List abandoned = leases.expiredDispatching(batchSize, staleAfter);
- for (var recipientDeliveryId : abandoned) {
+ List abandoned = leases.expiredDispatching(batchSize, staleAfter);
+ for (var lease : abandoned) {
+ RecipientDeliveryId recipientDeliveryId = lease.recipientDeliveryId();
var attemptsOfDelivery = attempts.attemptsOf(recipientDeliveryId);
if (attemptsOfDelivery.isEmpty()) {
// Crash before the attempt row: no provider call can have happened, so this is the one
// safe requeue. Leaving it DISPATCHING stranded the delivery permanently.
- requeue(recipientDeliveryId);
- handled++;
+ handled += requeue(lease);
continue;
}
if (attemptsOfDelivery.stream().allMatch(attempt -> attempt.completedAt().isPresent())) {
@@ -127,8 +133,7 @@ public final class LeaseRecoveryService {
// the delivery, and dispatch derives the next step from exactly those facts — an accepted
// submission stops as COMPLETED, an ambiguous one goes to reconciliation — so handing it
// back to dispatch cannot send a second copy of something the provider accepted.
- requeue(recipientDeliveryId);
- handled++;
+ handled += requeue(lease);
continue;
}
for (var attempt : attemptsOfDelivery) {
@@ -141,25 +146,25 @@ public final class LeaseRecoveryService {
// identically, so every abandoned attempt had to be reconciled. The pre-call row the
// dispatcher writes never lands here: it records the start as unknown, because it is
// still the stored state while the provider call runs.
- requeue(recipientDeliveryId);
+ handled += requeue(lease);
} else {
// Outside any transaction: reconciliation asks the provider over the network and opens
// its own boundaries for what it writes.
reconciliation.reconcile(attempt.id());
+ handled++;
}
- handled++;
}
}
}
return handled;
}
- private void requeue(RecipientDeliveryId recipientDeliveryId) {
- transactions.inWrite(
- () ->
- deliveries.transition(
- recipientDeliveryId,
- RecipientDeliveryState.READY_TO_DISPATCH,
- java.util.Optional.of(clock.instant())));
+ /** Requeues the abandoned dispatch; 1 when it was still abandoned and moved, 0 otherwise. */
+ private int requeue(RecipientLease abandoned) {
+ return transactions
+ .inWrite(() -> deliveries.requeueAbandoned(abandoned, clock.instant()))
+ .isPresent()
+ ? 1
+ : 0;
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java
index 101c9f35..c82fa5aa 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientDeliveryStorePort.java
@@ -49,4 +49,28 @@ public interface RecipientDeliveryStorePort {
RecipientDeliveryState state,
Optional nextDispatchAt,
RecipientLease lease);
+
+ /**
+ * Returns a job a dead worker left dispatching to the queue, only while it is still the abandoned
+ * dispatch recovery found.
+ *
+ * One conditional write on the abandoned lease — same owner, same fence, still {@code
+ * DISPATCHING}, lease not extended past what recovery saw — rather than a read followed by an
+ * unconditional transition. Recovery runs on every instance, and between finding the job and
+ * writing it another instance may have requeued it and a new worker claimed it, or the original
+ * holder may have finished; an unconditional write put such a job back to {@code
+ * READY_TO_DISPATCH} and it was dispatched again.
+ *
+ *
The target state is the operation's meaning, as {@code EXPIRED} is {@code
+ * RecipientLeaseStorePort.expireOverdue}'s, so it is not an argument. What recovery decides is
+ * which operation applies: this one when no attempt may have reached the provider, the
+ * reconciliation path otherwise.
+ *
+ * @param abandoned the lease as {@link RecipientLeaseStorePort#expiredDispatching} returned it
+ * @param nextDispatchAt when the job becomes due again
+ * @return the requeued job, or empty when it is no longer that abandoned dispatch and nothing was
+ * written
+ */
+ Optional requeueAbandoned(
+ RecipientLease abandoned, Instant nextDispatchAt);
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java
index 0b9962a3..5660e4d3 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLease.java
@@ -16,6 +16,10 @@ import java.util.Objects;
* that no longer exists and updates nothing. It travels with the lease rather than being re-read,
* because re-reading is the race.
*
+ * A store builds it from a row, never from a request, so a value it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
* @param recipientDeliveryId the job this lease is for
* @param owner the holder, including the process incarnation that claimed it
* @param fence the claim generation; a lower fence has been superseded
@@ -29,11 +33,11 @@ public record RecipientLease(
Objects.requireNonNull(owner, "owner");
Objects.requireNonNull(leaseUntil, "leaseUntil");
if (owner.isBlank()) {
- throw new IllegalArgumentException("owner");
+ throw new IllegalStateException("a lease needs an owner");
}
if (fence < 1) {
// A zero fence is the value an unclaimed row holds; a lease can never legitimately carry it.
- throw new IllegalArgumentException("fence");
+ throw new IllegalStateException("fence must be >= 1, was " + fence);
}
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java
index 634fabd7..da251a5d 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecipientLeaseStorePort.java
@@ -1,6 +1,5 @@
package dev.caskeleton.application.notification.platform.dispatch;
-import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
@@ -25,6 +24,9 @@ public interface RecipientLeaseStorePort {
/**
* Extend a lease that is still being worked on.
*
+ *
The template's dispatcher never calls it: a dispatch runs inside the lease it claimed. It is
+ * here for a deployment whose own dispatch outlasts one lease.
+ *
* @return the extended lease, or empty when this holder no longer owns the job
*/
Optional renew(RecipientLease lease, Duration leaseDuration);
@@ -40,8 +42,24 @@ public interface RecipientLeaseStorePort {
/** Whether this lease is still the current one, for checking before an external side effect. */
boolean stillHeld(RecipientLease lease);
- /** Jobs left {@code DISPATCHING} by a worker that died, for reconciliation. */
- List expiredDispatching(int limit, Duration olderThan);
+ /**
+ * Jobs left {@code DISPATCHING} by a worker that died, for recovery.
+ *
+ * Each comes back as the lease it was abandoned under — owner, fence and expiry as they stood
+ * when it was found. Recovery decides what to do from attempts it reads afterwards, and the lease
+ * is what lets its write say "only if this is still that abandoned dispatch": another instance's
+ * recovery, a new claim or the holder itself may have moved the job in between.
+ *
+ *
Only jobs a claim leased: an owner and a fence of at least one. A {@code DISPATCHING} job
+ * without them is no lease anyone holds — one left before the fence existed, or one moved there
+ * without a claim — and is not returned; offered, it would be a lease nobody can build, and the
+ * pass would fail on it every time.
+ *
+ * @param limit how many to return
+ * @param olderThan how long a lease must have been expired before the job counts as abandoned
+ * @return the abandoned leases, oldest expiry first
+ */
+ List expiredDispatching(int limit, Duration olderThan);
/**
* Retire deliveries whose window closed before anyone claimed them.
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java
new file mode 100644
index 00000000..2308ac3f
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsCommand.java
@@ -0,0 +1,6 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.command.Command;
+
+/** One reconciliation pass over the due jobs; its bounds are the use case's. */
+public record ReconcileOutstandingAttemptsCommand() implements Command {}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java
new file mode 100644
index 00000000..4edf380d
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCase.java
@@ -0,0 +1,185 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
+import dev.caskeleton.application.notification.platform.provider.ReconciliationResult;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Asks the provider what happened to attempts whose outcome is unknown.
+ *
+ * Reconciliation only ever ran when a lease expired and recovery walked past an incomplete
+ * attempt. An ambiguous submission whose worker exited cleanly — the common case, because a worker
+ * that records AMBIGUOUS and then finishes its shift has not crashed — was never asked about again.
+ * The delivery sat {@code RECONCILIATION_REQUIRED} indefinitely, which reads as a queue that
+ * stopped rather than as an outcome nobody knows.
+ *
+ *
An unsupported provider is not retried in a loop. Without a status-query capability the answer
+ * will not change, so the job is completed and the attempt stays visibly ambiguous for an operator
+ * — a busy loop against a capability that does not exist is how a background worker burns a
+ * connection pool while achieving nothing.
+ *
+ *
Every instance runs this worker, so the claim is a lease: the jobs a pass takes are this
+ * instance's until {@code claimLease} has passed, and no transaction is held while the provider is
+ * asked. A pass that outlives its lease may find a job taken over by another instance; the store
+ * then ignores this pass's write for it, so the job is asked about twice but recorded once.
+ *
+ *
A use case, not a worker of the notification adapter, which it used to be: when to ask again,
+ * when to give up and what each answer means are the platform's decisions, and the adapter's job is
+ * speaking provider protocols. The store writes each run in a write transaction of their own; the
+ * provider is asked with none open (round9 review 2026-11-12, RR9-05-06, RR9-05-08).
+ *
+ *
Not a bean. The notification composition's background workers construct it and keep it.
+ */
+@SystemInvoked(
+ reason =
+ "the notification background workers run it on their scheduler thread, which has no"
+ + " Authentication; an operator's run is ReconcileDeliveriesUseCase")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.WRITE_REPOSITORY,
+ // Each job asks a provider what happened.
+ externalOutboundAllowed = true)
+public class ReconcileOutstandingAttemptsUseCase
+ implements CommandUseCase {
+
+ private final ReconciliationJobStorePort jobs;
+ private final Function reconciler;
+ private final TransactionPort transactions;
+ private final Clock clock;
+ private final Duration retryBackoff;
+ private final Duration claimLease;
+ private final int batchSize;
+ private final int maxAttempts;
+
+ /**
+ * Creates the use case.
+ *
+ * The timing is derived here rather than in the assembly, because when to ask again and when
+ * to give up are this use case's decisions. A failed or crashed question waits one delivery lease
+ * before it is asked again. The claim lease reuses the delivery lease rather than adding a
+ * setting: one pass asks up to {@code batchSize} providers in turn, so a pass slower than the
+ * lease lets another instance re-ask its remaining jobs, and the claim fence keeps those answers
+ * recorded once. A job may ask once plus the delivery's additional attempts, the same budget a
+ * delivery has for being sent.
+ *
+ * @param jobs the outstanding questions
+ * @param reconciler asks one attempt's provider; it opens its own boundaries for what it writes
+ * @param transactions opens the write transaction around each job-store write
+ * @param clock the clock
+ * @param deliveryLease the delivery lease: the retry delay and how long one pass owns its jobs
+ * @param batchSize how many jobs one pass handles
+ * @param maxAdditionalAttempts how many more times a job may ask after its first question
+ * @throws IllegalStateException if a bound is not positive, which is the assembly's fault
+ */
+ public ReconcileOutstandingAttemptsUseCase(
+ ReconciliationJobStorePort jobs,
+ Function reconciler,
+ TransactionPort transactions,
+ Clock clock,
+ Duration deliveryLease,
+ int batchSize,
+ int maxAdditionalAttempts) {
+ this.jobs = Objects.requireNonNull(jobs, "jobs");
+ this.reconciler = Objects.requireNonNull(reconciler, "reconciler");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ this.clock = Objects.requireNonNull(clock, "clock");
+ Objects.requireNonNull(deliveryLease, "deliveryLease");
+ if (deliveryLease.isNegative() || deliveryLease.isZero()) {
+ throw new IllegalStateException("deliveryLease must be positive");
+ }
+ if (batchSize < 1) {
+ throw new IllegalStateException("batchSize must be positive");
+ }
+ if (maxAdditionalAttempts < 0) {
+ throw new IllegalStateException("maxAdditionalAttempts must not be negative");
+ }
+ this.retryBackoff = deliveryLease;
+ this.claimLease = deliveryLease;
+ this.batchSize = batchSize;
+ this.maxAttempts = maxAdditionalAttempts + 1;
+ }
+
+ /**
+ * Runs one pass.
+ *
+ * @return how many jobs reached a terminal decision
+ */
+ @Override
+ public Integer handle(ReconcileOutstandingAttemptsCommand command) {
+ Objects.requireNonNull(command, "command");
+ List due =
+ transactions.inWrite(() -> jobs.claimDue(batchSize, clock.instant(), claimLease));
+ int settled = 0;
+ for (ReconciliationJob job : due) {
+ // One job's failure is not the pass's: a provider that is refusing connections would
+ // otherwise stop every other provider's jobs behind it.
+ try {
+ if (handle(job)) {
+ settled++;
+ }
+ } catch (RuntimeException failure) {
+ reschedule(job, failure.getClass().getSimpleName(), clock.instant().plus(retryBackoff));
+ }
+ }
+ return settled;
+ }
+
+ private boolean handle(ReconciliationJob job) {
+ if (job.attempts() >= maxAttempts) {
+ // Asked enough times. Completing the job leaves the attempt ambiguous and visible rather
+ // than asking forever; an outcome that has not arrived after this many tries is an
+ // operator's decision, not a scheduler's.
+ transactions.inWrite(
+ () -> {
+ jobs.reschedule(job, "MAX_ATTEMPTS", clock.instant().plus(retryBackoff));
+ jobs.complete(job);
+ });
+ return true;
+ }
+ ReconciliationResult result = reconciler.apply(job.attemptId());
+ return switch (result) {
+ case ReconciliationResult.Confirmed ignored -> {
+ complete(job);
+ yield true;
+ }
+ case ReconciliationResult.Unsupported ignored -> {
+ // The capability does not exist; asking again cannot change that.
+ complete(job);
+ yield true;
+ }
+ case ReconciliationResult.StillUnknown stillUnknown -> {
+ reschedule(job, "STILL_UNKNOWN", stillUnknown.nextCheckAt());
+ yield false;
+ }
+ case ReconciliationResult.Failed failed -> {
+ if (failed.retryable()) {
+ reschedule(job, failed.code(), clock.instant().plus(retryBackoff));
+ yield false;
+ }
+ complete(job);
+ yield true;
+ }
+ };
+ }
+
+ private void complete(ReconciliationJob job) {
+ transactions.inWrite(() -> jobs.complete(job));
+ }
+
+ private void reschedule(ReconciliationJob job, String result, Instant nextCheckAt) {
+ transactions.inWrite(() -> jobs.reschedule(job, result, nextCheckAt));
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java
index c8545b62..e0d3e401 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/ReconciliationJob.java
@@ -24,6 +24,10 @@ import java.util.UUID;
* the store's writes after the claim match on it, so an instance that held the job past its lease
* cannot reschedule or remove it once another instance has claimed it since.
*
+ * A store builds it from a row, never from a request, so a value it refuses is a server fault:
+ * {@link IllegalStateException}, not the {@code IllegalArgumentException} the web layer would
+ * answer with a 400 (manual 06).
+ *
* @param id the job identity
* @param attemptId the attempt whose outcome is unknown
* @param providerProfileId the profile that must answer, since capability is per profile
@@ -55,10 +59,10 @@ public record ReconciliationJob(
Objects.requireNonNull(createdAt, "createdAt");
Objects.requireNonNull(updatedAt, "updatedAt");
if (attempts < 0) {
- throw new IllegalArgumentException("attempts must not be negative");
+ throw new IllegalStateException("attempts must not be negative");
}
if (claimFence < 0) {
- throw new IllegalArgumentException("claimFence must not be negative");
+ throw new IllegalStateException("claimFence must not be negative");
}
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java
new file mode 100644
index 00000000..6ed19414
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesCommand.java
@@ -0,0 +1,6 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.command.Command;
+
+/** One lease-recovery pass; its bounds are the recovery service's. */
+public record RecoverAbandonedDeliveriesCommand() implements Command {}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java
new file mode 100644
index 00000000..5d51446a
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCase.java
@@ -0,0 +1,47 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.security.SystemInvoked;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.usecase.CommandUseCase;
+import java.util.Objects;
+
+/**
+ * Returns deliveries a dead worker left dispatching to the queue, or to reconciliation when an
+ * attempt may have reached the provider.
+ *
+ *
{@code RepositoryAccess.NONE}: the pass is {@link LeaseRecoveryService}'s, which opens a
+ * boundary per delivery it settles and none around the provider question.
+ *
+ *
Not a bean. The notification composition's background workers construct it and keep it.
+ */
+@SystemInvoked(
+ reason =
+ "the notification background workers run it on their scheduler thread, which has no"
+ + " Authentication")
+@UseCaseCapability(
+ transactionMode = TransactionMode.WRITE,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.NONE,
+ // Recovery may ask a provider what happened to an attempt.
+ externalOutboundAllowed = true)
+public class RecoverAbandonedDeliveriesUseCase
+ implements CommandUseCase {
+
+ private final LeaseRecoveryService recovery;
+
+ public RecoverAbandonedDeliveriesUseCase(LeaseRecoveryService recovery) {
+ this.recovery = Objects.requireNonNull(recovery, "recovery");
+ }
+
+ /**
+ * @return how many deliveries the pass settled
+ */
+ @Override
+ public Integer handle(RecoverAbandonedDeliveriesCommand command) {
+ Objects.requireNonNull(command, "command");
+ return recovery.recoverOnce();
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java
new file mode 100644
index 00000000..b46a378e
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogQuery.java
@@ -0,0 +1,6 @@
+package dev.caskeleton.application.notification.platform.observation;
+
+import dev.caskeleton.application.query.Query;
+
+/** Asks how many deliveries are due and not yet sent; it has no parameters. */
+public record MeasureDispatchBacklogQuery() implements Query {}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java
new file mode 100644
index 00000000..efce336c
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/MeasureDispatchBacklogUseCase.java
@@ -0,0 +1,52 @@
+package dev.caskeleton.application.notification.platform.observation;
+
+import dev.caskeleton.application.capability.Idempotency;
+import dev.caskeleton.application.capability.RepositoryAccess;
+import dev.caskeleton.application.capability.UseCaseCapability;
+import dev.caskeleton.application.transaction.TransactionMode;
+import dev.caskeleton.application.transaction.TransactionPort;
+import dev.caskeleton.application.usecase.QueryUseCase;
+import java.util.Objects;
+
+/**
+ * Measures the dispatch backlog the scheduler publishes as its queue-depth gauge on every poll.
+ *
+ * The depth alone, one statement in a read transaction of its own: the full serving snapshot is
+ * several statements and is the health endpoint's. The scheduler read the serving-state port
+ * directly, with no use case and no read boundary in between (round9 review 2026-11-12, application
+ * structure). The notification composition constructs it for the scheduler; nothing else needs it.
+ */
+@UseCaseCapability(
+ transactionMode = TransactionMode.READ_ONLY,
+ idempotency = Idempotency.IDEMPOTENT,
+ repositoryAccess = RepositoryAccess.READ_REPOSITORY)
+public class MeasureDispatchBacklogUseCase
+ implements QueryUseCase {
+
+ private final NotificationServingStatePort servingState;
+ private final TransactionPort transactions;
+
+ /**
+ * Creates the use case.
+ *
+ * @param servingState the serving-state read model
+ * @param transactions opens the read transaction around the measurement
+ */
+ public MeasureDispatchBacklogUseCase(
+ NotificationServingStatePort servingState, TransactionPort transactions) {
+ this.servingState = Objects.requireNonNull(servingState, "servingState");
+ this.transactions = Objects.requireNonNull(transactions, "transactions");
+ }
+
+ /**
+ * Measures the backlog.
+ *
+ * @param query the measurement request
+ * @return how many deliveries are due and not yet sent
+ */
+ @Override
+ public Long handle(MeasureDispatchBacklogQuery query) {
+ Objects.requireNonNull(query, "query");
+ return transactions.inRead(servingState::backlogDepth);
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java
index ce48cf07..7ddece3f 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingState.java
@@ -12,7 +12,7 @@ import java.util.Objects;
* being applied reported exactly the same numbers as an idle healthy one. Readiness that cannot
* distinguish those two states is not readiness.
*
- * @param backlogDepth deliveries due now and not yet claimed
+ * @param backlogDepth deliveries a claim would take now: due, not expired, not under a live lease
* @param oldestDueAge how long the oldest of them has been due
* @param stuckLeases leases whose holder has not finished and whose expiry has passed
* @param pendingProjections provider events stored and not yet applied
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java
index 79a817aa..f6ae4f15 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/observation/NotificationServingStatePort.java
@@ -6,7 +6,25 @@ public interface NotificationServingStatePort {
/**
* Measure the backlog, the stuck leases and the projection and reconciliation lag.
*
+ * Several questions, for the health snapshot. A caller that needs only the queue depth asks
+ * {@link #backlogDepth()} instead: the scheduler reads the depth on every poll, and asking all of
+ * them that often ran four statements a poll, one of them over the whole provider-event table
+ * (round9 review 2026-11-12, RR9-01-14).
+ *
* @return the current state
*/
NotificationServingState currentState();
+
+ /**
+ * How many deliveries a claim would take now — due, not expired, and not under a live lease: the
+ * {@link NotificationServingState#backlogDepth()} of {@link #currentState()}, measured alone.
+ *
+ *
The default takes it from the full snapshot, so an implementation that can only measure
+ * everything at once still answers; a store that can count the backlog by itself overrides it.
+ *
+ * @return the backlog depth
+ */
+ default long backlogDepth() {
+ return currentState().backlogDepth();
+ }
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java
index 9953dca2..56088cdf 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/DefaultNotificationRetryPolicy.java
@@ -19,14 +19,15 @@ public final class DefaultNotificationRetryPolicy implements NotificationRetryPo
private final JitterSource jitter;
private final Duration reconcileDelay;
- public DefaultNotificationRetryPolicy(
- RetryBackoff backoff, JitterSource jitter, Duration reconcileDelay) {
- this.backoff = Objects.requireNonNull(backoff, "backoff");
+ /**
+ * @param timing the backoff and the reconcile delay, already checked by {@link RetryTiming}
+ * @param jitter the randomness the backoff draws from
+ */
+ public DefaultNotificationRetryPolicy(RetryTiming timing, JitterSource jitter) {
+ Objects.requireNonNull(timing, "timing");
+ this.backoff = timing.backoff();
this.jitter = Objects.requireNonNull(jitter, "jitter");
- this.reconcileDelay = Objects.requireNonNull(reconcileDelay, "reconcileDelay");
- if (reconcileDelay.isNegative()) {
- throw new IllegalArgumentException("reconcileDelay");
- }
+ this.reconcileDelay = timing.reconcileDelay();
}
@Override
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java
index 2e478427..fb8aa0b3 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/EligibilityResult.java
@@ -9,7 +9,7 @@ public record EligibilityResult(boolean allowed, List reasonCodes) {
public EligibilityResult {
reasonCodes = List.copyOf(Objects.requireNonNull(reasonCodes, "reasonCodes"));
if (!allowed && reasonCodes.isEmpty()) {
- throw new IllegalArgumentException("a block must carry at least one reason code");
+ throw new IllegalStateException("a block must carry at least one reason code");
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java
index f6ba8ab3..ab282fbb 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBackoff.java
@@ -30,10 +30,11 @@ public record RetryBackoff(Duration base, Duration max) {
Objects.requireNonNull(base, "base");
Objects.requireNonNull(max, "max");
if (base.isNegative() || base.isZero()) {
- throw new IllegalArgumentException("base");
+ throw new IllegalStateException("the backoff base must be positive, was " + base);
}
if (max.compareTo(base) < 0) {
- throw new IllegalArgumentException("max must not be smaller than base");
+ throw new IllegalStateException(
+ "the backoff max (" + max + ") must not be shorter than its base (" + base + ")");
}
}
@@ -42,7 +43,7 @@ public record RetryBackoff(Duration base, Duration max) {
Objects.requireNonNull(retryAfter, "retryAfter");
Objects.requireNonNull(jitter, "jitter");
if (attemptNumber < 1) {
- throw new IllegalArgumentException("attemptNumber");
+ throw new IllegalStateException("attemptNumber must be >= 1, was " + attemptNumber);
}
Duration jittered = new Backoff(base, max).fullJitter(attemptNumber, asRandom(jitter));
Duration computed = jittered.compareTo(base) < 0 ? base : jittered;
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java
index 796b56da..88f519b6 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryBudget.java
@@ -10,7 +10,8 @@ public record RetryBudget(int maxAdditionalAttempts, int consumed) {
public RetryBudget {
if (maxAdditionalAttempts < 0 || consumed < 0) {
- throw new IllegalArgumentException("budget values must not be negative");
+ throw new IllegalStateException(
+ "budget values must not be negative, were " + maxAdditionalAttempts + " and " + consumed);
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java
index 500992bf..f2b87992 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryContext.java
@@ -47,7 +47,7 @@ public record RetryContext(
Objects.requireNonNull(estimatedDispatchDuration, "estimatedDispatchDuration");
Objects.requireNonNull(now, "now");
if (attemptNumber < 1) {
- throw new IllegalArgumentException("attemptNumber");
+ throw new IllegalStateException("attemptNumber must be >= 1, was " + attemptNumber);
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java
index f38b9657..4bd55a18 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryDecision.java
@@ -12,7 +12,7 @@ public sealed interface RetryDecision {
public RetryAfter {
Objects.requireNonNull(delay, "delay");
if (delay.isNegative()) {
- throw new IllegalArgumentException("delay");
+ throw new IllegalStateException("delay must not be negative, was " + delay);
}
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java
new file mode 100644
index 00000000..4f6769ac
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RetryTiming.java
@@ -0,0 +1,31 @@
+package dev.caskeleton.application.notification.platform.policy;
+
+import java.time.Duration;
+import java.util.Objects;
+
+/**
+ * When a failed delivery is tried again: the backoff between attempts, and how long an ambiguous
+ * submission waits before the provider is asked what happened.
+ *
+ * One value so its rules have one home. The retry settings build it from their keys and the
+ * policy reads it; neither restates what a valid timing is.
+ *
+ * @param backoff the delay between attempts
+ * @param reconcileDelay how long after an ambiguous submission reconciliation first asks; zero asks
+ * at once
+ */
+public record RetryTiming(RetryBackoff backoff, Duration reconcileDelay) {
+
+ /** The shipped timing: the default backoff, reconciliation after a minute. */
+ public static final RetryTiming DEFAULT =
+ new RetryTiming(RetryBackoff.DEFAULT, Duration.ofMinutes(1));
+
+ public RetryTiming {
+ Objects.requireNonNull(backoff, "backoff");
+ Objects.requireNonNull(reconcileDelay, "reconcileDelay");
+ if (reconcileDelay.isNegative()) {
+ throw new IllegalStateException(
+ "the reconcile delay must not be negative, was " + reconcileDelay);
+ }
+ }
+}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java
index a55a3ee6..8773a4c3 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RouteCandidate.java
@@ -19,7 +19,7 @@ public record RouteCandidate(
Objects.requireNonNull(contactPointId, "contactPointId");
Objects.requireNonNull(providerProfileId, "providerProfileId");
if (routeIndex < 0) {
- throw new IllegalArgumentException("routeIndex");
+ throw new IllegalStateException("routeIndex must be >= 0, was " + routeIndex);
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java
index 3185d341..7b5b98f1 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingContext.java
@@ -27,7 +27,7 @@ public record RoutingContext(
Objects.requireNonNull(expiresAt, "expiresAt");
Objects.requireNonNull(now, "now");
if (routeCursor < 0) {
- throw new IllegalArgumentException("routeCursor");
+ throw new IllegalStateException("routeCursor must be >= 0, was " + routeCursor);
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java
index 218f6cd4..bbee04ea 100644
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/RoutingDecision.java
@@ -16,10 +16,10 @@ public record RoutingDecision(
Objects.requireNonNull(selected, "selected");
Objects.requireNonNull(reasonCode, "reasonCode");
if (reasonCode.isBlank()) {
- throw new IllegalArgumentException("reasonCode");
+ throw new IllegalStateException("a routing decision needs a reason code");
}
if (reconciliationRequired && (retryAllowed || fallbackAllowed)) {
- throw new IllegalArgumentException(
+ throw new IllegalStateException(
"an attempt awaiting reconciliation may not also be retried or fallen back");
}
}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java
new file mode 100644
index 00000000..a21e000c
--- /dev/null
+++ b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/policy/package-info.java
@@ -0,0 +1,16 @@
+/**
+ * The notification platform's delivery policy: who may be sent to (eligibility: consent,
+ * preference, suppression), what counts as the same request twice (deduplication), and which route
+ * to try and when to retry, reroute or stop.
+ *
+ *
The values the platform, or a policy it calls, assembles ({@link RetryTiming}, {@link
+ * RetryBackoff}, {@link RetryBudget}, {@link RetryContext}, {@link RetryDecision.RetryAfter},
+ * {@link RouteCandidate}, {@link RoutingContext}, {@link RoutingDecision}, {@link
+ * EligibilityResult}) are built from settings, the shipped defaults, its own counters and stored
+ * delivery state: what they refuse never comes from a request. A refusal is a server fault, {@link
+ * IllegalStateException}, not the {@code IllegalArgumentException} the web layer would answer with
+ * a 400 (manual 06). The records that carry what a request may send ({@link ConsentRecord}, {@link
+ * PreferenceRecord}, {@link RecipientIdentity}, {@link SuppressionEntry}) refuse with {@code
+ * IllegalArgumentException}.
+ */
+package dev.caskeleton.application.notification.platform.policy;
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java
deleted file mode 100644
index 43fb9527..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/BatchNotificationProviderAdapter.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package dev.caskeleton.application.notification.platform.provider;
-
-import java.util.List;
-import java.util.concurrent.CompletionStage;
-
-/**
- * Optional batching extension.
- *
- *
One transport call may carry many submissions, but the result list is positional: every
- * recipient keeps its own attempt and its own outcome.
- */
-public interface BatchNotificationProviderAdapter extends NotificationProviderAdapter {
-
- /** Submit a batch and return one result per input, in input order. */
- CompletionStage> submitBatch(List submissions);
-}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java
deleted file mode 100644
index a5ba086c..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/CollapseCapability.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package dev.caskeleton.application.notification.platform.provider;
-
-import dev.caskeleton.application.notification.platform.api.CollapseSpec;
-
-/** Maps a platform collapse hint onto a provider-native key, validating provider limits first. */
-public interface CollapseCapability {
-
- /** Translate the spec, or fail when the key violates a provider limit. */
- ProviderCollapseMapping map(CollapseSpec spec);
-}
diff --git a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java b/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java
deleted file mode 100644
index 121c940a..00000000
--- a/src/application/notification/src/main/java/dev/caskeleton/application/notification/platform/provider/ProviderCollapseMapping.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package dev.caskeleton.application.notification.platform.provider;
-
-import java.util.Objects;
-
-/**
- * Provider-native collapse key.
- *
- * {@code guaranteesSingleUserNotification} is always false: collapse replaces an undelivered
- * message inside the provider, it cannot recall one the device already showed.
- */
-public record ProviderCollapseMapping(String nativeKey, String headerOrFieldName) {
-
- public ProviderCollapseMapping {
- Objects.requireNonNull(nativeKey, "nativeKey");
- Objects.requireNonNull(headerOrFieldName, "headerOrFieldName");
- if (nativeKey.isBlank() || headerOrFieldName.isBlank()) {
- throw new IllegalArgumentException("collapse mapping must not be blank");
- }
- }
-
- /** Collapse never promises the user sees exactly one notification. */
- public boolean guaranteesSingleUserNotification() {
- return false;
- }
-}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java
index 80b5960d..fec63e1a 100644
--- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationExecutorTest.java
@@ -22,7 +22,7 @@ class AdminOperationExecutorTest {
private final InMemoryAdminOperationStore store = new InMemoryAdminOperationStore();
@Test
- @DisplayName("처음 claim한 operation만 claim을 쥔 채 한 쓰기 트랜잭션 안에서 부수 효과를 실행한다")
+ @DisplayName("처음 claim한 operation만 claim을 쥔 채 부수 효과를 실행하고, 트랜잭션은 열지 않는다")
void claimedOperationIsTheOnlyOutcomeThatRunsTheSideEffect() {
// given
DirectTransactionPort transactions = new DirectTransactionPort();
@@ -31,7 +31,7 @@ class AdminOperationExecutorTest {
// when
AdminOperationResult result =
- executor.execute(
+ executor.runClaimed(
"op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE", sideEffect(sideEffects));
// then
@@ -41,12 +41,10 @@ class AdminOperationExecutorTest {
sideEffects.get(),
store.claim("op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE"),
transactions.committed()))
- .as("the operation saves its own result; until then the id answers in-progress")
- .containsExactly(
- "op-1",
- 1,
- AdminOperationClaim.inProgress(),
- List.of(DirectTransactionPort.Boundary.WRITE));
+ .as(
+ "the operation saves its own result; until then the id answers in-progress. The"
+ + " use case owns the write transaction the claim runs in")
+ .containsExactly("op-1", 1, AdminOperationClaim.inProgress(), List.of());
}
@Test
@@ -62,7 +60,7 @@ class AdminOperationExecutorTest {
// when
AdminOperationResult replayed =
- executor.execute(
+ executor.runClaimed(
"op-1", FINGERPRINT, ACTOR, "ADMIN_PROVIDER_STATE", sideEffect(sideEffects));
// then
@@ -82,7 +80,7 @@ class AdminOperationExecutorTest {
Throwable thrown =
catchThrowable(
() ->
- executor.execute(
+ executor.runClaimed(
"op-1", FINGERPRINT, ACTOR, "ADMIN_REDRIVE", sideEffect(sideEffects)));
// then
@@ -103,7 +101,7 @@ class AdminOperationExecutorTest {
Throwable thrown =
catchThrowable(
() ->
- executor.execute(
+ executor.runClaimed(
"op-1", FINGERPRINT, ACTOR, "ADMIN_REDRIVE", sideEffect(sideEffects)));
// then
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java
similarity index 65%
rename from src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java
rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java
index d5f48645..1d33fa71 100644
--- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationCommandFingerprintTest.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/AdminOperationRequestFingerprintTest.java
@@ -7,17 +7,17 @@ import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.Test;
-class AdminOperationCommandFingerprintTest {
+class AdminOperationRequestFingerprintTest {
- private final AdminOperationCommandFingerprint fingerprints =
- new AdminOperationCommandFingerprint();
+ private final AdminOperationRequestFingerprint fingerprints =
+ new AdminOperationRequestFingerprint();
@Test
- void fingerprintIsStableSha256AndChangesWithCommandMeaning() {
+ void fingerprintIsStableSha256AndChangesWithRequestMeaning() {
DeliveryAttemptId attempt = new DeliveryAttemptId(UUID.fromString("00000000-0000-0000-0000-1"));
- RedriveCommand first = new RedriveCommand("op-1", attempt, "retry", true, false);
- RedriveCommand same = new RedriveCommand("another-id", attempt, "retry", true, false);
- RedriveCommand changed = new RedriveCommand("op-1", attempt, "different", true, false);
+ RedriveRequest first = new RedriveRequest("op-1", attempt, "retry", true, false);
+ RedriveRequest same = new RedriveRequest("another-id", attempt, "retry", true, false);
+ RedriveRequest changed = new RedriveRequest("op-1", attempt, "different", true, false);
assertThat(fingerprints.of(first)).matches("[0-9a-f]{64}");
assertThat(fingerprints.of(same)).isEqualTo(fingerprints.of(first));
@@ -31,10 +31,10 @@ class AdminOperationCommandFingerprintTest {
DeliveryAttemptId second =
new DeliveryAttemptId(UUID.fromString("00000000-0000-0000-0000-000000000002"));
- ReconcileCommand ordered =
- new ReconcileCommand("op-1", List.of(first, second), "reconcile", false);
- ReconcileCommand reversed =
- new ReconcileCommand("op-2", List.of(second, first), "reconcile", false);
+ ReconcileRequest ordered =
+ new ReconcileRequest("op-1", List.of(first, second), "reconcile", false);
+ ReconcileRequest reversed =
+ new ReconcileRequest("op-2", List.of(second, first), "reconcile", false);
assertThat(fingerprints.of(ordered)).isEqualTo(fingerprints.of(reversed));
}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java
similarity index 56%
rename from src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java
rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java
index 2a7e002b..23681873 100644
--- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminApplicationServiceTest.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/admin/NotificationAdminUseCasesTest.java
@@ -27,9 +27,9 @@ import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemor
import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue;
import dev.caskeleton.application.notification.platform.observation.NotificationAuditEvent;
import dev.caskeleton.application.notification.platform.observation.NotificationAuditPort;
-import dev.caskeleton.application.notification.platform.policy.SuppressionEntry;
-import dev.caskeleton.application.notification.platform.policy.SuppressionId;
-import dev.caskeleton.application.notification.platform.policy.SuppressionStorePort;
+import dev.caskeleton.application.notification.platform.policy.SuppressionReason;
+import dev.caskeleton.application.notification.platform.policy.SuppressionScope;
+import dev.caskeleton.application.notification.platform.policy.testkit.InMemorySuppressionStore;
import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
import dev.caskeleton.application.notification.platform.provider.ReconciliationResult;
import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
@@ -42,10 +42,14 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
-import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
-class NotificationAdminApplicationServiceTest {
+/**
+ * The four operator-plane use cases, each its own entrypoint with its own permission (round9 review
+ * 2026-11-12, RR9-05-07: one service implemented all four behind an interface no use-case rule
+ * saw).
+ */
+class NotificationAdminUseCasesTest {
private static final Clock CLOCK =
Clock.fixed(Instant.parse("2026-09-20T00:00:00Z"), ZoneOffset.UTC);
@@ -56,15 +60,16 @@ class NotificationAdminApplicationServiceTest {
@Test
void providerStateMutationClaimsBeforeCallingTheRuntime() {
Fixture fixture = new Fixture();
- SetProviderStateCommand command =
- new SetProviderStateCommand(
+ ProviderStateRequest request =
+ new ProviderStateRequest(
"op-1",
new ProviderProfileId("ses-primary"),
ProviderRuntimeState.DISABLED,
"provider maintenance",
false);
- AdminOperationResult result = fixture.service.setProviderState(command, ACTOR);
+ AdminOperationResult result =
+ fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR));
assertThat(result.operationId()).isEqualTo("op-1");
assertThat(fixture.operations.claims).isEqualTo(1);
@@ -77,18 +82,40 @@ class NotificationAdminApplicationServiceTest {
.containsExactly("ADMIN_PROVIDER_STATE");
}
+ @Test
+ void providerStateClaimAndChangeCommitInTheUseCasesOneWriteTransaction() {
+ // given
+ Fixture fixture = new Fixture();
+ ProviderStateRequest request =
+ new ProviderStateRequest(
+ "op-tx",
+ new ProviderProfileId("ses-primary"),
+ ProviderRuntimeState.DRAINING,
+ "provider maintenance",
+ false);
+
+ // when
+ fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR));
+
+ // then
+ assertThat(List.of(fixture.transactions.committed(), fixture.operations.claimsInTransaction))
+ .as("a claim outside the transaction would commit while the change rolled back")
+ .containsExactly(List.of(DirectTransactionPort.Boundary.WRITE), List.of(true));
+ }
+
@Test
void providerStateDryRunDoesNotClaimOrMutateRuntime() {
Fixture fixture = new Fixture();
- SetProviderStateCommand command =
- new SetProviderStateCommand(
+ ProviderStateRequest request =
+ new ProviderStateRequest(
"op-dry",
new ProviderProfileId("ses-primary"),
ProviderRuntimeState.DRAINING,
"inspect only",
true);
- AdminOperationResult result = fixture.service.setProviderState(command, ACTOR);
+ AdminOperationResult result =
+ fixture.providerState.handle(new ChangeProviderStateCommand(request, ACTOR));
assertThat(result.dryRun()).isTrue();
assertThat(fixture.operations.claims).isZero();
@@ -97,7 +124,6 @@ class NotificationAdminApplicationServiceTest {
}
@Test
- @DisplayName("tenant 범위 운영자는 다른 tenant의 attempt를 reconcile할 수 없다")
void aTenantScopedOperatorCannotReconcileAnotherTenantsAttempt() {
// given
Fixture fixture = new Fixture();
@@ -109,11 +135,13 @@ class NotificationAdminApplicationServiceTest {
"operator-b",
Set.of(NotificationAdminAuthority.RECONCILE),
Optional.of(new TenantId("tenant-b")));
- ReconcileCommand command =
- new ReconcileCommand("op-r", List.of(foreign.attemptId()), "stuck attempt", false);
+ ReconcileRequest request =
+ new ReconcileRequest("op-r", List.of(foreign.attemptId()), "stuck attempt", false);
// when
- Throwable failure = catchThrowable(() -> fixture.service.reconcile(command, tenantB));
+ Throwable failure =
+ catchThrowable(
+ () -> fixture.reconcile.handle(new ReconcileDeliveriesCommand(request, tenantB)));
// then
assertThat(failure)
@@ -124,7 +152,6 @@ class NotificationAdminApplicationServiceTest {
}
@Test
- @DisplayName("tenant 범위 운영자는 모든 tenant가 공유하는 provider 상태를 바꿀 수 없다")
void aTenantScopedOperatorCannotChangeAGlobalProviderState() {
// given
Fixture fixture = new Fixture();
@@ -133,8 +160,8 @@ class NotificationAdminApplicationServiceTest {
"operator-b",
Set.of(NotificationAdminAuthority.PROVIDER_CONTROL),
Optional.of(new TenantId("tenant-b")));
- SetProviderStateCommand command =
- new SetProviderStateCommand(
+ ProviderStateRequest request =
+ new ProviderStateRequest(
"op-p",
new ProviderProfileId("ses-primary"),
ProviderRuntimeState.DISABLED,
@@ -142,7 +169,9 @@ class NotificationAdminApplicationServiceTest {
false);
// when
- Throwable failure = catchThrowable(() -> fixture.service.setProviderState(command, tenantB));
+ Throwable failure =
+ catchThrowable(
+ () -> fixture.providerState.handle(new ChangeProviderStateCommand(request, tenantB)));
// then
assertThat(failure)
@@ -154,7 +183,6 @@ class NotificationAdminApplicationServiceTest {
}
@Test
- @DisplayName("운영자 reconcile은 DB 트랜잭션을 연 채로 provider에 묻지 않는다")
void reconcileAsksTheProviderWithNoTransactionOpen() {
// given
List transactionOpenDuringProviderCall = new ArrayList<>();
@@ -182,9 +210,10 @@ class NotificationAdminApplicationServiceTest {
"operator-1", Set.of(NotificationAdminAuthority.RECONCILE), Optional.empty());
// when
- fixture.service.reconcile(
- new ReconcileCommand("op-rc", List.of(attempt.attemptId()), "stuck attempt", false),
- operator);
+ fixture.reconcile.handle(
+ new ReconcileDeliveriesCommand(
+ new ReconcileRequest("op-rc", List.of(attempt.attemptId()), "stuck attempt", false),
+ operator));
// then
assertThat(transactionOpenDuringProviderCall)
@@ -192,6 +221,132 @@ class NotificationAdminApplicationServiceTest {
.containsExactly(false);
}
+ @Test
+ void redriveReturnsTheDeliveryToTheQueueUnderItsClaim() {
+ // given
+ Fixture fixture = new Fixture();
+ DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a"));
+ AttemptSnapshots.store(
+ fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED);
+ AdminActor operator =
+ new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty());
+ RedriveRequest request =
+ new RedriveRequest("op-rd", attempt.attemptId(), "provider outage", true, false);
+
+ // when
+ fixture.redrive.handle(new RedriveDeliveryCommand(request, operator));
+
+ // then
+ assertThat(
+ List.of(
+ fixture.recipients.find(attempt.recipientDeliveryId()).orElseThrow().state(),
+ fixture.operations.claimsInTransaction,
+ fixture.audit.events.stream().map(NotificationAuditEvent::action).toList()))
+ .containsExactly(
+ RecipientDeliveryState.READY_TO_DISPATCH, List.of(true), List.of("ADMIN_REDRIVE"));
+ }
+
+ @Test
+ void aRedriveReadsItsAttemptInsideTheWriteTransaction() {
+ // given
+ Fixture fixture = new Fixture();
+ DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a"));
+ AttemptSnapshots.store(
+ fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED);
+ List openWhileReading = new ArrayList<>();
+ fixture.attempts.beforeSnapshot(id -> openWhileReading.add(fixture.transactions.active()));
+ AdminActor operator =
+ new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty());
+
+ // when
+ fixture.redrive.handle(
+ new RedriveDeliveryCommand(
+ new RedriveRequest("op-rd-tx", attempt.attemptId(), "provider outage", true, false),
+ operator));
+
+ // then
+ assertThat(List.of(openWhileReading, fixture.transactions.committed()))
+ .as("the checks read the attempt the claim then redrives, in one transaction")
+ .containsExactly(List.of(true), List.of(DirectTransactionPort.Boundary.WRITE));
+ }
+
+ @Test
+ void aReconcileDryRunChecksItsTenantsInsideAReadTransaction() {
+ // given
+ Fixture fixture = new Fixture();
+ DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a"));
+ AttemptSnapshots.store(
+ fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.DISPATCHING);
+ List openWhileReading = new ArrayList<>();
+ fixture.attempts.beforeSnapshot(id -> openWhileReading.add(fixture.transactions.active()));
+ AdminActor operator =
+ new AdminActor(
+ "operator-1", Set.of(NotificationAdminAuthority.RECONCILE), Optional.empty());
+
+ // when
+ fixture.reconcile.handle(
+ new ReconcileDeliveriesCommand(
+ new ReconcileRequest("op-rc-tx", List.of(attempt.attemptId()), "stuck attempt", true),
+ operator));
+
+ // then
+ assertThat(List.of(openWhileReading, fixture.transactions.committed()))
+ .containsExactly(List.of(true), List.of(DirectTransactionPort.Boundary.READ));
+ }
+
+ @Test
+ void aRedriveDryRunReadsInOneReadTransactionAndClaimsNothing() {
+ // given
+ Fixture fixture = new Fixture();
+ DeliveryAttemptSnapshot attempt = attemptOf(new TenantId("tenant-a"));
+ AttemptSnapshots.store(
+ fixture.requests, fixture.attempts, attempt, RecipientDeliveryState.FAILED);
+ AdminActor operator =
+ new AdminActor("operator-1", Set.of(NotificationAdminAuthority.REDRIVE), Optional.empty());
+ RedriveRequest request =
+ new RedriveRequest("op-rd-dry", attempt.attemptId(), "provider outage", true, true);
+
+ // when
+ fixture.redrive.handle(new RedriveDeliveryCommand(request, operator));
+
+ // then
+ assertThat(List.of(fixture.transactions.committed(), fixture.operations.claimsInTransaction))
+ .containsExactly(List.of(DirectTransactionPort.Boundary.READ), List.of());
+ }
+
+ @Test
+ void anOperatorSuppressionIsStoredUnderItsClaim() {
+ // given
+ Fixture fixture = new Fixture();
+ TenantId tenant = new TenantId("tenant-a");
+ AdminActor operator =
+ new AdminActor(
+ "operator-1", Set.of(NotificationAdminAuthority.SUPPRESS), Optional.of(tenant));
+ SuppressionRequest request =
+ new SuppressionRequest(
+ "op-s",
+ tenant,
+ SuppressionScope.CONTACT_POINT,
+ SuppressionReason.ADMIN_BLOCK,
+ "c".repeat(64),
+ Optional.empty(),
+ false,
+ "reported abuse",
+ false);
+
+ // when
+ AdminOperationResult result =
+ fixture.suppression.handle(new ChangeSuppressionCommand(request, operator));
+
+ // then
+ assertThat(
+ List.of(
+ result.affected(),
+ fixture.suppressions.activeFor(tenant, "c".repeat(64), CLOCK.instant()).size(),
+ fixture.operations.claimsInTransaction))
+ .containsExactly(1, 1, List.of(true));
+ }
+
private static DeliveryAttemptSnapshot attemptOf(TenantId tenantId) {
return new DeliveryAttemptSnapshot(
new DeliveryAttemptId(UUID.fromString("00000000-0000-7000-8000-000000000011")),
@@ -225,7 +380,11 @@ class NotificationAdminApplicationServiceTest {
private final RecordingOperations operations = new RecordingOperations();
private final RecordingAudit audit = new RecordingAudit();
private final DirectTransactionPort transactions = new DirectTransactionPort();
- private final NotificationAdminApplicationService service;
+ private final InMemorySuppressionStore suppressions = new InMemorySuppressionStore();
+ private final ChangeProviderStateUseCase providerState;
+ private final ReconcileDeliveriesUseCase reconcile;
+ private final RedriveDeliveryUseCase redrive;
+ private final ChangeSuppressionUseCase suppression;
private Fixture() {
this(null);
@@ -235,24 +394,42 @@ class NotificationAdminApplicationServiceTest {
* @param gateway the provider the reconciliation asks, or null for one that supports nothing
*/
private Fixture(ReconciliationGatewayPort gateway) {
+ operations.transactions = transactions;
ReconciliationService reconciliation =
gateway == null
? PlatformCollaborators.reconciliation(attempts, recipients, CLOCK)
: PlatformCollaborators.reconciliation(attempts, recipients, gateway, CLOCK);
- service =
- new NotificationAdminApplicationService(
+ providerState =
+ new ChangeProviderStateUseCase(
+ new AdminAuthorizationGuard(), runtime, operations, audit, transactions, CLOCK);
+ redrive =
+ new RedriveDeliveryUseCase(
new AdminAuthorizationGuard(),
new DuplicateRiskGuard(),
attempts,
recipients,
- reconciliation,
- new UnusedSuppressionStore(),
- runtime,
+ operations,
+ audit,
+ transactions,
+ CLOCK);
+ suppression =
+ new ChangeSuppressionUseCase(
+ new AdminAuthorizationGuard(),
+ suppressions,
operations,
audit,
transactions,
new PlatformFakes.SequentialIds(),
CLOCK);
+ reconcile =
+ new ReconcileDeliveriesUseCase(
+ new AdminAuthorizationGuard(),
+ attempts,
+ reconciliation,
+ operations,
+ audit,
+ transactions,
+ CLOCK);
}
}
@@ -277,6 +454,8 @@ class NotificationAdminApplicationServiceTest {
}
private static final class RecordingOperations implements AdminOperationStorePort {
+ private final List claimsInTransaction = new ArrayList<>();
+ private DirectTransactionPort transactions;
private int claims;
private int saved;
@@ -293,9 +472,10 @@ class NotificationAdminApplicationServiceTest {
@Override
public AdminOperationClaim claim(
- String operationId, String commandFingerprint, AdminActor actor, String action) {
+ String operationId, String requestFingerprint, AdminActor actor, String action) {
claims++;
- assertThat(commandFingerprint).matches("[0-9a-f]{64}");
+ claimsInTransaction.add(transactions.active());
+ assertThat(requestFingerprint).matches("[0-9a-f]{64}");
return AdminOperationClaim.claimed();
}
}
@@ -308,25 +488,4 @@ class NotificationAdminApplicationServiceTest {
events.add(event);
}
}
-
- private static final class UnusedSuppressionStore implements SuppressionStorePort {
- @Override
- public List activeFor(
- dev.caskeleton.application.notification.platform.api.TenantId tenantId,
- String targetFingerprint,
- Instant at) {
- return List.of();
- }
-
- @Override
- public SuppressionEntry upsert(SuppressionEntry entry) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Optional remove(
- dev.caskeleton.application.notification.platform.api.TenantId tenantId, SuppressionId id) {
- return Optional.empty();
- }
- }
}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java
new file mode 100644
index 00000000..9014a229
--- /dev/null
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ClaimDueDeliveriesUseCaseTest.java
@@ -0,0 +1,90 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
+import dev.caskeleton.application.notification.platform.api.TenantId;
+import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows;
+import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The scheduler's claim, run through its {@code @SystemInvoked} use case.
+ *
+ * The claim used to run with no transaction of the scheduler's own: the store's statement was
+ * its whole boundary. It now commits in the use case's write transaction.
+ */
+class ClaimDueDeliveriesUseCaseTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z");
+ private static final Duration LEASE = Duration.ofMinutes(2);
+
+ private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore();
+ private final InMemoryRecipientQueue queue =
+ new InMemoryRecipientQueue(requests, Clock.fixed(NOW, ZoneOffset.UTC));
+ private final DirectTransactionPort transactions = new DirectTransactionPort();
+ private final ClaimDueDeliveriesUseCase useCase =
+ new ClaimDueDeliveriesUseCase(queue, transactions);
+
+ @Test
+ @DisplayName("due 발송은 쓰기 트랜잭션 하나에서 이 worker의 lease로 claim된다")
+ void aDueDeliveryIsClaimedForTheWorkerInOneWriteTransaction() {
+ // given
+ RecipientDeliveryId due = dueJob();
+
+ // when
+ List claimed =
+ useCase.handle(new ClaimDueDeliveriesCommand("worker-1", 10, LEASE));
+
+ // then
+ assertThat(
+ List.of(
+ claimed.stream().map(RecipientLease::recipientDeliveryId).toList(),
+ claimed.stream().map(RecipientLease::owner).toList(),
+ claimed.stream().map(RecipientLease::leaseUntil).toList(),
+ transactions.committed()))
+ .containsExactly(
+ List.of(due),
+ List.of("worker-1"),
+ List.of(NOW.plus(LEASE)),
+ List.of(DirectTransactionPort.Boundary.WRITE));
+ }
+
+ @Test
+ @DisplayName("한 번에 0개를 claim하라는 command는 조립의 잘못으로 실패한다")
+ void aClaimOfNothingIsTheAssemblysFault() {
+ // when
+ Throwable refused = catchThrowable(() -> new ClaimDueDeliveriesCommand("worker-1", 0, LEASE));
+
+ // then
+ assertThat(refused)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("limit must be positive");
+ }
+
+ private RecipientDeliveryId dueJob() {
+ var request =
+ NotificationRows.request(new TenantId("tenant-a"), "idem-" + UUID.randomUUID(), NOW);
+ var job =
+ NotificationRows.job(
+ request,
+ RecipientDeliveryState.READY_TO_DISPATCH,
+ Optional.of(NOW.minusSeconds(1)),
+ Optional.empty(),
+ NOW);
+ requests.insert(request, List.of(job));
+ return job.id();
+ }
+}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java
new file mode 100644
index 00000000..47d87614
--- /dev/null
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/DispatchClaimedDeliveryUseCaseTest.java
@@ -0,0 +1,78 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryContactPointStore;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryDeliveryAttemptStore;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue;
+import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy;
+import dev.caskeleton.application.notification.platform.policy.RetryDecision;
+import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine;
+import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.UUID;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The scheduler's dispatch of one claimed delivery, run through its {@code @SystemInvoked} use
+ * case. The use case holds no boundary of its own; the dispatch service opens each one.
+ */
+class DispatchClaimedDeliveryUseCaseTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z");
+ private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC);
+
+ private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore();
+ private final InMemoryRecipientQueue recipients = new InMemoryRecipientQueue(requests, CLOCK);
+ private final InMemoryDeliveryAttemptStore attempts = new InMemoryDeliveryAttemptStore(requests);
+ private final DirectTransactionPort transactions = new DirectTransactionPort();
+ private final PlatformFakes.UnreachableGateway gateway = new PlatformFakes.UnreachableGateway();
+
+ private final DispatchClaimedDeliveryUseCase useCase =
+ new DispatchClaimedDeliveryUseCase(
+ new NotificationDispatchService(
+ requests,
+ recipients,
+ attempts,
+ new DispatchPipeline(
+ new InMemoryContactPointStore(CLOCK),
+ NotificationEligibilityPolicy.permitAll(),
+ new PlatformFakes.RecordingMetrics()),
+ new RoutingDecisionEngine(),
+ context -> new RetryDecision.Stop("unused"),
+ new DeliveryAttemptFactory(new PlatformFakes.SequentialIds()),
+ PlatformCollaborators.outcomeRecorder(attempts, recipients),
+ gateway,
+ command -> {
+ throw new AssertionError("a lease whose delivery is gone renders nothing");
+ },
+ recipients,
+ transactions,
+ CLOCK,
+ 3,
+ Duration.ofSeconds(5)));
+
+ @Test
+ @DisplayName("delivery가 사라진 lease는 dispatch 서비스의 읽기·쓰기 경계에서 풀리고 provider에 닿지 않는다")
+ void aLeaseWhoseDeliveryIsGoneIsReleasedWithoutReachingTheProvider() {
+ // given
+ RecipientLease gone =
+ new RecipientLease(
+ new RecipientDeliveryId(UUID.randomUUID()), "worker-a", 1L, NOW.plusSeconds(30));
+
+ // when
+ useCase.handle(new DispatchClaimedDeliveryCommand(gone));
+
+ // then
+ assertThat(List.of(transactions.committed(), gateway.calls()))
+ .containsExactly(
+ List.of(DirectTransactionPort.Boundary.READ, DirectTransactionPort.Boundary.WRITE), 0);
+ }
+}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java
index 7ff29358..ed508150 100644
--- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/LeaseRecoveryServiceTest.java
@@ -186,6 +186,40 @@ class LeaseRecoveryServiceTest {
assertThat(recovery.recoverOnce()).isEqualTo(3);
}
+ @Test
+ void aDeliveryItsHolderFinishedAfterRecoveryFoundItIsNotRequeued() {
+ // given
+ RecipientDeliveryId id = jobOf(NOW, Optional.empty());
+ RecipientLease lease = queue.claim("worker-a", 10, Duration.ofSeconds(30)).getFirst();
+ workerDied();
+ // The holder completes the job between recovery finding it abandoned and recovery writing it
+ // — the window another instance or a slow worker uses in production.
+ attempts.beforeAttemptsOf(
+ delivery -> {
+ var unused =
+ queue.transitionHeldBy(
+ delivery, RecipientDeliveryState.COMPLETED, Optional.empty(), lease);
+ });
+ LeaseRecoveryService racing =
+ new LeaseRecoveryService(
+ queue,
+ attempts,
+ queue,
+ reconciled::add,
+ new DirectTransactionPort(),
+ CLOCK,
+ Duration.ofMinutes(1),
+ 10);
+
+ // when
+ racing.recoverOnce();
+
+ // then
+ assertThat(stateOf(id))
+ .as("the holder was only slow; recovery's requeue must not reopen what it finished")
+ .isEqualTo(RecipientDeliveryState.COMPLETED);
+ }
+
private static dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence
notStarted() {
return dev.caskeleton.application.notification.platform.provider.ProviderExecutionEvidence
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java
index e4e13f18..b5ab61a3 100644
--- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/NotificationDispatchServiceTest.java
@@ -32,8 +32,8 @@ import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemor
import dev.caskeleton.application.notification.platform.policy.DefaultNotificationRetryPolicy;
import dev.caskeleton.application.notification.platform.policy.JitterSource;
import dev.caskeleton.application.notification.platform.policy.NotificationEligibilityPolicy;
-import dev.caskeleton.application.notification.platform.policy.RetryBackoff;
import dev.caskeleton.application.notification.platform.policy.RetryDecision;
+import dev.caskeleton.application.notification.platform.policy.RetryTiming;
import dev.caskeleton.application.notification.platform.policy.RoutingDecisionEngine;
import dev.caskeleton.application.notification.platform.port.in.SubmitNotificationCommand;
import dev.caskeleton.application.notification.platform.provider.ProviderCapabilities;
@@ -70,7 +70,7 @@ class NotificationDispatchServiceTest {
new InMemoryRecipientQueue(requests, Clock.fixed(NOW, ZoneOffset.UTC));
InMemoryDeliveryAttemptStore attempts = new InMemoryDeliveryAttemptStore(requests);
DirectTransactionPort transactions = new DirectTransactionPort();
- FailingGateway gateway = new FailingGateway();
+ PlatformFakes.UnreachableGateway gateway = new PlatformFakes.UnreachableGateway();
NotificationDispatchService service =
new NotificationDispatchService(
requests,
@@ -104,7 +104,7 @@ class NotificationDispatchServiceTest {
// The WRITE boundary after the read is the release; the queue has no row to show it on.
assertThat(transactions.committed()).containsExactly(Boundary.READ, Boundary.WRITE);
- assertThat(gateway.calls).isZero();
+ assertThat(gateway.calls()).isZero();
}
@Test
@@ -500,8 +500,7 @@ class NotificationDispatchServiceTest {
NotificationEligibilityPolicy.permitAll(),
new PlatformFakes.RecordingMetrics()),
new RoutingDecisionEngine(),
- new DefaultNotificationRetryPolicy(
- RetryBackoff.DEFAULT, JitterSource.none(), Duration.ofMinutes(1)),
+ new DefaultNotificationRetryPolicy(RetryTiming.DEFAULT, JitterSource.none()),
new DeliveryAttemptFactory(new PlatformFakes.SequentialIds()),
PlatformCollaborators.outcomeRecorder(attempts, recipients),
gateway,
@@ -631,26 +630,4 @@ class NotificationDispatchServiceTest {
throw new AssertionError("nothing may be submitted through a profile that is gone");
}
}
-
- private static final class FailingGateway implements ProviderDispatchGatewayPort {
- private int calls;
-
- @Override
- public ProviderProfileSnapshot profile(ProviderProfileId profileId) {
- calls++;
- throw new AssertionError("gateway profile must not be reached");
- }
-
- @Override
- public ProviderRuntimeState state(ProviderProfileId profileId) {
- calls++;
- throw new AssertionError("gateway state must not be reached");
- }
-
- @Override
- public ProviderSubmissionResult submit(ProviderSubmission submission) {
- calls++;
- throw new AssertionError("provider submit must not be reached");
- }
- }
}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java
index a04d1631..7fc4750f 100644
--- a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/PlatformFakes.java
@@ -14,6 +14,10 @@ import dev.caskeleton.application.notification.platform.api.routing.DeliveryStra
import dev.caskeleton.application.notification.platform.api.routing.ExplicitChannel;
import dev.caskeleton.application.notification.platform.observation.NotificationMetricsPort;
import dev.caskeleton.application.notification.platform.policy.RouteCandidate;
+import dev.caskeleton.application.notification.platform.provider.ProviderProfileSnapshot;
+import dev.caskeleton.application.notification.platform.provider.ProviderRuntimeState;
+import dev.caskeleton.application.notification.platform.provider.ProviderSubmission;
+import dev.caskeleton.application.notification.platform.provider.ProviderSubmissionResult;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
@@ -35,6 +39,35 @@ public final class PlatformFakes {
private PlatformFakes() {}
+ /** A provider gateway the dispatch under test must not reach; it counts what reached it. */
+ public static final class UnreachableGateway implements ProviderDispatchGatewayPort {
+
+ private int calls;
+
+ /** How many calls reached the gateway. */
+ public int calls() {
+ return calls;
+ }
+
+ @Override
+ public ProviderProfileSnapshot profile(ProviderProfileId profileId) {
+ calls++;
+ throw new AssertionError("gateway profile must not be reached");
+ }
+
+ @Override
+ public ProviderRuntimeState state(ProviderProfileId profileId) {
+ calls++;
+ throw new AssertionError("gateway state must not be reached");
+ }
+
+ @Override
+ public ProviderSubmissionResult submit(ProviderSubmission submission) {
+ calls++;
+ throw new AssertionError("provider submit must not be reached");
+ }
+ }
+
/** Metrics sink that keeps what it was told, so tag discipline stays assertable. */
public static final class RecordingMetrics implements NotificationMetricsPort {
diff --git a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java
similarity index 50%
rename from src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java
rename to src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java
index ad25e7bd..2cbe1abb 100644
--- a/src/adapter/outbound/notification/src/test/java/dev/caskeleton/adapter/outbound/notification/platform/dispatch/ReconciliationJobWorkerTest.java
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/ReconcileOutstandingAttemptsUseCaseTest.java
@@ -1,4 +1,4 @@
-package dev.caskeleton.adapter.outbound.notification.platform.dispatch;
+package dev.caskeleton.application.notification.platform.dispatch;
import static org.assertj.core.api.Assertions.assertThat;
@@ -6,14 +6,14 @@ import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
import dev.caskeleton.application.notification.platform.api.NormalizedEventType;
import dev.caskeleton.application.notification.platform.api.NormalizedProviderEvent;
import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
-import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJob;
-import dev.caskeleton.application.notification.platform.dispatch.ReconciliationJobStorePort;
import dev.caskeleton.application.notification.platform.provider.ReconciliationResult;
+import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -29,117 +29,190 @@ import org.junit.jupiter.api.Test;
* in {@code RECONCILIATION_REQUIRED} permanently. The job table for exactly this has existed since
* V3 with no entity, no store and no worker.
*/
-class ReconciliationJobWorkerTest {
+class ReconcileOutstandingAttemptsUseCaseTest {
private static final Instant NOW = Instant.parse("2026-08-15T09:00:00Z");
private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC);
- private static final Duration CLAIM_LEASE = Duration.ofMinutes(2);
+ private static final Duration DELIVERY_LEASE = Duration.ofMinutes(2);
+ private static final int MAX_ADDITIONAL_ATTEMPTS = 2;
private final RecordingJobs jobs = new RecordingJobs();
- private final Map answers = new java.util.HashMap<>();
+ private final Map answers = new HashMap<>();
+ private final List asked = new ArrayList<>();
+ private final DirectTransactionPort transactions = new DirectTransactionPort();
- private final ReconciliationJobWorker worker =
- new ReconciliationJobWorker(
- jobs, answers::get, CLOCK, Duration.ofMinutes(5), CLAIM_LEASE, 10, 3);
+ private final ReconcileOutstandingAttemptsUseCase useCase =
+ new ReconcileOutstandingAttemptsUseCase(
+ jobs,
+ attemptId -> {
+ asked.add(attemptId);
+ return answers.get(attemptId);
+ },
+ transactions,
+ CLOCK,
+ DELIVERY_LEASE,
+ 10,
+ MAX_ADDITIONAL_ATTEMPTS);
@Test
- @DisplayName("a confirmed outcome completes the job")
void aConfirmedOutcomeCompletesTheJob() {
+ // given
ReconciliationJob job = due(0);
answers.put(job.attemptId(), new ReconciliationResult.Confirmed(event()));
- assertThat(worker.reconcileOnce()).isEqualTo(1);
- assertThat(jobs.completed).containsExactly(job.id());
+ // when
+ int settled = useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(List.of(settled, jobs.completed)).containsExactly(1, List.of(job.id()));
}
@Test
- @DisplayName("an unsupported provider is not asked in a loop")
- void anUnsupportedProviderIsNotAskedInALoop() {
+ @DisplayName("상태 조회가 없는 provider는 다시 묻지 않고 job을 끝낸다. 답이 바뀌지 않는다")
+ void anUnsupportedProviderIsCompletedRatherThanAskedAgain() {
+ // given
ReconciliationJob job = due(0);
answers.put(job.attemptId(), new ReconciliationResult.Unsupported());
- assertThat(worker.reconcileOnce()).isEqualTo(1);
- assertThat(jobs.completed)
- .as(
- "without a status-query capability the answer will not change; the attempt stays"
- + " visibly ambiguous for an operator instead")
- .containsExactly(job.id());
- assertThat(jobs.rescheduled).isEmpty();
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(List.of(jobs.completed, jobs.rescheduled))
+ .as("the attempt stays visibly ambiguous for an operator instead")
+ .containsExactly(List.of(job.id()), List.of());
}
@Test
- @DisplayName("a still-unknown outcome is asked again at the provider's own next time")
- void aStillUnknownOutcomeIsAskedAgain() {
+ void aStillUnknownOutcomeIsAskedAgainAtTheProvidersOwnNextTime() {
+ // given
ReconciliationJob job = due(0);
Instant providerSuggested = NOW.plus(Duration.ofHours(2));
answers.put(job.attemptId(), new ReconciliationResult.StillUnknown(providerSuggested));
- assertThat(worker.reconcileOnce()).isZero();
- assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), providerSuggested));
- assertThat(jobs.completed).isEmpty();
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(List.of(jobs.rescheduled, jobs.completed))
+ .containsExactly(List.of(Map.entry(job.id(), providerSuggested)), List.of());
}
@Test
- @DisplayName("a retryable failure is asked again after the backoff")
- void aRetryableFailureIsAskedAgain() {
+ void aRetryableFailureIsAskedAgainOneDeliveryLeaseLater() {
+ // given
ReconciliationJob job = due(0);
answers.put(job.attemptId(), new ReconciliationResult.Failed("PROVIDER_TIMEOUT", true));
- assertThat(worker.reconcileOnce()).isZero();
- assertThat(jobs.rescheduled)
- .containsExactly(Map.entry(job.id(), NOW.plus(Duration.ofMinutes(5))));
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(jobs.rescheduled).containsExactly(Map.entry(job.id(), NOW.plus(DELIVERY_LEASE)));
}
@Test
- @DisplayName("a non-retryable failure completes the job")
void aNonRetryableFailureCompletesTheJob() {
+ // given
ReconciliationJob job = due(0);
answers.put(job.attemptId(), new ReconciliationResult.Failed("ATTEMPT_UNKNOWN", false));
- assertThat(worker.reconcileOnce()).isEqualTo(1);
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
assertThat(jobs.completed).containsExactly(job.id());
}
@Test
- @DisplayName("a job that has asked enough times stops asking")
- void aJobThatHasAskedEnoughTimesStopsAsking() {
- ReconciliationJob job = due(3);
+ @DisplayName("첫 질문과 추가 시도 2번, 모두 3번을 물은 job은 더 묻지 않고 끝난다")
+ void aJobThatAskedOnceMoreThanItsAdditionalAttemptsStopsAsking() {
+ // given
+ ReconciliationJob job = due(MAX_ADDITIONAL_ATTEMPTS + 1);
- assertThat(worker.reconcileOnce()).isEqualTo(1);
- assertThat(jobs.completed)
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(List.of(asked, jobs.completed))
.as("an outcome that has not arrived after this many tries is an operator's decision")
- .containsExactly(job.id());
- assertThat(answers).isEmpty();
+ .containsExactly(List.of(), List.of(job.id()));
}
@Test
- @DisplayName("one job throwing does not stop the rest of the pass")
- void oneJobThrowingDoesNotStopTheRest() {
+ @DisplayName("2번 물은 job은 추가 시도가 남아 한 번 더 묻는다")
+ void aJobWithAnAdditionalAttemptLeftIsAskedAgain() {
+ // given
+ ReconciliationJob job = due(MAX_ADDITIONAL_ATTEMPTS);
+ answers.put(job.attemptId(), new ReconciliationResult.Unsupported());
+
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(asked).containsExactly(job.attemptId());
+ }
+
+ @Test
+ void oneJobThrowingDoesNotStopTheRestOfThePass() {
+ // given
+ // The failing job has no answer registered, so the reconciler returns null and the switch
+ // throws — which is exactly the shape of a provider refusing connections.
ReconciliationJob failing = due(0);
ReconciliationJob healthy = due(0);
answers.put(healthy.attemptId(), new ReconciliationResult.Confirmed(event()));
- // The failing job has no answer registered, so the reconciler returns null and the switch
- // throws — which is exactly the shape of a provider refusing connections.
- assertThat(worker.reconcileOnce()).isEqualTo(1);
- assertThat(jobs.completed).containsExactly(healthy.id());
- assertThat(jobs.rescheduled).hasSize(1);
- assertThat(jobs.rescheduled.get(0).getKey()).isEqualTo(failing.id());
+ // when
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(List.of(jobs.completed, jobs.rescheduled.stream().map(Map.Entry::getKey).toList()))
+ .containsExactly(List.of(healthy.id()), List.of(failing.id()));
}
@Test
- @DisplayName("한 번의 pass는 설정된 lease 동안 소유하도록 job을 claim한다")
- void aPassClaimsItsJobsForTheConfiguredLease() {
+ @DisplayName("한 번의 pass는 delivery lease 동안 소유하도록 job을 claim한다")
+ void aPassClaimsItsJobsForOneDeliveryLease() {
// given
due(0);
// when
- worker.reconcileOnce();
+ useCase.handle(new ReconcileOutstandingAttemptsCommand());
// then
assertThat(jobs.claimLeases)
.as("without a lease the claim ends with the statement and every instance takes the job")
- .containsExactly(CLAIM_LEASE);
+ .containsExactly(DELIVERY_LEASE);
+ }
+
+ @Test
+ @DisplayName("provider에는 트랜잭션 없이 묻고, job 기록은 그 앞뒤의 쓰기 트랜잭션에서 한다")
+ void theProviderIsAskedWithNoTransactionOpenAndTheJobIsRecordedInOne() {
+ // given
+ due(0);
+ List openWhileAsking = new ArrayList<>();
+ ReconcileOutstandingAttemptsUseCase observing =
+ new ReconcileOutstandingAttemptsUseCase(
+ jobs,
+ attemptId -> {
+ openWhileAsking.add(transactions.active());
+ return new ReconciliationResult.Unsupported();
+ },
+ transactions,
+ CLOCK,
+ DELIVERY_LEASE,
+ 10,
+ MAX_ADDITIONAL_ATTEMPTS);
+
+ // when
+ observing.handle(new ReconcileOutstandingAttemptsCommand());
+
+ // then
+ assertThat(List.of(openWhileAsking, transactions.committed()))
+ .as("the claim and the completion each commit; the provider call holds no connection")
+ .containsExactly(
+ List.of(false),
+ List.of(DirectTransactionPort.Boundary.WRITE, DirectTransactionPort.Boundary.WRITE));
}
private ReconciliationJob due(int attempts) {
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java
new file mode 100644
index 00000000..0065ffec
--- /dev/null
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/RecoverAbandonedDeliveriesUseCaseTest.java
@@ -0,0 +1,77 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
+import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
+import dev.caskeleton.application.notification.platform.api.TenantId;
+import dev.caskeleton.application.notification.platform.api.delivery.RecipientDeliveryState;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.ContractClock;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryDeliveryAttemptStore;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryNotificationRequestStore;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.InMemoryRecipientQueue;
+import dev.caskeleton.application.notification.platform.dispatch.testkit.NotificationRows;
+import dev.caskeleton.application.transaction.testkit.DirectTransactionPort;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/** The background workers' lease recovery, run through its {@code @SystemInvoked} use case. */
+class RecoverAbandonedDeliveriesUseCaseTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z");
+
+ private final InMemoryNotificationRequestStore requests = new InMemoryNotificationRequestStore();
+ private final ContractClock queueClock = new ContractClock(NOW);
+ private final InMemoryRecipientQueue queue = new InMemoryRecipientQueue(requests, queueClock);
+ private final List reconciled = new ArrayList<>();
+
+ private final RecoverAbandonedDeliveriesUseCase useCase =
+ new RecoverAbandonedDeliveriesUseCase(
+ new LeaseRecoveryService(
+ queue,
+ new InMemoryDeliveryAttemptStore(requests),
+ queue,
+ reconciled::add,
+ new DirectTransactionPort(),
+ Clock.fixed(NOW, ZoneOffset.UTC),
+ Duration.ofMinutes(1),
+ 10));
+
+ @Test
+ @DisplayName("worker가 attempt를 남기기 전에 죽은 delivery는 한 pass에서 다시 보낼 차례로 돌아간다")
+ void aDeliveryAbandonedBeforeItsAttemptIsRequeuedInOnePass() {
+ // given
+ RecipientDeliveryId abandoned = abandonedJob();
+ queueClock.moveTo(NOW.plus(Duration.ofMinutes(5)));
+
+ // when
+ int recovered = useCase.handle(new RecoverAbandonedDeliveriesCommand());
+
+ // then
+ assertThat(List.of(recovered, queue.find(abandoned).orElseThrow().state(), reconciled))
+ .containsExactly(1, RecipientDeliveryState.READY_TO_DISPATCH, List.of());
+ }
+
+ private RecipientDeliveryId abandonedJob() {
+ var request =
+ NotificationRows.request(new TenantId("tenant-a"), "idem-" + UUID.randomUUID(), NOW);
+ var job =
+ NotificationRows.job(
+ request,
+ RecipientDeliveryState.READY_TO_DISPATCH,
+ Optional.of(NOW),
+ Optional.empty(),
+ NOW);
+ requests.insert(request, List.of(job));
+ queue.claim("worker-a", 10, Duration.ofSeconds(30));
+ return job.id();
+ }
+}
diff --git a/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java
new file mode 100644
index 00000000..dd33283f
--- /dev/null
+++ b/src/application/notification/src/test/java/dev/caskeleton/application/notification/platform/dispatch/StoreBuiltRecordsTest.java
@@ -0,0 +1,76 @@
+package dev.caskeleton.application.notification.platform.dispatch;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+import dev.caskeleton.application.notification.platform.admin.AdminOperationClaim;
+import dev.caskeleton.application.notification.platform.api.DeliveryAttemptId;
+import dev.caskeleton.application.notification.platform.api.ProviderProfileId;
+import dev.caskeleton.application.notification.platform.api.RecipientDeliveryId;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Stream;
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+/**
+ * The leases, jobs and claims a notification store builds from its rows refuse a bad value as each
+ * record says.
+ */
+class StoreBuiltRecordsTest {
+
+ private static final Instant NOW = Instant.parse("2026-11-13T09:00:00Z");
+
+ static Stream