review-followup Wave 3 라운드 2(P18): outbox canonical 통일, composition 모듈, legacy Kafka seam 삭제

- 결정 7: `LegacyOutboxAppendPort`·legacy 저장 모델·relay·Kafka R0 seam 삭제, 템플릿 JSON encoder(`JsonIntegrationEventEncoder`),
  durable operation·sample 이벤트가 `OutboxAppendPort` + encoder로, persistence-jpa V19(pre-canonical 미발행 row 거부)
- 결정 3: outbox relay·멱등성 executor 조립을 `composition:outbox-relay`·`composition:idempotency`로,
  sample이 복사한 bootstrap 인프라를 `composition:service-runtime`으로(sample `SampleIdempotencyConfig` 복사본 삭제)
- persistence-jpa outbox·멱등성 패키지 feature 기준 이동, messaging outbox·inbox leaf를 reliability / reliability-jdbc-postgresql로
- messaging outbox header를 jsonb로(P18-new-1 — `\`로 끝나는 값이 다음 header를 삼키던 결함)
- 통합(충돌 22건): P25 A-10c(전략 선택 설정 삭제)를 service-runtime `DomainContextConfig`에 적용, P25 startup 예외 통합을
  composition 설정 테스트에, P25 `CommandUseCase.handle` 이름을 PG 통합 테스트에, P15 `EventJson`은 호출자 0이 되어 삭제
- root CLAUDE.md의 legacy seam 문장 수정(P18-new-2)

검증: ./gradlew check architectureCheck verifyDependencyLocks verifyPublicPathSnapshot :app-bootstrap:verifyEnvKeys
configContractCheck :app-bootstrap:integrationTest :sample-portfolio:integrationTest :sample-portfolio:systemTest
persistence-jpa jpaPlatformContractTest·postgresql{Idempotency,OutboxAppendTransactional,OutboxConcurrentRelay,OutboxRowLifecycle}IntegrationTest
:messaging:messaging-reliability-jdbc-postgresql:integrationTest --continue 통과(700 task)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
donghyeon-ka
2026-09-29 12:30:01 +09:00
co-authored by Claude Opus 5.5
parent b5e3c03777
commit 6397f52dc8
45 changed files with 298 additions and 532 deletions
@@ -0,0 +1,77 @@
package dev.caskeleton.messaging.inbox;
import dev.caskeleton.messaging.api.MessageId;
import dev.caskeleton.messaging.reliability.InboxRepository;
import java.time.Instant;
import java.util.Objects;
import java.util.function.Supplier;
/**
* Runs a handler's side effect at most once per message and consumer.
*
* <p>The reservation and the side effect must share one transaction. This class does not open that
* transaction itself — the caller supplies a runner that does — because the boundary belongs to the
* application's data access layer, and a nested or separate transaction here would silently break
* the guarantee while still looking correct.
*
* <p>A duplicate is not an error. It is the expected consequence of at-least-once delivery, so the
* skip path is a normal outcome rather than an exception.
*/
public final class IdempotentConsumer {
private final InboxRepository inbox;
private final TransactionRunner transactions;
/**
* Creates an idempotent consumer.
*
* @param inbox the inbox storage
* @param transactions runs work inside one database transaction
*/
public IdempotentConsumer(InboxRepository inbox, TransactionRunner transactions) {
this.inbox = Objects.requireNonNull(inbox, "inbox must not be null");
this.transactions = Objects.requireNonNull(transactions, "transactions must not be null");
}
/**
* Runs the side effect unless this message was already processed.
*
* @param <T> the side effect's result type
* @param messageId the logical message identity
* @param consumerId the consumer's stable identity
* @param now the current instant
* @param sideEffect the business work
* @return the outcome, including whether the work ran
*/
public <T> InboxOutcome<T> runOnce(
MessageId messageId, String consumerId, Instant now, Supplier<T> sideEffect) {
Objects.requireNonNull(messageId, "messageId must not be null");
Objects.requireNonNull(sideEffect, "sideEffect must not be null");
Objects.requireNonNull(now, "now must not be null");
if (consumerId == null || consumerId.isBlank()) {
throw new IllegalArgumentException("consumerId must not be blank");
}
return transactions.inTransaction(
() -> {
if (!inbox.reserve(messageId, consumerId, now)) {
return InboxOutcome.duplicate();
}
return InboxOutcome.processed(sideEffect.get());
});
}
/** Runs work inside exactly one database transaction. */
@FunctionalInterface
public interface TransactionRunner {
/**
* Runs the supplied work in a transaction and returns its result.
*
* @param <T> the result type
* @param work the work to run
* @return the work's result
*/
<T> T inTransaction(Supplier<T> work);
}
}
@@ -0,0 +1,65 @@
package dev.caskeleton.messaging.inbox;
import dev.caskeleton.messaging.reliability.InboxRepository;
import java.time.Instant;
import java.util.Objects;
/**
* Deletes inbox rows that are older than the retention policy allows.
*
* <p>Deletes in bounded batches. A single unbounded {@code DELETE} over a table that has been
* accumulating for weeks holds locks long enough to block the very reservations the inbox exists to
* serve, so the cleanup would cause the outage it is meant to prevent.
*
* <p>The policy is validated before the first deletion. Running a cleanup under a retention that is
* shorter than the redelivery window would actively create the duplicate-processing bug, so the job
* refuses to start rather than dutifully deleting the rows.
*/
public final class InboxCleanupJob {
/** How many rows one pass deletes before yielding. */
public static final int DEFAULT_BATCH_SIZE = 1_000;
private final InboxRepository inbox;
private final InboxRetentionPolicy policy;
private final int maxBatches;
/**
* Creates a cleanup job.
*
* @param inbox the inbox storage
* @param policy the retention policy
* @param maxBatches how many batches one run may delete
*/
public InboxCleanupJob(InboxRepository inbox, InboxRetentionPolicy policy, int maxBatches) {
this.inbox = Objects.requireNonNull(inbox, "inbox must not be null");
this.policy = Objects.requireNonNull(policy, "policy must not be null");
if (maxBatches < 1) {
throw new IllegalArgumentException("maxBatches must be at least 1");
}
this.maxBatches = maxBatches;
policy.validate();
}
/**
* Deletes expired rows, up to the configured batch ceiling.
*
* @param now the current instant
* @return how many rows were removed
*/
public int runOnce(Instant now) {
Objects.requireNonNull(now, "now must not be null");
Instant cutoff = policy.cutoff(now);
int removed = 0;
for (int batch = 0; batch < maxBatches; batch++) {
int deleted = inbox.purgeProcessedBefore(cutoff, DEFAULT_BATCH_SIZE);
removed += deleted;
// A short batch means the backlog is drained; continuing would just re-scan an empty range.
if (deleted < DEFAULT_BATCH_SIZE) {
break;
}
}
return removed;
}
}
@@ -0,0 +1,34 @@
package dev.caskeleton.messaging.inbox;
import java.util.Optional;
/**
* Whether a message's side effect ran or was suppressed as a duplicate.
*
* @param <T> the side effect's result type
* @param processed true when the side effect ran for the first time
* @param result the side effect's result, absent for a duplicate
*/
public record InboxOutcome<T>(boolean processed, Optional<T> result) {
/**
* Returns an outcome for a first-time delivery.
*
* @param <T> the result type
* @param value the side effect's result
* @return the processed outcome
*/
public static <T> InboxOutcome<T> processed(T value) {
return new InboxOutcome<>(true, Optional.ofNullable(value));
}
/**
* Returns an outcome for a suppressed duplicate.
*
* @param <T> the result type
* @return the duplicate outcome
*/
public static <T> InboxOutcome<T> duplicate() {
return new InboxOutcome<>(false, Optional.empty());
}
}
@@ -0,0 +1,78 @@
package dev.caskeleton.messaging.inbox;
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
/**
* How long inbox rows are kept, and the rule that makes the number safe.
*
* <p>Retention must exceed the broker's maximum redelivery window. That is not a tuning preference:
* a row pruned while the broker can still redeliver its message turns the inbox into a no-op for
* exactly that message, and the side effect runs a second time. The failure is silent, rare, and
* only happens under the conditions that already made the day bad.
*
* <p>The safety margin is multiplicative rather than additive so that it scales with the window
* itself. A stream whose redelivery window is measured in days needs more slack than one measured
* in minutes, for the same reason: the estimate of that window is proportionally less certain.
*
* @param retention how long a processed row is kept
* @param maximumRedeliveryWindow the longest the broker may redelivery a message
*/
public record InboxRetentionPolicy(Duration retention, Duration maximumRedeliveryWindow) {
/** How much longer than the redelivery window retention must be. */
public static final double REQUIRED_SAFETY_FACTOR = 2.0;
/** The default for a broker whose redelivery window is bounded by a one-day log retention. */
public static final Duration DEFAULT_RETENTION = Duration.ofDays(7);
public InboxRetentionPolicy {
Objects.requireNonNull(retention, "retention must not be null");
Objects.requireNonNull(maximumRedeliveryWindow, "maximumRedeliveryWindow must not be null");
if (retention.isNegative() || retention.isZero()) {
throw new IllegalArgumentException("retention must be positive");
}
if (maximumRedeliveryWindow.isNegative() || maximumRedeliveryWindow.isZero()) {
throw new IllegalArgumentException("maximumRedeliveryWindow must be positive");
}
}
/**
* Refuses a retention that could prune a row the broker can still redeliver.
*
* @throws MessagingConfigurationException when retention is too short
*/
public void validate() {
Duration required = required();
if (retention.compareTo(required) < 0) {
throw new MessagingConfigurationException(
"INBOX_RETENTION_TOO_SHORT",
"inbox retention of %s is below the %s required for a %s redelivery window; a pruned row"
.formatted(retention, required, maximumRedeliveryWindow)
+ " lets a late redelivery apply its effect twice");
}
}
/**
* Returns the shortest retention this redelivery window allows.
*
* @return the minimum safe retention
*/
public Duration required() {
return Duration.ofMillis(
Math.round(maximumRedeliveryWindow.toMillis() * REQUIRED_SAFETY_FACTOR));
}
/**
* Returns the cutoff before which rows may be deleted.
*
* @param now the current instant
* @return the retention cutoff
*/
public Instant cutoff(Instant now) {
Objects.requireNonNull(now, "now must not be null");
return now.minus(retention);
}
}
@@ -0,0 +1,104 @@
package dev.caskeleton.messaging.inbox;
import dev.caskeleton.messaging.api.delivery.HandleResult;
import dev.caskeleton.messaging.api.delivery.MessageDelivery;
import dev.caskeleton.messaging.api.error.FailureCategory;
import dev.caskeleton.messaging.api.error.FailureDescriptor;
import dev.caskeleton.messaging.reliability.IdempotentMessageHandler;
import dev.caskeleton.messaging.reliability.InboxResult;
import dev.caskeleton.messaging.reliability.TransactionalMessageAction;
import java.time.Instant;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.function.Supplier;
/**
* The JDBC-backed {@link IdempotentMessageHandler}.
*
* <p>Reservation and effect commit together, in the runner's single transaction. Everything else
* about this class follows from that: it cannot settle the message (settlement is not
* transactional), it cannot publish (the publish would survive a rollback), and it cannot catch and
* swallow the action's exception (the rollback is how the reservation is undone).
*
* <p>An already-reserved message returns success. Under at-least-once delivery a duplicate is the
* normal case, not an error; reporting failure would send a message whose work is complete to the
* dead-letter queue.
*
* @param <T> the payload type
*/
public final class TransactionalInboxHandler<T> implements IdempotentMessageHandler<T> {
private final IdempotentConsumer consumer;
private final Supplier<Instant> clock;
/**
* Creates a handler.
*
* @param consumer the reservation-and-effect runner
* @param clock supplies the current instant
*/
public TransactionalInboxHandler(IdempotentConsumer consumer, Supplier<Instant> clock) {
this.consumer = Objects.requireNonNull(consumer, "consumer must not be null");
this.clock = Objects.requireNonNull(clock, "clock must not be null");
}
@Override
public CompletionStage<HandleResult> handleOnce(
String consumerName, MessageDelivery<T> delivery, TransactionalMessageAction<T> action) {
Objects.requireNonNull(delivery, "delivery must not be null");
Objects.requireNonNull(action, "action must not be null");
if (consumerName == null || consumerName.isBlank()) {
throw new IllegalArgumentException("consumerName must not be blank");
}
try {
InboxOutcome<InboxResult> outcome =
consumer.runOnce(
delivery.message().messageId(),
consumerName,
clock.get(),
() -> {
try {
action.apply(delivery);
} catch (Exception failure) {
// Wrapped, not swallowed: the transaction runner has to see a throw to roll the
// reservation back along with the effect.
throw new ActionFailedException(failure);
}
return InboxResult.APPLIED;
});
return CompletableFuture.completedFuture(
outcome.processed() ? HandleResult.success() : duplicateIsSuccess());
} catch (ActionFailedException failure) {
return CompletableFuture.completedFuture(retryable(failure.getCause()));
}
}
private static HandleResult duplicateIsSuccess() {
// The effect already ran in an earlier delivery. Settling is correct; redelivering is not.
return HandleResult.success();
}
private static HandleResult retryable(Throwable cause) {
return new HandleResult.Retry(
new FailureDescriptor(
FailureCategory.TRANSIENT_INFRASTRUCTURE,
"INBOX_ACTION_FAILED",
true,
"the transactional action failed and its reservation was rolled back",
Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())));
}
/** Carries the action's checked exception out through the transaction runner. */
private static final class ActionFailedException extends RuntimeException {
private static final long serialVersionUID = 1L;
private ActionFailedException(Throwable cause) {
super(cause);
}
}
}
@@ -0,0 +1,62 @@
package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.reliability.OutboxRepository;
import java.time.Instant;
import java.util.Objects;
/**
* Deletes published outbox rows once they are past their retention.
*
* <p>Only {@code PUBLISHED} rows are eligible, and that restriction is the whole safety story. An
* {@code AMBIGUOUS} row is a message that may or may not have reached the broker; deleting it
* destroys the only record that it was ever meant to be sent. A {@code FAILED} row is the audit
* trail of a message that never went out. Both are the rows an operator most needs during an
* incident, which is exactly when a time-based sweep would otherwise remove them.
*/
public final class OutboxCleanupJob {
/** How many rows one delete statement removes before yielding. */
public static final int DEFAULT_BATCH_SIZE = 1_000;
private final OutboxRepository outbox;
private final OutboxProperties properties;
private final int maxBatches;
/**
* Creates a cleanup job.
*
* @param outbox the outbox storage
* @param properties the relay settings supplying the retention
* @param maxBatches how many delete batches one run may issue
*/
public OutboxCleanupJob(OutboxRepository outbox, OutboxProperties properties, int maxBatches) {
this.outbox = Objects.requireNonNull(outbox, "outbox must not be null");
this.properties = Objects.requireNonNull(properties, "properties must not be null");
if (maxBatches < 1) {
throw new IllegalArgumentException("maxBatches must be at least 1");
}
this.maxBatches = maxBatches;
}
/**
* Deletes expired published rows.
*
* @param now the current instant
* @return how many rows were removed
*/
public int runOnce(Instant now) {
Objects.requireNonNull(now, "now must not be null");
Instant cutoff = now.minus(properties.retentionAfterPublish());
int removed = 0;
for (int batch = 0; batch < maxBatches; batch++) {
int deleted = outbox.purgePublishedBefore(cutoff, DEFAULT_BATCH_SIZE);
removed += deleted;
// A short batch means the backlog is drained; asking again would re-scan an empty range.
if (deleted < DEFAULT_BATCH_SIZE) {
break;
}
}
return removed;
}
}
@@ -0,0 +1,124 @@
package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.api.MessageEnvelope;
import dev.caskeleton.messaging.api.ProducerId;
import dev.caskeleton.messaging.api.SchemaVersion;
import dev.caskeleton.messaging.api.header.HeaderName;
import dev.caskeleton.messaging.api.header.HeaderValue;
import dev.caskeleton.messaging.api.header.MessageHeaders;
import dev.caskeleton.messaging.api.header.ReservedHeaders;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.schema.EncodedMessage;
import dev.caskeleton.messaging.schema.SchemaReference;
import java.net.URI;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
/**
* Rebuilds a publishable envelope from an outbox row.
*
* <p>The identity comes from the row, never from a fresh mint. That is what makes a retry after an
* ambiguous publish safe: the broker's deduplication, or the consumer's Inbox, sees the same id and
* collapses the duplicate.
*
* <p>So does everything else the envelope carries. This used to rebuild correlation, causation,
* tenant, trace and the schema reference as empty, and read the routing keys out of the row's
* header map — so a message that travelled through the outbox reached its consumer with less
* provenance than one published directly, and the publish path became part of the message's
* meaning. The row now has columns for those facts and this reads them.
*
* <p>Reserved header names in the row are refused outright, with no exception for the routing keys.
* They had one, because the keys had nowhere else to live; a row could therefore write {@code
* msg.partition-key} and the same code path would have accepted {@code msg.id} if the check had
* been one line looser. Now that the keys are columns, the rule is the simple one: an outbox row
* cannot write into the platform's namespace at all.
*/
public final class OutboxEnvelopeFactory {
private final ProducerId producer;
/**
* Creates a factory for one service.
*
* @param producer the logical producing service, used for rows that name none
*/
public OutboxEnvelopeFactory(ProducerId producer) {
this.producer = Objects.requireNonNull(producer, "producer must not be null");
}
/**
* Builds the envelope for one outbox row.
*
* @param record the outbox row
* @return the encoded envelope
*/
public MessageEnvelope<EncodedMessage> toEnvelope(OutboxRecord record) {
Objects.requireNonNull(record, "record must not be null");
Map<HeaderName, HeaderValue> headers = new LinkedHashMap<>();
record
.headers()
.forEach(
(name, value) -> {
// The row's headers are application data. Promoting them straight into
// MessageHeaders.platform let a row carrying `msg.id` or `msg.type` reach the wire
// as if the platform had written it — and the consumer trusts the last such header,
// so a forged identity lands in another message's inbox deduplication.
if (ReservedHeaders.isReserved(name)) {
throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException(
"RESERVED_HEADER_IN_OUTBOX_ROW",
"outbox row header '"
+ name
+ "' uses the reserved namespace; canonical identity and metadata come from"
+ " the record's own columns, not from its header map");
}
headers.put(new HeaderName(name), new HeaderValue(value));
});
OutboxCanonicalMetadata metadata = record.metadata();
return new MessageEnvelope<>(
record.messageId(),
record.messageType(),
record.schemaVersion(),
record.createdAt(),
// The write instant stands in when the row records no separate occurrence. It is the
// closest true statement available: the business transaction that wrote the row is the one
// the fact occurred in.
Optional.of(metadata.occurredAt().orElse(record.createdAt())),
// A row written before the producer column existed names none, and the service that owns
// this relay is the only producer it can honestly be attributed to.
metadata.producer().orElse(producer),
metadata.correlationId(),
metadata.causationId(),
record.contentType(),
metadata.partitionKey(),
metadata.orderingKey(),
metadata.tenant(),
metadata.traceContext(),
MessageHeaders.platform(headers),
new EncodedMessage(
record.payload(), record.contentType(), schemaReference(record, metadata)));
}
/**
* Rebuilds the schema reference from the row's subject, the row's version, and its location.
*
* <p>The version is the row's {@code schema_version} column rather than a third copy of the same
* number: a reference whose version could disagree with the column the relay filters on is a
* reference nobody can trust.
*/
private static Optional<SchemaReference> schemaReference(
OutboxRecord record, OutboxCanonicalMetadata metadata) {
return metadata
.schemaSubject()
.map(
subject ->
new SchemaReference(
subject,
new SchemaVersion(record.schemaVersion().value()),
metadata.schemaUri().map(URI::create)));
}
}
@@ -0,0 +1,81 @@
package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
import java.time.Duration;
import java.util.Objects;
/**
* The relay's operational settings, and the relationships between them that have to hold.
*
* <p>The lease duration is the dangerous one. If it is shorter than the time a publish can take, a
* second relay claims the row while the first is still waiting for a confirm, and the message is
* published twice — under the same id, so consumers with an inbox survive it, but consumers without
* one do not. The constructor therefore requires the lease to exceed the publish timeout by a
* margin rather than merely to be positive.
*
* @param batchSize how many records one lease claims
* @param leaseDuration how long a claim holds
* @param publishTimeout the deadline for one publish
* @param pollInterval how often the relay looks for work
* @param retentionAfterPublish how long published rows are kept before deletion
* @param maxAttempts how many times a record is retried before it is parked
*/
public record OutboxProperties(
int batchSize,
Duration leaseDuration,
Duration publishTimeout,
Duration pollInterval,
Duration retentionAfterPublish,
int maxAttempts) {
/** How much longer than the publish timeout a lease must be. */
public static final double REQUIRED_LEASE_FACTOR = 2.0;
public OutboxProperties {
Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
Objects.requireNonNull(publishTimeout, "publishTimeout must not be null");
Objects.requireNonNull(pollInterval, "pollInterval must not be null");
Objects.requireNonNull(retentionAfterPublish, "retentionAfterPublish must not be null");
if (batchSize < 1) {
throw new IllegalArgumentException("batchSize must be at least 1");
}
if (maxAttempts < 1) {
throw new IllegalArgumentException("maxAttempts must be at least 1");
}
requirePositive(leaseDuration, "leaseDuration");
requirePositive(publishTimeout, "publishTimeout");
requirePositive(pollInterval, "pollInterval");
requirePositive(retentionAfterPublish, "retentionAfterPublish");
Duration minimumLease =
Duration.ofMillis(Math.round(publishTimeout.toMillis() * REQUIRED_LEASE_FACTOR));
if (leaseDuration.compareTo(minimumLease) < 0) {
throw new MessagingConfigurationException(
"OUTBOX_LEASE_TOO_SHORT",
"a lease of %s can expire while a %s publish is still in flight, letting a second relay"
.formatted(leaseDuration, publishTimeout)
+ " publish the same record; the minimum is %s".formatted(minimumLease));
}
}
/**
* Returns settings suited to a single relay against a healthy broker.
*
* @return the default properties
*/
public static OutboxProperties defaults() {
return new OutboxProperties(
100,
Duration.ofSeconds(30),
Duration.ofSeconds(5),
Duration.ofMillis(500),
Duration.ofDays(3),
10);
}
private static void requirePositive(Duration value, String field) {
if (value.isNegative() || value.isZero()) {
throw new IllegalArgumentException(field + " must be positive");
}
}
}
@@ -0,0 +1,277 @@
package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.api.MessageEnvelope;
import dev.caskeleton.messaging.api.destination.MessageDestination;
import dev.caskeleton.messaging.api.error.FailureCategory;
import dev.caskeleton.messaging.api.error.FailureDescriptor;
import dev.caskeleton.messaging.api.error.MessagingException;
import dev.caskeleton.messaging.api.publish.MessagePublisher;
import dev.caskeleton.messaging.api.publish.PublishCompletion;
import dev.caskeleton.messaging.api.publish.PublishEvidence;
import dev.caskeleton.messaging.api.publish.PublishOptions;
import dev.caskeleton.messaging.api.publish.PublishResult;
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
import dev.caskeleton.messaging.reliability.OutboxLease;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.reliability.OutboxRepository;
import dev.caskeleton.messaging.reliability.OutboxTransitionResult;
import dev.caskeleton.messaging.schema.EncodedMessage;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
/**
* Publishes outbox rows, treating an unknown outcome as retryable rather than final.
*
* <p>The relay's correctness rests on one rule: an ambiguous publish is retried <em>under the same
* message id</em>. Minting a new id would turn a possibly-delivered message into a
* definitely-second message, and no downstream deduplication could recover from it. Marking it
* failed instead would lose a message the broker may already hold.
*
* <p>The relay therefore guarantees at-least-once publication and nothing more. Effectively-once
* downstream effects come from pairing it with an Inbox — which is why the platform never
* advertises the outbox as exactly-once.
*/
public final class OutboxRelay {
private final OutboxRepository repository;
private final MessagePublisher publisher;
private final OutboxEnvelopeFactory envelopeFactory;
private final int batchSize;
private final String owner;
private final OutboxRetryScheduler scheduler;
private static String defaultOwner() {
return java.lang.ProcessHandle.current().pid()
+ "@"
+ java.util.UUID.randomUUID().toString().substring(0, 8);
}
private final Duration leaseDuration;
/**
* Creates a relay.
*
* @param repository the outbox storage
* @param publisher the platform publisher
* @param envelopeFactory rebuilds envelopes from rows
* @param batchSize how many rows to claim per pass
* @param leaseDuration how long a claim holds
*/
public OutboxRelay(
OutboxRepository repository,
MessagePublisher publisher,
OutboxEnvelopeFactory envelopeFactory,
int batchSize,
Duration leaseDuration) {
this(repository, publisher, envelopeFactory, batchSize, leaseDuration, defaultOwner());
}
/**
* A relay with an explicit owner identity.
*
* <p>The owner is half of the fencing predicate. It defaults to something stable per process
* rather than per object, because two relays in one JVM sharing an owner would each be able to
* overwrite the other's terminal writes — the exact failure the token exists to prevent.
*/
public OutboxRelay(
OutboxRepository repository,
MessagePublisher publisher,
OutboxEnvelopeFactory envelopeFactory,
int batchSize,
Duration leaseDuration,
String owner) {
this(
repository,
publisher,
envelopeFactory,
batchSize,
leaseDuration,
owner,
OutboxRetryScheduler.standard());
}
/**
* A relay with an explicit retry policy and a process-derived owner.
*
* @param repository the outbox storage
* @param publisher the platform publisher
* @param envelopeFactory rebuilds envelopes from rows
* @param batchSize how many rows to claim per pass
* @param leaseDuration how long a claim holds
* @param scheduler the retry policy
*/
public OutboxRelay(
OutboxRepository repository,
MessagePublisher publisher,
OutboxEnvelopeFactory envelopeFactory,
int batchSize,
Duration leaseDuration,
OutboxRetryScheduler scheduler) {
this(
repository,
publisher,
envelopeFactory,
batchSize,
leaseDuration,
defaultOwner(),
scheduler);
}
/**
* A relay with an explicit retry policy.
*
* <p>The scheduler was built by the auto-configuration and handed to nobody: the relay claimed
* every ambiguous row on the next pass regardless of its attempt count, so the configured budget
* of ten attempts was a number nothing read.
*/
public OutboxRelay(
OutboxRepository repository,
MessagePublisher publisher,
OutboxEnvelopeFactory envelopeFactory,
int batchSize,
Duration leaseDuration,
String owner,
OutboxRetryScheduler scheduler) {
this.owner = Objects.requireNonNull(owner, "owner must not be null");
this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null");
this.repository = Objects.requireNonNull(repository, "repository must not be null");
this.publisher = Objects.requireNonNull(publisher, "publisher must not be null");
this.envelopeFactory =
Objects.requireNonNull(envelopeFactory, "envelopeFactory must not be null");
this.leaseDuration = Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
if (batchSize < 1) {
throw new IllegalArgumentException("batchSize must be at least 1");
}
this.batchSize = batchSize;
}
/**
* Runs one relay pass.
*
* @param now the current instant
* @return what the pass did
*/
public OutboxRelayReport runOnce(Instant now) {
Objects.requireNonNull(now, "now must not be null");
// The claim carries an owner and a fencing token, and every write
// below is conditioned on them. Without that, a relay that stalled past its lease could record
// AMBIGUOUS over the PUBLISHED another relay had already written, making a confirmed message
// claimable again.
List<OutboxLease> batch =
repository.claimBatch(owner, batchSize, leaseDuration, now, scheduler.maxAttempts());
int published = 0;
int ambiguous = 0;
int failed = 0;
int stale = 0;
int exhausted = 0;
for (OutboxLease lease : batch) {
OutboxRecord record = lease.record();
MessageEnvelope<EncodedMessage> envelope;
try {
envelope = envelopeFactory.toEnvelope(record);
} catch (RuntimeException unpublishable) {
// A row that cannot become an envelope never will. Thrown out of the pass, it took the
// rest of the batch with it: those rows stayed IN_FLIGHT until their lease expired, and the
// same row was claimed and thrown again on the next pass, forever.
if (repository.markFailed(lease, failureCode(unpublishable), now)
== OutboxTransitionResult.APPLIED) {
failed++;
} else {
stale++;
}
continue;
}
PublishResult result = publish(record, envelope);
switch (result.completion()) {
case CONFIRMED -> {
if (repository.markPublished(lease, now) == OutboxTransitionResult.APPLIED) {
published++;
} else {
stale++;
}
}
case AMBIGUOUS -> {
String code = result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS");
// The attempt this pass just spent. The claim predicate and the row both count attempts
// after the transition, so the budget has to be judged on the same number the next claim
// will read, or the last attempt is spent twice.
int spent = record.attempts() + 1;
OutboxTransitionResult transition =
scheduler
.parkReason(spent)
// A row that has spent its budget without an answer is parked under its own
// status. Leaving it AMBIGUOUS makes it a row the claim predicate silently skips
// forever, which looks identical to a healthy backlog on every dashboard.
.map(reason -> repository.markExhausted(lease, reason, now))
.orElseGet(
() ->
repository.markAmbiguous(
lease, code, now, scheduler.nextAttemptAt(now, spent)));
if (transition == OutboxTransitionResult.APPLIED) {
if (scheduler.isExhausted(spent)) {
exhausted++;
} else {
ambiguous++;
}
} else {
stale++;
}
}
case REJECTED -> {
if (repository.markFailed(
lease, result.failure().map(failure -> failure.code()).orElse("REJECTED"), now)
== OutboxTransitionResult.APPLIED) {
failed++;
} else {
stale++;
}
}
// Unreachable while PublishCompletion has exactly these constants. Present so that a new
// completion fails loudly rather than leaving the row IN_FLIGHT with its lease ticking —
// which would look like a stalled relay rather than an unhandled case.
default ->
throw new IllegalStateException("unhandled publish completion: " + result.completion());
}
}
return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale, exhausted);
}
private PublishResult publish(OutboxRecord record, MessageEnvelope<EncodedMessage> envelope) {
MessageDestination<EncodedMessage> destination =
new MessageDestination<>(record.destination(), record.messageType(), EncodedMessage.class);
try {
return publisher
.publish(destination, envelope, PublishOptions.defaults())
.toCompletableFuture()
.join();
} catch (RuntimeException unknown) {
// An exception says nothing about whether the broker took the message, so the row is retried
// under the same id like any other unknown outcome — not failed, not left IN_FLIGHT.
return new PublishResult(
PublishCompletion.AMBIGUOUS,
PublishEvidence.ambiguous(),
RoutingOutcome.UNKNOWN,
Optional.empty(),
1,
Duration.ZERO,
Optional.of(
FailureDescriptor.of(
FailureCategory.AMBIGUOUS,
"PUBLISH_OUTCOME_UNKNOWN",
"the publisher threw " + unknown.getClass().getSimpleName())));
}
}
private static String failureCode(RuntimeException failure) {
return failure instanceof MessagingException known
? known.failure().code()
: "OUTBOX_ROW_UNPUBLISHABLE";
}
}
@@ -0,0 +1,50 @@
package dev.caskeleton.messaging.outbox;
/**
* What one relay pass did.
*
* <p>Ambiguous rows are counted separately from failures because they mean something operationally
* different: a rising ambiguous count is a broker confirmation problem, while a rising failed count
* is a contract or topology problem.
*
* <p>{@code staleLeases} is the count of terminal writes this pass could not apply because another
* relay had already taken the row. It is not an error and it is not a success: it means this worker
* published something whose outcome another worker has already recorded, so a non-zero value is the
* visible form of a duplicate publication.
*
* <p>{@code exhausted} is counted apart from {@code failed} for the same reason: a rejected row is
* a message the broker refused and an exhausted one is a message nobody ever answered for. The
* first needs a fix, the second needs a redrive, and a single counter for both tells an operator
* neither.
*
* @param leased how many rows were claimed
* @param published how many were confirmed
* @param ambiguous how many had an unknown outcome and stay retryable
* @param failed how many were definitively rejected
* @param staleLeases how many terminal writes were refused because the lease had been superseded
* @param exhausted how many spent their attempt budget without a confirmation and were parked
*/
public record OutboxRelayReport(
int leased, int published, int ambiguous, int failed, int staleLeases, int exhausted) {
public OutboxRelayReport {
if (leased < 0
|| published < 0
|| ambiguous < 0
|| failed < 0
|| staleLeases < 0
|| exhausted < 0) {
throw new IllegalArgumentException("relay counters must not be negative");
}
}
/** A report from a pass that parked nothing. */
public OutboxRelayReport(int leased, int published, int ambiguous, int failed, int staleLeases) {
this(leased, published, ambiguous, failed, staleLeases, 0);
}
/** A report from a pass that observed no stale leases. */
public OutboxRelayReport(int leased, int published, int ambiguous, int failed) {
this(leased, published, ambiguous, failed, 0, 0);
}
}
@@ -0,0 +1,206 @@
package dev.caskeleton.messaging.outbox;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
/**
* Runs relay passes on the schedule the retry policy decides.
*
* <p>The relay, its retry scheduler and the attempt budget all existed and nothing ever called
* {@code runOnce}. An outbox whose relay is never driven is the worst shape of all: the business
* transaction commits its row and reports success, and the message is never published — a silent,
* unbounded backlog rather than a visible failure.
*
* <p>Passes are self-scheduled rather than run at a fixed rate. A fixed rate keeps firing while the
* broker is down, and the pass after an empty or failed one is exactly where the policy's backoff
* belongs; a pass that published something resets to the base interval because work is flowing.
*
* <p>One thread, and the next pass is scheduled only after the previous returns. Overlapping passes
* from one process would have two claims racing on the same rows, which the fencing predicate would
* then reject as stale — correct, but it is duplicate publishing work bought at the cost of a
* duplicate publish.
*/
public final class OutboxRelayWorker implements AutoCloseable {
/** How long {@link #close()} waits for a running pass when no deadline is given. */
public static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30);
private static final System.Logger LOG = System.getLogger(OutboxRelayWorker.class.getName());
private final OutboxRelay relay;
private final OutboxRetryScheduler scheduler;
private final Supplier<Instant> clock;
private final ScheduledExecutorService passes;
private final AtomicBoolean running = new AtomicBoolean();
private final AtomicInteger unproductivePasses = new AtomicInteger();
private final AtomicLong completedPasses = new AtomicLong();
private final AtomicLong failedPasses = new AtomicLong();
/**
* Creates a worker driving one relay.
*
* @param relay the relay to run
* @param scheduler the policy deciding when the next pass runs
*/
public OutboxRelayWorker(OutboxRelay relay, OutboxRetryScheduler scheduler) {
this(relay, scheduler, Instant::now);
}
/**
* Creates a worker with an explicit clock, so a pass can be driven without waiting on wall time.
*
* @param relay the relay to run
* @param scheduler the policy deciding when the next pass runs
* @param clock the source of the current instant
*/
public OutboxRelayWorker(
OutboxRelay relay, OutboxRetryScheduler scheduler, Supplier<Instant> clock) {
this.relay = Objects.requireNonNull(relay, "relay must not be null");
this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null");
this.clock = Objects.requireNonNull(clock, "clock must not be null");
ScheduledThreadPoolExecutor executor =
new ScheduledThreadPoolExecutor(
1,
runnable -> {
// Daemon: a relay thread that outlives the application would keep publishing from a
// process that has already released its other resources.
Thread thread = new Thread(runnable, "messaging-outbox-relay");
thread.setDaemon(true);
return thread;
});
// A pass waiting out its backoff is dropped on shutdown rather than run afterwards. Left on,
// the default policy makes stop() wait for a pass that has not started and cannot do anything
// useful, so an orderly shutdown pays the whole poll interval before it can close.
executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
this.passes = executor;
}
/** Starts driving passes. Idempotent: a second call does not add a second schedule. */
public void start() {
if (running.compareAndSet(false, true)) {
schedule(Duration.ZERO);
}
}
/**
* Stops scheduling and lets the pass that is already running finish.
*
* <p>Draining rather than interrupting is the whole point. A pass killed between its claim and
* its terminal write leaves rows {@code IN_FLIGHT} holding a lease, and nothing may touch them
* until that lease expires — so an orderly shutdown would produce exactly the stall that a crash
* produces.
*
* @param deadline how long to wait for the running pass
* @return true when the running pass finished inside the deadline
*/
public boolean stop(Duration deadline) {
Objects.requireNonNull(deadline, "deadline must not be null");
running.set(false);
passes.shutdown();
try {
if (passes.awaitTermination(deadline.toMillis(), TimeUnit.MILLISECONDS)) {
return true;
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
// The deadline passed with a pass still running. Its rows stay leased until the lease expires,
// which is the outcome the deadline exists to bound rather than to prevent.
passes.shutdownNow();
return false;
}
@Override
public void close() {
stop(DEFAULT_DRAIN_DEADLINE);
}
/**
* Reports whether the worker is scheduling passes.
*
* @return true between {@link #start()} and {@link #stop(Duration)}
*/
public boolean isRunning() {
return running.get();
}
/**
* Returns how many passes have finished, successfully or not.
*
* @return the completed pass count
*/
public long completedPasses() {
return completedPasses.get();
}
/**
* Returns how many passes ended in an exception.
*
* @return the failed pass count
*/
public long failedPasses() {
return failedPasses.get();
}
// The scheduled future is not kept: a pass reports nothing to a caller and handles its own
// failure below, so the only thing holding it would achieve is a reference that has to be
// cleaned up.
@SuppressWarnings("FutureReturnValueIgnored")
private void schedule(Duration delay) {
if (!running.get()) {
return;
}
try {
passes.schedule(this::runPass, Math.max(0, delay.toMillis()), TimeUnit.MILLISECONDS);
} catch (RejectedExecutionException stopping) {
// stop() shut the executor between the running check and the submission. There is nothing to
// schedule onto and nothing to report: the worker is stopping, which is what was asked for.
running.set(false);
}
}
private void runPass() {
if (!running.get()) {
return;
}
int unproductive;
try {
OutboxRelayReport report = relay.runOnce(clock.get());
unproductive = report.published() > 0 ? reset() : unproductivePasses.incrementAndGet();
} catch (RuntimeException passFailed) {
// A failed pass must not stop the loop: the scheduled task's own exception would cancel every
// future pass, turning one broker error into a relay that never runs again. The failure is
// counted and the next pass backs off as if nothing was published, which is true. It is also
// logged: counted silently, a relay failing every pass looked exactly like an idle one.
long failures = failedPasses.incrementAndGet();
unproductive = unproductivePasses.incrementAndGet();
LOG.log(
System.Logger.Level.WARNING,
"outbox relay pass failed (" + failures + " failed so far); retrying with backoff",
passFailed);
}
completedPasses.incrementAndGet();
schedule(scheduler.backoff(unproductive));
}
private int reset() {
unproductivePasses.set(0);
return 0;
}
}
@@ -0,0 +1,134 @@
package dev.caskeleton.messaging.outbox;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.Optional;
/**
* Decides when a relay's next pass should run, and when a record has retried enough.
*
* <p>The same curve serves two different jobs and neither replaces the other. Backing off the
* <em>pass</em> is what lets a broker recover: an outage fails every record in the batch for the
* same reason, so retrying the whole backlog on the poll interval achieves nothing but load. But a
* pass-level wait is process memory — it is forgotten on restart and each replica keeps its own —
* so eligibility also has to be written on the row, or the backlog is retried as fast as the
* unluckiest replica polls.
*
* <p>Jitter is applied deterministically from the attempt count rather than randomly. Several relay
* instances that all started at deployment time would otherwise synchronise their retries into a
* thundering herd, and a random source would make the schedule impossible to test.
*/
public final class OutboxRetryScheduler {
private final Duration baseInterval;
private final Duration maxInterval;
private final int maxAttempts;
/**
* Creates a scheduler from the relay properties.
*
* @param properties the relay settings
* @param maxInterval the longest backoff between passes
*/
public OutboxRetryScheduler(OutboxProperties properties, Duration maxInterval) {
Objects.requireNonNull(properties, "properties must not be null");
Objects.requireNonNull(maxInterval, "maxInterval must not be null");
if (maxInterval.compareTo(properties.pollInterval()) < 0) {
throw new IllegalArgumentException("maxInterval must not be shorter than the poll interval");
}
this.baseInterval = properties.pollInterval();
this.maxInterval = maxInterval;
this.maxAttempts = properties.maxAttempts();
}
/**
* Returns when the next pass should run.
*
* <p>A pass that published something resets to the base interval: work is flowing, so there is no
* reason to wait.
*
* @param now the current instant
* @param consecutiveEmptyOrFailedPasses how many passes in a row did no useful work
* @return the next pass instant
*/
public Instant nextPassAt(Instant now, int consecutiveEmptyOrFailedPasses) {
Objects.requireNonNull(now, "now must not be null");
if (consecutiveEmptyOrFailedPasses < 0) {
throw new IllegalArgumentException("consecutiveEmptyOrFailedPasses must not be negative");
}
return now.plus(backoff(consecutiveEmptyOrFailedPasses));
}
/**
* Returns when a row that has spent this many attempts may be claimed again.
*
* <p>Written onto the row rather than held in the relay, because a deadline that lives in process
* memory is forgotten on restart and computed independently by every replica — which turns a
* backoff into every worker re-publishing the same backlog as fast as it polls.
*
* @param now the current instant
* @param attempts how many attempts the row has spent
* @return the earliest instant the row is eligible again
*/
public Instant nextAttemptAt(Instant now, int attempts) {
Objects.requireNonNull(now, "now must not be null");
if (attempts < 1) {
throw new IllegalArgumentException("a row that has been published once has spent an attempt");
}
return now.plus(backoff(attempts));
}
/** The platform's default policy, for a relay whose caller did not supply one. */
public static OutboxRetryScheduler standard() {
return new OutboxRetryScheduler(OutboxProperties.defaults(), java.time.Duration.ofMinutes(5));
}
/** The attempt budget a row may spend before it is parked. */
public int maxAttempts() {
return maxAttempts;
}
/**
* Returns the delay before the next pass.
*
* @param consecutiveEmptyOrFailedPasses how many passes in a row did no useful work
* @return the backoff delay
*/
public Duration backoff(int consecutiveEmptyOrFailedPasses) {
if (consecutiveEmptyOrFailedPasses <= 0) {
return baseInterval;
}
// Doubling, capped. Shifting rather than Math.pow so an unbounded counter cannot overflow into
// a negative duration on a long outage.
int exponent = Math.min(consecutiveEmptyOrFailedPasses, 20);
long scaled = baseInterval.toMillis() << exponent;
long capped = Math.min(scaled, maxInterval.toMillis());
long jittered = capped - (capped / 8) * (exponent % 3);
return Duration.ofMillis(Math.max(baseInterval.toMillis(), jittered));
}
/**
* Reports whether a record has exhausted its attempt budget.
*
* @param attempts how many times the record has been published
* @return true when it should be parked rather than retried
*/
public boolean isExhausted(int attempts) {
return attempts >= maxAttempts;
}
/**
* Returns the reason a record should be parked, when it should.
*
* @param attempts how many times the record has been published
* @return the sanitized reason, or empty when the record may still retry
*/
public Optional<String> parkReason(int attempts) {
return isExhausted(attempts)
? Optional.of(
"the record reached its %d attempt budget without a confirmed publish"
.formatted(maxAttempts))
: Optional.empty();
}
}