review-followup Wave 3 라운드 2(P18): outbox canonical 통일, composition 모듈, legacy Kafka seam 삭제
- 결정 7: `LegacyOutboxAppendPort`·legacy 저장 모델·relay·Kafka R0 seam 삭제, 템플릿 JSON encoder(`JsonIntegrationEventEncoder`),
durable operation·sample 이벤트가 `OutboxAppendPort` + encoder로, persistence-jpa V19(pre-canonical 미발행 row 거부)
- 결정 3: outbox relay·멱등성 executor 조립을 `composition:outbox-relay`·`composition:idempotency`로,
sample이 복사한 bootstrap 인프라를 `composition:service-runtime`으로(sample `SampleIdempotencyConfig` 복사본 삭제)
- persistence-jpa outbox·멱등성 패키지 feature 기준 이동, messaging outbox·inbox leaf를 reliability / reliability-jdbc-postgresql로
- messaging outbox header를 jsonb로(P18-new-1 — `\`로 끝나는 값이 다음 header를 삼키던 결함)
- 통합(충돌 22건): P25 A-10c(전략 선택 설정 삭제)를 service-runtime `DomainContextConfig`에 적용, P25 startup 예외 통합을
composition 설정 테스트에, P25 `CommandUseCase.handle` 이름을 PG 통합 테스트에, P15 `EventJson`은 호출자 0이 되어 삭제
- root CLAUDE.md의 legacy seam 문장 수정(P18-new-2)
검증: ./gradlew check architectureCheck verifyDependencyLocks verifyPublicPathSnapshot :app-bootstrap:verifyEnvKeys
configContractCheck :app-bootstrap:integrationTest :sample-portfolio:integrationTest :sample-portfolio:systemTest
persistence-jpa jpaPlatformContractTest·postgresql{Idempotency,OutboxAppendTransactional,OutboxConcurrentRelay,OutboxRowLifecycle}IntegrationTest
:messaging:messaging-reliability-jdbc-postgresql:integrationTest --continue 통과(700 task)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b5e3c03777
commit
6397f52dc8
+77
@@ -0,0 +1,77 @@
|
||||
package dev.caskeleton.messaging.inbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.reliability.InboxRepository;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Runs a handler's side effect at most once per message and consumer.
|
||||
*
|
||||
* <p>The reservation and the side effect must share one transaction. This class does not open that
|
||||
* transaction itself — the caller supplies a runner that does — because the boundary belongs to the
|
||||
* application's data access layer, and a nested or separate transaction here would silently break
|
||||
* the guarantee while still looking correct.
|
||||
*
|
||||
* <p>A duplicate is not an error. It is the expected consequence of at-least-once delivery, so the
|
||||
* skip path is a normal outcome rather than an exception.
|
||||
*/
|
||||
public final class IdempotentConsumer {
|
||||
|
||||
private final InboxRepository inbox;
|
||||
private final TransactionRunner transactions;
|
||||
|
||||
/**
|
||||
* Creates an idempotent consumer.
|
||||
*
|
||||
* @param inbox the inbox storage
|
||||
* @param transactions runs work inside one database transaction
|
||||
*/
|
||||
public IdempotentConsumer(InboxRepository inbox, TransactionRunner transactions) {
|
||||
this.inbox = Objects.requireNonNull(inbox, "inbox must not be null");
|
||||
this.transactions = Objects.requireNonNull(transactions, "transactions must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the side effect unless this message was already processed.
|
||||
*
|
||||
* @param <T> the side effect's result type
|
||||
* @param messageId the logical message identity
|
||||
* @param consumerId the consumer's stable identity
|
||||
* @param now the current instant
|
||||
* @param sideEffect the business work
|
||||
* @return the outcome, including whether the work ran
|
||||
*/
|
||||
public <T> InboxOutcome<T> runOnce(
|
||||
MessageId messageId, String consumerId, Instant now, Supplier<T> sideEffect) {
|
||||
Objects.requireNonNull(messageId, "messageId must not be null");
|
||||
Objects.requireNonNull(sideEffect, "sideEffect must not be null");
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
if (consumerId == null || consumerId.isBlank()) {
|
||||
throw new IllegalArgumentException("consumerId must not be blank");
|
||||
}
|
||||
|
||||
return transactions.inTransaction(
|
||||
() -> {
|
||||
if (!inbox.reserve(messageId, consumerId, now)) {
|
||||
return InboxOutcome.duplicate();
|
||||
}
|
||||
return InboxOutcome.processed(sideEffect.get());
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs work inside exactly one database transaction. */
|
||||
@FunctionalInterface
|
||||
public interface TransactionRunner {
|
||||
|
||||
/**
|
||||
* Runs the supplied work in a transaction and returns its result.
|
||||
*
|
||||
* @param <T> the result type
|
||||
* @param work the work to run
|
||||
* @return the work's result
|
||||
*/
|
||||
<T> T inTransaction(Supplier<T> work);
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package dev.caskeleton.messaging.inbox;
|
||||
|
||||
import dev.caskeleton.messaging.reliability.InboxRepository;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Deletes inbox rows that are older than the retention policy allows.
|
||||
*
|
||||
* <p>Deletes in bounded batches. A single unbounded {@code DELETE} over a table that has been
|
||||
* accumulating for weeks holds locks long enough to block the very reservations the inbox exists to
|
||||
* serve, so the cleanup would cause the outage it is meant to prevent.
|
||||
*
|
||||
* <p>The policy is validated before the first deletion. Running a cleanup under a retention that is
|
||||
* shorter than the redelivery window would actively create the duplicate-processing bug, so the job
|
||||
* refuses to start rather than dutifully deleting the rows.
|
||||
*/
|
||||
public final class InboxCleanupJob {
|
||||
|
||||
/** How many rows one pass deletes before yielding. */
|
||||
public static final int DEFAULT_BATCH_SIZE = 1_000;
|
||||
|
||||
private final InboxRepository inbox;
|
||||
private final InboxRetentionPolicy policy;
|
||||
private final int maxBatches;
|
||||
|
||||
/**
|
||||
* Creates a cleanup job.
|
||||
*
|
||||
* @param inbox the inbox storage
|
||||
* @param policy the retention policy
|
||||
* @param maxBatches how many batches one run may delete
|
||||
*/
|
||||
public InboxCleanupJob(InboxRepository inbox, InboxRetentionPolicy policy, int maxBatches) {
|
||||
this.inbox = Objects.requireNonNull(inbox, "inbox must not be null");
|
||||
this.policy = Objects.requireNonNull(policy, "policy must not be null");
|
||||
if (maxBatches < 1) {
|
||||
throw new IllegalArgumentException("maxBatches must be at least 1");
|
||||
}
|
||||
this.maxBatches = maxBatches;
|
||||
policy.validate();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes expired rows, up to the configured batch ceiling.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return how many rows were removed
|
||||
*/
|
||||
public int runOnce(Instant now) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
Instant cutoff = policy.cutoff(now);
|
||||
|
||||
int removed = 0;
|
||||
for (int batch = 0; batch < maxBatches; batch++) {
|
||||
int deleted = inbox.purgeProcessedBefore(cutoff, DEFAULT_BATCH_SIZE);
|
||||
removed += deleted;
|
||||
// A short batch means the backlog is drained; continuing would just re-scan an empty range.
|
||||
if (deleted < DEFAULT_BATCH_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package dev.caskeleton.messaging.inbox;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Whether a message's side effect ran or was suppressed as a duplicate.
|
||||
*
|
||||
* @param <T> the side effect's result type
|
||||
* @param processed true when the side effect ran for the first time
|
||||
* @param result the side effect's result, absent for a duplicate
|
||||
*/
|
||||
public record InboxOutcome<T>(boolean processed, Optional<T> result) {
|
||||
|
||||
/**
|
||||
* Returns an outcome for a first-time delivery.
|
||||
*
|
||||
* @param <T> the result type
|
||||
* @param value the side effect's result
|
||||
* @return the processed outcome
|
||||
*/
|
||||
public static <T> InboxOutcome<T> processed(T value) {
|
||||
return new InboxOutcome<>(true, Optional.ofNullable(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an outcome for a suppressed duplicate.
|
||||
*
|
||||
* @param <T> the result type
|
||||
* @return the duplicate outcome
|
||||
*/
|
||||
public static <T> InboxOutcome<T> duplicate() {
|
||||
return new InboxOutcome<>(false, Optional.empty());
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package dev.caskeleton.messaging.inbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* How long inbox rows are kept, and the rule that makes the number safe.
|
||||
*
|
||||
* <p>Retention must exceed the broker's maximum redelivery window. That is not a tuning preference:
|
||||
* a row pruned while the broker can still redeliver its message turns the inbox into a no-op for
|
||||
* exactly that message, and the side effect runs a second time. The failure is silent, rare, and
|
||||
* only happens under the conditions that already made the day bad.
|
||||
*
|
||||
* <p>The safety margin is multiplicative rather than additive so that it scales with the window
|
||||
* itself. A stream whose redelivery window is measured in days needs more slack than one measured
|
||||
* in minutes, for the same reason: the estimate of that window is proportionally less certain.
|
||||
*
|
||||
* @param retention how long a processed row is kept
|
||||
* @param maximumRedeliveryWindow the longest the broker may redelivery a message
|
||||
*/
|
||||
public record InboxRetentionPolicy(Duration retention, Duration maximumRedeliveryWindow) {
|
||||
|
||||
/** How much longer than the redelivery window retention must be. */
|
||||
public static final double REQUIRED_SAFETY_FACTOR = 2.0;
|
||||
|
||||
/** The default for a broker whose redelivery window is bounded by a one-day log retention. */
|
||||
public static final Duration DEFAULT_RETENTION = Duration.ofDays(7);
|
||||
|
||||
public InboxRetentionPolicy {
|
||||
Objects.requireNonNull(retention, "retention must not be null");
|
||||
Objects.requireNonNull(maximumRedeliveryWindow, "maximumRedeliveryWindow must not be null");
|
||||
if (retention.isNegative() || retention.isZero()) {
|
||||
throw new IllegalArgumentException("retention must be positive");
|
||||
}
|
||||
if (maximumRedeliveryWindow.isNegative() || maximumRedeliveryWindow.isZero()) {
|
||||
throw new IllegalArgumentException("maximumRedeliveryWindow must be positive");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a retention that could prune a row the broker can still redeliver.
|
||||
*
|
||||
* @throws MessagingConfigurationException when retention is too short
|
||||
*/
|
||||
public void validate() {
|
||||
Duration required = required();
|
||||
if (retention.compareTo(required) < 0) {
|
||||
throw new MessagingConfigurationException(
|
||||
"INBOX_RETENTION_TOO_SHORT",
|
||||
"inbox retention of %s is below the %s required for a %s redelivery window; a pruned row"
|
||||
.formatted(retention, required, maximumRedeliveryWindow)
|
||||
+ " lets a late redelivery apply its effect twice");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the shortest retention this redelivery window allows.
|
||||
*
|
||||
* @return the minimum safe retention
|
||||
*/
|
||||
public Duration required() {
|
||||
return Duration.ofMillis(
|
||||
Math.round(maximumRedeliveryWindow.toMillis() * REQUIRED_SAFETY_FACTOR));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the cutoff before which rows may be deleted.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return the retention cutoff
|
||||
*/
|
||||
public Instant cutoff(Instant now) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
return now.minus(retention);
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
package dev.caskeleton.messaging.inbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.delivery.HandleResult;
|
||||
import dev.caskeleton.messaging.api.delivery.MessageDelivery;
|
||||
import dev.caskeleton.messaging.api.error.FailureCategory;
|
||||
import dev.caskeleton.messaging.api.error.FailureDescriptor;
|
||||
import dev.caskeleton.messaging.reliability.IdempotentMessageHandler;
|
||||
import dev.caskeleton.messaging.reliability.InboxResult;
|
||||
import dev.caskeleton.messaging.reliability.TransactionalMessageAction;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* The JDBC-backed {@link IdempotentMessageHandler}.
|
||||
*
|
||||
* <p>Reservation and effect commit together, in the runner's single transaction. Everything else
|
||||
* about this class follows from that: it cannot settle the message (settlement is not
|
||||
* transactional), it cannot publish (the publish would survive a rollback), and it cannot catch and
|
||||
* swallow the action's exception (the rollback is how the reservation is undone).
|
||||
*
|
||||
* <p>An already-reserved message returns success. Under at-least-once delivery a duplicate is the
|
||||
* normal case, not an error; reporting failure would send a message whose work is complete to the
|
||||
* dead-letter queue.
|
||||
*
|
||||
* @param <T> the payload type
|
||||
*/
|
||||
public final class TransactionalInboxHandler<T> implements IdempotentMessageHandler<T> {
|
||||
|
||||
private final IdempotentConsumer consumer;
|
||||
private final Supplier<Instant> clock;
|
||||
|
||||
/**
|
||||
* Creates a handler.
|
||||
*
|
||||
* @param consumer the reservation-and-effect runner
|
||||
* @param clock supplies the current instant
|
||||
*/
|
||||
public TransactionalInboxHandler(IdempotentConsumer consumer, Supplier<Instant> clock) {
|
||||
this.consumer = Objects.requireNonNull(consumer, "consumer must not be null");
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionStage<HandleResult> handleOnce(
|
||||
String consumerName, MessageDelivery<T> delivery, TransactionalMessageAction<T> action) {
|
||||
Objects.requireNonNull(delivery, "delivery must not be null");
|
||||
Objects.requireNonNull(action, "action must not be null");
|
||||
if (consumerName == null || consumerName.isBlank()) {
|
||||
throw new IllegalArgumentException("consumerName must not be blank");
|
||||
}
|
||||
|
||||
try {
|
||||
InboxOutcome<InboxResult> outcome =
|
||||
consumer.runOnce(
|
||||
delivery.message().messageId(),
|
||||
consumerName,
|
||||
clock.get(),
|
||||
() -> {
|
||||
try {
|
||||
action.apply(delivery);
|
||||
} catch (Exception failure) {
|
||||
// Wrapped, not swallowed: the transaction runner has to see a throw to roll the
|
||||
// reservation back along with the effect.
|
||||
throw new ActionFailedException(failure);
|
||||
}
|
||||
return InboxResult.APPLIED;
|
||||
});
|
||||
|
||||
return CompletableFuture.completedFuture(
|
||||
outcome.processed() ? HandleResult.success() : duplicateIsSuccess());
|
||||
} catch (ActionFailedException failure) {
|
||||
return CompletableFuture.completedFuture(retryable(failure.getCause()));
|
||||
}
|
||||
}
|
||||
|
||||
private static HandleResult duplicateIsSuccess() {
|
||||
// The effect already ran in an earlier delivery. Settling is correct; redelivering is not.
|
||||
return HandleResult.success();
|
||||
}
|
||||
|
||||
private static HandleResult retryable(Throwable cause) {
|
||||
return new HandleResult.Retry(
|
||||
new FailureDescriptor(
|
||||
FailureCategory.TRANSIENT_INFRASTRUCTURE,
|
||||
"INBOX_ACTION_FAILED",
|
||||
true,
|
||||
"the transactional action failed and its reservation was rolled back",
|
||||
Optional.ofNullable(cause).map(value -> value.getClass().getSimpleName())));
|
||||
}
|
||||
|
||||
/** Carries the action's checked exception out through the transaction runner. */
|
||||
private static final class ActionFailedException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private ActionFailedException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.reliability.OutboxRepository;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Deletes published outbox rows once they are past their retention.
|
||||
*
|
||||
* <p>Only {@code PUBLISHED} rows are eligible, and that restriction is the whole safety story. An
|
||||
* {@code AMBIGUOUS} row is a message that may or may not have reached the broker; deleting it
|
||||
* destroys the only record that it was ever meant to be sent. A {@code FAILED} row is the audit
|
||||
* trail of a message that never went out. Both are the rows an operator most needs during an
|
||||
* incident, which is exactly when a time-based sweep would otherwise remove them.
|
||||
*/
|
||||
public final class OutboxCleanupJob {
|
||||
|
||||
/** How many rows one delete statement removes before yielding. */
|
||||
public static final int DEFAULT_BATCH_SIZE = 1_000;
|
||||
|
||||
private final OutboxRepository outbox;
|
||||
private final OutboxProperties properties;
|
||||
private final int maxBatches;
|
||||
|
||||
/**
|
||||
* Creates a cleanup job.
|
||||
*
|
||||
* @param outbox the outbox storage
|
||||
* @param properties the relay settings supplying the retention
|
||||
* @param maxBatches how many delete batches one run may issue
|
||||
*/
|
||||
public OutboxCleanupJob(OutboxRepository outbox, OutboxProperties properties, int maxBatches) {
|
||||
this.outbox = Objects.requireNonNull(outbox, "outbox must not be null");
|
||||
this.properties = Objects.requireNonNull(properties, "properties must not be null");
|
||||
if (maxBatches < 1) {
|
||||
throw new IllegalArgumentException("maxBatches must be at least 1");
|
||||
}
|
||||
this.maxBatches = maxBatches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes expired published rows.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return how many rows were removed
|
||||
*/
|
||||
public int runOnce(Instant now) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
Instant cutoff = now.minus(properties.retentionAfterPublish());
|
||||
|
||||
int removed = 0;
|
||||
for (int batch = 0; batch < maxBatches; batch++) {
|
||||
int deleted = outbox.purgePublishedBefore(cutoff, DEFAULT_BATCH_SIZE);
|
||||
removed += deleted;
|
||||
// A short batch means the backlog is drained; asking again would re-scan an empty range.
|
||||
if (deleted < DEFAULT_BATCH_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.MessageEnvelope;
|
||||
import dev.caskeleton.messaging.api.ProducerId;
|
||||
import dev.caskeleton.messaging.api.SchemaVersion;
|
||||
import dev.caskeleton.messaging.api.header.HeaderName;
|
||||
import dev.caskeleton.messaging.api.header.HeaderValue;
|
||||
import dev.caskeleton.messaging.api.header.MessageHeaders;
|
||||
import dev.caskeleton.messaging.api.header.ReservedHeaders;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.schema.EncodedMessage;
|
||||
import dev.caskeleton.messaging.schema.SchemaReference;
|
||||
import java.net.URI;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Rebuilds a publishable envelope from an outbox row.
|
||||
*
|
||||
* <p>The identity comes from the row, never from a fresh mint. That is what makes a retry after an
|
||||
* ambiguous publish safe: the broker's deduplication, or the consumer's Inbox, sees the same id and
|
||||
* collapses the duplicate.
|
||||
*
|
||||
* <p>So does everything else the envelope carries. This used to rebuild correlation, causation,
|
||||
* tenant, trace and the schema reference as empty, and read the routing keys out of the row's
|
||||
* header map — so a message that travelled through the outbox reached its consumer with less
|
||||
* provenance than one published directly, and the publish path became part of the message's
|
||||
* meaning. The row now has columns for those facts and this reads them.
|
||||
*
|
||||
* <p>Reserved header names in the row are refused outright, with no exception for the routing keys.
|
||||
* They had one, because the keys had nowhere else to live; a row could therefore write {@code
|
||||
* msg.partition-key} and the same code path would have accepted {@code msg.id} if the check had
|
||||
* been one line looser. Now that the keys are columns, the rule is the simple one: an outbox row
|
||||
* cannot write into the platform's namespace at all.
|
||||
*/
|
||||
public final class OutboxEnvelopeFactory {
|
||||
|
||||
private final ProducerId producer;
|
||||
|
||||
/**
|
||||
* Creates a factory for one service.
|
||||
*
|
||||
* @param producer the logical producing service, used for rows that name none
|
||||
*/
|
||||
public OutboxEnvelopeFactory(ProducerId producer) {
|
||||
this.producer = Objects.requireNonNull(producer, "producer must not be null");
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the envelope for one outbox row.
|
||||
*
|
||||
* @param record the outbox row
|
||||
* @return the encoded envelope
|
||||
*/
|
||||
public MessageEnvelope<EncodedMessage> toEnvelope(OutboxRecord record) {
|
||||
Objects.requireNonNull(record, "record must not be null");
|
||||
|
||||
Map<HeaderName, HeaderValue> headers = new LinkedHashMap<>();
|
||||
record
|
||||
.headers()
|
||||
.forEach(
|
||||
(name, value) -> {
|
||||
// The row's headers are application data. Promoting them straight into
|
||||
// MessageHeaders.platform let a row carrying `msg.id` or `msg.type` reach the wire
|
||||
// as if the platform had written it — and the consumer trusts the last such header,
|
||||
// so a forged identity lands in another message's inbox deduplication.
|
||||
if (ReservedHeaders.isReserved(name)) {
|
||||
throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException(
|
||||
"RESERVED_HEADER_IN_OUTBOX_ROW",
|
||||
"outbox row header '"
|
||||
+ name
|
||||
+ "' uses the reserved namespace; canonical identity and metadata come from"
|
||||
+ " the record's own columns, not from its header map");
|
||||
}
|
||||
headers.put(new HeaderName(name), new HeaderValue(value));
|
||||
});
|
||||
|
||||
OutboxCanonicalMetadata metadata = record.metadata();
|
||||
return new MessageEnvelope<>(
|
||||
record.messageId(),
|
||||
record.messageType(),
|
||||
record.schemaVersion(),
|
||||
record.createdAt(),
|
||||
// The write instant stands in when the row records no separate occurrence. It is the
|
||||
// closest true statement available: the business transaction that wrote the row is the one
|
||||
// the fact occurred in.
|
||||
Optional.of(metadata.occurredAt().orElse(record.createdAt())),
|
||||
// A row written before the producer column existed names none, and the service that owns
|
||||
// this relay is the only producer it can honestly be attributed to.
|
||||
metadata.producer().orElse(producer),
|
||||
metadata.correlationId(),
|
||||
metadata.causationId(),
|
||||
record.contentType(),
|
||||
metadata.partitionKey(),
|
||||
metadata.orderingKey(),
|
||||
metadata.tenant(),
|
||||
metadata.traceContext(),
|
||||
MessageHeaders.platform(headers),
|
||||
new EncodedMessage(
|
||||
record.payload(), record.contentType(), schemaReference(record, metadata)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds the schema reference from the row's subject, the row's version, and its location.
|
||||
*
|
||||
* <p>The version is the row's {@code schema_version} column rather than a third copy of the same
|
||||
* number: a reference whose version could disagree with the column the relay filters on is a
|
||||
* reference nobody can trust.
|
||||
*/
|
||||
private static Optional<SchemaReference> schemaReference(
|
||||
OutboxRecord record, OutboxCanonicalMetadata metadata) {
|
||||
return metadata
|
||||
.schemaSubject()
|
||||
.map(
|
||||
subject ->
|
||||
new SchemaReference(
|
||||
subject,
|
||||
new SchemaVersion(record.schemaVersion().value()),
|
||||
metadata.schemaUri().map(URI::create)));
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* The relay's operational settings, and the relationships between them that have to hold.
|
||||
*
|
||||
* <p>The lease duration is the dangerous one. If it is shorter than the time a publish can take, a
|
||||
* second relay claims the row while the first is still waiting for a confirm, and the message is
|
||||
* published twice — under the same id, so consumers with an inbox survive it, but consumers without
|
||||
* one do not. The constructor therefore requires the lease to exceed the publish timeout by a
|
||||
* margin rather than merely to be positive.
|
||||
*
|
||||
* @param batchSize how many records one lease claims
|
||||
* @param leaseDuration how long a claim holds
|
||||
* @param publishTimeout the deadline for one publish
|
||||
* @param pollInterval how often the relay looks for work
|
||||
* @param retentionAfterPublish how long published rows are kept before deletion
|
||||
* @param maxAttempts how many times a record is retried before it is parked
|
||||
*/
|
||||
public record OutboxProperties(
|
||||
int batchSize,
|
||||
Duration leaseDuration,
|
||||
Duration publishTimeout,
|
||||
Duration pollInterval,
|
||||
Duration retentionAfterPublish,
|
||||
int maxAttempts) {
|
||||
|
||||
/** How much longer than the publish timeout a lease must be. */
|
||||
public static final double REQUIRED_LEASE_FACTOR = 2.0;
|
||||
|
||||
public OutboxProperties {
|
||||
Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
|
||||
Objects.requireNonNull(publishTimeout, "publishTimeout must not be null");
|
||||
Objects.requireNonNull(pollInterval, "pollInterval must not be null");
|
||||
Objects.requireNonNull(retentionAfterPublish, "retentionAfterPublish must not be null");
|
||||
if (batchSize < 1) {
|
||||
throw new IllegalArgumentException("batchSize must be at least 1");
|
||||
}
|
||||
if (maxAttempts < 1) {
|
||||
throw new IllegalArgumentException("maxAttempts must be at least 1");
|
||||
}
|
||||
requirePositive(leaseDuration, "leaseDuration");
|
||||
requirePositive(publishTimeout, "publishTimeout");
|
||||
requirePositive(pollInterval, "pollInterval");
|
||||
requirePositive(retentionAfterPublish, "retentionAfterPublish");
|
||||
|
||||
Duration minimumLease =
|
||||
Duration.ofMillis(Math.round(publishTimeout.toMillis() * REQUIRED_LEASE_FACTOR));
|
||||
if (leaseDuration.compareTo(minimumLease) < 0) {
|
||||
throw new MessagingConfigurationException(
|
||||
"OUTBOX_LEASE_TOO_SHORT",
|
||||
"a lease of %s can expire while a %s publish is still in flight, letting a second relay"
|
||||
.formatted(leaseDuration, publishTimeout)
|
||||
+ " publish the same record; the minimum is %s".formatted(minimumLease));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns settings suited to a single relay against a healthy broker.
|
||||
*
|
||||
* @return the default properties
|
||||
*/
|
||||
public static OutboxProperties defaults() {
|
||||
return new OutboxProperties(
|
||||
100,
|
||||
Duration.ofSeconds(30),
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofMillis(500),
|
||||
Duration.ofDays(3),
|
||||
10);
|
||||
}
|
||||
|
||||
private static void requirePositive(Duration value, String field) {
|
||||
if (value.isNegative() || value.isZero()) {
|
||||
throw new IllegalArgumentException(field + " must be positive");
|
||||
}
|
||||
}
|
||||
}
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.MessageEnvelope;
|
||||
import dev.caskeleton.messaging.api.destination.MessageDestination;
|
||||
import dev.caskeleton.messaging.api.error.FailureCategory;
|
||||
import dev.caskeleton.messaging.api.error.FailureDescriptor;
|
||||
import dev.caskeleton.messaging.api.error.MessagingException;
|
||||
import dev.caskeleton.messaging.api.publish.MessagePublisher;
|
||||
import dev.caskeleton.messaging.api.publish.PublishCompletion;
|
||||
import dev.caskeleton.messaging.api.publish.PublishEvidence;
|
||||
import dev.caskeleton.messaging.api.publish.PublishOptions;
|
||||
import dev.caskeleton.messaging.api.publish.PublishResult;
|
||||
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
|
||||
import dev.caskeleton.messaging.reliability.OutboxLease;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRepository;
|
||||
import dev.caskeleton.messaging.reliability.OutboxTransitionResult;
|
||||
import dev.caskeleton.messaging.schema.EncodedMessage;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Publishes outbox rows, treating an unknown outcome as retryable rather than final.
|
||||
*
|
||||
* <p>The relay's correctness rests on one rule: an ambiguous publish is retried <em>under the same
|
||||
* message id</em>. Minting a new id would turn a possibly-delivered message into a
|
||||
* definitely-second message, and no downstream deduplication could recover from it. Marking it
|
||||
* failed instead would lose a message the broker may already hold.
|
||||
*
|
||||
* <p>The relay therefore guarantees at-least-once publication and nothing more. Effectively-once
|
||||
* downstream effects come from pairing it with an Inbox — which is why the platform never
|
||||
* advertises the outbox as exactly-once.
|
||||
*/
|
||||
public final class OutboxRelay {
|
||||
|
||||
private final OutboxRepository repository;
|
||||
private final MessagePublisher publisher;
|
||||
private final OutboxEnvelopeFactory envelopeFactory;
|
||||
private final int batchSize;
|
||||
|
||||
private final String owner;
|
||||
|
||||
private final OutboxRetryScheduler scheduler;
|
||||
|
||||
private static String defaultOwner() {
|
||||
return java.lang.ProcessHandle.current().pid()
|
||||
+ "@"
|
||||
+ java.util.UUID.randomUUID().toString().substring(0, 8);
|
||||
}
|
||||
|
||||
private final Duration leaseDuration;
|
||||
|
||||
/**
|
||||
* Creates a relay.
|
||||
*
|
||||
* @param repository the outbox storage
|
||||
* @param publisher the platform publisher
|
||||
* @param envelopeFactory rebuilds envelopes from rows
|
||||
* @param batchSize how many rows to claim per pass
|
||||
* @param leaseDuration how long a claim holds
|
||||
*/
|
||||
public OutboxRelay(
|
||||
OutboxRepository repository,
|
||||
MessagePublisher publisher,
|
||||
OutboxEnvelopeFactory envelopeFactory,
|
||||
int batchSize,
|
||||
Duration leaseDuration) {
|
||||
this(repository, publisher, envelopeFactory, batchSize, leaseDuration, defaultOwner());
|
||||
}
|
||||
|
||||
/**
|
||||
* A relay with an explicit owner identity.
|
||||
*
|
||||
* <p>The owner is half of the fencing predicate. It defaults to something stable per process
|
||||
* rather than per object, because two relays in one JVM sharing an owner would each be able to
|
||||
* overwrite the other's terminal writes — the exact failure the token exists to prevent.
|
||||
*/
|
||||
public OutboxRelay(
|
||||
OutboxRepository repository,
|
||||
MessagePublisher publisher,
|
||||
OutboxEnvelopeFactory envelopeFactory,
|
||||
int batchSize,
|
||||
Duration leaseDuration,
|
||||
String owner) {
|
||||
this(
|
||||
repository,
|
||||
publisher,
|
||||
envelopeFactory,
|
||||
batchSize,
|
||||
leaseDuration,
|
||||
owner,
|
||||
OutboxRetryScheduler.standard());
|
||||
}
|
||||
|
||||
/**
|
||||
* A relay with an explicit retry policy and a process-derived owner.
|
||||
*
|
||||
* @param repository the outbox storage
|
||||
* @param publisher the platform publisher
|
||||
* @param envelopeFactory rebuilds envelopes from rows
|
||||
* @param batchSize how many rows to claim per pass
|
||||
* @param leaseDuration how long a claim holds
|
||||
* @param scheduler the retry policy
|
||||
*/
|
||||
public OutboxRelay(
|
||||
OutboxRepository repository,
|
||||
MessagePublisher publisher,
|
||||
OutboxEnvelopeFactory envelopeFactory,
|
||||
int batchSize,
|
||||
Duration leaseDuration,
|
||||
OutboxRetryScheduler scheduler) {
|
||||
this(
|
||||
repository,
|
||||
publisher,
|
||||
envelopeFactory,
|
||||
batchSize,
|
||||
leaseDuration,
|
||||
defaultOwner(),
|
||||
scheduler);
|
||||
}
|
||||
|
||||
/**
|
||||
* A relay with an explicit retry policy.
|
||||
*
|
||||
* <p>The scheduler was built by the auto-configuration and handed to nobody: the relay claimed
|
||||
* every ambiguous row on the next pass regardless of its attempt count, so the configured budget
|
||||
* of ten attempts was a number nothing read.
|
||||
*/
|
||||
public OutboxRelay(
|
||||
OutboxRepository repository,
|
||||
MessagePublisher publisher,
|
||||
OutboxEnvelopeFactory envelopeFactory,
|
||||
int batchSize,
|
||||
Duration leaseDuration,
|
||||
String owner,
|
||||
OutboxRetryScheduler scheduler) {
|
||||
this.owner = Objects.requireNonNull(owner, "owner must not be null");
|
||||
this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null");
|
||||
this.repository = Objects.requireNonNull(repository, "repository must not be null");
|
||||
this.publisher = Objects.requireNonNull(publisher, "publisher must not be null");
|
||||
this.envelopeFactory =
|
||||
Objects.requireNonNull(envelopeFactory, "envelopeFactory must not be null");
|
||||
this.leaseDuration = Objects.requireNonNull(leaseDuration, "leaseDuration must not be null");
|
||||
if (batchSize < 1) {
|
||||
throw new IllegalArgumentException("batchSize must be at least 1");
|
||||
}
|
||||
this.batchSize = batchSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one relay pass.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @return what the pass did
|
||||
*/
|
||||
public OutboxRelayReport runOnce(Instant now) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
// The claim carries an owner and a fencing token, and every write
|
||||
// below is conditioned on them. Without that, a relay that stalled past its lease could record
|
||||
// AMBIGUOUS over the PUBLISHED another relay had already written, making a confirmed message
|
||||
// claimable again.
|
||||
List<OutboxLease> batch =
|
||||
repository.claimBatch(owner, batchSize, leaseDuration, now, scheduler.maxAttempts());
|
||||
|
||||
int published = 0;
|
||||
int ambiguous = 0;
|
||||
int failed = 0;
|
||||
int stale = 0;
|
||||
int exhausted = 0;
|
||||
|
||||
for (OutboxLease lease : batch) {
|
||||
OutboxRecord record = lease.record();
|
||||
MessageEnvelope<EncodedMessage> envelope;
|
||||
try {
|
||||
envelope = envelopeFactory.toEnvelope(record);
|
||||
} catch (RuntimeException unpublishable) {
|
||||
// A row that cannot become an envelope never will. Thrown out of the pass, it took the
|
||||
// rest of the batch with it: those rows stayed IN_FLIGHT until their lease expired, and the
|
||||
// same row was claimed and thrown again on the next pass, forever.
|
||||
if (repository.markFailed(lease, failureCode(unpublishable), now)
|
||||
== OutboxTransitionResult.APPLIED) {
|
||||
failed++;
|
||||
} else {
|
||||
stale++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
PublishResult result = publish(record, envelope);
|
||||
switch (result.completion()) {
|
||||
case CONFIRMED -> {
|
||||
if (repository.markPublished(lease, now) == OutboxTransitionResult.APPLIED) {
|
||||
published++;
|
||||
} else {
|
||||
stale++;
|
||||
}
|
||||
}
|
||||
case AMBIGUOUS -> {
|
||||
String code = result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS");
|
||||
// The attempt this pass just spent. The claim predicate and the row both count attempts
|
||||
// after the transition, so the budget has to be judged on the same number the next claim
|
||||
// will read, or the last attempt is spent twice.
|
||||
int spent = record.attempts() + 1;
|
||||
OutboxTransitionResult transition =
|
||||
scheduler
|
||||
.parkReason(spent)
|
||||
// A row that has spent its budget without an answer is parked under its own
|
||||
// status. Leaving it AMBIGUOUS makes it a row the claim predicate silently skips
|
||||
// forever, which looks identical to a healthy backlog on every dashboard.
|
||||
.map(reason -> repository.markExhausted(lease, reason, now))
|
||||
.orElseGet(
|
||||
() ->
|
||||
repository.markAmbiguous(
|
||||
lease, code, now, scheduler.nextAttemptAt(now, spent)));
|
||||
if (transition == OutboxTransitionResult.APPLIED) {
|
||||
if (scheduler.isExhausted(spent)) {
|
||||
exhausted++;
|
||||
} else {
|
||||
ambiguous++;
|
||||
}
|
||||
} else {
|
||||
stale++;
|
||||
}
|
||||
}
|
||||
case REJECTED -> {
|
||||
if (repository.markFailed(
|
||||
lease, result.failure().map(failure -> failure.code()).orElse("REJECTED"), now)
|
||||
== OutboxTransitionResult.APPLIED) {
|
||||
failed++;
|
||||
} else {
|
||||
stale++;
|
||||
}
|
||||
}
|
||||
// Unreachable while PublishCompletion has exactly these constants. Present so that a new
|
||||
// completion fails loudly rather than leaving the row IN_FLIGHT with its lease ticking —
|
||||
// which would look like a stalled relay rather than an unhandled case.
|
||||
default ->
|
||||
throw new IllegalStateException("unhandled publish completion: " + result.completion());
|
||||
}
|
||||
}
|
||||
return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale, exhausted);
|
||||
}
|
||||
|
||||
private PublishResult publish(OutboxRecord record, MessageEnvelope<EncodedMessage> envelope) {
|
||||
MessageDestination<EncodedMessage> destination =
|
||||
new MessageDestination<>(record.destination(), record.messageType(), EncodedMessage.class);
|
||||
try {
|
||||
return publisher
|
||||
.publish(destination, envelope, PublishOptions.defaults())
|
||||
.toCompletableFuture()
|
||||
.join();
|
||||
} catch (RuntimeException unknown) {
|
||||
// An exception says nothing about whether the broker took the message, so the row is retried
|
||||
// under the same id like any other unknown outcome — not failed, not left IN_FLIGHT.
|
||||
return new PublishResult(
|
||||
PublishCompletion.AMBIGUOUS,
|
||||
PublishEvidence.ambiguous(),
|
||||
RoutingOutcome.UNKNOWN,
|
||||
Optional.empty(),
|
||||
1,
|
||||
Duration.ZERO,
|
||||
Optional.of(
|
||||
FailureDescriptor.of(
|
||||
FailureCategory.AMBIGUOUS,
|
||||
"PUBLISH_OUTCOME_UNKNOWN",
|
||||
"the publisher threw " + unknown.getClass().getSimpleName())));
|
||||
}
|
||||
}
|
||||
|
||||
private static String failureCode(RuntimeException failure) {
|
||||
return failure instanceof MessagingException known
|
||||
? known.failure().code()
|
||||
: "OUTBOX_ROW_UNPUBLISHABLE";
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
/**
|
||||
* What one relay pass did.
|
||||
*
|
||||
* <p>Ambiguous rows are counted separately from failures because they mean something operationally
|
||||
* different: a rising ambiguous count is a broker confirmation problem, while a rising failed count
|
||||
* is a contract or topology problem.
|
||||
*
|
||||
* <p>{@code staleLeases} is the count of terminal writes this pass could not apply because another
|
||||
* relay had already taken the row. It is not an error and it is not a success: it means this worker
|
||||
* published something whose outcome another worker has already recorded, so a non-zero value is the
|
||||
* visible form of a duplicate publication.
|
||||
*
|
||||
* <p>{@code exhausted} is counted apart from {@code failed} for the same reason: a rejected row is
|
||||
* a message the broker refused and an exhausted one is a message nobody ever answered for. The
|
||||
* first needs a fix, the second needs a redrive, and a single counter for both tells an operator
|
||||
* neither.
|
||||
*
|
||||
* @param leased how many rows were claimed
|
||||
* @param published how many were confirmed
|
||||
* @param ambiguous how many had an unknown outcome and stay retryable
|
||||
* @param failed how many were definitively rejected
|
||||
* @param staleLeases how many terminal writes were refused because the lease had been superseded
|
||||
* @param exhausted how many spent their attempt budget without a confirmation and were parked
|
||||
*/
|
||||
public record OutboxRelayReport(
|
||||
int leased, int published, int ambiguous, int failed, int staleLeases, int exhausted) {
|
||||
|
||||
public OutboxRelayReport {
|
||||
if (leased < 0
|
||||
|| published < 0
|
||||
|| ambiguous < 0
|
||||
|| failed < 0
|
||||
|| staleLeases < 0
|
||||
|| exhausted < 0) {
|
||||
throw new IllegalArgumentException("relay counters must not be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/** A report from a pass that parked nothing. */
|
||||
public OutboxRelayReport(int leased, int published, int ambiguous, int failed, int staleLeases) {
|
||||
this(leased, published, ambiguous, failed, staleLeases, 0);
|
||||
}
|
||||
|
||||
/** A report from a pass that observed no stale leases. */
|
||||
public OutboxRelayReport(int leased, int published, int ambiguous, int failed) {
|
||||
this(leased, published, ambiguous, failed, 0, 0);
|
||||
}
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Runs relay passes on the schedule the retry policy decides.
|
||||
*
|
||||
* <p>The relay, its retry scheduler and the attempt budget all existed and nothing ever called
|
||||
* {@code runOnce}. An outbox whose relay is never driven is the worst shape of all: the business
|
||||
* transaction commits its row and reports success, and the message is never published — a silent,
|
||||
* unbounded backlog rather than a visible failure.
|
||||
*
|
||||
* <p>Passes are self-scheduled rather than run at a fixed rate. A fixed rate keeps firing while the
|
||||
* broker is down, and the pass after an empty or failed one is exactly where the policy's backoff
|
||||
* belongs; a pass that published something resets to the base interval because work is flowing.
|
||||
*
|
||||
* <p>One thread, and the next pass is scheduled only after the previous returns. Overlapping passes
|
||||
* from one process would have two claims racing on the same rows, which the fencing predicate would
|
||||
* then reject as stale — correct, but it is duplicate publishing work bought at the cost of a
|
||||
* duplicate publish.
|
||||
*/
|
||||
public final class OutboxRelayWorker implements AutoCloseable {
|
||||
|
||||
/** How long {@link #close()} waits for a running pass when no deadline is given. */
|
||||
public static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30);
|
||||
|
||||
private static final System.Logger LOG = System.getLogger(OutboxRelayWorker.class.getName());
|
||||
|
||||
private final OutboxRelay relay;
|
||||
|
||||
private final OutboxRetryScheduler scheduler;
|
||||
|
||||
private final Supplier<Instant> clock;
|
||||
|
||||
private final ScheduledExecutorService passes;
|
||||
|
||||
private final AtomicBoolean running = new AtomicBoolean();
|
||||
|
||||
private final AtomicInteger unproductivePasses = new AtomicInteger();
|
||||
|
||||
private final AtomicLong completedPasses = new AtomicLong();
|
||||
|
||||
private final AtomicLong failedPasses = new AtomicLong();
|
||||
|
||||
/**
|
||||
* Creates a worker driving one relay.
|
||||
*
|
||||
* @param relay the relay to run
|
||||
* @param scheduler the policy deciding when the next pass runs
|
||||
*/
|
||||
public OutboxRelayWorker(OutboxRelay relay, OutboxRetryScheduler scheduler) {
|
||||
this(relay, scheduler, Instant::now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a worker with an explicit clock, so a pass can be driven without waiting on wall time.
|
||||
*
|
||||
* @param relay the relay to run
|
||||
* @param scheduler the policy deciding when the next pass runs
|
||||
* @param clock the source of the current instant
|
||||
*/
|
||||
public OutboxRelayWorker(
|
||||
OutboxRelay relay, OutboxRetryScheduler scheduler, Supplier<Instant> clock) {
|
||||
this.relay = Objects.requireNonNull(relay, "relay must not be null");
|
||||
this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null");
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
ScheduledThreadPoolExecutor executor =
|
||||
new ScheduledThreadPoolExecutor(
|
||||
1,
|
||||
runnable -> {
|
||||
// Daemon: a relay thread that outlives the application would keep publishing from a
|
||||
// process that has already released its other resources.
|
||||
Thread thread = new Thread(runnable, "messaging-outbox-relay");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
// A pass waiting out its backoff is dropped on shutdown rather than run afterwards. Left on,
|
||||
// the default policy makes stop() wait for a pass that has not started and cannot do anything
|
||||
// useful, so an orderly shutdown pays the whole poll interval before it can close.
|
||||
executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
|
||||
this.passes = executor;
|
||||
}
|
||||
|
||||
/** Starts driving passes. Idempotent: a second call does not add a second schedule. */
|
||||
public void start() {
|
||||
if (running.compareAndSet(false, true)) {
|
||||
schedule(Duration.ZERO);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops scheduling and lets the pass that is already running finish.
|
||||
*
|
||||
* <p>Draining rather than interrupting is the whole point. A pass killed between its claim and
|
||||
* its terminal write leaves rows {@code IN_FLIGHT} holding a lease, and nothing may touch them
|
||||
* until that lease expires — so an orderly shutdown would produce exactly the stall that a crash
|
||||
* produces.
|
||||
*
|
||||
* @param deadline how long to wait for the running pass
|
||||
* @return true when the running pass finished inside the deadline
|
||||
*/
|
||||
public boolean stop(Duration deadline) {
|
||||
Objects.requireNonNull(deadline, "deadline must not be null");
|
||||
running.set(false);
|
||||
passes.shutdown();
|
||||
try {
|
||||
if (passes.awaitTermination(deadline.toMillis(), TimeUnit.MILLISECONDS)) {
|
||||
return true;
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
// The deadline passed with a pass still running. Its rows stay leased until the lease expires,
|
||||
// which is the outcome the deadline exists to bound rather than to prevent.
|
||||
passes.shutdownNow();
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
stop(DEFAULT_DRAIN_DEADLINE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the worker is scheduling passes.
|
||||
*
|
||||
* @return true between {@link #start()} and {@link #stop(Duration)}
|
||||
*/
|
||||
public boolean isRunning() {
|
||||
return running.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns how many passes have finished, successfully or not.
|
||||
*
|
||||
* @return the completed pass count
|
||||
*/
|
||||
public long completedPasses() {
|
||||
return completedPasses.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns how many passes ended in an exception.
|
||||
*
|
||||
* @return the failed pass count
|
||||
*/
|
||||
public long failedPasses() {
|
||||
return failedPasses.get();
|
||||
}
|
||||
|
||||
// The scheduled future is not kept: a pass reports nothing to a caller and handles its own
|
||||
// failure below, so the only thing holding it would achieve is a reference that has to be
|
||||
// cleaned up.
|
||||
@SuppressWarnings("FutureReturnValueIgnored")
|
||||
private void schedule(Duration delay) {
|
||||
if (!running.get()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
passes.schedule(this::runPass, Math.max(0, delay.toMillis()), TimeUnit.MILLISECONDS);
|
||||
} catch (RejectedExecutionException stopping) {
|
||||
// stop() shut the executor between the running check and the submission. There is nothing to
|
||||
// schedule onto and nothing to report: the worker is stopping, which is what was asked for.
|
||||
running.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void runPass() {
|
||||
if (!running.get()) {
|
||||
return;
|
||||
}
|
||||
int unproductive;
|
||||
try {
|
||||
OutboxRelayReport report = relay.runOnce(clock.get());
|
||||
unproductive = report.published() > 0 ? reset() : unproductivePasses.incrementAndGet();
|
||||
} catch (RuntimeException passFailed) {
|
||||
// A failed pass must not stop the loop: the scheduled task's own exception would cancel every
|
||||
// future pass, turning one broker error into a relay that never runs again. The failure is
|
||||
// counted and the next pass backs off as if nothing was published, which is true. It is also
|
||||
// logged: counted silently, a relay failing every pass looked exactly like an idle one.
|
||||
long failures = failedPasses.incrementAndGet();
|
||||
unproductive = unproductivePasses.incrementAndGet();
|
||||
LOG.log(
|
||||
System.Logger.Level.WARNING,
|
||||
"outbox relay pass failed (" + failures + " failed so far); retrying with backoff",
|
||||
passFailed);
|
||||
}
|
||||
completedPasses.incrementAndGet();
|
||||
schedule(scheduler.backoff(unproductive));
|
||||
}
|
||||
|
||||
private int reset() {
|
||||
unproductivePasses.set(0);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Decides when a relay's next pass should run, and when a record has retried enough.
|
||||
*
|
||||
* <p>The same curve serves two different jobs and neither replaces the other. Backing off the
|
||||
* <em>pass</em> is what lets a broker recover: an outage fails every record in the batch for the
|
||||
* same reason, so retrying the whole backlog on the poll interval achieves nothing but load. But a
|
||||
* pass-level wait is process memory — it is forgotten on restart and each replica keeps its own —
|
||||
* so eligibility also has to be written on the row, or the backlog is retried as fast as the
|
||||
* unluckiest replica polls.
|
||||
*
|
||||
* <p>Jitter is applied deterministically from the attempt count rather than randomly. Several relay
|
||||
* instances that all started at deployment time would otherwise synchronise their retries into a
|
||||
* thundering herd, and a random source would make the schedule impossible to test.
|
||||
*/
|
||||
public final class OutboxRetryScheduler {
|
||||
|
||||
private final Duration baseInterval;
|
||||
private final Duration maxInterval;
|
||||
private final int maxAttempts;
|
||||
|
||||
/**
|
||||
* Creates a scheduler from the relay properties.
|
||||
*
|
||||
* @param properties the relay settings
|
||||
* @param maxInterval the longest backoff between passes
|
||||
*/
|
||||
public OutboxRetryScheduler(OutboxProperties properties, Duration maxInterval) {
|
||||
Objects.requireNonNull(properties, "properties must not be null");
|
||||
Objects.requireNonNull(maxInterval, "maxInterval must not be null");
|
||||
if (maxInterval.compareTo(properties.pollInterval()) < 0) {
|
||||
throw new IllegalArgumentException("maxInterval must not be shorter than the poll interval");
|
||||
}
|
||||
this.baseInterval = properties.pollInterval();
|
||||
this.maxInterval = maxInterval;
|
||||
this.maxAttempts = properties.maxAttempts();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns when the next pass should run.
|
||||
*
|
||||
* <p>A pass that published something resets to the base interval: work is flowing, so there is no
|
||||
* reason to wait.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @param consecutiveEmptyOrFailedPasses how many passes in a row did no useful work
|
||||
* @return the next pass instant
|
||||
*/
|
||||
public Instant nextPassAt(Instant now, int consecutiveEmptyOrFailedPasses) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
if (consecutiveEmptyOrFailedPasses < 0) {
|
||||
throw new IllegalArgumentException("consecutiveEmptyOrFailedPasses must not be negative");
|
||||
}
|
||||
return now.plus(backoff(consecutiveEmptyOrFailedPasses));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns when a row that has spent this many attempts may be claimed again.
|
||||
*
|
||||
* <p>Written onto the row rather than held in the relay, because a deadline that lives in process
|
||||
* memory is forgotten on restart and computed independently by every replica — which turns a
|
||||
* backoff into every worker re-publishing the same backlog as fast as it polls.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @param attempts how many attempts the row has spent
|
||||
* @return the earliest instant the row is eligible again
|
||||
*/
|
||||
public Instant nextAttemptAt(Instant now, int attempts) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
if (attempts < 1) {
|
||||
throw new IllegalArgumentException("a row that has been published once has spent an attempt");
|
||||
}
|
||||
return now.plus(backoff(attempts));
|
||||
}
|
||||
|
||||
/** The platform's default policy, for a relay whose caller did not supply one. */
|
||||
public static OutboxRetryScheduler standard() {
|
||||
return new OutboxRetryScheduler(OutboxProperties.defaults(), java.time.Duration.ofMinutes(5));
|
||||
}
|
||||
|
||||
/** The attempt budget a row may spend before it is parked. */
|
||||
public int maxAttempts() {
|
||||
return maxAttempts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the delay before the next pass.
|
||||
*
|
||||
* @param consecutiveEmptyOrFailedPasses how many passes in a row did no useful work
|
||||
* @return the backoff delay
|
||||
*/
|
||||
public Duration backoff(int consecutiveEmptyOrFailedPasses) {
|
||||
if (consecutiveEmptyOrFailedPasses <= 0) {
|
||||
return baseInterval;
|
||||
}
|
||||
// Doubling, capped. Shifting rather than Math.pow so an unbounded counter cannot overflow into
|
||||
// a negative duration on a long outage.
|
||||
int exponent = Math.min(consecutiveEmptyOrFailedPasses, 20);
|
||||
long scaled = baseInterval.toMillis() << exponent;
|
||||
long capped = Math.min(scaled, maxInterval.toMillis());
|
||||
long jittered = capped - (capped / 8) * (exponent % 3);
|
||||
return Duration.ofMillis(Math.max(baseInterval.toMillis(), jittered));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether a record has exhausted its attempt budget.
|
||||
*
|
||||
* @param attempts how many times the record has been published
|
||||
* @return true when it should be parked rather than retried
|
||||
*/
|
||||
public boolean isExhausted(int attempts) {
|
||||
return attempts >= maxAttempts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the reason a record should be parked, when it should.
|
||||
*
|
||||
* @param attempts how many times the record has been published
|
||||
* @return the sanitized reason, or empty when the record may still retry
|
||||
*/
|
||||
public Optional<String> parkReason(int attempts) {
|
||||
return isExhausted(attempts)
|
||||
? Optional.of(
|
||||
"the record reached its %d attempt budget without a confirmed publish"
|
||||
.formatted(maxAttempts))
|
||||
: Optional.empty();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user