refactor: 각 어댑터터별 리펙토링 진행

This commit is contained in:
DongHyeonka
2026-08-24 18:26:40 +09:00
parent 16e33ce986
commit a261df2a2b
100 changed files with 7072 additions and 902 deletions
@@ -1,6 +1,7 @@
package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.api.header.ReservedHeaders;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -16,12 +17,29 @@ import java.util.Objects;
* <p>The platform contract does not change with the transport. The router emits the same reserved
* headers the polling relay does, so a consumer cannot tell which one published a message, and
* switching between them is a deployment decision rather than a contract change.
*
* <p>The message key and the topic are two different questions and used to be answered by one
* column. {@code destination} routed the topic <em>and</em> became the key, so every message on a
* topic carried the same key, landed on one partition, and made keyed ordering a claim with no
* mechanism behind it. The key now comes from {@code routing_key}, the stored column that states
* the partition-key-or-message-id fallback once for both relays, and {@code destination} stays what
* it always was: routing metadata.
*
* <p>One canonical field has no header here: the schema subject. There is no reserved header name
* for it, and inventing one that only this path wrote would replace a symmetric gap with an
* asymmetric one — the polling relay would carry a schema reference on the envelope and the CDC
* relay a header no adapter reads. It stays a column that the polling relay resolves, and closing
* it means adding the name to {@code ReservedHeaders} and teaching both broker header mappers to
* write and read it.
*/
public final class DebeziumOutboxEventRouter {
/** The outbox column Debezium uses as the message key. */
/** The outbox column Debezium routes the topic by. */
public static final String ROUTE_BY_COLUMN = "destination";
/** The outbox column Debezium uses as the message key. */
public static final String KEY_COLUMN = "routing_key";
/** The outbox column carrying the logical message id. */
public static final String ID_COLUMN = "message_id";
@@ -39,7 +57,7 @@ public final class DebeziumOutboxEventRouter {
Map<String, String> configuration = new LinkedHashMap<>();
configuration.put("transforms.outbox.type", "io.debezium.transforms.outbox.EventRouter");
configuration.put("transforms.outbox.table.field.event.id", ID_COLUMN);
configuration.put("transforms.outbox.table.field.event.key", ROUTE_BY_COLUMN);
configuration.put("transforms.outbox.table.field.event.key", KEY_COLUMN);
configuration.put("transforms.outbox.table.field.event.payload", PAYLOAD_COLUMN);
configuration.put("transforms.outbox.route.by.field", ROUTE_BY_COLUMN);
configuration.put(
@@ -51,7 +69,18 @@ public final class DebeziumOutboxEventRouter {
"message_type:header:" + ReservedHeaders.MESSAGE_TYPE,
"schema_version:header:" + ReservedHeaders.SCHEMA_VERSION,
"content_type:header:" + ReservedHeaders.CONTENT_TYPE,
"created_at:header:" + ReservedHeaders.PRODUCED_AT));
"created_at:header:" + ReservedHeaders.PRODUCED_AT,
"destination:header:" + ReservedHeaders.ORIGIN_DESTINATION,
"producer:header:" + ReservedHeaders.PRODUCER,
"occurred_at:header:" + ReservedHeaders.OCCURRED_AT,
"correlation_id:header:" + ReservedHeaders.CORRELATION_ID,
"causation_id:header:" + ReservedHeaders.CAUSATION_ID,
"tenant:header:" + ReservedHeaders.TENANT,
"partition_key:header:" + ReservedHeaders.PARTITION_KEY,
"ordering_key:header:" + ReservedHeaders.ORDERING_KEY,
"traceparent:header:" + ReservedHeaders.TRACEPARENT,
"tracestate:header:" + ReservedHeaders.TRACESTATE,
"baggage:header:" + ReservedHeaders.BAGGAGE));
return Map.copyOf(configuration);
}
@@ -71,6 +100,52 @@ public final class DebeziumOutboxEventRouter {
headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value()));
headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value());
headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString());
headers.put(ReservedHeaders.ORIGIN_DESTINATION, record.destination().value());
// Absent stays absent. Writing an empty string for a column the row does not have would make a
// message that carries no tenant indistinguishable from one whose tenant is the empty string,
// and the consumer would have to know which of the two this platform meant.
OutboxCanonicalMetadata metadata = record.metadata();
metadata.producer().ifPresent(value -> headers.put(ReservedHeaders.PRODUCER, value.value()));
metadata
.occurredAt()
.ifPresent(value -> headers.put(ReservedHeaders.OCCURRED_AT, value.toString()));
metadata
.correlationId()
.ifPresent(value -> headers.put(ReservedHeaders.CORRELATION_ID, value.value()));
metadata
.causationId()
.ifPresent(
value -> headers.put(ReservedHeaders.CAUSATION_ID, value.value().value().toString()));
metadata.tenant().ifPresent(value -> headers.put(ReservedHeaders.TENANT, value.tenantId()));
metadata.partitionKey().ifPresent(value -> headers.put(ReservedHeaders.PARTITION_KEY, value));
metadata.orderingKey().ifPresent(value -> headers.put(ReservedHeaders.ORDERING_KEY, value));
metadata
.traceContext()
.traceparent()
.ifPresent(value -> headers.put(ReservedHeaders.TRACEPARENT, value));
metadata
.traceContext()
.tracestate()
.ifPresent(value -> headers.put(ReservedHeaders.TRACESTATE, value));
metadata
.traceContext()
.baggage()
.ifPresent(value -> headers.put(ReservedHeaders.BAGGAGE, value));
return Map.copyOf(headers);
}
/**
* Returns the key a routed record carries.
*
* <p>The same value the {@code routing_key} column computes, stated here so a test can compare
* the two rather than trusting that the SQL and the Java agree.
*
* @param record the outbox row
* @return the broker message key
*/
public String routingKey(OutboxRecord record) {
Objects.requireNonNull(record, "record must not be null");
return record.metadata().partitionKey().orElseGet(() -> record.messageId().value().toString());
}
}
@@ -19,11 +19,17 @@ import java.util.Optional;
* <p>The message id comes from the row's {@code message_id} column, never from Debezium's own event
* id. Debezium mints a fresh event id per change record, so using it would break identity across a
* CDC re-snapshot — exactly the scenario where identity matters most.
*
* <p>The reserved headers come from {@link DebeziumOutboxEventRouter} rather than being rebuilt
* here. Two lists of the same headers is how the connector's configuration and the expectation this
* class encodes come to disagree, and the disagreement would be invisible: the test compares this
* output against the polling relay's, so it would pass while the connector emitted something else.
*/
public final class DebeziumOutboxRecordMapper {
private final String topicPrefix;
private final boolean aggregateIdAsPartitionKey;
private final DebeziumOutboxEventRouter router = new DebeziumOutboxEventRouter();
/**
* Creates a mapper from a CDC profile.
@@ -45,27 +51,29 @@ public final class DebeziumOutboxRecordMapper {
public DebeziumMappedRecord map(OutboxRecord record) {
Objects.requireNonNull(record, "record must not be null");
// Application headers first, canonical headers second, and the canonical ones win. The row's
// header map is application data; a row that wrote `msg.id` into it would otherwise overwrite
// the identity the platform put there, and the consumer trusts the last such header it sees.
// The write path refuses reserved names outright — this order is the second lock on the same
// door, because a CDC connector reading the table directly never passes through that check.
Map<String, String> headers = new LinkedHashMap<>(record.headers());
headers.put(ReservedHeaders.MESSAGE_ID, record.messageId().value().toString());
headers.put(ReservedHeaders.MESSAGE_TYPE, record.messageType().value());
headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value()));
headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value());
headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString());
headers.keySet().removeIf(ReservedHeaders::isReserved);
headers.putAll(router.reservedHeaders(record));
return new DebeziumMappedRecord(
topicPrefix + record.destination().value(),
partitionKey(record, headers),
partitionKey(record),
record.payload(),
headers);
}
private Optional<String> partitionKey(OutboxRecord record, Map<String, String> headers) {
private Optional<String> partitionKey(OutboxRecord record) {
if (!aggregateIdAsPartitionKey) {
return Optional.empty();
}
// The partition key header is the one the polling relay would also have set, so both relays
// land the same message on the same partition and neither reorders a keyed stream.
return Optional.ofNullable(headers.get(ReservedHeaders.PARTITION_KEY))
.or(() -> Optional.of(record.messageId().value().toString()));
// The same value the polling relay would use and the same value the `routing_key` column
// computes, so both relays land the same message on the same partition and neither reorders a
// keyed stream.
return Optional.of(router.routingKey(record));
}
}
@@ -1,11 +1,17 @@
package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.api.CausationId;
import dev.caskeleton.messaging.api.ContentType;
import dev.caskeleton.messaging.api.CorrelationId;
import dev.caskeleton.messaging.api.MessageId;
import dev.caskeleton.messaging.api.MessageType;
import dev.caskeleton.messaging.api.ProducerId;
import dev.caskeleton.messaging.api.SchemaVersion;
import dev.caskeleton.messaging.api.TenantContext;
import dev.caskeleton.messaging.api.TraceContext;
import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxLease;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.reliability.OutboxRepository;
@@ -45,10 +51,32 @@ public final class JdbcOutboxRepository implements OutboxRepository {
"""
INSERT INTO messaging_outbox (
message_id, destination, message_type, schema_version, content_type,
payload, headers, created_at, status, attempts)
VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?)
payload, headers, created_at, status, attempts,
producer, occurred_at, correlation_id, causation_id, tenant,
partition_key, ordering_key, schema_subject, schema_uri,
traceparent, tracestate, baggage)
VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?,
?, ?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?)
""";
/**
* The columns every read of a row selects.
*
* <p>One constant rather than four copies of the list. The lease, claim, find and reclaim
* statements each spelled it out, and a column added to one of them and not the others is a
* message that arrives with its tenant on the claim path and without it on the lease path — a
* difference that shows up as a data bug in whichever consumer happened to be reading.
*/
private static final String COLUMNS =
"""
message_id, destination, message_type, schema_version, content_type,
payload, headers, created_at, status, attempts, lease_expires_at, last_failure_code,
producer, occurred_at, correlation_id, causation_id, tenant,
partition_key, ordering_key, schema_subject, schema_uri,
traceparent, tracestate, baggage""";
private static final String LEASE =
"""
WITH claimable AS (
@@ -69,7 +97,10 @@ public final class JdbcOutboxRepository implements OutboxRepository {
WHERE o.message_id = c.message_id
RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type,
o.payload, o.headers, o.created_at, o.status, o.attempts,
o.lease_expires_at, o.last_failure_code
o.lease_expires_at, o.last_failure_code,
o.producer, o.occurred_at, o.correlation_id, o.causation_id, o.tenant,
o.partition_key, o.ordering_key, o.schema_subject, o.schema_uri,
o.traceparent, o.tracestate, o.baggage
""";
/**
@@ -104,7 +135,10 @@ public final class JdbcOutboxRepository implements OutboxRepository {
WHERE o.message_id = c.message_id
RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type,
o.payload, o.headers, o.created_at, o.status, o.attempts,
o.lease_expires_at, o.last_failure_code, o.lease_owner, o.lease_token
o.lease_expires_at, o.last_failure_code, o.lease_owner, o.lease_token,
o.producer, o.occurred_at, o.correlation_id, o.causation_id, o.tenant,
o.partition_key, o.ordering_key, o.schema_subject, o.schema_uri,
o.traceparent, o.tracestate, o.baggage
""";
private final DataSource dataSource;
@@ -144,6 +178,20 @@ public final class JdbcOutboxRepository implements OutboxRepository {
statement.setTimestamp(8, Timestamp.from(record.createdAt()));
statement.setString(9, record.status().name());
statement.setInt(10, record.attempts());
OutboxCanonicalMetadata metadata = record.metadata();
statement.setString(11, metadata.producer().map(ProducerId::value).orElse(null));
statement.setTimestamp(12, metadata.occurredAt().map(Timestamp::from).orElse(null));
statement.setString(13, metadata.correlationId().map(CorrelationId::value).orElse(null));
statement.setObject(
14, metadata.causationId().map(causation -> causation.value().value()).orElse(null));
statement.setString(15, metadata.tenant().map(TenantContext::tenantId).orElse(null));
statement.setString(16, metadata.partitionKey().orElse(null));
statement.setString(17, metadata.orderingKey().orElse(null));
statement.setString(18, metadata.schemaSubject().orElse(null));
statement.setString(19, metadata.schemaUri().orElse(null));
statement.setString(20, metadata.traceContext().traceparent().orElse(null));
statement.setString(21, metadata.traceContext().tracestate().orElse(null));
statement.setString(22, metadata.traceContext().baggage().orElse(null));
statement.executeUpdate();
} catch (SQLException exception) {
throw new MessagingConfigurationException(
@@ -425,9 +473,7 @@ public final class JdbcOutboxRepository implements OutboxRepository {
connection -> {
try (PreparedStatement statement =
connection.prepareStatement(
"SELECT message_id, destination, message_type, schema_version, content_type, "
+ "payload, headers, created_at, status, attempts, lease_expires_at, "
+ "last_failure_code FROM messaging_outbox WHERE message_id = ?")) {
"SELECT " + COLUMNS + " FROM messaging_outbox WHERE message_id = ?")) {
statement.setObject(1, messageId.value());
try (ResultSet results = statement.executeQuery()) {
return results.next() ? Optional.of(read(results)) : Optional.<OutboxRecord>empty();
@@ -509,6 +555,7 @@ public final class JdbcOutboxRepository implements OutboxRepository {
new ContentType(results.getString("content_type")),
results.getBytes("payload"),
fromJson(results.getString("headers")),
readMetadata(results),
results.getTimestamp("created_at").toInstant(),
OutboxStatus.valueOf(results.getString("status")),
results.getInt("attempts"),
@@ -516,6 +563,33 @@ public final class JdbcOutboxRepository implements OutboxRepository {
Optional.ofNullable(results.getString("last_failure_code")));
}
/**
* Rebuilds the canonical metadata from the row's own columns.
*
* <p>Absent columns stay absent. A row written before the columns existed rebuilds with empty
* metadata, which is the same thing a row written by a producer that had none rebuilds with — and
* the reader's behaviour has to be identical in both cases, because guessing a tenant or a
* correlation is worse than carrying none.
*/
private static OutboxCanonicalMetadata readMetadata(ResultSet results) throws SQLException {
Timestamp occurredAt = results.getTimestamp("occurred_at");
java.util.UUID causation = results.getObject("causation_id", java.util.UUID.class);
return new OutboxCanonicalMetadata(
Optional.ofNullable(results.getString("producer")).map(ProducerId::new),
Optional.ofNullable(occurredAt).map(Timestamp::toInstant),
Optional.ofNullable(results.getString("correlation_id")).map(CorrelationId::new),
Optional.ofNullable(causation).map(value -> new CausationId(new MessageId(value))),
Optional.ofNullable(results.getString("tenant")).map(TenantContext::new),
Optional.ofNullable(results.getString("partition_key")),
Optional.ofNullable(results.getString("ordering_key")),
Optional.ofNullable(results.getString("schema_subject")),
Optional.ofNullable(results.getString("schema_uri")),
new TraceContext(
Optional.ofNullable(results.getString("traceparent")),
Optional.ofNullable(results.getString("tracestate")),
Optional.ofNullable(results.getString("baggage"))));
}
private <T> T withConnection(ConnectionCallback<T> callback) {
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
@@ -2,13 +2,16 @@ package dev.caskeleton.messaging.outbox;
import dev.caskeleton.messaging.api.MessageEnvelope;
import dev.caskeleton.messaging.api.ProducerId;
import dev.caskeleton.messaging.api.TraceContext;
import dev.caskeleton.messaging.api.SchemaVersion;
import dev.caskeleton.messaging.api.header.HeaderName;
import dev.caskeleton.messaging.api.header.HeaderValue;
import dev.caskeleton.messaging.api.header.MessageHeaders;
import dev.caskeleton.messaging.api.header.ReservedHeaders;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.schema.EncodedMessage;
import dev.caskeleton.messaging.schema.SchemaReference;
import java.net.URI;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
@@ -20,6 +23,18 @@ import java.util.Optional;
* <p>The identity comes from the row, never from a fresh mint. That is what makes a retry after an
* ambiguous publish safe: the broker's deduplication, or the consumer's Inbox, sees the same id and
* collapses the duplicate.
*
* <p>So does everything else the envelope carries. This used to rebuild correlation, causation,
* tenant, trace and the schema reference as empty, and read the routing keys out of the row's
* header map — so a message that travelled through the outbox reached its consumer with less
* provenance than one published directly, and the publish path became part of the message's
* meaning. The row now has columns for those facts and this reads them.
*
* <p>Reserved header names in the row are refused outright, with no exception for the routing keys.
* They had one, because the keys had nowhere else to live; a row could therefore write {@code
* msg.partition-key} and the same code path would have accepted {@code msg.id} if the check had
* been one line looser. Now that the keys are columns, the rule is the simple one: an outbox row
* cannot write into the platform's namespace at all.
*/
public final class OutboxEnvelopeFactory {
@@ -28,7 +43,7 @@ public final class OutboxEnvelopeFactory {
/**
* Creates a factory for one service.
*
* @param producer the logical producing service
* @param producer the logical producing service, used for rows that name none
*/
public OutboxEnvelopeFactory(ProducerId producer) {
this.producer = Objects.requireNonNull(producer, "producer must not be null");
@@ -52,13 +67,7 @@ public final class OutboxEnvelopeFactory {
// MessageHeaders.platform let a row carrying `msg.id` or `msg.type` reach the wire
// as if the platform had written it — and the consumer trusts the last such header,
// so a forged identity lands in another message's inbox deduplication.
//
// Partition and ordering keys are the exception: they are reserved names the outbox
// legitimately carries as routing metadata, and they are read below rather than
// written as headers.
if (ReservedHeaders.isReserved(name)
&& !ReservedHeaders.PARTITION_KEY.equals(name)
&& !ReservedHeaders.ORDERING_KEY.equals(name)) {
if (ReservedHeaders.isReserved(name)) {
throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException(
"RESERVED_HEADER_IN_OUTBOX_ROW",
"outbox row header '"
@@ -66,26 +75,50 @@ public final class OutboxEnvelopeFactory {
+ "' uses the reserved namespace; canonical identity and metadata come from"
+ " the record's own columns, not from its header map");
}
if (!ReservedHeaders.isReserved(name)) {
headers.put(new HeaderName(name), new HeaderValue(value));
}
headers.put(new HeaderName(name), new HeaderValue(value));
});
OutboxCanonicalMetadata metadata = record.metadata();
return new MessageEnvelope<>(
record.messageId(),
record.messageType(),
record.schemaVersion(),
record.createdAt(),
Optional.of(record.createdAt()),
producer,
Optional.empty(),
Optional.empty(),
// The write instant stands in when the row records no separate occurrence. It is the
// closest true statement available: the business transaction that wrote the row is the one
// the fact occurred in.
Optional.of(metadata.occurredAt().orElse(record.createdAt())),
// A row written before the producer column existed names none, and the service that owns
// this relay is the only producer it can honestly be attributed to.
metadata.producer().orElse(producer),
metadata.correlationId(),
metadata.causationId(),
record.contentType(),
Optional.ofNullable(record.headers().get(ReservedHeaders.PARTITION_KEY)),
Optional.ofNullable(record.headers().get(ReservedHeaders.ORDERING_KEY)),
Optional.empty(),
TraceContext.none(),
metadata.partitionKey(),
metadata.orderingKey(),
metadata.tenant(),
metadata.traceContext(),
MessageHeaders.platform(headers),
new EncodedMessage(record.payload(), record.contentType(), Optional.empty()));
new EncodedMessage(
record.payload(), record.contentType(), schemaReference(record, metadata)));
}
/**
* Rebuilds the schema reference from the row's subject, the row's version, and its location.
*
* <p>The version is the row's {@code schema_version} column rather than a third copy of the same
* number: a reference whose version could disagree with the column the relay filters on is a
* reference nobody can trust.
*/
private static Optional<SchemaReference> schemaReference(
OutboxRecord record, OutboxCanonicalMetadata metadata) {
return metadata
.schemaSubject()
.map(
subject ->
new SchemaReference(
subject,
new SchemaVersion(record.schemaVersion().value()),
metadata.schemaUri().map(URI::create)));
}
}
@@ -87,6 +87,33 @@ public final class OutboxRelay {
OutboxRetryScheduler.standard());
}
/**
* A relay with an explicit retry policy and a process-derived owner.
*
* @param repository the outbox storage
* @param publisher the platform publisher
* @param envelopeFactory rebuilds envelopes from rows
* @param batchSize how many rows to claim per pass
* @param leaseDuration how long a claim holds
* @param scheduler the retry policy
*/
public OutboxRelay(
OutboxRepository repository,
MessagePublisher publisher,
OutboxEnvelopeFactory envelopeFactory,
int batchSize,
Duration leaseDuration,
OutboxRetryScheduler scheduler) {
this(
repository,
publisher,
envelopeFactory,
batchSize,
leaseDuration,
defaultOwner(),
scheduler);
}
/**
* A relay with an explicit retry policy.
*
@@ -134,6 +161,7 @@ public final class OutboxRelay {
int ambiguous = 0;
int failed = 0;
int stale = 0;
int exhausted = 0;
for (OutboxLease lease : batch) {
OutboxRecord record = lease.record();
@@ -147,10 +175,28 @@ public final class OutboxRelay {
}
}
case AMBIGUOUS -> {
if (repository.markAmbiguous(
lease, result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS"), now)
== OutboxTransitionResult.APPLIED) {
ambiguous++;
String code = result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS");
// The attempt this pass just spent. The claim predicate and the row both count attempts
// after the transition, so the budget has to be judged on the same number the next claim
// will read, or the last attempt is spent twice.
int spent = record.attempts() + 1;
OutboxTransitionResult transition =
scheduler
.parkReason(spent)
// A row that has spent its budget without an answer is parked under its own
// status. Leaving it AMBIGUOUS makes it a row the claim predicate silently skips
// forever, which looks identical to a healthy backlog on every dashboard.
.map(reason -> repository.markExhausted(lease, reason, now))
.orElseGet(
() ->
repository.markAmbiguous(
lease, code, now, scheduler.nextAttemptAt(now, spent)));
if (transition == OutboxTransitionResult.APPLIED) {
if (scheduler.isExhausted(spent)) {
exhausted++;
} else {
ambiguous++;
}
} else {
stale++;
}
@@ -171,7 +217,7 @@ public final class OutboxRelay {
throw new IllegalStateException("unhandled publish completion: " + result.completion());
}
}
return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale);
return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale, exhausted);
}
private PublishResult publish(OutboxRecord record) {
@@ -12,23 +12,39 @@ package dev.caskeleton.messaging.outbox;
* published something whose outcome another worker has already recorded, so a non-zero value is the
* visible form of a duplicate publication.
*
* <p>{@code exhausted} is counted apart from {@code failed} for the same reason: a rejected row is
* a message the broker refused and an exhausted one is a message nobody ever answered for. The
* first needs a fix, the second needs a redrive, and a single counter for both tells an operator
* neither.
*
* @param leased how many rows were claimed
* @param published how many were confirmed
* @param ambiguous how many had an unknown outcome and stay retryable
* @param failed how many were definitively rejected
* @param staleLeases how many terminal writes were refused because the lease had been superseded
* @param exhausted how many spent their attempt budget without a confirmation and were parked
*/
public record OutboxRelayReport(
int leased, int published, int ambiguous, int failed, int staleLeases) {
int leased, int published, int ambiguous, int failed, int staleLeases, int exhausted) {
public OutboxRelayReport {
if (leased < 0 || published < 0 || ambiguous < 0 || failed < 0 || staleLeases < 0) {
if (leased < 0
|| published < 0
|| ambiguous < 0
|| failed < 0
|| staleLeases < 0
|| exhausted < 0) {
throw new IllegalArgumentException("relay counters must not be negative");
}
}
/** A report from a pass that parked nothing. */
public OutboxRelayReport(int leased, int published, int ambiguous, int failed, int staleLeases) {
this(leased, published, ambiguous, failed, staleLeases, 0);
}
/** A report from a pass that observed no stale leases. */
public OutboxRelayReport(int leased, int published, int ambiguous, int failed) {
this(leased, published, ambiguous, failed, 0);
this(leased, published, ambiguous, failed, 0, 0);
}
}
@@ -0,0 +1,199 @@
package dev.caskeleton.messaging.outbox;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
/**
* Runs relay passes on the schedule the retry policy decides.
*
* <p>The relay, its retry scheduler and the attempt budget all existed and nothing ever called
* {@code runOnce}. An outbox whose relay is never driven is the worst shape of all: the business
* transaction commits its row and reports success, and the message is never published — a silent,
* unbounded backlog rather than a visible failure.
*
* <p>Passes are self-scheduled rather than run at a fixed rate. A fixed rate keeps firing while the
* broker is down, and the pass after an empty or failed one is exactly where the policy's backoff
* belongs; a pass that published something resets to the base interval because work is flowing.
*
* <p>One thread, and the next pass is scheduled only after the previous returns. Overlapping passes
* from one process would have two claims racing on the same rows, which the fencing predicate would
* then reject as stale — correct, but it is duplicate publishing work bought at the cost of a
* duplicate publish.
*/
public final class OutboxRelayWorker implements AutoCloseable {
/** How long {@link #close()} waits for a running pass when no deadline is given. */
public static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30);
private final OutboxRelay relay;
private final OutboxRetryScheduler scheduler;
private final Supplier<Instant> clock;
private final ScheduledExecutorService passes;
private final AtomicBoolean running = new AtomicBoolean();
private final AtomicInteger unproductivePasses = new AtomicInteger();
private final AtomicLong completedPasses = new AtomicLong();
private final AtomicLong failedPasses = new AtomicLong();
/**
* Creates a worker driving one relay.
*
* @param relay the relay to run
* @param scheduler the policy deciding when the next pass runs
*/
public OutboxRelayWorker(OutboxRelay relay, OutboxRetryScheduler scheduler) {
this(relay, scheduler, Instant::now);
}
/**
* Creates a worker with an explicit clock, so a pass can be driven without waiting on wall time.
*
* @param relay the relay to run
* @param scheduler the policy deciding when the next pass runs
* @param clock the source of the current instant
*/
public OutboxRelayWorker(
OutboxRelay relay, OutboxRetryScheduler scheduler, Supplier<Instant> clock) {
this.relay = Objects.requireNonNull(relay, "relay must not be null");
this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null");
this.clock = Objects.requireNonNull(clock, "clock must not be null");
ScheduledThreadPoolExecutor executor =
new ScheduledThreadPoolExecutor(
1,
runnable -> {
// Daemon: a relay thread that outlives the application would keep publishing from a
// process that has already released its other resources.
Thread thread = new Thread(runnable, "messaging-outbox-relay");
thread.setDaemon(true);
return thread;
});
// A pass waiting out its backoff is dropped on shutdown rather than run afterwards. Left on,
// the default policy makes stop() wait for a pass that has not started and cannot do anything
// useful, so an orderly shutdown pays the whole poll interval before it can close.
executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
this.passes = executor;
}
/** Starts driving passes. Idempotent: a second call does not add a second schedule. */
public void start() {
if (running.compareAndSet(false, true)) {
schedule(Duration.ZERO);
}
}
/**
* Stops scheduling and lets the pass that is already running finish.
*
* <p>Draining rather than interrupting is the whole point. A pass killed between its claim and
* its terminal write leaves rows {@code IN_FLIGHT} holding a lease, and nothing may touch them
* until that lease expires — so an orderly shutdown would produce exactly the stall that a crash
* produces.
*
* @param deadline how long to wait for the running pass
* @return true when the running pass finished inside the deadline
*/
public boolean stop(Duration deadline) {
Objects.requireNonNull(deadline, "deadline must not be null");
running.set(false);
passes.shutdown();
try {
if (passes.awaitTermination(deadline.toMillis(), TimeUnit.MILLISECONDS)) {
return true;
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
// The deadline passed with a pass still running. Its rows stay leased until the lease expires,
// which is the outcome the deadline exists to bound rather than to prevent.
passes.shutdownNow();
return false;
}
@Override
public void close() {
stop(DEFAULT_DRAIN_DEADLINE);
}
/**
* Reports whether the worker is scheduling passes.
*
* @return true between {@link #start()} and {@link #stop(Duration)}
*/
public boolean isRunning() {
return running.get();
}
/**
* Returns how many passes have finished, successfully or not.
*
* @return the completed pass count
*/
public long completedPasses() {
return completedPasses.get();
}
/**
* Returns how many passes ended in an exception.
*
* @return the failed pass count
*/
public long failedPasses() {
return failedPasses.get();
}
// The scheduled future is not kept: a pass reports nothing to a caller and handles its own
// failure below, so the only thing holding it would achieve is a reference that has to be
// cleaned up.
@SuppressWarnings("FutureReturnValueIgnored")
private void schedule(Duration delay) {
if (!running.get()) {
return;
}
try {
passes.schedule(this::runPass, Math.max(0, delay.toMillis()), TimeUnit.MILLISECONDS);
} catch (RejectedExecutionException stopping) {
// stop() shut the executor between the running check and the submission. There is nothing to
// schedule onto and nothing to report: the worker is stopping, which is what was asked for.
running.set(false);
}
}
private void runPass() {
if (!running.get()) {
return;
}
int unproductive;
try {
OutboxRelayReport report = relay.runOnce(clock.get());
unproductive = report.published() > 0 ? reset() : unproductivePasses.incrementAndGet();
} catch (RuntimeException passFailed) {
// A failed pass must not stop the loop: the scheduled task's own exception would cancel every
// future pass, turning one broker error into a relay that never runs again. The failure is
// counted and the next pass backs off as if nothing was published, which is true.
failedPasses.incrementAndGet();
unproductive = unproductivePasses.incrementAndGet();
}
completedPasses.incrementAndGet();
schedule(scheduler.backoff(unproductive));
}
private int reset() {
unproductivePasses.set(0);
return 0;
}
}
@@ -8,10 +8,12 @@ import java.util.Optional;
/**
* Decides when a relay's next pass should run, and when a record has retried enough.
*
* <p>The backoff is on the <em>relay</em>, not on the individual record. A broker outage fails
* every record in the batch for the same reason, so backing off per record would hammer the broker
* with the whole backlog on a schedule while achieving nothing; backing off the pass lets the
* broker recover.
* <p>The same curve serves two different jobs and neither replaces the other. Backing off the
* <em>pass</em> is what lets a broker recover: an outage fails every record in the batch for the
* same reason, so retrying the whole backlog on the poll interval achieves nothing but load. But a
* pass-level wait is process memory — it is forgotten on restart and each replica keeps its own —
* so eligibility also has to be written on the row, or the backlog is retried as fast as the
* unluckiest replica polls.
*
* <p>Jitter is applied deterministically from the attempt count rather than randomly. Several relay
* instances that all started at deployment time would otherwise synchronise their retries into a
@@ -58,6 +60,25 @@ public final class OutboxRetryScheduler {
return now.plus(backoff(consecutiveEmptyOrFailedPasses));
}
/**
* Returns when a row that has spent this many attempts may be claimed again.
*
* <p>Written onto the row rather than held in the relay, because a deadline that lives in process
* memory is forgotten on restart and computed independently by every replica — which turns a
* backoff into every worker re-publishing the same backlog as fast as it polls.
*
* @param now the current instant
* @param attempts how many attempts the row has spent
* @return the earliest instant the row is eligible again
*/
public Instant nextAttemptAt(Instant now, int attempts) {
Objects.requireNonNull(now, "now must not be null");
if (attempts < 1) {
throw new IllegalArgumentException("a row that has been published once has spent an attempt");
}
return now.plus(backoff(attempts));
}
/** The platform's default policy, for a relay whose caller did not supply one. */
public static OutboxRetryScheduler standard() {
return new OutboxRetryScheduler(OutboxProperties.defaults(), java.time.Duration.ofMinutes(5));
@@ -0,0 +1,66 @@
-- The envelope's canonical facts, as columns.
--
-- V1 stored identity, type, version, content type, payload and a free-form header map. Everything
-- else the envelope carries — producer, tenant, correlation, causation, the routing keys, the
-- schema the payload was written against, the active trace — had nowhere to go, so it was either
-- lost when the relay rebuilt the envelope or smuggled through the header map under the reserved
-- `msg.*` names the platform is supposed to own. Smuggling is the worse of the two: a row whose
-- header map contains `msg.id` overwrites another message's identity on the wire, and the consumer
-- trusts the last such header it sees.
--
-- Columns rather than a versioned envelope blob. Both round-trip the values faithfully; only one of
-- them lets the relay answer an operator's questions. "Which tenant is the backlog for", "which
-- correlation is stuck", "which rows carry a schema this consumer cannot read" are SELECTs against
-- this table if the fields are columns, and payload decoding of the whole backlog if they are not.
--
-- Every column is nullable. A row written before this migration has none of them, and a producer
-- that genuinely has no correlation writes none either; the reader treats both the same way, which
-- is to carry what is there and invent nothing.
ALTER TABLE messaging_outbox
ADD COLUMN producer VARCHAR(120),
ADD COLUMN occurred_at TIMESTAMPTZ,
ADD COLUMN correlation_id VARCHAR(160),
ADD COLUMN causation_id UUID,
ADD COLUMN tenant VARCHAR(64),
ADD COLUMN partition_key VARCHAR(1024),
ADD COLUMN ordering_key VARCHAR(1024),
ADD COLUMN schema_subject VARCHAR(240),
ADD COLUMN schema_uri VARCHAR(1024),
ADD COLUMN traceparent VARCHAR(64),
ADD COLUMN tracestate VARCHAR(512),
ADD COLUMN baggage VARCHAR(8192);
-- The same bound TenantContext enforces in Java. Stated here as well because the relay, the CDC
-- connector and any operator query read this table directly: a tenant slug that only the
-- application validates is a tenant slug that an INSERT from anywhere else can violate, and a
-- tenant id is the field every downstream authorization decision reads.
ALTER TABLE messaging_outbox
ADD CONSTRAINT ck_messaging_outbox_tenant
CHECK (tenant IS NULL OR tenant ~ '^[a-z0-9][a-z0-9._-]{0,63}$');
-- A schema location with no subject cannot be resolved: the reader would hold a URI and no way to
-- know what it is a schema for.
ALTER TABLE messaging_outbox
ADD CONSTRAINT ck_messaging_outbox_schema_reference
CHECK (schema_uri IS NULL OR schema_subject IS NOT NULL);
-- "Which tenant is the backlog for" is the first question asked when an outbox stops draining, and
-- before this column it could only be answered by decoding every payload. Partial, like the other
-- outbox indexes, because PUBLISHED rows accumulate until retention removes them and no operator
-- asks that question about work that already left.
CREATE INDEX ix_messaging_outbox_tenant_backlog
ON messaging_outbox (tenant, created_at)
WHERE status IN ('PENDING', 'AMBIGUOUS', 'IN_FLIGHT');
-- The key a relay puts on the wire, computed once and stored, so both relays agree.
--
-- Debezium's Event Router takes the message key from a column. It was pointed at `destination`,
-- which made the key the topic name — every message on a topic sharing one key, so every message
-- landing on one partition, and keyed ordering meaning nothing. The polling relay meanwhile used
-- the partition key when the row had one and the message id when it did not.
--
-- A generated column states that fallback once, in the place both relays read, instead of leaving
-- it as a rule each of them implements separately and one of them gets wrong.
ALTER TABLE messaging_outbox
ADD COLUMN routing_key TEXT
GENERATED ALWAYS AS (COALESCE(partition_key, message_id::TEXT)) STORED;
@@ -12,6 +12,7 @@ import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
import dev.caskeleton.messaging.api.header.ReservedHeaders;
import dev.caskeleton.messaging.outbox.DebeziumOutboxProfile.RelayMode;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.reliability.OutboxStatus;
import java.nio.charset.StandardCharsets;
@@ -26,6 +27,11 @@ class DebeziumOutboxRecordMapperTest {
private static final MessageId MESSAGE_ID = MessageId.newId();
private static OutboxRecord record(Map<String, String> headers) {
return record(headers, OutboxCanonicalMetadata.none());
}
private static OutboxRecord record(
Map<String, String> headers, OutboxCanonicalMetadata metadata) {
return new OutboxRecord(
MESSAGE_ID,
new DestinationName("orders.v1"),
@@ -34,6 +40,7 @@ class DebeziumOutboxRecordMapperTest {
ContentType.JSON,
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
headers,
metadata,
NOW,
OutboxStatus.PENDING,
0,
@@ -89,13 +96,71 @@ class DebeziumOutboxRecordMapperTest {
@Test
void aKeyedProfileUsesTheSamePartitionKeyThePollingRelayWouldHave() {
DebeziumMappedRecord mapped =
mapper(true).map(record(Map.of(ReservedHeaders.PARTITION_KEY, "customer-42")));
mapper(true)
.map(
record(
Map.of(),
OutboxCanonicalMetadata.none().withRoutingKeys("customer-42", "customer-42")));
assertThat(mapped.key())
.as("a different key would send the two relays' messages to different partitions")
.hasValue("customer-42");
}
@Test
void aKeyedProfileFallsBackToTheMessageIdWhenTheRowHasNoPartitionKey() {
assertThat(mapper(true).map(record(Map.of())).key())
.as("the fallback the routing_key column computes, stated in one place for both relays")
.hasValue(MESSAGE_ID.value().toString());
}
@Test
void theCanonicalColumnsBecomeReservedHeaders() {
OutboxCanonicalMetadata metadata =
OutboxCanonicalMetadata.none()
.withProducer(new dev.caskeleton.messaging.api.ProducerId("order-api"))
.withTenant(new dev.caskeleton.messaging.api.TenantContext("acme"))
.withRoutingKeys("customer-42", "customer-42");
Map<String, String> headers = mapper(false).map(record(Map.of(), metadata)).headers();
assertThat(headers)
.as("a relay that drops these makes the publish path part of the message's meaning")
.containsEntry(ReservedHeaders.PRODUCER, "order-api")
.containsEntry(ReservedHeaders.TENANT, "acme")
.containsEntry(ReservedHeaders.PARTITION_KEY, "customer-42")
.containsEntry(ReservedHeaders.ORDERING_KEY, "customer-42")
.containsEntry(ReservedHeaders.ORIGIN_DESTINATION, "orders.v1");
}
@Test
void aColumnTheRowDoesNotCarryProducesNoHeaderAtAll() {
assertThat(mapper(false).map(record(Map.of())).headers())
.as("an empty string would be indistinguishable from a tenant literally named ''")
.doesNotContainKeys(
ReservedHeaders.TENANT, ReservedHeaders.CORRELATION_ID, ReservedHeaders.TRACEPARENT);
}
@Test
void aForgedReservedHeaderOnTheRowCannotOverwriteTheCanonicalOne() {
Map<String, String> forged =
Map.of(ReservedHeaders.MESSAGE_ID, "00000000-0000-7000-8000-000000000000");
assertThat(mapper(false).map(record(forged)).headers())
.as("a CDC connector reads the table directly and never passes the write-path check")
.containsEntry(ReservedHeaders.MESSAGE_ID, MESSAGE_ID.value().toString());
}
@Test
void theRoutedKeyIsNotTheTopicName() {
// Literals, not the class's own constants: comparing a configuration value against the constant
// that produced it asserts that the router agrees with itself, which it always will.
assertThat(new DebeziumOutboxEventRouter().connectorConfiguration("prod."))
.as("keying by destination puts every message on a topic onto one partition")
.containsEntry("transforms.outbox.table.field.event.key", "routing_key")
.containsEntry("transforms.outbox.route.by.field", "destination");
}
@Test
void thePayloadIsUnchangedByTheTransform() {
assertThat(mapper(false).map(record(Map.of())).payload())
@@ -95,6 +95,7 @@ class JdbcOutboxTransactionRequirementTest {
ContentType.JSON,
"{}".getBytes(java.nio.charset.StandardCharsets.UTF_8),
Map.of(),
dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata.none(),
Instant.parse("2026-08-14T00:00:00Z"),
OutboxStatus.PENDING,
0,
@@ -0,0 +1,158 @@
package dev.caskeleton.messaging.outbox;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import dev.caskeleton.messaging.api.CausationId;
import dev.caskeleton.messaging.api.ContentType;
import dev.caskeleton.messaging.api.CorrelationId;
import dev.caskeleton.messaging.api.MessageEnvelope;
import dev.caskeleton.messaging.api.MessageId;
import dev.caskeleton.messaging.api.MessageType;
import dev.caskeleton.messaging.api.ProducerId;
import dev.caskeleton.messaging.api.SchemaVersion;
import dev.caskeleton.messaging.api.TenantContext;
import dev.caskeleton.messaging.api.TraceContext;
import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
import dev.caskeleton.messaging.api.header.ReservedHeaders;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.reliability.OutboxStatus;
import dev.caskeleton.messaging.schema.EncodedMessage;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Map;
import java.util.Optional;
import org.junit.jupiter.api.Test;
/**
* What survives a trip through the database.
*
* <p>The envelope this factory rebuilds used to be poorer than the one that was written: producer
* came from the relay's own configuration, and correlation, causation, tenant, trace and the schema
* reference were all rebuilt as empty. A message published directly and the same message published
* through the outbox therefore arrived at the consumer as two different messages, and which one a
* service got depended on a deployment decision it could not see.
*/
class OutboxEnvelopeFactoryTest {
private static final Instant WRITTEN_AT = Instant.parse("2026-08-10T09:15:00Z");
private static final Instant OCCURRED_AT = Instant.parse("2026-08-10T09:14:00Z");
private static final MessageId CAUSE = MessageId.newId();
private static final String TRACEPARENT =
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
private final OutboxEnvelopeFactory factory =
new OutboxEnvelopeFactory(new ProducerId("relay-owner"));
@Test
void everyCanonicalFieldOnTheRowReachesTheEnvelope() {
MessageEnvelope<EncodedMessage> envelope = factory.toEnvelope(record(fullMetadata(), Map.of()));
assertThat(envelope.producer()).isEqualTo(new ProducerId("order-api"));
assertThat(envelope.occurredAt()).contains(OCCURRED_AT);
assertThat(envelope.correlationId()).contains(new CorrelationId("checkout-7"));
assertThat(envelope.causationId()).contains(new CausationId(CAUSE));
assertThat(envelope.tenantContext()).contains(new TenantContext("acme"));
assertThat(envelope.partitionKey()).contains("customer-42");
assertThat(envelope.orderingKey()).contains("customer-42");
assertThat(envelope.traceContext().traceparent()).contains(TRACEPARENT);
assertThat(envelope.payload().schemaReference())
.hasValueSatisfying(
reference -> {
assertThat(reference.subject()).isEqualTo("order.created-value");
assertThat(reference.version()).isEqualTo(new SchemaVersion(1));
assertThat(reference.schemaUri()).contains(java.net.URI.create("urn:schema:1"));
});
}
@Test
void aTenantIsNeverReconstructedAsAbsent() {
assertThat(factory.toEnvelope(record(fullMetadata(), Map.of())).tenantContext())
.as("a message published by one tenant and consumed as belonging to none is an escalation")
.contains(new TenantContext("acme"));
}
@Test
void aRowWithoutAProducerIsAttributedToTheServiceThatOwnsTheRelay() {
MessageEnvelope<EncodedMessage> envelope =
factory.toEnvelope(record(OutboxCanonicalMetadata.none(), Map.of()));
assertThat(envelope.producer())
.as("a row written before the column existed has no honest producer but this one")
.isEqualTo(new ProducerId("relay-owner"));
assertThat(envelope.occurredAt())
.as("the transaction that wrote the row is the one the fact occurred in")
.contains(WRITTEN_AT);
assertThat(envelope.tenantContext()).isEmpty();
assertThat(envelope.payload().schemaReference()).isEmpty();
}
@Test
void applicationHeadersOnTheRowStillTravel() {
assertThat(
factory
.toEnvelope(record(OutboxCanonicalMetadata.none(), Map.of("x-tenant-tier", "gold")))
.headers()
.find("x-tenant-tier"))
.hasValueSatisfying(value -> assertThat(value.value()).isEqualTo("gold"));
}
@Test
void aReservedHeaderOnTheRowIsRefusedEvenWhenItIsARoutingKey() {
assertThatThrownBy(
() ->
factory.toEnvelope(
record(
OutboxCanonicalMetadata.none(),
Map.of(ReservedHeaders.PARTITION_KEY, "customer-42"))))
.as("the routing keys were the exception that kept the reserved-name door ajar")
.isInstanceOf(MessagingConfigurationException.class)
.hasMessageContaining("reserved namespace");
}
@Test
void aForgedMessageIdOnTheRowCannotOverwriteTheCanonicalOne() {
assertThatThrownBy(
() ->
factory.toEnvelope(
record(
OutboxCanonicalMetadata.none(),
Map.of(ReservedHeaders.MESSAGE_ID, MessageId.newId().value().toString()))))
.isInstanceOf(MessagingConfigurationException.class)
.hasMessageContaining(ReservedHeaders.MESSAGE_ID);
}
private static OutboxCanonicalMetadata fullMetadata() {
return new OutboxCanonicalMetadata(
Optional.of(new ProducerId("order-api")),
Optional.of(OCCURRED_AT),
Optional.of(new CorrelationId("checkout-7")),
Optional.of(new CausationId(CAUSE)),
Optional.of(new TenantContext("acme")),
Optional.of("customer-42"),
Optional.of("customer-42"),
Optional.of("order.created-value"),
Optional.of("urn:schema:1"),
new TraceContext(Optional.of(TRACEPARENT), Optional.empty(), Optional.empty()));
}
private static OutboxRecord record(
OutboxCanonicalMetadata metadata, Map<String, String> headers) {
return new OutboxRecord(
MessageId.newId(),
new DestinationName("order-events"),
new MessageType("order.created"),
new SchemaVersion(1),
ContentType.JSON,
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
headers,
metadata,
WRITTEN_AT,
OutboxStatus.PENDING,
0,
Optional.empty(),
Optional.empty());
}
}
@@ -7,6 +7,7 @@ import dev.caskeleton.messaging.api.MessageId;
import dev.caskeleton.messaging.api.MessageType;
import dev.caskeleton.messaging.api.SchemaVersion;
import dev.caskeleton.messaging.api.destination.DestinationName;
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.reliability.OutboxStatus;
import dev.caskeleton.messaging.testkit.DockerAvailability;
@@ -94,7 +95,9 @@ class OutboxPostgresIT {
assertThat(leasedRecord.messageId()).isEqualTo(record.messageId());
assertThat(leasedRecord.messageType()).isEqualTo(record.messageType());
assertThat(leasedRecord.payload()).isEqualTo(record.payload());
assertThat(leasedRecord.headers()).containsEntry("msg.partition-key", "acct-1");
assertThat(leasedRecord.metadata().partitionKey())
.as("the routing key is a column the relay can read without decoding the payload")
.contains("acct-1");
});
}
@@ -200,6 +203,12 @@ class OutboxPostgresIT {
}
private static OutboxRecord record() {
return record(
// The routing key is a column now, not a reserved header smuggled through the header map.
OutboxCanonicalMetadata.none().withRoutingKeys("acct-1", "acct-1"));
}
private static OutboxRecord record(OutboxCanonicalMetadata metadata) {
return new OutboxRecord(
MessageId.newId(),
new DestinationName("order-events"),
@@ -207,7 +216,8 @@ class OutboxPostgresIT {
new SchemaVersion(1),
ContentType.JSON,
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
Map.of("msg.partition-key", "acct-1"),
Map.of(),
metadata,
NOW.minusSeconds(60),
OutboxStatus.PENDING,
0,
@@ -223,9 +233,67 @@ class OutboxPostgresIT {
return source;
}
/**
* Every migration in the module, in version order.
*
* <p>Discovered rather than listed. The list said V1 and V2 above a comment claiming it was every
* migration, so V3 and V4 were never applied here — and the first symptom of that was not a
* missing check but this suite failing against a column the production schema has. A test that
* applies part of the schema certifies a database nobody deploys.
*/
private static java.util.List<String> migrations() throws IOException {
return java.util.List.of(
migration("V1__messaging_outbox.sql"), migration("V2__messaging_outbox_lease_fencing.sql"));
java.net.URL directory =
OutboxPostgresIT.class.getClassLoader().getResource("db/migration/messaging");
if (directory == null) {
throw new IOException("the messaging migrations are missing from the classpath");
}
java.io.File[] files =
new java.io.File(java.net.URI.create(directory.toString()))
.listFiles((dir, name) -> name.endsWith(".sql"));
if (files == null || files.length == 0) {
throw new IOException("no migration applied; the schema under test would be empty");
}
java.util.List<String> names =
java.util.Arrays.stream(files).map(java.io.File::getName).sorted().toList();
java.util.List<String> statements = new java.util.ArrayList<>();
for (String name : names) {
statements.add(migration(name));
}
return statements;
}
private int countWhere(String predicate) {
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement();
java.sql.ResultSet results =
statement.executeQuery("SELECT count(*) FROM messaging_outbox WHERE " + predicate)) {
results.next();
return results.getInt(1);
} catch (SQLException failure) {
throw new IllegalStateException("could not count outbox rows", failure);
}
}
private String routingKeyOf(MessageId messageId) {
try (Connection connection = dataSource.getConnection();
java.sql.PreparedStatement statement =
connection.prepareStatement(
"SELECT routing_key FROM messaging_outbox WHERE message_id = ?")) {
statement.setObject(1, messageId.value());
try (java.sql.ResultSet results = statement.executeQuery()) {
results.next();
return results.getString(1);
}
} catch (SQLException failure) {
throw new IllegalStateException("could not read the routing key", failure);
}
}
private void execute(String sql) throws SQLException {
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) {
statement.execute(sql);
}
}
private static String migration(String fileName) throws IOException {
@@ -385,6 +453,84 @@ class OutboxPostgresIT {
.isEqualTo(OutboxStatus.PUBLISHED);
}
@Test
void everyCanonicalColumnRoundTripsThroughTheDatabase() {
MessageId cause = MessageId.newId();
OutboxCanonicalMetadata written =
new OutboxCanonicalMetadata(
Optional.of(new dev.caskeleton.messaging.api.ProducerId("order-api")),
Optional.of(NOW.minusSeconds(120)),
Optional.of(new dev.caskeleton.messaging.api.CorrelationId("checkout-7")),
Optional.of(new dev.caskeleton.messaging.api.CausationId(cause)),
Optional.of(new dev.caskeleton.messaging.api.TenantContext("acme")),
Optional.of("customer-42"),
Optional.of("customer-42"),
Optional.of("order.created-value"),
Optional.of("urn:schema:1"),
new dev.caskeleton.messaging.api.TraceContext(
Optional.of("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
Optional.empty(),
Optional.empty()));
OutboxRecord record = record(written);
appendInTransaction(record);
assertThat(repository.find(record.messageId()).orElseThrow().metadata())
.as("what the relay reads back has to be what the business transaction wrote")
.isEqualTo(written);
}
@Test
void theRelayCanSelectOneTenantsBacklogWithoutDecodingAPayload() {
appendInTransaction(
record(
OutboxCanonicalMetadata.none()
.withTenant(new dev.caskeleton.messaging.api.TenantContext("acme"))));
appendInTransaction(
record(
OutboxCanonicalMetadata.none()
.withTenant(new dev.caskeleton.messaging.api.TenantContext("globex"))));
assertThat(countWhere("tenant = 'acme'"))
.as("the operational question an outbox backlog always raises, answerable as a SELECT")
.isEqualTo(1);
}
@Test
void theStoredRoutingKeyIsTheOneBothRelaysWouldUse() {
OutboxRecord keyed =
record(OutboxCanonicalMetadata.none().withRoutingKeys("customer-42", "customer-42"));
OutboxRecord unkeyed = record(OutboxCanonicalMetadata.none());
appendInTransaction(keyed);
appendInTransaction(unkeyed);
DebeziumOutboxEventRouter router = new DebeziumOutboxEventRouter();
assertThat(routingKeyOf(keyed.messageId()))
.as("the CDC key used to be the destination, so every message on a topic shared a key")
.isEqualTo(router.routingKey(keyed))
.isEqualTo("customer-42");
assertThat(routingKeyOf(unkeyed.messageId()))
.as("the fallback is stated once, in the column, and the Java agrees with it")
.isEqualTo(router.routingKey(unkeyed))
.isEqualTo(unkeyed.messageId().value().toString());
}
@Test
void aTenantThatBreaksTheSlugBoundIsRefusedByTheDatabase() {
org.assertj.core.api.Assertions.assertThatThrownBy(
() ->
execute(
"INSERT INTO messaging_outbox (message_id, destination, message_type, "
+ "schema_version, content_type, payload, created_at, status, attempts, "
+ "tenant) VALUES ('"
+ MessageId.newId().value()
+ "', 'order-events', 'order.created', 1, 'application/json', '\\x7b7d', "
+ "now(), 'PENDING', 0, 'Tenant With Spaces')"))
.as("a CDC connector and an operator both write this table without passing through Java")
.isInstanceOf(SQLException.class)
.hasMessageContaining("ck_messaging_outbox_tenant");
}
@Test
void aHeaderValueWithControlCharactersRoundTrips() {
OutboxRecord record =
@@ -398,7 +544,8 @@ class OutboxPostgresIT {
// A tab and a newline in a header value. Only backslash and quote were escaped, so
// this produced a document PostgreSQL refuses as invalid JSONB and the append failed
// with an error naming neither the header nor the row.
Map.of("msg.note", "line one\nline\ttwo"),
Map.of("x-note", "line one\nline\ttwo"),
OutboxCanonicalMetadata.none(),
NOW.minusSeconds(60),
OutboxStatus.PENDING,
0,
@@ -408,7 +555,7 @@ class OutboxPostgresIT {
appendInTransaction(record);
assertThat(repository.find(record.messageId()).orElseThrow().headers())
.containsEntry("msg.note", "line one\nline\ttwo");
.containsEntry("x-note", "line one\nline\ttwo");
}
private void appendInTransaction(OutboxRecord record) {
@@ -80,6 +80,41 @@ class OutboxRelayTest {
assertThat(publisher.publishedIds.get(1)).isEqualTo(record.messageId());
}
@Test
void anAmbiguousRowWaitsForItsBackoffBeforeItIsClaimedAgain() {
OutboxRecord record = OutboxFixtures.pending();
repository.append(record);
RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous();
relay(publisher).runOnce(NOW);
OutboxRelayReport tooSoon = relay(publisher).runOnce(NOW.plusMillis(100));
assertThat(tooSoon.leased())
.as("a backoff nothing writes on the row is a backoff the next pass cannot see")
.isZero();
assertThat(publisher.publishedIds).hasSize(1);
}
@Test
void aRowThatSpendsItsAttemptBudgetIsParkedRatherThanLeftRetryable() {
OutboxRecord record = OutboxFixtures.pending();
repository.append(record);
RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous();
OutboxRelay relay = relay(publisher, budgetOf(2));
relay.runOnce(NOW);
OutboxRelayReport last = relay.runOnce(NOW.plus(Duration.ofHours(1)));
assertThat(last.exhausted())
.as("a rejected row needs a fix and a spent one needs a redrive")
.isEqualTo(1);
assertThat(repository.find(record.messageId()).orElseThrow().status())
.isEqualTo(OutboxStatus.EXHAUSTED);
assertThat(relay.runOnce(NOW.plus(Duration.ofDays(1))).leased())
.as("only an operator redrive brings a parked row back")
.isZero();
}
@Test
void aRejectedPublishIsMarkedFailedAndNotRetried() {
OutboxRecord record = OutboxFixtures.pending();
@@ -135,6 +170,30 @@ class OutboxRelayTest {
100,
Duration.ofSeconds(30));
}
private OutboxRelay relay(MessagePublisher publisher, OutboxRetryScheduler scheduler) {
return new OutboxRelay(
repository,
publisher,
new OutboxEnvelopeFactory(new ProducerId("order-api")),
100,
Duration.ofSeconds(30),
"relay-under-test",
scheduler);
}
private static OutboxRetryScheduler budgetOf(int maxAttempts) {
OutboxProperties defaults = OutboxProperties.defaults();
return new OutboxRetryScheduler(
new OutboxProperties(
defaults.batchSize(),
defaults.leaseDuration(),
defaults.publishTimeout(),
defaults.pollInterval(),
defaults.retentionAfterPublish(),
maxAttempts),
Duration.ofMinutes(5));
}
}
/** Builds outbox rows. */
@@ -151,6 +210,7 @@ final class OutboxFixtures {
ContentType.JSON,
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
Map.of(),
dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata.none(),
Instant.parse("2026-08-10T09:14:00Z"),
OutboxStatus.PENDING,
0,
@@ -0,0 +1,163 @@
package dev.caskeleton.messaging.outbox;
import static org.assertj.core.api.Assertions.assertThat;
import dev.caskeleton.messaging.api.MessageEnvelope;
import dev.caskeleton.messaging.api.ProducerId;
import dev.caskeleton.messaging.api.destination.MessageDestination;
import dev.caskeleton.messaging.api.publish.ConfirmationLevel;
import dev.caskeleton.messaging.api.publish.MessagePublisher;
import dev.caskeleton.messaging.api.publish.PublishCompletion;
import dev.caskeleton.messaging.api.publish.PublishEvidence;
import dev.caskeleton.messaging.api.publish.PublishOptions;
import dev.caskeleton.messaging.api.publish.PublishResult;
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
import dev.caskeleton.messaging.reliability.OutboxRecord;
import dev.caskeleton.messaging.reliability.OutboxStatus;
import java.time.Duration;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
/**
* Whether anything actually drives the relay.
*
* <p>The relay, its scheduler and the attempt budget were all constructed by the auto-configuration
* and {@code runOnce} was called by no production code. A business transaction would commit its
* outbox row, report success, and the message would never be published — a silent backlog rather
* than a visible failure.
*/
class OutboxRelayWorkerTest {
private static final Duration PATIENCE = Duration.ofSeconds(5);
private final InMemoryOutboxRepository repository = new InMemoryOutboxRepository();
@Test
@DisplayName("a started worker publishes the backlog without anyone calling a pass")
void aStartedWorkerPublishesTheBacklog() throws InterruptedException {
OutboxRecord record = OutboxFixtures.pending();
repository.append(record);
OutboxRelayWorker worker = worker(RecordingOutboxPublisher.confirming());
worker.start();
try {
awaitStatus(record, OutboxStatus.PUBLISHED);
} finally {
worker.stop(PATIENCE);
}
assertThat(repository.find(record.messageId()).orElseThrow().status())
.isEqualTo(OutboxStatus.PUBLISHED);
assertThat(worker.completedPasses()).isPositive();
}
@Test
@DisplayName("stopping waits for the running pass instead of stranding its claimed rows")
void stoppingWaitsForTheRunningPass() throws InterruptedException {
OutboxRecord record = OutboxFixtures.pending();
repository.append(record);
// The publish is still in progress when stop() is called, so the row is claimed and IN_FLIGHT
// at that moment: the pass has to be allowed to finish or the lease strands the row.
SlowPublisher publisher = new SlowPublisher(Duration.ofMillis(300));
OutboxRelayWorker worker = worker(publisher);
worker.start();
assertThat(publisher.started.await(PATIENCE.toMillis(), TimeUnit.MILLISECONDS)).isTrue();
boolean drained = worker.stop(PATIENCE);
assertThat(drained).isTrue();
assertThat(repository.find(record.messageId()).orElseThrow().status())
.as("a pass killed between its claim and its terminal write leaves the row leased")
.isEqualTo(OutboxStatus.PUBLISHED);
}
@Test
@DisplayName("a pass that throws does not stop every pass after it")
void aFailedPassDoesNotEndTheLoop() throws InterruptedException {
repository.append(OutboxFixtures.pending());
OutboxRelayWorker worker = worker(new ThrowingPublisher());
worker.start();
try {
long deadline = System.nanoTime() + PATIENCE.toNanos();
while (worker.completedPasses() < 2 && System.nanoTime() < deadline) {
Thread.sleep(10);
}
} finally {
worker.stop(PATIENCE);
}
assertThat(worker.completedPasses())
.as("a scheduled task's own exception cancels every future run of it")
.isGreaterThanOrEqualTo(2);
assertThat(worker.failedPasses()).isPositive();
}
private OutboxRelayWorker worker(MessagePublisher publisher) {
OutboxRelay relay =
new OutboxRelay(
repository,
publisher,
new OutboxEnvelopeFactory(new ProducerId("order-api")),
100,
Duration.ofSeconds(30));
return new OutboxRelayWorker(relay, OutboxRetryScheduler.standard());
}
private void awaitStatus(OutboxRecord record, OutboxStatus expected) throws InterruptedException {
long deadline = System.nanoTime() + PATIENCE.toNanos();
while (System.nanoTime() < deadline) {
if (repository.find(record.messageId()).map(OutboxRecord::status).orElse(null) == expected) {
return;
}
Thread.sleep(10);
}
}
/** A publisher that is still publishing when shutdown begins. */
private static final class SlowPublisher implements MessagePublisher {
private final CountDownLatch started = new CountDownLatch(1);
private final Duration duration;
private SlowPublisher(Duration duration) {
this.duration = duration;
}
@Override
public <T> CompletionStage<PublishResult> publish(
MessageDestination<T> destination, MessageEnvelope<T> message, PublishOptions options) {
started.countDown();
try {
Thread.sleep(duration.toMillis());
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
return CompletableFuture.completedFuture(
new PublishResult(
PublishCompletion.CONFIRMED,
PublishEvidence.confirmed(ConfirmationLevel.BROKER_ACK),
RoutingOutcome.NOT_APPLICABLE,
Optional.empty(),
1,
Duration.ofMillis(1),
Optional.empty()));
}
}
/** A publisher whose failure reaches the worker as an exception rather than a result. */
private static final class ThrowingPublisher implements MessagePublisher {
@Override
public <T> CompletionStage<PublishResult> publish(
MessageDestination<T> destination, MessageEnvelope<T> message, PublishOptions options) {
throw new IllegalStateException("the broker connection is gone");
}
}
}