refactor: 각 어댑터터별 리펙토링 진행
This commit is contained in:
+78
-3
@@ -1,6 +1,7 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.header.ReservedHeaders;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
@@ -16,12 +17,29 @@ import java.util.Objects;
|
||||
* <p>The platform contract does not change with the transport. The router emits the same reserved
|
||||
* headers the polling relay does, so a consumer cannot tell which one published a message, and
|
||||
* switching between them is a deployment decision rather than a contract change.
|
||||
*
|
||||
* <p>The message key and the topic are two different questions and used to be answered by one
|
||||
* column. {@code destination} routed the topic <em>and</em> became the key, so every message on a
|
||||
* topic carried the same key, landed on one partition, and made keyed ordering a claim with no
|
||||
* mechanism behind it. The key now comes from {@code routing_key}, the stored column that states
|
||||
* the partition-key-or-message-id fallback once for both relays, and {@code destination} stays what
|
||||
* it always was: routing metadata.
|
||||
*
|
||||
* <p>One canonical field has no header here: the schema subject. There is no reserved header name
|
||||
* for it, and inventing one that only this path wrote would replace a symmetric gap with an
|
||||
* asymmetric one — the polling relay would carry a schema reference on the envelope and the CDC
|
||||
* relay a header no adapter reads. It stays a column that the polling relay resolves, and closing
|
||||
* it means adding the name to {@code ReservedHeaders} and teaching both broker header mappers to
|
||||
* write and read it.
|
||||
*/
|
||||
public final class DebeziumOutboxEventRouter {
|
||||
|
||||
/** The outbox column Debezium uses as the message key. */
|
||||
/** The outbox column Debezium routes the topic by. */
|
||||
public static final String ROUTE_BY_COLUMN = "destination";
|
||||
|
||||
/** The outbox column Debezium uses as the message key. */
|
||||
public static final String KEY_COLUMN = "routing_key";
|
||||
|
||||
/** The outbox column carrying the logical message id. */
|
||||
public static final String ID_COLUMN = "message_id";
|
||||
|
||||
@@ -39,7 +57,7 @@ public final class DebeziumOutboxEventRouter {
|
||||
Map<String, String> configuration = new LinkedHashMap<>();
|
||||
configuration.put("transforms.outbox.type", "io.debezium.transforms.outbox.EventRouter");
|
||||
configuration.put("transforms.outbox.table.field.event.id", ID_COLUMN);
|
||||
configuration.put("transforms.outbox.table.field.event.key", ROUTE_BY_COLUMN);
|
||||
configuration.put("transforms.outbox.table.field.event.key", KEY_COLUMN);
|
||||
configuration.put("transforms.outbox.table.field.event.payload", PAYLOAD_COLUMN);
|
||||
configuration.put("transforms.outbox.route.by.field", ROUTE_BY_COLUMN);
|
||||
configuration.put(
|
||||
@@ -51,7 +69,18 @@ public final class DebeziumOutboxEventRouter {
|
||||
"message_type:header:" + ReservedHeaders.MESSAGE_TYPE,
|
||||
"schema_version:header:" + ReservedHeaders.SCHEMA_VERSION,
|
||||
"content_type:header:" + ReservedHeaders.CONTENT_TYPE,
|
||||
"created_at:header:" + ReservedHeaders.PRODUCED_AT));
|
||||
"created_at:header:" + ReservedHeaders.PRODUCED_AT,
|
||||
"destination:header:" + ReservedHeaders.ORIGIN_DESTINATION,
|
||||
"producer:header:" + ReservedHeaders.PRODUCER,
|
||||
"occurred_at:header:" + ReservedHeaders.OCCURRED_AT,
|
||||
"correlation_id:header:" + ReservedHeaders.CORRELATION_ID,
|
||||
"causation_id:header:" + ReservedHeaders.CAUSATION_ID,
|
||||
"tenant:header:" + ReservedHeaders.TENANT,
|
||||
"partition_key:header:" + ReservedHeaders.PARTITION_KEY,
|
||||
"ordering_key:header:" + ReservedHeaders.ORDERING_KEY,
|
||||
"traceparent:header:" + ReservedHeaders.TRACEPARENT,
|
||||
"tracestate:header:" + ReservedHeaders.TRACESTATE,
|
||||
"baggage:header:" + ReservedHeaders.BAGGAGE));
|
||||
return Map.copyOf(configuration);
|
||||
}
|
||||
|
||||
@@ -71,6 +100,52 @@ public final class DebeziumOutboxEventRouter {
|
||||
headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value()));
|
||||
headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value());
|
||||
headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString());
|
||||
headers.put(ReservedHeaders.ORIGIN_DESTINATION, record.destination().value());
|
||||
|
||||
// Absent stays absent. Writing an empty string for a column the row does not have would make a
|
||||
// message that carries no tenant indistinguishable from one whose tenant is the empty string,
|
||||
// and the consumer would have to know which of the two this platform meant.
|
||||
OutboxCanonicalMetadata metadata = record.metadata();
|
||||
metadata.producer().ifPresent(value -> headers.put(ReservedHeaders.PRODUCER, value.value()));
|
||||
metadata
|
||||
.occurredAt()
|
||||
.ifPresent(value -> headers.put(ReservedHeaders.OCCURRED_AT, value.toString()));
|
||||
metadata
|
||||
.correlationId()
|
||||
.ifPresent(value -> headers.put(ReservedHeaders.CORRELATION_ID, value.value()));
|
||||
metadata
|
||||
.causationId()
|
||||
.ifPresent(
|
||||
value -> headers.put(ReservedHeaders.CAUSATION_ID, value.value().value().toString()));
|
||||
metadata.tenant().ifPresent(value -> headers.put(ReservedHeaders.TENANT, value.tenantId()));
|
||||
metadata.partitionKey().ifPresent(value -> headers.put(ReservedHeaders.PARTITION_KEY, value));
|
||||
metadata.orderingKey().ifPresent(value -> headers.put(ReservedHeaders.ORDERING_KEY, value));
|
||||
metadata
|
||||
.traceContext()
|
||||
.traceparent()
|
||||
.ifPresent(value -> headers.put(ReservedHeaders.TRACEPARENT, value));
|
||||
metadata
|
||||
.traceContext()
|
||||
.tracestate()
|
||||
.ifPresent(value -> headers.put(ReservedHeaders.TRACESTATE, value));
|
||||
metadata
|
||||
.traceContext()
|
||||
.baggage()
|
||||
.ifPresent(value -> headers.put(ReservedHeaders.BAGGAGE, value));
|
||||
return Map.copyOf(headers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the key a routed record carries.
|
||||
*
|
||||
* <p>The same value the {@code routing_key} column computes, stated here so a test can compare
|
||||
* the two rather than trusting that the SQL and the Java agree.
|
||||
*
|
||||
* @param record the outbox row
|
||||
* @return the broker message key
|
||||
*/
|
||||
public String routingKey(OutboxRecord record) {
|
||||
Objects.requireNonNull(record, "record must not be null");
|
||||
return record.metadata().partitionKey().orElseGet(() -> record.messageId().value().toString());
|
||||
}
|
||||
}
|
||||
|
||||
+19
-11
@@ -19,11 +19,17 @@ import java.util.Optional;
|
||||
* <p>The message id comes from the row's {@code message_id} column, never from Debezium's own event
|
||||
* id. Debezium mints a fresh event id per change record, so using it would break identity across a
|
||||
* CDC re-snapshot — exactly the scenario where identity matters most.
|
||||
*
|
||||
* <p>The reserved headers come from {@link DebeziumOutboxEventRouter} rather than being rebuilt
|
||||
* here. Two lists of the same headers is how the connector's configuration and the expectation this
|
||||
* class encodes come to disagree, and the disagreement would be invisible: the test compares this
|
||||
* output against the polling relay's, so it would pass while the connector emitted something else.
|
||||
*/
|
||||
public final class DebeziumOutboxRecordMapper {
|
||||
|
||||
private final String topicPrefix;
|
||||
private final boolean aggregateIdAsPartitionKey;
|
||||
private final DebeziumOutboxEventRouter router = new DebeziumOutboxEventRouter();
|
||||
|
||||
/**
|
||||
* Creates a mapper from a CDC profile.
|
||||
@@ -45,27 +51,29 @@ public final class DebeziumOutboxRecordMapper {
|
||||
public DebeziumMappedRecord map(OutboxRecord record) {
|
||||
Objects.requireNonNull(record, "record must not be null");
|
||||
|
||||
// Application headers first, canonical headers second, and the canonical ones win. The row's
|
||||
// header map is application data; a row that wrote `msg.id` into it would otherwise overwrite
|
||||
// the identity the platform put there, and the consumer trusts the last such header it sees.
|
||||
// The write path refuses reserved names outright — this order is the second lock on the same
|
||||
// door, because a CDC connector reading the table directly never passes through that check.
|
||||
Map<String, String> headers = new LinkedHashMap<>(record.headers());
|
||||
headers.put(ReservedHeaders.MESSAGE_ID, record.messageId().value().toString());
|
||||
headers.put(ReservedHeaders.MESSAGE_TYPE, record.messageType().value());
|
||||
headers.put(ReservedHeaders.SCHEMA_VERSION, Integer.toString(record.schemaVersion().value()));
|
||||
headers.put(ReservedHeaders.CONTENT_TYPE, record.contentType().value());
|
||||
headers.put(ReservedHeaders.PRODUCED_AT, record.createdAt().toString());
|
||||
headers.keySet().removeIf(ReservedHeaders::isReserved);
|
||||
headers.putAll(router.reservedHeaders(record));
|
||||
|
||||
return new DebeziumMappedRecord(
|
||||
topicPrefix + record.destination().value(),
|
||||
partitionKey(record, headers),
|
||||
partitionKey(record),
|
||||
record.payload(),
|
||||
headers);
|
||||
}
|
||||
|
||||
private Optional<String> partitionKey(OutboxRecord record, Map<String, String> headers) {
|
||||
private Optional<String> partitionKey(OutboxRecord record) {
|
||||
if (!aggregateIdAsPartitionKey) {
|
||||
return Optional.empty();
|
||||
}
|
||||
// The partition key header is the one the polling relay would also have set, so both relays
|
||||
// land the same message on the same partition and neither reorders a keyed stream.
|
||||
return Optional.ofNullable(headers.get(ReservedHeaders.PARTITION_KEY))
|
||||
.or(() -> Optional.of(record.messageId().value().toString()));
|
||||
// The same value the polling relay would use and the same value the `routing_key` column
|
||||
// computes, so both relays land the same message on the same partition and neither reorders a
|
||||
// keyed stream.
|
||||
return Optional.of(router.routingKey(record));
|
||||
}
|
||||
}
|
||||
|
||||
+81
-7
@@ -1,11 +1,17 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.CausationId;
|
||||
import dev.caskeleton.messaging.api.ContentType;
|
||||
import dev.caskeleton.messaging.api.CorrelationId;
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.MessageType;
|
||||
import dev.caskeleton.messaging.api.ProducerId;
|
||||
import dev.caskeleton.messaging.api.SchemaVersion;
|
||||
import dev.caskeleton.messaging.api.TenantContext;
|
||||
import dev.caskeleton.messaging.api.TraceContext;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxLease;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRepository;
|
||||
@@ -45,10 +51,32 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
"""
|
||||
INSERT INTO messaging_outbox (
|
||||
message_id, destination, message_type, schema_version, content_type,
|
||||
payload, headers, created_at, status, attempts)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?)
|
||||
payload, headers, created_at, status, attempts,
|
||||
producer, occurred_at, correlation_id, causation_id, tenant,
|
||||
partition_key, ordering_key, schema_subject, schema_uri,
|
||||
traceparent, tracestate, baggage)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?)
|
||||
""";
|
||||
|
||||
/**
|
||||
* The columns every read of a row selects.
|
||||
*
|
||||
* <p>One constant rather than four copies of the list. The lease, claim, find and reclaim
|
||||
* statements each spelled it out, and a column added to one of them and not the others is a
|
||||
* message that arrives with its tenant on the claim path and without it on the lease path — a
|
||||
* difference that shows up as a data bug in whichever consumer happened to be reading.
|
||||
*/
|
||||
private static final String COLUMNS =
|
||||
"""
|
||||
message_id, destination, message_type, schema_version, content_type,
|
||||
payload, headers, created_at, status, attempts, lease_expires_at, last_failure_code,
|
||||
producer, occurred_at, correlation_id, causation_id, tenant,
|
||||
partition_key, ordering_key, schema_subject, schema_uri,
|
||||
traceparent, tracestate, baggage""";
|
||||
|
||||
private static final String LEASE =
|
||||
"""
|
||||
WITH claimable AS (
|
||||
@@ -69,7 +97,10 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
WHERE o.message_id = c.message_id
|
||||
RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type,
|
||||
o.payload, o.headers, o.created_at, o.status, o.attempts,
|
||||
o.lease_expires_at, o.last_failure_code
|
||||
o.lease_expires_at, o.last_failure_code,
|
||||
o.producer, o.occurred_at, o.correlation_id, o.causation_id, o.tenant,
|
||||
o.partition_key, o.ordering_key, o.schema_subject, o.schema_uri,
|
||||
o.traceparent, o.tracestate, o.baggage
|
||||
""";
|
||||
|
||||
/**
|
||||
@@ -104,7 +135,10 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
WHERE o.message_id = c.message_id
|
||||
RETURNING o.message_id, o.destination, o.message_type, o.schema_version, o.content_type,
|
||||
o.payload, o.headers, o.created_at, o.status, o.attempts,
|
||||
o.lease_expires_at, o.last_failure_code, o.lease_owner, o.lease_token
|
||||
o.lease_expires_at, o.last_failure_code, o.lease_owner, o.lease_token,
|
||||
o.producer, o.occurred_at, o.correlation_id, o.causation_id, o.tenant,
|
||||
o.partition_key, o.ordering_key, o.schema_subject, o.schema_uri,
|
||||
o.traceparent, o.tracestate, o.baggage
|
||||
""";
|
||||
|
||||
private final DataSource dataSource;
|
||||
@@ -144,6 +178,20 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
statement.setTimestamp(8, Timestamp.from(record.createdAt()));
|
||||
statement.setString(9, record.status().name());
|
||||
statement.setInt(10, record.attempts());
|
||||
OutboxCanonicalMetadata metadata = record.metadata();
|
||||
statement.setString(11, metadata.producer().map(ProducerId::value).orElse(null));
|
||||
statement.setTimestamp(12, metadata.occurredAt().map(Timestamp::from).orElse(null));
|
||||
statement.setString(13, metadata.correlationId().map(CorrelationId::value).orElse(null));
|
||||
statement.setObject(
|
||||
14, metadata.causationId().map(causation -> causation.value().value()).orElse(null));
|
||||
statement.setString(15, metadata.tenant().map(TenantContext::tenantId).orElse(null));
|
||||
statement.setString(16, metadata.partitionKey().orElse(null));
|
||||
statement.setString(17, metadata.orderingKey().orElse(null));
|
||||
statement.setString(18, metadata.schemaSubject().orElse(null));
|
||||
statement.setString(19, metadata.schemaUri().orElse(null));
|
||||
statement.setString(20, metadata.traceContext().traceparent().orElse(null));
|
||||
statement.setString(21, metadata.traceContext().tracestate().orElse(null));
|
||||
statement.setString(22, metadata.traceContext().baggage().orElse(null));
|
||||
statement.executeUpdate();
|
||||
} catch (SQLException exception) {
|
||||
throw new MessagingConfigurationException(
|
||||
@@ -425,9 +473,7 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
connection -> {
|
||||
try (PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"SELECT message_id, destination, message_type, schema_version, content_type, "
|
||||
+ "payload, headers, created_at, status, attempts, lease_expires_at, "
|
||||
+ "last_failure_code FROM messaging_outbox WHERE message_id = ?")) {
|
||||
"SELECT " + COLUMNS + " FROM messaging_outbox WHERE message_id = ?")) {
|
||||
statement.setObject(1, messageId.value());
|
||||
try (ResultSet results = statement.executeQuery()) {
|
||||
return results.next() ? Optional.of(read(results)) : Optional.<OutboxRecord>empty();
|
||||
@@ -509,6 +555,7 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
new ContentType(results.getString("content_type")),
|
||||
results.getBytes("payload"),
|
||||
fromJson(results.getString("headers")),
|
||||
readMetadata(results),
|
||||
results.getTimestamp("created_at").toInstant(),
|
||||
OutboxStatus.valueOf(results.getString("status")),
|
||||
results.getInt("attempts"),
|
||||
@@ -516,6 +563,33 @@ public final class JdbcOutboxRepository implements OutboxRepository {
|
||||
Optional.ofNullable(results.getString("last_failure_code")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds the canonical metadata from the row's own columns.
|
||||
*
|
||||
* <p>Absent columns stay absent. A row written before the columns existed rebuilds with empty
|
||||
* metadata, which is the same thing a row written by a producer that had none rebuilds with — and
|
||||
* the reader's behaviour has to be identical in both cases, because guessing a tenant or a
|
||||
* correlation is worse than carrying none.
|
||||
*/
|
||||
private static OutboxCanonicalMetadata readMetadata(ResultSet results) throws SQLException {
|
||||
Timestamp occurredAt = results.getTimestamp("occurred_at");
|
||||
java.util.UUID causation = results.getObject("causation_id", java.util.UUID.class);
|
||||
return new OutboxCanonicalMetadata(
|
||||
Optional.ofNullable(results.getString("producer")).map(ProducerId::new),
|
||||
Optional.ofNullable(occurredAt).map(Timestamp::toInstant),
|
||||
Optional.ofNullable(results.getString("correlation_id")).map(CorrelationId::new),
|
||||
Optional.ofNullable(causation).map(value -> new CausationId(new MessageId(value))),
|
||||
Optional.ofNullable(results.getString("tenant")).map(TenantContext::new),
|
||||
Optional.ofNullable(results.getString("partition_key")),
|
||||
Optional.ofNullable(results.getString("ordering_key")),
|
||||
Optional.ofNullable(results.getString("schema_subject")),
|
||||
Optional.ofNullable(results.getString("schema_uri")),
|
||||
new TraceContext(
|
||||
Optional.ofNullable(results.getString("traceparent")),
|
||||
Optional.ofNullable(results.getString("tracestate")),
|
||||
Optional.ofNullable(results.getString("baggage"))));
|
||||
}
|
||||
|
||||
private <T> T withConnection(ConnectionCallback<T> callback) {
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
|
||||
+54
-21
@@ -2,13 +2,16 @@ package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import dev.caskeleton.messaging.api.MessageEnvelope;
|
||||
import dev.caskeleton.messaging.api.ProducerId;
|
||||
import dev.caskeleton.messaging.api.TraceContext;
|
||||
import dev.caskeleton.messaging.api.SchemaVersion;
|
||||
import dev.caskeleton.messaging.api.header.HeaderName;
|
||||
import dev.caskeleton.messaging.api.header.HeaderValue;
|
||||
import dev.caskeleton.messaging.api.header.MessageHeaders;
|
||||
import dev.caskeleton.messaging.api.header.ReservedHeaders;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.schema.EncodedMessage;
|
||||
import dev.caskeleton.messaging.schema.SchemaReference;
|
||||
import java.net.URI;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -20,6 +23,18 @@ import java.util.Optional;
|
||||
* <p>The identity comes from the row, never from a fresh mint. That is what makes a retry after an
|
||||
* ambiguous publish safe: the broker's deduplication, or the consumer's Inbox, sees the same id and
|
||||
* collapses the duplicate.
|
||||
*
|
||||
* <p>So does everything else the envelope carries. This used to rebuild correlation, causation,
|
||||
* tenant, trace and the schema reference as empty, and read the routing keys out of the row's
|
||||
* header map — so a message that travelled through the outbox reached its consumer with less
|
||||
* provenance than one published directly, and the publish path became part of the message's
|
||||
* meaning. The row now has columns for those facts and this reads them.
|
||||
*
|
||||
* <p>Reserved header names in the row are refused outright, with no exception for the routing keys.
|
||||
* They had one, because the keys had nowhere else to live; a row could therefore write {@code
|
||||
* msg.partition-key} and the same code path would have accepted {@code msg.id} if the check had
|
||||
* been one line looser. Now that the keys are columns, the rule is the simple one: an outbox row
|
||||
* cannot write into the platform's namespace at all.
|
||||
*/
|
||||
public final class OutboxEnvelopeFactory {
|
||||
|
||||
@@ -28,7 +43,7 @@ public final class OutboxEnvelopeFactory {
|
||||
/**
|
||||
* Creates a factory for one service.
|
||||
*
|
||||
* @param producer the logical producing service
|
||||
* @param producer the logical producing service, used for rows that name none
|
||||
*/
|
||||
public OutboxEnvelopeFactory(ProducerId producer) {
|
||||
this.producer = Objects.requireNonNull(producer, "producer must not be null");
|
||||
@@ -52,13 +67,7 @@ public final class OutboxEnvelopeFactory {
|
||||
// MessageHeaders.platform let a row carrying `msg.id` or `msg.type` reach the wire
|
||||
// as if the platform had written it — and the consumer trusts the last such header,
|
||||
// so a forged identity lands in another message's inbox deduplication.
|
||||
//
|
||||
// Partition and ordering keys are the exception: they are reserved names the outbox
|
||||
// legitimately carries as routing metadata, and they are read below rather than
|
||||
// written as headers.
|
||||
if (ReservedHeaders.isReserved(name)
|
||||
&& !ReservedHeaders.PARTITION_KEY.equals(name)
|
||||
&& !ReservedHeaders.ORDERING_KEY.equals(name)) {
|
||||
if (ReservedHeaders.isReserved(name)) {
|
||||
throw new dev.caskeleton.messaging.api.error.MessagingConfigurationException(
|
||||
"RESERVED_HEADER_IN_OUTBOX_ROW",
|
||||
"outbox row header '"
|
||||
@@ -66,26 +75,50 @@ public final class OutboxEnvelopeFactory {
|
||||
+ "' uses the reserved namespace; canonical identity and metadata come from"
|
||||
+ " the record's own columns, not from its header map");
|
||||
}
|
||||
if (!ReservedHeaders.isReserved(name)) {
|
||||
headers.put(new HeaderName(name), new HeaderValue(value));
|
||||
}
|
||||
headers.put(new HeaderName(name), new HeaderValue(value));
|
||||
});
|
||||
|
||||
OutboxCanonicalMetadata metadata = record.metadata();
|
||||
return new MessageEnvelope<>(
|
||||
record.messageId(),
|
||||
record.messageType(),
|
||||
record.schemaVersion(),
|
||||
record.createdAt(),
|
||||
Optional.of(record.createdAt()),
|
||||
producer,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
// The write instant stands in when the row records no separate occurrence. It is the
|
||||
// closest true statement available: the business transaction that wrote the row is the one
|
||||
// the fact occurred in.
|
||||
Optional.of(metadata.occurredAt().orElse(record.createdAt())),
|
||||
// A row written before the producer column existed names none, and the service that owns
|
||||
// this relay is the only producer it can honestly be attributed to.
|
||||
metadata.producer().orElse(producer),
|
||||
metadata.correlationId(),
|
||||
metadata.causationId(),
|
||||
record.contentType(),
|
||||
Optional.ofNullable(record.headers().get(ReservedHeaders.PARTITION_KEY)),
|
||||
Optional.ofNullable(record.headers().get(ReservedHeaders.ORDERING_KEY)),
|
||||
Optional.empty(),
|
||||
TraceContext.none(),
|
||||
metadata.partitionKey(),
|
||||
metadata.orderingKey(),
|
||||
metadata.tenant(),
|
||||
metadata.traceContext(),
|
||||
MessageHeaders.platform(headers),
|
||||
new EncodedMessage(record.payload(), record.contentType(), Optional.empty()));
|
||||
new EncodedMessage(
|
||||
record.payload(), record.contentType(), schemaReference(record, metadata)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds the schema reference from the row's subject, the row's version, and its location.
|
||||
*
|
||||
* <p>The version is the row's {@code schema_version} column rather than a third copy of the same
|
||||
* number: a reference whose version could disagree with the column the relay filters on is a
|
||||
* reference nobody can trust.
|
||||
*/
|
||||
private static Optional<SchemaReference> schemaReference(
|
||||
OutboxRecord record, OutboxCanonicalMetadata metadata) {
|
||||
return metadata
|
||||
.schemaSubject()
|
||||
.map(
|
||||
subject ->
|
||||
new SchemaReference(
|
||||
subject,
|
||||
new SchemaVersion(record.schemaVersion().value()),
|
||||
metadata.schemaUri().map(URI::create)));
|
||||
}
|
||||
}
|
||||
|
||||
+51
-5
@@ -87,6 +87,33 @@ public final class OutboxRelay {
|
||||
OutboxRetryScheduler.standard());
|
||||
}
|
||||
|
||||
/**
|
||||
* A relay with an explicit retry policy and a process-derived owner.
|
||||
*
|
||||
* @param repository the outbox storage
|
||||
* @param publisher the platform publisher
|
||||
* @param envelopeFactory rebuilds envelopes from rows
|
||||
* @param batchSize how many rows to claim per pass
|
||||
* @param leaseDuration how long a claim holds
|
||||
* @param scheduler the retry policy
|
||||
*/
|
||||
public OutboxRelay(
|
||||
OutboxRepository repository,
|
||||
MessagePublisher publisher,
|
||||
OutboxEnvelopeFactory envelopeFactory,
|
||||
int batchSize,
|
||||
Duration leaseDuration,
|
||||
OutboxRetryScheduler scheduler) {
|
||||
this(
|
||||
repository,
|
||||
publisher,
|
||||
envelopeFactory,
|
||||
batchSize,
|
||||
leaseDuration,
|
||||
defaultOwner(),
|
||||
scheduler);
|
||||
}
|
||||
|
||||
/**
|
||||
* A relay with an explicit retry policy.
|
||||
*
|
||||
@@ -134,6 +161,7 @@ public final class OutboxRelay {
|
||||
int ambiguous = 0;
|
||||
int failed = 0;
|
||||
int stale = 0;
|
||||
int exhausted = 0;
|
||||
|
||||
for (OutboxLease lease : batch) {
|
||||
OutboxRecord record = lease.record();
|
||||
@@ -147,10 +175,28 @@ public final class OutboxRelay {
|
||||
}
|
||||
}
|
||||
case AMBIGUOUS -> {
|
||||
if (repository.markAmbiguous(
|
||||
lease, result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS"), now)
|
||||
== OutboxTransitionResult.APPLIED) {
|
||||
ambiguous++;
|
||||
String code = result.failure().map(failure -> failure.code()).orElse("AMBIGUOUS");
|
||||
// The attempt this pass just spent. The claim predicate and the row both count attempts
|
||||
// after the transition, so the budget has to be judged on the same number the next claim
|
||||
// will read, or the last attempt is spent twice.
|
||||
int spent = record.attempts() + 1;
|
||||
OutboxTransitionResult transition =
|
||||
scheduler
|
||||
.parkReason(spent)
|
||||
// A row that has spent its budget without an answer is parked under its own
|
||||
// status. Leaving it AMBIGUOUS makes it a row the claim predicate silently skips
|
||||
// forever, which looks identical to a healthy backlog on every dashboard.
|
||||
.map(reason -> repository.markExhausted(lease, reason, now))
|
||||
.orElseGet(
|
||||
() ->
|
||||
repository.markAmbiguous(
|
||||
lease, code, now, scheduler.nextAttemptAt(now, spent)));
|
||||
if (transition == OutboxTransitionResult.APPLIED) {
|
||||
if (scheduler.isExhausted(spent)) {
|
||||
exhausted++;
|
||||
} else {
|
||||
ambiguous++;
|
||||
}
|
||||
} else {
|
||||
stale++;
|
||||
}
|
||||
@@ -171,7 +217,7 @@ public final class OutboxRelay {
|
||||
throw new IllegalStateException("unhandled publish completion: " + result.completion());
|
||||
}
|
||||
}
|
||||
return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale);
|
||||
return new OutboxRelayReport(batch.size(), published, ambiguous, failed, stale, exhausted);
|
||||
}
|
||||
|
||||
private PublishResult publish(OutboxRecord record) {
|
||||
|
||||
+19
-3
@@ -12,23 +12,39 @@ package dev.caskeleton.messaging.outbox;
|
||||
* published something whose outcome another worker has already recorded, so a non-zero value is the
|
||||
* visible form of a duplicate publication.
|
||||
*
|
||||
* <p>{@code exhausted} is counted apart from {@code failed} for the same reason: a rejected row is
|
||||
* a message the broker refused and an exhausted one is a message nobody ever answered for. The
|
||||
* first needs a fix, the second needs a redrive, and a single counter for both tells an operator
|
||||
* neither.
|
||||
*
|
||||
* @param leased how many rows were claimed
|
||||
* @param published how many were confirmed
|
||||
* @param ambiguous how many had an unknown outcome and stay retryable
|
||||
* @param failed how many were definitively rejected
|
||||
* @param staleLeases how many terminal writes were refused because the lease had been superseded
|
||||
* @param exhausted how many spent their attempt budget without a confirmation and were parked
|
||||
*/
|
||||
public record OutboxRelayReport(
|
||||
int leased, int published, int ambiguous, int failed, int staleLeases) {
|
||||
int leased, int published, int ambiguous, int failed, int staleLeases, int exhausted) {
|
||||
|
||||
public OutboxRelayReport {
|
||||
if (leased < 0 || published < 0 || ambiguous < 0 || failed < 0 || staleLeases < 0) {
|
||||
if (leased < 0
|
||||
|| published < 0
|
||||
|| ambiguous < 0
|
||||
|| failed < 0
|
||||
|| staleLeases < 0
|
||||
|| exhausted < 0) {
|
||||
throw new IllegalArgumentException("relay counters must not be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/** A report from a pass that parked nothing. */
|
||||
public OutboxRelayReport(int leased, int published, int ambiguous, int failed, int staleLeases) {
|
||||
this(leased, published, ambiguous, failed, staleLeases, 0);
|
||||
}
|
||||
|
||||
/** A report from a pass that observed no stale leases. */
|
||||
public OutboxRelayReport(int leased, int published, int ambiguous, int failed) {
|
||||
this(leased, published, ambiguous, failed, 0);
|
||||
this(leased, published, ambiguous, failed, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Runs relay passes on the schedule the retry policy decides.
|
||||
*
|
||||
* <p>The relay, its retry scheduler and the attempt budget all existed and nothing ever called
|
||||
* {@code runOnce}. An outbox whose relay is never driven is the worst shape of all: the business
|
||||
* transaction commits its row and reports success, and the message is never published — a silent,
|
||||
* unbounded backlog rather than a visible failure.
|
||||
*
|
||||
* <p>Passes are self-scheduled rather than run at a fixed rate. A fixed rate keeps firing while the
|
||||
* broker is down, and the pass after an empty or failed one is exactly where the policy's backoff
|
||||
* belongs; a pass that published something resets to the base interval because work is flowing.
|
||||
*
|
||||
* <p>One thread, and the next pass is scheduled only after the previous returns. Overlapping passes
|
||||
* from one process would have two claims racing on the same rows, which the fencing predicate would
|
||||
* then reject as stale — correct, but it is duplicate publishing work bought at the cost of a
|
||||
* duplicate publish.
|
||||
*/
|
||||
public final class OutboxRelayWorker implements AutoCloseable {
|
||||
|
||||
/** How long {@link #close()} waits for a running pass when no deadline is given. */
|
||||
public static final Duration DEFAULT_DRAIN_DEADLINE = Duration.ofSeconds(30);
|
||||
|
||||
private final OutboxRelay relay;
|
||||
|
||||
private final OutboxRetryScheduler scheduler;
|
||||
|
||||
private final Supplier<Instant> clock;
|
||||
|
||||
private final ScheduledExecutorService passes;
|
||||
|
||||
private final AtomicBoolean running = new AtomicBoolean();
|
||||
|
||||
private final AtomicInteger unproductivePasses = new AtomicInteger();
|
||||
|
||||
private final AtomicLong completedPasses = new AtomicLong();
|
||||
|
||||
private final AtomicLong failedPasses = new AtomicLong();
|
||||
|
||||
/**
|
||||
* Creates a worker driving one relay.
|
||||
*
|
||||
* @param relay the relay to run
|
||||
* @param scheduler the policy deciding when the next pass runs
|
||||
*/
|
||||
public OutboxRelayWorker(OutboxRelay relay, OutboxRetryScheduler scheduler) {
|
||||
this(relay, scheduler, Instant::now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a worker with an explicit clock, so a pass can be driven without waiting on wall time.
|
||||
*
|
||||
* @param relay the relay to run
|
||||
* @param scheduler the policy deciding when the next pass runs
|
||||
* @param clock the source of the current instant
|
||||
*/
|
||||
public OutboxRelayWorker(
|
||||
OutboxRelay relay, OutboxRetryScheduler scheduler, Supplier<Instant> clock) {
|
||||
this.relay = Objects.requireNonNull(relay, "relay must not be null");
|
||||
this.scheduler = Objects.requireNonNull(scheduler, "scheduler must not be null");
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
ScheduledThreadPoolExecutor executor =
|
||||
new ScheduledThreadPoolExecutor(
|
||||
1,
|
||||
runnable -> {
|
||||
// Daemon: a relay thread that outlives the application would keep publishing from a
|
||||
// process that has already released its other resources.
|
||||
Thread thread = new Thread(runnable, "messaging-outbox-relay");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
// A pass waiting out its backoff is dropped on shutdown rather than run afterwards. Left on,
|
||||
// the default policy makes stop() wait for a pass that has not started and cannot do anything
|
||||
// useful, so an orderly shutdown pays the whole poll interval before it can close.
|
||||
executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
|
||||
this.passes = executor;
|
||||
}
|
||||
|
||||
/** Starts driving passes. Idempotent: a second call does not add a second schedule. */
|
||||
public void start() {
|
||||
if (running.compareAndSet(false, true)) {
|
||||
schedule(Duration.ZERO);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops scheduling and lets the pass that is already running finish.
|
||||
*
|
||||
* <p>Draining rather than interrupting is the whole point. A pass killed between its claim and
|
||||
* its terminal write leaves rows {@code IN_FLIGHT} holding a lease, and nothing may touch them
|
||||
* until that lease expires — so an orderly shutdown would produce exactly the stall that a crash
|
||||
* produces.
|
||||
*
|
||||
* @param deadline how long to wait for the running pass
|
||||
* @return true when the running pass finished inside the deadline
|
||||
*/
|
||||
public boolean stop(Duration deadline) {
|
||||
Objects.requireNonNull(deadline, "deadline must not be null");
|
||||
running.set(false);
|
||||
passes.shutdown();
|
||||
try {
|
||||
if (passes.awaitTermination(deadline.toMillis(), TimeUnit.MILLISECONDS)) {
|
||||
return true;
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
// The deadline passed with a pass still running. Its rows stay leased until the lease expires,
|
||||
// which is the outcome the deadline exists to bound rather than to prevent.
|
||||
passes.shutdownNow();
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
stop(DEFAULT_DRAIN_DEADLINE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the worker is scheduling passes.
|
||||
*
|
||||
* @return true between {@link #start()} and {@link #stop(Duration)}
|
||||
*/
|
||||
public boolean isRunning() {
|
||||
return running.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns how many passes have finished, successfully or not.
|
||||
*
|
||||
* @return the completed pass count
|
||||
*/
|
||||
public long completedPasses() {
|
||||
return completedPasses.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns how many passes ended in an exception.
|
||||
*
|
||||
* @return the failed pass count
|
||||
*/
|
||||
public long failedPasses() {
|
||||
return failedPasses.get();
|
||||
}
|
||||
|
||||
// The scheduled future is not kept: a pass reports nothing to a caller and handles its own
|
||||
// failure below, so the only thing holding it would achieve is a reference that has to be
|
||||
// cleaned up.
|
||||
@SuppressWarnings("FutureReturnValueIgnored")
|
||||
private void schedule(Duration delay) {
|
||||
if (!running.get()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
passes.schedule(this::runPass, Math.max(0, delay.toMillis()), TimeUnit.MILLISECONDS);
|
||||
} catch (RejectedExecutionException stopping) {
|
||||
// stop() shut the executor between the running check and the submission. There is nothing to
|
||||
// schedule onto and nothing to report: the worker is stopping, which is what was asked for.
|
||||
running.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void runPass() {
|
||||
if (!running.get()) {
|
||||
return;
|
||||
}
|
||||
int unproductive;
|
||||
try {
|
||||
OutboxRelayReport report = relay.runOnce(clock.get());
|
||||
unproductive = report.published() > 0 ? reset() : unproductivePasses.incrementAndGet();
|
||||
} catch (RuntimeException passFailed) {
|
||||
// A failed pass must not stop the loop: the scheduled task's own exception would cancel every
|
||||
// future pass, turning one broker error into a relay that never runs again. The failure is
|
||||
// counted and the next pass backs off as if nothing was published, which is true.
|
||||
failedPasses.incrementAndGet();
|
||||
unproductive = unproductivePasses.incrementAndGet();
|
||||
}
|
||||
completedPasses.incrementAndGet();
|
||||
schedule(scheduler.backoff(unproductive));
|
||||
}
|
||||
|
||||
private int reset() {
|
||||
unproductivePasses.set(0);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
+25
-4
@@ -8,10 +8,12 @@ import java.util.Optional;
|
||||
/**
|
||||
* Decides when a relay's next pass should run, and when a record has retried enough.
|
||||
*
|
||||
* <p>The backoff is on the <em>relay</em>, not on the individual record. A broker outage fails
|
||||
* every record in the batch for the same reason, so backing off per record would hammer the broker
|
||||
* with the whole backlog on a schedule while achieving nothing; backing off the pass lets the
|
||||
* broker recover.
|
||||
* <p>The same curve serves two different jobs and neither replaces the other. Backing off the
|
||||
* <em>pass</em> is what lets a broker recover: an outage fails every record in the batch for the
|
||||
* same reason, so retrying the whole backlog on the poll interval achieves nothing but load. But a
|
||||
* pass-level wait is process memory — it is forgotten on restart and each replica keeps its own —
|
||||
* so eligibility also has to be written on the row, or the backlog is retried as fast as the
|
||||
* unluckiest replica polls.
|
||||
*
|
||||
* <p>Jitter is applied deterministically from the attempt count rather than randomly. Several relay
|
||||
* instances that all started at deployment time would otherwise synchronise their retries into a
|
||||
@@ -58,6 +60,25 @@ public final class OutboxRetryScheduler {
|
||||
return now.plus(backoff(consecutiveEmptyOrFailedPasses));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns when a row that has spent this many attempts may be claimed again.
|
||||
*
|
||||
* <p>Written onto the row rather than held in the relay, because a deadline that lives in process
|
||||
* memory is forgotten on restart and computed independently by every replica — which turns a
|
||||
* backoff into every worker re-publishing the same backlog as fast as it polls.
|
||||
*
|
||||
* @param now the current instant
|
||||
* @param attempts how many attempts the row has spent
|
||||
* @return the earliest instant the row is eligible again
|
||||
*/
|
||||
public Instant nextAttemptAt(Instant now, int attempts) {
|
||||
Objects.requireNonNull(now, "now must not be null");
|
||||
if (attempts < 1) {
|
||||
throw new IllegalArgumentException("a row that has been published once has spent an attempt");
|
||||
}
|
||||
return now.plus(backoff(attempts));
|
||||
}
|
||||
|
||||
/** The platform's default policy, for a relay whose caller did not supply one. */
|
||||
public static OutboxRetryScheduler standard() {
|
||||
return new OutboxRetryScheduler(OutboxProperties.defaults(), java.time.Duration.ofMinutes(5));
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
-- The envelope's canonical facts, as columns.
|
||||
--
|
||||
-- V1 stored identity, type, version, content type, payload and a free-form header map. Everything
|
||||
-- else the envelope carries — producer, tenant, correlation, causation, the routing keys, the
|
||||
-- schema the payload was written against, the active trace — had nowhere to go, so it was either
|
||||
-- lost when the relay rebuilt the envelope or smuggled through the header map under the reserved
|
||||
-- `msg.*` names the platform is supposed to own. Smuggling is the worse of the two: a row whose
|
||||
-- header map contains `msg.id` overwrites another message's identity on the wire, and the consumer
|
||||
-- trusts the last such header it sees.
|
||||
--
|
||||
-- Columns rather than a versioned envelope blob. Both round-trip the values faithfully; only one of
|
||||
-- them lets the relay answer an operator's questions. "Which tenant is the backlog for", "which
|
||||
-- correlation is stuck", "which rows carry a schema this consumer cannot read" are SELECTs against
|
||||
-- this table if the fields are columns, and payload decoding of the whole backlog if they are not.
|
||||
--
|
||||
-- Every column is nullable. A row written before this migration has none of them, and a producer
|
||||
-- that genuinely has no correlation writes none either; the reader treats both the same way, which
|
||||
-- is to carry what is there and invent nothing.
|
||||
ALTER TABLE messaging_outbox
|
||||
ADD COLUMN producer VARCHAR(120),
|
||||
ADD COLUMN occurred_at TIMESTAMPTZ,
|
||||
ADD COLUMN correlation_id VARCHAR(160),
|
||||
ADD COLUMN causation_id UUID,
|
||||
ADD COLUMN tenant VARCHAR(64),
|
||||
ADD COLUMN partition_key VARCHAR(1024),
|
||||
ADD COLUMN ordering_key VARCHAR(1024),
|
||||
ADD COLUMN schema_subject VARCHAR(240),
|
||||
ADD COLUMN schema_uri VARCHAR(1024),
|
||||
ADD COLUMN traceparent VARCHAR(64),
|
||||
ADD COLUMN tracestate VARCHAR(512),
|
||||
ADD COLUMN baggage VARCHAR(8192);
|
||||
|
||||
-- The same bound TenantContext enforces in Java. Stated here as well because the relay, the CDC
|
||||
-- connector and any operator query read this table directly: a tenant slug that only the
|
||||
-- application validates is a tenant slug that an INSERT from anywhere else can violate, and a
|
||||
-- tenant id is the field every downstream authorization decision reads.
|
||||
ALTER TABLE messaging_outbox
|
||||
ADD CONSTRAINT ck_messaging_outbox_tenant
|
||||
CHECK (tenant IS NULL OR tenant ~ '^[a-z0-9][a-z0-9._-]{0,63}$');
|
||||
|
||||
-- A schema location with no subject cannot be resolved: the reader would hold a URI and no way to
|
||||
-- know what it is a schema for.
|
||||
ALTER TABLE messaging_outbox
|
||||
ADD CONSTRAINT ck_messaging_outbox_schema_reference
|
||||
CHECK (schema_uri IS NULL OR schema_subject IS NOT NULL);
|
||||
|
||||
-- "Which tenant is the backlog for" is the first question asked when an outbox stops draining, and
|
||||
-- before this column it could only be answered by decoding every payload. Partial, like the other
|
||||
-- outbox indexes, because PUBLISHED rows accumulate until retention removes them and no operator
|
||||
-- asks that question about work that already left.
|
||||
CREATE INDEX ix_messaging_outbox_tenant_backlog
|
||||
ON messaging_outbox (tenant, created_at)
|
||||
WHERE status IN ('PENDING', 'AMBIGUOUS', 'IN_FLIGHT');
|
||||
|
||||
-- The key a relay puts on the wire, computed once and stored, so both relays agree.
|
||||
--
|
||||
-- Debezium's Event Router takes the message key from a column. It was pointed at `destination`,
|
||||
-- which made the key the topic name — every message on a topic sharing one key, so every message
|
||||
-- landing on one partition, and keyed ordering meaning nothing. The polling relay meanwhile used
|
||||
-- the partition key when the row had one and the message id when it did not.
|
||||
--
|
||||
-- A generated column states that fallback once, in the place both relays read, instead of leaving
|
||||
-- it as a rule each of them implements separately and one of them gets wrong.
|
||||
ALTER TABLE messaging_outbox
|
||||
ADD COLUMN routing_key TEXT
|
||||
GENERATED ALWAYS AS (COALESCE(partition_key, message_id::TEXT)) STORED;
|
||||
+66
-1
@@ -12,6 +12,7 @@ import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import dev.caskeleton.messaging.api.header.ReservedHeaders;
|
||||
import dev.caskeleton.messaging.outbox.DebeziumOutboxProfile.RelayMode;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.reliability.OutboxStatus;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -26,6 +27,11 @@ class DebeziumOutboxRecordMapperTest {
|
||||
private static final MessageId MESSAGE_ID = MessageId.newId();
|
||||
|
||||
private static OutboxRecord record(Map<String, String> headers) {
|
||||
return record(headers, OutboxCanonicalMetadata.none());
|
||||
}
|
||||
|
||||
private static OutboxRecord record(
|
||||
Map<String, String> headers, OutboxCanonicalMetadata metadata) {
|
||||
return new OutboxRecord(
|
||||
MESSAGE_ID,
|
||||
new DestinationName("orders.v1"),
|
||||
@@ -34,6 +40,7 @@ class DebeziumOutboxRecordMapperTest {
|
||||
ContentType.JSON,
|
||||
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
|
||||
headers,
|
||||
metadata,
|
||||
NOW,
|
||||
OutboxStatus.PENDING,
|
||||
0,
|
||||
@@ -89,13 +96,71 @@ class DebeziumOutboxRecordMapperTest {
|
||||
@Test
|
||||
void aKeyedProfileUsesTheSamePartitionKeyThePollingRelayWouldHave() {
|
||||
DebeziumMappedRecord mapped =
|
||||
mapper(true).map(record(Map.of(ReservedHeaders.PARTITION_KEY, "customer-42")));
|
||||
mapper(true)
|
||||
.map(
|
||||
record(
|
||||
Map.of(),
|
||||
OutboxCanonicalMetadata.none().withRoutingKeys("customer-42", "customer-42")));
|
||||
|
||||
assertThat(mapped.key())
|
||||
.as("a different key would send the two relays' messages to different partitions")
|
||||
.hasValue("customer-42");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aKeyedProfileFallsBackToTheMessageIdWhenTheRowHasNoPartitionKey() {
|
||||
assertThat(mapper(true).map(record(Map.of())).key())
|
||||
.as("the fallback the routing_key column computes, stated in one place for both relays")
|
||||
.hasValue(MESSAGE_ID.value().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void theCanonicalColumnsBecomeReservedHeaders() {
|
||||
OutboxCanonicalMetadata metadata =
|
||||
OutboxCanonicalMetadata.none()
|
||||
.withProducer(new dev.caskeleton.messaging.api.ProducerId("order-api"))
|
||||
.withTenant(new dev.caskeleton.messaging.api.TenantContext("acme"))
|
||||
.withRoutingKeys("customer-42", "customer-42");
|
||||
|
||||
Map<String, String> headers = mapper(false).map(record(Map.of(), metadata)).headers();
|
||||
|
||||
assertThat(headers)
|
||||
.as("a relay that drops these makes the publish path part of the message's meaning")
|
||||
.containsEntry(ReservedHeaders.PRODUCER, "order-api")
|
||||
.containsEntry(ReservedHeaders.TENANT, "acme")
|
||||
.containsEntry(ReservedHeaders.PARTITION_KEY, "customer-42")
|
||||
.containsEntry(ReservedHeaders.ORDERING_KEY, "customer-42")
|
||||
.containsEntry(ReservedHeaders.ORIGIN_DESTINATION, "orders.v1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aColumnTheRowDoesNotCarryProducesNoHeaderAtAll() {
|
||||
assertThat(mapper(false).map(record(Map.of())).headers())
|
||||
.as("an empty string would be indistinguishable from a tenant literally named ''")
|
||||
.doesNotContainKeys(
|
||||
ReservedHeaders.TENANT, ReservedHeaders.CORRELATION_ID, ReservedHeaders.TRACEPARENT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aForgedReservedHeaderOnTheRowCannotOverwriteTheCanonicalOne() {
|
||||
Map<String, String> forged =
|
||||
Map.of(ReservedHeaders.MESSAGE_ID, "00000000-0000-7000-8000-000000000000");
|
||||
|
||||
assertThat(mapper(false).map(record(forged)).headers())
|
||||
.as("a CDC connector reads the table directly and never passes the write-path check")
|
||||
.containsEntry(ReservedHeaders.MESSAGE_ID, MESSAGE_ID.value().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void theRoutedKeyIsNotTheTopicName() {
|
||||
// Literals, not the class's own constants: comparing a configuration value against the constant
|
||||
// that produced it asserts that the router agrees with itself, which it always will.
|
||||
assertThat(new DebeziumOutboxEventRouter().connectorConfiguration("prod."))
|
||||
.as("keying by destination puts every message on a topic onto one partition")
|
||||
.containsEntry("transforms.outbox.table.field.event.key", "routing_key")
|
||||
.containsEntry("transforms.outbox.route.by.field", "destination");
|
||||
}
|
||||
|
||||
@Test
|
||||
void thePayloadIsUnchangedByTheTransform() {
|
||||
assertThat(mapper(false).map(record(Map.of())).payload())
|
||||
|
||||
+1
@@ -95,6 +95,7 @@ class JdbcOutboxTransactionRequirementTest {
|
||||
ContentType.JSON,
|
||||
"{}".getBytes(java.nio.charset.StandardCharsets.UTF_8),
|
||||
Map.of(),
|
||||
dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata.none(),
|
||||
Instant.parse("2026-08-14T00:00:00Z"),
|
||||
OutboxStatus.PENDING,
|
||||
0,
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import dev.caskeleton.messaging.api.CausationId;
|
||||
import dev.caskeleton.messaging.api.ContentType;
|
||||
import dev.caskeleton.messaging.api.CorrelationId;
|
||||
import dev.caskeleton.messaging.api.MessageEnvelope;
|
||||
import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.MessageType;
|
||||
import dev.caskeleton.messaging.api.ProducerId;
|
||||
import dev.caskeleton.messaging.api.SchemaVersion;
|
||||
import dev.caskeleton.messaging.api.TenantContext;
|
||||
import dev.caskeleton.messaging.api.TraceContext;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.api.error.MessagingConfigurationException;
|
||||
import dev.caskeleton.messaging.api.header.ReservedHeaders;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.reliability.OutboxStatus;
|
||||
import dev.caskeleton.messaging.schema.EncodedMessage;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* What survives a trip through the database.
|
||||
*
|
||||
* <p>The envelope this factory rebuilds used to be poorer than the one that was written: producer
|
||||
* came from the relay's own configuration, and correlation, causation, tenant, trace and the schema
|
||||
* reference were all rebuilt as empty. A message published directly and the same message published
|
||||
* through the outbox therefore arrived at the consumer as two different messages, and which one a
|
||||
* service got depended on a deployment decision it could not see.
|
||||
*/
|
||||
class OutboxEnvelopeFactoryTest {
|
||||
|
||||
private static final Instant WRITTEN_AT = Instant.parse("2026-08-10T09:15:00Z");
|
||||
private static final Instant OCCURRED_AT = Instant.parse("2026-08-10T09:14:00Z");
|
||||
private static final MessageId CAUSE = MessageId.newId();
|
||||
private static final String TRACEPARENT =
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
|
||||
|
||||
private final OutboxEnvelopeFactory factory =
|
||||
new OutboxEnvelopeFactory(new ProducerId("relay-owner"));
|
||||
|
||||
@Test
|
||||
void everyCanonicalFieldOnTheRowReachesTheEnvelope() {
|
||||
MessageEnvelope<EncodedMessage> envelope = factory.toEnvelope(record(fullMetadata(), Map.of()));
|
||||
|
||||
assertThat(envelope.producer()).isEqualTo(new ProducerId("order-api"));
|
||||
assertThat(envelope.occurredAt()).contains(OCCURRED_AT);
|
||||
assertThat(envelope.correlationId()).contains(new CorrelationId("checkout-7"));
|
||||
assertThat(envelope.causationId()).contains(new CausationId(CAUSE));
|
||||
assertThat(envelope.tenantContext()).contains(new TenantContext("acme"));
|
||||
assertThat(envelope.partitionKey()).contains("customer-42");
|
||||
assertThat(envelope.orderingKey()).contains("customer-42");
|
||||
assertThat(envelope.traceContext().traceparent()).contains(TRACEPARENT);
|
||||
assertThat(envelope.payload().schemaReference())
|
||||
.hasValueSatisfying(
|
||||
reference -> {
|
||||
assertThat(reference.subject()).isEqualTo("order.created-value");
|
||||
assertThat(reference.version()).isEqualTo(new SchemaVersion(1));
|
||||
assertThat(reference.schemaUri()).contains(java.net.URI.create("urn:schema:1"));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTenantIsNeverReconstructedAsAbsent() {
|
||||
assertThat(factory.toEnvelope(record(fullMetadata(), Map.of())).tenantContext())
|
||||
.as("a message published by one tenant and consumed as belonging to none is an escalation")
|
||||
.contains(new TenantContext("acme"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRowWithoutAProducerIsAttributedToTheServiceThatOwnsTheRelay() {
|
||||
MessageEnvelope<EncodedMessage> envelope =
|
||||
factory.toEnvelope(record(OutboxCanonicalMetadata.none(), Map.of()));
|
||||
|
||||
assertThat(envelope.producer())
|
||||
.as("a row written before the column existed has no honest producer but this one")
|
||||
.isEqualTo(new ProducerId("relay-owner"));
|
||||
assertThat(envelope.occurredAt())
|
||||
.as("the transaction that wrote the row is the one the fact occurred in")
|
||||
.contains(WRITTEN_AT);
|
||||
assertThat(envelope.tenantContext()).isEmpty();
|
||||
assertThat(envelope.payload().schemaReference()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void applicationHeadersOnTheRowStillTravel() {
|
||||
assertThat(
|
||||
factory
|
||||
.toEnvelope(record(OutboxCanonicalMetadata.none(), Map.of("x-tenant-tier", "gold")))
|
||||
.headers()
|
||||
.find("x-tenant-tier"))
|
||||
.hasValueSatisfying(value -> assertThat(value.value()).isEqualTo("gold"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aReservedHeaderOnTheRowIsRefusedEvenWhenItIsARoutingKey() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
factory.toEnvelope(
|
||||
record(
|
||||
OutboxCanonicalMetadata.none(),
|
||||
Map.of(ReservedHeaders.PARTITION_KEY, "customer-42"))))
|
||||
.as("the routing keys were the exception that kept the reserved-name door ajar")
|
||||
.isInstanceOf(MessagingConfigurationException.class)
|
||||
.hasMessageContaining("reserved namespace");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aForgedMessageIdOnTheRowCannotOverwriteTheCanonicalOne() {
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
factory.toEnvelope(
|
||||
record(
|
||||
OutboxCanonicalMetadata.none(),
|
||||
Map.of(ReservedHeaders.MESSAGE_ID, MessageId.newId().value().toString()))))
|
||||
.isInstanceOf(MessagingConfigurationException.class)
|
||||
.hasMessageContaining(ReservedHeaders.MESSAGE_ID);
|
||||
}
|
||||
|
||||
private static OutboxCanonicalMetadata fullMetadata() {
|
||||
return new OutboxCanonicalMetadata(
|
||||
Optional.of(new ProducerId("order-api")),
|
||||
Optional.of(OCCURRED_AT),
|
||||
Optional.of(new CorrelationId("checkout-7")),
|
||||
Optional.of(new CausationId(CAUSE)),
|
||||
Optional.of(new TenantContext("acme")),
|
||||
Optional.of("customer-42"),
|
||||
Optional.of("customer-42"),
|
||||
Optional.of("order.created-value"),
|
||||
Optional.of("urn:schema:1"),
|
||||
new TraceContext(Optional.of(TRACEPARENT), Optional.empty(), Optional.empty()));
|
||||
}
|
||||
|
||||
private static OutboxRecord record(
|
||||
OutboxCanonicalMetadata metadata, Map<String, String> headers) {
|
||||
return new OutboxRecord(
|
||||
MessageId.newId(),
|
||||
new DestinationName("order-events"),
|
||||
new MessageType("order.created"),
|
||||
new SchemaVersion(1),
|
||||
ContentType.JSON,
|
||||
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
|
||||
headers,
|
||||
metadata,
|
||||
WRITTEN_AT,
|
||||
OutboxStatus.PENDING,
|
||||
0,
|
||||
Optional.empty(),
|
||||
Optional.empty());
|
||||
}
|
||||
}
|
||||
+153
-6
@@ -7,6 +7,7 @@ import dev.caskeleton.messaging.api.MessageId;
|
||||
import dev.caskeleton.messaging.api.MessageType;
|
||||
import dev.caskeleton.messaging.api.SchemaVersion;
|
||||
import dev.caskeleton.messaging.api.destination.DestinationName;
|
||||
import dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.reliability.OutboxStatus;
|
||||
import dev.caskeleton.messaging.testkit.DockerAvailability;
|
||||
@@ -94,7 +95,9 @@ class OutboxPostgresIT {
|
||||
assertThat(leasedRecord.messageId()).isEqualTo(record.messageId());
|
||||
assertThat(leasedRecord.messageType()).isEqualTo(record.messageType());
|
||||
assertThat(leasedRecord.payload()).isEqualTo(record.payload());
|
||||
assertThat(leasedRecord.headers()).containsEntry("msg.partition-key", "acct-1");
|
||||
assertThat(leasedRecord.metadata().partitionKey())
|
||||
.as("the routing key is a column the relay can read without decoding the payload")
|
||||
.contains("acct-1");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -200,6 +203,12 @@ class OutboxPostgresIT {
|
||||
}
|
||||
|
||||
private static OutboxRecord record() {
|
||||
return record(
|
||||
// The routing key is a column now, not a reserved header smuggled through the header map.
|
||||
OutboxCanonicalMetadata.none().withRoutingKeys("acct-1", "acct-1"));
|
||||
}
|
||||
|
||||
private static OutboxRecord record(OutboxCanonicalMetadata metadata) {
|
||||
return new OutboxRecord(
|
||||
MessageId.newId(),
|
||||
new DestinationName("order-events"),
|
||||
@@ -207,7 +216,8 @@ class OutboxPostgresIT {
|
||||
new SchemaVersion(1),
|
||||
ContentType.JSON,
|
||||
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
|
||||
Map.of("msg.partition-key", "acct-1"),
|
||||
Map.of(),
|
||||
metadata,
|
||||
NOW.minusSeconds(60),
|
||||
OutboxStatus.PENDING,
|
||||
0,
|
||||
@@ -223,9 +233,67 @@ class OutboxPostgresIT {
|
||||
return source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every migration in the module, in version order.
|
||||
*
|
||||
* <p>Discovered rather than listed. The list said V1 and V2 above a comment claiming it was every
|
||||
* migration, so V3 and V4 were never applied here — and the first symptom of that was not a
|
||||
* missing check but this suite failing against a column the production schema has. A test that
|
||||
* applies part of the schema certifies a database nobody deploys.
|
||||
*/
|
||||
private static java.util.List<String> migrations() throws IOException {
|
||||
return java.util.List.of(
|
||||
migration("V1__messaging_outbox.sql"), migration("V2__messaging_outbox_lease_fencing.sql"));
|
||||
java.net.URL directory =
|
||||
OutboxPostgresIT.class.getClassLoader().getResource("db/migration/messaging");
|
||||
if (directory == null) {
|
||||
throw new IOException("the messaging migrations are missing from the classpath");
|
||||
}
|
||||
java.io.File[] files =
|
||||
new java.io.File(java.net.URI.create(directory.toString()))
|
||||
.listFiles((dir, name) -> name.endsWith(".sql"));
|
||||
if (files == null || files.length == 0) {
|
||||
throw new IOException("no migration applied; the schema under test would be empty");
|
||||
}
|
||||
java.util.List<String> names =
|
||||
java.util.Arrays.stream(files).map(java.io.File::getName).sorted().toList();
|
||||
java.util.List<String> statements = new java.util.ArrayList<>();
|
||||
for (String name : names) {
|
||||
statements.add(migration(name));
|
||||
}
|
||||
return statements;
|
||||
}
|
||||
|
||||
private int countWhere(String predicate) {
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
Statement statement = connection.createStatement();
|
||||
java.sql.ResultSet results =
|
||||
statement.executeQuery("SELECT count(*) FROM messaging_outbox WHERE " + predicate)) {
|
||||
results.next();
|
||||
return results.getInt(1);
|
||||
} catch (SQLException failure) {
|
||||
throw new IllegalStateException("could not count outbox rows", failure);
|
||||
}
|
||||
}
|
||||
|
||||
private String routingKeyOf(MessageId messageId) {
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
java.sql.PreparedStatement statement =
|
||||
connection.prepareStatement(
|
||||
"SELECT routing_key FROM messaging_outbox WHERE message_id = ?")) {
|
||||
statement.setObject(1, messageId.value());
|
||||
try (java.sql.ResultSet results = statement.executeQuery()) {
|
||||
results.next();
|
||||
return results.getString(1);
|
||||
}
|
||||
} catch (SQLException failure) {
|
||||
throw new IllegalStateException("could not read the routing key", failure);
|
||||
}
|
||||
}
|
||||
|
||||
private void execute(String sql) throws SQLException {
|
||||
try (Connection connection = dataSource.getConnection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private static String migration(String fileName) throws IOException {
|
||||
@@ -385,6 +453,84 @@ class OutboxPostgresIT {
|
||||
.isEqualTo(OutboxStatus.PUBLISHED);
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyCanonicalColumnRoundTripsThroughTheDatabase() {
|
||||
MessageId cause = MessageId.newId();
|
||||
OutboxCanonicalMetadata written =
|
||||
new OutboxCanonicalMetadata(
|
||||
Optional.of(new dev.caskeleton.messaging.api.ProducerId("order-api")),
|
||||
Optional.of(NOW.minusSeconds(120)),
|
||||
Optional.of(new dev.caskeleton.messaging.api.CorrelationId("checkout-7")),
|
||||
Optional.of(new dev.caskeleton.messaging.api.CausationId(cause)),
|
||||
Optional.of(new dev.caskeleton.messaging.api.TenantContext("acme")),
|
||||
Optional.of("customer-42"),
|
||||
Optional.of("customer-42"),
|
||||
Optional.of("order.created-value"),
|
||||
Optional.of("urn:schema:1"),
|
||||
new dev.caskeleton.messaging.api.TraceContext(
|
||||
Optional.of("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
|
||||
Optional.empty(),
|
||||
Optional.empty()));
|
||||
OutboxRecord record = record(written);
|
||||
|
||||
appendInTransaction(record);
|
||||
|
||||
assertThat(repository.find(record.messageId()).orElseThrow().metadata())
|
||||
.as("what the relay reads back has to be what the business transaction wrote")
|
||||
.isEqualTo(written);
|
||||
}
|
||||
|
||||
@Test
|
||||
void theRelayCanSelectOneTenantsBacklogWithoutDecodingAPayload() {
|
||||
appendInTransaction(
|
||||
record(
|
||||
OutboxCanonicalMetadata.none()
|
||||
.withTenant(new dev.caskeleton.messaging.api.TenantContext("acme"))));
|
||||
appendInTransaction(
|
||||
record(
|
||||
OutboxCanonicalMetadata.none()
|
||||
.withTenant(new dev.caskeleton.messaging.api.TenantContext("globex"))));
|
||||
|
||||
assertThat(countWhere("tenant = 'acme'"))
|
||||
.as("the operational question an outbox backlog always raises, answerable as a SELECT")
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void theStoredRoutingKeyIsTheOneBothRelaysWouldUse() {
|
||||
OutboxRecord keyed =
|
||||
record(OutboxCanonicalMetadata.none().withRoutingKeys("customer-42", "customer-42"));
|
||||
OutboxRecord unkeyed = record(OutboxCanonicalMetadata.none());
|
||||
appendInTransaction(keyed);
|
||||
appendInTransaction(unkeyed);
|
||||
|
||||
DebeziumOutboxEventRouter router = new DebeziumOutboxEventRouter();
|
||||
assertThat(routingKeyOf(keyed.messageId()))
|
||||
.as("the CDC key used to be the destination, so every message on a topic shared a key")
|
||||
.isEqualTo(router.routingKey(keyed))
|
||||
.isEqualTo("customer-42");
|
||||
assertThat(routingKeyOf(unkeyed.messageId()))
|
||||
.as("the fallback is stated once, in the column, and the Java agrees with it")
|
||||
.isEqualTo(router.routingKey(unkeyed))
|
||||
.isEqualTo(unkeyed.messageId().value().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTenantThatBreaksTheSlugBoundIsRefusedByTheDatabase() {
|
||||
org.assertj.core.api.Assertions.assertThatThrownBy(
|
||||
() ->
|
||||
execute(
|
||||
"INSERT INTO messaging_outbox (message_id, destination, message_type, "
|
||||
+ "schema_version, content_type, payload, created_at, status, attempts, "
|
||||
+ "tenant) VALUES ('"
|
||||
+ MessageId.newId().value()
|
||||
+ "', 'order-events', 'order.created', 1, 'application/json', '\\x7b7d', "
|
||||
+ "now(), 'PENDING', 0, 'Tenant With Spaces')"))
|
||||
.as("a CDC connector and an operator both write this table without passing through Java")
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("ck_messaging_outbox_tenant");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aHeaderValueWithControlCharactersRoundTrips() {
|
||||
OutboxRecord record =
|
||||
@@ -398,7 +544,8 @@ class OutboxPostgresIT {
|
||||
// A tab and a newline in a header value. Only backslash and quote were escaped, so
|
||||
// this produced a document PostgreSQL refuses as invalid JSONB and the append failed
|
||||
// with an error naming neither the header nor the row.
|
||||
Map.of("msg.note", "line one\nline\ttwo"),
|
||||
Map.of("x-note", "line one\nline\ttwo"),
|
||||
OutboxCanonicalMetadata.none(),
|
||||
NOW.minusSeconds(60),
|
||||
OutboxStatus.PENDING,
|
||||
0,
|
||||
@@ -408,7 +555,7 @@ class OutboxPostgresIT {
|
||||
appendInTransaction(record);
|
||||
|
||||
assertThat(repository.find(record.messageId()).orElseThrow().headers())
|
||||
.containsEntry("msg.note", "line one\nline\ttwo");
|
||||
.containsEntry("x-note", "line one\nline\ttwo");
|
||||
}
|
||||
|
||||
private void appendInTransaction(OutboxRecord record) {
|
||||
|
||||
+60
@@ -80,6 +80,41 @@ class OutboxRelayTest {
|
||||
assertThat(publisher.publishedIds.get(1)).isEqualTo(record.messageId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void anAmbiguousRowWaitsForItsBackoffBeforeItIsClaimedAgain() {
|
||||
OutboxRecord record = OutboxFixtures.pending();
|
||||
repository.append(record);
|
||||
RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous();
|
||||
|
||||
relay(publisher).runOnce(NOW);
|
||||
OutboxRelayReport tooSoon = relay(publisher).runOnce(NOW.plusMillis(100));
|
||||
|
||||
assertThat(tooSoon.leased())
|
||||
.as("a backoff nothing writes on the row is a backoff the next pass cannot see")
|
||||
.isZero();
|
||||
assertThat(publisher.publishedIds).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRowThatSpendsItsAttemptBudgetIsParkedRatherThanLeftRetryable() {
|
||||
OutboxRecord record = OutboxFixtures.pending();
|
||||
repository.append(record);
|
||||
RecordingOutboxPublisher publisher = RecordingOutboxPublisher.ambiguous();
|
||||
OutboxRelay relay = relay(publisher, budgetOf(2));
|
||||
|
||||
relay.runOnce(NOW);
|
||||
OutboxRelayReport last = relay.runOnce(NOW.plus(Duration.ofHours(1)));
|
||||
|
||||
assertThat(last.exhausted())
|
||||
.as("a rejected row needs a fix and a spent one needs a redrive")
|
||||
.isEqualTo(1);
|
||||
assertThat(repository.find(record.messageId()).orElseThrow().status())
|
||||
.isEqualTo(OutboxStatus.EXHAUSTED);
|
||||
assertThat(relay.runOnce(NOW.plus(Duration.ofDays(1))).leased())
|
||||
.as("only an operator redrive brings a parked row back")
|
||||
.isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aRejectedPublishIsMarkedFailedAndNotRetried() {
|
||||
OutboxRecord record = OutboxFixtures.pending();
|
||||
@@ -135,6 +170,30 @@ class OutboxRelayTest {
|
||||
100,
|
||||
Duration.ofSeconds(30));
|
||||
}
|
||||
|
||||
private OutboxRelay relay(MessagePublisher publisher, OutboxRetryScheduler scheduler) {
|
||||
return new OutboxRelay(
|
||||
repository,
|
||||
publisher,
|
||||
new OutboxEnvelopeFactory(new ProducerId("order-api")),
|
||||
100,
|
||||
Duration.ofSeconds(30),
|
||||
"relay-under-test",
|
||||
scheduler);
|
||||
}
|
||||
|
||||
private static OutboxRetryScheduler budgetOf(int maxAttempts) {
|
||||
OutboxProperties defaults = OutboxProperties.defaults();
|
||||
return new OutboxRetryScheduler(
|
||||
new OutboxProperties(
|
||||
defaults.batchSize(),
|
||||
defaults.leaseDuration(),
|
||||
defaults.publishTimeout(),
|
||||
defaults.pollInterval(),
|
||||
defaults.retentionAfterPublish(),
|
||||
maxAttempts),
|
||||
Duration.ofMinutes(5));
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds outbox rows. */
|
||||
@@ -151,6 +210,7 @@ final class OutboxFixtures {
|
||||
ContentType.JSON,
|
||||
"{\"orderId\":\"o-1\"}".getBytes(StandardCharsets.UTF_8),
|
||||
Map.of(),
|
||||
dev.caskeleton.messaging.reliability.OutboxCanonicalMetadata.none(),
|
||||
Instant.parse("2026-08-10T09:14:00Z"),
|
||||
OutboxStatus.PENDING,
|
||||
0,
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
package dev.caskeleton.messaging.outbox;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import dev.caskeleton.messaging.api.MessageEnvelope;
|
||||
import dev.caskeleton.messaging.api.ProducerId;
|
||||
import dev.caskeleton.messaging.api.destination.MessageDestination;
|
||||
import dev.caskeleton.messaging.api.publish.ConfirmationLevel;
|
||||
import dev.caskeleton.messaging.api.publish.MessagePublisher;
|
||||
import dev.caskeleton.messaging.api.publish.PublishCompletion;
|
||||
import dev.caskeleton.messaging.api.publish.PublishEvidence;
|
||||
import dev.caskeleton.messaging.api.publish.PublishOptions;
|
||||
import dev.caskeleton.messaging.api.publish.PublishResult;
|
||||
import dev.caskeleton.messaging.api.publish.RoutingOutcome;
|
||||
import dev.caskeleton.messaging.reliability.OutboxRecord;
|
||||
import dev.caskeleton.messaging.reliability.OutboxStatus;
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Whether anything actually drives the relay.
|
||||
*
|
||||
* <p>The relay, its scheduler and the attempt budget were all constructed by the auto-configuration
|
||||
* and {@code runOnce} was called by no production code. A business transaction would commit its
|
||||
* outbox row, report success, and the message would never be published — a silent backlog rather
|
||||
* than a visible failure.
|
||||
*/
|
||||
class OutboxRelayWorkerTest {
|
||||
|
||||
private static final Duration PATIENCE = Duration.ofSeconds(5);
|
||||
|
||||
private final InMemoryOutboxRepository repository = new InMemoryOutboxRepository();
|
||||
|
||||
@Test
|
||||
@DisplayName("a started worker publishes the backlog without anyone calling a pass")
|
||||
void aStartedWorkerPublishesTheBacklog() throws InterruptedException {
|
||||
OutboxRecord record = OutboxFixtures.pending();
|
||||
repository.append(record);
|
||||
OutboxRelayWorker worker = worker(RecordingOutboxPublisher.confirming());
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
awaitStatus(record, OutboxStatus.PUBLISHED);
|
||||
} finally {
|
||||
worker.stop(PATIENCE);
|
||||
}
|
||||
|
||||
assertThat(repository.find(record.messageId()).orElseThrow().status())
|
||||
.isEqualTo(OutboxStatus.PUBLISHED);
|
||||
assertThat(worker.completedPasses()).isPositive();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("stopping waits for the running pass instead of stranding its claimed rows")
|
||||
void stoppingWaitsForTheRunningPass() throws InterruptedException {
|
||||
OutboxRecord record = OutboxFixtures.pending();
|
||||
repository.append(record);
|
||||
// The publish is still in progress when stop() is called, so the row is claimed and IN_FLIGHT
|
||||
// at that moment: the pass has to be allowed to finish or the lease strands the row.
|
||||
SlowPublisher publisher = new SlowPublisher(Duration.ofMillis(300));
|
||||
OutboxRelayWorker worker = worker(publisher);
|
||||
|
||||
worker.start();
|
||||
assertThat(publisher.started.await(PATIENCE.toMillis(), TimeUnit.MILLISECONDS)).isTrue();
|
||||
boolean drained = worker.stop(PATIENCE);
|
||||
|
||||
assertThat(drained).isTrue();
|
||||
assertThat(repository.find(record.messageId()).orElseThrow().status())
|
||||
.as("a pass killed between its claim and its terminal write leaves the row leased")
|
||||
.isEqualTo(OutboxStatus.PUBLISHED);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("a pass that throws does not stop every pass after it")
|
||||
void aFailedPassDoesNotEndTheLoop() throws InterruptedException {
|
||||
repository.append(OutboxFixtures.pending());
|
||||
OutboxRelayWorker worker = worker(new ThrowingPublisher());
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
long deadline = System.nanoTime() + PATIENCE.toNanos();
|
||||
while (worker.completedPasses() < 2 && System.nanoTime() < deadline) {
|
||||
Thread.sleep(10);
|
||||
}
|
||||
} finally {
|
||||
worker.stop(PATIENCE);
|
||||
}
|
||||
|
||||
assertThat(worker.completedPasses())
|
||||
.as("a scheduled task's own exception cancels every future run of it")
|
||||
.isGreaterThanOrEqualTo(2);
|
||||
assertThat(worker.failedPasses()).isPositive();
|
||||
}
|
||||
|
||||
private OutboxRelayWorker worker(MessagePublisher publisher) {
|
||||
OutboxRelay relay =
|
||||
new OutboxRelay(
|
||||
repository,
|
||||
publisher,
|
||||
new OutboxEnvelopeFactory(new ProducerId("order-api")),
|
||||
100,
|
||||
Duration.ofSeconds(30));
|
||||
return new OutboxRelayWorker(relay, OutboxRetryScheduler.standard());
|
||||
}
|
||||
|
||||
private void awaitStatus(OutboxRecord record, OutboxStatus expected) throws InterruptedException {
|
||||
long deadline = System.nanoTime() + PATIENCE.toNanos();
|
||||
while (System.nanoTime() < deadline) {
|
||||
if (repository.find(record.messageId()).map(OutboxRecord::status).orElse(null) == expected) {
|
||||
return;
|
||||
}
|
||||
Thread.sleep(10);
|
||||
}
|
||||
}
|
||||
|
||||
/** A publisher that is still publishing when shutdown begins. */
|
||||
private static final class SlowPublisher implements MessagePublisher {
|
||||
|
||||
private final CountDownLatch started = new CountDownLatch(1);
|
||||
|
||||
private final Duration duration;
|
||||
|
||||
private SlowPublisher(Duration duration) {
|
||||
this.duration = duration;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> CompletionStage<PublishResult> publish(
|
||||
MessageDestination<T> destination, MessageEnvelope<T> message, PublishOptions options) {
|
||||
started.countDown();
|
||||
try {
|
||||
Thread.sleep(duration.toMillis());
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return CompletableFuture.completedFuture(
|
||||
new PublishResult(
|
||||
PublishCompletion.CONFIRMED,
|
||||
PublishEvidence.confirmed(ConfirmationLevel.BROKER_ACK),
|
||||
RoutingOutcome.NOT_APPLICABLE,
|
||||
Optional.empty(),
|
||||
1,
|
||||
Duration.ofMillis(1),
|
||||
Optional.empty()));
|
||||
}
|
||||
}
|
||||
|
||||
/** A publisher whose failure reaches the worker as an exception rather than a result. */
|
||||
private static final class ThrowingPublisher implements MessagePublisher {
|
||||
|
||||
@Override
|
||||
public <T> CompletionStage<PublishResult> publish(
|
||||
MessageDestination<T> destination, MessageEnvelope<T> message, PublishOptions options) {
|
||||
throw new IllegalStateException("the broker connection is gone");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user