diff --git a/docs/messaging/delivery-guarantees.md b/docs/messaging/delivery-guarantees.md index 518aa3e..d980932 100644 --- a/docs/messaging/delivery-guarantees.md +++ b/docs/messaging/delivery-guarantees.md @@ -19,7 +19,12 @@ public enum PublishCompletion { CONFIRMED, REJECTED, AMBIGUOUS } ``` `REJECTED`와 `AMBIGUOUS`를 하나의 "실패"로 합치면 중복 주문이 만들어진다. -전자는 broker가 저장하지 않았음이 **확정**되어 포기해도 안전하고, 후자는 그렇지 않다. +전자는 broker가 저장하지 않았음이 **확정**되어 중복 걱정 없이 다시 보내거나 포기할 수 있고, 후자는 그렇지 않다. + +`REJECTED`가 곧 "포기해도 된다"는 뜻은 아니다. 거부 이유가 그 순간에 속하면(종료 중 +`SHUTTING_DOWN`, permit 대기 초과, `PUBLISH_RUNTIME_UNAVAILABLE`, `KAFKA_TRANSPORT_CLOSED`, Kafka +인증·인가 실패) `failure().retryable()`이 true이고, 호출자는 같은 메시지를 나중에 다시 보내야 한다. +포기해도 되는 것은 `retryable=false`인 거부뿐이다. | 상황 | 결과 | |---|---| diff --git a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java index e80cff4..ce84b54 100644 --- a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java +++ b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransport.java @@ -119,7 +119,16 @@ public final class KafkaMessagingTransport implements MessagingTransport { Objects.requireNonNull(request, "request must not be null"); if (closed.get()) { - return completed(rejectedLocally("KAFKA_TRANSPORT_CLOSED", "the transport is shutting down")); + // Not the message's fault: the next instance, or this one after a restart, will take it. A + // caller that trusts the retryable flag must not park it. + return completed( + new TransportPublishResult( + PublishResult.rejected( + Duration.ZERO, + FailureDescriptor.of( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "KAFKA_TRANSPORT_CLOSED", + "the transport is shutting down")))); } int size = request.envelope().payload().size(); int limit = request.profile().payload().maxBytes(); diff --git a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java index 3436871..0eea955 100644 --- a/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java +++ b/messaging/messaging-kafka/src/main/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifier.java @@ -19,8 +19,10 @@ import org.apache.kafka.common.errors.UnsupportedVersionException; *

The split is between failures that prove the record was not stored and failures that prove * nothing. Serialization, an invalid topic, authentication, authorization, fencing, and an * over-large record are all rejections: the broker never took the record, so abandoning the attempt - * is safe. Everything else — a delivery timeout above all — is ambiguous, because the record may - * already be replicated and only the acknowledgement was lost. + * is safe. Of those, authentication and authorization are marked retryable, because they clear on + * their own once a credential or an ACL is in place. Everything else — a delivery timeout above all + * — is ambiguous, because the record may already be replicated and only the acknowledgement was + * lost. * *

The default is deliberately ambiguous rather than rejected. Guessing "rejected" on an unknown * error is what turns one lost confirmation into two orders. @@ -37,7 +39,8 @@ public final class KafkaPublishFailureClassifier { public PublishResult classify(Throwable error, Duration elapsed) { Throwable cause = unwrap(error); if (isDefinitiveRejection(cause)) { - return PublishResult.rejected(elapsed, descriptor(cause, rejectionCategory(cause), false)); + return PublishResult.rejected( + elapsed, descriptor(cause, rejectionCategory(cause), mayPassOnRetry(cause))); } return PublishResult.ambiguous(elapsed, descriptor(cause, FailureCategory.AMBIGUOUS, false)); } @@ -58,6 +61,21 @@ public final class KafkaPublishFailureClassifier { || cause instanceof UnsupportedVersionException; } + /** + * Reports whether a definitive rejection may pass if the same record is sent again later. + * + *

Authentication and authorization prove the broker did not store the record, but not that it + * will refuse it again: a credential being rotated, or an ACL deployed after the application, + * fails every publish this way for a few minutes and then stops. Marked not retryable, a + * publisher that trusts the flag — the application's outbox relay does — parks every record sent + * in those minutes. A credential that is simply wrong is retried too, and the outbox then holds + * its rows FAILED with an alert every retry — recoverable once the credential is fixed, where + * DEAD rows needed a manual reset each. + */ + private static boolean mayPassOnRetry(Throwable cause) { + return cause instanceof AuthenticationException || cause instanceof AuthorizationException; + } + private static FailureCategory rejectionCategory(Throwable cause) { if (cause instanceof AuthenticationException) { return FailureCategory.AUTHENTICATION; diff --git a/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransportTest.java b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransportTest.java new file mode 100644 index 0000000..d50ce60 --- /dev/null +++ b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaMessagingTransportTest.java @@ -0,0 +1,43 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishOptions; +import dev.caskeleton.messaging.api.publish.PublishResult; +import dev.caskeleton.messaging.transport.TransportPublishRequest; +import org.apache.kafka.clients.producer.MockProducer; +import org.apache.kafka.clients.producer.Partitioner; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** What a closed Kafka transport answers a publish that arrives during shutdown. */ +class KafkaMessagingTransportTest { + + @Test + @DisplayName("닫힌 transport의 거부는 다음 인스턴스가 받을 수 있으므로 다시 시도할 수 있다고 표시한다") + void aPublishToAClosedTransportIsARefusalThatMayBeRetried() { + // given: the transport closed while a relay cycle was still publishing + KafkaMessagingTransport transport = + new KafkaMessagingTransport( + "kafka-primary", + 1, + new MockProducer<>( + false, (Partitioner) null, new ByteArraySerializer(), new ByteArraySerializer())); + transport.close(); + TransportPublishRequest request = + new TransportPublishRequest( + KafkaContractHarness.profile(), + KafkaFixtures.orderCreatedEnvelope(), + PublishOptions.defaults()); + + // when + PublishResult result = transport.publish(request).toCompletableFuture().join().result(); + + // then + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()).get().extracting(FailureDescriptor::retryable).isEqualTo(true); + } +} diff --git a/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifierTest.java b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifierTest.java new file mode 100644 index 0000000..cdbae5a --- /dev/null +++ b/messaging/messaging-kafka/src/test/java/dev/caskeleton/messaging/kafka/KafkaPublishFailureClassifierTest.java @@ -0,0 +1,69 @@ +package dev.caskeleton.messaging.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.caskeleton.messaging.api.error.FailureDescriptor; +import dev.caskeleton.messaging.api.publish.PublishCompletion; +import dev.caskeleton.messaging.api.publish.PublishResult; +import java.time.Duration; +import org.apache.kafka.common.errors.AuthenticationException; +import org.apache.kafka.common.errors.AuthorizationException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Which producer failures a publisher may retry. + * + *

Authentication and authorization failures prove the broker did not store the record, so they + * are rejections. They are not proof the record will be refused again: a credential being rotated + * or an ACL deployed after the application fails this way for a few minutes and then stops. + */ +class KafkaPublishFailureClassifierTest { + + private final KafkaPublishFailureClassifier classifier = new KafkaPublishFailureClassifier(); + + @Test + @DisplayName("인증 실패는 저장되지 않은 거부지만 다시 시도할 수 있다고 표시한다") + void anAuthenticationFailureIsARejectionThatMayBeRetried() { + // given + AuthenticationException rotatingCredential = + new AuthenticationException("SASL authentication failed"); + + // when + PublishResult result = classifier.classify(rotatingCredential, Duration.ZERO); + + // then + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()).get().extracting(FailureDescriptor::retryable).isEqualTo(true); + } + + @Test + @DisplayName("인가 실패는 저장되지 않은 거부지만 다시 시도할 수 있다고 표시한다") + void anAuthorizationFailureIsARejectionThatMayBeRetried() { + // given + AuthorizationException aclNotYetDeployed = new TopicAuthorizationException("orders"); + + // when + PublishResult result = classifier.classify(aclNotYetDeployed, Duration.ZERO); + + // then + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()).get().extracting(FailureDescriptor::retryable).isEqualTo(true); + } + + @Test + @DisplayName("존재하지 않는 topic은 다시 시도해도 같으므로 다시 시도할 수 없다고 표시한다") + void anInvalidTopicIsARejectionThatMayNotBeRetried() { + // given + InvalidTopicException noSuchTopic = new InvalidTopicException("no such topic"); + + // when + PublishResult result = classifier.classify(noSuchTopic, Duration.ZERO); + + // then + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()).get().extracting(FailureDescriptor::retryable).isEqualTo(false); + } +} diff --git a/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java b/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java index 0f6d9e2..a51f9fe 100644 --- a/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java +++ b/messaging/messaging-runtime-core/src/main/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisher.java @@ -263,7 +263,17 @@ public final class DefaultMessagePublisher implements MessagePublisher, EncodedM lease.close(); } admission.complete(destination.name().value()); - return rejected("PUBLISH_RUNTIME_UNAVAILABLE", sanitized(beforeTheSend), startedAt); + // No runtime to send through — the registry closed during shutdown, or none is installed + // yet. Nothing about the message was wrong, so the refusal is marked retryable: a caller that + // trusts the flag, as the application's outbox relay does, retries it instead of parking it. + return refused( + new FailureDescriptor( + FailureCategory.TRANSIENT_INFRASTRUCTURE, + "PUBLISH_RUNTIME_UNAVAILABLE", + true, + sanitized(beforeTheSend), + Optional.empty()), + startedAt); } } diff --git a/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java b/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java index 4ac4854..5a9b750 100644 --- a/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java +++ b/messaging/messaging-runtime-core/src/test/java/dev/caskeleton/messaging/runtime/DefaultMessagePublisherTest.java @@ -420,6 +420,37 @@ class DefaultMessagePublisherTest { assertThat(admission.inFlight()).isZero(); } + @Test + @DisplayName("runtime이 없어서 생긴 거부는 다시 시도할 수 있다고 표시한다") + void aRefusalForAnUnavailableRuntimeMayBeRetried() { + // given: the runtime registry closed during shutdown, before this publish acquired it + DefaultMessagePublisher publisher = + new DefaultMessagePublisher( + registry(), + allowOrders(), + new SingleCodecRegistry(), + admission, + new FailingRuntimeRegistry(), + transport); + + // when + PublishResult result = + publisher + .publish( + new MessageDestination<>(ORDERS, new MessageType("order.created"), String.class), + envelope(), + PublishOptions.defaults()) + .toCompletableFuture() + .join(); + + // then: nothing about the message was wrong, so a later attempt may succeed + assertThat(result.completion()).isEqualTo(PublishCompletion.REJECTED); + assertThat(result.failure()) + .get() + .extracting(FailureDescriptor::code, FailureDescriptor::retryable) + .containsExactly("PUBLISH_RUNTIME_UNAVAILABLE", true); + } + @Test @DisplayName("an unregistered destination names itself") void anUnregisteredDestinationNamesItself() {